jdk-sandbox: comparison src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp

equal deleted inserted replaced

-:ea43db53de91
+:ed12027517c0
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 */
 #include "precompiled.hpp"
+#include "opto/castnode.hpp"
 #include "opto/compile.hpp"
-#include "opto/castnode.hpp"
 #include "opto/escape.hpp"
 #include "opto/graphKit.hpp"
-#include "opto/idealKit.hpp"
 #include "opto/loopnode.hpp"
+#include "opto/machnode.hpp"
 #include "opto/macro.hpp"
+#include "opto/memnode.hpp"
+#include "opto/movenode.hpp"
 #include "opto/node.hpp"
+#include "opto/phase.hpp"
+#include "opto/phaseX.hpp"
+#include "opto/rootnode.hpp"
 #include "opto/type.hpp"
+#include "utilities/copy.hpp"
+#include "utilities/growableArray.hpp"
 #include "utilities/macros.hpp"
 #include "gc/z/zBarrierSet.hpp"
 #include "gc/z/c2/zBarrierSetC2.hpp"
 #include "gc/z/zThreadLocalData.hpp"
 #include "gc/z/zBarrierSetRuntime.hpp"
 // 3. This step checks for the phi corresponding to an optimized load barrier expansion
 if (node->is_Phi()) {
 PhiNode* phi = node->as_Phi();
 Node* n = phi->in(1);
-if (n != NULL && (n->is_LoadBarrierSlowReg() ||  n->is_LoadBarrierWeakSlowReg())) {
+if (n != NULL && n->is_LoadBarrierSlowReg()) {
 return true;
 }
 }
 return false;
 if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) {
 igvn->_worklist.push(node);
 }
 }
-void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) {
+static bool load_require_barrier(LoadNode* load)      { return ((load->barrier_data() & RequireBarrier) != 0); }
-// Look for dominating barriers on the same address only once all
+static bool load_has_weak_barrier(LoadNode* load)     { return ((load->barrier_data() & WeakBarrier) != 0); }
-// other loop opts are over. Loop opts may cause a safepoint to be
+static bool load_has_expanded_barrier(LoadNode* load) { return ((load->barrier_data() & ExpandedBarrier) != 0); }
-// inserted between a barrier and its dominating barrier.
+static void load_set_expanded_barrier(LoadNode* load) { return load->set_barrier_data(ExpandedBarrier); }
-Compile* C = Compile::current();
-ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2();
+static void load_set_barrier(LoadNode* load, bool weak)    {
-ZBarrierSetC2State* s = bs->state();
+if (weak) {
-if (s->load_barrier_count() >= 2) {
+load->set_barrier_data(WeakBarrier);
-Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]);
+} else {
-PhaseIdealLoop::optimize(igvn, LoopOptsLastRound);
+load->set_barrier_data(RequireBarrier);
-if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
+}
-}
-}
-void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {
-// Permanent temporary workaround
-// Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is
-// removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to
-// the worklist. Unless we add them explicitly they are not guaranteed to end up there.
-ZBarrierSetC2State* s = state();
-for (int i = 0; i < s->load_barrier_count(); i++) {
-LoadBarrierNode* n = s->load_barrier_node(i);
-worklist->push(n);
-}
-}
-const TypeFunc* ZBarrierSetC2::load_barrier_Type() const {
-const Type** fields;
-// Create input types (domain)
-fields = TypeTuple::fields(2);
-fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
-fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM;
-const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
-// Create result type (range)
-fields = TypeTuple::fields(1);
-fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM;
-const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
-return TypeFunc::make(domain, range);
 }
 // == LoadBarrierNode ==
 LoadBarrierNode::LoadBarrierNode(Compile* C,
 Node* c,
 Node* mem,
 Node* val,
 Node* adr,
-bool weak,
+bool weak) :
-bool writeback,
-bool oop_reload_allowed) :
 MultiNode(Number_of_Inputs),
-_weak(weak),
+_weak(weak) {
-_writeback(writeback),
-_oop_reload_allowed(oop_reload_allowed) {
 init_req(Control, c);
 init_req(Memory, mem);
 init_req(Oop, val);
 init_req(Address, adr);
 init_req(Similar, C->top());
 const Type *LoadBarrierNode::Value(PhaseGVN *phase) const {
 const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
 const Type* val_t = phase->type(in(Oop));
 floadbarrier[Control] = Type::CONTROL;
-floadbarrier[Memory] = Type::MEMORY;
+floadbarrier[Memory]  = Type::MEMORY;
-floadbarrier[Oop] = val_t;
+floadbarrier[Oop]     = val_t;
 return TypeTuple::make(Number_of_Outputs, floadbarrier);
 }
 bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) {
 if (phase != NULL) {
 return false;
 }
 LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) {
+if (is_weak()) {
+// Weak barriers can't be eliminated
+return NULL;
+}
 Node* val = in(LoadBarrierNode::Oop);
 if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) {
 LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier();
 assert(lb->in(Address) == in(Address), "");
 // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier.
 return u->as_LoadBarrier();
 }
 }
 }
-if (ZVerifyLoadBarriers || can_be_eliminated()) {
+if (can_be_eliminated()) {
 return NULL;
 }
 if (!look_for_similar) {
 return NULL;
 }
 }
 }
 if (ok) {
 assert(dom_found, "");
-return u->as_LoadBarrier();;
+return u->as_LoadBarrier();
 }
 break;
 }
 }
 }
 return NULL;
 }
 void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const {
 // Change to that barrier may affect a dominated barrier so re-push those
+assert(!is_weak(), "sanity");
 Node* val = in(LoadBarrierNode::Oop);
 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
 Node* u = val->fast_out(i);
 if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) {
 }
 }
 }
 Node *LoadBarrierNode::Identity(PhaseGVN *phase) {
-if (!phase->C->directive()->ZOptimizeLoadBarriersOption) {
-return this;
-}
-bool redundant_addr = false;
 LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false);
 if (dominating_barrier != NULL) {
+assert(!is_weak(), "Weak barriers cant be eliminated");
 assert(dominating_barrier->in(Oop) == in(Oop), "");
 return dominating_barrier;
 }
 return this;
 Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
 if (remove_dead_region(phase, can_reshape)) {
 return this;
 }
-Node* val = in(Oop);
+Node *val = in(Oop);
-Node* mem = in(Memory);
+Node *mem = in(Memory);
-Node* ctrl = in(Control);
+Node *ctrl = in(Control);
-Node* adr = in(Address);
 assert(val->Opcode() != Op_LoadN, "");
+assert(val->Opcode() != Op_DecodeN, "");
 if (mem->is_MergeMem()) {
-Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
+Node *new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
 set_req(Memory, new_mem);
 if (mem->outcnt() == 0 && can_reshape) {
 phase->is_IterGVN()->_worklist.push(mem);
 }
 return this;
 }
-bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption;
+LoadBarrierNode *dominating_barrier = NULL;
-LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL;
+if (!is_weak()) {
-if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {
+dominating_barrier = has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress());
-assert(in(Address) == dominating_barrier->in(Address), "");
+if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {
-set_req(Similar, dominating_barrier->proj_out(Oop));
+assert(in(Address) == dominating_barrier->in(Address), "");
-return this;
+set_req(Similar, dominating_barrier->proj_out(Oop));
-}
+return this;
+}
-bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) ||
+}
-(can_reshape && (dominating_barrier != NULL || !has_true_uses()));
+bool eliminate = can_reshape && (dominating_barrier != NULL || !has_true_uses());
 if (eliminate) {
 if (can_reshape) {
 PhaseIterGVN* igvn = phase->is_IterGVN();
 Node* out_ctrl = proj_out_or_null(Control);
 Node* out_res = proj_out_or_null(Oop);
 // That transformation may cause the Similar edge on the load barrier to be invalid
 fix_similar_in_uses(igvn);
 if (out_res != NULL) {
 if (dominating_barrier != NULL) {
+assert(!is_weak(), "Sanity");
 igvn->replace_node(out_res, dominating_barrier->proj_out(Oop));
 } else {
 igvn->replace_node(out_res, val);
 }
 }
 }
 return new ConINode(TypeInt::ZERO);
 }
 // If the Similar edge is no longer a load barrier, clear it
 Node* similar = in(Similar);
 if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) {
 set_req(Similar, phase->C->top());
 return this;
 }
-if (can_reshape) {
+if (can_reshape && !is_weak()) {
 // If this barrier is linked through the Similar edge by a
 // dominated barrier and both barriers have the same Oop field,
 // the dominated barrier can go away, so push it for reprocessing.
 // We also want to avoid a barrier to depend on another dominating
 // barrier through its Similar edge that itself depend on another
 Node* out_res = proj_out(Oop);
 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
 Node* u = out_res->fast_out(i);
 if (u->is_LoadBarrier() && u->in(Similar) == out_res &&
 (u->in(Oop) == val || !u->in(Similar)->is_top())) {
+assert(!u->as_LoadBarrier()->is_weak(), "Sanity");
 igvn->_worklist.push(u);
 }
 }
 push_dominated_barriers(igvn);
 }
 return NULL;
 }
 }
 }
 bool LoadBarrierNode::has_true_uses() const {
 Node* out_res = proj_out_or_null(Oop);
-if (out_res == NULL) {
+if (out_res != NULL) {
-return false;
+for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
-}
+Node *u = out_res->fast_out(i);
+if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {
-for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
+return true;
-Node* u = out_res->fast_out(i);
+}
-if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {
+}
-return true;
+}
-}
-}
 return false;
-}
-// == Accesses ==
-Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicParseAccess& access) const {
-assert(!UseCompressedOops, "Not allowed");
-CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access();
-PhaseGVN& gvn = access.gvn();
-Compile* C = Compile::current();
-GraphKit* kit = access.kit();
-Node* in_ctrl     = cas->in(MemNode::Control);
-Node* in_mem      = cas->in(MemNode::Memory);
-Node* in_adr      = cas->in(MemNode::Address);
-Node* in_val      = cas->in(MemNode::ValueIn);
-Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn);
-float likely                   = PROB_LIKELY(0.999);
-const TypePtr *adr_type        = gvn.type(in_adr)->isa_ptr();
-Compile::AliasType* alias_type = C->alias_type(adr_type);
-int alias_idx                  = C->get_alias_index(adr_type);
-// Outer check - true: continue, false: load and check
-Node* region   = new RegionNode(3);
-Node* phi      = new PhiNode(region, TypeInt::BOOL);
-Node* phi_mem  = new PhiNode(region, Type::MEMORY, adr_type);
-// Inner check - is the healed ref equal to the expected
-Node* region2  = new RegionNode(3);
-Node* phi2     = new PhiNode(region2, TypeInt::BOOL);
-Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type);
-// CAS node returns 0 or 1
-Node* cmp     = gvn.transform(new CmpINode(cas, kit->intcon(0)));
-Node* bol     = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
-IfNode* iff   = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
-Node* then    = gvn.transform(new IfTrueNode(iff));
-Node* elsen   = gvn.transform(new IfFalseNode(iff));
-Node* scmemproj1   = gvn.transform(new SCMemProjNode(cas));
-kit->set_memory(scmemproj1, alias_idx);
-phi_mem->init_req(1, scmemproj1);
-phi_mem2->init_req(2, scmemproj1);
-// CAS fail - reload and heal oop
-Node* reload      = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
-Node* barrier     = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
-Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
-Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
-// Check load
-Node* tmpX    = gvn.transform(new CastP2XNode(NULL, barrierdata));
-Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
-Node* cmp2    = gvn.transform(new CmpXNode(tmpX, in_expX));
-Node *bol2    = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
-IfNode* iff2  = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
-Node* then2   = gvn.transform(new IfTrueNode(iff2));
-Node* elsen2  = gvn.transform(new IfFalseNode(iff2));
-// redo CAS
-Node* cas2       = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order()));
-Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2));
-kit->set_control(elsen2);
-kit->set_memory(scmemproj2, alias_idx);
-// Merge inner flow - check if healed oop was equal too expected.
-region2->set_req(1, kit->control());
-region2->set_req(2, then2);
-phi2->set_req(1, cas2);
-phi2->set_req(2, kit->intcon(0));
-phi_mem2->init_req(1, scmemproj2);
-kit->set_memory(phi_mem2, alias_idx);
-// Merge outer flow - then check if first CAS succeeded
-region->set_req(1, then);
-region->set_req(2, region2);
-phi->set_req(1, kit->intcon(1));
-phi->set_req(2, phi2);
-phi_mem->init_req(2, phi_mem2);
-kit->set_memory(phi_mem, alias_idx);
-gvn.transform(region2);
-gvn.transform(phi2);
-gvn.transform(phi_mem2);
-gvn.transform(region);
-gvn.transform(phi);
-gvn.transform(phi_mem);
-kit->set_control(region);
-kit->insert_mem_bar(Op_MemBarCPUOrder);
-return phi;
-}
-Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicParseAccess& access) const {
-CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access();
-GraphKit* kit = access.kit();
-PhaseGVN& gvn = kit->gvn();
-Compile* C = Compile::current();
-Node* in_ctrl     = cmpx->in(MemNode::Control);
-Node* in_mem      = cmpx->in(MemNode::Memory);
-Node* in_adr      = cmpx->in(MemNode::Address);
-Node* in_val      = cmpx->in(MemNode::ValueIn);
-Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn);
-float likely                   = PROB_LIKELY(0.999);
-const TypePtr *adr_type        = cmpx->get_ptr_type();
-Compile::AliasType* alias_type = C->alias_type(adr_type);
-int alias_idx                  = C->get_alias_index(adr_type);
-// Outer check - true: continue, false: load and check
-Node* region  = new RegionNode(3);
-Node* phi     = new PhiNode(region, adr_type);
-// Inner check - is the healed ref equal to the expected
-Node* region2 = new RegionNode(3);
-Node* phi2    = new PhiNode(region2, adr_type);
-// Check if cmpx succeeded
-Node* cmp     = gvn.transform(new CmpPNode(cmpx, in_expected));
-Node* bol     = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool();
-IfNode* iff   = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
-Node* then    = gvn.transform(new IfTrueNode(iff));
-Node* elsen   = gvn.transform(new IfFalseNode(iff));
-Node* scmemproj1  = gvn.transform(new SCMemProjNode(cmpx));
-kit->set_memory(scmemproj1, alias_idx);
-// CAS fail - reload and heal oop
-Node* reload      = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
-Node* barrier     = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
-Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
-Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
-// Check load
-Node* tmpX    = gvn.transform(new CastP2XNode(NULL, barrierdata));
-Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
-Node* cmp2    = gvn.transform(new CmpXNode(tmpX, in_expX));
-Node *bol2    = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
-IfNode* iff2  = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
-Node* then2   = gvn.transform(new IfTrueNode(iff2));
-Node* elsen2  = gvn.transform(new IfFalseNode(iff2));
-// Redo CAS
-Node* cmpx2      = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order()));
-Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2));
-kit->set_control(elsen2);
-kit->set_memory(scmemproj2, alias_idx);
-// Merge inner flow - check if healed oop was equal too expected.
-region2->set_req(1, kit->control());
-region2->set_req(2, then2);
-phi2->set_req(1, cmpx2);
-phi2->set_req(2, barrierdata);
-// Merge outer flow - then check if first cas succeeded
-region->set_req(1, then);
-region->set_req(2, region2);
-phi->set_req(1, cmpx);
-phi->set_req(2, phi2);
-gvn.transform(region2);
-gvn.transform(phi2);
-gvn.transform(region);
-gvn.transform(phi);
-kit->set_control(region);
-kit->set_memory(in_mem, alias_idx);
-kit->insert_mem_bar(Op_MemBarCPUOrder);
-return phi;
-}
-Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const {
-PhaseGVN& gvn = kit->gvn();
-Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed);
-Node* transformed_barrier = gvn.transform(barrier);
-if (transformed_barrier->is_LoadBarrier()) {
-if (barrier == transformed_barrier) {
-kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)));
-}
-Node* result = gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop));
-return result;
-} else {
-return val;
-}
 }
 static bool barrier_needed(C2Access& access) {
 return ZBarrierSet::barrier_needed(access.decorators(), access.type());
 }
 if (!barrier_needed(access)) {
 return p;
 }
 bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;
+if (p->isa_Load()) {
-assert(access.is_parse_access(), "entry not supported at optimization time");
+load_set_barrier(p->as_Load(), weak);
-C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
+}
-GraphKit* kit = parse_access.kit();
+return p;
-PhaseGVN& gvn = kit->gvn();
-Node* adr = access.addr().node();
-Node* heap_base_oop = access.base();
-bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0;
-if (unsafe) {
-if (!ZVerifyLoadBarriers) {
-p = load_barrier(kit, p, adr);
-} else {
-if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) {
-p = load_barrier(kit, p, adr);
-} else {
-IdealKit ideal(kit);
-IdealVariable res(ideal);
-#define __ ideal.
-__ declarations_done();
-__ set(res, p);
-__ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); {
-kit->sync_kit(ideal);
-p = load_barrier(kit, p, adr);
-__ set(res, p);
-__ sync_kit(kit);
-} __ end_if();
-kit->final_sync(ideal);
-p = __ value(res);
-#undef __
-}
-}
-return p;
-} else {
-return load_barrier(parse_access.kit(), p, access.addr().node(), weak, true, true);
-}
 }
 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
 Node* new_val, const Type* val_type) const {
 Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
-if (!barrier_needed(access)) {
+LoadStoreNode* lsn = result->as_LoadStore();
-return result;
+if (barrier_needed(access)) {
-}
+lsn->set_has_barrier();
+}
-access.set_needs_pinning(false);
+return lsn;
-return make_cmpx_loadbarrier(access);
 }
 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
 Node* new_val, const Type* value_type) const {
 Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
-if (!barrier_needed(access)) {
+LoadStoreNode* lsn = result->as_LoadStore();
-return result;
+if (barrier_needed(access)) {
-}
+lsn->set_has_barrier();
+}
-Node* load_store = access.raw_access();
+return lsn;
-bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0;
-bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR);
-if (!expected_is_null) {
-if (weak_cas) {
-access.set_needs_pinning(false);
-load_store = make_cas_loadbarrier(access);
-} else {
-access.set_needs_pinning(false);
-load_store = make_cas_loadbarrier(access);
-}
-}
-return load_store;
 }
 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
-if (!barrier_needed(access)) {
+LoadStoreNode* lsn = result->as_LoadStore();
-return result;
+if (barrier_needed(access)) {
-}
+lsn->set_has_barrier();
+}
-Node* load_store = access.raw_access();
+return lsn;
-Node* adr = access.addr().node();
-assert(access.is_parse_access(), "entry not supported at optimization time");
-C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
-return load_barrier(parse_access.kit(), load_store, adr, false, false, false);
 }
 // == Macro Expansion ==
+// Optimized, low spill, loadbarrier variant using stub specialized on register used
 void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const {
-Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
-Node* in_mem  = barrier->in(LoadBarrierNode::Memory);
-Node* in_val  = barrier->in(LoadBarrierNode::Oop);
-Node* in_adr  = barrier->in(LoadBarrierNode::Address);
-Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
-Node* out_res  = barrier->proj_out(LoadBarrierNode::Oop);
 PhaseIterGVN &igvn = phase->igvn();
-if (ZVerifyLoadBarriers) {
-igvn.replace_node(out_res, in_val);
-igvn.replace_node(out_ctrl, in_ctrl);
-return;
-}
-if (barrier->can_be_eliminated()) {
-// Clone and pin the load for this barrier below the dominating
-// barrier: the load cannot be allowed to float above the
-// dominating barrier
-Node* load = in_val;
-if (load->is_Load()) {
-Node* new_load = load->clone();
-Node* addp = new_load->in(MemNode::Address);
-assert(addp->is_AddP() || addp->is_Phi() || addp->is_Load(), "bad address");
-Node* cast = new CastPPNode(addp, igvn.type(addp), true);
-Node* ctrl = NULL;
-Node* similar = barrier->in(LoadBarrierNode::Similar);
-if (similar->is_Phi()) {
-// already expanded
-ctrl = similar->in(0);
-} else {
-assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape");
-ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control);
-}
-assert(ctrl != NULL, "bad control");
-cast->set_req(0, ctrl);
-igvn.transform(cast);
-new_load->set_req(MemNode::Address, cast);
-igvn.transform(new_load);
-igvn.replace_node(out_res, new_load);
-igvn.replace_node(out_ctrl, in_ctrl);
-return;
-}
-// cannot eliminate
-}
-// There are two cases that require the basic loadbarrier
-// 1) When the writeback of a healed oop must be avoided (swap)
-// 2) When we must guarantee that no reload of is done (swap, cas, cmpx)
-if (!barrier->is_writeback()) {
-assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop");
-}
-if (!barrier->oop_reload_allowed()) {
-expand_loadbarrier_basic(phase, barrier);
-} else {
-expand_loadbarrier_optimized(phase, barrier);
-}
-}
-// Basic loadbarrier using conventional argument passing
-void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
-PhaseIterGVN &igvn = phase->igvn();
-Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
-Node* in_mem  = barrier->in(LoadBarrierNode::Memory);
-Node* in_val  = barrier->in(LoadBarrierNode::Oop);
-Node* in_adr  = barrier->in(LoadBarrierNode::Address);
-Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
-Node* out_res  = barrier->proj_out(LoadBarrierNode::Oop);
 float unlikely  = PROB_UNLIKELY(0.999);
-const Type* in_val_maybe_null_t = igvn.type(in_val);
-Node* jthread = igvn.transform(new ThreadLocalNode());
-Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
-Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
-Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
-Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
-Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
-Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
-IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
-Node* then = igvn.transform(new IfTrueNode(iff));
-Node* elsen = igvn.transform(new IfFalseNode(iff));
-Node* result_region;
-Node* result_val;
-result_region = new RegionNode(3);
-result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM);
-result_region->set_req(1, elsen);
-Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t));
-res->init_req(0, elsen);
-result_val->set_req(1, res);
-const TypeFunc *tf = load_barrier_Type();
-Node* call;
-if (barrier->is_weak()) {
-call = new CallLeafNode(tf,
-ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(),
-"ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded",
-TypeRawPtr::BOTTOM);
-} else {
-call = new CallLeafNode(tf,
-ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(),
-"ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded",
-TypeRawPtr::BOTTOM);
-}
-call->init_req(TypeFunc::Control, then);
-call->init_req(TypeFunc::I_O    , phase->top());
-call->init_req(TypeFunc::Memory , in_mem);
-call->init_req(TypeFunc::FramePtr, phase->top());
-call->init_req(TypeFunc::ReturnAdr, phase->top());
-call->init_req(TypeFunc::Parms+0, in_val);
-if (barrier->is_writeback()) {
-call->init_req(TypeFunc::Parms+1, in_adr);
-} else {
-// When slow path is called with a null address, the healed oop will not be written back
-call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT));
-}
-call = igvn.transform(call);
-Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control));
-res = igvn.transform(new ProjNode(call, TypeFunc::Parms));
-res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t));
-result_region->set_req(2, ctrl);
-result_val->set_req(2, res);
-result_region = igvn.transform(result_region);
-result_val = igvn.transform(result_val);
-if (out_ctrl != NULL) { // Added if cond
-igvn.replace_node(out_ctrl, result_region);
-}
-igvn.replace_node(out_res, result_val);
-}
-// Optimized, low spill, loadbarrier variant using stub specialized on register used
-void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
-PhaseIterGVN &igvn = phase->igvn();
-#ifdef PRINT_NODE_TRAVERSALS
-Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop);
-#endif
 Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
 Node* in_mem = barrier->in(LoadBarrierNode::Memory);
 Node* in_val = barrier->in(LoadBarrierNode::Oop);
 Node* in_adr = barrier->in(LoadBarrierNode::Address);
-Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
+Node* out_ctrl = barrier->proj_out_or_null(LoadBarrierNode::Control);
 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
 assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null");
-#ifdef PRINT_NODE_TRAVERSALS
-tty->print("\n\n\nBefore barrier optimization:\n");
-traverse(barrier, out_ctrl, out_res, -1);
-tty->print("\nBefore barrier optimization:  preceding_barrier_node\n");
-traverse(preceding_barrier_node, out_ctrl, out_res, -1);
-#endif
-float unlikely  = PROB_UNLIKELY(0.999);
 Node* jthread = igvn.transform(new ThreadLocalNode());
 Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
 Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr,
 TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(),
 Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
 IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
 Node* then = igvn.transform(new IfTrueNode(iff));
 Node* elsen = igvn.transform(new IfFalseNode(iff));
-Node* slow_path_surrogate;
+Node* new_loadp = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
-if (!barrier->is_weak()) {
+(const TypePtr*) in_val->bottom_type(), MemNode::unordered, barrier->is_weak()));
-slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
-(const TypePtr*) in_val->bottom_type(), MemNode::unordered));
-} else {
-slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
-(const TypePtr*) in_val->bottom_type(), MemNode::unordered));
-}
-Node *new_loadp;
-new_loadp = slow_path_surrogate;
 // Create the final region/phi pair to converge cntl/data paths to downstream code
 Node* result_region = igvn.transform(new RegionNode(3));
 result_region->set_req(1, then);
 result_region->set_req(2, elsen);
 Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM));
 result_phi->set_req(1, new_loadp);
 result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop));
-// Finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution
+if (out_ctrl != NULL) {
-// igvn.replace_node(out_ctrl, result_region);
-if (out_ctrl != NULL) { // added if cond
 igvn.replace_node(out_ctrl, result_region);
 }
 igvn.replace_node(out_res, result_phi);
 assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!");
-#ifdef PRINT_NODE_TRAVERSALS
+igvn.remove_dead_node(barrier);
-tty->print("\nAfter barrier optimization:  old out_ctrl\n");
+igvn.remove_dead_node(out_ctrl);
-traverse(out_ctrl, out_ctrl, out_res, -1);
+igvn.remove_dead_node(out_res);
-tty->print("\nAfter barrier optimization:  old out_res\n");
-traverse(out_res, out_ctrl, out_res, -1);
-tty->print("\nAfter barrier optimization:  old barrier\n");
-traverse(barrier, out_ctrl, out_res, -1);
-tty->print("\nAfter barrier optimization:  preceding_barrier_node\n");
-traverse(preceding_barrier_node, result_region, result_phi, -1);
-#endif
 assert(is_gc_barrier_node(result_phi), "sanity");
 assert(step_over_gc_barrier(result_phi) == in_val, "sanity");
+phase->C->print_method(PHASE_BARRIER_EXPANSION, 4, barrier->_idx);
 }
 bool ZBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
 ZBarrierSetC2State* s = state();
 if (s->load_barrier_count() > 0) {
 PhaseMacroExpand macro(igvn);
-#ifdef ASSERT
-verify_gc_barriers(false);
-#endif
 int skipped = 0;
 while (s->load_barrier_count() > skipped) {
 int load_barrier_count = s->load_barrier_count();
 LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped);
 if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
 }
 return false;
 }
-// == Loop optimization ==
-static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
-PhaseIterGVN &igvn = phase->igvn();
-Compile* C = Compile::current();
-LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round);
-if (lb2 == NULL) {
-return false;
-}
-if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) {
-assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), "Invalid address");
-igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop));
-C->set_major_progress();
-return false;
-}
-// That transformation may cause the Similar edge on dominated load barriers to be invalid
-lb->fix_similar_in_uses(&igvn);
-Node* val = lb->proj_out(LoadBarrierNode::Oop);
-assert(lb2->has_true_uses(), "Invalid uses");
-assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), "Invalid oop");
-phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
-phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
-igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop));
-return true;
-}
-static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) {
-assert(dom->is_Region() || i == -1, "");
-Node* m = mem;
-while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) {
-if (m->is_Mem()) {
-assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, "");
-m = m->in(MemNode::Memory);
-} else if (m->is_MergeMem()) {
-m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
-} else if (m->is_Phi()) {
-if (m->in(0) == dom && i != -1) {
-m = m->in(i);
-break;
-} else {
-m = m->in(LoopNode::EntryControl);
-}
-} else if (m->is_Proj()) {
-m = m->in(0);
-} else if (m->is_SafePoint() || m->is_MemBar()) {
-m = m->in(TypeFunc::Memory);
-} else {
-#ifdef ASSERT
-m->dump();
-#endif
-ShouldNotReachHere();
-}
-}
-return m;
-}
-static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) {
-PhaseIterGVN &igvn = phase->igvn();
-Compile* C = Compile::current();
-Node* the_clone = lb->clone();
-the_clone->set_req(LoadBarrierNode::Control, ctl);
-the_clone->set_req(LoadBarrierNode::Memory, mem);
-if (oop_in != NULL) {
-the_clone->set_req(LoadBarrierNode::Oop, oop_in);
-}
-LoadBarrierNode* new_lb = the_clone->as_LoadBarrier();
-igvn.register_new_node_with_optimizer(new_lb);
-IdealLoopTree *loop = phase->get_loop(new_lb->in(0));
-phase->set_ctrl(new_lb, new_lb->in(0));
-phase->set_loop(new_lb, loop);
-phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1);
-if (!loop->_child) {
-loop->_body.push(new_lb);
-}
-Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control);
-igvn.register_new_node_with_optimizer(proj_ctl);
-phase->set_ctrl(proj_ctl, proj_ctl->in(0));
-phase->set_loop(proj_ctl, loop);
-phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1);
-if (!loop->_child) {
-loop->_body.push(proj_ctl);
-}
-Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop);
-phase->register_new_node(proj_oop, new_lb);
-if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) {
-LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier();
-if (!phase->is_dominator(similar, ctl)) {
-igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top());
-}
-}
-return new_lb;
-}
-static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) {
-PhaseIterGVN &igvn = phase->igvn();
-Node* val = lb->proj_out(LoadBarrierNode::Oop);
-igvn.replace_node(val, new_val);
-phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
-phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
-}
-static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
-PhaseIterGVN &igvn = phase->igvn();
-Compile* C = Compile::current();
-if (lb->in(LoadBarrierNode::Oop)->is_Phi()) {
-Node* oop_phi = lb->in(LoadBarrierNode::Oop);
-if ((oop_phi->req() != 3) || (oop_phi->in(2) == oop_phi)) {
-// Ignore phis with only one input
-return false;
-}
-if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)),
-oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) {
-// That transformation may cause the Similar edge on dominated load barriers to be invalid
-lb->fix_similar_in_uses(&igvn);
-RegionNode* region = oop_phi->in(0)->as_Region();
-int backedge = LoopNode::LoopBackControl;
-if (region->is_Loop() && region->in(backedge)->is_Proj() && region->in(backedge)->in(0)->is_If()) {
-Node* c = region->in(backedge)->in(0)->in(0);
-assert(c->unique_ctrl_out() == region->in(backedge)->in(0), "");
-Node* oop = lb->in(LoadBarrierNode::Oop)->in(backedge);
-Node* oop_c = phase->has_ctrl(oop) ? phase->get_ctrl(oop) : oop;
-if (!phase->is_dominator(oop_c, c)) {
-return false;
-}
-}
-// If the node on the backedge above the phi is the node itself - we have a self loop.
-// Don't clone - this will be folded later.
-if (oop_phi->in(LoopNode::LoopBackControl) == lb->proj_out(LoadBarrierNode::Oop)) {
-return false;
-}
-bool is_strip_mined = region->is_CountedLoop() && region->as_CountedLoop()->is_strip_mined();
-Node *phi = oop_phi->clone();
-for (uint i = 1; i < region->req(); i++) {
-Node* ctrl = region->in(i);
-if (ctrl != C->top()) {
-assert(!phase->is_dominator(ctrl, region) || region->is_Loop(), "");
-Node* mem = lb->in(LoadBarrierNode::Memory);
-Node* m = find_dominating_memory(phase, mem, region, i);
-if (region->is_Loop() && i == LoopNode::LoopBackControl && ctrl->is_Proj() && ctrl->in(0)->is_If()) {
-ctrl = ctrl->in(0)->in(0);
-} else if (region->is_Loop() && is_strip_mined) {
-// If this is a strip mined loop, control must move above OuterStripMinedLoop
-assert(i == LoopNode::EntryControl, "check");
-assert(ctrl->is_OuterStripMinedLoop(), "sanity");
-ctrl = ctrl->as_OuterStripMinedLoop()->in(LoopNode::EntryControl);
-}
-LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, ctrl, m, lb->in(LoadBarrierNode::Oop)->in(i));
-Node* out_ctrl = new_lb->proj_out(LoadBarrierNode::Control);
-if (is_strip_mined && (i == LoopNode::EntryControl)) {
-assert(region->in(i)->is_OuterStripMinedLoop(), "");
-igvn.replace_input_of(region->in(i), i, out_ctrl);
-phase->set_idom(region->in(i), out_ctrl, phase->dom_depth(out_ctrl));
-} else if (ctrl == region->in(i)) {
-igvn.replace_input_of(region, i, out_ctrl);
-// Only update the idom if is the loop entry we are updating
-// - A loop backedge doesn't change the idom
-if (region->is_Loop() && i == LoopNode::EntryControl) {
-phase->set_idom(region, out_ctrl, phase->dom_depth(out_ctrl));
-}
-} else {
-Node* iff = region->in(i)->in(0);
-igvn.replace_input_of(iff, 0, out_ctrl);
-phase->set_idom(iff, out_ctrl, phase->dom_depth(out_ctrl)+1);
-}
-phi->set_req(i, new_lb->proj_out(LoadBarrierNode::Oop));
-}
-}
-phase->register_new_node(phi, region);
-replace_barrier(phase, lb, phi);
-if (region->is_Loop()) {
-// Load barrier moved to the back edge of the Loop may now
-// have a safepoint on the path to the barrier on the Similar
-// edge
-igvn.replace_input_of(phi->in(LoopNode::LoopBackControl)->in(0), LoadBarrierNode::Similar, C->top());
-Node* head = region->in(LoopNode::EntryControl);
-phase->set_idom(region, head, phase->dom_depth(head)+1);
-phase->recompute_dom_depth();
-if (head->is_CountedLoop() && head->as_CountedLoop()->is_main_loop()) {
-head->as_CountedLoop()->set_normal_loop();
-}
-}
-return true;
-}
-}
-return false;
-}
-static bool move_out_of_loop(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
-PhaseIterGVN &igvn = phase->igvn();
-IdealLoopTree *lb_loop = phase->get_loop(lb->in(0));
-if (lb_loop != phase->ltree_root() && !lb_loop->_irreducible) {
-Node* oop_ctrl = phase->get_ctrl(lb->in(LoadBarrierNode::Oop));
-IdealLoopTree *oop_loop = phase->get_loop(oop_ctrl);
-IdealLoopTree* adr_loop = phase->get_loop(phase->get_ctrl(lb->in(LoadBarrierNode::Address)));
-if (!lb_loop->is_member(oop_loop) && !lb_loop->is_member(adr_loop)) {
-// That transformation may cause the Similar edge on dominated load barriers to be invalid
-lb->fix_similar_in_uses(&igvn);
-Node* head = lb_loop->_head;
-assert(head->is_Loop(), "");
-if (phase->is_dominator(head, oop_ctrl)) {
-assert(oop_ctrl->Opcode() == Op_CProj && oop_ctrl->in(0)->Opcode() == Op_NeverBranch, "");
-assert(lb_loop->is_member(phase->get_loop(oop_ctrl->in(0)->in(0))), "");
-return false;
-}
-if (head->is_CountedLoop()) {
-CountedLoopNode* cloop = head->as_CountedLoop();
-if (cloop->is_main_loop()) {
-cloop->set_normal_loop();
-}
-// When we are moving barrier out of a counted loop,
-// make sure we move it all the way out of the strip mined outer loop.
-if (cloop->is_strip_mined()) {
-head = cloop->outer_loop();
-}
-}
-Node* mem = lb->in(LoadBarrierNode::Memory);
-Node* m = find_dominating_memory(phase, mem, head, -1);
-LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, head->in(LoopNode::EntryControl), m, NULL);
-assert(phase->idom(head) == head->in(LoopNode::EntryControl), "");
-Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
-igvn.replace_input_of(head, LoopNode::EntryControl, proj_ctl);
-phase->set_idom(head, proj_ctl, phase->dom_depth(proj_ctl) + 1);
-replace_barrier(phase, lb, new_lb->proj_out(LoadBarrierNode::Oop));
-phase->recompute_dom_depth();
-return true;
-}
-}
-return false;
-}
-static bool common_barriers(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
-PhaseIterGVN &igvn = phase->igvn();
-Node* in_val = lb->in(LoadBarrierNode::Oop);
-for (DUIterator_Fast imax, i = in_val->fast_outs(imax); i < imax; i++) {
-Node* u = in_val->fast_out(i);
-if (u != lb && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
-Node* this_ctrl = lb->in(LoadBarrierNode::Control);
-Node* other_ctrl = u->in(LoadBarrierNode::Control);
-Node* lca = phase->dom_lca(this_ctrl, other_ctrl);
-Node* proj1 = NULL;
-Node* proj2 = NULL;
-bool ok = (lb->in(LoadBarrierNode::Address) == u->in(LoadBarrierNode::Address));
-while (this_ctrl != lca && ok) {
-if (this_ctrl->in(0) != NULL &&
-this_ctrl->in(0)->is_MultiBranch()) {
-if (this_ctrl->in(0)->in(0) == lca) {
-assert(proj1 == NULL, "");
-assert(this_ctrl->is_Proj(), "");
-proj1 = this_ctrl;
-} else if (!(this_ctrl->in(0)->is_If() && this_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
-ok = false;
-}
-}
-this_ctrl = phase->idom(this_ctrl);
-}
-while (other_ctrl != lca && ok) {
-if (other_ctrl->in(0) != NULL &&
-other_ctrl->in(0)->is_MultiBranch()) {
-if (other_ctrl->in(0)->in(0) == lca) {
-assert(other_ctrl->is_Proj(), "");
-assert(proj2 == NULL, "");
-proj2 = other_ctrl;
-} else if (!(other_ctrl->in(0)->is_If() && other_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
-ok = false;
-}
-}
-other_ctrl = phase->idom(other_ctrl);
-}
-assert(proj1 == NULL || proj2 == NULL || proj1->in(0) == proj2->in(0), "");
-if (ok && proj1 && proj2 && proj1 != proj2 && proj1->in(0)->is_If()) {
-// That transformation may cause the Similar edge on dominated load barriers to be invalid
-lb->fix_similar_in_uses(&igvn);
-u->as_LoadBarrier()->fix_similar_in_uses(&igvn);
-Node* split = lca->unique_ctrl_out();
-assert(split->in(0) == lca, "");
-Node* mem = lb->in(LoadBarrierNode::Memory);
-Node* m = find_dominating_memory(phase, mem, split, -1);
-LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, lca, m, NULL);
-Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
-igvn.replace_input_of(split, 0, new_lb->proj_out(LoadBarrierNode::Control));
-phase->set_idom(split, proj_ctl, phase->dom_depth(proj_ctl)+1);
-Node* proj_oop = new_lb->proj_out(LoadBarrierNode::Oop);
-replace_barrier(phase, lb, proj_oop);
-replace_barrier(phase, u->as_LoadBarrier(), proj_oop);
-phase->recompute_dom_depth();
-return true;
-}
-}
-}
-return false;
-}
-void ZBarrierSetC2::loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round) {
-if (!Compile::current()->directive()->ZOptimizeLoadBarriersOption) {
-return;
-}
-if (!node->is_LoadBarrier()) {
-return;
-}
-if (!node->as_LoadBarrier()->has_true_uses()) {
-return;
-}
-if (replace_with_dominating_barrier(phase, node->as_LoadBarrier(), last_round)) {
-return;
-}
-if (split_barrier_thru_phi(phase, node->as_LoadBarrier())) {
-return;
-}
-if (move_out_of_loop(phase, node->as_LoadBarrier())) {
-return;
-}
-if (common_barriers(phase, node->as_LoadBarrier())) {
-return;
-}
-}
 Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const {
 Node* node = c;
 // 1. This step follows potential oop projections of a load barrier before expansion
 if (node->is_Proj()) {
 // 3. This step checks for the phi corresponding to an optimized load barrier expansion
 if (node->is_Phi()) {
 PhiNode* phi = node->as_Phi();
 Node* n = phi->in(1);
-if (n != NULL && (n->is_LoadBarrierSlowReg() ||  n->is_LoadBarrierWeakSlowReg())) {
+if (n != NULL && n->is_LoadBarrierSlowReg()) {
 assert(c == node, "projections from step 1 should only be seen before macro expansion");
 return phi->in(2);
 }
 }
 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
 return type == T_OBJECT || type == T_ARRAY;
 }
 bool ZBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
-if (opcode != Op_LoadBarrierSlowReg &&
+switch (opcode) {
-opcode != Op_LoadBarrierWeakSlowReg) {
+case Op_LoadBarrier:
-return false;
+assert(0, "There should be no load barriers left");
-}
+case Op_ZGetAndSetP:
+case Op_ZCompareAndExchangeP:
+case Op_ZCompareAndSwapP:
+case Op_ZWeakCompareAndSwapP:
+case Op_LoadBarrierSlowReg:
 #ifdef ASSERT
 if (VerifyOptoOopOffsets) {
-MemNode* mem  = n->as_Mem();
+MemNode *mem = n->as_Mem();
 // Check to see if address types have grounded out somehow.
-const TypeInstPtr* tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr();
+const TypeInstPtr *tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr();
-ciInstanceKlass* k = tp->klass()->as_instance_klass();
+ciInstanceKlass *k = tp->klass()->as_instance_klass();
 bool oop_offset_is_sane = k->contains_field_offset(tp->offset());
 assert(!tp || oop_offset_is_sane, "");
 }
 #endif
+return true;
-return true;
+default:
+return false;
+}
 }
 bool ZBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const {
-if (opcode == Op_CallLeaf &&
+switch(opcode) {
-(n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() ||
+case Op_CallLeaf:
-n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr())) {
+if (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() ||
-mem_op = true;
+n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr()) {
-mem_addr_idx = TypeFunc::Parms + 1;
+mem_op = true;
-return true;
+mem_addr_idx = TypeFunc::Parms + 1;
-}
+return true;
+}
 return false;
+default:
+return false;
+}
+}
+bool ZBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const {
+switch(opcode) {
+case Op_ZCompareAndExchangeP:
+case Op_ZCompareAndSwapP:
+case Op_ZWeakCompareAndSwapP: {
+Node *mem = n->in(MemNode::Address);
+Node *keepalive = n->in(5);
+Node *pair1 = new BinaryNode(mem, keepalive);
+Node *newval = n->in(MemNode::ValueIn);
+Node *oldval = n->in(LoadStoreConditionalNode::ExpectedIn);
+Node *pair2 = new BinaryNode(oldval, newval);
+n->set_req(MemNode::Address, pair1);
+n->set_req(MemNode::ValueIn, pair2);
+n->del_req(5);
+n->del_req(LoadStoreConditionalNode::ExpectedIn);
+return true;
+}
+case Op_ZGetAndSetP: {
+Node *keepalive = n->in(4);
+Node *newval = n->in(MemNode::ValueIn);
+Node *pair = new BinaryNode(newval, keepalive);
+n->set_req(MemNode::ValueIn, pair);
+n->del_req(4);
+return true;
+}
+default:
+return false;
+}
 }
 // == Verification ==
 #ifdef ASSERT
 return true;
 }
 void ZBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
-if (phase == BarrierSetC2::BeforeCodeGen) return;
+switch(phase) {
-bool post_parse = phase == BarrierSetC2::BeforeOptimize;
+case BarrierSetC2::BeforeOptimize:
-verify_gc_barriers(post_parse);
+case BarrierSetC2::BeforeLateInsertion:
-}
+assert(state()->load_barrier_count() == 0, "No barriers inserted yet");
+break;
+case BarrierSetC2::BeforeMacroExpand:
+// Barrier placement should be set by now.
+verify_gc_barriers(false /*post_parse*/);
+break;
+case BarrierSetC2::BeforeCodeGen:
+// Barriers has been fully expanded.
+assert(state()->load_barrier_count() == 0, "No more macro barriers");
+break;
+default:
+assert(0, "Phase without verification");
+}
+}
+// post_parse implies that there might be load barriers without uses after parsing
+// That only applies when adding barriers at parse time.
 void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const {
 ZBarrierSetC2State* s = state();
 Compile* C = Compile::current();
 ResourceMark rm;
 VectorSet visited(Thread::current()->resource_area());
 for (int i = 0; i < s->load_barrier_count(); i++) {
 LoadBarrierNode* n = s->load_barrier_node(i);
 // The dominating barrier on the same address if it exists and
 // this barrier must not be applied on the value from the same
 (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) &&
 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)),
 "broken similar edge");
-assert(post_parse || n->as_LoadBarrier()->has_true_uses(),
+assert(n->as_LoadBarrier()->has_true_uses(),
 "found unneeded load barrier");
 // Several load barrier nodes chained through their Similar edge
 // break the code that remove the barriers in final graph reshape.
 assert(n->in(LoadBarrierNode::Similar)->is_top() ||
 if (!n->in(LoadBarrierNode::Similar)->is_top()) {
 ResourceMark rm;
 Unique_Node_List wq;
 Node* other = n->in(LoadBarrierNode::Similar)->in(0);
 wq.push(n);
-bool ok = true;
-bool dom_found = false;
 for (uint next = 0; next < wq.size(); ++next) {
-Node *n = wq.at(next);
+Node *nn = wq.at(next);
-assert(n->is_CFG(), "");
+assert(nn->is_CFG(), "");
-assert(!n->is_SafePoint(), "");
+assert(!nn->is_SafePoint(), "");
-if (n == other) {
+if (nn == other) {
 continue;
 }
-if (n->is_Region()) {
+if (nn->is_Region()) {
-for (uint i = 1; i < n->req(); i++) {
+for (uint i = 1; i < nn->req(); i++) {
-Node* m = n->in(i);
+Node* m = nn->in(i);
 if (m != NULL) {
 wq.push(m);
 }
 }
 } else {
-Node* m = n->in(0);
+Node* m = nn->in(0);
 if (m != NULL) {
 wq.push(m);
 }
 }
 }
 }
+}
-if (ZVerifyLoadBarriers) {
+}
-if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) {
-visited.Clear();
+#endif // end verification code
-bool found = look_for_barrier(n, post_parse, visited);
-if (!found) {
+static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl);
-n->dump(1);
-n->dump(-3);
+// This code is cloning all uses of a load that is between a call and the catch blocks,
-stringStream ss;
+// to each use.
-C->method()->print_short_name(&ss);
-tty->print_cr("-%s-", ss.as_string());
+static bool fixup_uses_in_catch(PhaseIdealLoop *phase, Node *start_ctrl, Node *node) {
-assert(found, "");
+if (!phase->has_ctrl(node)) {
+// This node is floating - doesn't need to be cloned.
+assert(node != start_ctrl, "check");
+return false;
+}
+Node* ctrl = phase->get_ctrl(node);
+if (ctrl != start_ctrl) {
+// We are in a successor block - the node is ok.
+return false; // Unwind
+}
+// Process successor nodes
+int outcnt = node->outcnt();
+for (int i = 0; i < outcnt; i++) {
+Node* n = node->raw_out(0);
+assert(!n->is_LoadBarrier(), "Sanity");
+// Calling recursively, visiting leafs first
+fixup_uses_in_catch(phase, start_ctrl, n);
+}
+// Now all successors are outside
+// - Clone this node to both successors
+int no_succs = node->outcnt();
+assert(!node->is_Store(), "Stores not expected here");
+// In some very rare cases a load that doesn't need a barrier will end up here
+// Treat it as a LoadP and the insertion of phis will be done correctly.
+if (node->is_Load()) {
+assert(node->as_Load()->barrier_data() == 0, "Sanity");
+call_catch_cleanup_one(phase, node->as_Load(), phase->get_ctrl(node));
+} else {
+for (DUIterator_Fast jmax, i = node->fast_outs(jmax); i < jmax; i++) {
+Node* use = node->fast_out(i);
+Node* clone = node->clone();
+assert(clone->outcnt() == 0, "");
+assert(use->find_edge(node) != -1, "check");
+phase->igvn().rehash_node_delayed(use);
+use->replace_edge(node, clone);
+Node* new_ctrl;
+if (use->is_block_start()) {
+new_ctrl = use;
+} else if (use->is_CFG()) {
+new_ctrl = use->in(0);
+assert (new_ctrl != NULL, "");
+} else {
+new_ctrl = phase->get_ctrl(use);
+}
+phase->set_ctrl(clone, new_ctrl);
+if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr("  Clone op %i as %i to control %i", node->_idx, clone->_idx, new_ctrl->_idx);
+phase->igvn().register_new_node_with_optimizer(clone);
+--i, --jmax;
+}
+assert(node->outcnt() == 0, "must be empty now");
+// Node node is dead.
+phase->igvn().remove_dead_node(node);
+}
+return true; // unwind - return if a use was processed
+}
+// Clone a load to a specific catch_proj
+static Node* clone_load_to_catchproj(PhaseIdealLoop* phase, Node* load, Node* catch_proj) {
+Node* cloned_load = load->clone();
+cloned_load->set_req(0, catch_proj);      // set explicit control
+phase->set_ctrl(cloned_load, catch_proj); // update
+if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr("  Clone LOAD %i as %i to control %i", load->_idx, cloned_load->_idx, catch_proj->_idx);
+phase->igvn().register_new_node_with_optimizer(cloned_load);
+return cloned_load;
+}
+static Node* get_dominating_region(PhaseIdealLoop* phase, Node* node, Node* stop) {
+Node* region = node;
+while (!region->isa_Region()) {
+Node *up = phase->idom(region);
+assert(up != region, "Must not loop");
+assert(up != stop,   "Must not find original control");
+region = up;
+}
+return region;
+}
+// Clone this load to each catch block
+static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) {
+bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
+phase->igvn().set_delay_transform(true);
+// Verify pre conditions
+assert(ctrl->isa_Proj() && ctrl->in(0)->isa_Call(), "Must be a call proj");
+assert(ctrl->raw_out(0)->isa_Catch(), "Must be a catch");
+if (ctrl->raw_out(0)->isa_Catch()->outcnt() == 1) {
+if (trace) tty->print_cr("Cleaning up catch: Skipping load %i, call with single catch", load->_idx);
+return;
+}
+// Process the loads successor nodes - if any is between
+// the call and the catch blocks, they need to be cloned to.
+// This is done recursively
+int outcnt = load->outcnt();
+uint index = 0;
+for (int i = 0; i < outcnt; i++) {
+if (index < load->outcnt()) {
+Node *n = load->raw_out(index);
+assert(!n->is_LoadBarrier(), "Sanity");
+if (!fixup_uses_in_catch(phase, ctrl, n)) {
+// if no successor was cloned, progress to next out.
+index++;
+}
+}
+}
+// Now all the loads uses has been cloned down
+// Only thing left is to clone the loads, but they must end up
+// first in the catch blocks.
+// We clone the loads oo the catch blocks only when needed.
+// An array is used to map the catch blocks to each lazily cloned load.
+// In that way no extra unnecessary loads are cloned.
+// Any use dominated by original block must have an phi and a region added
+Node* catch_node = ctrl->raw_out(0);
+int number_of_catch_projs = catch_node->outcnt();
+Node** proj_to_load_mapping = NEW_RESOURCE_ARRAY(Node*, number_of_catch_projs);
+Copy::zero_to_bytes(proj_to_load_mapping, sizeof(Node*) * number_of_catch_projs);
+// The phi_map is used to keep track of where phis have already been inserted
+int phi_map_len = phase->C->unique();
+Node** phi_map = NEW_RESOURCE_ARRAY(Node*, phi_map_len);
+Copy::zero_to_bytes(phi_map, sizeof(Node*) * phi_map_len);
+for (unsigned int i = 0; i  < load->outcnt(); i++) {
+Node* load_use_control = NULL;
+Node* load_use = load->raw_out(i);
+if (phase->has_ctrl(load_use)) {
+load_use_control = phase->get_ctrl(load_use);
+} else {
+load_use_control = load_use->in(0);
+}
+assert(load_use_control != NULL, "sanity");
+if (trace) tty->print_cr("  Handling use: %i, with control: %i", load_use->_idx, load_use_control->_idx);
+// Some times the loads use is a phi. For them we need to determine from which catch block
+// the use is defined.
+bool load_use_is_phi = false;
+unsigned int load_use_phi_index = 0;
+Node* phi_ctrl = NULL;
+if (load_use->is_Phi()) {
+// Find phi input that matches load
+for (unsigned int u = 1; u < load_use->req(); u++) {
+if (load_use->in(u) == load) {
+load_use_is_phi = true;
+load_use_phi_index = u;
+assert(load_use->in(0)->is_Region(), "Region or broken");
+phi_ctrl = load_use->in(0)->in(u);
+assert(phi_ctrl->is_CFG(), "check");
+assert(phi_ctrl != load,   "check");
+break;
 }
 }
-}
+assert(load_use_is_phi,        "must find");
-}
+assert(load_use_phi_index > 0, "sanity");
 }
-#endif
+// For each load use, see which catch projs dominates, create load clone lazily and reconnect
+bool found_dominating_catchproj = false;
-bool ZBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const {
+for (int c = 0; c < number_of_catch_projs; c++) {
-switch (opcode) {
+Node* catchproj = catch_node->raw_out(c);
-case Op_LoadBarrierSlowReg:
+assert(catchproj != NULL && catchproj->isa_CatchProj(), "Sanity");
-case Op_LoadBarrierWeakSlowReg:
-conn_graph->add_objload_to_connection_graph(n, delayed_worklist);
+if (!phase->is_dominator(catchproj, load_use_control)) {
-return true;
+if (load_use_is_phi && phase->is_dominator(catchproj, phi_ctrl)) {
+// The loads use is local to the catchproj.
-case Op_Proj:
+// fall out and replace load with catch-local load clone.
-if (n->as_Proj()->_con != LoadBarrierNode::Oop || !n->in(0)->is_LoadBarrier()) {
+} else {
-return false;
+continue;
 }
-conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), delayed_worklist);
+}
-return true;
+assert(!found_dominating_catchproj, "Max one should match");
-}
+// Clone loads to catch projs
-return false;
+Node* load_clone = proj_to_load_mapping[c];
-}
+if (load_clone == NULL) {
+load_clone = clone_load_to_catchproj(phase, load, catchproj);
-bool ZBarrierSetC2::escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const {
+proj_to_load_mapping[c] = load_clone;
-switch (opcode) {
+}
-case Op_LoadBarrierSlowReg:
+phase->igvn().rehash_node_delayed(load_use);
-case Op_LoadBarrierWeakSlowReg:
-if (gvn->type(n)->make_ptr() == NULL) {
+if (load_use_is_phi) {
-return false;
+// phis are special - the load is defined from a specific control flow
-}
+load_use->set_req(load_use_phi_index, load_clone);
-conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(MemNode::Address), NULL);
+} else {
-return true;
+// Multipe edges can be replaced at once - on calls for example
+load_use->replace_edge(load, load_clone);
-case Op_Proj:
+}
-if (n->as_Proj()->_con != LoadBarrierNode::Oop || !n->in(0)->is_LoadBarrier()) {
+--i; // more than one edge can have been removed, but the next is in later iterations
-return false;
-}
+// We could break the for-loop after finding a dominating match.
-conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), NULL);
+// But keep iterating to catch any bad idom early.
-return true;
+found_dominating_catchproj = true;
 }
-return false;
+// We found no single catchproj that dominated the use - The use is at a point after
-}
+// where control flow from multiple catch projs have merged. We will have to create
+// phi nodes before the use and tie the output from the cloned loads together. It
+// can be a single phi or a number of chained phis, depending on control flow
+if (!found_dominating_catchproj) {
+// Use phi-control if use is a phi
+if (load_use_is_phi) {
+load_use_control = phi_ctrl;
+}
+assert(phase->is_dominator(ctrl, load_use_control), "Common use but no dominator");
+// Clone a load on all paths
+for (int c = 0; c < number_of_catch_projs; c++) {
+Node* catchproj = catch_node->raw_out(c);
+Node* load_clone = proj_to_load_mapping[c];
+if (load_clone == NULL) {
+load_clone = clone_load_to_catchproj(phase, load, catchproj);
+proj_to_load_mapping[c] = load_clone;
+}
+}
+// Move up dominator tree from use until dom front is reached
+Node* next_region = get_dominating_region(phase, load_use_control, ctrl);
+while (phase->idom(next_region) != catch_node) {
+next_region = phase->idom(next_region);
+if (trace) tty->print_cr("Moving up idom to region ctrl %i", next_region->_idx);
+}
+assert(phase->is_dominator(catch_node, next_region), "Sanity");
+// Create or reuse phi node that collect all cloned loads and feed it to the use.
+Node* test_phi = phi_map[next_region->_idx];
+if ((test_phi != NULL) && test_phi->is_Phi()) {
+// Reuse an already created phi
+if (trace) tty->print_cr("    Using cached Phi %i on load_use %i", test_phi->_idx, load_use->_idx);
+phase->igvn().rehash_node_delayed(load_use);
+load_use->replace_edge(load, test_phi);
+// Now this use is done
+} else {
+// Otherwise we need to create one or more phis
+PhiNode* next_phi = new PhiNode(next_region, load->type());
+phi_map[next_region->_idx] = next_phi; // cache new phi
+phase->igvn().rehash_node_delayed(load_use);
+load_use->replace_edge(load, next_phi);
+int dominators_of_region = 0;
+do {
+// New phi, connect to region and add all loads as in.
+Node* region = next_region;
+assert(region->isa_Region() && region->req() > 2, "Catch dead region nodes");
+PhiNode* new_phi = next_phi;
+if (trace) tty->print_cr("Created Phi %i on load %i with control %i", new_phi->_idx, load->_idx, region->_idx);
+// Need to add all cloned loads to the phi, taking care that the right path is matched
+dominators_of_region = 0; // reset for new region
+for (unsigned int reg_i = 1; reg_i < region->req(); reg_i++) {
+Node* region_pred = region->in(reg_i);
+assert(region_pred->is_CFG(), "check");
+bool pred_has_dominator = false;
+for (int c = 0; c < number_of_catch_projs; c++) {
+Node* catchproj = catch_node->raw_out(c);
+if (phase->is_dominator(catchproj, region_pred)) {
+new_phi->set_req(reg_i, proj_to_load_mapping[c]);
+if (trace) tty->print_cr(" - Phi in(%i) set to load %i", reg_i, proj_to_load_mapping[c]->_idx);
+pred_has_dominator = true;
+dominators_of_region++;
+break;
+}
+}
+// Sometimes we need to chain several phis.
+if (!pred_has_dominator) {
+assert(dominators_of_region <= 1, "More than one region can't require extra phi");
+if (trace) tty->print_cr(" - Region %i pred %i not dominated by catch proj", region->_idx, region_pred->_idx);
+// Continue search on on this region_pred
+// - walk up to next region
+// - create a new phi and connect to first new_phi
+next_region = get_dominating_region(phase, region_pred, ctrl);
+// Lookup if there already is a phi, create a new otherwise
+Node* test_phi = phi_map[next_region->_idx];
+if ((test_phi != NULL) && test_phi->is_Phi()) {
+next_phi = test_phi->isa_Phi();
+dominators_of_region++; // record that a match was found and that we are done
+if (trace) tty->print_cr("    Using cached phi Phi %i on control %i", next_phi->_idx, next_region->_idx);
+} else {
+next_phi = new PhiNode(next_region, load->type());
+phi_map[next_region->_idx] = next_phi;
+}
+new_phi->set_req(reg_i, next_phi);
+}
+}
+new_phi->set_req(0, region);
+phase->igvn().register_new_node_with_optimizer(new_phi);
+phase->set_ctrl(new_phi, region);
+assert(dominators_of_region != 0, "Must have found one this iteration");
+} while (dominators_of_region == 1);
+}
+--i;
+}
+} // end of loop over uses
+assert(load->outcnt() == 0, "All uses should be handled");
+phase->igvn().remove_dead_node(load);
+phase->C->print_method(PHASE_CALL_CATCH_CLEANUP, 4, load->_idx);
+// Now we should be home
+phase->igvn().set_delay_transform(false);
+}
+// Sort out the loads that are between a call ant its catch blocks
+static void process_catch_cleanup_candidate(PhaseIdealLoop* phase, LoadNode* load) {
+bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
+Node* ctrl = phase->get_ctrl(load);
+if (!ctrl->is_Proj() || (ctrl->in(0) == NULL) || !ctrl->in(0)->isa_Call()) {
+return;
+}
+Node* catch_node = ctrl->isa_Proj()->raw_out(0);
+if (catch_node->is_Catch()) {
+if (catch_node->outcnt() > 1) {
+call_catch_cleanup_one(phase, load, ctrl);
+} else {
+if (trace) tty->print_cr("Call catch cleanup with only one catch: load %i ", load->_idx);
+}
+}
+}
+void ZBarrierSetC2::barrier_insertion_phase(Compile* C, PhaseIterGVN& igvn) const {
+PhaseIdealLoop::optimize(igvn, LoopOptsZBarrierInsertion);
+if (C->failing())  return;
+}
+bool ZBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const {
+if (mode == LoopOptsZBarrierInsertion) {
+// First make sure all loads between call and catch are moved to the catch block
+clean_catch_blocks(phase);
+// Then expand barriers on all loads
+insert_load_barriers(phase);
+// Handle all Unsafe that need barriers.
+insert_barriers_on_unsafe(phase);
+phase->C->clear_major_progress();
+return true;
+} else {
+return false;
+}
+}
+static bool can_simplify_cas(LoadStoreNode* node) {
+if (node->isa_LoadStoreConditional()) {
+Node *expected_in = node->as_LoadStoreConditional()->in(LoadStoreConditionalNode::ExpectedIn);
+return (expected_in->get_ptr_type() == TypePtr::NULL_PTR);
+} else {
+return false;
+}
+}
+static void insert_barrier_before_unsafe(PhaseIdealLoop* phase, LoadStoreNode* old_node) {
+Compile *C = phase->C;
+PhaseIterGVN &igvn = phase->igvn();
+LoadStoreNode* zclone = NULL;
+bool is_weak = false;
+Node *in_ctrl = old_node->in(MemNode::Control);
+Node *in_mem  = old_node->in(MemNode::Memory);
+Node *in_adr  = old_node->in(MemNode::Address);
+Node *in_val  = old_node->in(MemNode::ValueIn);
+const TypePtr *adr_type = old_node->adr_type();
+const TypePtr* load_type = TypeOopPtr::BOTTOM; // The type for the load we are adding
+switch (old_node->Opcode()) {
+case Op_CompareAndExchangeP: {
+zclone = new ZCompareAndExchangePNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn),
+adr_type, old_node->get_ptr_type(), ((CompareAndExchangeNode*)old_node)->order());
+load_type = old_node->bottom_type()->is_ptr();
+break;
+}
+case Op_WeakCompareAndSwapP: {
+if (can_simplify_cas(old_node)) {
+break;
+}
+is_weak  = true;
+zclone = new ZWeakCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn),
+((CompareAndSwapNode*)old_node)->order());
+adr_type = TypePtr::BOTTOM;
+break;
+}
+case Op_CompareAndSwapP: {
+if (can_simplify_cas(old_node)) {
+break;
+}
+zclone = new ZCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn),
+((CompareAndSwapNode*)old_node)->order());
+adr_type = TypePtr::BOTTOM;
+break;
+}
+case Op_GetAndSetP: {
+zclone = new ZGetAndSetPNode(in_ctrl, in_mem, in_adr, in_val, old_node->adr_type(), old_node->get_ptr_type());
+load_type = old_node->bottom_type()->is_ptr();
+break;
+}
+}
+if (zclone != NULL) {
+igvn.register_new_node_with_optimizer(zclone, old_node);
+// Make load
+LoadPNode *load = new LoadPNode(NULL, in_mem, in_adr, adr_type, load_type, MemNode::unordered,
+LoadNode::DependsOnlyOnTest);
+load_set_expanded_barrier(load);
+igvn.register_new_node_with_optimizer(load);
+igvn.replace_node(old_node, zclone);
+Node *barrier = new LoadBarrierNode(C, NULL, in_mem, load, in_adr, is_weak);
+Node *barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop);
+Node *barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control);
+igvn.register_new_node_with_optimizer(barrier);
+igvn.register_new_node_with_optimizer(barrier_val);
+igvn.register_new_node_with_optimizer(barrier_ctrl);
+// loop over all of in_ctrl usages and move to barrier_ctrl
+for (DUIterator_Last imin, i = in_ctrl->last_outs(imin); i >= imin; --i) {
+Node *use = in_ctrl->last_out(i);
+uint l;
+for (l = 0; use->in(l) != in_ctrl; l++) {}
+igvn.replace_input_of(use, l, barrier_ctrl);
+}
+load->set_req(MemNode::Control, in_ctrl);
+barrier->set_req(LoadBarrierNode::Control, in_ctrl);
+zclone->add_req(barrier_val); // add req as keep alive.
+C->print_method(PHASE_ADD_UNSAFE_BARRIER, 4, zclone->_idx);
+}
+}
+void ZBarrierSetC2::insert_barriers_on_unsafe(PhaseIdealLoop* phase) const {
+Compile *C = phase->C;
+PhaseIterGVN &igvn = phase->igvn();
+uint new_ids = C->unique();
+VectorSet visited(Thread::current()->resource_area());
+GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL);
+nodeStack.push(C->root());
+visited.test_set(C->root()->_idx);
+// Traverse all nodes, visit all unsafe ops that require a barrier
+while (nodeStack.length() > 0) {
+Node *n = nodeStack.pop();
+bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup
+if (is_old_node) {
+if (n->is_LoadStore()) {
+LoadStoreNode* lsn = n->as_LoadStore();
+if (lsn->has_barrier()) {
+BasicType bt = lsn->in(MemNode::Address)->bottom_type()->basic_type();
+assert ((bt == T_OBJECT || bt == T_ARRAY), "Sanity test");
+insert_barrier_before_unsafe(phase, lsn);
+}
+}
+}
+for (uint i = 0; i < n->len(); i++) {
+if (n->in(i)) {
+if (!visited.test_set(n->in(i)->_idx)) {
+nodeStack.push(n->in(i));
+}
+}
+}
+}
+igvn.optimize();
+C->print_method(PHASE_ADD_UNSAFE_BARRIER, 2);
+}
+// The purpose of ZBarrierSetC2::clean_catch_blocks is to prepare the IR for
+// splicing in load barrier nodes.
+//
+// The problem is that we might have instructions between a call and its catch nodes.
+// (This is usually handled in PhaseCFG:call_catch_cleanup, which clones mach nodes in
+// already scheduled blocks.) We can't have loads that require barriers there,
+// because we need to splice in new control flow, and that would violate the IR.
+//
+// clean_catch_blocks find all Loads that require a barrier and clone them and any
+// dependent instructions to each use. The loads must be in the beginning of the catch block
+// before any store.
+//
+// Sometimes the loads use will be at a place dominated by all catch blocks, then we need
+// a load in each catch block, and a Phi at the dominated use.
+void ZBarrierSetC2::clean_catch_blocks(PhaseIdealLoop* phase) const {
+Compile *C = phase->C;
+uint new_ids = C->unique();
+PhaseIterGVN &igvn = phase->igvn();
+VectorSet visited(Thread::current()->resource_area());
+GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL);
+nodeStack.push(C->root());
+visited.test_set(C->root()->_idx);
+// Traverse all nodes, visit all loads that require a barrier
+while(nodeStack.length() > 0) {
+Node *n = nodeStack.pop();
+bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup
+if (n->is_Load() && is_old_node) {
+LoadNode* load = n->isa_Load();
+// only care about loads that will have a barrier
+if (load_require_barrier(load)) {
+process_catch_cleanup_candidate(phase, load);
+}
+}
+for (uint i = 0; i < n->len(); i++) {
+if (n->in(i)) {
+if (!visited.test_set(n->in(i)->_idx)) {
+nodeStack.push(n->in(i));
+}
+}
+}
+}
+C->print_method(PHASE_CALL_CATCH_CLEANUP, 2);
+}
+class DomDepthCompareClosure : public CompareClosure<LoadNode*> {
+PhaseIdealLoop* _phase;
+public:
+DomDepthCompareClosure(PhaseIdealLoop* phase) : _phase(phase) { }
+int do_compare(LoadNode* const &n1, LoadNode* const &n2) {
+int d1 = _phase->dom_depth(_phase->get_ctrl(n1));
+int d2 = _phase->dom_depth(_phase->get_ctrl(n2));
+if (d1 == d2) {
+// Compare index if the depth is the same, ensures all entries are unique.
+return n1->_idx - n2->_idx;
+} else {
+return d2 - d1;
+}
+}
+};
+// Traverse graph and add all loadPs to list, sorted by dom depth
+void gather_loadnodes_sorted(PhaseIdealLoop* phase, GrowableArray<LoadNode*>* loadList) {
+VectorSet visited(Thread::current()->resource_area());
+GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL);
+DomDepthCompareClosure ddcc(phase);
+nodeStack.push(phase->C->root());
+while(nodeStack.length() > 0) {
+Node *n = nodeStack.pop();
+if (visited.test(n->_idx)) {
+continue;
+}
+if (n->isa_Load()) {
+LoadNode *load = n->as_Load();
+if (load_require_barrier(load)) {
+assert(phase->get_ctrl(load) != NULL, "sanity");
+assert(phase->dom_depth(phase->get_ctrl(load)) != 0, "sanity");
+loadList->insert_sorted(&ddcc, load);
+}
+}
+visited.set(n->_idx);
+for (uint i = 0; i < n->req(); i++) {
+if (n->in(i)) {
+if (!visited.test(n->in(i)->_idx)) {
+nodeStack.push(n->in(i));
+}
+}
+}
+}
+}
+// Add LoadBarriers to all LoadPs
+void ZBarrierSetC2::insert_load_barriers(PhaseIdealLoop* phase) const {
+bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
+GrowableArray<LoadNode *> loadList(Thread::current()->resource_area(), 0, 0, NULL);
+gather_loadnodes_sorted(phase, &loadList);
+PhaseIterGVN &igvn = phase->igvn();
+int count = 0;
+for (GrowableArrayIterator<LoadNode *> loadIter = loadList.begin(); loadIter != loadList.end(); ++loadIter) {
+LoadNode *load = *loadIter;
+if (load_has_expanded_barrier(load)) {
+continue;
+}
+do {
+// Insert a barrier on a loadP
+// if another load is found that needs to be expanded first, retry on that one
+LoadNode* result = insert_one_loadbarrier(phase, load, phase->get_ctrl(load));
+while (result != NULL) {
+result = insert_one_loadbarrier(phase, result, phase->get_ctrl(result));
+}
+} while (!load_has_expanded_barrier(load));
+}
+phase->C->print_method(PHASE_INSERT_BARRIER, 2);
+}
+void push_antidependent_stores(PhaseIdealLoop* phase, Node_Stack& nodestack, LoadNode* start_load) {
+// push all stores on the same mem, that can_alias
+// Any load found must be handled first
+PhaseIterGVN &igvn = phase->igvn();
+int load_alias_idx = igvn.C->get_alias_index(start_load->adr_type());
+Node *mem = start_load->in(1);
+for (DUIterator_Fast imax, u = mem->fast_outs(imax); u < imax; u++) {
+Node *mem_use = mem->fast_out(u);
+if (mem_use == start_load) continue;
+if (!mem_use->is_Store()) continue;
+if (!phase->has_ctrl(mem_use)) continue;
+if (phase->get_ctrl(mem_use) != phase->get_ctrl(start_load)) continue;
+// add any aliasing store in this block
+StoreNode *store = mem_use->isa_Store();
+const TypePtr *adr_type = store->adr_type();
+if (igvn.C->can_alias(adr_type, load_alias_idx)) {
+nodestack.push(store, 0);
+}
+}
+}
+LoadNode* ZBarrierSetC2::insert_one_loadbarrier(PhaseIdealLoop* phase, LoadNode* start_load, Node* ctrl) const {
+bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
+PhaseIterGVN &igvn = phase->igvn();
+// Check for other loadPs at the same loop depth that is reachable by a DFS
+// - if found - return it. It needs to be inserted first
+// - otherwise proceed and insert barrier
+VectorSet visited(Thread::current()->resource_area());
+Node_Stack nodestack(100);
+nodestack.push(start_load, 0);
+push_antidependent_stores(phase, nodestack, start_load);
+while(!nodestack.is_empty()) {
+Node* n = nodestack.node(); // peek
+nodestack.pop();
+if (visited.test(n->_idx)) {
+continue;
+}
+if (n->is_Load() && n != start_load && load_require_barrier(n->as_Load()) && !load_has_expanded_barrier(n->as_Load())) {
+// Found another load that needs a barrier in the same block. Must expand later loads first.
+if (trace) tty->print_cr(" * Found LoadP %i on DFS", n->_idx);
+return n->as_Load(); // return node that should be expanded first
+}
+if (!phase->has_ctrl(n)) continue;
+if (phase->get_ctrl(n) != phase->get_ctrl(start_load)) continue;
+if (n->is_Phi()) continue;
+visited.set(n->_idx);
+// push all children
+for (DUIterator_Fast imax, ii = n->fast_outs(imax); ii < imax; ii++) {
+Node* c = n->fast_out(ii);
+if (c != NULL) {
+nodestack.push(c, 0);
+}
+}
+}
+insert_one_loadbarrier_inner(phase, start_load, ctrl, visited);
+return NULL;
+}
+void ZBarrierSetC2::insert_one_loadbarrier_inner(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl, VectorSet visited2) const {
+PhaseIterGVN &igvn = phase->igvn();
+Compile* C = igvn.C;
+bool trace = C->directive()->ZTraceLoadBarriersOption;
+// create barrier
+Node* barrier = new LoadBarrierNode(C, NULL, load->in(LoadNode::Memory), NULL, load->in(LoadNode::Address), load_has_weak_barrier(load));
+Node* barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop);
+Node* barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control);
+if (trace) tty->print_cr("Insert load %i with barrier: %i and ctrl : %i", load->_idx, barrier->_idx, ctrl->_idx);
+// Splice control
+// - insert barrier control diamond between loads ctrl and ctrl successor on path to block end.
+// - If control successor is a catch, step over to next.
+Node* ctrl_succ = NULL;
+for (DUIterator_Fast imax, j = ctrl->fast_outs(imax); j < imax; j++) {
+Node* tmp = ctrl->fast_out(j);
+// - CFG nodes is the ones we are going to splice (1 only!)
+// - Phi nodes will continue to hang from the region node!
+// - self loops should be skipped
+if (tmp->is_Phi() || tmp == ctrl) {
+continue;
+}
+if (tmp->is_CFG()) {
+assert(ctrl_succ == NULL, "There can be only one");
+ctrl_succ = tmp;
+continue;
+}
+}
+// Now splice control
+assert(ctrl_succ != load, "sanity");
+assert(ctrl_succ != NULL, "Broken IR");
+bool found = false;
+for(uint k = 0; k < ctrl_succ->req(); k++) {
+if (ctrl_succ->in(k) == ctrl) {
+assert(!found, "sanity");
+if (trace) tty->print_cr(" Move CFG ctrl_succ %i to barrier_ctrl", ctrl_succ->_idx);
+igvn.replace_input_of(ctrl_succ, k, barrier_ctrl);
+found = true;
+k--;
+}
+}
+// For all successors of ctrl - move all visited to become successors of barrier_ctrl instead
+for (DUIterator_Fast imax, r = ctrl->fast_outs(imax); r < imax; r++) {
+Node* tmp = ctrl->fast_out(r);
+if (visited2.test(tmp->_idx) && (tmp != load)) {
+if (trace) tty->print_cr(" Move ctrl_succ %i to barrier_ctrl", tmp->_idx);
+igvn.replace_input_of(tmp, 0, barrier_ctrl);
+--r; --imax;
+}
+}
+// Move the loads user to the barrier
+for (DUIterator_Fast imax, i = load->fast_outs(imax); i < imax; i++) {
+Node* u = load->fast_out(i);
+if (u->isa_LoadBarrier()) {
+continue;
+}
+// find correct input  - replace with iterator?
+for(uint j = 0; j < u->req(); j++) {
+if (u->in(j) == load) {
+igvn.replace_input_of(u, j, barrier_val);
+--i; --imax; // Adjust the iterator of the *outer* loop
+break; // some nodes (calls) might have several uses from the same node
+}
+}
+}
+// Connect barrier to load and control
+barrier->set_req(LoadBarrierNode::Oop, load);
+barrier->set_req(LoadBarrierNode::Control, ctrl);
+igvn.rehash_node_delayed(load);
+igvn.register_new_node_with_optimizer(barrier);
+igvn.register_new_node_with_optimizer(barrier_val);
+igvn.register_new_node_with_optimizer(barrier_ctrl);
+load_set_expanded_barrier(load);
+C->print_method(PHASE_INSERT_BARRIER, 3, load->_idx);
+}
+// The bad_mask in the ThreadLocalData shouldn't have an anti-dep-check.
+// The bad_mask address if of type TypeRawPtr, but that will alias
+// InitializeNodes until the type system is expanded.
+bool ZBarrierSetC2::needs_anti_dependence_check(const Node* node) const {
+MachNode* mnode = node->as_Mach();
+if (mnode != NULL) {
+intptr_t offset = 0;
+const TypePtr *adr_type2 = NULL;
+const Node* base = mnode->get_base_and_disp(offset, adr_type2);
+if ((base != NULL) &&
+(base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_ThreadLocal) &&
+(offset == in_bytes(ZThreadLocalData::address_bad_mask_offset()))) {
+return false;
+}
+}
+return true;
+}

changeset 55307	ed12027517c0
parent 55123	7b70f269fe0d
child 55321	ddda023e6f66