--- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp Thu Oct 17 20:27:44 2019 +0100
+++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp Thu Oct 17 20:53:35 2019 +0100
@@ -22,666 +22,157 @@
*/
#include "precompiled.hpp"
-#include "opto/compile.hpp"
-#include "opto/castnode.hpp"
-#include "opto/escape.hpp"
-#include "opto/graphKit.hpp"
-#include "opto/idealKit.hpp"
-#include "opto/loopnode.hpp"
-#include "opto/macro.hpp"
-#include "opto/node.hpp"
-#include "opto/type.hpp"
-#include "utilities/macros.hpp"
+#include "classfile/javaClasses.hpp"
+#include "gc/z/c2/zBarrierSetC2.hpp"
#include "gc/z/zBarrierSet.hpp"
-#include "gc/z/c2/zBarrierSetC2.hpp"
-#include "gc/z/zThreadLocalData.hpp"
+#include "gc/z/zBarrierSetAssembler.hpp"
#include "gc/z/zBarrierSetRuntime.hpp"
+#include "opto/block.hpp"
+#include "opto/compile.hpp"
+#include "opto/graphKit.hpp"
+#include "opto/machnode.hpp"
+#include "opto/memnode.hpp"
+#include "opto/node.hpp"
+#include "opto/regalloc.hpp"
+#include "opto/rootnode.hpp"
+#include "utilities/growableArray.hpp"
+#include "utilities/macros.hpp"
-ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena) :
- _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8, 0, NULL)) {}
+class ZBarrierSetC2State : public ResourceObj {
+private:
+ GrowableArray<ZLoadBarrierStubC2*>* _stubs;
+ Node_Array _live;
-int ZBarrierSetC2State::load_barrier_count() const {
- return _load_barrier_nodes->length();
-}
+public:
+ ZBarrierSetC2State(Arena* arena) :
+ _stubs(new (arena) GrowableArray<ZLoadBarrierStubC2*>(arena, 8, 0, NULL)),
+ _live(arena) {}
-void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) {
- assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list");
- _load_barrier_nodes->append(n);
-}
-
-void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) {
- // this function may be called twice for a node so check
- // that the node is in the array before attempting to remove it
- if (_load_barrier_nodes->contains(n)) {
- _load_barrier_nodes->remove(n);
+ GrowableArray<ZLoadBarrierStubC2*>* stubs() {
+ return _stubs;
}
-}
-LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const {
- return _load_barrier_nodes->at(idx);
-}
+ RegMask* live(const Node* node) {
+ if (!node->is_Mach()) {
+ // Don't need liveness for non-MachNodes
+ return NULL;
+ }
-void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
- return new(comp_arena) ZBarrierSetC2State(comp_arena);
-}
+ const MachNode* const mach = node->as_Mach();
+ if (mach->barrier_data() != ZLoadBarrierStrong &&
+ mach->barrier_data() != ZLoadBarrierWeak) {
+ // Don't need liveness data for nodes without barriers
+ return NULL;
+ }
-ZBarrierSetC2State* ZBarrierSetC2::state() const {
+ RegMask* live = (RegMask*)_live[node->_idx];
+ if (live == NULL) {
+ live = new (Compile::current()->comp_arena()->Amalloc_D(sizeof(RegMask))) RegMask();
+ _live.map(node->_idx, (Node*)live);
+ }
+
+ return live;
+ }
+};
+
+static ZBarrierSetC2State* barrier_set_state() {
return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
}
-bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const {
- // 1. This step follows potential oop projections of a load barrier before expansion
- if (node->is_Proj()) {
- node = node->in(0);
- }
-
- // 2. This step checks for unexpanded load barriers
- if (node->is_LoadBarrier()) {
- return true;
- }
-
- // 3. This step checks for the phi corresponding to an optimized load barrier expansion
- if (node->is_Phi()) {
- PhiNode* phi = node->as_Phi();
- Node* n = phi->in(1);
- if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) {
- return true;
- }
- }
-
- return false;
-}
-
-void ZBarrierSetC2::register_potential_barrier_node(Node* node) const {
- if (node->is_LoadBarrier()) {
- state()->add_load_barrier_node(node->as_LoadBarrier());
- }
-}
-
-void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
- if (node->is_LoadBarrier()) {
- state()->remove_load_barrier_node(node->as_LoadBarrier());
- }
-}
-
-void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {
- // Remove useless LoadBarrier nodes
- ZBarrierSetC2State* s = state();
- for (int i = s->load_barrier_count()-1; i >= 0; i--) {
- LoadBarrierNode* n = s->load_barrier_node(i);
- if (!useful.member(n)) {
- unregister_potential_barrier_node(n);
- }
- }
-}
-
-void ZBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
- if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) {
- igvn->_worklist.push(node);
- }
-}
-
-void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) {
- // Look for dominating barriers on the same address only once all
- // other loop opts are over. Loop opts may cause a safepoint to be
- // inserted between a barrier and its dominating barrier.
- Compile* C = Compile::current();
- ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2();
- ZBarrierSetC2State* s = bs->state();
- if (s->load_barrier_count() >= 2) {
- Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]);
- PhaseIdealLoop::optimize(igvn, LoopOptsLastRound);
- if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
- }
-}
-
-void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {
- // Permanent temporary workaround
- // Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is
- // removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to
- // the worklist. Unless we add them explicitly they are not guaranteed to end up there.
- ZBarrierSetC2State* s = state();
-
- for (int i = 0; i < s->load_barrier_count(); i++) {
- LoadBarrierNode* n = s->load_barrier_node(i);
- worklist->push(n);
- }
-}
-
-const TypeFunc* ZBarrierSetC2::load_barrier_Type() const {
- const Type** fields;
-
- // Create input types (domain)
- fields = TypeTuple::fields(2);
- fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
- fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM;
- const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
-
- // Create result type (range)
- fields = TypeTuple::fields(1);
- fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM;
- const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
-
- return TypeFunc::make(domain, range);
-}
-
-// == LoadBarrierNode ==
-
-LoadBarrierNode::LoadBarrierNode(Compile* C,
- Node* c,
- Node* mem,
- Node* val,
- Node* adr,
- bool weak,
- bool writeback,
- bool oop_reload_allowed) :
- MultiNode(Number_of_Inputs),
- _weak(weak),
- _writeback(writeback),
- _oop_reload_allowed(oop_reload_allowed) {
- init_req(Control, c);
- init_req(Memory, mem);
- init_req(Oop, val);
- init_req(Address, adr);
- init_req(Similar, C->top());
-
- init_class_id(Class_LoadBarrier);
- BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
- bs->register_potential_barrier_node(this);
-}
-
-uint LoadBarrierNode::size_of() const {
- return sizeof(*this);
-}
-
-bool LoadBarrierNode::cmp(const Node& n) const {
- ShouldNotReachHere();
- return false;
-}
-
-const Type *LoadBarrierNode::bottom_type() const {
- const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
- Node* in_oop = in(Oop);
- floadbarrier[Control] = Type::CONTROL;
- floadbarrier[Memory] = Type::MEMORY;
- floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type();
- return TypeTuple::make(Number_of_Outputs, floadbarrier);
-}
-
-const TypePtr* LoadBarrierNode::adr_type() const {
- ShouldNotReachHere();
- return NULL;
-}
-
-const Type *LoadBarrierNode::Value(PhaseGVN *phase) const {
- const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
- const Type* val_t = phase->type(in(Oop));
- floadbarrier[Control] = Type::CONTROL;
- floadbarrier[Memory] = Type::MEMORY;
- floadbarrier[Oop] = val_t;
- return TypeTuple::make(Number_of_Outputs, floadbarrier);
-}
-
-bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) {
- if (phase != NULL) {
- return phase->is_dominator(d, n);
- }
-
- for (int i = 0; i < 10 && n != NULL; i++) {
- n = IfNode::up_one_dom(n, linear_only);
- if (n == d) {
- return true;
- }
- }
-
- return false;
-}
-
-LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) {
- Node* val = in(LoadBarrierNode::Oop);
- if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) {
- LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier();
- assert(lb->in(Address) == in(Address), "");
- // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier.
- if (lb->in(Oop) == in(Oop)) {
- return lb;
- }
- // Follow chain of load barrier through Similar edges
- while (!lb->in(Similar)->is_top()) {
- lb = lb->in(Similar)->in(0)->as_LoadBarrier();
- assert(lb->in(Address) == in(Address), "");
- }
- if (lb != in(Similar)->in(0)) {
- return lb;
- }
- }
- for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
- Node* u = val->fast_out(i);
- if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) {
- Node* this_ctrl = in(LoadBarrierNode::Control);
- Node* other_ctrl = u->in(LoadBarrierNode::Control);
- if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
- return u->as_LoadBarrier();
- }
- }
- }
-
- if (ZVerifyLoadBarriers || can_be_eliminated()) {
- return NULL;
- }
-
- if (!look_for_similar) {
- return NULL;
- }
-
- Node* addr = in(LoadBarrierNode::Address);
- for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
- Node* u = addr->fast_out(i);
- if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
- Node* this_ctrl = in(LoadBarrierNode::Control);
- Node* other_ctrl = u->in(LoadBarrierNode::Control);
- if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
- ResourceMark rm;
- Unique_Node_List wq;
- wq.push(in(LoadBarrierNode::Control));
- bool ok = true;
- bool dom_found = false;
- for (uint next = 0; next < wq.size(); ++next) {
- Node *n = wq.at(next);
- if (n->is_top()) {
- return NULL;
- }
- assert(n->is_CFG(), "");
- if (n->is_SafePoint()) {
- ok = false;
- break;
- }
- if (n == u) {
- dom_found = true;
- continue;
- }
- if (n->is_Region()) {
- for (uint i = 1; i < n->req(); i++) {
- Node* m = n->in(i);
- if (m != NULL) {
- wq.push(m);
- }
- }
- } else {
- Node* m = n->in(0);
- if (m != NULL) {
- wq.push(m);
- }
- }
- }
- if (ok) {
- assert(dom_found, "");
- return u->as_LoadBarrier();;
- }
- break;
- }
- }
- }
-
- return NULL;
-}
-
-void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const {
- // Change to that barrier may affect a dominated barrier so re-push those
- Node* val = in(LoadBarrierNode::Oop);
-
- for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
- Node* u = val->fast_out(i);
- if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) {
- Node* this_ctrl = in(Control);
- Node* other_ctrl = u->in(Control);
- if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
- igvn->_worklist.push(u);
- }
- }
-
- Node* addr = in(LoadBarrierNode::Address);
- for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
- Node* u = addr->fast_out(i);
- if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) {
- Node* this_ctrl = in(Control);
- Node* other_ctrl = u->in(Control);
- if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
- igvn->_worklist.push(u);
- }
- }
- }
- }
-}
-
-Node *LoadBarrierNode::Identity(PhaseGVN *phase) {
- if (!phase->C->directive()->ZOptimizeLoadBarriersOption) {
- return this;
- }
-
- bool redundant_addr = false;
- LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false);
- if (dominating_barrier != NULL) {
- assert(dominating_barrier->in(Oop) == in(Oop), "");
- return dominating_barrier;
- }
-
- return this;
-}
-
-Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
- if (remove_dead_region(phase, can_reshape)) {
- return this;
+ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) {
+ ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref, tmp, weak);
+ if (!Compile::current()->in_scratch_emit_size()) {
+ barrier_set_state()->stubs()->append(stub);
}
- Node* val = in(Oop);
- Node* mem = in(Memory);
- Node* ctrl = in(Control);
- Node* adr = in(Address);
- assert(val->Opcode() != Op_LoadN, "");
-
- if (mem->is_MergeMem()) {
- Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
- set_req(Memory, new_mem);
- if (mem->outcnt() == 0 && can_reshape) {
- phase->is_IterGVN()->_worklist.push(mem);
- }
-
- return this;
- }
-
- bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption;
- LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL;
- if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {
- assert(in(Address) == dominating_barrier->in(Address), "");
- set_req(Similar, dominating_barrier->proj_out(Oop));
- return this;
- }
-
- bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) ||
- (can_reshape && (dominating_barrier != NULL || !has_true_uses()));
-
- if (eliminate) {
- if (can_reshape) {
- PhaseIterGVN* igvn = phase->is_IterGVN();
- Node* out_ctrl = proj_out_or_null(Control);
- Node* out_res = proj_out_or_null(Oop);
-
- if (out_ctrl != NULL) {
- igvn->replace_node(out_ctrl, ctrl);
- }
+ return stub;
+}
- // That transformation may cause the Similar edge on the load barrier to be invalid
- fix_similar_in_uses(igvn);
- if (out_res != NULL) {
- if (dominating_barrier != NULL) {
- igvn->replace_node(out_res, dominating_barrier->proj_out(Oop));
- } else {
- igvn->replace_node(out_res, val);
- }
- }
- }
-
- return new ConINode(TypeInt::ZERO);
- }
-
- // If the Similar edge is no longer a load barrier, clear it
- Node* similar = in(Similar);
- if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) {
- set_req(Similar, phase->C->top());
- return this;
- }
+ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) :
+ _node(node),
+ _ref_addr(ref_addr),
+ _ref(ref),
+ _tmp(tmp),
+ _weak(weak),
+ _entry(),
+ _continuation() {
+ assert_different_registers(ref, ref_addr.base());
+ assert_different_registers(ref, ref_addr.index());
+}
- if (can_reshape) {
- // If this barrier is linked through the Similar edge by a
- // dominated barrier and both barriers have the same Oop field,
- // the dominated barrier can go away, so push it for reprocessing.
- // We also want to avoid a barrier to depend on another dominating
- // barrier through its Similar edge that itself depend on another
- // barrier through its Similar edge and rather have the first
- // depend on the third.
- PhaseIterGVN* igvn = phase->is_IterGVN();
- Node* out_res = proj_out(Oop);
- for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
- Node* u = out_res->fast_out(i);
- if (u->is_LoadBarrier() && u->in(Similar) == out_res &&
- (u->in(Oop) == val || !u->in(Similar)->is_top())) {
- igvn->_worklist.push(u);
- }
- }
-
- push_dominated_barriers(igvn);
- }
-
- return NULL;
+Address ZLoadBarrierStubC2::ref_addr() const {
+ return _ref_addr;
}
-uint LoadBarrierNode::match_edge(uint idx) const {
- ShouldNotReachHere();
- return 0;
+Register ZLoadBarrierStubC2::ref() const {
+ return _ref;
+}
+
+Register ZLoadBarrierStubC2::tmp() const {
+ return _tmp;
}
-void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) {
- Node* out_res = proj_out_or_null(Oop);
- if (out_res == NULL) {
- return;
- }
-
- for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
- Node* u = out_res->fast_out(i);
- if (u->is_LoadBarrier() && u->in(Similar) == out_res) {
- igvn->replace_input_of(u, Similar, igvn->C->top());
- --i;
- --imax;
- }
- }
+address ZLoadBarrierStubC2::slow_path() const {
+ const DecoratorSet decorators = _weak ? ON_WEAK_OOP_REF : ON_STRONG_OOP_REF;
+ return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators);
}
-bool LoadBarrierNode::has_true_uses() const {
- Node* out_res = proj_out_or_null(Oop);
- if (out_res == NULL) {
- return false;
- }
+RegMask& ZLoadBarrierStubC2::live() const {
+ return *barrier_set_state()->live(_node);
+}
- for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
- Node* u = out_res->fast_out(i);
- if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {
- return true;
- }
- }
-
- return false;
+Label* ZLoadBarrierStubC2::entry() {
+ // The _entry will never be bound when in_scratch_emit_size() is true.
+ // However, we still need to return a label that is not bound now, but
+ // will eventually be bound. Any lable will do, as it will only act as
+ // a placeholder, so we return the _continuation label.
+ return Compile::current()->in_scratch_emit_size() ? &_continuation : &_entry;
}
-// == Accesses ==
-
-Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicParseAccess& access) const {
- assert(!UseCompressedOops, "Not allowed");
- CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access();
- PhaseGVN& gvn = access.gvn();
- Compile* C = Compile::current();
- GraphKit* kit = access.kit();
-
- Node* in_ctrl = cas->in(MemNode::Control);
- Node* in_mem = cas->in(MemNode::Memory);
- Node* in_adr = cas->in(MemNode::Address);
- Node* in_val = cas->in(MemNode::ValueIn);
- Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn);
-
- float likely = PROB_LIKELY(0.999);
-
- const TypePtr *adr_type = gvn.type(in_adr)->isa_ptr();
- Compile::AliasType* alias_type = C->alias_type(adr_type);
- int alias_idx = C->get_alias_index(adr_type);
-
- // Outer check - true: continue, false: load and check
- Node* region = new RegionNode(3);
- Node* phi = new PhiNode(region, TypeInt::BOOL);
- Node* phi_mem = new PhiNode(region, Type::MEMORY, adr_type);
-
- // Inner check - is the healed ref equal to the expected
- Node* region2 = new RegionNode(3);
- Node* phi2 = new PhiNode(region2, TypeInt::BOOL);
- Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type);
-
- // CAS node returns 0 or 1
- Node* cmp = gvn.transform(new CmpINode(cas, kit->intcon(0)));
- Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
- IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
- Node* then = gvn.transform(new IfTrueNode(iff));
- Node* elsen = gvn.transform(new IfFalseNode(iff));
-
- Node* scmemproj1 = gvn.transform(new SCMemProjNode(cas));
-
- kit->set_memory(scmemproj1, alias_idx);
- phi_mem->init_req(1, scmemproj1);
- phi_mem2->init_req(2, scmemproj1);
+Label* ZLoadBarrierStubC2::continuation() {
+ return &_continuation;
+}
- // CAS fail - reload and heal oop
- Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
- Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
- Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
- Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
-
- // Check load
- Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata));
- Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
- Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX));
- Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
- IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
- Node* then2 = gvn.transform(new IfTrueNode(iff2));
- Node* elsen2 = gvn.transform(new IfFalseNode(iff2));
-
- // redo CAS
- Node* cas2 = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order()));
- Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2));
- kit->set_control(elsen2);
- kit->set_memory(scmemproj2, alias_idx);
+void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
+ return new (comp_arena) ZBarrierSetC2State(comp_arena);
+}
- // Merge inner flow - check if healed oop was equal too expected.
- region2->set_req(1, kit->control());
- region2->set_req(2, then2);
- phi2->set_req(1, cas2);
- phi2->set_req(2, kit->intcon(0));
- phi_mem2->init_req(1, scmemproj2);
- kit->set_memory(phi_mem2, alias_idx);
-
- // Merge outer flow - then check if first CAS succeeded
- region->set_req(1, then);
- region->set_req(2, region2);
- phi->set_req(1, kit->intcon(1));
- phi->set_req(2, phi2);
- phi_mem->init_req(2, phi_mem2);
- kit->set_memory(phi_mem, alias_idx);
-
- gvn.transform(region2);
- gvn.transform(phi2);
- gvn.transform(phi_mem2);
- gvn.transform(region);
- gvn.transform(phi);
- gvn.transform(phi_mem);
-
- kit->set_control(region);
- kit->insert_mem_bar(Op_MemBarCPUOrder);
-
- return phi;
+void ZBarrierSetC2::late_barrier_analysis() const {
+ analyze_dominating_barriers();
+ compute_liveness_at_stubs();
}
-Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicParseAccess& access) const {
- CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access();
- GraphKit* kit = access.kit();
- PhaseGVN& gvn = kit->gvn();
- Compile* C = Compile::current();
-
- Node* in_ctrl = cmpx->in(MemNode::Control);
- Node* in_mem = cmpx->in(MemNode::Memory);
- Node* in_adr = cmpx->in(MemNode::Address);
- Node* in_val = cmpx->in(MemNode::ValueIn);
- Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn);
-
- float likely = PROB_LIKELY(0.999);
-
- const TypePtr *adr_type = cmpx->get_ptr_type();
- Compile::AliasType* alias_type = C->alias_type(adr_type);
- int alias_idx = C->get_alias_index(adr_type);
-
- // Outer check - true: continue, false: load and check
- Node* region = new RegionNode(3);
- Node* phi = new PhiNode(region, adr_type);
-
- // Inner check - is the healed ref equal to the expected
- Node* region2 = new RegionNode(3);
- Node* phi2 = new PhiNode(region2, adr_type);
-
- // Check if cmpx succeeded
- Node* cmp = gvn.transform(new CmpPNode(cmpx, in_expected));
- Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool();
- IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
- Node* then = gvn.transform(new IfTrueNode(iff));
- Node* elsen = gvn.transform(new IfFalseNode(iff));
-
- Node* scmemproj1 = gvn.transform(new SCMemProjNode(cmpx));
- kit->set_memory(scmemproj1, alias_idx);
+void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
+ MacroAssembler masm(&cb);
+ GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
- // CAS fail - reload and heal oop
- Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
- Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
- Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
- Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
-
- // Check load
- Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata));
- Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
- Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX));
- Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
- IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
- Node* then2 = gvn.transform(new IfTrueNode(iff2));
- Node* elsen2 = gvn.transform(new IfFalseNode(iff2));
-
- // Redo CAS
- Node* cmpx2 = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order()));
- Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2));
- kit->set_control(elsen2);
- kit->set_memory(scmemproj2, alias_idx);
+ for (int i = 0; i < stubs->length(); i++) {
+ // Make sure there is enough space in the code buffer
+ if (cb.insts()->maybe_expand_to_ensure_remaining(Compile::MAX_inst_size) && cb.blob() == NULL) {
+ ciEnv::current()->record_failure("CodeCache is full");
+ return;
+ }
- // Merge inner flow - check if healed oop was equal too expected.
- region2->set_req(1, kit->control());
- region2->set_req(2, then2);
- phi2->set_req(1, cmpx2);
- phi2->set_req(2, barrierdata);
+ ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
+ }
- // Merge outer flow - then check if first cas succeeded
- region->set_req(1, then);
- region->set_req(2, region2);
- phi->set_req(1, cmpx);
- phi->set_req(2, phi2);
-
- gvn.transform(region2);
- gvn.transform(phi2);
- gvn.transform(region);
- gvn.transform(phi);
-
- kit->set_control(region);
- kit->set_memory(in_mem, alias_idx);
- kit->insert_mem_bar(Op_MemBarCPUOrder);
-
- return phi;
+ masm.flush();
}
-Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const {
- PhaseGVN& gvn = kit->gvn();
- Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed);
- Node* transformed_barrier = gvn.transform(barrier);
+int ZBarrierSetC2::estimate_stub_size() const {
+ Compile* const C = Compile::current();
+ BufferBlob* const blob = C->scratch_buffer_blob();
+ GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
+ int size = 0;
- if (transformed_barrier->is_LoadBarrier()) {
- if (barrier == transformed_barrier) {
- kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)));
- }
- Node* result = gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop));
- return result;
- } else {
- return val;
+ for (int i = 0; i < stubs->length(); i++) {
+ CodeBuffer cb(blob->content_begin(), (address)C->scratch_locs_memory() - blob->content_begin());
+ MacroAssembler masm(&cb);
+ ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
+ size += cb.insts_size();
}
+
+ return size;
}
static bool barrier_needed(C2Access& access) {
@@ -689,956 +180,252 @@
}
Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
- Node* p = BarrierSetC2::load_at_resolved(access, val_type);
- if (!barrier_needed(access)) {
- return p;
+ Node* result = BarrierSetC2::load_at_resolved(access, val_type);
+ if (barrier_needed(access) && access.raw_access()->is_Mem()) {
+ if ((access.decorators() & ON_WEAK_OOP_REF) != 0) {
+ access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierWeak);
+ } else {
+ access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierStrong);
+ }
}
- bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;
-
- assert(access.is_parse_access(), "entry not supported at optimization time");
- C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
- GraphKit* kit = parse_access.kit();
- PhaseGVN& gvn = kit->gvn();
- Node* adr = access.addr().node();
- Node* heap_base_oop = access.base();
- bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0;
- if (unsafe) {
- if (!ZVerifyLoadBarriers) {
- p = load_barrier(kit, p, adr);
- } else {
- if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) {
- p = load_barrier(kit, p, adr);
- } else {
- IdealKit ideal(kit);
- IdealVariable res(ideal);
-#define __ ideal.
- __ declarations_done();
- __ set(res, p);
- __ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); {
- kit->sync_kit(ideal);
- p = load_barrier(kit, p, adr);
- __ set(res, p);
- __ sync_kit(kit);
- } __ end_if();
- kit->final_sync(ideal);
- p = __ value(res);
-#undef __
- }
- }
- return p;
- } else {
- return load_barrier(parse_access.kit(), p, access.addr().node(), weak, true, true);
- }
+ return result;
}
Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* val_type) const {
Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
- if (!barrier_needed(access)) {
- return result;
+ if (barrier_needed(access)) {
+ access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);
}
-
- access.set_needs_pinning(false);
- return make_cmpx_loadbarrier(access);
+ return result;
}
Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const {
Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
- if (!barrier_needed(access)) {
- return result;
+ if (barrier_needed(access)) {
+ access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);
}
-
- Node* load_store = access.raw_access();
- bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0;
- bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR);
-
- if (!expected_is_null) {
- if (weak_cas) {
- access.set_needs_pinning(false);
- load_store = make_cas_loadbarrier(access);
- } else {
- access.set_needs_pinning(false);
- load_store = make_cas_loadbarrier(access);
- }
- }
-
- return load_store;
+ return result;
}
Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
- if (!barrier_needed(access)) {
- return result;
- }
-
- Node* load_store = access.raw_access();
- Node* adr = access.addr().node();
-
- assert(access.is_parse_access(), "entry not supported at optimization time");
- C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
- return load_barrier(parse_access.kit(), load_store, adr, false, false, false);
-}
-
-// == Macro Expansion ==
-
-void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const {
- Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
- Node* in_mem = barrier->in(LoadBarrierNode::Memory);
- Node* in_val = barrier->in(LoadBarrierNode::Oop);
- Node* in_adr = barrier->in(LoadBarrierNode::Address);
-
- Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
- Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
-
- PhaseIterGVN &igvn = phase->igvn();
-
- if (ZVerifyLoadBarriers) {
- igvn.replace_node(out_res, in_val);
- igvn.replace_node(out_ctrl, in_ctrl);
- return;
+ if (barrier_needed(access)) {
+ access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);
}
-
- if (barrier->can_be_eliminated()) {
- // Clone and pin the load for this barrier below the dominating
- // barrier: the load cannot be allowed to float above the
- // dominating barrier
- Node* load = in_val;
-
- if (load->is_Load()) {
- Node* new_load = load->clone();
- Node* addp = new_load->in(MemNode::Address);
- assert(addp->is_AddP() || addp->is_Phi() || addp->is_Load(), "bad address");
- Node* cast = new CastPPNode(addp, igvn.type(addp), true);
- Node* ctrl = NULL;
- Node* similar = barrier->in(LoadBarrierNode::Similar);
- if (similar->is_Phi()) {
- // already expanded
- ctrl = similar->in(0);
- } else {
- assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape");
- ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control);
- }
- assert(ctrl != NULL, "bad control");
- cast->set_req(0, ctrl);
- igvn.transform(cast);
- new_load->set_req(MemNode::Address, cast);
- igvn.transform(new_load);
-
- igvn.replace_node(out_res, new_load);
- igvn.replace_node(out_ctrl, in_ctrl);
- return;
- }
- // cannot eliminate
- }
-
- // There are two cases that require the basic loadbarrier
- // 1) When the writeback of a healed oop must be avoided (swap)
- // 2) When we must guarantee that no reload of is done (swap, cas, cmpx)
- if (!barrier->is_writeback()) {
- assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop");
- }
-
- if (!barrier->oop_reload_allowed()) {
- expand_loadbarrier_basic(phase, barrier);
- } else {
- expand_loadbarrier_optimized(phase, barrier);
- }
+ return result;
}
-// Basic loadbarrier using conventional argument passing
-void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
- PhaseIterGVN &igvn = phase->igvn();
-
- Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
- Node* in_mem = barrier->in(LoadBarrierNode::Memory);
- Node* in_val = barrier->in(LoadBarrierNode::Oop);
- Node* in_adr = barrier->in(LoadBarrierNode::Address);
-
- Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
- Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
-
- float unlikely = PROB_UNLIKELY(0.999);
- const Type* in_val_maybe_null_t = igvn.type(in_val);
-
- Node* jthread = igvn.transform(new ThreadLocalNode());
- Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
- Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
- Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
- Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
- Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
- Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
- IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
- Node* then = igvn.transform(new IfTrueNode(iff));
- Node* elsen = igvn.transform(new IfFalseNode(iff));
-
- Node* result_region;
- Node* result_val;
-
- result_region = new RegionNode(3);
- result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM);
-
- result_region->set_req(1, elsen);
- Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t));
- res->init_req(0, elsen);
- result_val->set_req(1, res);
-
- const TypeFunc *tf = load_barrier_Type();
- Node* call;
- if (barrier->is_weak()) {
- call = new CallLeafNode(tf,
- ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(),
- "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded",
- TypeRawPtr::BOTTOM);
- } else {
- call = new CallLeafNode(tf,
- ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(),
- "ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded",
- TypeRawPtr::BOTTOM);
- }
-
- call->init_req(TypeFunc::Control, then);
- call->init_req(TypeFunc::I_O , phase->top());
- call->init_req(TypeFunc::Memory , in_mem);
- call->init_req(TypeFunc::FramePtr, phase->top());
- call->init_req(TypeFunc::ReturnAdr, phase->top());
- call->init_req(TypeFunc::Parms+0, in_val);
- if (barrier->is_writeback()) {
- call->init_req(TypeFunc::Parms+1, in_adr);
- } else {
- // When slow path is called with a null address, the healed oop will not be written back
- call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT));
- }
- call = igvn.transform(call);
-
- Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control));
- res = igvn.transform(new ProjNode(call, TypeFunc::Parms));
- res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t));
-
- result_region->set_req(2, ctrl);
- result_val->set_req(2, res);
-
- result_region = igvn.transform(result_region);
- result_val = igvn.transform(result_val);
-
- if (out_ctrl != NULL) { // Added if cond
- igvn.replace_node(out_ctrl, result_region);
- }
- igvn.replace_node(out_res, result_val);
+bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type,
+ bool is_clone, ArrayCopyPhase phase) const {
+ return type == T_OBJECT || type == T_ARRAY;
}
-// Optimized, low spill, loadbarrier variant using stub specialized on register used
-void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
- PhaseIterGVN &igvn = phase->igvn();
-#ifdef PRINT_NODE_TRAVERSALS
- Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop);
-#endif
-
- Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
- Node* in_mem = barrier->in(LoadBarrierNode::Memory);
- Node* in_val = barrier->in(LoadBarrierNode::Oop);
- Node* in_adr = barrier->in(LoadBarrierNode::Address);
-
- Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
- Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
-
- assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null");
-
-#ifdef PRINT_NODE_TRAVERSALS
- tty->print("\n\n\nBefore barrier optimization:\n");
- traverse(barrier, out_ctrl, out_res, -1);
-
- tty->print("\nBefore barrier optimization: preceding_barrier_node\n");
- traverse(preceding_barrier_node, out_ctrl, out_res, -1);
-#endif
-
- float unlikely = PROB_UNLIKELY(0.999);
-
- Node* jthread = igvn.transform(new ThreadLocalNode());
- Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
- Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr,
- TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(),
- MemNode::unordered));
- Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
- Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
- Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
- Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
- IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
- Node* then = igvn.transform(new IfTrueNode(iff));
- Node* elsen = igvn.transform(new IfFalseNode(iff));
-
- Node* slow_path_surrogate;
- if (!barrier->is_weak()) {
- slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
- (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
- } else {
- slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
- (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
- }
-
- Node *new_loadp;
- new_loadp = slow_path_surrogate;
- // Create the final region/phi pair to converge cntl/data paths to downstream code
- Node* result_region = igvn.transform(new RegionNode(3));
- result_region->set_req(1, then);
- result_region->set_req(2, elsen);
-
- Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM));
- result_phi->set_req(1, new_loadp);
- result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop));
+// == Dominating barrier elision ==
- // Finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution
- // igvn.replace_node(out_ctrl, result_region);
- if (out_ctrl != NULL) { // added if cond
- igvn.replace_node(out_ctrl, result_region);
- }
- igvn.replace_node(out_res, result_phi);
-
- assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!");
-
-#ifdef PRINT_NODE_TRAVERSALS
- tty->print("\nAfter barrier optimization: old out_ctrl\n");
- traverse(out_ctrl, out_ctrl, out_res, -1);
- tty->print("\nAfter barrier optimization: old out_res\n");
- traverse(out_res, out_ctrl, out_res, -1);
- tty->print("\nAfter barrier optimization: old barrier\n");
- traverse(barrier, out_ctrl, out_res, -1);
- tty->print("\nAfter barrier optimization: preceding_barrier_node\n");
- traverse(preceding_barrier_node, result_region, result_phi, -1);
-#endif
-
- assert(is_gc_barrier_node(result_phi), "sanity");
- assert(step_over_gc_barrier(result_phi) == in_val, "sanity");
-}
-
-bool ZBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
- ZBarrierSetC2State* s = state();
- if (s->load_barrier_count() > 0) {
- PhaseMacroExpand macro(igvn);
-#ifdef ASSERT
- verify_gc_barriers(false);
-#endif
- int skipped = 0;
- while (s->load_barrier_count() > skipped) {
- int load_barrier_count = s->load_barrier_count();
- LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped);
- if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
- // Node is unreachable, so don't try to expand it
- s->remove_load_barrier_node(n);
- continue;
- }
- if (!n->can_be_eliminated()) {
- skipped++;
- continue;
- }
- expand_loadbarrier_node(¯o, n);
- assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
- if (C->failing()) {
- return true;
- }
- }
- while (s->load_barrier_count() > 0) {
- int load_barrier_count = s->load_barrier_count();
- LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1);
- assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already");
- assert(!n->can_be_eliminated(), "should have been processed already");
- expand_loadbarrier_node(¯o, n);
- assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
- if (C->failing()) {
- return true;
- }
- }
- igvn.set_delay_transform(false);
- igvn.optimize();
- if (C->failing()) {
+static bool block_has_safepoint(const Block* block, uint from, uint to) {
+ for (uint i = from; i < to; i++) {
+ if (block->get_node(i)->is_MachSafePoint()) {
+ // Safepoint found
return true;
}
}
+ // Safepoint not found
return false;
}
-// == Loop optimization ==
-
-static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
- PhaseIterGVN &igvn = phase->igvn();
- Compile* C = Compile::current();
-
- LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round);
- if (lb2 == NULL) {
- return false;
- }
-
- if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) {
- assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), "Invalid address");
- igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop));
- C->set_major_progress();
- return false;
- }
-
- // That transformation may cause the Similar edge on dominated load barriers to be invalid
- lb->fix_similar_in_uses(&igvn);
-
- Node* val = lb->proj_out(LoadBarrierNode::Oop);
- assert(lb2->has_true_uses(), "Invalid uses");
- assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), "Invalid oop");
- phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
- phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
- igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop));
-
- return true;
+static bool block_has_safepoint(const Block* block) {
+ return block_has_safepoint(block, 0, block->number_of_nodes());
}
-static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) {
- assert(dom->is_Region() || i == -1, "");
-
- Node* m = mem;
- while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) {
- if (m->is_Mem()) {
- assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, "");
- m = m->in(MemNode::Memory);
- } else if (m->is_MergeMem()) {
- m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
- } else if (m->is_Phi()) {
- if (m->in(0) == dom && i != -1) {
- m = m->in(i);
- break;
- } else {
- m = m->in(LoopNode::EntryControl);
- }
- } else if (m->is_Proj()) {
- m = m->in(0);
- } else if (m->is_SafePoint() || m->is_MemBar()) {
- m = m->in(TypeFunc::Memory);
- } else {
-#ifdef ASSERT
- m->dump();
-#endif
- ShouldNotReachHere();
+static uint block_index(const Block* block, const Node* node) {
+ for (uint j = 0; j < block->number_of_nodes(); ++j) {
+ if (block->get_node(j) == node) {
+ return j;
}
}
-
- return m;
+ ShouldNotReachHere();
+ return 0;
}
-static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) {
- PhaseIterGVN &igvn = phase->igvn();
- Compile* C = Compile::current();
- Node* the_clone = lb->clone();
- the_clone->set_req(LoadBarrierNode::Control, ctl);
- the_clone->set_req(LoadBarrierNode::Memory, mem);
- if (oop_in != NULL) {
- the_clone->set_req(LoadBarrierNode::Oop, oop_in);
- }
-
- LoadBarrierNode* new_lb = the_clone->as_LoadBarrier();
- igvn.register_new_node_with_optimizer(new_lb);
- IdealLoopTree *loop = phase->get_loop(new_lb->in(0));
- phase->set_ctrl(new_lb, new_lb->in(0));
- phase->set_loop(new_lb, loop);
- phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1);
- if (!loop->_child) {
- loop->_body.push(new_lb);
- }
-
- Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control);
- igvn.register_new_node_with_optimizer(proj_ctl);
- phase->set_ctrl(proj_ctl, proj_ctl->in(0));
- phase->set_loop(proj_ctl, loop);
- phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1);
- if (!loop->_child) {
- loop->_body.push(proj_ctl);
- }
-
- Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop);
- phase->register_new_node(proj_oop, new_lb);
-
- if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) {
- LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier();
- if (!phase->is_dominator(similar, ctl)) {
- igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top());
- }
- }
-
- return new_lb;
-}
+void ZBarrierSetC2::analyze_dominating_barriers() const {
+ ResourceMark rm;
+ Compile* const C = Compile::current();
+ PhaseCFG* const cfg = C->cfg();
+ Block_List worklist;
+ Node_List mem_ops;
+ Node_List barrier_loads;
-static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) {
- PhaseIterGVN &igvn = phase->igvn();
- Node* val = lb->proj_out(LoadBarrierNode::Oop);
- igvn.replace_node(val, new_val);
- phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
- phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
-}
-
-static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
- PhaseIterGVN &igvn = phase->igvn();
- Compile* C = Compile::current();
-
- if (lb->in(LoadBarrierNode::Oop)->is_Phi()) {
- Node* oop_phi = lb->in(LoadBarrierNode::Oop);
-
- if ((oop_phi->req() != 3) || (oop_phi->in(2) == oop_phi)) {
- // Ignore phis with only one input
- return false;
- }
-
- if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)),
- oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) {
- // That transformation may cause the Similar edge on dominated load barriers to be invalid
- lb->fix_similar_in_uses(&igvn);
-
- RegionNode* region = oop_phi->in(0)->as_Region();
-
- int backedge = LoopNode::LoopBackControl;
- if (region->is_Loop() && region->in(backedge)->is_Proj() && region->in(backedge)->in(0)->is_If()) {
- Node* c = region->in(backedge)->in(0)->in(0);
- assert(c->unique_ctrl_out() == region->in(backedge)->in(0), "");
- Node* oop = lb->in(LoadBarrierNode::Oop)->in(backedge);
- Node* oop_c = phase->has_ctrl(oop) ? phase->get_ctrl(oop) : oop;
- if (!phase->is_dominator(oop_c, c)) {
- return false;
- }
- }
-
- // If the node on the backedge above the phi is the node itself - we have a self loop.
- // Don't clone - this will be folded later.
- if (oop_phi->in(LoopNode::LoopBackControl) == lb->proj_out(LoadBarrierNode::Oop)) {
- return false;
+ // Step 1 - Find accesses, and track them in lists
+ for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
+ const Block* const block = cfg->get_block(i);
+ for (uint j = 0; j < block->number_of_nodes(); ++j) {
+ const Node* const node = block->get_node(j);
+ if (!node->is_Mach()) {
+ continue;
}
- bool is_strip_mined = region->is_CountedLoop() && region->as_CountedLoop()->is_strip_mined();
- Node *phi = oop_phi->clone();
-
- for (uint i = 1; i < region->req(); i++) {
- Node* ctrl = region->in(i);
- if (ctrl != C->top()) {
- assert(!phase->is_dominator(ctrl, region) || region->is_Loop(), "");
-
- Node* mem = lb->in(LoadBarrierNode::Memory);
- Node* m = find_dominating_memory(phase, mem, region, i);
-
- if (region->is_Loop() && i == LoopNode::LoopBackControl && ctrl->is_Proj() && ctrl->in(0)->is_If()) {
- ctrl = ctrl->in(0)->in(0);
- } else if (region->is_Loop() && is_strip_mined) {
- // If this is a strip mined loop, control must move above OuterStripMinedLoop
- assert(i == LoopNode::EntryControl, "check");
- assert(ctrl->is_OuterStripMinedLoop(), "sanity");
- ctrl = ctrl->as_OuterStripMinedLoop()->in(LoopNode::EntryControl);
- }
-
- LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, ctrl, m, lb->in(LoadBarrierNode::Oop)->in(i));
- Node* out_ctrl = new_lb->proj_out(LoadBarrierNode::Control);
-
- if (is_strip_mined && (i == LoopNode::EntryControl)) {
- assert(region->in(i)->is_OuterStripMinedLoop(), "");
- igvn.replace_input_of(region->in(i), i, out_ctrl);
- phase->set_idom(region->in(i), out_ctrl, phase->dom_depth(out_ctrl));
- } else if (ctrl == region->in(i)) {
- igvn.replace_input_of(region, i, out_ctrl);
- // Only update the idom if is the loop entry we are updating
- // - A loop backedge doesn't change the idom
- if (region->is_Loop() && i == LoopNode::EntryControl) {
- phase->set_idom(region, out_ctrl, phase->dom_depth(out_ctrl));
- }
- } else {
- Node* iff = region->in(i)->in(0);
- igvn.replace_input_of(iff, 0, out_ctrl);
- phase->set_idom(iff, out_ctrl, phase->dom_depth(out_ctrl)+1);
- }
- phi->set_req(i, new_lb->proj_out(LoadBarrierNode::Oop));
- }
- }
- phase->register_new_node(phi, region);
- replace_barrier(phase, lb, phi);
-
- if (region->is_Loop()) {
- // Load barrier moved to the back edge of the Loop may now
- // have a safepoint on the path to the barrier on the Similar
- // edge
- igvn.replace_input_of(phi->in(LoopNode::LoopBackControl)->in(0), LoadBarrierNode::Similar, C->top());
- Node* head = region->in(LoopNode::EntryControl);
- phase->set_idom(region, head, phase->dom_depth(head)+1);
- phase->recompute_dom_depth();
- if (head->is_CountedLoop() && head->as_CountedLoop()->is_main_loop()) {
- head->as_CountedLoop()->set_normal_loop();
- }
- }
-
- return true;
- }
- }
-
- return false;
-}
-
-static bool move_out_of_loop(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
- PhaseIterGVN &igvn = phase->igvn();
- IdealLoopTree *lb_loop = phase->get_loop(lb->in(0));
- if (lb_loop != phase->ltree_root() && !lb_loop->_irreducible) {
- Node* oop_ctrl = phase->get_ctrl(lb->in(LoadBarrierNode::Oop));
- IdealLoopTree *oop_loop = phase->get_loop(oop_ctrl);
- IdealLoopTree* adr_loop = phase->get_loop(phase->get_ctrl(lb->in(LoadBarrierNode::Address)));
- if (!lb_loop->is_member(oop_loop) && !lb_loop->is_member(adr_loop)) {
- // That transformation may cause the Similar edge on dominated load barriers to be invalid
- lb->fix_similar_in_uses(&igvn);
-
- Node* head = lb_loop->_head;
- assert(head->is_Loop(), "");
-
- if (phase->is_dominator(head, oop_ctrl)) {
- assert(oop_ctrl->Opcode() == Op_CProj && oop_ctrl->in(0)->Opcode() == Op_NeverBranch, "");
- assert(lb_loop->is_member(phase->get_loop(oop_ctrl->in(0)->in(0))), "");
- return false;
- }
-
- if (head->is_CountedLoop()) {
- CountedLoopNode* cloop = head->as_CountedLoop();
- if (cloop->is_main_loop()) {
- cloop->set_normal_loop();
+ MachNode* const mach = node->as_Mach();
+ switch (mach->ideal_Opcode()) {
+ case Op_LoadP:
+ case Op_CompareAndExchangeP:
+ case Op_CompareAndSwapP:
+ case Op_GetAndSetP:
+ if (mach->barrier_data() == ZLoadBarrierStrong) {
+ barrier_loads.push(mach);
}
- // When we are moving barrier out of a counted loop,
- // make sure we move it all the way out of the strip mined outer loop.
- if (cloop->is_strip_mined()) {
- head = cloop->outer_loop();
- }
- }
-
- Node* mem = lb->in(LoadBarrierNode::Memory);
- Node* m = find_dominating_memory(phase, mem, head, -1);
-
- LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, head->in(LoopNode::EntryControl), m, NULL);
-
- assert(phase->idom(head) == head->in(LoopNode::EntryControl), "");
- Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
- igvn.replace_input_of(head, LoopNode::EntryControl, proj_ctl);
- phase->set_idom(head, proj_ctl, phase->dom_depth(proj_ctl) + 1);
-
- replace_barrier(phase, lb, new_lb->proj_out(LoadBarrierNode::Oop));
-
- phase->recompute_dom_depth();
-
- return true;
- }
- }
-
- return false;
-}
-
-static bool common_barriers(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
- PhaseIterGVN &igvn = phase->igvn();
- Node* in_val = lb->in(LoadBarrierNode::Oop);
- for (DUIterator_Fast imax, i = in_val->fast_outs(imax); i < imax; i++) {
- Node* u = in_val->fast_out(i);
- if (u != lb && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
- Node* this_ctrl = lb->in(LoadBarrierNode::Control);
- Node* other_ctrl = u->in(LoadBarrierNode::Control);
-
- Node* lca = phase->dom_lca(this_ctrl, other_ctrl);
- bool ok = true;
-
- Node* proj1 = NULL;
- Node* proj2 = NULL;
+ case Op_StoreP:
+ mem_ops.push(mach);
+ break;
- while (this_ctrl != lca && ok) {
- if (this_ctrl->in(0) != NULL &&
- this_ctrl->in(0)->is_MultiBranch()) {
- if (this_ctrl->in(0)->in(0) == lca) {
- assert(proj1 == NULL, "");
- assert(this_ctrl->is_Proj(), "");
- proj1 = this_ctrl;
- } else if (!(this_ctrl->in(0)->is_If() && this_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
- ok = false;
- }
- }
- this_ctrl = phase->idom(this_ctrl);
- }
- while (other_ctrl != lca && ok) {
- if (other_ctrl->in(0) != NULL &&
- other_ctrl->in(0)->is_MultiBranch()) {
- if (other_ctrl->in(0)->in(0) == lca) {
- assert(other_ctrl->is_Proj(), "");
- assert(proj2 == NULL, "");
- proj2 = other_ctrl;
- } else if (!(other_ctrl->in(0)->is_If() && other_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
- ok = false;
- }
- }
- other_ctrl = phase->idom(other_ctrl);
- }
- assert(proj1 == NULL || proj2 == NULL || proj1->in(0) == proj2->in(0), "");
- if (ok && proj1 && proj2 && proj1 != proj2 && proj1->in(0)->is_If()) {
- // That transformation may cause the Similar edge on dominated load barriers to be invalid
- lb->fix_similar_in_uses(&igvn);
- u->as_LoadBarrier()->fix_similar_in_uses(&igvn);
-
- Node* split = lca->unique_ctrl_out();
- assert(split->in(0) == lca, "");
-
- Node* mem = lb->in(LoadBarrierNode::Memory);
- Node* m = find_dominating_memory(phase, mem, split, -1);
- LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, lca, m, NULL);
-
- Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
- igvn.replace_input_of(split, 0, new_lb->proj_out(LoadBarrierNode::Control));
- phase->set_idom(split, proj_ctl, phase->dom_depth(proj_ctl)+1);
-
- Node* proj_oop = new_lb->proj_out(LoadBarrierNode::Oop);
- replace_barrier(phase, lb, proj_oop);
- replace_barrier(phase, u->as_LoadBarrier(), proj_oop);
-
- phase->recompute_dom_depth();
-
- return true;
+ default:
+ break;
}
}
}
- return false;
-}
-
-void ZBarrierSetC2::loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round) {
- if (!Compile::current()->directive()->ZOptimizeLoadBarriersOption) {
- return;
- }
-
- if (!node->is_LoadBarrier()) {
- return;
- }
-
- if (!node->as_LoadBarrier()->has_true_uses()) {
- return;
- }
-
- if (replace_with_dominating_barrier(phase, node->as_LoadBarrier(), last_round)) {
- return;
- }
-
- if (split_barrier_thru_phi(phase, node->as_LoadBarrier())) {
- return;
- }
-
- if (move_out_of_loop(phase, node->as_LoadBarrier())) {
- return;
- }
-
- if (common_barriers(phase, node->as_LoadBarrier())) {
- return;
- }
-}
-
-Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const {
- Node* node = c;
-
- // 1. This step follows potential oop projections of a load barrier before expansion
- if (node->is_Proj()) {
- node = node->in(0);
- }
+ // Step 2 - Find dominating accesses for each load
+ for (uint i = 0; i < barrier_loads.size(); i++) {
+ MachNode* const load = barrier_loads.at(i)->as_Mach();
+ const TypePtr* load_adr_type = NULL;
+ intptr_t load_offset = 0;
+ const Node* const load_obj = load->get_base_and_disp(load_offset, load_adr_type);
+ Block* const load_block = cfg->get_block_for_node(load);
+ const uint load_index = block_index(load_block, load);
- // 2. This step checks for unexpanded load barriers
- if (node->is_LoadBarrier()) {
- return node->in(LoadBarrierNode::Oop);
- }
-
- // 3. This step checks for the phi corresponding to an optimized load barrier expansion
- if (node->is_Phi()) {
- PhiNode* phi = node->as_Phi();
- Node* n = phi->in(1);
- if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) {
- assert(c == node, "projections from step 1 should only be seen before macro expansion");
- return phi->in(2);
- }
- }
-
- return c;
-}
+ for (uint j = 0; j < mem_ops.size(); j++) {
+ MachNode* mem = mem_ops.at(j)->as_Mach();
+ const TypePtr* mem_adr_type = NULL;
+ intptr_t mem_offset = 0;
+ const Node* mem_obj = mem_obj = mem->get_base_and_disp(mem_offset, mem_adr_type);
+ Block* mem_block = cfg->get_block_for_node(mem);
+ uint mem_index = block_index(mem_block, mem);
-bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
- return type == T_OBJECT || type == T_ARRAY;
-}
-
-bool ZBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
- if (opcode != Op_LoadBarrierSlowReg &&
- opcode != Op_LoadBarrierWeakSlowReg) {
- return false;
- }
+ if (load_obj == NodeSentinel || mem_obj == NodeSentinel ||
+ load_obj == NULL || mem_obj == NULL ||
+ load_offset < 0 || mem_offset < 0) {
+ continue;
+ }
-#ifdef ASSERT
- if (VerifyOptoOopOffsets) {
- MemNode* mem = n->as_Mem();
- // Check to see if address types have grounded out somehow.
- const TypeInstPtr* tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr();
- ciInstanceKlass* k = tp->klass()->as_instance_klass();
- bool oop_offset_is_sane = k->contains_field_offset(tp->offset());
- assert(!tp || oop_offset_is_sane, "");
- }
-#endif
-
- return true;
-}
+ if (mem_obj != load_obj || mem_offset != load_offset) {
+ // Not the same addresses, not a candidate
+ continue;
+ }
-bool ZBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const {
- if (opcode == Op_CallLeaf &&
- (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() ||
- n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr())) {
- mem_op = true;
- mem_addr_idx = TypeFunc::Parms + 1;
- return true;
- }
-
- return false;
-}
-
-// == Verification ==
-
-#ifdef ASSERT
-
-static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) {
- if (visited.test_set(n->_idx)) {
- return true;
- }
-
- for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
- Node* u = n->fast_out(i);
- if (u->is_LoadBarrier()) {
- } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) {
- if (!look_for_barrier(u, post_parse, visited)) {
- return false;
- }
- } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) {
- if (!look_for_barrier(u, post_parse, visited)) {
- return false;
- }
- } else if (u->Opcode() != Op_SCMemProj) {
- tty->print("bad use"); u->dump();
- return false;
- }
- }
-
- return true;
-}
+ if (load_block == mem_block) {
+ // Earlier accesses in the same block
+ if (mem_index < load_index && !block_has_safepoint(mem_block, mem_index + 1, load_index)) {
+ load->set_barrier_data(ZLoadBarrierElided);
+ }
+ } else if (mem_block->dominates(load_block)) {
+ // Dominating block? Look around for safepoints
+ ResourceMark rm;
+ Block_List stack;
+ VectorSet visited(Thread::current()->resource_area());
+ stack.push(load_block);
+ bool safepoint_found = block_has_safepoint(load_block);
+ while (!safepoint_found && stack.size() > 0) {
+ Block* block = stack.pop();
+ if (visited.test_set(block->_pre_order)) {
+ continue;
+ }
+ if (block_has_safepoint(block)) {
+ safepoint_found = true;
+ break;
+ }
+ if (block == mem_block) {
+ continue;
+ }
-void ZBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
- if (phase == BarrierSetC2::BeforeCodeGen) return;
- bool post_parse = phase == BarrierSetC2::BeforeOptimize;
- verify_gc_barriers(post_parse);
-}
-
-void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const {
- ZBarrierSetC2State* s = state();
- Compile* C = Compile::current();
- ResourceMark rm;
- VectorSet visited(Thread::current()->resource_area());
- for (int i = 0; i < s->load_barrier_count(); i++) {
- LoadBarrierNode* n = s->load_barrier_node(i);
-
- // The dominating barrier on the same address if it exists and
- // this barrier must not be applied on the value from the same
- // load otherwise the value is not reloaded before it's used the
- // second time.
- assert(n->in(LoadBarrierNode::Similar)->is_top() ||
- (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
- n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) &&
- n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)),
- "broken similar edge");
-
- assert(post_parse || n->as_LoadBarrier()->has_true_uses(),
- "found unneeded load barrier");
-
- // Several load barrier nodes chained through their Similar edge
- // break the code that remove the barriers in final graph reshape.
- assert(n->in(LoadBarrierNode::Similar)->is_top() ||
- (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
- n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()),
- "chain of Similar load barriers");
-
- if (!n->in(LoadBarrierNode::Similar)->is_top()) {
- ResourceMark rm;
- Unique_Node_List wq;
- Node* other = n->in(LoadBarrierNode::Similar)->in(0);
- wq.push(n);
- bool ok = true;
- bool dom_found = false;
- for (uint next = 0; next < wq.size(); ++next) {
- Node *n = wq.at(next);
- assert(n->is_CFG(), "");
- assert(!n->is_SafePoint(), "");
-
- if (n == other) {
- continue;
+ // Push predecessor blocks
+ for (uint p = 1; p < block->num_preds(); ++p) {
+ Block* pred = cfg->get_block_for_node(block->pred(p));
+ stack.push(pred);
+ }
}
- if (n->is_Region()) {
- for (uint i = 1; i < n->req(); i++) {
- Node* m = n->in(i);
- if (m != NULL) {
- wq.push(m);
- }
- }
- } else {
- Node* m = n->in(0);
- if (m != NULL) {
- wq.push(m);
- }
- }
- }
- }
-
- if (ZVerifyLoadBarriers) {
- if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) {
- visited.Clear();
- bool found = look_for_barrier(n, post_parse, visited);
- if (!found) {
- n->dump(1);
- n->dump(-3);
- stringStream ss;
- C->method()->print_short_name(&ss);
- tty->print_cr("-%s-", ss.as_string());
- assert(found, "");
+ if (!safepoint_found) {
+ load->set_barrier_data(ZLoadBarrierElided);
}
}
}
}
}
-#endif
+// == Reduced spilling optimization ==
-bool ZBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const {
- switch (opcode) {
- case Op_LoadBarrierSlowReg:
- case Op_LoadBarrierWeakSlowReg:
- conn_graph->add_objload_to_connection_graph(n, delayed_worklist);
- return true;
+void ZBarrierSetC2::compute_liveness_at_stubs() const {
+ ResourceMark rm;
+ Compile* const C = Compile::current();
+ Arena* const A = Thread::current()->resource_area();
+ PhaseCFG* const cfg = C->cfg();
+ PhaseRegAlloc* const regalloc = C->regalloc();
+ RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask));
+ ZBarrierSetAssembler* const bs = ZBarrierSet::assembler();
+ Block_List worklist;
- case Op_Proj:
- if (n->as_Proj()->_con != LoadBarrierNode::Oop || !n->in(0)->is_LoadBarrier()) {
- return false;
- }
- conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), delayed_worklist);
- return true;
+ for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
+ new ((void*)(live + i)) RegMask();
+ worklist.push(cfg->get_block(i));
}
- return false;
-}
+ while (worklist.size() > 0) {
+ const Block* const block = worklist.pop();
+ RegMask& old_live = live[block->_pre_order];
+ RegMask new_live;
+
+ // Initialize to union of successors
+ for (uint i = 0; i < block->_num_succs; i++) {
+ const uint succ_id = block->_succs[i]->_pre_order;
+ new_live.OR(live[succ_id]);
+ }
-bool ZBarrierSetC2::escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const {
- switch (opcode) {
- case Op_LoadBarrierSlowReg:
- case Op_LoadBarrierWeakSlowReg:
- if (gvn->type(n)->make_ptr() == NULL) {
- return false;
+ // Walk block backwards, computing liveness
+ for (int i = block->number_of_nodes() - 1; i >= 0; --i) {
+ const Node* const node = block->get_node(i);
+
+ // Remove def bits
+ const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node));
+ const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node));
+ if (first != OptoReg::Bad) {
+ new_live.Remove(first);
+ }
+ if (second != OptoReg::Bad) {
+ new_live.Remove(second);
}
- conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(MemNode::Address), NULL);
- return true;
- case Op_Proj:
- if (n->as_Proj()->_con != LoadBarrierNode::Oop || !n->in(0)->is_LoadBarrier()) {
- return false;
+ // Add use bits
+ for (uint j = 1; j < node->req(); ++j) {
+ const Node* const use = node->in(j);
+ const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use));
+ const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use));
+ if (first != OptoReg::Bad) {
+ new_live.Insert(first);
+ }
+ if (second != OptoReg::Bad) {
+ new_live.Insert(second);
+ }
}
- conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), NULL);
- return true;
+
+ // If this node tracks liveness, update it
+ RegMask* const regs = barrier_set_state()->live(node);
+ if (regs != NULL) {
+ regs->OR(new_live);
+ }
+ }
+
+ // Now at block top, see if we have any changes
+ new_live.SUBTRACT(old_live);
+ if (new_live.is_NotEmpty()) {
+ // Liveness has refined, update and propagate to prior blocks
+ old_live.OR(new_live);
+ for (uint i = 1; i < block->num_preds(); ++i) {
+ Block* const pred = cfg->get_block_for_node(block->pred(i));
+ worklist.push(pred);
+ }
+ }
}
-
- return false;
}