--- a/src/hotspot/share/ci/ciInstanceKlass.cpp Tue Nov 06 15:23:52 2018 +0800
+++ b/src/hotspot/share/ci/ciInstanceKlass.cpp Tue Nov 06 10:01:27 2018 +0100
@@ -554,6 +554,12 @@
_has_injected_fields = has_injected_fields;
}
+bool ciInstanceKlass::has_object_fields() const {
+ GUARDED_VM_ENTRY(
+ return get_instanceKlass()->nonstatic_oop_map_size() > 0;
+ );
+}
+
// ------------------------------------------------------------------
// ciInstanceKlass::find_method
//
--- a/src/hotspot/share/ci/ciInstanceKlass.hpp Tue Nov 06 15:23:52 2018 +0800
+++ b/src/hotspot/share/ci/ciInstanceKlass.hpp Tue Nov 06 10:01:27 2018 +0100
@@ -202,6 +202,8 @@
return _has_injected_fields > 0 ? true : false;
}
+ bool has_object_fields() const;
+
// nth nonstatic field (presented by ascending address)
ciField* nonstatic_field_at(int i) {
assert(_nonstatic_fields != NULL, "");
--- a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp Tue Nov 06 15:23:52 2018 +0800
+++ b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp Tue Nov 06 10:01:27 2018 +0100
@@ -594,8 +594,6 @@
Node* G1BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
DecoratorSet decorators = access.decorators();
- GraphKit* kit = access.kit();
-
Node* adr = access.addr().node();
Node* obj = access.base();
@@ -606,7 +604,8 @@
bool is_unordered = (decorators & MO_UNORDERED) != 0;
bool need_cpu_mem_bar = !is_unordered || mismatched || !in_heap;
- Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : kit->top();
+ Node* top = Compile::current()->top();
+ Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
Node* load = CardTableBarrierSetC2::load_at_resolved(access, val_type);
// If we are reading the value of the referent field of a Reference
@@ -616,12 +615,16 @@
// Also we need to add memory barrier to prevent commoning reads
// from this field across safepoint since GC can change its value.
bool need_read_barrier = in_heap && (on_weak ||
- (unknown && offset != kit->top() && obj != kit->top()));
+ (unknown && offset != top && obj != top));
if (!access.is_oop() || !need_read_barrier) {
return load;
}
+ assert(access.is_parse_access(), "entry not supported at optimization time");
+ C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
+ GraphKit* kit = parse_access.kit();
+
if (on_weak) {
// Use the pre-barrier to record the value in the referent field
pre_barrier(kit, false /* do_load */,
--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp Tue Nov 06 15:23:52 2018 +0800
+++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp Tue Nov 06 10:01:27 2018 +0100
@@ -35,10 +35,12 @@
// By default this is a no-op.
void BarrierSetC2::resolve_address(C2Access& access) const { }
-void* C2Access::barrier_set_state() const {
+void* C2ParseAccess::barrier_set_state() const {
return _kit->barrier_set_state();
}
+PhaseGVN& C2ParseAccess::gvn() const { return _kit->gvn(); }
+
bool C2Access::needs_cpu_membar() const {
bool mismatched = (_decorators & C2_MISMATCHED) != 0;
bool is_unordered = (_decorators & MO_UNORDERED) != 0;
@@ -70,7 +72,6 @@
Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
DecoratorSet decorators = access.decorators();
- GraphKit* kit = access.kit();
bool mismatched = (decorators & C2_MISMATCHED) != 0;
bool unaligned = (decorators & C2_UNALIGNED) != 0;
@@ -79,22 +80,49 @@
bool in_native = (decorators & IN_NATIVE) != 0;
assert(!in_native, "not supported yet");
- if (access.type() == T_DOUBLE) {
- Node* new_val = kit->dstore_rounding(val.node());
- val.set_node(new_val);
- }
-
MemNode::MemOrd mo = access.mem_node_mo();
- Node* store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(),
+ Node* store;
+ if (access.is_parse_access()) {
+ C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
+
+ GraphKit* kit = parse_access.kit();
+ if (access.type() == T_DOUBLE) {
+ Node* new_val = kit->dstore_rounding(val.node());
+ val.set_node(new_val);
+ }
+
+ store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(),
access.addr().type(), mo, requires_atomic_access, unaligned, mismatched);
- access.set_raw_access(store);
+ access.set_raw_access(store);
+ } else {
+ assert(!requires_atomic_access, "not yet supported");
+ assert(access.is_opt_access(), "either parse or opt access");
+ C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
+ Node* ctl = opt_access.ctl();
+ MergeMemNode* mm = opt_access.mem();
+ PhaseGVN& gvn = opt_access.gvn();
+ const TypePtr* adr_type = access.addr().type();
+ int alias = gvn.C->get_alias_index(adr_type);
+ Node* mem = mm->memory_at(alias);
+
+ StoreNode* st = StoreNode::make(gvn, ctl, mem, access.addr().node(), adr_type, val.node(), access.type(), mo);
+ if (unaligned) {
+ st->set_unaligned_access();
+ }
+ if (mismatched) {
+ st->set_mismatched_access();
+ }
+ store = gvn.transform(st);
+ if (store == st) {
+ mm->set_memory_at(alias, st);
+ }
+ }
return store;
}
Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
DecoratorSet decorators = access.decorators();
- GraphKit* kit = access.kit();
Node* adr = access.addr().node();
const TypePtr* adr_type = access.addr().type();
@@ -109,16 +137,31 @@
MemNode::MemOrd mo = access.mem_node_mo();
LoadNode::ControlDependency dep = pinned ? LoadNode::Pinned : LoadNode::DependsOnlyOnTest;
- Node* control = control_dependent ? kit->control() : NULL;
Node* load;
- if (in_native) {
- load = kit->make_load(control, adr, val_type, access.type(), mo);
+ if (access.is_parse_access()) {
+ C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
+ GraphKit* kit = parse_access.kit();
+ Node* control = control_dependent ? kit->control() : NULL;
+
+ if (in_native) {
+ load = kit->make_load(control, adr, val_type, access.type(), mo);
+ } else {
+ load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo,
+ dep, requires_atomic_access, unaligned, mismatched);
+ }
+ access.set_raw_access(load);
} else {
- load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo,
- dep, requires_atomic_access, unaligned, mismatched);
+ assert(!requires_atomic_access, "not yet supported");
+ assert(access.is_opt_access(), "either parse or opt access");
+ C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
+ Node* control = control_dependent ? opt_access.ctl() : NULL;
+ MergeMemNode* mm = opt_access.mem();
+ PhaseGVN& gvn = opt_access.gvn();
+ Node* mem = mm->memory_at(gvn.C->get_alias_index(adr_type));
+ load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep, unaligned, mismatched);
+ load = gvn.transform(load);
}
- access.set_raw_access(load);
return load;
}
@@ -130,7 +173,11 @@
public:
C2AccessFence(C2Access& access) :
_access(access), _leading_membar(NULL) {
- GraphKit* kit = access.kit();
+ GraphKit* kit = NULL;
+ if (access.is_parse_access()) {
+ C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
+ kit = parse_access.kit();
+ }
DecoratorSet decorators = access.decorators();
bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
@@ -141,6 +188,7 @@
bool is_release = (decorators & MO_RELEASE) != 0;
if (is_atomic) {
+ assert(kit != NULL, "unsupported at optimization time");
// Memory-model-wise, a LoadStore acts like a little synchronized
// block, so needs barriers on each side. These don't translate
// into actual barriers on most machines, but we still need rest of
@@ -159,6 +207,7 @@
// floating down past the volatile write. Also prevents commoning
// another volatile read.
if (is_volatile || is_release) {
+ assert(kit != NULL, "unsupported at optimization time");
_leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
}
} else {
@@ -168,11 +217,13 @@
// so there's no problems making a strong assert about mixing users
// of safe & unsafe memory.
if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) {
+ assert(kit != NULL, "unsupported at optimization time");
_leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
}
}
if (access.needs_cpu_membar()) {
+ assert(kit != NULL, "unsupported at optimization time");
kit->insert_mem_bar(Op_MemBarCPUOrder);
}
@@ -185,7 +236,11 @@
}
~C2AccessFence() {
- GraphKit* kit = _access.kit();
+ GraphKit* kit = NULL;
+ if (_access.is_parse_access()) {
+ C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(_access);
+ kit = parse_access.kit();
+ }
DecoratorSet decorators = _access.decorators();
bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
@@ -202,6 +257,7 @@
}
if (is_atomic) {
+ assert(kit != NULL, "unsupported at optimization time");
if (is_acquire || is_volatile) {
Node* n = _access.raw_access();
Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
@@ -212,6 +268,7 @@
} else if (is_write) {
// If not multiple copy atomic, we do the MemBarVolatile before the load.
if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) {
+ assert(kit != NULL, "unsupported at optimization time");
Node* n = _access.raw_access();
Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar
if (_leading_membar != NULL) {
@@ -220,6 +277,7 @@
}
} else {
if (is_volatile || is_acquire) {
+ assert(kit != NULL, "unsupported at optimization time");
Node* n = _access.raw_access();
assert(_leading_membar == NULL || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected");
Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
@@ -295,7 +353,7 @@
if (!needs_cpu_membar() && adr_type->isa_instptr()) {
assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null");
intptr_t offset = Type::OffsetBot;
- AddPNode::Ideal_base_and_offset(adr, &_kit->gvn(), offset);
+ AddPNode::Ideal_base_and_offset(adr, &gvn(), offset);
if (offset >= 0) {
int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->klass()->layout_helper());
if (offset < s) {
@@ -310,26 +368,28 @@
//--------------------------- atomic operations---------------------------------
-void BarrierSetC2::pin_atomic_op(C2AtomicAccess& access) const {
+void BarrierSetC2::pin_atomic_op(C2AtomicParseAccess& access) const {
if (!access.needs_pinning()) {
return;
}
// SCMemProjNodes represent the memory state of a LoadStore. Their
// main role is to prevent LoadStore nodes from being optimized away
// when their results aren't used.
- GraphKit* kit = access.kit();
+ assert(access.is_parse_access(), "entry not supported at optimization time");
+ C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
+ GraphKit* kit = parse_access.kit();
Node* load_store = access.raw_access();
assert(load_store != NULL, "must pin atomic op");
Node* proj = kit->gvn().transform(new SCMemProjNode(load_store));
kit->set_memory(proj, access.alias_idx());
}
-void C2AtomicAccess::set_memory() {
+void C2AtomicParseAccess::set_memory() {
Node *mem = _kit->memory(_alias_idx);
_memory = mem;
}
-Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
+Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const {
GraphKit* kit = access.kit();
MemNode::MemOrd mo = access.mem_node_mo();
@@ -386,7 +446,7 @@
return load_store;
}
-Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
+Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const {
GraphKit* kit = access.kit();
DecoratorSet decorators = access.decorators();
@@ -460,7 +520,7 @@
return load_store;
}
-Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
+Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
GraphKit* kit = access.kit();
Node* mem = access.memory();
Node* adr = access.addr().node();
@@ -508,7 +568,7 @@
return load_store;
}
-Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
+Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
Node* load_store = NULL;
GraphKit* kit = access.kit();
Node* adr = access.addr().node();
@@ -538,27 +598,27 @@
return load_store;
}
-Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicAccess& access, Node* expected_val,
+Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const {
C2AccessFence fence(access);
resolve_address(access);
return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
}
-Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicAccess& access, Node* expected_val,
+Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const {
C2AccessFence fence(access);
resolve_address(access);
return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
}
-Node* BarrierSetC2::atomic_xchg_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
+Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
C2AccessFence fence(access);
resolve_address(access);
return atomic_xchg_at_resolved(access, new_val, value_type);
}
-Node* BarrierSetC2::atomic_add_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
+Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
C2AccessFence fence(access);
resolve_address(access);
return atomic_add_at_resolved(access, new_val, value_type);
@@ -594,7 +654,7 @@
const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
- ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, NULL, dst_base, NULL, countx, false, false);
+ ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, NULL, dst_base, NULL, countx, true, false);
ac->set_clonebasic();
Node* n = kit->gvn().transform(ac);
if (n == ac) {
@@ -731,3 +791,8 @@
}
return fast_oop;
}
+
+void BarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const {
+ // no barrier
+ igvn.replace_node(ac, call);
+}
--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp Tue Nov 06 15:23:52 2018 +0800
+++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp Tue Nov 06 10:01:27 2018 +0100
@@ -49,6 +49,10 @@
const DecoratorSet C2_WRITE_ACCESS = DECORATOR_LAST << 7;
// This denotes that the access reads state.
const DecoratorSet C2_READ_ACCESS = DECORATOR_LAST << 8;
+// A nearby allocation?
+const DecoratorSet C2_TIGHLY_COUPLED_ALLOC = DECORATOR_LAST << 9;
+// Loads and stores from an arraycopy being optimized
+const DecoratorSet C2_ARRAY_COPY = DECORATOR_LAST << 10;
class GraphKit;
class IdealKit;
@@ -88,7 +92,6 @@
// BarrierSetC2 backend hierarchy, for loads and stores, to reduce boiler plate.
class C2Access: public StackObj {
protected:
- GraphKit* _kit;
DecoratorSet _decorators;
BasicType _type;
Node* _base;
@@ -96,22 +99,17 @@
Node* _raw_access;
void fixup_decorators();
- void* barrier_set_state() const;
public:
- C2Access(GraphKit* kit, DecoratorSet decorators,
+ C2Access(DecoratorSet decorators,
BasicType type, Node* base, C2AccessValuePtr& addr) :
- _kit(kit),
_decorators(decorators),
_type(type),
_base(base),
_addr(addr),
_raw_access(NULL)
- {
- fixup_decorators();
- }
+ {}
- GraphKit* kit() const { return _kit; }
DecoratorSet decorators() const { return _decorators; }
Node* base() const { return _base; }
C2AccessValuePtr& addr() const { return _addr; }
@@ -126,23 +124,48 @@
MemNode::MemOrd mem_node_mo() const;
bool needs_cpu_membar() const;
+ virtual PhaseGVN& gvn() const = 0;
+ virtual bool is_parse_access() const { return false; }
+ virtual bool is_opt_access() const { return false; }
+};
+
+// C2Access for parse time calls to the BarrierSetC2 backend.
+class C2ParseAccess: public C2Access {
+protected:
+ GraphKit* _kit;
+
+ void* barrier_set_state() const;
+
+public:
+ C2ParseAccess(GraphKit* kit, DecoratorSet decorators,
+ BasicType type, Node* base, C2AccessValuePtr& addr) :
+ C2Access(decorators, type, base, addr),
+ _kit(kit) {
+ fixup_decorators();
+ }
+
+ GraphKit* kit() const { return _kit; }
+
template <typename T>
T barrier_set_state_as() const {
return reinterpret_cast<T>(barrier_set_state());
}
+
+ virtual PhaseGVN& gvn() const;
+ virtual bool is_parse_access() const { return true; }
};
// This class wraps a bunch of context parameters thare are passed around in the
// BarrierSetC2 backend hierarchy, for atomic accesses, to reduce boiler plate.
-class C2AtomicAccess: public C2Access {
+class C2AtomicParseAccess: public C2ParseAccess {
Node* _memory;
uint _alias_idx;
bool _needs_pinning;
public:
- C2AtomicAccess(GraphKit* kit, DecoratorSet decorators, BasicType type,
+ C2AtomicParseAccess(GraphKit* kit, DecoratorSet decorators, BasicType type,
Node* base, C2AccessValuePtr& addr, uint alias_idx) :
- C2Access(kit, decorators, type, base, addr),
+ C2ParseAccess(kit, decorators, type, base, addr),
_memory(NULL),
_alias_idx(alias_idx),
_needs_pinning(true) {}
@@ -157,6 +180,31 @@
void set_needs_pinning(bool value) { _needs_pinning = value; }
};
+// C2Access for optimization time calls to the BarrierSetC2 backend.
+class C2OptAccess: public C2Access {
+ PhaseGVN& _gvn;
+ MergeMemNode* _mem;
+ Node* _ctl;
+
+public:
+ C2OptAccess(PhaseGVN& gvn, Node* ctl, MergeMemNode* mem, DecoratorSet decorators,
+ BasicType type, Node* base, C2AccessValuePtr& addr) :
+ C2Access(decorators, type, base, addr),
+ _gvn(gvn), _mem(mem), _ctl(ctl) {
+ fixup_decorators();
+ }
+
+
+ MergeMemNode* mem() const { return _mem; }
+ Node* ctl() const { return _ctl; }
+ // void set_mem(Node* mem) { _mem = mem; }
+ void set_ctl(Node* ctl) { _ctl = ctl; }
+
+ virtual PhaseGVN& gvn() const { return _gvn; }
+ virtual bool is_opt_access() const { return true; }
+};
+
+
// This is the top-level class for the backend of the Access API in C2.
// The top-level class is responsible for performing raw accesses. The
// various GC barrier sets inherit from the BarrierSetC2 class to sprinkle
@@ -167,25 +215,25 @@
virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const;
virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const;
- virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
+ virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* val_type) const;
- virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
+ virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const;
- virtual Node* atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const;
- virtual Node* atomic_add_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const;
- void pin_atomic_op(C2AtomicAccess& access) const;
+ virtual Node* atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const;
+ virtual Node* atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const;
+ void pin_atomic_op(C2AtomicParseAccess& access) const;
public:
// This is the entry-point for the backend to perform accesses through the Access API.
virtual Node* store_at(C2Access& access, C2AccessValue& val) const;
virtual Node* load_at(C2Access& access, const Type* val_type) const;
- virtual Node* atomic_cmpxchg_val_at(C2AtomicAccess& access, Node* expected_val,
+ virtual Node* atomic_cmpxchg_val_at(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* val_type) const;
- virtual Node* atomic_cmpxchg_bool_at(C2AtomicAccess& access, Node* expected_val,
+ virtual Node* atomic_cmpxchg_bool_at(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* val_type) const;
- virtual Node* atomic_xchg_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const;
- virtual Node* atomic_add_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const;
+ virtual Node* atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const;
+ virtual Node* atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const;
virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const;
@@ -203,6 +251,7 @@
Expansion
};
virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const { return false; }
+ virtual void clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const;
// Support for GC barriers emitted during parsing
virtual bool has_load_barriers() const { return false; }
--- a/src/hotspot/share/gc/shared/c2/modRefBarrierSetC2.cpp Tue Nov 06 15:23:52 2018 +0800
+++ b/src/hotspot/share/gc/shared/c2/modRefBarrierSetC2.cpp Tue Nov 06 10:01:27 2018 +0100
@@ -32,7 +32,6 @@
Node* ModRefBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
DecoratorSet decorators = access.decorators();
- GraphKit* kit = access.kit();
const TypePtr* adr_type = access.addr().type();
Node* adr = access.addr().node();
@@ -41,11 +40,16 @@
bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
bool in_heap = (decorators & IN_HEAP) != 0;
bool use_precise = is_array || anonymous;
+ bool tighly_coupled_alloc = (decorators & C2_TIGHLY_COUPLED_ALLOC) != 0;
- if (!access.is_oop() || (!in_heap && !anonymous)) {
+ if (!access.is_oop() || tighly_coupled_alloc || (!in_heap && !anonymous)) {
return BarrierSetC2::store_at_resolved(access, val);
}
+ assert(access.is_parse_access(), "entry not supported at optimization time");
+ C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
+ GraphKit* kit = parse_access.kit();
+
uint adr_idx = kit->C->get_alias_index(adr_type);
assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
@@ -58,7 +62,7 @@
return store;
}
-Node* ModRefBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
+Node* ModRefBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const {
GraphKit* kit = access.kit();
@@ -78,7 +82,7 @@
return result;
}
-Node* ModRefBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
+Node* ModRefBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const {
GraphKit* kit = access.kit();
@@ -114,7 +118,7 @@
return load_store;
}
-Node* ModRefBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
+Node* ModRefBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
GraphKit* kit = access.kit();
Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, value_type);
--- a/src/hotspot/share/gc/shared/c2/modRefBarrierSetC2.hpp Tue Nov 06 15:23:52 2018 +0800
+++ b/src/hotspot/share/gc/shared/c2/modRefBarrierSetC2.hpp Tue Nov 06 10:01:27 2018 +0100
@@ -54,11 +54,11 @@
virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const;
- virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
+ virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const;
- virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
+ virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const;
- virtual Node* atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const;
+ virtual Node* atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const;
};
#endif // SHARE_GC_SHARED_C2_MODREFBARRIERSETC2_HPP
--- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp Tue Nov 06 15:23:52 2018 +0800
+++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp Tue Nov 06 10:01:27 2018 +0100
@@ -474,10 +474,10 @@
// == Accesses ==
-Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicAccess& access) const {
+Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicParseAccess& access) const {
assert(!UseCompressedOops, "Not allowed");
CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access();
- PhaseGVN& gvn = access.kit()->gvn();
+ PhaseGVN& gvn = access.gvn();
Compile* C = Compile::current();
GraphKit* kit = access.kit();
@@ -566,7 +566,7 @@
return phi;
}
-Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicAccess& access) const {
+Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicParseAccess& access) const {
CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access();
GraphKit* kit = access.kit();
PhaseGVN& gvn = kit->gvn();
@@ -665,7 +665,7 @@
}
}
-static bool barrier_needed(C2Access access) {
+static bool barrier_needed(C2Access& access) {
return ZBarrierSet::barrier_needed(access.decorators(), access.type());
}
@@ -677,7 +677,9 @@
bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;
- GraphKit* kit = access.kit();
+ assert(access.is_parse_access(), "entry not supported at optimization time");
+ C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
+ GraphKit* kit = parse_access.kit();
PhaseGVN& gvn = kit->gvn();
Node* adr = access.addr().node();
Node* heap_base_oop = access.base();
@@ -707,11 +709,11 @@
}
return p;
} else {
- return load_barrier(access.kit(), p, access.addr().node(), weak, true, true);
+ return load_barrier(parse_access.kit(), p, access.addr().node(), weak, true, true);
}
}
-Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
+Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* val_type) const {
Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
if (!barrier_needed(access)) {
@@ -722,7 +724,7 @@
return make_cmpx_loadbarrier(access);
}
-Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
+Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const {
Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
if (!barrier_needed(access)) {
@@ -746,7 +748,7 @@
return load_store;
}
-Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const {
+Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
if (!barrier_needed(access)) {
return result;
@@ -755,7 +757,9 @@
Node* load_store = access.raw_access();
Node* adr = access.addr().node();
- return load_barrier(access.kit(), load_store, adr, false, false, false);
+ assert(access.is_parse_access(), "entry not supported at optimization time");
+ C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
+ return load_barrier(parse_access.kit(), load_store, adr, false, false, false);
}
// == Macro Expansion ==
--- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp Tue Nov 06 15:23:52 2018 +0800
+++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp Tue Nov 06 10:01:27 2018 +0100
@@ -156,8 +156,8 @@
class ZBarrierSetC2 : public BarrierSetC2 {
private:
ZBarrierSetC2State* state() const;
- Node* make_cas_loadbarrier(C2AtomicAccess& access) const;
- Node* make_cmpx_loadbarrier(C2AtomicAccess& access) const;
+ Node* make_cas_loadbarrier(C2AtomicParseAccess& access) const;
+ Node* make_cmpx_loadbarrier(C2AtomicParseAccess& access) const;
void expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const;
void expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const;
void expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const;
@@ -165,15 +165,15 @@
protected:
virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const;
- virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access,
+ virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access,
Node* expected_val,
Node* new_val,
const Type* val_type) const;
- virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access,
+ virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access,
Node* expected_val,
Node* new_val,
const Type* value_type) const;
- virtual Node* atomic_xchg_at_resolved(C2AtomicAccess& access,
+ virtual Node* atomic_xchg_at_resolved(C2AtomicParseAccess& access,
Node* new_val,
const Type* val_type) const;
--- a/src/hotspot/share/opto/arraycopynode.cpp Tue Nov 06 15:23:52 2018 +0800
+++ b/src/hotspot/share/opto/arraycopynode.cpp Tue Nov 06 10:01:27 2018 +0100
@@ -148,6 +148,28 @@
return get_length_if_constant(phase);
}
+Node* ArrayCopyNode::load(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* adr, const TypePtr* adr_type, const Type *type, BasicType bt) {
+ DecoratorSet decorators = C2_READ_ACCESS | C2_CONTROL_DEPENDENT_LOAD | IN_HEAP | C2_ARRAY_COPY;
+ C2AccessValuePtr addr(adr, adr_type);
+ C2OptAccess access(*phase, ctl, mem, decorators, bt, adr->in(AddPNode::Base), addr);
+ Node* res = bs->load_at(access, type);
+ ctl = access.ctl();
+ return res;
+}
+
+void ArrayCopyNode::store(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* adr, const TypePtr* adr_type, Node* val, const Type *type, BasicType bt) {
+ DecoratorSet decorators = C2_WRITE_ACCESS | IN_HEAP | C2_ARRAY_COPY;
+ if (is_alloc_tightly_coupled()) {
+ decorators |= C2_TIGHLY_COUPLED_ALLOC;
+ }
+ C2AccessValuePtr addr(adr, adr_type);
+ C2AccessValue value(val, type);
+ C2OptAccess access(*phase, ctl, mem, decorators, bt, adr->in(AddPNode::Base), addr);
+ bs->store_at(access, value);
+ ctl = access.ctl();
+}
+
+
Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int count) {
if (!is_clonebasic()) {
return NULL;
@@ -182,6 +204,7 @@
ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
assert(ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem, "too many fields");
+ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
for (int i = 0; i < count; i++) {
ciField* field = ik->nonstatic_field_at(i);
int fieldidx = phase->C->alias_type(field)->index();
@@ -203,11 +226,8 @@
type = Type::get_const_basic_type(bt);
}
- Node* v = LoadNode::make(*phase, ctl, mem->memory_at(fieldidx), next_src, adr_type, type, bt, MemNode::unordered);
- v = phase->transform(v);
- Node* s = StoreNode::make(*phase, ctl, mem->memory_at(fieldidx), next_dest, adr_type, v, bt, MemNode::unordered);
- s = phase->transform(s);
- mem->set_memory_at(fieldidx, s);
+ Node* v = load(bs, phase, ctl, mem, next_src, adr_type, type, bt);
+ store(bs, phase, ctl, mem, next_dest, adr_type, v, type, bt);
}
if (!finish_transform(phase, can_reshape, ctl, mem)) {
@@ -368,28 +388,18 @@
if (!forward_ctl->is_top()) {
// copy forward
mm = mm->clone()->as_MergeMem();
- uint alias_idx_src = phase->C->get_alias_index(atp_src);
- uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
- Node *start_mem_src = mm->memory_at(alias_idx_src);
- Node *start_mem_dest = mm->memory_at(alias_idx_dest);
- Node* mem = start_mem_dest;
- bool same_alias = (alias_idx_src == alias_idx_dest);
if (count > 0) {
- Node* v = LoadNode::make(*phase, forward_ctl, start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
- v = phase->transform(v);
- mem = StoreNode::make(*phase, forward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
- mem = phase->transform(mem);
+ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
+ Node* v = load(bs, phase, forward_ctl, mm, adr_src, atp_src, value_type, copy_type);
+ store(bs, phase, forward_ctl, mm, adr_dest, atp_dest, v, value_type, copy_type);
for (int i = 1; i < count; i++) {
Node* off = phase->MakeConX(type2aelembytes(copy_type) * i);
Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));
Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));
- v = LoadNode::make(*phase, forward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered);
- v = phase->transform(v);
- mem = StoreNode::make(*phase, forward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);
- mem = phase->transform(mem);
+ v = load(bs, phase, forward_ctl, mm, next_src, atp_src, value_type, copy_type);
+ store(bs, phase, forward_ctl, mm, next_dest, atp_dest, v, value_type, copy_type);
}
- mm->set_memory_at(alias_idx_dest, mem);
} else if(can_reshape) {
PhaseIterGVN* igvn = phase->is_IterGVN();
igvn->_worklist.push(adr_src);
@@ -416,31 +426,20 @@
if (!backward_ctl->is_top()) {
// copy backward
mm = mm->clone()->as_MergeMem();
- uint alias_idx_src = phase->C->get_alias_index(atp_src);
- uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
- Node *start_mem_src = mm->memory_at(alias_idx_src);
- Node *start_mem_dest = mm->memory_at(alias_idx_dest);
- Node* mem = start_mem_dest;
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
assert(copy_type != T_OBJECT || !bs->array_copy_requires_gc_barriers(false, T_OBJECT, false, BarrierSetC2::Optimization), "only tightly coupled allocations for object arrays");
- bool same_alias = (alias_idx_src == alias_idx_dest);
if (count > 0) {
for (int i = count-1; i >= 1; i--) {
Node* off = phase->MakeConX(type2aelembytes(copy_type) * i);
Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));
Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));
- Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered);
- v = phase->transform(v);
- mem = StoreNode::make(*phase, backward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);
- mem = phase->transform(mem);
+ Node* v = load(bs, phase, backward_ctl, mm, next_src, atp_src, value_type, copy_type);
+ store(bs, phase, backward_ctl, mm, next_dest, atp_dest, v, value_type, copy_type);
}
- Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
- v = phase->transform(v);
- mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
- mem = phase->transform(mem);
- mm->set_memory_at(alias_idx_dest, mem);
+ Node* v = load(bs, phase, backward_ctl, mm, adr_src, atp_src, value_type, copy_type);
+ store(bs, phase, backward_ctl, mm, adr_dest, atp_dest, v, value_type, copy_type);
} else if(can_reshape) {
PhaseIterGVN* igvn = phase->is_IterGVN();
igvn->_worklist.push(adr_src);
--- a/src/hotspot/share/opto/arraycopynode.hpp Tue Nov 06 15:23:52 2018 +0800
+++ b/src/hotspot/share/opto/arraycopynode.hpp Tue Nov 06 10:01:27 2018 +0100
@@ -110,6 +110,9 @@
Node* ctl, Node *mem);
static bool may_modify_helper(const TypeOopPtr *t_oop, Node* n, PhaseTransform *phase, CallNode*& call);
+ static Node* load(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* addr, const TypePtr* adr_type, const Type *type, BasicType bt);
+ void store(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* addr, const TypePtr* adr_type, Node* val, const Type *type, BasicType bt);
+
public:
enum {
--- a/src/hotspot/share/opto/graphKit.cpp Tue Nov 06 15:23:52 2018 +0800
+++ b/src/hotspot/share/opto/graphKit.cpp Tue Nov 06 10:01:27 2018 +0100
@@ -1566,7 +1566,7 @@
C2AccessValuePtr addr(adr, adr_type);
C2AccessValue value(val, val_type);
- C2Access access(this, decorators | C2_WRITE_ACCESS, bt, obj, addr);
+ C2ParseAccess access(this, decorators | C2_WRITE_ACCESS, bt, obj, addr);
if (access.is_raw()) {
return _barrier_set->BarrierSetC2::store_at(access, value);
} else {
@@ -1585,7 +1585,7 @@
}
C2AccessValuePtr addr(adr, adr_type);
- C2Access access(this, decorators | C2_READ_ACCESS, bt, obj, addr);
+ C2ParseAccess access(this, decorators | C2_READ_ACCESS, bt, obj, addr);
if (access.is_raw()) {
return _barrier_set->BarrierSetC2::load_at(access, val_type);
} else {
@@ -1602,7 +1602,7 @@
}
C2AccessValuePtr addr(adr, NULL);
- C2Access access(this, decorators | C2_READ_ACCESS, bt, NULL, addr);
+ C2ParseAccess access(this, decorators | C2_READ_ACCESS, bt, NULL, addr);
if (access.is_raw()) {
return _barrier_set->BarrierSetC2::load_at(access, val_type);
} else {
@@ -1620,7 +1620,7 @@
BasicType bt,
DecoratorSet decorators) {
C2AccessValuePtr addr(adr, adr_type);
- C2AtomicAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS,
+ C2AtomicParseAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS,
bt, obj, addr, alias_idx);
if (access.is_raw()) {
return _barrier_set->BarrierSetC2::atomic_cmpxchg_val_at(access, expected_val, new_val, value_type);
@@ -1639,7 +1639,7 @@
BasicType bt,
DecoratorSet decorators) {
C2AccessValuePtr addr(adr, adr_type);
- C2AtomicAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS,
+ C2AtomicParseAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS,
bt, obj, addr, alias_idx);
if (access.is_raw()) {
return _barrier_set->BarrierSetC2::atomic_cmpxchg_bool_at(access, expected_val, new_val, value_type);
@@ -1657,7 +1657,7 @@
BasicType bt,
DecoratorSet decorators) {
C2AccessValuePtr addr(adr, adr_type);
- C2AtomicAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS,
+ C2AtomicParseAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS,
bt, obj, addr, alias_idx);
if (access.is_raw()) {
return _barrier_set->BarrierSetC2::atomic_xchg_at(access, new_val, value_type);
@@ -1675,7 +1675,7 @@
BasicType bt,
DecoratorSet decorators) {
C2AccessValuePtr addr(adr, adr_type);
- C2AtomicAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, bt, obj, addr, alias_idx);
+ C2AtomicParseAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, bt, obj, addr, alias_idx);
if (access.is_raw()) {
return _barrier_set->BarrierSetC2::atomic_add_at(access, new_val, value_type);
} else {
--- a/src/hotspot/share/opto/macro.cpp Tue Nov 06 15:23:52 2018 +0800
+++ b/src/hotspot/share/opto/macro.cpp Tue Nov 06 10:01:27 2018 +0100
@@ -940,6 +940,7 @@
}
k -= (oc2 - use->outcnt());
}
+ _igvn.remove_dead_node(use);
} else if (use->is_ArrayCopy()) {
// Disconnect ArrayCopy node
ArrayCopyNode* ac = use->as_ArrayCopy();
--- a/src/hotspot/share/opto/macroArrayCopy.cpp Tue Nov 06 15:23:52 2018 +0800
+++ b/src/hotspot/share/opto/macroArrayCopy.cpp Tue Nov 06 10:01:27 2018 +0100
@@ -72,7 +72,6 @@
Node* parm2, Node* parm3,
Node* parm4, Node* parm5,
Node* parm6, Node* parm7) {
- int size = call_type->domain()->cnt();
Node* call = new CallLeafNoFPNode(call_type, call_addr, call_name, adr_type);
call->init_req(TypeFunc::Control, ctrl);
call->init_req(TypeFunc::I_O , top());
@@ -1107,7 +1106,9 @@
Node* call = make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, src, dest, length XTOP);
transform_later(call);
- _igvn.replace_node(ac, call);
+ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
+ bs->clone_barrier_at_expansion(ac, call, _igvn);
+
return;
} else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_cloneoop()) {
Node* mem = ac->in(TypeFunc::Memory);