--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Thu Oct 17 20:27:44 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Thu Oct 17 20:53:35 2019 +0100
@@ -38,6 +38,7 @@
#include "opto/movenode.hpp"
#include "opto/narrowptrnode.hpp"
#include "opto/rootnode.hpp"
+#include "opto/runtime.hpp"
ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
@@ -297,7 +298,7 @@
bool ShenandoahBarrierSetC2::is_shenandoah_lrb_call(Node* call) {
return call->is_CallLeaf() &&
- call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT);
+ call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier);
}
bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
@@ -462,7 +463,7 @@
const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
const Type **fields = TypeTuple::fields(1);
- fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
+ fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; // src oop
const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
// create result type (range)
@@ -473,9 +474,11 @@
}
const TypeFunc* ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type() {
- const Type **fields = TypeTuple::fields(1);
+ const Type **fields = TypeTuple::fields(2);
fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
- const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
+ fields[TypeFunc::Parms+1] = TypeRawPtr::BOTTOM; // original load address
+
+ const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
// create result type (range)
fields = TypeTuple::fields(1);
@@ -544,7 +547,7 @@
if (access.is_oop()) {
if (ShenandoahLoadRefBarrier) {
- load = new ShenandoahLoadReferenceBarrierNode(NULL, load);
+ load = new ShenandoahLoadReferenceBarrierNode(NULL, load, (decorators & IN_NATIVE) != 0);
if (access.is_parse_access()) {
load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
} else {
@@ -629,7 +632,7 @@
load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
}
#endif
- load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store));
+ load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store, false));
return load_store;
}
return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
@@ -697,7 +700,7 @@
}
Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
if (access.is_oop()) {
- result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result));
+ result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result, false));
shenandoah_write_barrier_pre(kit, false /* do_load */,
NULL, NULL, max_juint, NULL, NULL,
result /* pre_val */, T_OBJECT);
@@ -705,11 +708,6 @@
return result;
}
-void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
- assert(!src->is_AddP(), "unexpected input");
- BarrierSetC2::clone(kit, src, dst, size, is_array);
-}
-
// Support for GC barriers emitted during parsing
bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) return true;
@@ -755,7 +753,7 @@
}
bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
- bool is_oop = type == T_OBJECT || type == T_ARRAY;
+ bool is_oop = is_reference_type(type);
if (!is_oop) {
return false;
}
@@ -771,9 +769,8 @@
return true;
}
-bool ShenandoahBarrierSetC2::clone_needs_postbarrier(ArrayCopyNode *ac, PhaseIterGVN& igvn) {
- Node* src = ac->in(ArrayCopyNode::Src);
- const TypeOopPtr* src_type = igvn.type(src)->is_oopptr();
+bool ShenandoahBarrierSetC2::clone_needs_barrier(Node* src, PhaseGVN& gvn) {
+ const TypeOopPtr* src_type = gvn.type(src)->is_oopptr();
if (src_type->isa_instptr() != NULL) {
ciInstanceKlass* ik = src_type->klass()->as_instance_klass();
if ((src_type->klass_is_exact() || (!ik->is_interface() && !ik->has_subklass())) && !ik->has_injected_fields()) {
@@ -781,7 +778,7 @@
return true;
} else {
if (!src_type->klass_is_exact()) {
- igvn.C->dependencies()->assert_leaf_type(ik);
+ Compile::current()->dependencies()->assert_leaf_type(ik);
}
}
} else {
@@ -789,7 +786,7 @@
}
} else if (src_type->isa_aryptr()) {
BasicType src_elem = src_type->klass()->as_array_klass()->element_type()->basic_type();
- if (src_elem == T_OBJECT || src_elem == T_ARRAY) {
+ if (is_reference_type(src_elem)) {
return true;
}
} else {
@@ -798,42 +795,79 @@
return false;
}
-void ShenandoahBarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const {
- assert(ac->is_clonebasic(), "no other kind of arraycopy here");
-
- if (!clone_needs_postbarrier(ac, igvn)) {
- BarrierSetC2::clone_barrier_at_expansion(ac, call, igvn);
- return;
- }
-
- const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
- Node* c = new ProjNode(call,TypeFunc::Control);
- c = igvn.transform(c);
- Node* m = new ProjNode(call, TypeFunc::Memory);
- m = igvn.transform(m);
-
+void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
+ Node* ctrl = ac->in(TypeFunc::Control);
+ Node* mem = ac->in(TypeFunc::Memory);
+ Node* src = ac->in(ArrayCopyNode::Src);
+ Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
Node* dest = ac->in(ArrayCopyNode::Dest);
- assert(dest->is_AddP(), "bad input");
- Node* barrier_call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),
- CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),
- "shenandoah_clone_barrier", raw_adr_type);
- barrier_call->init_req(TypeFunc::Control, c);
- barrier_call->init_req(TypeFunc::I_O , igvn.C->top());
- barrier_call->init_req(TypeFunc::Memory , m);
- barrier_call->init_req(TypeFunc::ReturnAdr, igvn.C->top());
- barrier_call->init_req(TypeFunc::FramePtr, igvn.C->top());
- barrier_call->init_req(TypeFunc::Parms+0, dest->in(AddPNode::Base));
+ Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
+ Node* length = ac->in(ArrayCopyNode::Length);
+ assert (src_offset == NULL && dest_offset == NULL, "for clone offsets should be null");
+ assert (src->is_AddP(), "for clone the src should be the interior ptr");
+ assert (dest->is_AddP(), "for clone the dst should be the interior ptr");
+
+ if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
+ // Check if heap is has forwarded objects. If it does, we need to call into the special
+ // routine that would fix up source references before we can continue.
+
+ enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
+ Node* region = new RegionNode(PATH_LIMIT);
+ Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
+
+ Node* thread = phase->transform_later(new ThreadLocalNode());
+ Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+ Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
+
+ uint gc_state_idx = Compile::AliasIdxRaw;
+ const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
+ debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
+
+ Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
+ Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
+ Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
+ Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
+
+ IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
+ Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
+ Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
- barrier_call = igvn.transform(barrier_call);
- c = new ProjNode(barrier_call,TypeFunc::Control);
- c = igvn.transform(c);
- m = new ProjNode(barrier_call, TypeFunc::Memory);
- m = igvn.transform(m);
+ // Heap is stable, no need to do anything additional
+ region->init_req(_heap_stable, stable_ctrl);
+ mem_phi->init_req(_heap_stable, mem);
+
+ // Heap is unstable, call into clone barrier stub
+ Node* call = phase->make_leaf_call(unstable_ctrl, mem,
+ ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),
+ CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),
+ "shenandoah_clone",
+ TypeRawPtr::BOTTOM,
+ src->in(AddPNode::Base));
+ call = phase->transform_later(call);
+
+ ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
+ mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
+ region->init_req(_heap_unstable, ctrl);
+ mem_phi->init_req(_heap_unstable, mem);
- Node* out_c = ac->proj_out(TypeFunc::Control);
- Node* out_m = ac->proj_out(TypeFunc::Memory);
- igvn.replace_node(out_c, c);
- igvn.replace_node(out_m, m);
+ // Wire up the actual arraycopy stub now
+ ctrl = phase->transform_later(region);
+ mem = phase->transform_later(mem_phi);
+
+ const char* name = "arraycopy";
+ call = phase->make_leaf_call(ctrl, mem,
+ OptoRuntime::fast_arraycopy_Type(),
+ phase->basictype2arraycopy(T_LONG, NULL, NULL, true, name, true),
+ name, TypeRawPtr::BOTTOM,
+ src, dest, length
+ LP64_ONLY(COMMA phase->top()));
+ call = phase->transform_later(call);
+
+ // Hook up the whole thing into the graph
+ phase->igvn().replace_node(ac, call);
+ } else {
+ BarrierSetC2::clone_at_expansion(phase, ac);
+ }
}
@@ -912,8 +946,6 @@
}
}
-void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {}
-
void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena);
}
@@ -928,7 +960,7 @@
#ifdef ASSERT
void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
- if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeExpand) {
+ if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeMacroExpand) {
ShenandoahBarrierC2Support::verify(Compile::current()->root());
} else if (phase == BarrierSetC2::BeforeCodeGen) {
// Verify G1 pre-barriers
@@ -1196,10 +1228,6 @@
}
-bool ShenandoahBarrierSetC2::escape_is_barrier_node(Node* n) const {
- return n->Opcode() == Op_ShenandoahLoadReferenceBarrier;
-}
-
bool ShenandoahBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const {
switch (opcode) {
case Op_ShenandoahCompareAndExchangeP: