--- a/src/hotspot/share/opto/macro.cpp Thu Oct 17 20:27:44 2019 +0100
+++ b/src/hotspot/share/opto/macro.cpp Thu Oct 17 20:53:35 2019 +0100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -348,25 +348,49 @@
}
Node* res = NULL;
if (ac->is_clonebasic()) {
+ assert(ac->in(ArrayCopyNode::Src) != ac->in(ArrayCopyNode::Dest), "clone source equals destination");
Node* base = ac->in(ArrayCopyNode::Src)->in(AddPNode::Base);
Node* adr = _igvn.transform(new AddPNode(base, base, MakeConX(offset)));
const TypePtr* adr_type = _igvn.type(base)->is_ptr()->add_offset(offset);
- res = LoadNode::make(_igvn, ctl, mem, adr, adr_type, type, bt, MemNode::unordered, LoadNode::Pinned);
+ res = LoadNode::make(_igvn, ctl, mem, adr, adr_type, type, bt, MemNode::unordered, LoadNode::UnknownControl);
} else {
if (ac->modifies(offset, offset, &_igvn, true)) {
assert(ac->in(ArrayCopyNode::Dest) == alloc->result_cast(), "arraycopy destination should be allocation's result");
- uint shift = exact_log2(type2aelembytes(bt));
- Node* diff = _igvn.transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
+ uint shift = exact_log2(type2aelembytes(bt));
+ Node* src_pos = ac->in(ArrayCopyNode::SrcPos);
+ Node* dest_pos = ac->in(ArrayCopyNode::DestPos);
+ const TypeInt* src_pos_t = _igvn.type(src_pos)->is_int();
+ const TypeInt* dest_pos_t = _igvn.type(dest_pos)->is_int();
+
+ Node* adr = NULL;
+ const TypePtr* adr_type = NULL;
+ if (src_pos_t->is_con() && dest_pos_t->is_con()) {
+ intptr_t off = ((src_pos_t->get_con() - dest_pos_t->get_con()) << shift) + offset;
+ Node* base = ac->in(ArrayCopyNode::Src);
+ adr = _igvn.transform(new AddPNode(base, base, MakeConX(off)));
+ adr_type = _igvn.type(base)->is_ptr()->add_offset(off);
+ if (ac->in(ArrayCopyNode::Src) == ac->in(ArrayCopyNode::Dest)) {
+ // Don't emit a new load from src if src == dst but try to get the value from memory instead
+ return value_from_mem(ac->in(TypeFunc::Memory), ctl, ft, ftype, adr_type->isa_oopptr(), alloc);
+ }
+ } else {
+ Node* diff = _igvn.transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
#ifdef _LP64
- diff = _igvn.transform(new ConvI2LNode(diff));
+ diff = _igvn.transform(new ConvI2LNode(diff));
#endif
- diff = _igvn.transform(new LShiftXNode(diff, intcon(shift)));
+ diff = _igvn.transform(new LShiftXNode(diff, intcon(shift)));
- Node* off = _igvn.transform(new AddXNode(MakeConX(offset), diff));
- Node* base = ac->in(ArrayCopyNode::Src);
- Node* adr = _igvn.transform(new AddPNode(base, base, off));
- const TypePtr* adr_type = _igvn.type(base)->is_ptr()->add_offset(offset);
- res = LoadNode::make(_igvn, ctl, mem, adr, adr_type, type, bt, MemNode::unordered, LoadNode::Pinned);
+ Node* off = _igvn.transform(new AddXNode(MakeConX(offset), diff));
+ Node* base = ac->in(ArrayCopyNode::Src);
+ adr = _igvn.transform(new AddPNode(base, base, off));
+ adr_type = _igvn.type(base)->is_ptr()->add_offset(Type::OffsetBot);
+ if (ac->in(ArrayCopyNode::Src) == ac->in(ArrayCopyNode::Dest)) {
+ // Non constant offset in the array: we can't statically
+ // determine the value
+ return NULL;
+ }
+ }
+ res = LoadNode::make(_igvn, ctl, mem, adr, adr_type, type, bt, MemNode::unordered, LoadNode::UnknownControl);
}
}
if (res != NULL) {
@@ -497,7 +521,6 @@
Arena *a = Thread::current()->resource_area();
VectorSet visited(a);
-
bool done = sfpt_mem == alloc_mem;
Node *mem = sfpt_mem;
while (!done) {
@@ -787,7 +810,7 @@
const Type *field_type;
// The next code is taken from Parse::do_get_xxx().
- if (basic_elem_type == T_OBJECT || basic_elem_type == T_ARRAY) {
+ if (is_reference_type(basic_elem_type)) {
if (!elem_type->is_loaded()) {
field_type = TypeInstPtr::BOTTOM;
} else if (field != NULL && field->is_static_constant()) {
@@ -1008,8 +1031,17 @@
assert(init->outcnt() <= 2, "only a control and memory projection expected");
Node *ctrl_proj = init->proj_out_or_null(TypeFunc::Control);
if (ctrl_proj != NULL) {
- assert(init->in(TypeFunc::Control) == _fallthroughcatchproj, "allocation control projection");
- _igvn.replace_node(ctrl_proj, _fallthroughcatchproj);
+ _igvn.replace_node(ctrl_proj, init->in(TypeFunc::Control));
+#ifdef ASSERT
+ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
+ Node* tmp = init->in(TypeFunc::Control);
+ while (bs->is_gc_barrier_node(tmp)) {
+ Node* tmp2 = bs->step_over_gc_barrier_ctrl(tmp);
+ assert(tmp != tmp2, "Must make progress");
+ tmp = tmp2;
+ }
+ assert(tmp == _fallthroughcatchproj, "allocation control projection");
+#endif
}
Node *mem_proj = init->proj_out_or_null(TypeFunc::Memory);
if (mem_proj != NULL) {
@@ -1620,14 +1652,11 @@
Node* size_in_bytes) {
InitializeNode* init = alloc->initialization();
// Store the klass & mark bits
- Node* mark_node = NULL;
- // For now only enable fast locking for non-array types
- if (UseBiasedLocking && (length == NULL)) {
- mark_node = make_load(control, rawmem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
- } else {
- mark_node = makecon(TypeRawPtr::make((address)markOopDesc::prototype()));
+ Node* mark_node = alloc->make_ideal_mark(&_igvn, object, control, rawmem);
+ if (!mark_node->is_Con()) {
+ transform_later(mark_node);
}
- rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS);
+ rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, TypeX_X->basic_type());
rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
int header_size = alloc->minimum_header_size(); // conservatively small
@@ -2183,8 +2212,8 @@
// Get fast path - mark word has the biased lock pattern.
ctrl = opt_bits_test(ctrl, fast_lock_region, 1, mark_node,
- markOopDesc::biased_lock_mask_in_place,
- markOopDesc::biased_lock_pattern, true);
+ markWord::biased_lock_mask_in_place,
+ markWord::biased_lock_pattern, true);
// fast_lock_region->in(1) is set to slow path.
fast_lock_mem_phi->init_req(1, mem);
@@ -2212,8 +2241,9 @@
Node* x_node = transform_later(new XorXNode(o_node, mark_node));
// Get slow path - mark word does NOT match the value.
+ STATIC_ASSERT(markWord::age_mask_in_place <= INT_MAX);
Node* not_biased_ctrl = opt_bits_test(ctrl, region, 3, x_node,
- (~markOopDesc::age_mask_in_place), 0);
+ (~(int)markWord::age_mask_in_place), 0);
// region->in(3) is set to fast path - the object is biased to the current thread.
mem_phi->init_req(3, mem);
@@ -2224,7 +2254,7 @@
// First, check biased pattern.
// Get fast path - _prototype_header has the same biased lock pattern.
ctrl = opt_bits_test(not_biased_ctrl, fast_lock_region, 2, x_node,
- markOopDesc::biased_lock_mask_in_place, 0, true);
+ markWord::biased_lock_mask_in_place, 0, true);
not_biased_ctrl = fast_lock_region->in(2); // Slow path
// fast_lock_region->in(2) - the prototype header is no longer biased
@@ -2246,7 +2276,7 @@
// Get slow path - mark word does NOT match epoch bits.
Node* epoch_ctrl = opt_bits_test(ctrl, rebiased_region, 1, x_node,
- markOopDesc::epoch_mask_in_place, 0);
+ markWord::epoch_mask_in_place, 0);
// The epoch of the current bias is not valid, attempt to rebias the object
// toward the current thread.
rebiased_region->init_req(2, epoch_ctrl);
@@ -2256,9 +2286,9 @@
// rebiased_region->in(1) is set to fast path.
// The epoch of the current bias is still valid but we know
// nothing about the owner; it might be set or it might be clear.
- Node* cmask = MakeConX(markOopDesc::biased_lock_mask_in_place |
- markOopDesc::age_mask_in_place |
- markOopDesc::epoch_mask_in_place);
+ Node* cmask = MakeConX(markWord::biased_lock_mask_in_place |
+ markWord::age_mask_in_place |
+ markWord::epoch_mask_in_place);
Node* old = transform_later(new AndXNode(mark_node, cmask));
cast_thread = transform_later(new CastP2XNode(ctrl, thread));
Node* new_mark = transform_later(new OrXNode(cast_thread, old));
@@ -2373,8 +2403,8 @@
Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type());
ctrl = opt_bits_test(ctrl, region, 3, mark_node,
- markOopDesc::biased_lock_mask_in_place,
- markOopDesc::biased_lock_pattern);
+ markWord::biased_lock_mask_in_place,
+ markWord::biased_lock_pattern);
} else {
region = new RegionNode(3);
// create a Phi for the memory state
@@ -2563,15 +2593,36 @@
if (_igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
// node is unreachable, so don't try to expand it
C->remove_macro_node(n);
- } else if (n->is_ArrayCopy()){
- int macro_count = C->macro_count();
+ continue;
+ }
+ int macro_count = C->macro_count();
+ switch (n->class_id()) {
+ case Node::Class_Lock:
+ expand_lock_node(n->as_Lock());
+ assert(C->macro_count() < macro_count, "must have deleted a node from macro list");
+ break;
+ case Node::Class_Unlock:
+ expand_unlock_node(n->as_Unlock());
+ assert(C->macro_count() < macro_count, "must have deleted a node from macro list");
+ break;
+ case Node::Class_ArrayCopy:
expand_arraycopy_node(n->as_ArrayCopy());
assert(C->macro_count() < macro_count, "must have deleted a node from macro list");
+ break;
}
if (C->failing()) return true;
macro_idx --;
}
+ // All nodes except Allocate nodes are expanded now. There could be
+ // new optimization opportunities (such as folding newly created
+ // load from a just allocated object). Run IGVN.
+ _igvn.set_delay_transform(false);
+ _igvn.optimize();
+ if (C->failing()) return true;
+
+ _igvn.set_delay_transform(true);
+
// expand "macro" nodes
// nodes are removed from the macro list as they are processed
while (C->macro_count() > 0) {
@@ -2590,12 +2641,6 @@
case Node::Class_AllocateArray:
expand_allocate_array(n->as_AllocateArray());
break;
- case Node::Class_Lock:
- expand_lock_node(n->as_Lock());
- break;
- case Node::Class_Unlock:
- expand_unlock_node(n->as_Unlock());
- break;
default:
assert(false, "unknown node type in macro list");
}