jdk-sandbox: comparison src/hotspot/share/opto/library

equal deleted inserted replaced

-:d9bc8557ae16
+:ffa644980dff
 bool inline_notify(vmIntrinsics::ID id);
 Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y);
 // This returns Type::AnyPtr, RawPtr, or OopPtr.
 int classify_unsafe_addr(Node* &base, Node* &offset, BasicType type);
 Node* make_unsafe_address(Node*& base, Node* offset, BasicType type = T_ILLEGAL, bool can_cast = false);
-// Helper for inline_unsafe_access.
-// Generates the guards that check whether the result of
-// Unsafe.getObject should be recorded in an SATB log buffer.
-void insert_pre_barrier(Node* base_oop, Node* offset, Node* pre_val, bool need_mem_bar);
 typedef enum { Relaxed, Opaque, Volatile, Acquire, Release } AccessKind;
+DecoratorSet mo_decorator_for_access_kind(AccessKind kind);
 bool inline_unsafe_access(bool is_store, BasicType type, AccessKind kind, bool is_unaligned);
 static bool klass_needs_init_guard(Node* kls);
 bool inline_unsafe_allocate();
 bool inline_unsafe_newArray(bool uninitialized);
 bool inline_unsafe_copyMemory();
 bool inline_native_subtype_check();
 bool inline_native_getLength();
 bool inline_array_copyOf(bool is_copyOfRange);
 bool inline_array_equals(StrIntrinsicNode::ArgEnc ae);
 bool inline_preconditions_checkIndex();
-void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark);
+void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array);
 bool inline_native_clone(bool is_virtual);
 bool inline_native_Reflection_getCallerClass();
 // Helper function for inlining native object hash method
 bool inline_native_hashcode(bool is_virtual, bool is_static);
 bool inline_native_getClass();
 JVMState* arraycopy_restore_alloc_state(AllocateArrayNode* alloc, int& saved_reexecute_sp);
 void arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms, int saved_reexecute_sp,
 uint new_idx);
 typedef enum { LS_get_add, LS_get_set, LS_cmp_swap, LS_cmp_swap_weak, LS_cmp_exchange } LoadStoreKind;
-MemNode::MemOrd access_kind_to_memord_LS(AccessKind access_kind, bool is_store);
-MemNode::MemOrd access_kind_to_memord(AccessKind access_kind);
 bool inline_unsafe_load_store(BasicType type,  LoadStoreKind kind, AccessKind access_kind);
 bool inline_unsafe_fence(vmIntrinsics::ID id);
 bool inline_onspinwait();
 bool inline_fp_conversions(vmIntrinsics::ID id);
 bool inline_number_methods(vmIntrinsics::ID id);
 return true;
 }
 //----------------------------inline_unsafe_access----------------------------
-// Helper that guards and inserts a pre-barrier.
-void LibraryCallKit::insert_pre_barrier(Node* base_oop, Node* offset,
-Node* pre_val, bool need_mem_bar) {
-// We could be accessing the referent field of a reference object. If so, when G1
-// is enabled, we need to log the value in the referent field in an SATB buffer.
-// This routine performs some compile time filters and generates suitable
-// runtime filters that guard the pre-barrier code.
-// Also add memory barrier for non volatile load from the referent field
-// to prevent commoning of loads across safepoint.
-if (!UseG1GC && !need_mem_bar)
-return;
-// Some compile time checks.
-// If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
-const TypeX* otype = offset->find_intptr_t_type();
-if (otype != NULL && otype->is_con() &&
-otype->get_con() != java_lang_ref_Reference::referent_offset) {
-// Constant offset but not the reference_offset so just return
-return;
-}
-// We only need to generate the runtime guards for instances.
-const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
-if (btype != NULL) {
-if (btype->isa_aryptr()) {
-// Array type so nothing to do
-return;
-}
-const TypeInstPtr* itype = btype->isa_instptr();
-if (itype != NULL) {
-// Can the klass of base_oop be statically determined to be
-// _not_ a sub-class of Reference and _not_ Object?
-ciKlass* klass = itype->klass();
-if ( klass->is_loaded() &&
-!klass->is_subtype_of(env()->Reference_klass()) &&
-!env()->Object_klass()->is_subtype_of(klass)) {
-return;
-}
-}
-}
-// The compile time filters did not reject base_oop/offset so
-// we need to generate the following runtime filters
-//
-// if (offset == java_lang_ref_Reference::_reference_offset) {
-//   if (instance_of(base, java.lang.ref.Reference)) {
-//     pre_barrier(_, pre_val, ...);
-//   }
-// }
-float likely   = PROB_LIKELY(  0.999);
-float unlikely = PROB_UNLIKELY(0.999);
-IdealKit ideal(this);
-#define __ ideal.
-Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);
-__ if_then(offset, BoolTest::eq, referent_off, unlikely); {
-// Update graphKit memory and control from IdealKit.
-sync_kit(ideal);
-Node* ref_klass_con = makecon(TypeKlassPtr::make(env()->Reference_klass()));
-Node* is_instof = gen_instanceof(base_oop, ref_klass_con);
-// Update IdealKit memory and control from graphKit.
-__ sync_kit(this);
-Node* one = __ ConI(1);
-// is_instof == 0 if base_oop == NULL
-__ if_then(is_instof, BoolTest::eq, one, unlikely); {
-// Update graphKit from IdeakKit.
-sync_kit(ideal);
-// Use the pre-barrier to record the value in the referent field
-pre_barrier(false /* do_load */,
-__ ctrl(),
-NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
-pre_val /* pre_val */,
-T_OBJECT);
-if (need_mem_bar) {
-// Add memory barrier to prevent commoning reads from this field
-// across safepoint since GC can change its value.
-insert_mem_bar(Op_MemBarCPUOrder);
-}
-// Update IdealKit from graphKit.
-__ sync_kit(this);
-} __ end_if(); // _ref_type != ref_none
-} __ end_if(); // offset == referent_offset
-// Final sync IdealKit and GraphKit.
-final_sync(ideal);
-#undef __
-}
 const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type) {
 // Attempt to infer a sharper value type from the offset and base type.
 ciKlass* sharpened_klass = NULL;
 // See if it is an instance field, with an object type.
 return tjp;
 }
 return NULL;
 }
+DecoratorSet LibraryCallKit::mo_decorator_for_access_kind(AccessKind kind) {
+switch (kind) {
+case Relaxed:
+return MO_UNORDERED;
+case Opaque:
+return MO_RELAXED;
+case Acquire:
+return MO_ACQUIRE;
+case Release:
+return MO_RELEASE;
+case Volatile:
+return MO_SEQ_CST;
+default:
+ShouldNotReachHere();
+return 0;
+}
+}
 bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, const AccessKind kind, const bool unaligned) {
 if (callee()->is_static())  return false;  // caller must have the capability!
+DecoratorSet decorators = C2_UNSAFE_ACCESS;
 guarantee(!is_store || kind != Acquire, "Acquire accesses can be produced only for loads");
 guarantee( is_store || kind != Release, "Release accesses can be produced only for stores");
 assert(type != T_OBJECT || !unaligned, "unaligned access not supported with object type");
+if (type == T_OBJECT || type == T_ARRAY) {
+decorators |= ON_UNKNOWN_OOP_REF;
+}
+if (unaligned) {
+decorators |= C2_UNALIGNED;
+}
 #ifndef PRODUCT
 {
 ResourceMark rm;
 // Check the signatures.
 }
 // Can base be NULL? Otherwise, always on-heap access.
 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop));
+if (!can_access_non_heap) {
+decorators |= IN_HEAP;
+}
 val = is_store ? argument(4) : NULL;
 const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
 // Try to categorize the address.
 mismatched = true; // conservatively mark all "wide" on-heap accesses as mismatched
 }
 assert(!mismatched || alias_type->adr_type()->is_oopptr(), "off-heap access can't be mismatched");
+if (mismatched) {
+decorators |= C2_MISMATCHED;
+}
 // First guess at the value type.
 const Type *value_type = Type::get_const_basic_type(type);
-// We will need memory barriers unless we can determine a unique
-// alias category for this reference.  (Note:  If for some reason
-// the barriers get omitted and the unsafe reference begins to "pollute"
-// the alias analysis of the rest of the graph, either Compile::can_alias
-// or Compile::must_alias will throw a diagnostic assert.)
-bool need_mem_bar = false;
-switch (kind) {
-case Relaxed:
-need_mem_bar = (mismatched && !adr_type->isa_aryptr()) || can_access_non_heap;
-break;
-case Opaque:
-// Opaque uses CPUOrder membars for protection against code movement.
-case Acquire:
-case Release:
-case Volatile:
-need_mem_bar = true;
-break;
-default:
-ShouldNotReachHere();
-}
-// Some accesses require access atomicity for all types, notably longs and doubles.
-// When AlwaysAtomicAccesses is enabled, all accesses are atomic.
-bool requires_atomic_access = false;
-switch (kind) {
-case Relaxed:
-requires_atomic_access = AlwaysAtomicAccesses;
-break;
-case Opaque:
-// Opaque accesses are atomic.
-case Acquire:
-case Release:
-case Volatile:
-requires_atomic_access = true;
-break;
-default:
-ShouldNotReachHere();
-}
 // Figure out the memory ordering.
-// Acquire/Release/Volatile accesses require marking the loads/stores with MemOrd
+decorators |= mo_decorator_for_access_kind(kind);
-MemNode::MemOrd mo = access_kind_to_memord_LS(kind, is_store);
-// If we are reading the value of the referent field of a Reference
-// object (either by using Unsafe directly or through reflection)
-// then, if G1 is enabled, we need to record the referent in an
-// SATB log buffer using the pre-barrier mechanism.
-// Also we need to add memory barrier to prevent commoning reads
-// from this field across safepoint since GC can change its value.
-bool need_read_barrier = !is_store &&
-offset != top() && heap_base_oop != top();
 if (!is_store && type == T_OBJECT) {
 const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type);
 if (tjp != NULL) {
 value_type = tjp;
 }
 // Heap pointers get a null-check from the interpreter,
 // as a courtesy.  However, this is not guaranteed by Unsafe,
 // and it is not possible to fully distinguish unintended nulls
 // from intended ones in this API.
-// We need to emit leading and trailing CPU membars (see below) in
-// addition to memory membars for special access modes. This is a little
-// too strong, but avoids the need to insert per-alias-type
-// volatile membars (for stores; compare Parse::do_put_xxx), which
-// we cannot do effectively here because we probably only have a
-// rough approximation of type.
-switch(kind) {
-case Relaxed:
-case Opaque:
-case Acquire:
-break;
-case Release:
-case Volatile:
-if (is_store) {
-insert_mem_bar(Op_MemBarRelease);
-} else {
-if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
-insert_mem_bar(Op_MemBarVolatile);
-}
-}
-break;
-default:
-ShouldNotReachHere();
-}
-// Memory barrier to prevent normal and 'unsafe' accesses from
-// bypassing each other.  Happens after null checks, so the
-// exception paths do not take memory state from the memory barrier,
-// so there's no problems making a strong assert about mixing users
-// of safe & unsafe memory.
-if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
 if (!is_store) {
 Node* p = NULL;
 // Try to constant fold a load from a constant field
 ciField* field = alias_type->field();
 if (heap_base_oop != top() && field != NULL && field->is_constant() && !mismatched) {
 // final or stable field
 p = make_constant_from_field(field, heap_base_oop);
 }
-if (p == NULL) {
-// To be valid, unsafe loads may depend on other conditions than
+if (p == NULL) { // Could not constant fold the load
-// the one that guards them: pin the Load node
+p = access_load_at(heap_base_oop, adr, adr_type, value_type, type, decorators);
-LoadNode::ControlDependency dep = LoadNode::Pinned;
+// Normalize the value returned by getBoolean in the following cases
-Node* ctrl = control();
+if (type == T_BOOLEAN &&
-// non volatile loads may be able to float
+(mismatched ||
-if (!need_mem_bar && adr_type->isa_instptr()) {
+heap_base_oop == top() ||                  // - heap_base_oop is NULL or
-assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null");
+(can_access_non_heap && field == NULL))    // - heap_base_oop is potentially NULL
-intptr_t offset = Type::OffsetBot;
+//   and the unsafe access is made to large offset
-AddPNode::Ideal_base_and_offset(adr, &_gvn, offset);
+//   (i.e., larger than the maximum offset necessary for any
-if (offset >= 0) {
+//   field access)
-int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->klass()->layout_helper());
-if (offset < s) {
-// Guaranteed to be a valid access, no need to pin it
-dep = LoadNode::DependsOnlyOnTest;
-ctrl = NULL;
-}
-}
-}
-p = make_load(ctrl, adr, value_type, type, adr_type, mo, dep, requires_atomic_access, unaligned, mismatched);
-// load value
-switch (type) {
-case T_BOOLEAN:
-{
-// Normalize the value returned by getBoolean in the following cases
-if (mismatched ||
-heap_base_oop == top() ||                            // - heap_base_oop is NULL or
-(can_access_non_heap && alias_type->field() == NULL) // - heap_base_oop is potentially NULL
-//   and the unsafe access is made to large offset
-//   (i.e., larger than the maximum offset necessary for any
-//   field access)
 ) {
 IdealKit ideal = IdealKit(this);
 #define __ ideal.
 IdealVariable normalized_result(ideal);
 __ declarations_done();
 __ set(normalized_result, ideal.ConI(1));
 ideal.end_if();
 final_sync(ideal);
 p = __ value(normalized_result);
 #undef __
-}
 }
-case T_CHAR:
+}
-case T_BYTE:
+if (type == T_ADDRESS) {
-case T_SHORT:
+p = gvn().transform(new CastP2XNode(NULL, p));
-case T_INT:
+p = ConvX2UL(p);
-case T_LONG:
-case T_FLOAT:
-case T_DOUBLE:
-break;
-case T_OBJECT:
-if (need_read_barrier) {
-// We do not require a mem bar inside pre_barrier if need_mem_bar
-// is set: the barriers would be emitted by us.
-insert_pre_barrier(heap_base_oop, offset, p, !need_mem_bar);
-}
-break;
-case T_ADDRESS:
-// Cast to an int type.
-p = _gvn.transform(new CastP2XNode(NULL, p));
-p = ConvX2UL(p);
-break;
-default:
-fatal("unexpected type %d: %s", type, type2name(type));
-break;
-}
 }
 // The load node has the control of the preceding MemBarCPUOrder.  All
 // following nodes will have the control of the MemBarCPUOrder inserted at
 // the end of this method.  So, pushing the load onto the stack at a later
 // point is fine.
 set_result(p);
 } else {
-// place effect of store into memory
+if (bt == T_ADDRESS) {
-switch (type) {
-case T_DOUBLE:
-val = dstore_rounding(val);
-break;
-case T_ADDRESS:
 // Repackage the long as a pointer.
 val = ConvL2X(val);
-val = _gvn.transform(new CastX2PNode(val));
+val = gvn().transform(new CastX2PNode(val));
-break;
+}
-default:
+access_store_at(control(), heap_base_oop, adr, adr_type, val, value_type, type, decorators);
-break;
+}
-}
-if (type == T_OBJECT) {
-store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type, mo, mismatched);
-} else {
-store_to_memory(control(), adr, val, type, adr_type, mo, requires_atomic_access, unaligned, mismatched);
-}
-}
-switch(kind) {
-case Relaxed:
-case Opaque:
-case Release:
-break;
-case Acquire:
-case Volatile:
-if (!is_store) {
-insert_mem_bar(Op_MemBarAcquire);
-} else {
-if (!support_IRIW_for_not_multiple_copy_atomic_cpu) {
-insert_mem_bar(Op_MemBarVolatile);
-}
-}
-break;
-default:
-ShouldNotReachHere();
-}
-if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
 return true;
 }
 //----------------------------inline_unsafe_load_store----------------------------
 // possible are retained from inline_unsafe_access though to make
 // the correspondences clearer. - dl
 if (callee()->is_static())  return false;  // caller must have the capability!
+DecoratorSet decorators = C2_UNSAFE_ACCESS;
+decorators |= mo_decorator_for_access_kind(access_kind);
 #ifndef PRODUCT
 BasicType rtype;
 {
 ResourceMark rm;
 // Check the signatures.
 return true;
 }
 int alias_idx = C->get_alias_index(adr_type);
-// Memory-model-wise, a LoadStore acts like a little synchronized
+if (type == T_OBJECT || type == T_ARRAY) {
-// block, so needs barriers on each side.  These don't translate
+decorators |= IN_HEAP | ON_UNKNOWN_OOP_REF;
-// into actual barriers on most machines, but we still need rest of
-// compiler to respect ordering.
-switch (access_kind) {
-case Relaxed:
-case Acquire:
-break;
-case Release:
-insert_mem_bar(Op_MemBarRelease);
-break;
-case Volatile:
-if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
-insert_mem_bar(Op_MemBarVolatile);
-} else {
-insert_mem_bar(Op_MemBarRelease);
-}
-break;
-default:
-ShouldNotReachHere();
-}
-insert_mem_bar(Op_MemBarCPUOrder);
-// Figure out the memory ordering.
-MemNode::MemOrd mo = access_kind_to_memord(access_kind);
-// 4984716: MemBars must be inserted before this
-//          memory node in order to avoid a false
-//          dependency which will confuse the scheduler.
-Node *mem = memory(alias_idx);
-// For now, we handle only those cases that actually exist: ints,
-// longs, and Object. Adding others should be straightforward.
-Node* load_store = NULL;
-switch(type) {
-case T_BYTE:
-switch(kind) {
-case LS_get_add:
-load_store = _gvn.transform(new GetAndAddBNode(control(), mem, adr, newval, adr_type));
-break;
-case LS_get_set:
-load_store = _gvn.transform(new GetAndSetBNode(control(), mem, adr, newval, adr_type));
-break;
-case LS_cmp_swap_weak:
-load_store = _gvn.transform(new WeakCompareAndSwapBNode(control(), mem, adr, newval, oldval, mo));
-break;
-case LS_cmp_swap:
-load_store = _gvn.transform(new CompareAndSwapBNode(control(), mem, adr, newval, oldval, mo));
-break;
-case LS_cmp_exchange:
-load_store = _gvn.transform(new CompareAndExchangeBNode(control(), mem, adr, newval, oldval, adr_type, mo));
-break;
-default:
-ShouldNotReachHere();
-}
-break;
-case T_SHORT:
-switch(kind) {
-case LS_get_add:
-load_store = _gvn.transform(new GetAndAddSNode(control(), mem, adr, newval, adr_type));
-break;
-case LS_get_set:
-load_store = _gvn.transform(new GetAndSetSNode(control(), mem, adr, newval, adr_type));
-break;
-case LS_cmp_swap_weak:
-load_store = _gvn.transform(new WeakCompareAndSwapSNode(control(), mem, adr, newval, oldval, mo));
-break;
-case LS_cmp_swap:
-load_store = _gvn.transform(new CompareAndSwapSNode(control(), mem, adr, newval, oldval, mo));
-break;
-case LS_cmp_exchange:
-load_store = _gvn.transform(new CompareAndExchangeSNode(control(), mem, adr, newval, oldval, adr_type, mo));
-break;
-default:
-ShouldNotReachHere();
-}
-break;
-case T_INT:
-switch(kind) {
-case LS_get_add:
-load_store = _gvn.transform(new GetAndAddINode(control(), mem, adr, newval, adr_type));
-break;
-case LS_get_set:
-load_store = _gvn.transform(new GetAndSetINode(control(), mem, adr, newval, adr_type));
-break;
-case LS_cmp_swap_weak:
-load_store = _gvn.transform(new WeakCompareAndSwapINode(control(), mem, adr, newval, oldval, mo));
-break;
-case LS_cmp_swap:
-load_store = _gvn.transform(new CompareAndSwapINode(control(), mem, adr, newval, oldval, mo));
-break;
-case LS_cmp_exchange:
-load_store = _gvn.transform(new CompareAndExchangeINode(control(), mem, adr, newval, oldval, adr_type, mo));
-break;
-default:
-ShouldNotReachHere();
-}
-break;
-case T_LONG:
-switch(kind) {
-case LS_get_add:
-load_store = _gvn.transform(new GetAndAddLNode(control(), mem, adr, newval, adr_type));
-break;
-case LS_get_set:
-load_store = _gvn.transform(new GetAndSetLNode(control(), mem, adr, newval, adr_type));
-break;
-case LS_cmp_swap_weak:
-load_store = _gvn.transform(new WeakCompareAndSwapLNode(control(), mem, adr, newval, oldval, mo));
-break;
-case LS_cmp_swap:
-load_store = _gvn.transform(new CompareAndSwapLNode(control(), mem, adr, newval, oldval, mo));
-break;
-case LS_cmp_exchange:
-load_store = _gvn.transform(new CompareAndExchangeLNode(control(), mem, adr, newval, oldval, adr_type, mo));
-break;
-default:
-ShouldNotReachHere();
-}
-break;
-case T_OBJECT:
 // Transformation of a value which could be NULL pointer (CastPP #NULL)
 // could be delayed during Parse (for example, in adjust_map_after_if()).
 // Execute transformation here to avoid barrier generation in such case.
 if (_gvn.type(newval) == TypePtr::NULL_PTR)
 newval = _gvn.makecon(TypePtr::NULL_PTR);
-// Reference stores need a store barrier.
+if (oldval != NULL && _gvn.type(oldval) == TypePtr::NULL_PTR) {
-switch(kind) {
+// Refine the value to a null constant, when it is known to be null
-case LS_get_set: {
+oldval = _gvn.makecon(TypePtr::NULL_PTR);
-// If pre-barrier must execute before the oop store, old value will require do_load here.
+}
-if (!can_move_pre_barrier()) {
+}
-pre_barrier(true /* do_load*/,
-control(), base, adr, alias_idx, newval, value_type->make_oopptr(),
+Node* result = NULL;
-NULL /* pre_val*/,
+switch (kind) {
-T_OBJECT);
+case LS_cmp_exchange: {
-} // Else move pre_barrier to use load_store value, see below.
+result = access_atomic_cmpxchg_val_at(control(), base, adr, adr_type, alias_idx,
-break;
+oldval, newval, value_type, type, decorators);
-}
-case LS_cmp_swap_weak:
-case LS_cmp_swap:
-case LS_cmp_exchange: {
-// Same as for newval above:
-if (_gvn.type(oldval) == TypePtr::NULL_PTR) {
-oldval = _gvn.makecon(TypePtr::NULL_PTR);
-}
-// The only known value which might get overwritten is oldval.
-pre_barrier(false /* do_load */,
-control(), NULL, NULL, max_juint, NULL, NULL,
-oldval /* pre_val */,
-T_OBJECT);
-break;
-}
-default:
-ShouldNotReachHere();
-}
-#ifdef _LP64
-if (adr->bottom_type()->is_ptr_to_narrowoop()) {
-Node *newval_enc = _gvn.transform(new EncodePNode(newval, newval->bottom_type()->make_narrowoop()));
-switch(kind) {
-case LS_get_set:
-load_store = _gvn.transform(new GetAndSetNNode(control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop()));
-break;
-case LS_cmp_swap_weak: {
-Node *oldval_enc = _gvn.transform(new EncodePNode(oldval, oldval->bottom_type()->make_narrowoop()));
-load_store = _gvn.transform(new WeakCompareAndSwapNNode(control(), mem, adr, newval_enc, oldval_enc, mo));
-break;
-}
-case LS_cmp_swap: {
-Node *oldval_enc = _gvn.transform(new EncodePNode(oldval, oldval->bottom_type()->make_narrowoop()));
-load_store = _gvn.transform(new CompareAndSwapNNode(control(), mem, adr, newval_enc, oldval_enc, mo));
-break;
-}
-case LS_cmp_exchange: {
-Node *oldval_enc = _gvn.transform(new EncodePNode(oldval, oldval->bottom_type()->make_narrowoop()));
-load_store = _gvn.transform(new CompareAndExchangeNNode(control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
-break;
-}
-default:
-ShouldNotReachHere();
-}
-} else
-#endif
-switch (kind) {
-case LS_get_set:
-load_store = _gvn.transform(new GetAndSetPNode(control(), mem, adr, newval, adr_type, value_type->is_oopptr()));
-break;
-case LS_cmp_swap_weak:
-load_store = _gvn.transform(new WeakCompareAndSwapPNode(control(), mem, adr, newval, oldval, mo));
-break;
-case LS_cmp_swap:
-load_store = _gvn.transform(new CompareAndSwapPNode(control(), mem, adr, newval, oldval, mo));
-break;
-case LS_cmp_exchange:
-load_store = _gvn.transform(new CompareAndExchangePNode(control(), mem, adr, newval, oldval, adr_type, value_type->is_oopptr(), mo));
-break;
-default:
-ShouldNotReachHere();
-}
-// Emit the post barrier only when the actual store happened. This makes sense
-// to check only for LS_cmp_* that can fail to set the value.
-// LS_cmp_exchange does not produce any branches by default, so there is no
-// boolean result to piggyback on. TODO: When we merge CompareAndSwap with
-// CompareAndExchange and move branches here, it would make sense to conditionalize
-// post_barriers for LS_cmp_exchange as well.
-//
-// CAS success path is marked more likely since we anticipate this is a performance
-// critical path, while CAS failure path can use the penalty for going through unlikely
-// path as backoff. Which is still better than doing a store barrier there.
-switch (kind) {
-case LS_get_set:
-case LS_cmp_exchange: {
-post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
-break;
-}
-case LS_cmp_swap_weak:
-case LS_cmp_swap: {
-IdealKit ideal(this);
-ideal.if_then(load_store, BoolTest::ne, ideal.ConI(0), PROB_STATIC_FREQUENT); {
-sync_kit(ideal);
-post_barrier(ideal.ctrl(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
-ideal.sync_kit(this);
-} ideal.end_if();
-final_sync(ideal);
-break;
-}
-default:
-ShouldNotReachHere();
-}
-break;
-default:
-fatal("unexpected type %d: %s", type, type2name(type));
-break;
-}
-// SCMemProjNodes represent the memory state of a LoadStore. Their
-// main role is to prevent LoadStore nodes from being optimized away
-// when their results aren't used.
-Node* proj = _gvn.transform(new SCMemProjNode(load_store));
-set_memory(proj, alias_idx);
-if (type == T_OBJECT && (kind == LS_get_set || kind == LS_cmp_exchange)) {
-#ifdef _LP64
-if (adr->bottom_type()->is_ptr_to_narrowoop()) {
-load_store = _gvn.transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
-}
-#endif
-if (can_move_pre_barrier() && kind == LS_get_set) {
-// Don't need to load pre_val. The old value is returned by load_store.
-// The pre_barrier can execute after the xchg as long as no safepoint
-// gets inserted between them.
-pre_barrier(false /* do_load */,
-control(), NULL, NULL, max_juint, NULL, NULL,
-load_store /* pre_val */,
-T_OBJECT);
-}
-}
-// Add the trailing membar surrounding the access
-insert_mem_bar(Op_MemBarCPUOrder);
-switch (access_kind) {
-case Relaxed:
-case Release:
-break; // do nothing
-case Acquire:
-case Volatile:
-insert_mem_bar(Op_MemBarAcquire);
-// !support_IRIW_for_not_multiple_copy_atomic_cpu handled in platform code
 break;
+}
+case LS_cmp_swap_weak:
+decorators |= C2_WEAK_CMPXCHG;
+case LS_cmp_swap: {
+result = access_atomic_cmpxchg_bool_at(control(), base, adr, adr_type, alias_idx,
+oldval, newval, value_type, type, decorators);
+break;
+}
+case LS_get_set: {
+result = access_atomic_xchg_at(control(), base, adr, adr_type, alias_idx,
+newval, value_type, type, decorators);
+break;
+}
+case LS_get_add: {
+result = access_atomic_add_at(control(), base, adr, adr_type, alias_idx,
+newval, value_type, type, decorators);
+break;
+}
 default:
 ShouldNotReachHere();
 }
-assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
+assert(type2size[result->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
-set_result(load_store);
+set_result(result);
 return true;
-}
-MemNode::MemOrd LibraryCallKit::access_kind_to_memord_LS(AccessKind kind, bool is_store) {
-MemNode::MemOrd mo = MemNode::unset;
-switch(kind) {
-case Opaque:
-case Relaxed:  mo = MemNode::unordered; break;
-case Acquire:  mo = MemNode::acquire;   break;
-case Release:  mo = MemNode::release;   break;
-case Volatile: mo = is_store ? MemNode::release : MemNode::acquire; break;
-default:
-ShouldNotReachHere();
-}
-guarantee(mo != MemNode::unset, "Should select memory ordering");
-return mo;
-}
-MemNode::MemOrd LibraryCallKit::access_kind_to_memord(AccessKind kind) {
-MemNode::MemOrd mo = MemNode::unset;
-switch(kind) {
-case Opaque:
-case Relaxed:  mo = MemNode::unordered; break;
-case Acquire:  mo = MemNode::acquire;   break;
-case Release:  mo = MemNode::release;   break;
-case Volatile: mo = MemNode::seqcst;    break;
-default:
-ShouldNotReachHere();
-}
-guarantee(mo != MemNode::unset, "Should select memory ordering");
-return mo;
 }
 bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) {
 // Regardless of form, don't allow previous ld/st to move down,
 // then issue acquire, release, or volatile mem_bar.
 return true;
 }
 //------------------------clone_coping-----------------------------------
 // Helper function for inline_native_clone.
-void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) {
+void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array) {
 assert(obj_size != NULL, "");
 Node* raw_obj = alloc_obj->in(1);
 assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
 AllocateNode* alloc = NULL;
 alloc->initialization()->set_complete_with_arraycopy();
 }
 // Copy the fastest available way.
 // TODO: generate fields copies for small objects instead.
-Node* src  = obj;
-Node* dest = alloc_obj;
 Node* size = _gvn.transform(obj_size);
-// Exclude the header but include array length to copy by 8 bytes words.
+access_clone(control(), obj, alloc_obj, size, is_array);
-// Can't use base_offset_in_bytes(bt) since basic type is unknown.
-int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
-instanceOopDesc::base_offset_in_bytes();
-// base_off:
-// 8  - 32-bit VM
-// 12 - 64-bit VM, compressed klass
-// 16 - 64-bit VM, normal klass
-if (base_off % BytesPerLong != 0) {
-assert(UseCompressedClassPointers, "");
-if (is_array) {
-// Exclude length to copy by 8 bytes words.
-base_off += sizeof(int);
-} else {
-// Include klass to copy by 8 bytes words.
-base_off = instanceOopDesc::klass_offset_in_bytes();
-}
-assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
-}
-src  = basic_plus_adr(src,  base_off);
-dest = basic_plus_adr(dest, base_off);
-// Compute the length also, if needed:
-Node* countx = size;
-countx = _gvn.transform(new SubXNode(countx, MakeConX(base_off)));
-countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong) ));
-const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
-ArrayCopyNode* ac = ArrayCopyNode::make(this, false, src, NULL, dest, NULL, countx, false, false);
-ac->set_clonebasic();
-Node* n = _gvn.transform(ac);
-if (n == ac) {
-set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
-} else {
-set_all_memory(n);
-}
-// If necessary, emit some card marks afterwards.  (Non-arrays only.)
-if (card_mark) {
-assert(!is_array, "");
-// Put in store barrier for any and all oops we are sticking
-// into this object.  (We could avoid this if we could prove
-// that the object type contains no oop fields at all.)
-Node* no_particular_value = NULL;
-Node* no_particular_field = NULL;
-int raw_adr_idx = Compile::AliasIdxRaw;
-post_barrier(control(),
-memory(raw_adr_type),
-alloc_obj,
-no_particular_field,
-raw_adr_idx,
-no_particular_value,
-T_OBJECT,
-false);
-}
 // Do not let reads from the cloned object float above the arraycopy.
 if (alloc != NULL) {
 // Do not let stores that initialize this object be reordered with
 // a subsequent store that would make this object accessible by
 result_val             = new PhiNode(result_reg, TypeInstPtr::NOTNULL);
 PhiNode*    result_i_o = new PhiNode(result_reg, Type::ABIO);
 PhiNode*    result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
 record_for_igvn(result_reg);
-const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
-int raw_adr_idx = Compile::AliasIdxRaw;
 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
 if (array_ctl != NULL) {
 // It's an array.
 PreserveJVMState pjvms(this);
 set_control(array_ctl);
 Node* obj_length = load_array_length(obj);
 Node* obj_size  = NULL;
 Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size);  // no arguments to push
-if (!use_ReduceInitialCardMarks()) {
+BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
+if (bs->array_copy_requires_gc_barriers(T_OBJECT)) {
 // If it is an oop array, it requires very special treatment,
-// because card marking is required on each card of the array.
+// because gc barriers are required when accessing the array.
 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL);
 if (is_obja != NULL) {
 PreserveJVMState pjvms2(this);
 set_control(is_obja);
 // Generate a direct call to the right arraycopy function(s).
 result_val->init_req(_objArray_path, alloc_obj);
 result_i_o ->set_req(_objArray_path, i_o());
 result_mem ->set_req(_objArray_path, reset_memory());
 }
 }
-// Otherwise, there are no card marks to worry about.
+// Otherwise, there are no barriers to worry about.
 // (We can dispense with card marks if we know the allocation
 //  comes out of eden (TLAB)...  In fact, ReduceInitialCardMarks
 //  causes the non-eden paths to take compensating steps to
 //  simulate a fresh allocation, so that no further
 //  card marks are required in compiled code to initialize
 //  the object.)
 if (!stopped()) {
-copy_to_clone(obj, alloc_obj, obj_size, true, false);
+copy_to_clone(obj, alloc_obj, obj_size, true);
 // Present the results of the copy.
 result_reg->init_req(_array_path, control());
 result_val->init_req(_array_path, alloc_obj);
 result_i_o ->set_req(_array_path, i_o());
 // Need to deoptimize on exception from allocation since Object.clone intrinsic
 // is reexecuted if deoptimization occurs and there could be problems when merging
 // exception state between multiple Object.clone versions (reexecute=true vs reexecute=false).
 Node* alloc_obj = new_instance(obj_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true);
-copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks());
+copy_to_clone(obj, alloc_obj, obj_size, false);
 // Present the results of the slow call.
 result_reg->init_req(_instance_path, control());
 result_val->init_req(_instance_path, alloc_obj);
 result_i_o ->set_req(_instance_path, i_o());
 // Get the argument:
 Node* reference_obj = null_check_receiver();
 if (stopped()) return true;
+const TypeInstPtr* tinst = _gvn.type(reference_obj)->isa_instptr();
+assert(tinst != NULL, "obj is null");
+assert(tinst->klass()->is_loaded(), "obj is not loaded");
+ciInstanceKlass* referenceKlass = tinst->klass()->as_instance_klass();
+ciField* field = referenceKlass->get_field_by_name(ciSymbol::make("referent"),
+ciSymbol::make("Ljava/lang/Object;"),
+false);
+assert (field != NULL, "undefined field");
 Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
+const TypePtr* adr_type = C->alias_type(field)->adr_type();
 ciInstanceKlass* klass = env()->Object_klass();
 const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
-Node* no_ctrl = NULL;
+DecoratorSet decorators = IN_HEAP | ON_WEAK_OOP_REF;
-Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT, MemNode::unordered);
+Node* result = access_load_at(reference_obj, adr, adr_type, object_type, T_OBJECT, decorators);
-// Use the pre-barrier to record the value in the referent field
-pre_barrier(false /* do_load */,
-control(),
-NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
-result /* pre_val */,
-T_OBJECT);
 // Add memory barrier to prevent commoning reads from this field
 // across safepoint since GC can change its value.
 insert_mem_bar(Op_MemBarCPUOrder);
 set_result(result);
 type = TypeOopPtr::make_from_klass(field_klass->as_klass());
 } else {
 type = Type::get_const_basic_type(bt);
 }
-if (support_IRIW_for_not_multiple_copy_atomic_cpu && is_vol) {
+DecoratorSet decorators = IN_HEAP;
-insert_mem_bar(Op_MemBarVolatile);   // StoreLoad barrier
-}
-// Build the load.
-MemNode::MemOrd mo = is_vol ? MemNode::acquire : MemNode::unordered;
-Node* loadedField = make_load(NULL, adr, type, bt, adr_type, mo, LoadNode::DependsOnlyOnTest, is_vol);
-// If reference is volatile, prevent following memory ops from
-// floating up past the volatile read.  Also prevents commoning
-// another volatile read.
 if (is_vol) {
-// Memory barrier includes bogus read of value to force load BEFORE membar
+decorators |= MO_SEQ_CST;
-insert_mem_bar(Op_MemBarAcquire, loadedField);
+}
-}
-return loadedField;
+return access_load_at(fromObj, adr, adr_type, type, bt, decorators);
 }
 Node * LibraryCallKit::field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
 bool is_exact = true, bool is_static = false,
 ciInstanceKlass * fromKls = NULL) {

changeset 50180	ffa644980dff
parent 50113	caf115bb98ad
child 50399	ee87876734c9