8061553: Contended Locking fast enter bucket
Summary: JEP-143/JDK-8061553 Contended Locking fast enter bucket
Reviewed-by: dholmes, acorn
Contributed-by: dave.dice@oracle.com, karen.kinnear@oracle.com, daniel.daugherty@oracle.com
--- a/hotspot/src/cpu/sparc/vm/macroAssembler_sparc.cpp Fri Feb 13 13:17:13 2015 +0100
+++ b/hotspot/src/cpu/sparc/vm/macroAssembler_sparc.cpp Fri Feb 13 10:01:42 2015 -0800
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -2943,24 +2943,14 @@
}
bind (IsInflated);
- if (EmitSync & 64) {
- // If m->owner != null goto IsLocked
- // Test-and-CAS vs CAS
- // Pessimistic form avoids futile (doomed) CAS attempts
- // The optimistic form avoids RTS->RTO cache line upgrades.
- ld_ptr(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), Rscratch);
- andcc(Rscratch, Rscratch, G0);
- brx(Assembler::notZero, false, Assembler::pn, done);
- delayed()->nop();
- // m->owner == null : it's unlocked.
- }
// Try to CAS m->owner from null to Self
// Invariant: if we acquire the lock then _recursions should be 0.
add(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), Rmark);
mov(G2_thread, Rscratch);
cas_ptr(Rmark, G0, Rscratch);
- cmp(Rscratch, G0);
+ andcc(Rscratch, Rscratch, G0); // set ICCs for done: icc.zf iff success
+ // set icc.zf : 1=success 0=failure
// ST box->displaced_header = NonZero.
// Any non-zero value suffices:
// markOopDesc::unused_mark(), G2_thread, RBox, RScratch, rsp, etc.
--- a/hotspot/src/cpu/x86/vm/macroAssembler_x86.cpp Fri Feb 13 13:17:13 2015 +0100
+++ b/hotspot/src/cpu/x86/vm/macroAssembler_x86.cpp Fri Feb 13 10:01:42 2015 -0800
@@ -1718,27 +1718,6 @@
// Force all sync thru slow-path: slow_enter() and slow_exit()
movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
cmpptr (rsp, (int32_t)NULL_WORD);
- } else
- if (EmitSync & 2) {
- Label DONE_LABEL ;
- if (UseBiasedLocking) {
- // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.
- biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, counters);
- }
-
- movptr(tmpReg, Address(objReg, 0)); // fetch markword
- orptr (tmpReg, 0x1);
- movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
- if (os::is_MP()) {
- lock();
- }
- cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
- jccb(Assembler::equal, DONE_LABEL);
- // Recursive locking
- subptr(tmpReg, rsp);
- andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
- movptr(Address(boxReg, 0), tmpReg);
- bind(DONE_LABEL);
} else {
// Possible cases that we'll encounter in fast_lock
// ------------------------------------------------
@@ -1923,29 +1902,19 @@
}
#else // _LP64
// It's inflated
-
- // TODO: someday avoid the ST-before-CAS penalty by
- // relocating (deferring) the following ST.
- // We should also think about trying a CAS without having
- // fetched _owner. If the CAS is successful we may
- // avoid an RTO->RTS upgrade on the $line.
-
- // Without cast to int32_t a movptr will destroy r10 which is typically obj
- movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
-
- movptr (boxReg, tmpReg);
- movptr(tmpReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
- testptr(tmpReg, tmpReg);
- jccb (Assembler::notZero, DONE_LABEL);
-
- // It's inflated and appears unlocked
+ movq(scrReg, tmpReg);
+ xorq(tmpReg, tmpReg);
+
if (os::is_MP()) {
lock();
}
- cmpxchgptr(r15_thread, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
+ cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
+ // Unconditionally set box->_displaced_header = markOopDesc::unused_mark().
+ // Without cast to int32_t movptr will destroy r10 which is typically obj.
+ movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
// Intentional fall-through into DONE_LABEL ...
+ // Propagate ICC.ZF from CAS above into DONE_LABEL.
#endif // _LP64
-
#if INCLUDE_RTM_OPT
} // use_rtm()
#endif
--- a/hotspot/src/share/vm/runtime/interfaceSupport.hpp Fri Feb 13 13:17:13 2015 +0100
+++ b/hotspot/src/share/vm/runtime/interfaceSupport.hpp Fri Feb 13 10:01:42 2015 -0800
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -511,6 +511,12 @@
Thread* THREAD = thread; \
debug_only(VMEntryWrapper __vew;)
+#define JRT_BLOCK_NO_ASYNC \
+ { \
+ ThreadInVMfromJavaNoAsyncException __tiv(thread); \
+ Thread* THREAD = thread; \
+ debug_only(VMEntryWrapper __vew;)
+
#define JRT_BLOCK_END }
#define JRT_END }
--- a/hotspot/src/share/vm/runtime/sharedRuntime.cpp Fri Feb 13 13:17:13 2015 +0100
+++ b/hotspot/src/share/vm/runtime/sharedRuntime.cpp Fri Feb 13 10:01:42 2015 -0800
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1792,7 +1792,17 @@
// Handles the uncommon case in locking, i.e., contention or an inflated lock.
-JRT_ENTRY_NO_ASYNC(void, SharedRuntime::complete_monitor_locking_C(oopDesc* _obj, BasicLock* lock, JavaThread* thread))
+JRT_BLOCK_ENTRY(void, SharedRuntime::complete_monitor_locking_C(oopDesc* _obj, BasicLock* lock, JavaThread* thread))
+ if (!SafepointSynchronize::is_synchronizing()) {
+ // Only try quick_enter() if we're not trying to reach a safepoint
+ // so that the calling thread reaches the safepoint more quickly.
+ if (ObjectSynchronizer::quick_enter(_obj, thread, lock)) return;
+ }
+ // NO_ASYNC required because an async exception on the state transition destructor
+ // would leave you with the lock held and it would never be released.
+ // The normal monitorenter NullPointerException is thrown without acquiring a lock
+ // and the model is that an exception implies the method failed.
+ JRT_BLOCK_NO_ASYNC
oop obj(_obj);
if (PrintBiasedLockingStatistics) {
Atomic::inc(BiasedLocking::slow_path_entry_count_addr());
@@ -1805,6 +1815,7 @@
ObjectSynchronizer::slow_enter(h_obj, lock, CHECK);
}
assert(!HAS_PENDING_EXCEPTION, "Should have no exception here");
+ JRT_BLOCK_END
JRT_END
// Handles the uncommon cases of monitor unlocking in compiled code
--- a/hotspot/src/share/vm/runtime/synchronizer.cpp Fri Feb 13 13:17:13 2015 +0100
+++ b/hotspot/src/share/vm/runtime/synchronizer.cpp Fri Feb 13 10:01:42 2015 -0800
@@ -122,6 +122,70 @@
static volatile int MonitorPopulation = 0; // # Extant -- in circulation
#define CHAINMARKER (cast_to_oop<intptr_t>(-1))
+
+// =====================> Quick functions
+
+// The quick_* forms are special fast-path variants used to improve
+// performance. In the simplest case, a "quick_*" implementation could
+// simply return false, in which case the caller will perform the necessary
+// state transitions and call the slow-path form.
+// The fast-path is designed to handle frequently arising cases in an efficient
+// manner and is just a degenerate "optimistic" variant of the slow-path.
+// returns true -- to indicate the call was satisfied.
+// returns false -- to indicate the call needs the services of the slow-path.
+// A no-loitering ordinance is in effect for code in the quick_* family
+// operators: safepoints or indefinite blocking (blocking that might span a
+// safepoint) are forbidden. Generally the thread_state() is _in_Java upon
+// entry.
+
+// The LockNode emitted directly at the synchronization site would have
+// been too big if it were to have included support for the cases of inflated
+// recursive enter and exit, so they go here instead.
+// Note that we can't safely call AsyncPrintJavaStack() from within
+// quick_enter() as our thread state remains _in_Java.
+
+bool ObjectSynchronizer::quick_enter(oop obj, Thread * Self,
+ BasicLock * Lock) {
+ assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
+ assert(Self->is_Java_thread(), "invariant");
+ assert(((JavaThread *) Self)->thread_state() == _thread_in_Java, "invariant");
+ No_Safepoint_Verifier nsv;
+ if (obj == NULL) return false; // Need to throw NPE
+ const markOop mark = obj->mark();
+
+ if (mark->has_monitor()) {
+ ObjectMonitor * const m = mark->monitor();
+ assert(m->object() == obj, "invariant");
+ Thread * const owner = (Thread *) m->_owner;
+
+ // Lock contention and Transactional Lock Elision (TLE) diagnostics
+ // and observability
+ // Case: light contention possibly amenable to TLE
+ // Case: TLE inimical operations such as nested/recursive synchronization
+
+ if (owner == Self) {
+ m->_recursions++;
+ return true;
+ }
+
+ if (owner == NULL &&
+ Atomic::cmpxchg_ptr(Self, &(m->_owner), NULL) == NULL) {
+ assert(m->_recursions == 0, "invariant");
+ assert(m->_owner == Self, "invariant");
+ return true;
+ }
+ }
+
+ // Note that we could inflate in quick_enter.
+ // This is likely a useful optimization
+ // Critically, in quick_enter() we must not:
+ // -- perform bias revocation, or
+ // -- block indefinitely, or
+ // -- reach a safepoint
+
+ return false; // revert to slow-path
+}
+
// -----------------------------------------------------------------------------
// Fast Monitor Enter/Exit
// This the fast monitor enter. The interpreter and compiler use
--- a/hotspot/src/share/vm/runtime/synchronizer.hpp Fri Feb 13 13:17:13 2015 +0100
+++ b/hotspot/src/share/vm/runtime/synchronizer.hpp Fri Feb 13 10:01:42 2015 -0800
@@ -72,6 +72,8 @@
static void notify(Handle obj, TRAPS);
static void notifyall(Handle obj, TRAPS);
+ static bool quick_enter(oop obj, Thread* Self, BasicLock* Lock);
+
// Special internal-use-only method for use by JVM infrastructure
// that needs to wait() on a java-level object but that can't risk
// throwing unexpected InterruptedExecutionExceptions.