/*

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "runtime/mutex.hpp"

#include "runtime/osThread.hpp"

#include "runtime/thread.inline.hpp"

#include "utilities/events.hpp"

#ifdef TARGET_OS_FAMILY_linux

# include "mutex_linux.inline.hpp"

#endif

#ifdef TARGET_OS_FAMILY_solaris

# include "mutex_solaris.inline.hpp"

#endif

#ifdef TARGET_OS_FAMILY_windows

# include "mutex_windows.inline.hpp"

#endif

#ifdef TARGET_OS_FAMILY_bsd

# include "mutex_bsd.inline.hpp"

#endif

// o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o

//

// Native Monitor-Mutex locking - theory of operations

//

// * Native Monitors are completely unrelated to Java-level monitors,

//   although the "back-end" slow-path implementations share a common lineage.

//   See objectMonitor:: in synchronizer.cpp.

//   Native Monitors do *not* support nesting or recursion but otherwise

//   they're basically Hoare-flavor monitors.

//

// * A thread acquires ownership of a Monitor/Mutex by CASing the LockByte

//   in the _LockWord from zero to non-zero.  Note that the _Owner field

//   is advisory and is used only to verify that the thread calling unlock()

//   is indeed the last thread to have acquired the lock.

//

// * Contending threads "push" themselves onto the front of the contention

//   queue -- called the cxq -- with CAS and then spin/park.

//   The _LockWord contains the LockByte as well as the pointer to the head

//   of the cxq.  Colocating the LockByte with the cxq precludes certain races.

//

// * Using a separately addressable LockByte allows for CAS:MEMBAR or CAS:0

//   idioms.  We currently use MEMBAR in the uncontended unlock() path, as

//   MEMBAR often has less latency than CAS.  If warranted, we could switch to

//   a CAS:0 mode, using timers to close the resultant race, as is done

//   with Java Monitors in synchronizer.cpp.

//

//   See the following for a discussion of the relative cost of atomics (CAS)

//   MEMBAR, and ways to eliminate such instructions from the common-case paths:

//   -- http://blogs.sun.com/dave/entry/biased_locking_in_hotspot

//   -- http://blogs.sun.com/dave/resource/MustangSync.pdf

//   -- http://blogs.sun.com/dave/resource/synchronization-public2.pdf

//   -- synchronizer.cpp

//

// * Overall goals - desiderata

//   1. Minimize context switching

//   2. Minimize lock migration

//   3. Minimize CPI -- affinity and locality

//   4. Minimize the execution of high-latency instructions such as CAS or MEMBAR

//   5. Minimize outer lock hold times

//   6. Behave gracefully on a loaded system

//

// * Thread flow and list residency:

//

//   Contention queue --> EntryList --> OnDeck --> Owner --> !Owner

//   [..resident on monitor list..]

//   [...........contending..................]

//

//   -- The contention queue (cxq) contains recently-arrived threads (RATs).

//      Threads on the cxq eventually drain into the EntryList.

//   -- Invariant: a thread appears on at most one list -- cxq, EntryList

//      or WaitSet -- at any one time.

//   -- For a given monitor there can be at most one "OnDeck" thread at any

//      given time but if needbe this particular invariant could be relaxed.

//

// * The WaitSet and EntryList linked lists are composed of ParkEvents.

//   I use ParkEvent instead of threads as ParkEvents are immortal and

//   type-stable, meaning we can safely unpark() a possibly stale

//   list element in the unlock()-path.  (That's benign).

//

// * Succession policy - providing for progress:

//

//   As necessary, the unlock()ing thread identifies, unlinks, and unparks

//   an "heir presumptive" tentative successor thread from the EntryList.

//   This becomes the so-called "OnDeck" thread, of which there can be only

//   one at any given time for a given monitor.  The wakee will recontend

//   for ownership of monitor.

//

//   Succession is provided for by a policy of competitive handoff.

//   The exiting thread does _not_ grant or pass ownership to the

//   successor thread.  (This is also referred to as "handoff" succession").

//   Instead the exiting thread releases ownership and possibly wakes

//   a successor, so the successor can (re)compete for ownership of the lock.

//

//   Competitive handoff provides excellent overall throughput at the expense

//   of short-term fairness.  If fairness is a concern then one remedy might

//   be to add an AcquireCounter field to the monitor.  After a thread acquires

//   the lock it will decrement the AcquireCounter field.  When the count

//   reaches 0 the thread would reset the AcquireCounter variable, abdicate

//   the lock directly to some thread on the EntryList, and then move itself to the

//   tail of the EntryList.

//

//   But in practice most threads engage or otherwise participate in resource

//   bounded producer-consumer relationships, so lock domination is not usually

//   a practical concern.  Recall too, that in general it's easier to construct

//   a fair lock from a fast lock, but not vice-versa.

//

// * The cxq can have multiple concurrent "pushers" but only one concurrent

//   detaching thread.  This mechanism is immune from the ABA corruption.

//   More precisely, the CAS-based "push" onto cxq is ABA-oblivious.

//   We use OnDeck as a pseudo-lock to enforce the at-most-one detaching

//   thread constraint.

//

// * Taken together, the cxq and the EntryList constitute or form a

//   single logical queue of threads stalled trying to acquire the lock.

//   We use two distinct lists to reduce heat on the list ends.

//   Threads in lock() enqueue onto cxq while threads in unlock() will

//   dequeue from the EntryList.  (c.f. Michael Scott's "2Q" algorithm).

//   A key desideratum is to minimize queue & monitor metadata manipulation

//   that occurs while holding the "outer" monitor lock -- that is, we want to

//   minimize monitor lock holds times.

//

//   The EntryList is ordered by the prevailing queue discipline and

//   can be organized in any convenient fashion, such as a doubly-linked list or

//   a circular doubly-linked list.  If we need a priority queue then something akin

//   to Solaris' sleepq would work nicely.  Viz.,

//   -- http://agg.eng/ws/on10_nightly/source/usr/src/uts/common/os/sleepq.c.

//   -- http://cvs.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/uts/common/os/sleepq.c

//   Queue discipline is enforced at ::unlock() time, when the unlocking thread

//   drains the cxq into the EntryList, and orders or reorders the threads on the

//   EntryList accordingly.

//

//   Barring "lock barging", this mechanism provides fair cyclic ordering,

//   somewhat similar to an elevator-scan.

//

// * OnDeck

//   --  For a given monitor there can be at most one OnDeck thread at any given

//       instant.  The OnDeck thread is contending for the lock, but has been

//       unlinked from the EntryList and cxq by some previous unlock() operations.

//       Once a thread has been designated the OnDeck thread it will remain so

//       until it manages to acquire the lock -- being OnDeck is a stable property.

//   --  Threads on the EntryList or cxq are _not allowed to attempt lock acquisition.

//   --  OnDeck also serves as an "inner lock" as follows.  Threads in unlock() will, after

//       having cleared the LockByte and dropped the outer lock,  attempt to "trylock"

//       OnDeck by CASing the field from null to non-null.  If successful, that thread

//       is then responsible for progress and succession and can use CAS to detach and

//       drain the cxq into the EntryList.  By convention, only this thread, the holder of

//       the OnDeck inner lock, can manipulate the EntryList or detach and drain the

//       RATs on the cxq into the EntryList.  This avoids ABA corruption on the cxq as

//       we allow multiple concurrent "push" operations but restrict detach concurrency

//       to at most one thread.  Having selected and detached a successor, the thread then

//       changes the OnDeck to refer to that successor, and then unparks the successor.

//       That successor will eventually acquire the lock and clear OnDeck.  Beware

//       that the OnDeck usage as a lock is asymmetric.  A thread in unlock() transiently

//       "acquires" OnDeck, performs queue manipulations, passes OnDeck to some successor,

//       and then the successor eventually "drops" OnDeck.  Note that there's never

//       any sense of contention on the inner lock, however.  Threads never contend

//       or wait for the inner lock.

//   --  OnDeck provides for futile wakeup throttling a described in section 3.3 of

//       See http://www.usenix.org/events/jvm01/full_papers/dice/dice.pdf

//       In a sense, OnDeck subsumes the ObjectMonitor _Succ and ObjectWaiter

//       TState fields found in Java-level objectMonitors.  (See synchronizer.cpp).

//

// * Waiting threads reside on the WaitSet list -- wait() puts

//   the caller onto the WaitSet.  Notify() or notifyAll() simply

//   transfers threads from the WaitSet to either the EntryList or cxq.

//   Subsequent unlock() operations will eventually unpark the notifyee.

//   Unparking a notifee in notify() proper is inefficient - if we were to do so

//   it's likely the notifyee would simply impale itself on the lock held

//   by the notifier.

//

// * The mechanism is obstruction-free in that if the holder of the transient

//   OnDeck lock in unlock() is preempted or otherwise stalls, other threads

//   can still acquire and release the outer lock and continue to make progress.

//   At worst, waking of already blocked contending threads may be delayed,

//   but nothing worse.  (We only use "trylock" operations on the inner OnDeck

//   lock).

//

// * Note that thread-local storage must be initialized before a thread

//   uses Native monitors or mutexes.  The native monitor-mutex subsystem

//   depends on Thread::current().

//

// * The monitor synchronization subsystem avoids the use of native

//   synchronization primitives except for the narrow platform-specific

//   park-unpark abstraction.  See the comments in os_solaris.cpp regarding

//   the semantics of park-unpark.  Put another way, this monitor implementation

//   depends only on atomic operations and park-unpark.  The monitor subsystem

//   manages all RUNNING->BLOCKED and BLOCKED->READY transitions while the

//   underlying OS manages the READY<->RUN transitions.

//

// * The memory consistency model provide by lock()-unlock() is at least as

//   strong or stronger than the Java Memory model defined by JSR-133.

//   That is, we guarantee at least entry consistency, if not stronger.

//   See http://g.oswego.edu/dl/jmm/cookbook.html.

//

// * Thread:: currently contains a set of purpose-specific ParkEvents:

//   _MutexEvent, _ParkEvent, etc.  A better approach might be to do away with

//   the purpose-specific ParkEvents and instead implement a general per-thread

//   stack of available ParkEvents which we could provision on-demand.  The

//   stack acts as a local cache to avoid excessive calls to ParkEvent::Allocate()

//   and ::Release().  A thread would simply pop an element from the local stack before it

//   enqueued or park()ed.  When the contention was over the thread would

//   push the no-longer-needed ParkEvent back onto its stack.

//

// * A slightly reduced form of ILock() and IUnlock() have been partially

//   model-checked (Murphi) for safety and progress at T=1,2,3 and 4.

//   It'd be interesting to see if TLA/TLC could be useful as well.

//

// * Mutex-Monitor is a low-level "leaf" subsystem.  That is, the monitor

//   code should never call other code in the JVM that might itself need to

//   acquire monitors or mutexes.  That's true *except* in the case of the

//   ThreadBlockInVM state transition wrappers.  The ThreadBlockInVM DTOR handles

//   mutator reentry (ingress) by checking for a pending safepoint in which case it will

//   call SafepointSynchronize::block(), which in turn may call Safepoint_lock->lock(), etc.

//   In that particular case a call to lock() for a given Monitor can end up recursively

//   calling lock() on another monitor.   While distasteful, this is largely benign

//   as the calls come from jacket that wraps lock(), and not from deep within lock() itself.

//

//   It's unfortunate that native mutexes and thread state transitions were convolved.

//   They're really separate concerns and should have remained that way.  Melding

//   them together was facile -- a bit too facile.   The current implementation badly

//   conflates the two concerns.

//

// * TODO-FIXME:

//

//   -- Add DTRACE probes for contended acquire, contended acquired, contended unlock

//      We should also add DTRACE probes in the ParkEvent subsystem for

//      Park-entry, Park-exit, and Unpark.

//

//   -- We have an excess of mutex-like constructs in the JVM, namely:

//      1. objectMonitors for Java-level synchronization (synchronizer.cpp)

//      2. low-level muxAcquire and muxRelease

//      3. low-level spinAcquire and spinRelease

//      4. native Mutex:: and Monitor::

//      5. jvm_raw_lock() and _unlock()

//      6. JVMTI raw monitors -- distinct from (5) despite having a confusingly

//         similar name.

//

// o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o-o

// CASPTR() uses the canonical argument order that dominates in the literature.

// Our internal cmpxchg_ptr() uses a bastardized ordering to accommodate Sun .il templates.

#define CASPTR(a,c,s) intptr_t(Atomic::cmpxchg_ptr ((void *)(s),(void *)(a),(void *)(c)))

#define UNS(x) (uintptr_t(x))

#define TRACE(m) { static volatile int ctr = 0 ; int x = ++ctr ; if ((x & (x-1))==0) { ::printf ("%d:%s\n", x, #m); ::fflush(stdout); }}

// Simplistic low-quality Marsaglia SHIFT-XOR RNG.

// Bijective except for the trailing mask operation.

// Useful for spin loops as the compiler can't optimize it away.

static inline jint MarsagliaXORV (jint x) {

  if (x == 0) x = 1|os::random() ;

  x ^= x << 6;

  x ^= ((unsigned)x) >> 21;

  x ^= x << 7 ;

  return x & 0x7FFFFFFF ;

}

static inline jint MarsagliaXOR (jint * const a) {

  jint x = *a ;

  if (x == 0) x = UNS(a)|1 ;

  x ^= x << 6;

  x ^= ((unsigned)x) >> 21;

  x ^= x << 7 ;

  *a = x ;

  return x & 0x7FFFFFFF ;

}

static int Stall (int its) {

  static volatile jint rv = 1 ;

  volatile int OnFrame = 0 ;

  jint v = rv ^ UNS(OnFrame) ;

  while (--its >= 0) {

    v = MarsagliaXORV (v) ;

  }

  // Make this impossible for the compiler to optimize away,

  // but (mostly) avoid W coherency sharing on MP systems.

  if (v == 0x12345) rv = v ;

  return v ;

}

int Monitor::TryLock () {

  intptr_t v = _LockWord.FullWord ;

  for (;;) {

    if ((v & _LBIT) != 0) return 0 ;

    const intptr_t u = CASPTR (&_LockWord, v, v|_LBIT) ;

    if (v == u) return 1 ;

    v = u ;

  }

}

int Monitor::TryFast () {

  // Optimistic fast-path form ...

  // Fast-path attempt for the common uncontended case.

  // Avoid RTS->RTO $ coherence upgrade on typical SMP systems.

  intptr_t v = CASPTR (&_LockWord, 0, _LBIT) ;  // agro ...

  if (v == 0) return 1 ;

  for (;;) {

    if ((v & _LBIT) != 0) return 0 ;

    const intptr_t u = CASPTR (&_LockWord, v, v|_LBIT) ;

    if (v == u) return 1 ;

    v = u ;

  }

}

int Monitor::ILocked () {

  const intptr_t w = _LockWord.FullWord & 0xFF ;

  assert (w == 0 || w == _LBIT, "invariant") ;

  return w == _LBIT ;

}

// Polite TATAS spinlock with exponential backoff - bounded spin.

// Ideally we'd use processor cycles, time or vtime to control

// the loop, but we currently use iterations.

// All the constants within were derived empirically but work over

// over the spectrum of J2SE reference platforms.

// On Niagara-class systems the back-off is unnecessary but

// is relatively harmless.  (At worst it'll slightly retard

// acquisition times).  The back-off is critical for older SMP systems

// where constant fetching of the LockWord would otherwise impair

// scalability.

//

// Clamp spinning at approximately 1/2 of a context-switch round-trip.

// See synchronizer.cpp for details and rationale.

int Monitor::TrySpin (Thread * const Self) {

  if (TryLock())    return 1 ;

  if (!os::is_MP()) return 0 ;

  int Probes  = 0 ;

  int Delay   = 0 ;

  int Steps   = 0 ;

  int SpinMax = NativeMonitorSpinLimit ;

  int flgs    = NativeMonitorFlags ;

  for (;;) {

    intptr_t v = _LockWord.FullWord;

    if ((v & _LBIT) == 0) {

      if (CASPTR (&_LockWord, v, v|_LBIT) == v) {

        return 1 ;

      }

      continue ;

    }

    if ((flgs & 8) == 0) {

      SpinPause () ;

    }

    // Periodically increase Delay -- variable Delay form

    // conceptually: delay *= 1 + 1/Exponent

    ++ Probes;

    if (Probes > SpinMax) return 0 ;

    if ((Probes & 0x7) == 0) {

      Delay = ((Delay << 1)|1) & 0x7FF ;

      // CONSIDER: Delay += 1 + (Delay/4); Delay &= 0x7FF ;

    }

    if (flgs & 2) continue ;

    // Consider checking _owner's schedctl state, if OFFPROC abort spin.

    // If the owner is OFFPROC then it's unlike that the lock will be dropped

    // in a timely fashion, which suggests that spinning would not be fruitful

    // or profitable.

    // Stall for "Delay" time units - iterations in the current implementation.

    // Avoid generating coherency traffic while stalled.

    // Possible ways to delay:

    //   PAUSE, SLEEP, MEMBAR #sync, MEMBAR #halt,

    //   wr %g0,%asi, gethrtime, rdstick, rdtick, rdtsc, etc. ...

    // Note that on Niagara-class systems we want to minimize STs in the

    // spin loop.  N1 and brethren write-around the L1$ over the xbar into the L2$.

    // Furthermore, they don't have a W$ like traditional SPARC processors.

    // We currently use a Marsaglia Shift-Xor RNG loop.

    Steps += Delay ;

    if (Self != NULL) {

      jint rv = Self->rng[0] ;

      for (int k = Delay ; --k >= 0; ) {

        rv = MarsagliaXORV (rv) ;

        if ((flgs & 4) == 0 && SafepointSynchronize::do_call_back()) return 0 ;

      }

      Self->rng[0] = rv ;

    } else {

      Stall (Delay) ;

    }

  }

}

static int ParkCommon (ParkEvent * ev, jlong timo) {

  // Diagnostic support - periodically unwedge blocked threads

  intx nmt = NativeMonitorTimeout ;

  if (nmt > 0 && (nmt < timo || timo <= 0)) {

     timo = nmt ;

  }

  int err = OS_OK ;

  if (0 == timo) {

    ev->park() ;

  } else {

    err = ev->park(timo) ;

  }

  return err ;

}

inline int Monitor::AcquireOrPush (ParkEvent * ESelf) {

  intptr_t v = _LockWord.FullWord ;

  for (;;) {

    if ((v & _LBIT) == 0) {

      const intptr_t u = CASPTR (&_LockWord, v, v|_LBIT) ;

      if (u == v) return 1 ;        // indicate acquired

      v = u ;

    } else {

      // Anticipate success ...

      ESelf->ListNext = (ParkEvent *) (v & ~_LBIT) ;

      const intptr_t u = CASPTR (&_LockWord, v, intptr_t(ESelf)|_LBIT) ;

      if (u == v) return 0 ;        // indicate pushed onto cxq

      v = u ;

    }

    // Interference - LockWord change - just retry

  }

}

// ILock and IWait are the lowest level primitive internal blocking

// synchronization functions.  The callers of IWait and ILock must have

// performed any needed state transitions beforehand.

// IWait and ILock may directly call park() without any concern for thread state.

// Note that ILock and IWait do *not* access _owner.

// _owner is a higher-level logical concept.

void Monitor::ILock (Thread * Self) {

  assert (_OnDeck != Self->_MutexEvent, "invariant") ;

  if (TryFast()) {

 Exeunt:

    assert (ILocked(), "invariant") ;

    return ;

  }

  ParkEvent * const ESelf = Self->_MutexEvent ;

  assert (_OnDeck != ESelf, "invariant") ;

  // As an optimization, spinners could conditionally try to set ONDECK to _LBIT

  // Synchronizer.cpp uses a similar optimization.

  if (TrySpin (Self)) goto Exeunt ;

  // Slow-path - the lock is contended.

  // Either Enqueue Self on cxq or acquire the outer lock.

  // LockWord encoding = (cxq,LOCKBYTE)

  ESelf->reset() ;

  OrderAccess::fence() ;

  // Optional optimization ... try barging on the inner lock

  if ((NativeMonitorFlags & 32) && CASPTR (&_OnDeck, NULL, UNS(Self)) == 0) {

    goto OnDeck_LOOP ;

  }

  if (AcquireOrPush (ESelf)) goto Exeunt ;

  // At any given time there is at most one ondeck thread.

  // ondeck implies not resident on cxq and not resident on EntryList

  // Only the OnDeck thread can try to acquire -- contended for -- the lock.

  // CONSIDER: use Self->OnDeck instead of m->OnDeck.

  // Deschedule Self so that others may run.

  while (_OnDeck != ESelf) {

    ParkCommon (ESelf, 0) ;

  }

  // Self is now in the ONDECK position and will remain so until it

  // manages to acquire the lock.

 OnDeck_LOOP:

  for (;;) {

    assert (_OnDeck == ESelf, "invariant") ;

    if (TrySpin (Self)) break ;

    // CONSIDER: if ESelf->TryPark() && TryLock() break ...

    // It's probably wise to spin only if we *actually* blocked

    // CONSIDER: check the lockbyte, if it remains set then

    // preemptively drain the cxq into the EntryList.

    // The best place and time to perform queue operations -- lock metadata --

    // is _before having acquired the outer lock, while waiting for the lock to drop.

    ParkCommon (ESelf, 0) ;

  }

  assert (_OnDeck == ESelf, "invariant") ;

  _OnDeck = NULL ;

  // Note that we current drop the inner lock (clear OnDeck) in the slow-path

  // epilog immediately after having acquired the outer lock.

  // But instead we could consider the following optimizations:

  // A. Shift or defer dropping the inner lock until the subsequent IUnlock() operation.

  //    This might avoid potential reacquisition of the inner lock in IUlock().

  // B. While still holding the inner lock, attempt to opportunistically select

  //    and unlink the next ONDECK thread from the EntryList.

  //    If successful, set ONDECK to refer to that thread, otherwise clear ONDECK.

  //    It's critical that the select-and-unlink operation run in constant-time as

  //    it executes when holding the outer lock and may artificially increase the

  //    effective length of the critical section.

  // Note that (A) and (B) are tantamount to succession by direct handoff for

  // the inner lock.

  goto Exeunt ;

}

void Monitor::IUnlock (bool RelaxAssert) {

  assert (ILocked(), "invariant") ;