hotspot/src/share/vm/runtime/mutex.cpp
changeset 25069 c937c5e883c5
parent 24424 2658d7834c6e
child 25353 806fe0d44237
equal deleted inserted replaced
25068:de4b1c96dca3 25069:c937c5e883c5
   267 // CASPTR() uses the canonical argument order that dominates in the literature.
   267 // CASPTR() uses the canonical argument order that dominates in the literature.
   268 // Our internal cmpxchg_ptr() uses a bastardized ordering to accommodate Sun .il templates.
   268 // Our internal cmpxchg_ptr() uses a bastardized ordering to accommodate Sun .il templates.
   269 
   269 
   270 #define CASPTR(a,c,s) intptr_t(Atomic::cmpxchg_ptr ((void *)(s),(void *)(a),(void *)(c)))
   270 #define CASPTR(a,c,s) intptr_t(Atomic::cmpxchg_ptr ((void *)(s),(void *)(a),(void *)(c)))
   271 #define UNS(x) (uintptr_t(x))
   271 #define UNS(x) (uintptr_t(x))
   272 #define TRACE(m) { static volatile int ctr = 0 ; int x = ++ctr ; if ((x & (x-1))==0) { ::printf ("%d:%s\n", x, #m); ::fflush(stdout); }}
   272 #define TRACE(m) { static volatile int ctr = 0; int x = ++ctr; if ((x & (x-1))==0) { ::printf ("%d:%s\n", x, #m); ::fflush(stdout); }}
   273 
   273 
   274 // Simplistic low-quality Marsaglia SHIFT-XOR RNG.
   274 // Simplistic low-quality Marsaglia SHIFT-XOR RNG.
   275 // Bijective except for the trailing mask operation.
   275 // Bijective except for the trailing mask operation.
   276 // Useful for spin loops as the compiler can't optimize it away.
   276 // Useful for spin loops as the compiler can't optimize it away.
   277 
   277 
   278 static inline jint MarsagliaXORV (jint x) {
   278 static inline jint MarsagliaXORV (jint x) {
   279   if (x == 0) x = 1|os::random() ;
   279   if (x == 0) x = 1|os::random();
   280   x ^= x << 6;
   280   x ^= x << 6;
   281   x ^= ((unsigned)x) >> 21;
   281   x ^= ((unsigned)x) >> 21;
   282   x ^= x << 7 ;
   282   x ^= x << 7;
   283   return x & 0x7FFFFFFF ;
   283   return x & 0x7FFFFFFF;
   284 }
   284 }
   285 
   285 
   286 static int Stall (int its) {
   286 static int Stall (int its) {
   287   static volatile jint rv = 1 ;
   287   static volatile jint rv = 1;
   288   volatile int OnFrame = 0 ;
   288   volatile int OnFrame = 0;
   289   jint v = rv ^ UNS(OnFrame) ;
   289   jint v = rv ^ UNS(OnFrame);
   290   while (--its >= 0) {
   290   while (--its >= 0) {
   291     v = MarsagliaXORV (v) ;
   291     v = MarsagliaXORV(v);
   292   }
   292   }
   293   // Make this impossible for the compiler to optimize away,
   293   // Make this impossible for the compiler to optimize away,
   294   // but (mostly) avoid W coherency sharing on MP systems.
   294   // but (mostly) avoid W coherency sharing on MP systems.
   295   if (v == 0x12345) rv = v ;
   295   if (v == 0x12345) rv = v;
   296   return v ;
   296   return v;
   297 }
   297 }
   298 
   298 
   299 int Monitor::TryLock () {
   299 int Monitor::TryLock() {
   300   intptr_t v = _LockWord.FullWord ;
   300   intptr_t v = _LockWord.FullWord;
   301   for (;;) {
   301   for (;;) {
   302     if ((v & _LBIT) != 0) return 0 ;
   302     if ((v & _LBIT) != 0) return 0;
   303     const intptr_t u = CASPTR (&_LockWord, v, v|_LBIT) ;
   303     const intptr_t u = CASPTR(&_LockWord, v, v|_LBIT);
   304     if (v == u) return 1 ;
   304     if (v == u) return 1;
   305     v = u ;
   305     v = u;
   306   }
   306   }
   307 }
   307 }
   308 
   308 
   309 int Monitor::TryFast () {
   309 int Monitor::TryFast() {
   310   // Optimistic fast-path form ...
   310   // Optimistic fast-path form ...
   311   // Fast-path attempt for the common uncontended case.
   311   // Fast-path attempt for the common uncontended case.
   312   // Avoid RTS->RTO $ coherence upgrade on typical SMP systems.
   312   // Avoid RTS->RTO $ coherence upgrade on typical SMP systems.
   313   intptr_t v = CASPTR (&_LockWord, 0, _LBIT) ;  // agro ...
   313   intptr_t v = CASPTR(&_LockWord, 0, _LBIT);  // agro ...
   314   if (v == 0) return 1 ;
   314   if (v == 0) return 1;
   315 
   315 
   316   for (;;) {
   316   for (;;) {
   317     if ((v & _LBIT) != 0) return 0 ;
   317     if ((v & _LBIT) != 0) return 0;
   318     const intptr_t u = CASPTR (&_LockWord, v, v|_LBIT) ;
   318     const intptr_t u = CASPTR(&_LockWord, v, v|_LBIT);
   319     if (v == u) return 1 ;
   319     if (v == u) return 1;
   320     v = u ;
   320     v = u;
   321   }
   321   }
   322 }
   322 }
   323 
   323 
   324 int Monitor::ILocked () {
   324 int Monitor::ILocked() {
   325   const intptr_t w = _LockWord.FullWord & 0xFF ;
   325   const intptr_t w = _LockWord.FullWord & 0xFF;
   326   assert (w == 0 || w == _LBIT, "invariant") ;
   326   assert(w == 0 || w == _LBIT, "invariant");
   327   return w == _LBIT ;
   327   return w == _LBIT;
   328 }
   328 }
   329 
   329 
   330 // Polite TATAS spinlock with exponential backoff - bounded spin.
   330 // Polite TATAS spinlock with exponential backoff - bounded spin.
   331 // Ideally we'd use processor cycles, time or vtime to control
   331 // Ideally we'd use processor cycles, time or vtime to control
   332 // the loop, but we currently use iterations.
   332 // the loop, but we currently use iterations.
   340 //
   340 //
   341 // Clamp spinning at approximately 1/2 of a context-switch round-trip.
   341 // Clamp spinning at approximately 1/2 of a context-switch round-trip.
   342 // See synchronizer.cpp for details and rationale.
   342 // See synchronizer.cpp for details and rationale.
   343 
   343 
   344 int Monitor::TrySpin (Thread * const Self) {
   344 int Monitor::TrySpin (Thread * const Self) {
   345   if (TryLock())    return 1 ;
   345   if (TryLock())    return 1;
   346   if (!os::is_MP()) return 0 ;
   346   if (!os::is_MP()) return 0;
   347 
   347 
   348   int Probes  = 0 ;
   348   int Probes  = 0;
   349   int Delay   = 0 ;
   349   int Delay   = 0;
   350   int Steps   = 0 ;
   350   int Steps   = 0;
   351   int SpinMax = NativeMonitorSpinLimit ;
   351   int SpinMax = NativeMonitorSpinLimit;
   352   int flgs    = NativeMonitorFlags ;
   352   int flgs    = NativeMonitorFlags;
   353   for (;;) {
   353   for (;;) {
   354     intptr_t v = _LockWord.FullWord;
   354     intptr_t v = _LockWord.FullWord;
   355     if ((v & _LBIT) == 0) {
   355     if ((v & _LBIT) == 0) {
   356       if (CASPTR (&_LockWord, v, v|_LBIT) == v) {
   356       if (CASPTR (&_LockWord, v, v|_LBIT) == v) {
   357         return 1 ;
   357         return 1;
   358       }
   358       }
   359       continue ;
   359       continue;
   360     }
   360     }
   361 
   361 
   362     if ((flgs & 8) == 0) {
   362     if ((flgs & 8) == 0) {
   363       SpinPause () ;
   363       SpinPause();
   364     }
   364     }
   365 
   365 
   366     // Periodically increase Delay -- variable Delay form
   366     // Periodically increase Delay -- variable Delay form
   367     // conceptually: delay *= 1 + 1/Exponent
   367     // conceptually: delay *= 1 + 1/Exponent
   368     ++ Probes;
   368     ++Probes;
   369     if (Probes > SpinMax) return 0 ;
   369     if (Probes > SpinMax) return 0;
   370 
   370 
   371     if ((Probes & 0x7) == 0) {
   371     if ((Probes & 0x7) == 0) {
   372       Delay = ((Delay << 1)|1) & 0x7FF ;
   372       Delay = ((Delay << 1)|1) & 0x7FF;
   373       // CONSIDER: Delay += 1 + (Delay/4); Delay &= 0x7FF ;
   373       // CONSIDER: Delay += 1 + (Delay/4); Delay &= 0x7FF ;
   374     }
   374     }
   375 
   375 
   376     if (flgs & 2) continue ;
   376     if (flgs & 2) continue;
   377 
   377 
   378     // Consider checking _owner's schedctl state, if OFFPROC abort spin.
   378     // Consider checking _owner's schedctl state, if OFFPROC abort spin.
   379     // If the owner is OFFPROC then it's unlike that the lock will be dropped
   379     // If the owner is OFFPROC then it's unlike that the lock will be dropped
   380     // in a timely fashion, which suggests that spinning would not be fruitful
   380     // in a timely fashion, which suggests that spinning would not be fruitful
   381     // or profitable.
   381     // or profitable.
   387     //   wr %g0,%asi, gethrtime, rdstick, rdtick, rdtsc, etc. ...
   387     //   wr %g0,%asi, gethrtime, rdstick, rdtick, rdtsc, etc. ...
   388     // Note that on Niagara-class systems we want to minimize STs in the
   388     // Note that on Niagara-class systems we want to minimize STs in the
   389     // spin loop.  N1 and brethren write-around the L1$ over the xbar into the L2$.
   389     // spin loop.  N1 and brethren write-around the L1$ over the xbar into the L2$.
   390     // Furthermore, they don't have a W$ like traditional SPARC processors.
   390     // Furthermore, they don't have a W$ like traditional SPARC processors.
   391     // We currently use a Marsaglia Shift-Xor RNG loop.
   391     // We currently use a Marsaglia Shift-Xor RNG loop.
   392     Steps += Delay ;
   392     Steps += Delay;
   393     if (Self != NULL) {
   393     if (Self != NULL) {
   394       jint rv = Self->rng[0] ;
   394       jint rv = Self->rng[0];
   395       for (int k = Delay ; --k >= 0; ) {
   395       for (int k = Delay; --k >= 0;) {
   396         rv = MarsagliaXORV (rv) ;
   396         rv = MarsagliaXORV(rv);
   397         if ((flgs & 4) == 0 && SafepointSynchronize::do_call_back()) return 0 ;
   397         if ((flgs & 4) == 0 && SafepointSynchronize::do_call_back()) return 0;
   398       }
   398       }
   399       Self->rng[0] = rv ;
   399       Self->rng[0] = rv;
   400     } else {
   400     } else {
   401       Stall (Delay) ;
   401       Stall(Delay);
   402     }
   402     }
   403   }
   403   }
   404 }
   404 }
   405 
   405 
   406 static int ParkCommon (ParkEvent * ev, jlong timo) {
   406 static int ParkCommon (ParkEvent * ev, jlong timo) {
   407   // Diagnostic support - periodically unwedge blocked threads
   407   // Diagnostic support - periodically unwedge blocked threads
   408   intx nmt = NativeMonitorTimeout ;
   408   intx nmt = NativeMonitorTimeout;
   409   if (nmt > 0 && (nmt < timo || timo <= 0)) {
   409   if (nmt > 0 && (nmt < timo || timo <= 0)) {
   410      timo = nmt ;
   410      timo = nmt;
   411   }
   411   }
   412   int err = OS_OK ;
   412   int err = OS_OK;
   413   if (0 == timo) {
   413   if (0 == timo) {
   414     ev->park() ;
   414     ev->park();
   415   } else {
   415   } else {
   416     err = ev->park(timo) ;
   416     err = ev->park(timo);
   417   }
   417   }
   418   return err ;
   418   return err;
   419 }
   419 }
   420 
   420 
   421 inline int Monitor::AcquireOrPush (ParkEvent * ESelf) {
   421 inline int Monitor::AcquireOrPush (ParkEvent * ESelf) {
   422   intptr_t v = _LockWord.FullWord ;
   422   intptr_t v = _LockWord.FullWord;
   423   for (;;) {
   423   for (;;) {
   424     if ((v & _LBIT) == 0) {
   424     if ((v & _LBIT) == 0) {
   425       const intptr_t u = CASPTR (&_LockWord, v, v|_LBIT) ;
   425       const intptr_t u = CASPTR(&_LockWord, v, v|_LBIT);
   426       if (u == v) return 1 ;        // indicate acquired
   426       if (u == v) return 1;        // indicate acquired
   427       v = u ;
   427       v = u;
   428     } else {
   428     } else {
   429       // Anticipate success ...
   429       // Anticipate success ...
   430       ESelf->ListNext = (ParkEvent *) (v & ~_LBIT) ;
   430       ESelf->ListNext = (ParkEvent *)(v & ~_LBIT);
   431       const intptr_t u = CASPTR (&_LockWord, v, intptr_t(ESelf)|_LBIT) ;
   431       const intptr_t u = CASPTR(&_LockWord, v, intptr_t(ESelf)|_LBIT);
   432       if (u == v) return 0 ;        // indicate pushed onto cxq
   432       if (u == v) return 0;        // indicate pushed onto cxq
   433       v = u ;
   433       v = u;
   434     }
   434     }
   435     // Interference - LockWord change - just retry
   435     // Interference - LockWord change - just retry
   436   }
   436   }
   437 }
   437 }
   438 
   438 
   442 // IWait and ILock may directly call park() without any concern for thread state.
   442 // IWait and ILock may directly call park() without any concern for thread state.
   443 // Note that ILock and IWait do *not* access _owner.
   443 // Note that ILock and IWait do *not* access _owner.
   444 // _owner is a higher-level logical concept.
   444 // _owner is a higher-level logical concept.
   445 
   445 
   446 void Monitor::ILock (Thread * Self) {
   446 void Monitor::ILock (Thread * Self) {
   447   assert (_OnDeck != Self->_MutexEvent, "invariant") ;
   447   assert(_OnDeck != Self->_MutexEvent, "invariant");
   448 
   448 
   449   if (TryFast()) {
   449   if (TryFast()) {
   450  Exeunt:
   450  Exeunt:
   451     assert (ILocked(), "invariant") ;
   451     assert(ILocked(), "invariant");
   452     return ;
   452     return;
   453   }
   453   }
   454 
   454 
   455   ParkEvent * const ESelf = Self->_MutexEvent ;
   455   ParkEvent * const ESelf = Self->_MutexEvent;
   456   assert (_OnDeck != ESelf, "invariant") ;
   456   assert(_OnDeck != ESelf, "invariant");
   457 
   457 
   458   // As an optimization, spinners could conditionally try to set ONDECK to _LBIT
   458   // As an optimization, spinners could conditionally try to set ONDECK to _LBIT
   459   // Synchronizer.cpp uses a similar optimization.
   459   // Synchronizer.cpp uses a similar optimization.
   460   if (TrySpin (Self)) goto Exeunt ;
   460   if (TrySpin(Self)) goto Exeunt;
   461 
   461 
   462   // Slow-path - the lock is contended.
   462   // Slow-path - the lock is contended.
   463   // Either Enqueue Self on cxq or acquire the outer lock.
   463   // Either Enqueue Self on cxq or acquire the outer lock.
   464   // LockWord encoding = (cxq,LOCKBYTE)
   464   // LockWord encoding = (cxq,LOCKBYTE)
   465   ESelf->reset() ;
   465   ESelf->reset();
   466   OrderAccess::fence() ;
   466   OrderAccess::fence();
   467 
   467 
   468   // Optional optimization ... try barging on the inner lock
   468   // Optional optimization ... try barging on the inner lock
   469   if ((NativeMonitorFlags & 32) && CASPTR (&_OnDeck, NULL, UNS(Self)) == 0) {
   469   if ((NativeMonitorFlags & 32) && CASPTR (&_OnDeck, NULL, UNS(Self)) == 0) {
   470     goto OnDeck_LOOP ;
   470     goto OnDeck_LOOP;
   471   }
   471   }
   472 
   472 
   473   if (AcquireOrPush (ESelf)) goto Exeunt ;
   473   if (AcquireOrPush(ESelf)) goto Exeunt;
   474 
   474 
   475   // At any given time there is at most one ondeck thread.
   475   // At any given time there is at most one ondeck thread.
   476   // ondeck implies not resident on cxq and not resident on EntryList
   476   // ondeck implies not resident on cxq and not resident on EntryList
   477   // Only the OnDeck thread can try to acquire -- contended for -- the lock.
   477   // Only the OnDeck thread can try to acquire -- contended for -- the lock.
   478   // CONSIDER: use Self->OnDeck instead of m->OnDeck.
   478   // CONSIDER: use Self->OnDeck instead of m->OnDeck.
   479   // Deschedule Self so that others may run.
   479   // Deschedule Self so that others may run.
   480   while (_OnDeck != ESelf) {
   480   while (_OnDeck != ESelf) {
   481     ParkCommon (ESelf, 0) ;
   481     ParkCommon(ESelf, 0);
   482   }
   482   }
   483 
   483 
   484   // Self is now in the ONDECK position and will remain so until it
   484   // Self is now in the ONDECK position and will remain so until it
   485   // manages to acquire the lock.
   485   // manages to acquire the lock.
   486  OnDeck_LOOP:
   486  OnDeck_LOOP:
   487   for (;;) {
   487   for (;;) {
   488     assert (_OnDeck == ESelf, "invariant") ;
   488     assert(_OnDeck == ESelf, "invariant");
   489     if (TrySpin (Self)) break ;
   489     if (TrySpin(Self)) break;
   490     // CONSIDER: if ESelf->TryPark() && TryLock() break ...
   490     // CONSIDER: if ESelf->TryPark() && TryLock() break ...
   491     // It's probably wise to spin only if we *actually* blocked
   491     // It's probably wise to spin only if we *actually* blocked
   492     // CONSIDER: check the lockbyte, if it remains set then
   492     // CONSIDER: check the lockbyte, if it remains set then
   493     // preemptively drain the cxq into the EntryList.
   493     // preemptively drain the cxq into the EntryList.
   494     // The best place and time to perform queue operations -- lock metadata --
   494     // The best place and time to perform queue operations -- lock metadata --
   495     // is _before having acquired the outer lock, while waiting for the lock to drop.
   495     // is _before having acquired the outer lock, while waiting for the lock to drop.
   496     ParkCommon (ESelf, 0) ;
   496     ParkCommon(ESelf, 0);
   497   }
   497   }
   498 
   498 
   499   assert (_OnDeck == ESelf, "invariant") ;
   499   assert(_OnDeck == ESelf, "invariant");
   500   _OnDeck = NULL ;
   500   _OnDeck = NULL;
   501 
   501 
   502   // Note that we current drop the inner lock (clear OnDeck) in the slow-path
   502   // Note that we current drop the inner lock (clear OnDeck) in the slow-path
   503   // epilogue immediately after having acquired the outer lock.
   503   // epilogue immediately after having acquired the outer lock.
   504   // But instead we could consider the following optimizations:
   504   // But instead we could consider the following optimizations:
   505   // A. Shift or defer dropping the inner lock until the subsequent IUnlock() operation.
   505   // A. Shift or defer dropping the inner lock until the subsequent IUnlock() operation.
   510   //    It's critical that the select-and-unlink operation run in constant-time as
   510   //    It's critical that the select-and-unlink operation run in constant-time as
   511   //    it executes when holding the outer lock and may artificially increase the
   511   //    it executes when holding the outer lock and may artificially increase the
   512   //    effective length of the critical section.
   512   //    effective length of the critical section.
   513   // Note that (A) and (B) are tantamount to succession by direct handoff for
   513   // Note that (A) and (B) are tantamount to succession by direct handoff for
   514   // the inner lock.
   514   // the inner lock.
   515   goto Exeunt ;
   515   goto Exeunt;
   516 }
   516 }
   517 
   517 
   518 void Monitor::IUnlock (bool RelaxAssert) {
   518 void Monitor::IUnlock (bool RelaxAssert) {
   519   assert (ILocked(), "invariant") ;
   519   assert(ILocked(), "invariant");
   520   // Conceptually we need a MEMBAR #storestore|#loadstore barrier or fence immediately
   520   // Conceptually we need a MEMBAR #storestore|#loadstore barrier or fence immediately
   521   // before the store that releases the lock.  Crucially, all the stores and loads in the
   521   // before the store that releases the lock.  Crucially, all the stores and loads in the
   522   // critical section must be globally visible before the store of 0 into the lock-word
   522   // critical section must be globally visible before the store of 0 into the lock-word
   523   // that releases the lock becomes globally visible.  That is, memory accesses in the
   523   // that releases the lock becomes globally visible.  That is, memory accesses in the
   524   // critical section should not be allowed to bypass or overtake the following ST that
   524   // critical section should not be allowed to bypass or overtake the following ST that
   530   // Note that the OrderAccess::storeload() fence that appears after unlock store
   530   // Note that the OrderAccess::storeload() fence that appears after unlock store
   531   // provides for progress conditions and succession and is _not related to exclusion
   531   // provides for progress conditions and succession and is _not related to exclusion
   532   // safety or lock release consistency.
   532   // safety or lock release consistency.
   533   OrderAccess::release_store(&_LockWord.Bytes[_LSBINDEX], 0); // drop outer lock
   533   OrderAccess::release_store(&_LockWord.Bytes[_LSBINDEX], 0); // drop outer lock
   534 
   534 
   535   OrderAccess::storeload ();
   535   OrderAccess::storeload();
   536   ParkEvent * const w = _OnDeck ;
   536   ParkEvent * const w = _OnDeck;
   537   assert (RelaxAssert || w != Thread::current()->_MutexEvent, "invariant") ;
   537   assert(RelaxAssert || w != Thread::current()->_MutexEvent, "invariant");
   538   if (w != NULL) {
   538   if (w != NULL) {
   539     // Either we have a valid ondeck thread or ondeck is transiently "locked"
   539     // Either we have a valid ondeck thread or ondeck is transiently "locked"
   540     // by some exiting thread as it arranges for succession.  The LSBit of
   540     // by some exiting thread as it arranges for succession.  The LSBit of
   541     // OnDeck allows us to discriminate two cases.  If the latter, the
   541     // OnDeck allows us to discriminate two cases.  If the latter, the
   542     // responsibility for progress and succession lies with that other thread.
   542     // responsibility for progress and succession lies with that other thread.
   547     // In that case the following unpark() is harmless and the worst that'll happen
   547     // In that case the following unpark() is harmless and the worst that'll happen
   548     // is a spurious return from a park() operation.  Critically, if "w" _is stale,
   548     // is a spurious return from a park() operation.  Critically, if "w" _is stale,
   549     // then progress is known to have occurred as that means the thread associated
   549     // then progress is known to have occurred as that means the thread associated
   550     // with "w" acquired the lock.  In that case this thread need take no further
   550     // with "w" acquired the lock.  In that case this thread need take no further
   551     // action to guarantee progress.
   551     // action to guarantee progress.
   552     if ((UNS(w) & _LBIT) == 0) w->unpark() ;
   552     if ((UNS(w) & _LBIT) == 0) w->unpark();
   553     return ;
   553     return;
   554   }
   554   }
   555 
   555 
   556   intptr_t cxq = _LockWord.FullWord ;
   556   intptr_t cxq = _LockWord.FullWord;
   557   if (((cxq & ~_LBIT)|UNS(_EntryList)) == 0) {
   557   if (((cxq & ~_LBIT)|UNS(_EntryList)) == 0) {
   558     return ;      // normal fast-path exit - cxq and EntryList both empty
   558     return;      // normal fast-path exit - cxq and EntryList both empty
   559   }
   559   }
   560   if (cxq & _LBIT) {
   560   if (cxq & _LBIT) {
   561     // Optional optimization ...
   561     // Optional optimization ...
   562     // Some other thread acquired the lock in the window since this
   562     // Some other thread acquired the lock in the window since this
   563     // thread released it.  Succession is now that thread's responsibility.
   563     // thread released it.  Succession is now that thread's responsibility.
   564     return ;
   564     return;
   565   }
   565   }
   566 
   566 
   567  Succession:
   567  Succession:
   568   // Slow-path exit - this thread must ensure succession and progress.
   568   // Slow-path exit - this thread must ensure succession and progress.
   569   // OnDeck serves as lock to protect cxq and EntryList.
   569   // OnDeck serves as lock to protect cxq and EntryList.
   573   // Consider protecting this critical section with schedctl on Solaris.
   573   // Consider protecting this critical section with schedctl on Solaris.
   574   // Unlike a normal lock, however, the exiting thread "locks" OnDeck,
   574   // Unlike a normal lock, however, the exiting thread "locks" OnDeck,
   575   // picks a successor and marks that thread as OnDeck.  That successor
   575   // picks a successor and marks that thread as OnDeck.  That successor
   576   // thread will then clear OnDeck once it eventually acquires the outer lock.
   576   // thread will then clear OnDeck once it eventually acquires the outer lock.
   577   if (CASPTR (&_OnDeck, NULL, _LBIT) != UNS(NULL)) {
   577   if (CASPTR (&_OnDeck, NULL, _LBIT) != UNS(NULL)) {
   578     return ;
   578     return;
   579   }
   579   }
   580 
   580 
   581   ParkEvent * List = _EntryList ;
   581   ParkEvent * List = _EntryList;
   582   if (List != NULL) {
   582   if (List != NULL) {
   583     // Transfer the head of the EntryList to the OnDeck position.
   583     // Transfer the head of the EntryList to the OnDeck position.
   584     // Once OnDeck, a thread stays OnDeck until it acquires the lock.
   584     // Once OnDeck, a thread stays OnDeck until it acquires the lock.
   585     // For a given lock there is at most OnDeck thread at any one instant.
   585     // For a given lock there is at most OnDeck thread at any one instant.
   586    WakeOne:
   586    WakeOne:
   587     assert (List == _EntryList, "invariant") ;
   587     assert(List == _EntryList, "invariant");
   588     ParkEvent * const w = List ;
   588     ParkEvent * const w = List;
   589     assert (RelaxAssert || w != Thread::current()->_MutexEvent, "invariant") ;
   589     assert(RelaxAssert || w != Thread::current()->_MutexEvent, "invariant");
   590     _EntryList = w->ListNext ;
   590     _EntryList = w->ListNext;
   591     // as a diagnostic measure consider setting w->_ListNext = BAD
   591     // as a diagnostic measure consider setting w->_ListNext = BAD
   592     assert (UNS(_OnDeck) == _LBIT, "invariant") ;
   592     assert(UNS(_OnDeck) == _LBIT, "invariant");
   593     _OnDeck = w ;           // pass OnDeck to w.
   593     _OnDeck = w;           // pass OnDeck to w.
   594                             // w will clear OnDeck once it acquires the outer lock
   594                             // w will clear OnDeck once it acquires the outer lock
   595 
   595 
   596     // Another optional optimization ...
   596     // Another optional optimization ...
   597     // For heavily contended locks it's not uncommon that some other
   597     // For heavily contended locks it's not uncommon that some other
   598     // thread acquired the lock while this thread was arranging succession.
   598     // thread acquired the lock while this thread was arranging succession.
   599     // Try to defer the unpark() operation - Delegate the responsibility
   599     // Try to defer the unpark() operation - Delegate the responsibility
   600     // for unpark()ing the OnDeck thread to the current or subsequent owners
   600     // for unpark()ing the OnDeck thread to the current or subsequent owners
   601     // That is, the new owner is responsible for unparking the OnDeck thread.
   601     // That is, the new owner is responsible for unparking the OnDeck thread.
   602     OrderAccess::storeload() ;
   602     OrderAccess::storeload();
   603     cxq = _LockWord.FullWord ;
   603     cxq = _LockWord.FullWord;
   604     if (cxq & _LBIT) return ;
   604     if (cxq & _LBIT) return;
   605 
   605 
   606     w->unpark() ;
   606     w->unpark();
   607     return ;
   607     return;
   608   }
   608   }
   609 
   609 
   610   cxq = _LockWord.FullWord ;
   610   cxq = _LockWord.FullWord;
   611   if ((cxq & ~_LBIT) != 0) {
   611   if ((cxq & ~_LBIT) != 0) {
   612     // The EntryList is empty but the cxq is populated.
   612     // The EntryList is empty but the cxq is populated.
   613     // drain RATs from cxq into EntryList
   613     // drain RATs from cxq into EntryList
   614     // Detach RATs segment with CAS and then merge into EntryList
   614     // Detach RATs segment with CAS and then merge into EntryList
   615     for (;;) {
   615     for (;;) {
   616       // optional optimization - if locked, the owner is responsible for succession
   616       // optional optimization - if locked, the owner is responsible for succession
   617       if (cxq & _LBIT) goto Punt ;
   617       if (cxq & _LBIT) goto Punt;
   618       const intptr_t vfy = CASPTR (&_LockWord, cxq, cxq & _LBIT) ;
   618       const intptr_t vfy = CASPTR(&_LockWord, cxq, cxq & _LBIT);
   619       if (vfy == cxq) break ;
   619       if (vfy == cxq) break;
   620       cxq = vfy ;
   620       cxq = vfy;
   621       // Interference - LockWord changed - Just retry
   621       // Interference - LockWord changed - Just retry
   622       // We can see concurrent interference from contending threads
   622       // We can see concurrent interference from contending threads
   623       // pushing themselves onto the cxq or from lock-unlock operations.
   623       // pushing themselves onto the cxq or from lock-unlock operations.
   624       // From the perspective of this thread, EntryList is stable and
   624       // From the perspective of this thread, EntryList is stable and
   625       // the cxq is prepend-only -- the head is volatile but the interior
   625       // the cxq is prepend-only -- the head is volatile but the interior
   637 
   637 
   638     // We don't currently reorder the cxq segment as we move it onto
   638     // We don't currently reorder the cxq segment as we move it onto
   639     // the EntryList, but it might make sense to reverse the order
   639     // the EntryList, but it might make sense to reverse the order
   640     // or perhaps sort by thread priority.  See the comments in
   640     // or perhaps sort by thread priority.  See the comments in
   641     // synchronizer.cpp objectMonitor::exit().
   641     // synchronizer.cpp objectMonitor::exit().
   642     assert (_EntryList == NULL, "invariant") ;
   642     assert(_EntryList == NULL, "invariant");
   643     _EntryList = List = (ParkEvent *)(cxq & ~_LBIT) ;
   643     _EntryList = List = (ParkEvent *)(cxq & ~_LBIT);
   644     assert (List != NULL, "invariant") ;
   644     assert(List != NULL, "invariant");
   645     goto WakeOne ;
   645     goto WakeOne;
   646   }
   646   }
   647 
   647 
   648   // cxq|EntryList is empty.
   648   // cxq|EntryList is empty.
   649   // w == NULL implies that cxq|EntryList == NULL in the past.
   649   // w == NULL implies that cxq|EntryList == NULL in the past.
   650   // Possible race - rare inopportune interleaving.
   650   // Possible race - rare inopportune interleaving.
   651   // A thread could have added itself to cxq since this thread previously checked.
   651   // A thread could have added itself to cxq since this thread previously checked.
   652   // Detect and recover by refetching cxq.
   652   // Detect and recover by refetching cxq.
   653  Punt:
   653  Punt:
   654   assert (UNS(_OnDeck) == _LBIT, "invariant") ;
   654   assert(UNS(_OnDeck) == _LBIT, "invariant");
   655   _OnDeck = NULL ;            // Release inner lock.
   655   _OnDeck = NULL;            // Release inner lock.
   656   OrderAccess::storeload();   // Dekker duality - pivot point
   656   OrderAccess::storeload();   // Dekker duality - pivot point
   657 
   657 
   658   // Resample LockWord/cxq to recover from possible race.
   658   // Resample LockWord/cxq to recover from possible race.
   659   // For instance, while this thread T1 held OnDeck, some other thread T2 might
   659   // For instance, while this thread T1 held OnDeck, some other thread T2 might
   660   // acquire the outer lock.  Another thread T3 might try to acquire the outer
   660   // acquire the outer lock.  Another thread T3 might try to acquire the outer
   663   // T1 is and remains responsible for ensuring succession of T3.
   663   // T1 is and remains responsible for ensuring succession of T3.
   664   //
   664   //
   665   // Note that we don't need to recheck EntryList, just cxq.
   665   // Note that we don't need to recheck EntryList, just cxq.
   666   // If threads moved onto EntryList since we dropped OnDeck
   666   // If threads moved onto EntryList since we dropped OnDeck
   667   // that implies some other thread forced succession.
   667   // that implies some other thread forced succession.
   668   cxq = _LockWord.FullWord ;
   668   cxq = _LockWord.FullWord;
   669   if ((cxq & ~_LBIT) != 0 && (cxq & _LBIT) == 0) {
   669   if ((cxq & ~_LBIT) != 0 && (cxq & _LBIT) == 0) {
   670     goto Succession ;         // potential race -- re-run succession
   670     goto Succession;         // potential race -- re-run succession
   671   }
   671   }
   672   return ;
   672   return;
   673 }
   673 }
   674 
   674 
   675 bool Monitor::notify() {
   675 bool Monitor::notify() {
   676   assert (_owner == Thread::current(), "invariant") ;
   676   assert(_owner == Thread::current(), "invariant");
   677   assert (ILocked(), "invariant") ;
   677   assert(ILocked(), "invariant");
   678   if (_WaitSet == NULL) return true ;
   678   if (_WaitSet == NULL) return true;
   679   NotifyCount ++ ;
   679   NotifyCount++;
   680 
   680 
   681   // Transfer one thread from the WaitSet to the EntryList or cxq.
   681   // Transfer one thread from the WaitSet to the EntryList or cxq.
   682   // Currently we just unlink the head of the WaitSet and prepend to the cxq.
   682   // Currently we just unlink the head of the WaitSet and prepend to the cxq.
   683   // And of course we could just unlink it and unpark it, too, but
   683   // And of course we could just unlink it and unpark it, too, but
   684   // in that case it'd likely impale itself on the reentry.
   684   // in that case it'd likely impale itself on the reentry.
   685   Thread::muxAcquire (_WaitLock, "notify:WaitLock") ;
   685   Thread::muxAcquire(_WaitLock, "notify:WaitLock");
   686   ParkEvent * nfy = _WaitSet ;
   686   ParkEvent * nfy = _WaitSet;
   687   if (nfy != NULL) {                  // DCL idiom
   687   if (nfy != NULL) {                  // DCL idiom
   688     _WaitSet = nfy->ListNext ;
   688     _WaitSet = nfy->ListNext;
   689     assert (nfy->Notified == 0, "invariant") ;
   689     assert(nfy->Notified == 0, "invariant");
   690     // push nfy onto the cxq
   690     // push nfy onto the cxq
   691     for (;;) {
   691     for (;;) {
   692       const intptr_t v = _LockWord.FullWord ;
   692       const intptr_t v = _LockWord.FullWord;
   693       assert ((v & 0xFF) == _LBIT, "invariant") ;
   693       assert((v & 0xFF) == _LBIT, "invariant");
   694       nfy->ListNext = (ParkEvent *)(v & ~_LBIT);
   694       nfy->ListNext = (ParkEvent *)(v & ~_LBIT);
   695       if (CASPTR (&_LockWord, v, UNS(nfy)|_LBIT) == v) break;
   695       if (CASPTR (&_LockWord, v, UNS(nfy)|_LBIT) == v) break;
   696       // interference - _LockWord changed -- just retry
   696       // interference - _LockWord changed -- just retry
   697     }
   697     }
   698     // Note that setting Notified before pushing nfy onto the cxq is
   698     // Note that setting Notified before pushing nfy onto the cxq is
   699     // also legal and safe, but the safety properties are much more
   699     // also legal and safe, but the safety properties are much more
   700     // subtle, so for the sake of code stewardship ...
   700     // subtle, so for the sake of code stewardship ...
   701     OrderAccess::fence() ;
   701     OrderAccess::fence();
   702     nfy->Notified = 1;
   702     nfy->Notified = 1;
   703   }
   703   }
   704   Thread::muxRelease (_WaitLock) ;
   704   Thread::muxRelease(_WaitLock);
   705   if (nfy != NULL && (NativeMonitorFlags & 16)) {
   705   if (nfy != NULL && (NativeMonitorFlags & 16)) {
   706     // Experimental code ... light up the wakee in the hope that this thread (the owner)
   706     // Experimental code ... light up the wakee in the hope that this thread (the owner)
   707     // will drop the lock just about the time the wakee comes ONPROC.
   707     // will drop the lock just about the time the wakee comes ONPROC.
   708     nfy->unpark() ;
   708     nfy->unpark();
   709   }
   709   }
   710   assert (ILocked(), "invariant") ;
   710   assert(ILocked(), "invariant");
   711   return true ;
   711   return true;
   712 }
   712 }
   713 
   713 
   714 // Currently notifyAll() transfers the waiters one-at-a-time from the waitset
   714 // Currently notifyAll() transfers the waiters one-at-a-time from the waitset
   715 // to the cxq.  This could be done more efficiently with a single bulk en-mass transfer,
   715 // to the cxq.  This could be done more efficiently with a single bulk en-mass transfer,
   716 // but in practice notifyAll() for large #s of threads is rare and not time-critical.
   716 // but in practice notifyAll() for large #s of threads is rare and not time-critical.
   717 // Beware too, that we invert the order of the waiters.  Lets say that the
   717 // Beware too, that we invert the order of the waiters.  Lets say that the
   718 // waitset is "ABCD" and the cxq is "XYZ".  After a notifyAll() the waitset
   718 // waitset is "ABCD" and the cxq is "XYZ".  After a notifyAll() the waitset
   719 // will be empty and the cxq will be "DCBAXYZ".  This is benign, of course.
   719 // will be empty and the cxq will be "DCBAXYZ".  This is benign, of course.
   720 
   720 
   721 bool Monitor::notify_all() {
   721 bool Monitor::notify_all() {
   722   assert (_owner == Thread::current(), "invariant") ;
   722   assert(_owner == Thread::current(), "invariant");
   723   assert (ILocked(), "invariant") ;
   723   assert(ILocked(), "invariant");
   724   while (_WaitSet != NULL) notify() ;
   724   while (_WaitSet != NULL) notify();
   725   return true ;
   725   return true;
   726 }
   726 }
   727 
   727 
   728 int Monitor::IWait (Thread * Self, jlong timo) {
   728 int Monitor::IWait (Thread * Self, jlong timo) {
   729   assert (ILocked(), "invariant") ;
   729   assert(ILocked(), "invariant");
   730 
   730 
   731   // Phases:
   731   // Phases:
   732   // 1. Enqueue Self on WaitSet - currently prepend
   732   // 1. Enqueue Self on WaitSet - currently prepend
   733   // 2. unlock - drop the outer lock
   733   // 2. unlock - drop the outer lock
   734   // 3. wait for either notification or timeout
   734   // 3. wait for either notification or timeout
   735   // 4. lock - reentry - reacquire the outer lock
   735   // 4. lock - reentry - reacquire the outer lock
   736 
   736 
   737   ParkEvent * const ESelf = Self->_MutexEvent ;
   737   ParkEvent * const ESelf = Self->_MutexEvent;
   738   ESelf->Notified = 0 ;
   738   ESelf->Notified = 0;
   739   ESelf->reset() ;
   739   ESelf->reset();
   740   OrderAccess::fence() ;
   740   OrderAccess::fence();
   741 
   741 
   742   // Add Self to WaitSet
   742   // Add Self to WaitSet
   743   // Ideally only the holder of the outer lock would manipulate the WaitSet -
   743   // Ideally only the holder of the outer lock would manipulate the WaitSet -
   744   // That is, the outer lock would implicitly protect the WaitSet.
   744   // That is, the outer lock would implicitly protect the WaitSet.
   745   // But if a thread in wait() encounters a timeout it will need to dequeue itself
   745   // But if a thread in wait() encounters a timeout it will need to dequeue itself
   764   // We could also deconstruct the ParkEvent into a "pure" event and add a
   764   // We could also deconstruct the ParkEvent into a "pure" event and add a
   765   // new immortal/TSM "ListElement" class that referred to ParkEvents.
   765   // new immortal/TSM "ListElement" class that referred to ParkEvents.
   766   // In that case we could have one ListElement on the WaitSet and another
   766   // In that case we could have one ListElement on the WaitSet and another
   767   // on the EntryList, with both referring to the same pure Event.
   767   // on the EntryList, with both referring to the same pure Event.
   768 
   768 
   769   Thread::muxAcquire (_WaitLock, "wait:WaitLock:Add") ;
   769   Thread::muxAcquire(_WaitLock, "wait:WaitLock:Add");
   770   ESelf->ListNext = _WaitSet ;
   770   ESelf->ListNext = _WaitSet;
   771   _WaitSet = ESelf ;
   771   _WaitSet = ESelf;
   772   Thread::muxRelease (_WaitLock) ;
   772   Thread::muxRelease(_WaitLock);
   773 
   773 
   774   // Release the outer lock
   774   // Release the outer lock
   775   // We call IUnlock (RelaxAssert=true) as a thread T1 might
   775   // We call IUnlock (RelaxAssert=true) as a thread T1 might
   776   // enqueue itself on the WaitSet, call IUnlock(), drop the lock,
   776   // enqueue itself on the WaitSet, call IUnlock(), drop the lock,
   777   // and then stall before it can attempt to wake a successor.
   777   // and then stall before it can attempt to wake a successor.
   779   // T1 from the WaitSet to the cxq.  T2 then drops the lock.  T1 resumes,
   779   // T1 from the WaitSet to the cxq.  T2 then drops the lock.  T1 resumes,
   780   // and then finds *itself* on the cxq.  During the course of a normal
   780   // and then finds *itself* on the cxq.  During the course of a normal
   781   // IUnlock() call a thread should _never find itself on the EntryList
   781   // IUnlock() call a thread should _never find itself on the EntryList
   782   // or cxq, but in the case of wait() it's possible.
   782   // or cxq, but in the case of wait() it's possible.
   783   // See synchronizer.cpp objectMonitor::wait().
   783   // See synchronizer.cpp objectMonitor::wait().
   784   IUnlock (true) ;
   784   IUnlock(true);
   785 
   785 
   786   // Wait for either notification or timeout
   786   // Wait for either notification or timeout
   787   // Beware that in some circumstances we might propagate
   787   // Beware that in some circumstances we might propagate
   788   // spurious wakeups back to the caller.
   788   // spurious wakeups back to the caller.
   789 
   789 
   790   for (;;) {
   790   for (;;) {
   791     if (ESelf->Notified) break ;
   791     if (ESelf->Notified) break;
   792     int err = ParkCommon (ESelf, timo) ;
   792     int err = ParkCommon(ESelf, timo);
   793     if (err == OS_TIMEOUT || (NativeMonitorFlags & 1)) break ;
   793     if (err == OS_TIMEOUT || (NativeMonitorFlags & 1)) break;
   794   }
   794   }
   795 
   795 
   796   // Prepare for reentry - if necessary, remove ESelf from WaitSet
   796   // Prepare for reentry - if necessary, remove ESelf from WaitSet
   797   // ESelf can be:
   797   // ESelf can be:
   798   // 1. Still on the WaitSet.  This can happen if we exited the loop by timeout.
   798   // 1. Still on the WaitSet.  This can happen if we exited the loop by timeout.
   799   // 2. On the cxq or EntryList
   799   // 2. On the cxq or EntryList
   800   // 3. Not resident on cxq, EntryList or WaitSet, but in the OnDeck position.
   800   // 3. Not resident on cxq, EntryList or WaitSet, but in the OnDeck position.
   801 
   801 
   802   OrderAccess::fence() ;
   802   OrderAccess::fence();
   803   int WasOnWaitSet = 0 ;
   803   int WasOnWaitSet = 0;
   804   if (ESelf->Notified == 0) {
   804   if (ESelf->Notified == 0) {
   805     Thread::muxAcquire (_WaitLock, "wait:WaitLock:remove") ;
   805     Thread::muxAcquire(_WaitLock, "wait:WaitLock:remove");
   806     if (ESelf->Notified == 0) {     // DCL idiom
   806     if (ESelf->Notified == 0) {     // DCL idiom
   807       assert (_OnDeck != ESelf, "invariant") ;   // can't be both OnDeck and on WaitSet
   807       assert(_OnDeck != ESelf, "invariant");   // can't be both OnDeck and on WaitSet
   808       // ESelf is resident on the WaitSet -- unlink it.
   808       // ESelf is resident on the WaitSet -- unlink it.
   809       // A doubly-linked list would be better here so we can unlink in constant-time.
   809       // A doubly-linked list would be better here so we can unlink in constant-time.
   810       // We have to unlink before we potentially recontend as ESelf might otherwise
   810       // We have to unlink before we potentially recontend as ESelf might otherwise
   811       // end up on the cxq|EntryList -- it can't be on two lists at once.
   811       // end up on the cxq|EntryList -- it can't be on two lists at once.
   812       ParkEvent * p = _WaitSet ;
   812       ParkEvent * p = _WaitSet;
   813       ParkEvent * q = NULL ;            // classic q chases p
   813       ParkEvent * q = NULL;            // classic q chases p
   814       while (p != NULL && p != ESelf) {
   814       while (p != NULL && p != ESelf) {
   815         q = p ;
   815         q = p;
   816         p = p->ListNext ;
   816         p = p->ListNext;
   817       }
   817       }
   818       assert (p == ESelf, "invariant") ;
   818       assert(p == ESelf, "invariant");
   819       if (p == _WaitSet) {      // found at head
   819       if (p == _WaitSet) {      // found at head
   820         assert (q == NULL, "invariant") ;
   820         assert(q == NULL, "invariant");
   821         _WaitSet = p->ListNext ;
   821         _WaitSet = p->ListNext;
   822       } else {                  // found in interior
   822       } else {                  // found in interior
   823         assert (q->ListNext == p, "invariant") ;
   823         assert(q->ListNext == p, "invariant");
   824         q->ListNext = p->ListNext ;
   824         q->ListNext = p->ListNext;
   825       }
   825       }
   826       WasOnWaitSet = 1 ;        // We were *not* notified but instead encountered timeout
   826       WasOnWaitSet = 1;        // We were *not* notified but instead encountered timeout
   827     }
   827     }
   828     Thread::muxRelease (_WaitLock) ;
   828     Thread::muxRelease(_WaitLock);
   829   }
   829   }
   830 
   830 
   831   // Reentry phase - reacquire the lock
   831   // Reentry phase - reacquire the lock
   832   if (WasOnWaitSet) {
   832   if (WasOnWaitSet) {
   833     // ESelf was previously on the WaitSet but we just unlinked it above
   833     // ESelf was previously on the WaitSet but we just unlinked it above
   834     // because of a timeout.  ESelf is not resident on any list and is not OnDeck
   834     // because of a timeout.  ESelf is not resident on any list and is not OnDeck
   835     assert (_OnDeck != ESelf, "invariant") ;
   835     assert(_OnDeck != ESelf, "invariant");
   836     ILock (Self) ;
   836     ILock(Self);
   837   } else {
   837   } else {
   838     // A prior notify() operation moved ESelf from the WaitSet to the cxq.
   838     // A prior notify() operation moved ESelf from the WaitSet to the cxq.
   839     // ESelf is now on the cxq, EntryList or at the OnDeck position.
   839     // ESelf is now on the cxq, EntryList or at the OnDeck position.
   840     // The following fragment is extracted from Monitor::ILock()
   840     // The following fragment is extracted from Monitor::ILock()
   841     for (;;) {
   841     for (;;) {
   842       if (_OnDeck == ESelf && TrySpin(Self)) break ;
   842       if (_OnDeck == ESelf && TrySpin(Self)) break;
   843       ParkCommon (ESelf, 0) ;
   843       ParkCommon(ESelf, 0);
   844     }
   844     }
   845     assert (_OnDeck == ESelf, "invariant") ;
   845     assert(_OnDeck == ESelf, "invariant");
   846     _OnDeck = NULL ;
   846     _OnDeck = NULL;
   847   }
   847   }
   848 
   848 
   849   assert (ILocked(), "invariant") ;
   849   assert(ILocked(), "invariant");
   850   return WasOnWaitSet != 0 ;        // return true IFF timeout
   850   return WasOnWaitSet != 0;        // return true IFF timeout
   851 }
   851 }
   852 
   852 
   853 
   853 
   854 // ON THE VMTHREAD SNEAKING PAST HELD LOCKS:
   854 // ON THE VMTHREAD SNEAKING PAST HELD LOCKS:
   855 // In particular, there are certain types of global lock that may be held
   855 // In particular, there are certain types of global lock that may be held
   894     Self->clear_unhandled_oops();
   894     Self->clear_unhandled_oops();
   895   }
   895   }
   896 #endif // CHECK_UNHANDLED_OOPS
   896 #endif // CHECK_UNHANDLED_OOPS
   897 
   897 
   898   debug_only(check_prelock_state(Self));
   898   debug_only(check_prelock_state(Self));
   899   assert (_owner != Self              , "invariant") ;
   899   assert(_owner != Self              , "invariant");
   900   assert (_OnDeck != Self->_MutexEvent, "invariant") ;
   900   assert(_OnDeck != Self->_MutexEvent, "invariant");
   901 
   901 
   902   if (TryFast()) {
   902   if (TryFast()) {
   903  Exeunt:
   903  Exeunt:
   904     assert (ILocked(), "invariant") ;
   904     assert(ILocked(), "invariant");
   905     assert (owner() == NULL, "invariant");
   905     assert(owner() == NULL, "invariant");
   906     set_owner (Self);
   906     set_owner(Self);
   907     return ;
   907     return;
   908   }
   908   }
   909 
   909 
   910   // The lock is contended ...
   910   // The lock is contended ...
   911 
   911 
   912   bool can_sneak = Self->is_VM_thread() && SafepointSynchronize::is_at_safepoint();
   912   bool can_sneak = Self->is_VM_thread() && SafepointSynchronize::is_at_safepoint();
   914     // a java thread has locked the lock but has not entered the
   914     // a java thread has locked the lock but has not entered the
   915     // critical region -- let's just pretend we've locked the lock
   915     // critical region -- let's just pretend we've locked the lock
   916     // and go on.  we note this with _snuck so we can also
   916     // and go on.  we note this with _snuck so we can also
   917     // pretend to unlock when the time comes.
   917     // pretend to unlock when the time comes.
   918     _snuck = true;
   918     _snuck = true;
   919     goto Exeunt ;
   919     goto Exeunt;
   920   }
   920   }
   921 
   921 
   922   // Try a brief spin to avoid passing thru thread state transition ...
   922   // Try a brief spin to avoid passing thru thread state transition ...
   923   if (TrySpin (Self)) goto Exeunt ;
   923   if (TrySpin(Self)) goto Exeunt;
   924 
   924 
   925   check_block_state(Self);
   925   check_block_state(Self);
   926   if (Self->is_Java_thread()) {
   926   if (Self->is_Java_thread()) {
   927     // Horrible dictu - we suffer through a state transition
   927     // Horrible dictu - we suffer through a state transition
   928     assert(rank() > Mutex::special, "Potential deadlock with special or lesser rank mutex");
   928     assert(rank() > Mutex::special, "Potential deadlock with special or lesser rank mutex");
   929     ThreadBlockInVM tbivm ((JavaThread *) Self) ;
   929     ThreadBlockInVM tbivm((JavaThread *) Self);
   930     ILock (Self) ;
   930     ILock(Self);
   931   } else {
   931   } else {
   932     // Mirabile dictu
   932     // Mirabile dictu
   933     ILock (Self) ;
   933     ILock(Self);
   934   }
   934   }
   935   goto Exeunt ;
   935   goto Exeunt;
   936 }
   936 }
   937 
   937 
   938 void Monitor::lock() {
   938 void Monitor::lock() {
   939   this->lock(Thread::current());
   939   this->lock(Thread::current());
   940 }
   940 }
   943 // Should ONLY be used by safepoint code and other code
   943 // Should ONLY be used by safepoint code and other code
   944 // that is guaranteed not to block while running inside the VM. If this is called with
   944 // that is guaranteed not to block while running inside the VM. If this is called with
   945 // thread state set to be in VM, the safepoint synchronization code will deadlock!
   945 // thread state set to be in VM, the safepoint synchronization code will deadlock!
   946 
   946 
   947 void Monitor::lock_without_safepoint_check (Thread * Self) {
   947 void Monitor::lock_without_safepoint_check (Thread * Self) {
   948   assert (_owner != Self, "invariant") ;
   948   assert(_owner != Self, "invariant");
   949   ILock (Self) ;
   949   ILock(Self);
   950   assert (_owner == NULL, "invariant");
   950   assert(_owner == NULL, "invariant");
   951   set_owner (Self);
   951   set_owner(Self);
   952 }
   952 }
   953 
   953 
   954 void Monitor::lock_without_safepoint_check () {
   954 void Monitor::lock_without_safepoint_check() {
   955   lock_without_safepoint_check (Thread::current()) ;
   955   lock_without_safepoint_check(Thread::current());
   956 }
   956 }
   957 
   957 
   958 
   958 
   959 // Returns true if thread succeeds in grabbing the lock, otherwise false.
   959 // Returns true if thread succeeds in grabbing the lock, otherwise false.
   960 
   960 
   974     return true;
   974     return true;
   975   }
   975   }
   976 
   976 
   977   if (TryLock()) {
   977   if (TryLock()) {
   978     // We got the lock
   978     // We got the lock
   979     assert (_owner == NULL, "invariant");
   979     assert(_owner == NULL, "invariant");
   980     set_owner (Self);
   980     set_owner(Self);
   981     return true;
   981     return true;
   982   }
   982   }
   983   return false;
   983   return false;
   984 }
   984 }
   985 
   985 
   986 void Monitor::unlock() {
   986 void Monitor::unlock() {
   987   assert (_owner  == Thread::current(), "invariant") ;
   987   assert(_owner  == Thread::current(), "invariant");
   988   assert (_OnDeck != Thread::current()->_MutexEvent , "invariant") ;
   988   assert(_OnDeck != Thread::current()->_MutexEvent , "invariant");
   989   set_owner (NULL) ;
   989   set_owner(NULL);
   990   if (_snuck) {
   990   if (_snuck) {
   991     assert(SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread(), "sneak");
   991     assert(SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread(), "sneak");
   992     _snuck = false;
   992     _snuck = false;
   993     return ;
   993     return;
   994   }
   994   }
   995   IUnlock (false) ;
   995   IUnlock(false);
   996 }
   996 }
   997 
   997 
   998 // Yet another degenerate version of Monitor::lock() or lock_without_safepoint_check()
   998 // Yet another degenerate version of Monitor::lock() or lock_without_safepoint_check()
   999 // jvm_raw_lock() and _unlock() can be called by non-Java threads via JVM_RawMonitorEnter.
   999 // jvm_raw_lock() and _unlock() can be called by non-Java threads via JVM_RawMonitorEnter.
  1000 //
  1000 //
  1018 void Monitor::jvm_raw_lock() {
  1018 void Monitor::jvm_raw_lock() {
  1019   assert(rank() == native, "invariant");
  1019   assert(rank() == native, "invariant");
  1020 
  1020 
  1021   if (TryLock()) {
  1021   if (TryLock()) {
  1022  Exeunt:
  1022  Exeunt:
  1023     assert (ILocked(), "invariant") ;
  1023     assert(ILocked(), "invariant");
  1024     assert (_owner == NULL, "invariant");
  1024     assert(_owner == NULL, "invariant");
  1025     // This can potentially be called by non-java Threads. Thus, the ThreadLocalStorage
  1025     // This can potentially be called by non-java Threads. Thus, the ThreadLocalStorage
  1026     // might return NULL. Don't call set_owner since it will break on an NULL owner
  1026     // might return NULL. Don't call set_owner since it will break on an NULL owner
  1027     // Consider installing a non-null "ANON" distinguished value instead of just NULL.
  1027     // Consider installing a non-null "ANON" distinguished value instead of just NULL.
  1028     _owner = ThreadLocalStorage::thread();
  1028     _owner = ThreadLocalStorage::thread();
  1029     return ;
  1029     return;
  1030   }
  1030   }
  1031 
  1031 
  1032   if (TrySpin(NULL)) goto Exeunt ;
  1032   if (TrySpin(NULL)) goto Exeunt;
  1033 
  1033 
  1034   // slow-path - apparent contention
  1034   // slow-path - apparent contention
  1035   // Allocate a ParkEvent for transient use.
  1035   // Allocate a ParkEvent for transient use.
  1036   // The ParkEvent remains associated with this thread until
  1036   // The ParkEvent remains associated with this thread until
  1037   // the time the thread manages to acquire the lock.
  1037   // the time the thread manages to acquire the lock.
  1038   ParkEvent * const ESelf = ParkEvent::Allocate(NULL) ;
  1038   ParkEvent * const ESelf = ParkEvent::Allocate(NULL);
  1039   ESelf->reset() ;
  1039   ESelf->reset();
  1040   OrderAccess::storeload() ;
  1040   OrderAccess::storeload();
  1041 
  1041 
  1042   // Either Enqueue Self on cxq or acquire the outer lock.
  1042   // Either Enqueue Self on cxq or acquire the outer lock.
  1043   if (AcquireOrPush (ESelf)) {
  1043   if (AcquireOrPush (ESelf)) {
  1044     ParkEvent::Release (ESelf) ;      // surrender the ParkEvent
  1044     ParkEvent::Release(ESelf);      // surrender the ParkEvent
  1045     goto Exeunt ;
  1045     goto Exeunt;
  1046   }
  1046   }
  1047 
  1047 
  1048   // At any given time there is at most one ondeck thread.
  1048   // At any given time there is at most one ondeck thread.
  1049   // ondeck implies not resident on cxq and not resident on EntryList
  1049   // ondeck implies not resident on cxq and not resident on EntryList
  1050   // Only the OnDeck thread can try to acquire -- contended for -- the lock.
  1050   // Only the OnDeck thread can try to acquire -- contended for -- the lock.
  1051   // CONSIDER: use Self->OnDeck instead of m->OnDeck.
  1051   // CONSIDER: use Self->OnDeck instead of m->OnDeck.
  1052   for (;;) {
  1052   for (;;) {
  1053     if (_OnDeck == ESelf && TrySpin(NULL)) break ;
  1053     if (_OnDeck == ESelf && TrySpin(NULL)) break;
  1054     ParkCommon (ESelf, 0) ;
  1054     ParkCommon(ESelf, 0);
  1055   }
  1055   }
  1056 
  1056 
  1057   assert (_OnDeck == ESelf, "invariant") ;
  1057   assert(_OnDeck == ESelf, "invariant");
  1058   _OnDeck = NULL ;
  1058   _OnDeck = NULL;
  1059   ParkEvent::Release (ESelf) ;      // surrender the ParkEvent
  1059   ParkEvent::Release(ESelf);      // surrender the ParkEvent
  1060   goto Exeunt ;
  1060   goto Exeunt;
  1061 }
  1061 }
  1062 
  1062 
  1063 void Monitor::jvm_raw_unlock() {
  1063 void Monitor::jvm_raw_unlock() {
  1064   // Nearly the same as Monitor::unlock() ...
  1064   // Nearly the same as Monitor::unlock() ...
  1065   // directly set _owner instead of using set_owner(null)
  1065   // directly set _owner instead of using set_owner(null)
  1066   _owner = NULL ;
  1066   _owner = NULL;
  1067   if (_snuck) {         // ???
  1067   if (_snuck) {         // ???
  1068     assert(SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread(), "sneak");
  1068     assert(SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread(), "sneak");
  1069     _snuck = false;
  1069     _snuck = false;
  1070     return ;
  1070     return;
  1071   }
  1071   }
  1072   IUnlock(false) ;
  1072   IUnlock(false);
  1073 }
  1073 }
  1074 
  1074 
  1075 bool Monitor::wait(bool no_safepoint_check, long timeout, bool as_suspend_equivalent) {
  1075 bool Monitor::wait(bool no_safepoint_check, long timeout, bool as_suspend_equivalent) {
  1076   Thread * const Self = Thread::current() ;
  1076   Thread * const Self = Thread::current();
  1077   assert (_owner == Self, "invariant") ;
  1077   assert(_owner == Self, "invariant");
  1078   assert (ILocked(), "invariant") ;
  1078   assert(ILocked(), "invariant");
  1079 
  1079 
  1080   // as_suspend_equivalent logically implies !no_safepoint_check
  1080   // as_suspend_equivalent logically implies !no_safepoint_check
  1081   guarantee (!as_suspend_equivalent || !no_safepoint_check, "invariant") ;
  1081   guarantee(!as_suspend_equivalent || !no_safepoint_check, "invariant");
  1082   // !no_safepoint_check logically implies java_thread
  1082   // !no_safepoint_check logically implies java_thread
  1083   guarantee (no_safepoint_check || Self->is_Java_thread(), "invariant") ;
  1083   guarantee(no_safepoint_check || Self->is_Java_thread(), "invariant");
  1084 
  1084 
  1085   #ifdef ASSERT
  1085   #ifdef ASSERT
  1086     Monitor * least = get_least_ranked_lock_besides_this(Self->owned_locks());
  1086     Monitor * least = get_least_ranked_lock_besides_this(Self->owned_locks());
  1087     assert(least != this, "Specification of get_least_... call above");
  1087     assert(least != this, "Specification of get_least_... call above");
  1088     if (least != NULL && least->rank() <= special) {
  1088     if (least != NULL && least->rank() <= special) {
  1091                  name(), rank(), least->name(), least->rank());
  1091                  name(), rank(), least->name(), least->rank());
  1092       assert(false, "Shouldn't block(wait) while holding a lock of rank special");
  1092       assert(false, "Shouldn't block(wait) while holding a lock of rank special");
  1093     }
  1093     }
  1094   #endif // ASSERT
  1094   #endif // ASSERT
  1095 
  1095 
  1096   int wait_status ;
  1096   int wait_status;
  1097   // conceptually set the owner to NULL in anticipation of
  1097   // conceptually set the owner to NULL in anticipation of
  1098   // abdicating the lock in wait
  1098   // abdicating the lock in wait
  1099   set_owner(NULL);
  1099   set_owner(NULL);
  1100   if (no_safepoint_check) {
  1100   if (no_safepoint_check) {
  1101     wait_status = IWait (Self, timeout) ;
  1101     wait_status = IWait(Self, timeout);
  1102   } else {
  1102   } else {
  1103     assert (Self->is_Java_thread(), "invariant") ;
  1103     assert(Self->is_Java_thread(), "invariant");
  1104     JavaThread *jt = (JavaThread *)Self;
  1104     JavaThread *jt = (JavaThread *)Self;
  1105 
  1105 
  1106     // Enter safepoint region - ornate and Rococo ...
  1106     // Enter safepoint region - ornate and Rococo ...
  1107     ThreadBlockInVM tbivm(jt);
  1107     ThreadBlockInVM tbivm(jt);
  1108     OSThreadWaitState osts(Self->osthread(), false /* not Object.wait() */);
  1108     OSThreadWaitState osts(Self->osthread(), false /* not Object.wait() */);
  1111       jt->set_suspend_equivalent();
  1111       jt->set_suspend_equivalent();
  1112       // cleared by handle_special_suspend_equivalent_condition() or
  1112       // cleared by handle_special_suspend_equivalent_condition() or
  1113       // java_suspend_self()
  1113       // java_suspend_self()
  1114     }
  1114     }
  1115 
  1115 
  1116     wait_status = IWait (Self, timeout) ;
  1116     wait_status = IWait(Self, timeout);
  1117 
  1117 
  1118     // were we externally suspended while we were waiting?
  1118     // were we externally suspended while we were waiting?
  1119     if (as_suspend_equivalent && jt->handle_special_suspend_equivalent_condition()) {
  1119     if (as_suspend_equivalent && jt->handle_special_suspend_equivalent_condition()) {
  1120       // Our event wait has finished and we own the lock, but
  1120       // Our event wait has finished and we own the lock, but
  1121       // while we were waiting another thread suspended us. We don't
  1121       // while we were waiting another thread suspended us. We don't
  1122       // want to hold the lock while suspended because that
  1122       // want to hold the lock while suspended because that
  1123       // would surprise the thread that suspended us.
  1123       // would surprise the thread that suspended us.
  1124       assert (ILocked(), "invariant") ;
  1124       assert(ILocked(), "invariant");
  1125       IUnlock (true) ;
  1125       IUnlock(true);
  1126       jt->java_suspend_self();
  1126       jt->java_suspend_self();
  1127       ILock (Self) ;
  1127       ILock(Self);
  1128       assert (ILocked(), "invariant") ;
  1128       assert(ILocked(), "invariant");
  1129     }
  1129     }
  1130   }
  1130   }
  1131 
  1131 
  1132   // Conceptually reestablish ownership of the lock.
  1132   // Conceptually reestablish ownership of the lock.
  1133   // The "real" lock -- the LockByte -- was reacquired by IWait().
  1133   // The "real" lock -- the LockByte -- was reacquired by IWait().
  1134   assert (ILocked(), "invariant") ;
  1134   assert(ILocked(), "invariant");
  1135   assert (_owner == NULL, "invariant") ;
  1135   assert(_owner == NULL, "invariant");
  1136   set_owner (Self) ;
  1136   set_owner(Self);
  1137   return wait_status != 0 ;          // return true IFF timeout
  1137   return wait_status != 0;          // return true IFF timeout
  1138 }
  1138 }
  1139 
  1139 
  1140 Monitor::~Monitor() {
  1140 Monitor::~Monitor() {
  1141   assert ((UNS(_owner)|UNS(_LockWord.FullWord)|UNS(_EntryList)|UNS(_WaitSet)|UNS(_OnDeck)) == 0, "") ;
  1141   assert((UNS(_owner)|UNS(_LockWord.FullWord)|UNS(_EntryList)|UNS(_WaitSet)|UNS(_OnDeck)) == 0, "");
  1142 }
  1142 }
  1143 
  1143 
  1144 void Monitor::ClearMonitor (Monitor * m, const char *name) {
  1144 void Monitor::ClearMonitor (Monitor * m, const char *name) {
  1145   m->_owner             = NULL ;
  1145   m->_owner             = NULL;
  1146   m->_snuck             = false ;
  1146   m->_snuck             = false;
  1147   if (name == NULL) {
  1147   if (name == NULL) {
  1148     strcpy(m->_name, "UNKNOWN") ;
  1148     strcpy(m->_name, "UNKNOWN");
  1149   } else {
  1149   } else {
  1150     strncpy(m->_name, name, MONITOR_NAME_LEN - 1);
  1150     strncpy(m->_name, name, MONITOR_NAME_LEN - 1);
  1151     m->_name[MONITOR_NAME_LEN - 1] = '\0';
  1151     m->_name[MONITOR_NAME_LEN - 1] = '\0';
  1152   }
  1152   }
  1153   m->_LockWord.FullWord = 0 ;
  1153   m->_LockWord.FullWord = 0;
  1154   m->_EntryList         = NULL ;
  1154   m->_EntryList         = NULL;
  1155   m->_OnDeck            = NULL ;
  1155   m->_OnDeck            = NULL;
  1156   m->_WaitSet           = NULL ;
  1156   m->_WaitSet           = NULL;
  1157   m->_WaitLock[0]       = 0 ;
  1157   m->_WaitLock[0]       = 0;
  1158 }
  1158 }
  1159 
  1159 
  1160 Monitor::Monitor() { ClearMonitor(this); }
  1160 Monitor::Monitor() { ClearMonitor(this); }
  1161 
  1161 
  1162 Monitor::Monitor (int Rank, const char * name, bool allow_vm_block) {
  1162 Monitor::Monitor (int Rank, const char * name, bool allow_vm_block) {
  1163   ClearMonitor (this, name) ;
  1163   ClearMonitor(this, name);
  1164 #ifdef ASSERT
  1164 #ifdef ASSERT
  1165   _allow_vm_block  = allow_vm_block;
  1165   _allow_vm_block  = allow_vm_block;
  1166   _rank            = Rank ;
  1166   _rank            = Rank;
  1167 #endif
  1167 #endif
  1168 }
  1168 }
  1169 
  1169 
  1170 Mutex::~Mutex() {
  1170 Mutex::~Mutex() {
  1171   assert ((UNS(_owner)|UNS(_LockWord.FullWord)|UNS(_EntryList)|UNS(_WaitSet)|UNS(_OnDeck)) == 0, "") ;
  1171   assert((UNS(_owner)|UNS(_LockWord.FullWord)|UNS(_EntryList)|UNS(_WaitSet)|UNS(_OnDeck)) == 0, "");
  1172 }
  1172 }
  1173 
  1173 
  1174 Mutex::Mutex (int Rank, const char * name, bool allow_vm_block) {
  1174 Mutex::Mutex (int Rank, const char * name, bool allow_vm_block) {
  1175   ClearMonitor ((Monitor *) this, name) ;
  1175   ClearMonitor((Monitor *) this, name);
  1176 #ifdef ASSERT
  1176 #ifdef ASSERT
  1177  _allow_vm_block   = allow_vm_block;
  1177  _allow_vm_block   = allow_vm_block;
  1178  _rank             = Rank ;
  1178  _rank             = Rank;
  1179 #endif
  1179 #endif
  1180 }
  1180 }
  1181 
  1181 
  1182 bool Monitor::owned_by_self() const {
  1182 bool Monitor::owned_by_self() const {
  1183   bool ret = _owner == Thread::current();
  1183   bool ret = _owner == Thread::current();
  1184   assert (!ret || _LockWord.Bytes[_LSBINDEX] != 0, "invariant") ;
  1184   assert(!ret || _LockWord.Bytes[_LSBINDEX] != 0, "invariant");
  1185   return ret;
  1185   return ret;
  1186 }
  1186 }
  1187 
  1187 
  1188 void Monitor::print_on_error(outputStream* st) const {
  1188 void Monitor::print_on_error(outputStream* st) const {
  1189   st->print("[" PTR_FORMAT, this);
  1189   st->print("[" PTR_FORMAT, this);