450 // Retry doesn't make as much sense because the lock was just acquired. |
450 // Retry doesn't make as much sense because the lock was just acquired. |
451 return -1; |
451 return -1; |
452 } |
452 } |
453 |
453 |
454 void NOINLINE ObjectMonitor::EnterI (TRAPS) { |
454 void NOINLINE ObjectMonitor::EnterI (TRAPS) { |
455 Thread * const Self = THREAD; |
455 Thread * const Self = THREAD; |
456 assert(Self->is_Java_thread(), "invariant"); |
456 assert(Self->is_Java_thread(), "invariant"); |
457 assert(((JavaThread *) Self)->thread_state() == _thread_blocked , "invariant"); |
457 assert(((JavaThread *) Self)->thread_state() == _thread_blocked , "invariant"); |
458 |
458 |
459 // Try the lock - TATAS |
459 // Try the lock - TATAS |
|
460 if (TryLock (Self) > 0) { |
|
461 assert(_succ != Self , "invariant"); |
|
462 assert(_owner == Self , "invariant"); |
|
463 assert(_Responsible != Self , "invariant"); |
|
464 return; |
|
465 } |
|
466 |
|
467 DeferredInitialize(); |
|
468 |
|
469 // We try one round of spinning *before* enqueueing Self. |
|
470 // |
|
471 // If the _owner is ready but OFFPROC we could use a YieldTo() |
|
472 // operation to donate the remainder of this thread's quantum |
|
473 // to the owner. This has subtle but beneficial affinity |
|
474 // effects. |
|
475 |
|
476 if (TrySpin (Self) > 0) { |
|
477 assert(_owner == Self , "invariant"); |
|
478 assert(_succ != Self , "invariant"); |
|
479 assert(_Responsible != Self , "invariant"); |
|
480 return; |
|
481 } |
|
482 |
|
483 // The Spin failed -- Enqueue and park the thread ... |
|
484 assert(_succ != Self , "invariant"); |
|
485 assert(_owner != Self , "invariant"); |
|
486 assert(_Responsible != Self , "invariant"); |
|
487 |
|
488 // Enqueue "Self" on ObjectMonitor's _cxq. |
|
489 // |
|
490 // Node acts as a proxy for Self. |
|
491 // As an aside, if were to ever rewrite the synchronization code mostly |
|
492 // in Java, WaitNodes, ObjectMonitors, and Events would become 1st-class |
|
493 // Java objects. This would avoid awkward lifecycle and liveness issues, |
|
494 // as well as eliminate a subset of ABA issues. |
|
495 // TODO: eliminate ObjectWaiter and enqueue either Threads or Events. |
|
496 // |
|
497 |
|
498 ObjectWaiter node(Self); |
|
499 Self->_ParkEvent->reset(); |
|
500 node._prev = (ObjectWaiter *) 0xBAD; |
|
501 node.TState = ObjectWaiter::TS_CXQ; |
|
502 |
|
503 // Push "Self" onto the front of the _cxq. |
|
504 // Once on cxq/EntryList, Self stays on-queue until it acquires the lock. |
|
505 // Note that spinning tends to reduce the rate at which threads |
|
506 // enqueue and dequeue on EntryList|cxq. |
|
507 ObjectWaiter * nxt; |
|
508 for (;;) { |
|
509 node._next = nxt = _cxq; |
|
510 if (Atomic::cmpxchg_ptr(&node, &_cxq, nxt) == nxt) break; |
|
511 |
|
512 // Interference - the CAS failed because _cxq changed. Just retry. |
|
513 // As an optional optimization we retry the lock. |
460 if (TryLock (Self) > 0) { |
514 if (TryLock (Self) > 0) { |
461 assert(_succ != Self , "invariant"); |
515 assert(_succ != Self , "invariant"); |
462 assert(_owner == Self , "invariant"); |
516 assert(_owner == Self , "invariant"); |
463 assert(_Responsible != Self , "invariant"); |
517 assert(_Responsible != Self , "invariant"); |
464 return; |
518 return; |
465 } |
519 } |
466 |
520 } |
467 DeferredInitialize(); |
521 |
468 |
522 // Check for cxq|EntryList edge transition to non-null. This indicates |
469 // We try one round of spinning *before* enqueueing Self. |
523 // the onset of contention. While contention persists exiting threads |
|
524 // will use a ST:MEMBAR:LD 1-1 exit protocol. When contention abates exit |
|
525 // operations revert to the faster 1-0 mode. This enter operation may interleave |
|
526 // (race) a concurrent 1-0 exit operation, resulting in stranding, so we |
|
527 // arrange for one of the contending thread to use a timed park() operations |
|
528 // to detect and recover from the race. (Stranding is form of progress failure |
|
529 // where the monitor is unlocked but all the contending threads remain parked). |
|
530 // That is, at least one of the contended threads will periodically poll _owner. |
|
531 // One of the contending threads will become the designated "Responsible" thread. |
|
532 // The Responsible thread uses a timed park instead of a normal indefinite park |
|
533 // operation -- it periodically wakes and checks for and recovers from potential |
|
534 // strandings admitted by 1-0 exit operations. We need at most one Responsible |
|
535 // thread per-monitor at any given moment. Only threads on cxq|EntryList may |
|
536 // be responsible for a monitor. |
|
537 // |
|
538 // Currently, one of the contended threads takes on the added role of "Responsible". |
|
539 // A viable alternative would be to use a dedicated "stranding checker" thread |
|
540 // that periodically iterated over all the threads (or active monitors) and unparked |
|
541 // successors where there was risk of stranding. This would help eliminate the |
|
542 // timer scalability issues we see on some platforms as we'd only have one thread |
|
543 // -- the checker -- parked on a timer. |
|
544 |
|
545 if ((SyncFlags & 16) == 0 && nxt == NULL && _EntryList == NULL) { |
|
546 // Try to assume the role of responsible thread for the monitor. |
|
547 // CONSIDER: ST vs CAS vs { if (Responsible==null) Responsible=Self } |
|
548 Atomic::cmpxchg_ptr(Self, &_Responsible, NULL); |
|
549 } |
|
550 |
|
551 // The lock might have been released while this thread was occupied queueing |
|
552 // itself onto _cxq. To close the race and avoid "stranding" and |
|
553 // progress-liveness failure we must resample-retry _owner before parking. |
|
554 // Note the Dekker/Lamport duality: ST cxq; MEMBAR; LD Owner. |
|
555 // In this case the ST-MEMBAR is accomplished with CAS(). |
|
556 // |
|
557 // TODO: Defer all thread state transitions until park-time. |
|
558 // Since state transitions are heavy and inefficient we'd like |
|
559 // to defer the state transitions until absolutely necessary, |
|
560 // and in doing so avoid some transitions ... |
|
561 |
|
562 TEVENT(Inflated enter - Contention); |
|
563 int nWakeups = 0; |
|
564 int RecheckInterval = 1; |
|
565 |
|
566 for (;;) { |
|
567 |
|
568 if (TryLock(Self) > 0) break; |
|
569 assert(_owner != Self, "invariant"); |
|
570 |
|
571 if ((SyncFlags & 2) && _Responsible == NULL) { |
|
572 Atomic::cmpxchg_ptr(Self, &_Responsible, NULL); |
|
573 } |
|
574 |
|
575 // park self |
|
576 if (_Responsible == Self || (SyncFlags & 1)) { |
|
577 TEVENT(Inflated enter - park TIMED); |
|
578 Self->_ParkEvent->park((jlong) RecheckInterval); |
|
579 // Increase the RecheckInterval, but clamp the value. |
|
580 RecheckInterval *= 8; |
|
581 if (RecheckInterval > 1000) RecheckInterval = 1000; |
|
582 } else { |
|
583 TEVENT(Inflated enter - park UNTIMED); |
|
584 Self->_ParkEvent->park(); |
|
585 } |
|
586 |
|
587 if (TryLock(Self) > 0) break; |
|
588 |
|
589 // The lock is still contested. |
|
590 // Keep a tally of the # of futile wakeups. |
|
591 // Note that the counter is not protected by a lock or updated by atomics. |
|
592 // That is by design - we trade "lossy" counters which are exposed to |
|
593 // races during updates for a lower probe effect. |
|
594 TEVENT(Inflated enter - Futile wakeup); |
|
595 if (ObjectMonitor::_sync_FutileWakeups != NULL) { |
|
596 ObjectMonitor::_sync_FutileWakeups->inc(); |
|
597 } |
|
598 ++nWakeups; |
|
599 |
|
600 // Assuming this is not a spurious wakeup we'll normally find _succ == Self. |
|
601 // We can defer clearing _succ until after the spin completes |
|
602 // TrySpin() must tolerate being called with _succ == Self. |
|
603 // Try yet another round of adaptive spinning. |
|
604 if ((Knob_SpinAfterFutile & 1) && TrySpin(Self) > 0) break; |
|
605 |
|
606 // We can find that we were unpark()ed and redesignated _succ while |
|
607 // we were spinning. That's harmless. If we iterate and call park(), |
|
608 // park() will consume the event and return immediately and we'll |
|
609 // just spin again. This pattern can repeat, leaving _succ to simply |
|
610 // spin on a CPU. Enable Knob_ResetEvent to clear pending unparks(). |
|
611 // Alternately, we can sample fired() here, and if set, forgo spinning |
|
612 // in the next iteration. |
|
613 |
|
614 if ((Knob_ResetEvent & 1) && Self->_ParkEvent->fired()) { |
|
615 Self->_ParkEvent->reset(); |
|
616 OrderAccess::fence(); |
|
617 } |
|
618 if (_succ == Self) _succ = NULL; |
|
619 |
|
620 // Invariant: after clearing _succ a thread *must* retry _owner before parking. |
|
621 OrderAccess::fence(); |
|
622 } |
|
623 |
|
624 // Egress : |
|
625 // Self has acquired the lock -- Unlink Self from the cxq or EntryList. |
|
626 // Normally we'll find Self on the EntryList . |
|
627 // From the perspective of the lock owner (this thread), the |
|
628 // EntryList is stable and cxq is prepend-only. |
|
629 // The head of cxq is volatile but the interior is stable. |
|
630 // In addition, Self.TState is stable. |
|
631 |
|
632 assert(_owner == Self , "invariant"); |
|
633 assert(object() != NULL , "invariant"); |
|
634 // I'd like to write: |
|
635 // guarantee (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ; |
|
636 // but as we're at a safepoint that's not safe. |
|
637 |
|
638 UnlinkAfterAcquire(Self, &node); |
|
639 if (_succ == Self) _succ = NULL; |
|
640 |
|
641 assert(_succ != Self, "invariant"); |
|
642 if (_Responsible == Self) { |
|
643 _Responsible = NULL; |
|
644 OrderAccess::fence(); // Dekker pivot-point |
|
645 |
|
646 // We may leave threads on cxq|EntryList without a designated |
|
647 // "Responsible" thread. This is benign. When this thread subsequently |
|
648 // exits the monitor it can "see" such preexisting "old" threads -- |
|
649 // threads that arrived on the cxq|EntryList before the fence, above -- |
|
650 // by LDing cxq|EntryList. Newly arrived threads -- that is, threads |
|
651 // that arrive on cxq after the ST:MEMBAR, above -- will set Responsible |
|
652 // non-null and elect a new "Responsible" timer thread. |
470 // |
653 // |
471 // If the _owner is ready but OFFPROC we could use a YieldTo() |
654 // This thread executes: |
472 // operation to donate the remainder of this thread's quantum |
655 // ST Responsible=null; MEMBAR (in enter epilogue - here) |
473 // to the owner. This has subtle but beneficial affinity |
656 // LD cxq|EntryList (in subsequent exit) |
474 // effects. |
|
475 |
|
476 if (TrySpin (Self) > 0) { |
|
477 assert(_owner == Self , "invariant"); |
|
478 assert(_succ != Self , "invariant"); |
|
479 assert(_Responsible != Self , "invariant"); |
|
480 return; |
|
481 } |
|
482 |
|
483 // The Spin failed -- Enqueue and park the thread ... |
|
484 assert(_succ != Self , "invariant"); |
|
485 assert(_owner != Self , "invariant"); |
|
486 assert(_Responsible != Self , "invariant"); |
|
487 |
|
488 // Enqueue "Self" on ObjectMonitor's _cxq. |
|
489 // |
657 // |
490 // Node acts as a proxy for Self. |
658 // Entering threads in the slow/contended path execute: |
491 // As an aside, if were to ever rewrite the synchronization code mostly |
659 // ST cxq=nonnull; MEMBAR; LD Responsible (in enter prolog) |
492 // in Java, WaitNodes, ObjectMonitors, and Events would become 1st-class |
660 // The (ST cxq; MEMBAR) is accomplished with CAS(). |
493 // Java objects. This would avoid awkward lifecycle and liveness issues, |
|
494 // as well as eliminate a subset of ABA issues. |
|
495 // TODO: eliminate ObjectWaiter and enqueue either Threads or Events. |
|
496 // |
661 // |
497 |
662 // The MEMBAR, above, prevents the LD of cxq|EntryList in the subsequent |
498 ObjectWaiter node(Self); |
663 // exit operation from floating above the ST Responsible=null. |
499 Self->_ParkEvent->reset(); |
664 } |
500 node._prev = (ObjectWaiter *) 0xBAD; |
665 |
501 node.TState = ObjectWaiter::TS_CXQ; |
666 // We've acquired ownership with CAS(). |
502 |
667 // CAS is serializing -- it has MEMBAR/FENCE-equivalent semantics. |
503 // Push "Self" onto the front of the _cxq. |
668 // But since the CAS() this thread may have also stored into _succ, |
504 // Once on cxq/EntryList, Self stays on-queue until it acquires the lock. |
669 // EntryList, cxq or Responsible. These meta-data updates must be |
505 // Note that spinning tends to reduce the rate at which threads |
670 // visible __before this thread subsequently drops the lock. |
506 // enqueue and dequeue on EntryList|cxq. |
671 // Consider what could occur if we didn't enforce this constraint -- |
507 ObjectWaiter * nxt; |
672 // STs to monitor meta-data and user-data could reorder with (become |
508 for (;;) { |
673 // visible after) the ST in exit that drops ownership of the lock. |
509 node._next = nxt = _cxq; |
674 // Some other thread could then acquire the lock, but observe inconsistent |
510 if (Atomic::cmpxchg_ptr(&node, &_cxq, nxt) == nxt) break; |
675 // or old monitor meta-data and heap data. That violates the JMM. |
511 |
676 // To that end, the 1-0 exit() operation must have at least STST|LDST |
512 // Interference - the CAS failed because _cxq changed. Just retry. |
677 // "release" barrier semantics. Specifically, there must be at least a |
513 // As an optional optimization we retry the lock. |
678 // STST|LDST barrier in exit() before the ST of null into _owner that drops |
514 if (TryLock (Self) > 0) { |
679 // the lock. The barrier ensures that changes to monitor meta-data and data |
515 assert(_succ != Self , "invariant"); |
680 // protected by the lock will be visible before we release the lock, and |
516 assert(_owner == Self , "invariant"); |
681 // therefore before some other thread (CPU) has a chance to acquire the lock. |
517 assert(_Responsible != Self , "invariant"); |
682 // See also: http://gee.cs.oswego.edu/dl/jmm/cookbook.html. |
518 return; |
683 // |
519 } |
684 // Critically, any prior STs to _succ or EntryList must be visible before |
520 } |
685 // the ST of null into _owner in the *subsequent* (following) corresponding |
521 |
686 // monitorexit. Recall too, that in 1-0 mode monitorexit does not necessarily |
522 // Check for cxq|EntryList edge transition to non-null. This indicates |
687 // execute a serializing instruction. |
523 // the onset of contention. While contention persists exiting threads |
688 |
524 // will use a ST:MEMBAR:LD 1-1 exit protocol. When contention abates exit |
689 if (SyncFlags & 8) { |
525 // operations revert to the faster 1-0 mode. This enter operation may interleave |
690 OrderAccess::fence(); |
526 // (race) a concurrent 1-0 exit operation, resulting in stranding, so we |
691 } |
527 // arrange for one of the contending thread to use a timed park() operations |
692 return; |
528 // to detect and recover from the race. (Stranding is form of progress failure |
|
529 // where the monitor is unlocked but all the contending threads remain parked). |
|
530 // That is, at least one of the contended threads will periodically poll _owner. |
|
531 // One of the contending threads will become the designated "Responsible" thread. |
|
532 // The Responsible thread uses a timed park instead of a normal indefinite park |
|
533 // operation -- it periodically wakes and checks for and recovers from potential |
|
534 // strandings admitted by 1-0 exit operations. We need at most one Responsible |
|
535 // thread per-monitor at any given moment. Only threads on cxq|EntryList may |
|
536 // be responsible for a monitor. |
|
537 // |
|
538 // Currently, one of the contended threads takes on the added role of "Responsible". |
|
539 // A viable alternative would be to use a dedicated "stranding checker" thread |
|
540 // that periodically iterated over all the threads (or active monitors) and unparked |
|
541 // successors where there was risk of stranding. This would help eliminate the |
|
542 // timer scalability issues we see on some platforms as we'd only have one thread |
|
543 // -- the checker -- parked on a timer. |
|
544 |
|
545 if ((SyncFlags & 16) == 0 && nxt == NULL && _EntryList == NULL) { |
|
546 // Try to assume the role of responsible thread for the monitor. |
|
547 // CONSIDER: ST vs CAS vs { if (Responsible==null) Responsible=Self } |
|
548 Atomic::cmpxchg_ptr(Self, &_Responsible, NULL); |
|
549 } |
|
550 |
|
551 // The lock might have been released while this thread was occupied queueing |
|
552 // itself onto _cxq. To close the race and avoid "stranding" and |
|
553 // progress-liveness failure we must resample-retry _owner before parking. |
|
554 // Note the Dekker/Lamport duality: ST cxq; MEMBAR; LD Owner. |
|
555 // In this case the ST-MEMBAR is accomplished with CAS(). |
|
556 // |
|
557 // TODO: Defer all thread state transitions until park-time. |
|
558 // Since state transitions are heavy and inefficient we'd like |
|
559 // to defer the state transitions until absolutely necessary, |
|
560 // and in doing so avoid some transitions ... |
|
561 |
|
562 TEVENT(Inflated enter - Contention); |
|
563 int nWakeups = 0; |
|
564 int RecheckInterval = 1; |
|
565 |
|
566 for (;;) { |
|
567 |
|
568 if (TryLock(Self) > 0) break; |
|
569 assert(_owner != Self, "invariant"); |
|
570 |
|
571 if ((SyncFlags & 2) && _Responsible == NULL) { |
|
572 Atomic::cmpxchg_ptr(Self, &_Responsible, NULL); |
|
573 } |
|
574 |
|
575 // park self |
|
576 if (_Responsible == Self || (SyncFlags & 1)) { |
|
577 TEVENT(Inflated enter - park TIMED); |
|
578 Self->_ParkEvent->park((jlong) RecheckInterval); |
|
579 // Increase the RecheckInterval, but clamp the value. |
|
580 RecheckInterval *= 8; |
|
581 if (RecheckInterval > 1000) RecheckInterval = 1000; |
|
582 } else { |
|
583 TEVENT(Inflated enter - park UNTIMED); |
|
584 Self->_ParkEvent->park(); |
|
585 } |
|
586 |
|
587 if (TryLock(Self) > 0) break; |
|
588 |
|
589 // The lock is still contested. |
|
590 // Keep a tally of the # of futile wakeups. |
|
591 // Note that the counter is not protected by a lock or updated by atomics. |
|
592 // That is by design - we trade "lossy" counters which are exposed to |
|
593 // races during updates for a lower probe effect. |
|
594 TEVENT(Inflated enter - Futile wakeup); |
|
595 if (ObjectMonitor::_sync_FutileWakeups != NULL) { |
|
596 ObjectMonitor::_sync_FutileWakeups->inc(); |
|
597 } |
|
598 ++nWakeups; |
|
599 |
|
600 // Assuming this is not a spurious wakeup we'll normally find _succ == Self. |
|
601 // We can defer clearing _succ until after the spin completes |
|
602 // TrySpin() must tolerate being called with _succ == Self. |
|
603 // Try yet another round of adaptive spinning. |
|
604 if ((Knob_SpinAfterFutile & 1) && TrySpin(Self) > 0) break; |
|
605 |
|
606 // We can find that we were unpark()ed and redesignated _succ while |
|
607 // we were spinning. That's harmless. If we iterate and call park(), |
|
608 // park() will consume the event and return immediately and we'll |
|
609 // just spin again. This pattern can repeat, leaving _succ to simply |
|
610 // spin on a CPU. Enable Knob_ResetEvent to clear pending unparks(). |
|
611 // Alternately, we can sample fired() here, and if set, forgo spinning |
|
612 // in the next iteration. |
|
613 |
|
614 if ((Knob_ResetEvent & 1) && Self->_ParkEvent->fired()) { |
|
615 Self->_ParkEvent->reset(); |
|
616 OrderAccess::fence(); |
|
617 } |
|
618 if (_succ == Self) _succ = NULL; |
|
619 |
|
620 // Invariant: after clearing _succ a thread *must* retry _owner before parking. |
|
621 OrderAccess::fence(); |
|
622 } |
|
623 |
|
624 // Egress : |
|
625 // Self has acquired the lock -- Unlink Self from the cxq or EntryList. |
|
626 // Normally we'll find Self on the EntryList . |
|
627 // From the perspective of the lock owner (this thread), the |
|
628 // EntryList is stable and cxq is prepend-only. |
|
629 // The head of cxq is volatile but the interior is stable. |
|
630 // In addition, Self.TState is stable. |
|
631 |
|
632 assert(_owner == Self , "invariant"); |
|
633 assert(object() != NULL , "invariant"); |
|
634 // I'd like to write: |
|
635 // guarantee (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ; |
|
636 // but as we're at a safepoint that's not safe. |
|
637 |
|
638 UnlinkAfterAcquire(Self, &node); |
|
639 if (_succ == Self) _succ = NULL; |
|
640 |
|
641 assert(_succ != Self, "invariant"); |
|
642 if (_Responsible == Self) { |
|
643 _Responsible = NULL; |
|
644 OrderAccess::fence(); // Dekker pivot-point |
|
645 |
|
646 // We may leave threads on cxq|EntryList without a designated |
|
647 // "Responsible" thread. This is benign. When this thread subsequently |
|
648 // exits the monitor it can "see" such preexisting "old" threads -- |
|
649 // threads that arrived on the cxq|EntryList before the fence, above -- |
|
650 // by LDing cxq|EntryList. Newly arrived threads -- that is, threads |
|
651 // that arrive on cxq after the ST:MEMBAR, above -- will set Responsible |
|
652 // non-null and elect a new "Responsible" timer thread. |
|
653 // |
|
654 // This thread executes: |
|
655 // ST Responsible=null; MEMBAR (in enter epilogue - here) |
|
656 // LD cxq|EntryList (in subsequent exit) |
|
657 // |
|
658 // Entering threads in the slow/contended path execute: |
|
659 // ST cxq=nonnull; MEMBAR; LD Responsible (in enter prolog) |
|
660 // The (ST cxq; MEMBAR) is accomplished with CAS(). |
|
661 // |
|
662 // The MEMBAR, above, prevents the LD of cxq|EntryList in the subsequent |
|
663 // exit operation from floating above the ST Responsible=null. |
|
664 } |
|
665 |
|
666 // We've acquired ownership with CAS(). |
|
667 // CAS is serializing -- it has MEMBAR/FENCE-equivalent semantics. |
|
668 // But since the CAS() this thread may have also stored into _succ, |
|
669 // EntryList, cxq or Responsible. These meta-data updates must be |
|
670 // visible __before this thread subsequently drops the lock. |
|
671 // Consider what could occur if we didn't enforce this constraint -- |
|
672 // STs to monitor meta-data and user-data could reorder with (become |
|
673 // visible after) the ST in exit that drops ownership of the lock. |
|
674 // Some other thread could then acquire the lock, but observe inconsistent |
|
675 // or old monitor meta-data and heap data. That violates the JMM. |
|
676 // To that end, the 1-0 exit() operation must have at least STST|LDST |
|
677 // "release" barrier semantics. Specifically, there must be at least a |
|
678 // STST|LDST barrier in exit() before the ST of null into _owner that drops |
|
679 // the lock. The barrier ensures that changes to monitor meta-data and data |
|
680 // protected by the lock will be visible before we release the lock, and |
|
681 // therefore before some other thread (CPU) has a chance to acquire the lock. |
|
682 // See also: http://gee.cs.oswego.edu/dl/jmm/cookbook.html. |
|
683 // |
|
684 // Critically, any prior STs to _succ or EntryList must be visible before |
|
685 // the ST of null into _owner in the *subsequent* (following) corresponding |
|
686 // monitorexit. Recall too, that in 1-0 mode monitorexit does not necessarily |
|
687 // execute a serializing instruction. |
|
688 |
|
689 if (SyncFlags & 8) { |
|
690 OrderAccess::fence(); |
|
691 } |
|
692 return; |
|
693 } |
693 } |
694 |
694 |
695 // ReenterI() is a specialized inline form of the latter half of the |
695 // ReenterI() is a specialized inline form of the latter half of the |
696 // contended slow-path from EnterI(). We use ReenterI() only for |
696 // contended slow-path from EnterI(). We use ReenterI() only for |
697 // monitor reentry in wait(). |
697 // monitor reentry in wait(). |
699 // In the future we should reconcile EnterI() and ReenterI(), adding |
699 // In the future we should reconcile EnterI() and ReenterI(), adding |
700 // Knob_Reset and Knob_SpinAfterFutile support and restructuring the |
700 // Knob_Reset and Knob_SpinAfterFutile support and restructuring the |
701 // loop accordingly. |
701 // loop accordingly. |
702 |
702 |
703 void NOINLINE ObjectMonitor::ReenterI (Thread * Self, ObjectWaiter * SelfNode) { |
703 void NOINLINE ObjectMonitor::ReenterI (Thread * Self, ObjectWaiter * SelfNode) { |
704 assert(Self != NULL , "invariant"); |
704 assert(Self != NULL , "invariant"); |
705 assert(SelfNode != NULL , "invariant"); |
705 assert(SelfNode != NULL , "invariant"); |
706 assert(SelfNode->_thread == Self , "invariant"); |
706 assert(SelfNode->_thread == Self , "invariant"); |
707 assert(_waiters > 0 , "invariant"); |
707 assert(_waiters > 0 , "invariant"); |
708 assert(((oop)(object()))->mark() == markOopDesc::encode(this) , "invariant"); |
708 assert(((oop)(object()))->mark() == markOopDesc::encode(this) , "invariant"); |
709 assert(((JavaThread *)Self)->thread_state() != _thread_blocked, "invariant"); |
709 assert(((JavaThread *)Self)->thread_state() != _thread_blocked, "invariant"); |
710 JavaThread * jt = (JavaThread *) Self; |
710 JavaThread * jt = (JavaThread *) Self; |
711 |
711 |
712 int nWakeups = 0; |
712 int nWakeups = 0; |
713 for (;;) { |
713 for (;;) { |
714 ObjectWaiter::TStates v = SelfNode->TState; |
714 ObjectWaiter::TStates v = SelfNode->TState; |
715 guarantee(v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant"); |
715 guarantee(v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant"); |
716 assert(_owner != Self, "invariant"); |
716 assert(_owner != Self, "invariant"); |
717 |
717 |
718 if (TryLock(Self) > 0) break; |
718 if (TryLock(Self) > 0) break; |
719 if (TrySpin(Self) > 0) break; |
719 if (TrySpin(Self) > 0) break; |
720 |
720 |
721 TEVENT(Wait Reentry - parking); |
721 TEVENT(Wait Reentry - parking); |
722 |
722 |
723 // State transition wrappers around park() ... |
723 // State transition wrappers around park() ... |
724 // ReenterI() wisely defers state transitions until |
724 // ReenterI() wisely defers state transitions until |
725 // it's clear we must park the thread. |
725 // it's clear we must park the thread. |
726 { |
726 { |
727 OSThreadContendState osts(Self->osthread()); |
727 OSThreadContendState osts(Self->osthread()); |
728 ThreadBlockInVM tbivm(jt); |
728 ThreadBlockInVM tbivm(jt); |
729 |
729 |
730 // cleared by handle_special_suspend_equivalent_condition() |
730 // cleared by handle_special_suspend_equivalent_condition() |
731 // or java_suspend_self() |
731 // or java_suspend_self() |
732 jt->set_suspend_equivalent(); |
732 jt->set_suspend_equivalent(); |
733 if (SyncFlags & 1) { |
733 if (SyncFlags & 1) { |
734 Self->_ParkEvent->park((jlong)1000); |
734 Self->_ParkEvent->park((jlong)1000); |
735 } else { |
735 } else { |
736 Self->_ParkEvent->park(); |
736 Self->_ParkEvent->park(); |
737 } |
737 } |
738 |
738 |
739 // were we externally suspended while we were waiting? |
739 // were we externally suspended while we were waiting? |
740 for (;;) { |
740 for (;;) { |
741 if (!ExitSuspendEquivalent(jt)) break; |
741 if (!ExitSuspendEquivalent(jt)) break; |
742 if (_succ == Self) { _succ = NULL; OrderAccess::fence(); } |
742 if (_succ == Self) { _succ = NULL; OrderAccess::fence(); } |
743 jt->java_suspend_self(); |
743 jt->java_suspend_self(); |
744 jt->set_suspend_equivalent(); |
744 jt->set_suspend_equivalent(); |
745 } |
745 } |
746 } |
746 } |
747 |
747 |
748 // Try again, but just so we distinguish between futile wakeups and |
748 // Try again, but just so we distinguish between futile wakeups and |
749 // successful wakeups. The following test isn't algorithmically |
749 // successful wakeups. The following test isn't algorithmically |
750 // necessary, but it helps us maintain sensible statistics. |
750 // necessary, but it helps us maintain sensible statistics. |
751 if (TryLock(Self) > 0) break; |
751 if (TryLock(Self) > 0) break; |
752 |
752 |
753 // The lock is still contested. |
753 // The lock is still contested. |
754 // Keep a tally of the # of futile wakeups. |
754 // Keep a tally of the # of futile wakeups. |
755 // Note that the counter is not protected by a lock or updated by atomics. |
755 // Note that the counter is not protected by a lock or updated by atomics. |
756 // That is by design - we trade "lossy" counters which are exposed to |
756 // That is by design - we trade "lossy" counters which are exposed to |
757 // races during updates for a lower probe effect. |
757 // races during updates for a lower probe effect. |
758 TEVENT(Wait Reentry - futile wakeup); |
758 TEVENT(Wait Reentry - futile wakeup); |
759 ++nWakeups; |
759 ++nWakeups; |
760 |
760 |
761 // Assuming this is not a spurious wakeup we'll normally |
761 // Assuming this is not a spurious wakeup we'll normally |
762 // find that _succ == Self. |
762 // find that _succ == Self. |
763 if (_succ == Self) _succ = NULL; |
|
764 |
|
765 // Invariant: after clearing _succ a contending thread |
|
766 // *must* retry _owner before parking. |
|
767 OrderAccess::fence(); |
|
768 |
|
769 if (ObjectMonitor::_sync_FutileWakeups != NULL) { |
|
770 ObjectMonitor::_sync_FutileWakeups->inc(); |
|
771 } |
|
772 } |
|
773 |
|
774 // Self has acquired the lock -- Unlink Self from the cxq or EntryList . |
|
775 // Normally we'll find Self on the EntryList. |
|
776 // Unlinking from the EntryList is constant-time and atomic-free. |
|
777 // From the perspective of the lock owner (this thread), the |
|
778 // EntryList is stable and cxq is prepend-only. |
|
779 // The head of cxq is volatile but the interior is stable. |
|
780 // In addition, Self.TState is stable. |
|
781 |
|
782 assert(_owner == Self, "invariant"); |
|
783 assert(((oop)(object()))->mark() == markOopDesc::encode(this), "invariant"); |
|
784 UnlinkAfterAcquire(Self, SelfNode); |
|
785 if (_succ == Self) _succ = NULL; |
763 if (_succ == Self) _succ = NULL; |
786 assert(_succ != Self, "invariant"); |
764 |
787 SelfNode->TState = ObjectWaiter::TS_RUN; |
765 // Invariant: after clearing _succ a contending thread |
788 OrderAccess::fence(); // see comments at the end of EnterI() |
766 // *must* retry _owner before parking. |
|
767 OrderAccess::fence(); |
|
768 |
|
769 if (ObjectMonitor::_sync_FutileWakeups != NULL) { |
|
770 ObjectMonitor::_sync_FutileWakeups->inc(); |
|
771 } |
|
772 } |
|
773 |
|
774 // Self has acquired the lock -- Unlink Self from the cxq or EntryList . |
|
775 // Normally we'll find Self on the EntryList. |
|
776 // Unlinking from the EntryList is constant-time and atomic-free. |
|
777 // From the perspective of the lock owner (this thread), the |
|
778 // EntryList is stable and cxq is prepend-only. |
|
779 // The head of cxq is volatile but the interior is stable. |
|
780 // In addition, Self.TState is stable. |
|
781 |
|
782 assert(_owner == Self, "invariant"); |
|
783 assert(((oop)(object()))->mark() == markOopDesc::encode(this), "invariant"); |
|
784 UnlinkAfterAcquire(Self, SelfNode); |
|
785 if (_succ == Self) _succ = NULL; |
|
786 assert(_succ != Self, "invariant"); |
|
787 SelfNode->TState = ObjectWaiter::TS_RUN; |
|
788 OrderAccess::fence(); // see comments at the end of EnterI() |
789 } |
789 } |
790 |
790 |
791 // By convention we unlink a contending thread from EntryList|cxq immediately |
791 // By convention we unlink a contending thread from EntryList|cxq immediately |
792 // after the thread acquires the lock in ::enter(). Equally, we could defer |
792 // after the thread acquires the lock in ::enter(). Equally, we could defer |
793 // unlinking the thread until ::exit()-time. |
793 // unlinking the thread until ::exit()-time. |
794 |
794 |
795 void ObjectMonitor::UnlinkAfterAcquire (Thread * Self, ObjectWaiter * SelfNode) |
795 void ObjectMonitor::UnlinkAfterAcquire (Thread * Self, ObjectWaiter * SelfNode) |
796 { |
796 { |
797 assert(_owner == Self, "invariant"); |
797 assert(_owner == Self, "invariant"); |
798 assert(SelfNode->_thread == Self, "invariant"); |
798 assert(SelfNode->_thread == Self, "invariant"); |
799 |
799 |
800 if (SelfNode->TState == ObjectWaiter::TS_ENTER) { |
800 if (SelfNode->TState == ObjectWaiter::TS_ENTER) { |
801 // Normal case: remove Self from the DLL EntryList . |
801 // Normal case: remove Self from the DLL EntryList . |
802 // This is a constant-time operation. |
802 // This is a constant-time operation. |
803 ObjectWaiter * nxt = SelfNode->_next; |
803 ObjectWaiter * nxt = SelfNode->_next; |
804 ObjectWaiter * prv = SelfNode->_prev; |
804 ObjectWaiter * prv = SelfNode->_prev; |
805 if (nxt != NULL) nxt->_prev = prv; |
805 if (nxt != NULL) nxt->_prev = prv; |
806 if (prv != NULL) prv->_next = nxt; |
806 if (prv != NULL) prv->_next = nxt; |
807 if (SelfNode == _EntryList) _EntryList = nxt; |
807 if (SelfNode == _EntryList) _EntryList = nxt; |
808 assert(nxt == NULL || nxt->TState == ObjectWaiter::TS_ENTER, "invariant"); |
808 assert(nxt == NULL || nxt->TState == ObjectWaiter::TS_ENTER, "invariant"); |
809 assert(prv == NULL || prv->TState == ObjectWaiter::TS_ENTER, "invariant"); |
809 assert(prv == NULL || prv->TState == ObjectWaiter::TS_ENTER, "invariant"); |
810 TEVENT(Unlink from EntryList); |
810 TEVENT(Unlink from EntryList); |
811 } else { |
811 } else { |
812 assert(SelfNode->TState == ObjectWaiter::TS_CXQ, "invariant"); |
812 assert(SelfNode->TState == ObjectWaiter::TS_CXQ, "invariant"); |
813 // Inopportune interleaving -- Self is still on the cxq. |
813 // Inopportune interleaving -- Self is still on the cxq. |
814 // This usually means the enqueue of self raced an exiting thread. |
814 // This usually means the enqueue of self raced an exiting thread. |
815 // Normally we'll find Self near the front of the cxq, so |
815 // Normally we'll find Self near the front of the cxq, so |
816 // dequeueing is typically fast. If needbe we can accelerate |
816 // dequeueing is typically fast. If needbe we can accelerate |
817 // this with some MCS/CHL-like bidirectional list hints and advisory |
817 // this with some MCS/CHL-like bidirectional list hints and advisory |
818 // back-links so dequeueing from the interior will normally operate |
818 // back-links so dequeueing from the interior will normally operate |
819 // in constant-time. |
819 // in constant-time. |
820 // Dequeue Self from either the head (with CAS) or from the interior |
820 // Dequeue Self from either the head (with CAS) or from the interior |
821 // with a linear-time scan and normal non-atomic memory operations. |
821 // with a linear-time scan and normal non-atomic memory operations. |
822 // CONSIDER: if Self is on the cxq then simply drain cxq into EntryList |
822 // CONSIDER: if Self is on the cxq then simply drain cxq into EntryList |
823 // and then unlink Self from EntryList. We have to drain eventually, |
823 // and then unlink Self from EntryList. We have to drain eventually, |
824 // so it might as well be now. |
824 // so it might as well be now. |
825 |
825 |
826 ObjectWaiter * v = _cxq; |
826 ObjectWaiter * v = _cxq; |
827 assert(v != NULL, "invariant"); |
827 assert(v != NULL, "invariant"); |
828 if (v != SelfNode || Atomic::cmpxchg_ptr (SelfNode->_next, &_cxq, v) != v) { |
828 if (v != SelfNode || Atomic::cmpxchg_ptr (SelfNode->_next, &_cxq, v) != v) { |
829 // The CAS above can fail from interference IFF a "RAT" arrived. |
829 // The CAS above can fail from interference IFF a "RAT" arrived. |
830 // In that case Self must be in the interior and can no longer be |
830 // In that case Self must be in the interior and can no longer be |
831 // at the head of cxq. |
831 // at the head of cxq. |
832 if (v == SelfNode) { |
832 if (v == SelfNode) { |
833 assert(_cxq != v, "invariant"); |
833 assert(_cxq != v, "invariant"); |
834 v = _cxq; // CAS above failed - start scan at head of list |
834 v = _cxq; // CAS above failed - start scan at head of list |
835 } |
835 } |
836 ObjectWaiter * p; |
836 ObjectWaiter * p; |
837 ObjectWaiter * q = NULL; |
837 ObjectWaiter * q = NULL; |
838 for (p = v; p != NULL && p != SelfNode; p = p->_next) { |
838 for (p = v; p != NULL && p != SelfNode; p = p->_next) { |
839 q = p; |
839 q = p; |
840 assert(p->TState == ObjectWaiter::TS_CXQ, "invariant"); |
840 assert(p->TState == ObjectWaiter::TS_CXQ, "invariant"); |
841 } |
841 } |
842 assert(v != SelfNode, "invariant"); |
842 assert(v != SelfNode, "invariant"); |
843 assert(p == SelfNode, "Node not found on cxq"); |
843 assert(p == SelfNode, "Node not found on cxq"); |
844 assert(p != _cxq, "invariant"); |
844 assert(p != _cxq, "invariant"); |
845 assert(q != NULL, "invariant"); |
845 assert(q != NULL, "invariant"); |
846 assert(q->_next == p, "invariant"); |
846 assert(q->_next == p, "invariant"); |
847 q->_next = p->_next; |
847 q->_next = p->_next; |
848 } |
848 } |
849 TEVENT(Unlink from cxq); |
849 TEVENT(Unlink from cxq); |
850 } |
850 } |
851 |
851 |
852 #ifdef ASSERT |
852 #ifdef ASSERT |
853 // Diagnostic hygiene ... |
853 // Diagnostic hygiene ... |
854 SelfNode->_prev = (ObjectWaiter *) 0xBAD; |
854 SelfNode->_prev = (ObjectWaiter *) 0xBAD; |
855 SelfNode->_next = (ObjectWaiter *) 0xBAD; |
855 SelfNode->_next = (ObjectWaiter *) 0xBAD; |
856 SelfNode->TState = ObjectWaiter::TS_RUN; |
856 SelfNode->TState = ObjectWaiter::TS_RUN; |
857 #endif |
857 #endif |
858 } |
858 } |
859 |
859 |
860 // ----------------------------------------------------------------------------- |
860 // ----------------------------------------------------------------------------- |
861 // Exit support |
861 // Exit support |
913 // then wake a thread unnecessarily. This is benign, and we've |
913 // then wake a thread unnecessarily. This is benign, and we've |
914 // structured the code so the windows are short and the frequency |
914 // structured the code so the windows are short and the frequency |
915 // of such futile wakups is low. |
915 // of such futile wakups is low. |
916 |
916 |
917 void NOINLINE ObjectMonitor::exit(bool not_suspended, TRAPS) { |
917 void NOINLINE ObjectMonitor::exit(bool not_suspended, TRAPS) { |
918 Thread * const Self = THREAD; |
918 Thread * const Self = THREAD; |
919 if (THREAD != _owner) { |
919 if (THREAD != _owner) { |
920 if (THREAD->is_lock_owned((address) _owner)) { |
920 if (THREAD->is_lock_owned((address) _owner)) { |
921 // Transmute _owner from a BasicLock pointer to a Thread address. |
921 // Transmute _owner from a BasicLock pointer to a Thread address. |
922 // We don't need to hold _mutex for this transition. |
922 // We don't need to hold _mutex for this transition. |
923 // Non-null to Non-null is safe as long as all readers can |
923 // Non-null to Non-null is safe as long as all readers can |
924 // tolerate either flavor. |
924 // tolerate either flavor. |
925 assert(_recursions == 0, "invariant"); |
925 assert(_recursions == 0, "invariant"); |
926 _owner = THREAD; |
926 _owner = THREAD; |
927 _recursions = 0; |
927 _recursions = 0; |
928 OwnerIsThread = 1; |
928 OwnerIsThread = 1; |
929 } else { |
929 } else { |
930 // Apparent unbalanced locking ... |
930 // Apparent unbalanced locking ... |
931 // Naively we'd like to throw IllegalMonitorStateException. |
931 // Naively we'd like to throw IllegalMonitorStateException. |
932 // As a practical matter we can neither allocate nor throw an |
932 // As a practical matter we can neither allocate nor throw an |
933 // exception as ::exit() can be called from leaf routines. |
933 // exception as ::exit() can be called from leaf routines. |
934 // see x86_32.ad Fast_Unlock() and the I1 and I2 properties. |
934 // see x86_32.ad Fast_Unlock() and the I1 and I2 properties. |
935 // Upon deeper reflection, however, in a properly run JVM the only |
935 // Upon deeper reflection, however, in a properly run JVM the only |
936 // way we should encounter this situation is in the presence of |
936 // way we should encounter this situation is in the presence of |
937 // unbalanced JNI locking. TODO: CheckJNICalls. |
937 // unbalanced JNI locking. TODO: CheckJNICalls. |
938 // See also: CR4414101 |
938 // See also: CR4414101 |
939 TEVENT(Exit - Throw IMSX); |
939 TEVENT(Exit - Throw IMSX); |
940 assert(false, "Non-balanced monitor enter/exit! Likely JNI locking"); |
940 assert(false, "Non-balanced monitor enter/exit! Likely JNI locking"); |
941 return; |
941 return; |
942 } |
942 } |
943 } |
943 } |
944 |
944 |
945 if (_recursions != 0) { |
945 if (_recursions != 0) { |
946 _recursions--; // this is simple recursive enter |
946 _recursions--; // this is simple recursive enter |
947 TEVENT(Inflated exit - recursive); |
947 TEVENT(Inflated exit - recursive); |
948 return; |
948 return; |
949 } |
949 } |
950 |
950 |
951 // Invariant: after setting Responsible=null an thread must execute |
951 // Invariant: after setting Responsible=null an thread must execute |
952 // a MEMBAR or other serializing instruction before fetching EntryList|cxq. |
952 // a MEMBAR or other serializing instruction before fetching EntryList|cxq. |
953 if ((SyncFlags & 4) == 0) { |
953 if ((SyncFlags & 4) == 0) { |
954 _Responsible = NULL; |
954 _Responsible = NULL; |
955 } |
955 } |
956 |
956 |
957 #if INCLUDE_TRACE |
957 #if INCLUDE_TRACE |
958 // get the owner's thread id for the MonitorEnter event |
958 // get the owner's thread id for the MonitorEnter event |
959 // if it is enabled and the thread isn't suspended |
959 // if it is enabled and the thread isn't suspended |
960 if (not_suspended && Tracing::is_event_enabled(TraceJavaMonitorEnterEvent)) { |
960 if (not_suspended && Tracing::is_event_enabled(TraceJavaMonitorEnterEvent)) { |
961 _previous_owner_tid = SharedRuntime::get_java_tid(Self); |
961 _previous_owner_tid = SharedRuntime::get_java_tid(Self); |
962 } |
962 } |
963 #endif |
963 #endif |
964 |
964 |
965 for (;;) { |
965 for (;;) { |
966 assert(THREAD == _owner, "invariant"); |
966 assert(THREAD == _owner, "invariant"); |
967 |
967 |
968 |
968 |
969 if (Knob_ExitPolicy == 0) { |
969 if (Knob_ExitPolicy == 0) { |
970 // release semantics: prior loads and stores from within the critical section |
970 // release semantics: prior loads and stores from within the critical section |
971 // must not float (reorder) past the following store that drops the lock. |
971 // must not float (reorder) past the following store that drops the lock. |
972 // On SPARC that requires MEMBAR #loadstore|#storestore. |
972 // On SPARC that requires MEMBAR #loadstore|#storestore. |
973 // But of course in TSO #loadstore|#storestore is not required. |
973 // But of course in TSO #loadstore|#storestore is not required. |
974 // I'd like to write one of the following: |
974 // I'd like to write one of the following: |
975 // A. OrderAccess::release() ; _owner = NULL |
975 // A. OrderAccess::release() ; _owner = NULL |
976 // B. OrderAccess::loadstore(); OrderAccess::storestore(); _owner = NULL; |
976 // B. OrderAccess::loadstore(); OrderAccess::storestore(); _owner = NULL; |
977 // Unfortunately OrderAccess::release() and OrderAccess::loadstore() both |
977 // Unfortunately OrderAccess::release() and OrderAccess::loadstore() both |
978 // store into a _dummy variable. That store is not needed, but can result |
978 // store into a _dummy variable. That store is not needed, but can result |
979 // in massive wasteful coherency traffic on classic SMP systems. |
979 // in massive wasteful coherency traffic on classic SMP systems. |
980 // Instead, I use release_store(), which is implemented as just a simple |
980 // Instead, I use release_store(), which is implemented as just a simple |
981 // ST on x64, x86 and SPARC. |
981 // ST on x64, x86 and SPARC. |
982 OrderAccess::release_store_ptr(&_owner, NULL); // drop the lock |
982 OrderAccess::release_store_ptr(&_owner, NULL); // drop the lock |
983 OrderAccess::storeload(); // See if we need to wake a successor |
983 OrderAccess::storeload(); // See if we need to wake a successor |
984 if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) { |
984 if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) { |
985 TEVENT(Inflated exit - simple egress); |
985 TEVENT(Inflated exit - simple egress); |
986 return; |
986 return; |
987 } |
987 } |
988 TEVENT(Inflated exit - complex egress); |
988 TEVENT(Inflated exit - complex egress); |
989 // Other threads are blocked trying to acquire the lock. |
989 // Other threads are blocked trying to acquire the lock. |
990 |
990 |
991 // Normally the exiting thread is responsible for ensuring succession, |
991 // Normally the exiting thread is responsible for ensuring succession, |
992 // but if other successors are ready or other entering threads are spinning |
992 // but if other successors are ready or other entering threads are spinning |
993 // then this thread can simply store NULL into _owner and exit without |
993 // then this thread can simply store NULL into _owner and exit without |
994 // waking a successor. The existence of spinners or ready successors |
994 // waking a successor. The existence of spinners or ready successors |
995 // guarantees proper succession (liveness). Responsibility passes to the |
995 // guarantees proper succession (liveness). Responsibility passes to the |
996 // ready or running successors. The exiting thread delegates the duty. |
996 // ready or running successors. The exiting thread delegates the duty. |
997 // More precisely, if a successor already exists this thread is absolved |
997 // More precisely, if a successor already exists this thread is absolved |
998 // of the responsibility of waking (unparking) one. |
998 // of the responsibility of waking (unparking) one. |
999 // |
999 // |
1000 // The _succ variable is critical to reducing futile wakeup frequency. |
1000 // The _succ variable is critical to reducing futile wakeup frequency. |
1001 // _succ identifies the "heir presumptive" thread that has been made |
1001 // _succ identifies the "heir presumptive" thread that has been made |
1002 // ready (unparked) but that has not yet run. We need only one such |
1002 // ready (unparked) but that has not yet run. We need only one such |
1003 // successor thread to guarantee progress. |
1003 // successor thread to guarantee progress. |
1004 // See http://www.usenix.org/events/jvm01/full_papers/dice/dice.pdf |
1004 // See http://www.usenix.org/events/jvm01/full_papers/dice/dice.pdf |
1005 // section 3.3 "Futile Wakeup Throttling" for details. |
1005 // section 3.3 "Futile Wakeup Throttling" for details. |
1006 // |
1006 // |
1007 // Note that spinners in Enter() also set _succ non-null. |
1007 // Note that spinners in Enter() also set _succ non-null. |
1008 // In the current implementation spinners opportunistically set |
1008 // In the current implementation spinners opportunistically set |
1009 // _succ so that exiting threads might avoid waking a successor. |
1009 // _succ so that exiting threads might avoid waking a successor. |
1010 // Another less appealing alternative would be for the exiting thread |
1010 // Another less appealing alternative would be for the exiting thread |
1011 // to drop the lock and then spin briefly to see if a spinner managed |
1011 // to drop the lock and then spin briefly to see if a spinner managed |
1012 // to acquire the lock. If so, the exiting thread could exit |
1012 // to acquire the lock. If so, the exiting thread could exit |
1013 // immediately without waking a successor, otherwise the exiting |
1013 // immediately without waking a successor, otherwise the exiting |
1014 // thread would need to dequeue and wake a successor. |
1014 // thread would need to dequeue and wake a successor. |
1015 // (Note that we'd need to make the post-drop spin short, but no |
1015 // (Note that we'd need to make the post-drop spin short, but no |
1016 // shorter than the worst-case round-trip cache-line migration time. |
1016 // shorter than the worst-case round-trip cache-line migration time. |
1017 // The dropped lock needs to become visible to the spinner, and then |
1017 // The dropped lock needs to become visible to the spinner, and then |
1018 // the acquisition of the lock by the spinner must become visible to |
1018 // the acquisition of the lock by the spinner must become visible to |
1019 // the exiting thread). |
1019 // the exiting thread). |
1020 // |
1020 // |
1021 |
1021 |
1022 // It appears that an heir-presumptive (successor) must be made ready. |
1022 // It appears that an heir-presumptive (successor) must be made ready. |
1023 // Only the current lock owner can manipulate the EntryList or |
1023 // Only the current lock owner can manipulate the EntryList or |
1024 // drain _cxq, so we need to reacquire the lock. If we fail |
1024 // drain _cxq, so we need to reacquire the lock. If we fail |
1025 // to reacquire the lock the responsibility for ensuring succession |
1025 // to reacquire the lock the responsibility for ensuring succession |
1026 // falls to the new owner. |
1026 // falls to the new owner. |
1027 // |
1027 // |
1028 if (Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) != NULL) { |
1028 if (Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) != NULL) { |
1029 return; |
1029 return; |
1030 } |
1030 } |
1031 TEVENT(Exit - Reacquired); |
1031 TEVENT(Exit - Reacquired); |
|
1032 } else { |
|
1033 if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) { |
|
1034 OrderAccess::release_store_ptr(&_owner, NULL); // drop the lock |
|
1035 OrderAccess::storeload(); |
|
1036 // Ratify the previously observed values. |
|
1037 if (_cxq == NULL || _succ != NULL) { |
|
1038 TEVENT(Inflated exit - simple egress); |
|
1039 return; |
|
1040 } |
|
1041 |
|
1042 // inopportune interleaving -- the exiting thread (this thread) |
|
1043 // in the fast-exit path raced an entering thread in the slow-enter |
|
1044 // path. |
|
1045 // We have two choices: |
|
1046 // A. Try to reacquire the lock. |
|
1047 // If the CAS() fails return immediately, otherwise |
|
1048 // we either restart/rerun the exit operation, or simply |
|
1049 // fall-through into the code below which wakes a successor. |
|
1050 // B. If the elements forming the EntryList|cxq are TSM |
|
1051 // we could simply unpark() the lead thread and return |
|
1052 // without having set _succ. |
|
1053 if (Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) != NULL) { |
|
1054 TEVENT(Inflated exit - reacquired succeeded); |
|
1055 return; |
|
1056 } |
|
1057 TEVENT(Inflated exit - reacquired failed); |
1032 } else { |
1058 } else { |
1033 if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) { |
1059 TEVENT(Inflated exit - complex egress); |
1034 OrderAccess::release_store_ptr(&_owner, NULL); // drop the lock |
1060 } |
1035 OrderAccess::storeload(); |
1061 } |
1036 // Ratify the previously observed values. |
1062 |
1037 if (_cxq == NULL || _succ != NULL) { |
1063 guarantee(_owner == THREAD, "invariant"); |
1038 TEVENT(Inflated exit - simple egress); |
1064 |
1039 return; |
1065 ObjectWaiter * w = NULL; |
1040 } |
1066 int QMode = Knob_QMode; |
1041 |
1067 |
1042 // inopportune interleaving -- the exiting thread (this thread) |
1068 if (QMode == 2 && _cxq != NULL) { |
1043 // in the fast-exit path raced an entering thread in the slow-enter |
1069 // QMode == 2 : cxq has precedence over EntryList. |
1044 // path. |
1070 // Try to directly wake a successor from the cxq. |
1045 // We have two choices: |
1071 // If successful, the successor will need to unlink itself from cxq. |
1046 // A. Try to reacquire the lock. |
|
1047 // If the CAS() fails return immediately, otherwise |
|
1048 // we either restart/rerun the exit operation, or simply |
|
1049 // fall-through into the code below which wakes a successor. |
|
1050 // B. If the elements forming the EntryList|cxq are TSM |
|
1051 // we could simply unpark() the lead thread and return |
|
1052 // without having set _succ. |
|
1053 if (Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) != NULL) { |
|
1054 TEVENT(Inflated exit - reacquired succeeded); |
|
1055 return; |
|
1056 } |
|
1057 TEVENT(Inflated exit - reacquired failed); |
|
1058 } else { |
|
1059 TEVENT(Inflated exit - complex egress); |
|
1060 } |
|
1061 } |
|
1062 |
|
1063 guarantee(_owner == THREAD, "invariant"); |
|
1064 |
|
1065 ObjectWaiter * w = NULL; |
|
1066 int QMode = Knob_QMode; |
|
1067 |
|
1068 if (QMode == 2 && _cxq != NULL) { |
|
1069 // QMode == 2 : cxq has precedence over EntryList. |
|
1070 // Try to directly wake a successor from the cxq. |
|
1071 // If successful, the successor will need to unlink itself from cxq. |
|
1072 w = _cxq; |
|
1073 assert(w != NULL, "invariant"); |
|
1074 assert(w->TState == ObjectWaiter::TS_CXQ, "Invariant"); |
|
1075 ExitEpilog(Self, w); |
|
1076 return; |
|
1077 } |
|
1078 |
|
1079 if (QMode == 3 && _cxq != NULL) { |
|
1080 // Aggressively drain cxq into EntryList at the first opportunity. |
|
1081 // This policy ensure that recently-run threads live at the head of EntryList. |
|
1082 // Drain _cxq into EntryList - bulk transfer. |
|
1083 // First, detach _cxq. |
|
1084 // The following loop is tantamount to: w = swap (&cxq, NULL) |
|
1085 w = _cxq; |
|
1086 for (;;) { |
|
1087 assert(w != NULL, "Invariant"); |
|
1088 ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr(NULL, &_cxq, w); |
|
1089 if (u == w) break; |
|
1090 w = u; |
|
1091 } |
|
1092 assert(w != NULL , "invariant"); |
|
1093 |
|
1094 ObjectWaiter * q = NULL; |
|
1095 ObjectWaiter * p; |
|
1096 for (p = w; p != NULL; p = p->_next) { |
|
1097 guarantee(p->TState == ObjectWaiter::TS_CXQ, "Invariant"); |
|
1098 p->TState = ObjectWaiter::TS_ENTER; |
|
1099 p->_prev = q; |
|
1100 q = p; |
|
1101 } |
|
1102 |
|
1103 // Append the RATs to the EntryList |
|
1104 // TODO: organize EntryList as a CDLL so we can locate the tail in constant-time. |
|
1105 ObjectWaiter * Tail; |
|
1106 for (Tail = _EntryList; Tail != NULL && Tail->_next != NULL; Tail = Tail->_next); |
|
1107 if (Tail == NULL) { |
|
1108 _EntryList = w; |
|
1109 } else { |
|
1110 Tail->_next = w; |
|
1111 w->_prev = Tail; |
|
1112 } |
|
1113 |
|
1114 // Fall thru into code that tries to wake a successor from EntryList |
|
1115 } |
|
1116 |
|
1117 if (QMode == 4 && _cxq != NULL) { |
|
1118 // Aggressively drain cxq into EntryList at the first opportunity. |
|
1119 // This policy ensure that recently-run threads live at the head of EntryList. |
|
1120 |
|
1121 // Drain _cxq into EntryList - bulk transfer. |
|
1122 // First, detach _cxq. |
|
1123 // The following loop is tantamount to: w = swap (&cxq, NULL) |
|
1124 w = _cxq; |
|
1125 for (;;) { |
|
1126 assert(w != NULL, "Invariant"); |
|
1127 ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr(NULL, &_cxq, w); |
|
1128 if (u == w) break; |
|
1129 w = u; |
|
1130 } |
|
1131 assert(w != NULL , "invariant"); |
|
1132 |
|
1133 ObjectWaiter * q = NULL; |
|
1134 ObjectWaiter * p; |
|
1135 for (p = w; p != NULL; p = p->_next) { |
|
1136 guarantee(p->TState == ObjectWaiter::TS_CXQ, "Invariant"); |
|
1137 p->TState = ObjectWaiter::TS_ENTER; |
|
1138 p->_prev = q; |
|
1139 q = p; |
|
1140 } |
|
1141 |
|
1142 // Prepend the RATs to the EntryList |
|
1143 if (_EntryList != NULL) { |
|
1144 q->_next = _EntryList; |
|
1145 _EntryList->_prev = q; |
|
1146 } |
|
1147 _EntryList = w; |
|
1148 |
|
1149 // Fall thru into code that tries to wake a successor from EntryList |
|
1150 } |
|
1151 |
|
1152 w = _EntryList; |
|
1153 if (w != NULL) { |
|
1154 // I'd like to write: guarantee (w->_thread != Self). |
|
1155 // But in practice an exiting thread may find itself on the EntryList. |
|
1156 // Let's say thread T1 calls O.wait(). Wait() enqueues T1 on O's waitset and |
|
1157 // then calls exit(). Exit release the lock by setting O._owner to NULL. |
|
1158 // Let's say T1 then stalls. T2 acquires O and calls O.notify(). The |
|
1159 // notify() operation moves T1 from O's waitset to O's EntryList. T2 then |
|
1160 // release the lock "O". T2 resumes immediately after the ST of null into |
|
1161 // _owner, above. T2 notices that the EntryList is populated, so it |
|
1162 // reacquires the lock and then finds itself on the EntryList. |
|
1163 // Given all that, we have to tolerate the circumstance where "w" is |
|
1164 // associated with Self. |
|
1165 assert(w->TState == ObjectWaiter::TS_ENTER, "invariant"); |
|
1166 ExitEpilog(Self, w); |
|
1167 return; |
|
1168 } |
|
1169 |
|
1170 // If we find that both _cxq and EntryList are null then just |
|
1171 // re-run the exit protocol from the top. |
|
1172 w = _cxq; |
1072 w = _cxq; |
1173 if (w == NULL) continue; |
1073 assert(w != NULL, "invariant"); |
1174 |
1074 assert(w->TState == ObjectWaiter::TS_CXQ, "Invariant"); |
|
1075 ExitEpilog(Self, w); |
|
1076 return; |
|
1077 } |
|
1078 |
|
1079 if (QMode == 3 && _cxq != NULL) { |
|
1080 // Aggressively drain cxq into EntryList at the first opportunity. |
|
1081 // This policy ensure that recently-run threads live at the head of EntryList. |
1175 // Drain _cxq into EntryList - bulk transfer. |
1082 // Drain _cxq into EntryList - bulk transfer. |
1176 // First, detach _cxq. |
1083 // First, detach _cxq. |
1177 // The following loop is tantamount to: w = swap (&cxq, NULL) |
1084 // The following loop is tantamount to: w = swap (&cxq, NULL) |
|
1085 w = _cxq; |
1178 for (;;) { |
1086 for (;;) { |
1179 assert(w != NULL, "Invariant"); |
1087 assert(w != NULL, "Invariant"); |
1180 ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr(NULL, &_cxq, w); |
1088 ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr(NULL, &_cxq, w); |
1181 if (u == w) break; |
1089 if (u == w) break; |
1182 w = u; |
1090 w = u; |
1183 } |
1091 } |
1184 TEVENT(Inflated exit - drain cxq into EntryList); |
|
1185 |
|
1186 assert(w != NULL , "invariant"); |
1092 assert(w != NULL , "invariant"); |
1187 assert(_EntryList == NULL , "invariant"); |
1093 |
1188 |
1094 ObjectWaiter * q = NULL; |
1189 // Convert the LIFO SLL anchored by _cxq into a DLL. |
1095 ObjectWaiter * p; |
1190 // The list reorganization step operates in O(LENGTH(w)) time. |
1096 for (p = w; p != NULL; p = p->_next) { |
1191 // It's critical that this step operate quickly as |
1097 guarantee(p->TState == ObjectWaiter::TS_CXQ, "Invariant"); |
1192 // "Self" still holds the outer-lock, restricting parallelism |
1098 p->TState = ObjectWaiter::TS_ENTER; |
1193 // and effectively lengthening the critical section. |
1099 p->_prev = q; |
1194 // Invariant: s chases t chases u. |
1100 q = p; |
1195 // TODO-FIXME: consider changing EntryList from a DLL to a CDLL so |
1101 } |
1196 // we have faster access to the tail. |
1102 |
1197 |
1103 // Append the RATs to the EntryList |
1198 if (QMode == 1) { |
1104 // TODO: organize EntryList as a CDLL so we can locate the tail in constant-time. |
1199 // QMode == 1 : drain cxq to EntryList, reversing order |
1105 ObjectWaiter * Tail; |
1200 // We also reverse the order of the list. |
1106 for (Tail = _EntryList; Tail != NULL && Tail->_next != NULL; Tail = Tail->_next); |
1201 ObjectWaiter * s = NULL; |
1107 if (Tail == NULL) { |
1202 ObjectWaiter * t = w; |
1108 _EntryList = w; |
1203 ObjectWaiter * u = NULL; |
|
1204 while (t != NULL) { |
|
1205 guarantee(t->TState == ObjectWaiter::TS_CXQ, "invariant"); |
|
1206 t->TState = ObjectWaiter::TS_ENTER; |
|
1207 u = t->_next; |
|
1208 t->_prev = u; |
|
1209 t->_next = s; |
|
1210 s = t; |
|
1211 t = u; |
|
1212 } |
|
1213 _EntryList = s; |
|
1214 assert(s != NULL, "invariant"); |
|
1215 } else { |
1109 } else { |
1216 // QMode == 0 or QMode == 2 |
1110 Tail->_next = w; |
1217 _EntryList = w; |
1111 w->_prev = Tail; |
1218 ObjectWaiter * q = NULL; |
1112 } |
1219 ObjectWaiter * p; |
1113 |
1220 for (p = w; p != NULL; p = p->_next) { |
1114 // Fall thru into code that tries to wake a successor from EntryList |
1221 guarantee(p->TState == ObjectWaiter::TS_CXQ, "Invariant"); |
1115 } |
1222 p->TState = ObjectWaiter::TS_ENTER; |
1116 |
1223 p->_prev = q; |
1117 if (QMode == 4 && _cxq != NULL) { |
1224 q = p; |
1118 // Aggressively drain cxq into EntryList at the first opportunity. |
1225 } |
1119 // This policy ensure that recently-run threads live at the head of EntryList. |
1226 } |
1120 |
1227 |
1121 // Drain _cxq into EntryList - bulk transfer. |
1228 // In 1-0 mode we need: ST EntryList; MEMBAR #storestore; ST _owner = NULL |
1122 // First, detach _cxq. |
1229 // The MEMBAR is satisfied by the release_store() operation in ExitEpilog(). |
1123 // The following loop is tantamount to: w = swap (&cxq, NULL) |
1230 |
1124 w = _cxq; |
1231 // See if we can abdicate to a spinner instead of waking a thread. |
1125 for (;;) { |
1232 // A primary goal of the implementation is to reduce the |
1126 assert(w != NULL, "Invariant"); |
1233 // context-switch rate. |
1127 ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr(NULL, &_cxq, w); |
1234 if (_succ != NULL) continue; |
1128 if (u == w) break; |
1235 |
1129 w = u; |
1236 w = _EntryList; |
1130 } |
1237 if (w != NULL) { |
1131 assert(w != NULL , "invariant"); |
1238 guarantee(w->TState == ObjectWaiter::TS_ENTER, "invariant"); |
1132 |
1239 ExitEpilog(Self, w); |
1133 ObjectWaiter * q = NULL; |
1240 return; |
1134 ObjectWaiter * p; |
1241 } |
1135 for (p = w; p != NULL; p = p->_next) { |
1242 } |
1136 guarantee(p->TState == ObjectWaiter::TS_CXQ, "Invariant"); |
|
1137 p->TState = ObjectWaiter::TS_ENTER; |
|
1138 p->_prev = q; |
|
1139 q = p; |
|
1140 } |
|
1141 |
|
1142 // Prepend the RATs to the EntryList |
|
1143 if (_EntryList != NULL) { |
|
1144 q->_next = _EntryList; |
|
1145 _EntryList->_prev = q; |
|
1146 } |
|
1147 _EntryList = w; |
|
1148 |
|
1149 // Fall thru into code that tries to wake a successor from EntryList |
|
1150 } |
|
1151 |
|
1152 w = _EntryList; |
|
1153 if (w != NULL) { |
|
1154 // I'd like to write: guarantee (w->_thread != Self). |
|
1155 // But in practice an exiting thread may find itself on the EntryList. |
|
1156 // Let's say thread T1 calls O.wait(). Wait() enqueues T1 on O's waitset and |
|
1157 // then calls exit(). Exit release the lock by setting O._owner to NULL. |
|
1158 // Let's say T1 then stalls. T2 acquires O and calls O.notify(). The |
|
1159 // notify() operation moves T1 from O's waitset to O's EntryList. T2 then |
|
1160 // release the lock "O". T2 resumes immediately after the ST of null into |
|
1161 // _owner, above. T2 notices that the EntryList is populated, so it |
|
1162 // reacquires the lock and then finds itself on the EntryList. |
|
1163 // Given all that, we have to tolerate the circumstance where "w" is |
|
1164 // associated with Self. |
|
1165 assert(w->TState == ObjectWaiter::TS_ENTER, "invariant"); |
|
1166 ExitEpilog(Self, w); |
|
1167 return; |
|
1168 } |
|
1169 |
|
1170 // If we find that both _cxq and EntryList are null then just |
|
1171 // re-run the exit protocol from the top. |
|
1172 w = _cxq; |
|
1173 if (w == NULL) continue; |
|
1174 |
|
1175 // Drain _cxq into EntryList - bulk transfer. |
|
1176 // First, detach _cxq. |
|
1177 // The following loop is tantamount to: w = swap (&cxq, NULL) |
|
1178 for (;;) { |
|
1179 assert(w != NULL, "Invariant"); |
|
1180 ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr(NULL, &_cxq, w); |
|
1181 if (u == w) break; |
|
1182 w = u; |
|
1183 } |
|
1184 TEVENT(Inflated exit - drain cxq into EntryList); |
|
1185 |
|
1186 assert(w != NULL , "invariant"); |
|
1187 assert(_EntryList == NULL , "invariant"); |
|
1188 |
|
1189 // Convert the LIFO SLL anchored by _cxq into a DLL. |
|
1190 // The list reorganization step operates in O(LENGTH(w)) time. |
|
1191 // It's critical that this step operate quickly as |
|
1192 // "Self" still holds the outer-lock, restricting parallelism |
|
1193 // and effectively lengthening the critical section. |
|
1194 // Invariant: s chases t chases u. |
|
1195 // TODO-FIXME: consider changing EntryList from a DLL to a CDLL so |
|
1196 // we have faster access to the tail. |
|
1197 |
|
1198 if (QMode == 1) { |
|
1199 // QMode == 1 : drain cxq to EntryList, reversing order |
|
1200 // We also reverse the order of the list. |
|
1201 ObjectWaiter * s = NULL; |
|
1202 ObjectWaiter * t = w; |
|
1203 ObjectWaiter * u = NULL; |
|
1204 while (t != NULL) { |
|
1205 guarantee(t->TState == ObjectWaiter::TS_CXQ, "invariant"); |
|
1206 t->TState = ObjectWaiter::TS_ENTER; |
|
1207 u = t->_next; |
|
1208 t->_prev = u; |
|
1209 t->_next = s; |
|
1210 s = t; |
|
1211 t = u; |
|
1212 } |
|
1213 _EntryList = s; |
|
1214 assert(s != NULL, "invariant"); |
|
1215 } else { |
|
1216 // QMode == 0 or QMode == 2 |
|
1217 _EntryList = w; |
|
1218 ObjectWaiter * q = NULL; |
|
1219 ObjectWaiter * p; |
|
1220 for (p = w; p != NULL; p = p->_next) { |
|
1221 guarantee(p->TState == ObjectWaiter::TS_CXQ, "Invariant"); |
|
1222 p->TState = ObjectWaiter::TS_ENTER; |
|
1223 p->_prev = q; |
|
1224 q = p; |
|
1225 } |
|
1226 } |
|
1227 |
|
1228 // In 1-0 mode we need: ST EntryList; MEMBAR #storestore; ST _owner = NULL |
|
1229 // The MEMBAR is satisfied by the release_store() operation in ExitEpilog(). |
|
1230 |
|
1231 // See if we can abdicate to a spinner instead of waking a thread. |
|
1232 // A primary goal of the implementation is to reduce the |
|
1233 // context-switch rate. |
|
1234 if (_succ != NULL) continue; |
|
1235 |
|
1236 w = _EntryList; |
|
1237 if (w != NULL) { |
|
1238 guarantee(w->TState == ObjectWaiter::TS_ENTER, "invariant"); |
|
1239 ExitEpilog(Self, w); |
|
1240 return; |
|
1241 } |
|
1242 } |
1243 } |
1243 } |
1244 |
1244 |
1245 // ExitSuspendEquivalent: |
1245 // ExitSuspendEquivalent: |
1246 // A faster alternate to handle_special_suspend_equivalent_condition() |
1246 // A faster alternate to handle_special_suspend_equivalent_condition() |
1247 // |
1247 // |
1427 // Wait/Notify/NotifyAll |
1427 // Wait/Notify/NotifyAll |
1428 // |
1428 // |
1429 // Note: a subset of changes to ObjectMonitor::wait() |
1429 // Note: a subset of changes to ObjectMonitor::wait() |
1430 // will need to be replicated in complete_exit |
1430 // will need to be replicated in complete_exit |
1431 void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { |
1431 void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { |
1432 Thread * const Self = THREAD; |
1432 Thread * const Self = THREAD; |
1433 assert(Self->is_Java_thread(), "Must be Java thread!"); |
1433 assert(Self->is_Java_thread(), "Must be Java thread!"); |
1434 JavaThread *jt = (JavaThread *)THREAD; |
1434 JavaThread *jt = (JavaThread *)THREAD; |
1435 |
1435 |
1436 DeferredInitialize(); |
1436 DeferredInitialize(); |
1437 |
1437 |
1438 // Throw IMSX or IEX. |
1438 // Throw IMSX or IEX. |
1439 CHECK_OWNER(); |
1439 CHECK_OWNER(); |
1440 |
1440 |
1441 EventJavaMonitorWait event; |
1441 EventJavaMonitorWait event; |
1442 |
1442 |
1443 // check for a pending interrupt |
1443 // check for a pending interrupt |
1444 if (interruptible && Thread::is_interrupted(Self, true) && !HAS_PENDING_EXCEPTION) { |
1444 if (interruptible && Thread::is_interrupted(Self, true) && !HAS_PENDING_EXCEPTION) { |
1445 // post monitor waited event. Note that this is past-tense, we are done waiting. |
1445 // post monitor waited event. Note that this is past-tense, we are done waiting. |
1446 if (JvmtiExport::should_post_monitor_waited()) { |
1446 if (JvmtiExport::should_post_monitor_waited()) { |
1447 // Note: 'false' parameter is passed here because the |
1447 // Note: 'false' parameter is passed here because the |
1448 // wait was not timed out due to thread interrupt. |
1448 // wait was not timed out due to thread interrupt. |
1449 JvmtiExport::post_monitor_waited(jt, this, false); |
1449 JvmtiExport::post_monitor_waited(jt, this, false); |
1450 |
1450 |
1451 // In this short circuit of the monitor wait protocol, the |
1451 // In this short circuit of the monitor wait protocol, the |
1452 // current thread never drops ownership of the monitor and |
1452 // current thread never drops ownership of the monitor and |
1453 // never gets added to the wait queue so the current thread |
1453 // never gets added to the wait queue so the current thread |
1454 // cannot be made the successor. This means that the |
1454 // cannot be made the successor. This means that the |
1455 // JVMTI_EVENT_MONITOR_WAITED event handler cannot accidentally |
1455 // JVMTI_EVENT_MONITOR_WAITED event handler cannot accidentally |
1456 // consume an unpark() meant for the ParkEvent associated with |
1456 // consume an unpark() meant for the ParkEvent associated with |
1457 // this ObjectMonitor. |
1457 // this ObjectMonitor. |
1458 } |
1458 } |
1459 if (event.should_commit()) { |
1459 if (event.should_commit()) { |
1460 post_monitor_wait_event(&event, 0, millis, false); |
1460 post_monitor_wait_event(&event, 0, millis, false); |
1461 } |
1461 } |
1462 TEVENT(Wait - Throw IEX); |
1462 TEVENT(Wait - Throw IEX); |
1463 THROW(vmSymbols::java_lang_InterruptedException()); |
1463 THROW(vmSymbols::java_lang_InterruptedException()); |
1464 return; |
1464 return; |
1465 } |
1465 } |
1466 |
1466 |
1467 TEVENT(Wait); |
1467 TEVENT(Wait); |
1468 |
1468 |
1469 assert(Self->_Stalled == 0, "invariant"); |
1469 assert(Self->_Stalled == 0, "invariant"); |
1470 Self->_Stalled = intptr_t(this); |
1470 Self->_Stalled = intptr_t(this); |
1471 jt->set_current_waiting_monitor(this); |
1471 jt->set_current_waiting_monitor(this); |
1472 |
1472 |
1473 // create a node to be put into the queue |
1473 // create a node to be put into the queue |
1474 // Critically, after we reset() the event but prior to park(), we must check |
1474 // Critically, after we reset() the event but prior to park(), we must check |
1475 // for a pending interrupt. |
1475 // for a pending interrupt. |
1476 ObjectWaiter node(Self); |
1476 ObjectWaiter node(Self); |
1477 node.TState = ObjectWaiter::TS_WAIT; |
1477 node.TState = ObjectWaiter::TS_WAIT; |
1478 Self->_ParkEvent->reset(); |
1478 Self->_ParkEvent->reset(); |
1479 OrderAccess::fence(); // ST into Event; membar ; LD interrupted-flag |
1479 OrderAccess::fence(); // ST into Event; membar ; LD interrupted-flag |
1480 |
1480 |
1481 // Enter the waiting queue, which is a circular doubly linked list in this case |
1481 // Enter the waiting queue, which is a circular doubly linked list in this case |
1482 // but it could be a priority queue or any data structure. |
1482 // but it could be a priority queue or any data structure. |
1483 // _WaitSetLock protects the wait queue. Normally the wait queue is accessed only |
1483 // _WaitSetLock protects the wait queue. Normally the wait queue is accessed only |
1484 // by the the owner of the monitor *except* in the case where park() |
1484 // by the the owner of the monitor *except* in the case where park() |
1485 // returns because of a timeout of interrupt. Contention is exceptionally rare |
1485 // returns because of a timeout of interrupt. Contention is exceptionally rare |
1486 // so we use a simple spin-lock instead of a heavier-weight blocking lock. |
1486 // so we use a simple spin-lock instead of a heavier-weight blocking lock. |
1487 |
1487 |
1488 Thread::SpinAcquire(&_WaitSetLock, "WaitSet - add"); |
1488 Thread::SpinAcquire(&_WaitSetLock, "WaitSet - add"); |
1489 AddWaiter(&node); |
1489 AddWaiter(&node); |
1490 Thread::SpinRelease(&_WaitSetLock); |
1490 Thread::SpinRelease(&_WaitSetLock); |
1491 |
1491 |
1492 if ((SyncFlags & 4) == 0) { |
1492 if ((SyncFlags & 4) == 0) { |
1493 _Responsible = NULL; |
1493 _Responsible = NULL; |
1494 } |
1494 } |
1495 intptr_t save = _recursions; // record the old recursion count |
1495 intptr_t save = _recursions; // record the old recursion count |
1496 _waiters++; // increment the number of waiters |
1496 _waiters++; // increment the number of waiters |
1497 _recursions = 0; // set the recursion level to be 1 |
1497 _recursions = 0; // set the recursion level to be 1 |
1498 exit(true, Self); // exit the monitor |
1498 exit(true, Self); // exit the monitor |
1499 guarantee(_owner != Self, "invariant"); |
1499 guarantee(_owner != Self, "invariant"); |
1500 |
1500 |
1501 // The thread is on the WaitSet list - now park() it. |
1501 // The thread is on the WaitSet list - now park() it. |
1502 // On MP systems it's conceivable that a brief spin before we park |
1502 // On MP systems it's conceivable that a brief spin before we park |
1503 // could be profitable. |
1503 // could be profitable. |
1504 // |
1504 // |
1505 // TODO-FIXME: change the following logic to a loop of the form |
1505 // TODO-FIXME: change the following logic to a loop of the form |
1506 // while (!timeout && !interrupted && _notified == 0) park() |
1506 // while (!timeout && !interrupted && _notified == 0) park() |
1507 |
1507 |
1508 int ret = OS_OK; |
1508 int ret = OS_OK; |
1509 int WasNotified = 0; |
1509 int WasNotified = 0; |
1510 { // State transition wrappers |
1510 { // State transition wrappers |
1511 OSThread* osthread = Self->osthread(); |
1511 OSThread* osthread = Self->osthread(); |
1512 OSThreadWaitState osts(osthread, true); |
1512 OSThreadWaitState osts(osthread, true); |
1513 { |
1513 { |
1514 ThreadBlockInVM tbivm(jt); |
1514 ThreadBlockInVM tbivm(jt); |
1515 // Thread is in thread_blocked state and oop access is unsafe. |
1515 // Thread is in thread_blocked state and oop access is unsafe. |
1516 jt->set_suspend_equivalent(); |
1516 jt->set_suspend_equivalent(); |
1517 |
1517 |
1518 if (interruptible && (Thread::is_interrupted(THREAD, false) || HAS_PENDING_EXCEPTION)) { |
1518 if (interruptible && (Thread::is_interrupted(THREAD, false) || HAS_PENDING_EXCEPTION)) { |
1519 // Intentionally empty |
1519 // Intentionally empty |
1520 } else |
1520 } else |
1521 if (node._notified == 0) { |
1521 if (node._notified == 0) { |
1522 if (millis <= 0) { |
1522 if (millis <= 0) { |
1523 Self->_ParkEvent->park(); |
1523 Self->_ParkEvent->park(); |
1524 } else { |
1524 } else { |
1525 ret = Self->_ParkEvent->park(millis); |
1525 ret = Self->_ParkEvent->park(millis); |
1526 } |
1526 } |
1527 } |
1527 } |
1528 |
1528 |
1529 // were we externally suspended while we were waiting? |
1529 // were we externally suspended while we were waiting? |
1530 if (ExitSuspendEquivalent (jt)) { |
1530 if (ExitSuspendEquivalent (jt)) { |
1531 // TODO-FIXME: add -- if succ == Self then succ = null. |
1531 // TODO-FIXME: add -- if succ == Self then succ = null. |
1532 jt->java_suspend_self(); |
1532 jt->java_suspend_self(); |
1533 } |
1533 } |
1534 |
1534 |
1535 } // Exit thread safepoint: transition _thread_blocked -> _thread_in_vm |
1535 } // Exit thread safepoint: transition _thread_blocked -> _thread_in_vm |
1536 |
1536 |
1537 |
1537 |
1538 // Node may be on the WaitSet, the EntryList (or cxq), or in transition |
1538 // Node may be on the WaitSet, the EntryList (or cxq), or in transition |
1539 // from the WaitSet to the EntryList. |
1539 // from the WaitSet to the EntryList. |
1540 // See if we need to remove Node from the WaitSet. |
1540 // See if we need to remove Node from the WaitSet. |
1541 // We use double-checked locking to avoid grabbing _WaitSetLock |
1541 // We use double-checked locking to avoid grabbing _WaitSetLock |
1542 // if the thread is not on the wait queue. |
1542 // if the thread is not on the wait queue. |
1543 // |
1543 // |
1544 // Note that we don't need a fence before the fetch of TState. |
1544 // Note that we don't need a fence before the fetch of TState. |
1545 // In the worst case we'll fetch a old-stale value of TS_WAIT previously |
1545 // In the worst case we'll fetch a old-stale value of TS_WAIT previously |
1546 // written by the is thread. (perhaps the fetch might even be satisfied |
1546 // written by the is thread. (perhaps the fetch might even be satisfied |
1547 // by a look-aside into the processor's own store buffer, although given |
1547 // by a look-aside into the processor's own store buffer, although given |
1548 // the length of the code path between the prior ST and this load that's |
1548 // the length of the code path between the prior ST and this load that's |
1549 // highly unlikely). If the following LD fetches a stale TS_WAIT value |
1549 // highly unlikely). If the following LD fetches a stale TS_WAIT value |
1550 // then we'll acquire the lock and then re-fetch a fresh TState value. |
1550 // then we'll acquire the lock and then re-fetch a fresh TState value. |
1551 // That is, we fail toward safety. |
1551 // That is, we fail toward safety. |
1552 |
1552 |
1553 if (node.TState == ObjectWaiter::TS_WAIT) { |
1553 if (node.TState == ObjectWaiter::TS_WAIT) { |
1554 Thread::SpinAcquire(&_WaitSetLock, "WaitSet - unlink"); |
1554 Thread::SpinAcquire(&_WaitSetLock, "WaitSet - unlink"); |
1555 if (node.TState == ObjectWaiter::TS_WAIT) { |
1555 if (node.TState == ObjectWaiter::TS_WAIT) { |
1556 DequeueSpecificWaiter(&node); // unlink from WaitSet |
1556 DequeueSpecificWaiter(&node); // unlink from WaitSet |
1557 assert(node._notified == 0, "invariant"); |
1557 assert(node._notified == 0, "invariant"); |
1558 node.TState = ObjectWaiter::TS_RUN; |
1558 node.TState = ObjectWaiter::TS_RUN; |
1559 } |
1559 } |
1560 Thread::SpinRelease(&_WaitSetLock); |
1560 Thread::SpinRelease(&_WaitSetLock); |
1561 } |
1561 } |
1562 |
1562 |
1563 // The thread is now either on off-list (TS_RUN), |
1563 // The thread is now either on off-list (TS_RUN), |
1564 // on the EntryList (TS_ENTER), or on the cxq (TS_CXQ). |
1564 // on the EntryList (TS_ENTER), or on the cxq (TS_CXQ). |
1565 // The Node's TState variable is stable from the perspective of this thread. |
1565 // The Node's TState variable is stable from the perspective of this thread. |
1566 // No other threads will asynchronously modify TState. |
1566 // No other threads will asynchronously modify TState. |
1567 guarantee(node.TState != ObjectWaiter::TS_WAIT, "invariant"); |
1567 guarantee(node.TState != ObjectWaiter::TS_WAIT, "invariant"); |
1568 OrderAccess::loadload(); |
1568 OrderAccess::loadload(); |
1569 if (_succ == Self) _succ = NULL; |
1569 if (_succ == Self) _succ = NULL; |
1570 WasNotified = node._notified; |
1570 WasNotified = node._notified; |
1571 |
1571 |
1572 // Reentry phase -- reacquire the monitor. |
1572 // Reentry phase -- reacquire the monitor. |
1573 // re-enter contended monitor after object.wait(). |
1573 // re-enter contended monitor after object.wait(). |
1574 // retain OBJECT_WAIT state until re-enter successfully completes |
1574 // retain OBJECT_WAIT state until re-enter successfully completes |
1575 // Thread state is thread_in_vm and oop access is again safe, |
1575 // Thread state is thread_in_vm and oop access is again safe, |
1576 // although the raw address of the object may have changed. |
1576 // although the raw address of the object may have changed. |
1577 // (Don't cache naked oops over safepoints, of course). |
1577 // (Don't cache naked oops over safepoints, of course). |
1578 |
1578 |
1579 // post monitor waited event. Note that this is past-tense, we are done waiting. |
1579 // post monitor waited event. Note that this is past-tense, we are done waiting. |
1580 if (JvmtiExport::should_post_monitor_waited()) { |
1580 if (JvmtiExport::should_post_monitor_waited()) { |
1581 JvmtiExport::post_monitor_waited(jt, this, ret == OS_TIMEOUT); |
1581 JvmtiExport::post_monitor_waited(jt, this, ret == OS_TIMEOUT); |
1582 |
1582 |
1583 if (node._notified != 0 && _succ == Self) { |
1583 if (node._notified != 0 && _succ == Self) { |
1584 // In this part of the monitor wait-notify-reenter protocol it |
1584 // In this part of the monitor wait-notify-reenter protocol it |
1585 // is possible (and normal) for another thread to do a fastpath |
1585 // is possible (and normal) for another thread to do a fastpath |
1586 // monitor enter-exit while this thread is still trying to get |
1586 // monitor enter-exit while this thread is still trying to get |
1587 // to the reenter portion of the protocol. |
1587 // to the reenter portion of the protocol. |
1588 // |
1588 // |
1589 // The ObjectMonitor was notified and the current thread is |
1589 // The ObjectMonitor was notified and the current thread is |
1590 // the successor which also means that an unpark() has already |
1590 // the successor which also means that an unpark() has already |
1591 // been done. The JVMTI_EVENT_MONITOR_WAITED event handler can |
1591 // been done. The JVMTI_EVENT_MONITOR_WAITED event handler can |
1592 // consume the unpark() that was done when the successor was |
1592 // consume the unpark() that was done when the successor was |
1593 // set because the same ParkEvent is shared between Java |
1593 // set because the same ParkEvent is shared between Java |
1594 // monitors and JVM/TI RawMonitors (for now). |
1594 // monitors and JVM/TI RawMonitors (for now). |
1595 // |
1595 // |
1596 // We redo the unpark() to ensure forward progress, i.e., we |
1596 // We redo the unpark() to ensure forward progress, i.e., we |
1597 // don't want all pending threads hanging (parked) with none |
1597 // don't want all pending threads hanging (parked) with none |
1598 // entering the unlocked monitor. |
1598 // entering the unlocked monitor. |
1599 node._event->unpark(); |
1599 node._event->unpark(); |
1600 } |
1600 } |
1601 } |
1601 } |
1602 |
1602 |
1603 if (event.should_commit()) { |
1603 if (event.should_commit()) { |
1604 post_monitor_wait_event(&event, node._notifier_tid, millis, ret == OS_TIMEOUT); |
1604 post_monitor_wait_event(&event, node._notifier_tid, millis, ret == OS_TIMEOUT); |
1605 } |
1605 } |
1606 |
1606 |
1607 OrderAccess::fence(); |
1607 OrderAccess::fence(); |
1608 |
1608 |
1609 assert(Self->_Stalled != 0, "invariant"); |
1609 assert(Self->_Stalled != 0, "invariant"); |
1610 Self->_Stalled = 0; |
1610 Self->_Stalled = 0; |
1611 |
1611 |
1612 assert(_owner != Self, "invariant"); |
1612 assert(_owner != Self, "invariant"); |
1613 ObjectWaiter::TStates v = node.TState; |
1613 ObjectWaiter::TStates v = node.TState; |
1614 if (v == ObjectWaiter::TS_RUN) { |
1614 if (v == ObjectWaiter::TS_RUN) { |
1615 enter(Self); |
1615 enter(Self); |
1616 } else { |
1616 } else { |
1617 guarantee(v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant"); |
1617 guarantee(v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant"); |
1618 ReenterI(Self, &node); |
1618 ReenterI(Self, &node); |
1619 node.wait_reenter_end(this); |
1619 node.wait_reenter_end(this); |
1620 } |
1620 } |
1621 |
1621 |
1622 // Self has reacquired the lock. |
1622 // Self has reacquired the lock. |
1623 // Lifecycle - the node representing Self must not appear on any queues. |
1623 // Lifecycle - the node representing Self must not appear on any queues. |
1624 // Node is about to go out-of-scope, but even if it were immortal we wouldn't |
1624 // Node is about to go out-of-scope, but even if it were immortal we wouldn't |
1625 // want residual elements associated with this thread left on any lists. |
1625 // want residual elements associated with this thread left on any lists. |
1626 guarantee(node.TState == ObjectWaiter::TS_RUN, "invariant"); |
1626 guarantee(node.TState == ObjectWaiter::TS_RUN, "invariant"); |
1627 assert(_owner == Self, "invariant"); |
1627 assert(_owner == Self, "invariant"); |
1628 assert(_succ != Self , "invariant"); |
1628 assert(_succ != Self , "invariant"); |
1629 } // OSThreadWaitState() |
1629 } // OSThreadWaitState() |
1630 |
1630 |
1631 jt->set_current_waiting_monitor(NULL); |
1631 jt->set_current_waiting_monitor(NULL); |
1632 |
1632 |
1633 guarantee(_recursions == 0, "invariant"); |
1633 guarantee(_recursions == 0, "invariant"); |
1634 _recursions = save; // restore the old recursion count |
1634 _recursions = save; // restore the old recursion count |
1635 _waiters--; // decrement the number of waiters |
1635 _waiters--; // decrement the number of waiters |
1636 |
1636 |
1637 // Verify a few postconditions |
1637 // Verify a few postconditions |
1638 assert(_owner == Self , "invariant"); |
1638 assert(_owner == Self , "invariant"); |
1639 assert(_succ != Self , "invariant"); |
1639 assert(_succ != Self , "invariant"); |
1640 assert(((oop)(object()))->mark() == markOopDesc::encode(this), "invariant"); |
1640 assert(((oop)(object()))->mark() == markOopDesc::encode(this), "invariant"); |
1641 |
1641 |
1642 if (SyncFlags & 32) { |
1642 if (SyncFlags & 32) { |
1643 OrderAccess::fence(); |
1643 OrderAccess::fence(); |
1644 } |
1644 } |
1645 |
1645 |
1646 // check if the notification happened |
1646 // check if the notification happened |
1647 if (!WasNotified) { |
1647 if (!WasNotified) { |
1648 // no, it could be timeout or Thread.interrupt() or both |
1648 // no, it could be timeout or Thread.interrupt() or both |
1649 // check for interrupt event, otherwise it is timeout |
1649 // check for interrupt event, otherwise it is timeout |
1650 if (interruptible && Thread::is_interrupted(Self, true) && !HAS_PENDING_EXCEPTION) { |
1650 if (interruptible && Thread::is_interrupted(Self, true) && !HAS_PENDING_EXCEPTION) { |
1651 TEVENT(Wait - throw IEX from epilog); |
1651 TEVENT(Wait - throw IEX from epilog); |
1652 THROW(vmSymbols::java_lang_InterruptedException()); |
1652 THROW(vmSymbols::java_lang_InterruptedException()); |
1653 } |
1653 } |
1654 } |
1654 } |
1655 |
1655 |
1656 // NOTE: Spurious wake up will be consider as timeout. |
1656 // NOTE: Spurious wake up will be consider as timeout. |
1657 // Monitor notify has precedence over thread interrupt. |
1657 // Monitor notify has precedence over thread interrupt. |
1658 } |
1658 } |
1659 |
1659 |
1660 |
1660 |
1661 // Consider: |
1661 // Consider: |
1662 // If the lock is cool (cxq == null && succ == null) and we're on an MP system |
1662 // If the lock is cool (cxq == null && succ == null) and we're on an MP system |
1664 // we might just dequeue a thread from the WaitSet and directly unpark() it. |
1664 // we might just dequeue a thread from the WaitSet and directly unpark() it. |
1665 |
1665 |
1666 void ObjectMonitor::notify(TRAPS) { |
1666 void ObjectMonitor::notify(TRAPS) { |
1667 CHECK_OWNER(); |
1667 CHECK_OWNER(); |
1668 if (_WaitSet == NULL) { |
1668 if (_WaitSet == NULL) { |
1669 TEVENT(Empty-Notify); |
1669 TEVENT(Empty-Notify); |
1670 return; |
1670 return; |
1671 } |
1671 } |
1672 DTRACE_MONITOR_PROBE(notify, this, object(), THREAD); |
1672 DTRACE_MONITOR_PROBE(notify, this, object(), THREAD); |
1673 |
1673 |
1674 int Policy = Knob_MoveNotifyee; |
1674 int Policy = Knob_MoveNotifyee; |
1675 |
1675 |
1676 Thread::SpinAcquire(&_WaitSetLock, "WaitSet - notify"); |
1676 Thread::SpinAcquire(&_WaitSetLock, "WaitSet - notify"); |
1677 ObjectWaiter * iterator = DequeueWaiter(); |
1677 ObjectWaiter * iterator = DequeueWaiter(); |
1678 if (iterator != NULL) { |
1678 if (iterator != NULL) { |
1679 TEVENT(Notify1 - Transfer); |
1679 TEVENT(Notify1 - Transfer); |
1680 guarantee(iterator->TState == ObjectWaiter::TS_WAIT, "invariant"); |
1680 guarantee(iterator->TState == ObjectWaiter::TS_WAIT, "invariant"); |
1681 guarantee(iterator->_notified == 0, "invariant"); |
1681 guarantee(iterator->_notified == 0, "invariant"); |
1682 if (Policy != 4) { |
1682 if (Policy != 4) { |
1683 iterator->TState = ObjectWaiter::TS_ENTER; |
1683 iterator->TState = ObjectWaiter::TS_ENTER; |
1684 } |
1684 } |
1685 iterator->_notified = 1; |
1685 iterator->_notified = 1; |
1686 Thread * Self = THREAD; |
1686 Thread * Self = THREAD; |
1687 iterator->_notifier_tid = Self->osthread()->thread_id(); |
1687 iterator->_notifier_tid = Self->osthread()->thread_id(); |
1688 |
1688 |
1689 ObjectWaiter * List = _EntryList; |
1689 ObjectWaiter * List = _EntryList; |
1690 if (List != NULL) { |
1690 if (List != NULL) { |
1691 assert(List->_prev == NULL, "invariant"); |
1691 assert(List->_prev == NULL, "invariant"); |
1692 assert(List->TState == ObjectWaiter::TS_ENTER, "invariant"); |
1692 assert(List->TState == ObjectWaiter::TS_ENTER, "invariant"); |
1693 assert(List != iterator, "invariant"); |
1693 assert(List != iterator, "invariant"); |
1694 } |
1694 } |
1695 |
1695 |
1696 if (Policy == 0) { // prepend to EntryList |
1696 if (Policy == 0) { // prepend to EntryList |
1697 if (List == NULL) { |
1697 if (List == NULL) { |
1698 iterator->_next = iterator->_prev = NULL; |
1698 iterator->_next = iterator->_prev = NULL; |
1699 _EntryList = iterator; |
1699 _EntryList = iterator; |
1700 } else { |
1700 } else { |
1701 List->_prev = iterator; |
1701 List->_prev = iterator; |
1702 iterator->_next = List; |
1702 iterator->_next = List; |
1703 iterator->_prev = NULL; |
1703 iterator->_prev = NULL; |
1704 _EntryList = iterator; |
1704 _EntryList = iterator; |
1705 } |
1705 } |
1706 } else |
1706 } else |
1707 if (Policy == 1) { // append to EntryList |
1707 if (Policy == 1) { // append to EntryList |
1708 if (List == NULL) { |
1708 if (List == NULL) { |
1709 iterator->_next = iterator->_prev = NULL; |
1709 iterator->_next = iterator->_prev = NULL; |
1710 _EntryList = iterator; |
1710 _EntryList = iterator; |
1711 } else { |
1711 } else { |
1712 // CONSIDER: finding the tail currently requires a linear-time walk of |
1712 // CONSIDER: finding the tail currently requires a linear-time walk of |
1713 // the EntryList. We can make tail access constant-time by converting to |
1713 // the EntryList. We can make tail access constant-time by converting to |
1714 // a CDLL instead of using our current DLL. |
1714 // a CDLL instead of using our current DLL. |
1715 ObjectWaiter * Tail; |
1715 ObjectWaiter * Tail; |
1716 for (Tail = List; Tail->_next != NULL; Tail = Tail->_next); |
1716 for (Tail = List; Tail->_next != NULL; Tail = Tail->_next); |
1717 assert(Tail != NULL && Tail->_next == NULL, "invariant"); |
1717 assert(Tail != NULL && Tail->_next == NULL, "invariant"); |
1718 Tail->_next = iterator; |
1718 Tail->_next = iterator; |
1719 iterator->_prev = Tail; |
1719 iterator->_prev = Tail; |
1720 iterator->_next = NULL; |
1720 iterator->_next = NULL; |
1721 } |
1721 } |
1722 } else |
1722 } else |
1723 if (Policy == 2) { // prepend to cxq |
1723 if (Policy == 2) { // prepend to cxq |
1724 // prepend to cxq |
1724 // prepend to cxq |
1725 if (List == NULL) { |
1725 if (List == NULL) { |
1726 iterator->_next = iterator->_prev = NULL; |
1726 iterator->_next = iterator->_prev = NULL; |
1727 _EntryList = iterator; |
1727 _EntryList = iterator; |
1728 } else { |
1728 } else { |
1729 iterator->TState = ObjectWaiter::TS_CXQ; |
|
1730 for (;;) { |
|
1731 ObjectWaiter * Front = _cxq; |
|
1732 iterator->_next = Front; |
|
1733 if (Atomic::cmpxchg_ptr (iterator, &_cxq, Front) == Front) { |
|
1734 break; |
|
1735 } |
|
1736 } |
|
1737 } |
|
1738 } else |
|
1739 if (Policy == 3) { // append to cxq |
|
1740 iterator->TState = ObjectWaiter::TS_CXQ; |
1729 iterator->TState = ObjectWaiter::TS_CXQ; |
1741 for (;;) { |
1730 for (;;) { |
1742 ObjectWaiter * Tail; |
1731 ObjectWaiter * Front = _cxq; |
1743 Tail = _cxq; |
1732 iterator->_next = Front; |
1744 if (Tail == NULL) { |
1733 if (Atomic::cmpxchg_ptr (iterator, &_cxq, Front) == Front) { |
1745 iterator->_next = NULL; |
1734 break; |
1746 if (Atomic::cmpxchg_ptr (iterator, &_cxq, NULL) == NULL) { |
1735 } |
1747 break; |
|
1748 } |
|
1749 } else { |
|
1750 while (Tail->_next != NULL) Tail = Tail->_next; |
|
1751 Tail->_next = iterator; |
|
1752 iterator->_prev = Tail; |
|
1753 iterator->_next = NULL; |
|
1754 break; |
|
1755 } |
|
1756 } |
1736 } |
1757 } else { |
1737 } |
1758 ParkEvent * ev = iterator->_event; |
1738 } else |
1759 iterator->TState = ObjectWaiter::TS_RUN; |
1739 if (Policy == 3) { // append to cxq |
1760 OrderAccess::fence(); |
1740 iterator->TState = ObjectWaiter::TS_CXQ; |
1761 ev->unpark(); |
1741 for (;;) { |
1762 } |
1742 ObjectWaiter * Tail; |
1763 |
1743 Tail = _cxq; |
1764 if (Policy < 4) { |
1744 if (Tail == NULL) { |
1765 iterator->wait_reenter_begin(this); |
1745 iterator->_next = NULL; |
1766 } |
1746 if (Atomic::cmpxchg_ptr (iterator, &_cxq, NULL) == NULL) { |
1767 |
1747 break; |
1768 // _WaitSetLock protects the wait queue, not the EntryList. We could |
1748 } |
1769 // move the add-to-EntryList operation, above, outside the critical section |
1749 } else { |
1770 // protected by _WaitSetLock. In practice that's not useful. With the |
1750 while (Tail->_next != NULL) Tail = Tail->_next; |
1771 // exception of wait() timeouts and interrupts the monitor owner |
1751 Tail->_next = iterator; |
1772 // is the only thread that grabs _WaitSetLock. There's almost no contention |
1752 iterator->_prev = Tail; |
1773 // on _WaitSetLock so it's not profitable to reduce the length of the |
1753 iterator->_next = NULL; |
1774 // critical section. |
1754 break; |
|
1755 } |
|
1756 } |
|
1757 } else { |
|
1758 ParkEvent * ev = iterator->_event; |
|
1759 iterator->TState = ObjectWaiter::TS_RUN; |
|
1760 OrderAccess::fence(); |
|
1761 ev->unpark(); |
|
1762 } |
|
1763 |
|
1764 if (Policy < 4) { |
|
1765 iterator->wait_reenter_begin(this); |
|
1766 } |
|
1767 |
|
1768 // _WaitSetLock protects the wait queue, not the EntryList. We could |
|
1769 // move the add-to-EntryList operation, above, outside the critical section |
|
1770 // protected by _WaitSetLock. In practice that's not useful. With the |
|
1771 // exception of wait() timeouts and interrupts the monitor owner |
|
1772 // is the only thread that grabs _WaitSetLock. There's almost no contention |
|
1773 // on _WaitSetLock so it's not profitable to reduce the length of the |
|
1774 // critical section. |
1775 } |
1775 } |
1776 |
1776 |
1777 Thread::SpinRelease(&_WaitSetLock); |
1777 Thread::SpinRelease(&_WaitSetLock); |
1778 |
1778 |
1779 if (iterator != NULL && ObjectMonitor::_sync_Notifications != NULL) { |
1779 if (iterator != NULL && ObjectMonitor::_sync_Notifications != NULL) { |
1780 ObjectMonitor::_sync_Notifications->inc(); |
1780 ObjectMonitor::_sync_Notifications->inc(); |
1781 } |
1781 } |
1782 } |
1782 } |
1783 |
1783 |
1784 |
1784 |
1785 void ObjectMonitor::notifyAll(TRAPS) { |
1785 void ObjectMonitor::notifyAll(TRAPS) { |
1786 CHECK_OWNER(); |
1786 CHECK_OWNER(); |
1787 ObjectWaiter* iterator; |
1787 ObjectWaiter* iterator; |
1788 if (_WaitSet == NULL) { |
1788 if (_WaitSet == NULL) { |
1789 TEVENT(Empty-NotifyAll); |
1789 TEVENT(Empty-NotifyAll); |
1790 return; |
1790 return; |
1791 } |
1791 } |
1792 DTRACE_MONITOR_PROBE(notifyAll, this, object(), THREAD); |
1792 DTRACE_MONITOR_PROBE(notifyAll, this, object(), THREAD); |
1793 |
1793 |
1794 int Policy = Knob_MoveNotifyee; |
1794 int Policy = Knob_MoveNotifyee; |
1795 int Tally = 0; |
1795 int Tally = 0; |
1796 Thread::SpinAcquire(&_WaitSetLock, "WaitSet - notifyall"); |
1796 Thread::SpinAcquire(&_WaitSetLock, "WaitSet - notifyall"); |
1797 |
1797 |
1798 for (;;) { |
1798 for (;;) { |
1799 iterator = DequeueWaiter(); |
1799 iterator = DequeueWaiter(); |
1800 if (iterator == NULL) break; |
1800 if (iterator == NULL) break; |
1801 TEVENT(NotifyAll - Transfer1); |
1801 TEVENT(NotifyAll - Transfer1); |
1802 ++Tally; |
1802 ++Tally; |
1803 |
1803 |
1804 // Disposition - what might we do with iterator ? |
1804 // Disposition - what might we do with iterator ? |
1805 // a. add it directly to the EntryList - either tail or head. |
1805 // a. add it directly to the EntryList - either tail or head. |
1806 // b. push it onto the front of the _cxq. |
1806 // b. push it onto the front of the _cxq. |
1807 // For now we use (a). |
1807 // For now we use (a). |
1808 |
1808 |
1809 guarantee(iterator->TState == ObjectWaiter::TS_WAIT, "invariant"); |
1809 guarantee(iterator->TState == ObjectWaiter::TS_WAIT, "invariant"); |
1810 guarantee(iterator->_notified == 0, "invariant"); |
1810 guarantee(iterator->_notified == 0, "invariant"); |
1811 iterator->_notified = 1; |
1811 iterator->_notified = 1; |
1812 Thread * Self = THREAD; |
1812 Thread * Self = THREAD; |
1813 iterator->_notifier_tid = Self->osthread()->thread_id(); |
1813 iterator->_notifier_tid = Self->osthread()->thread_id(); |
1814 if (Policy != 4) { |
1814 if (Policy != 4) { |
1815 iterator->TState = ObjectWaiter::TS_ENTER; |
1815 iterator->TState = ObjectWaiter::TS_ENTER; |
1816 } |
1816 } |
1817 |
1817 |
1818 ObjectWaiter * List = _EntryList; |
1818 ObjectWaiter * List = _EntryList; |
1819 if (List != NULL) { |
1819 if (List != NULL) { |
1820 assert(List->_prev == NULL, "invariant"); |
1820 assert(List->_prev == NULL, "invariant"); |
1821 assert(List->TState == ObjectWaiter::TS_ENTER, "invariant"); |
1821 assert(List->TState == ObjectWaiter::TS_ENTER, "invariant"); |
1822 assert(List != iterator, "invariant"); |
1822 assert(List != iterator, "invariant"); |
1823 } |
1823 } |
1824 |
1824 |
1825 if (Policy == 0) { // prepend to EntryList |
1825 if (Policy == 0) { // prepend to EntryList |
1826 if (List == NULL) { |
1826 if (List == NULL) { |
1827 iterator->_next = iterator->_prev = NULL; |
1827 iterator->_next = iterator->_prev = NULL; |
1828 _EntryList = iterator; |
1828 _EntryList = iterator; |
1829 } else { |
1829 } else { |
1830 List->_prev = iterator; |
1830 List->_prev = iterator; |
1831 iterator->_next = List; |
1831 iterator->_next = List; |
1832 iterator->_prev = NULL; |
1832 iterator->_prev = NULL; |
1833 _EntryList = iterator; |
1833 _EntryList = iterator; |
|
1834 } |
|
1835 } else |
|
1836 if (Policy == 1) { // append to EntryList |
|
1837 if (List == NULL) { |
|
1838 iterator->_next = iterator->_prev = NULL; |
|
1839 _EntryList = iterator; |
|
1840 } else { |
|
1841 // CONSIDER: finding the tail currently requires a linear-time walk of |
|
1842 // the EntryList. We can make tail access constant-time by converting to |
|
1843 // a CDLL instead of using our current DLL. |
|
1844 ObjectWaiter * Tail; |
|
1845 for (Tail = List; Tail->_next != NULL; Tail = Tail->_next); |
|
1846 assert(Tail != NULL && Tail->_next == NULL, "invariant"); |
|
1847 Tail->_next = iterator; |
|
1848 iterator->_prev = Tail; |
|
1849 iterator->_next = NULL; |
|
1850 } |
|
1851 } else |
|
1852 if (Policy == 2) { // prepend to cxq |
|
1853 // prepend to cxq |
|
1854 iterator->TState = ObjectWaiter::TS_CXQ; |
|
1855 for (;;) { |
|
1856 ObjectWaiter * Front = _cxq; |
|
1857 iterator->_next = Front; |
|
1858 if (Atomic::cmpxchg_ptr (iterator, &_cxq, Front) == Front) { |
|
1859 break; |
1834 } |
1860 } |
1835 } else |
1861 } |
1836 if (Policy == 1) { // append to EntryList |
1862 } else |
1837 if (List == NULL) { |
1863 if (Policy == 3) { // append to cxq |
1838 iterator->_next = iterator->_prev = NULL; |
1864 iterator->TState = ObjectWaiter::TS_CXQ; |
1839 _EntryList = iterator; |
1865 for (;;) { |
1840 } else { |
1866 ObjectWaiter * Tail; |
1841 // CONSIDER: finding the tail currently requires a linear-time walk of |
1867 Tail = _cxq; |
1842 // the EntryList. We can make tail access constant-time by converting to |
1868 if (Tail == NULL) { |
1843 // a CDLL instead of using our current DLL. |
1869 iterator->_next = NULL; |
1844 ObjectWaiter * Tail; |
1870 if (Atomic::cmpxchg_ptr (iterator, &_cxq, NULL) == NULL) { |
1845 for (Tail = List; Tail->_next != NULL; Tail = Tail->_next); |
1871 break; |
1846 assert(Tail != NULL && Tail->_next == NULL, "invariant"); |
1872 } |
1847 Tail->_next = iterator; |
1873 } else { |
1848 iterator->_prev = Tail; |
1874 while (Tail->_next != NULL) Tail = Tail->_next; |
1849 iterator->_next = NULL; |
1875 Tail->_next = iterator; |
|
1876 iterator->_prev = Tail; |
|
1877 iterator->_next = NULL; |
|
1878 break; |
1850 } |
1879 } |
1851 } else |
1880 } |
1852 if (Policy == 2) { // prepend to cxq |
1881 } else { |
1853 // prepend to cxq |
1882 ParkEvent * ev = iterator->_event; |
1854 iterator->TState = ObjectWaiter::TS_CXQ; |
1883 iterator->TState = ObjectWaiter::TS_RUN; |
1855 for (;;) { |
1884 OrderAccess::fence(); |
1856 ObjectWaiter * Front = _cxq; |
1885 ev->unpark(); |
1857 iterator->_next = Front; |
1886 } |
1858 if (Atomic::cmpxchg_ptr (iterator, &_cxq, Front) == Front) { |
1887 |
1859 break; |
1888 if (Policy < 4) { |
1860 } |
1889 iterator->wait_reenter_begin(this); |
1861 } |
1890 } |
1862 } else |
1891 |
1863 if (Policy == 3) { // append to cxq |
1892 // _WaitSetLock protects the wait queue, not the EntryList. We could |
1864 iterator->TState = ObjectWaiter::TS_CXQ; |
1893 // move the add-to-EntryList operation, above, outside the critical section |
1865 for (;;) { |
1894 // protected by _WaitSetLock. In practice that's not useful. With the |
1866 ObjectWaiter * Tail; |
1895 // exception of wait() timeouts and interrupts the monitor owner |
1867 Tail = _cxq; |
1896 // is the only thread that grabs _WaitSetLock. There's almost no contention |
1868 if (Tail == NULL) { |
1897 // on _WaitSetLock so it's not profitable to reduce the length of the |
1869 iterator->_next = NULL; |
1898 // critical section. |
1870 if (Atomic::cmpxchg_ptr (iterator, &_cxq, NULL) == NULL) { |
|
1871 break; |
|
1872 } |
|
1873 } else { |
|
1874 while (Tail->_next != NULL) Tail = Tail->_next; |
|
1875 Tail->_next = iterator; |
|
1876 iterator->_prev = Tail; |
|
1877 iterator->_next = NULL; |
|
1878 break; |
|
1879 } |
|
1880 } |
|
1881 } else { |
|
1882 ParkEvent * ev = iterator->_event; |
|
1883 iterator->TState = ObjectWaiter::TS_RUN; |
|
1884 OrderAccess::fence(); |
|
1885 ev->unpark(); |
|
1886 } |
|
1887 |
|
1888 if (Policy < 4) { |
|
1889 iterator->wait_reenter_begin(this); |
|
1890 } |
|
1891 |
|
1892 // _WaitSetLock protects the wait queue, not the EntryList. We could |
|
1893 // move the add-to-EntryList operation, above, outside the critical section |
|
1894 // protected by _WaitSetLock. In practice that's not useful. With the |
|
1895 // exception of wait() timeouts and interrupts the monitor owner |
|
1896 // is the only thread that grabs _WaitSetLock. There's almost no contention |
|
1897 // on _WaitSetLock so it's not profitable to reduce the length of the |
|
1898 // critical section. |
|
1899 } |
1899 } |
1900 |
1900 |
1901 Thread::SpinRelease(&_WaitSetLock); |
1901 Thread::SpinRelease(&_WaitSetLock); |
1902 |
1902 |
1903 if (Tally != 0 && ObjectMonitor::_sync_Notifications != NULL) { |
1903 if (Tally != 0 && ObjectMonitor::_sync_Notifications != NULL) { |
1904 ObjectMonitor::_sync_Notifications->inc(Tally); |
1904 ObjectMonitor::_sync_Notifications->inc(Tally); |
1905 } |
1905 } |
1906 } |
1906 } |
1907 |
1907 |
1908 // ----------------------------------------------------------------------------- |
1908 // ----------------------------------------------------------------------------- |
1909 // Adaptive Spinning Support |
1909 // Adaptive Spinning Support |
1977 // Spinning: Fixed frequency (100%), vary duration |
1977 // Spinning: Fixed frequency (100%), vary duration |
1978 |
1978 |
1979 |
1979 |
1980 int ObjectMonitor::TrySpin_VaryDuration (Thread * Self) { |
1980 int ObjectMonitor::TrySpin_VaryDuration (Thread * Self) { |
1981 |
1981 |
1982 // Dumb, brutal spin. Good for comparative measurements against adaptive spinning. |
1982 // Dumb, brutal spin. Good for comparative measurements against adaptive spinning. |
1983 int ctr = Knob_FixedSpin; |
1983 int ctr = Knob_FixedSpin; |
1984 if (ctr != 0) { |
1984 if (ctr != 0) { |
1985 while (--ctr >= 0) { |
1985 while (--ctr >= 0) { |
1986 if (TryLock(Self) > 0) return 1; |
1986 if (TryLock(Self) > 0) return 1; |
1987 SpinPause(); |
1987 SpinPause(); |
|
1988 } |
|
1989 return 0; |
|
1990 } |
|
1991 |
|
1992 for (ctr = Knob_PreSpin + 1; --ctr >= 0;) { |
|
1993 if (TryLock(Self) > 0) { |
|
1994 // Increase _SpinDuration ... |
|
1995 // Note that we don't clamp SpinDuration precisely at SpinLimit. |
|
1996 // Raising _SpurDuration to the poverty line is key. |
|
1997 int x = _SpinDuration; |
|
1998 if (x < Knob_SpinLimit) { |
|
1999 if (x < Knob_Poverty) x = Knob_Poverty; |
|
2000 _SpinDuration = x + Knob_BonusB; |
|
2001 } |
|
2002 return 1; |
|
2003 } |
|
2004 SpinPause(); |
|
2005 } |
|
2006 |
|
2007 // Admission control - verify preconditions for spinning |
|
2008 // |
|
2009 // We always spin a little bit, just to prevent _SpinDuration == 0 from |
|
2010 // becoming an absorbing state. Put another way, we spin briefly to |
|
2011 // sample, just in case the system load, parallelism, contention, or lock |
|
2012 // modality changed. |
|
2013 // |
|
2014 // Consider the following alternative: |
|
2015 // Periodically set _SpinDuration = _SpinLimit and try a long/full |
|
2016 // spin attempt. "Periodically" might mean after a tally of |
|
2017 // the # of failed spin attempts (or iterations) reaches some threshold. |
|
2018 // This takes us into the realm of 1-out-of-N spinning, where we |
|
2019 // hold the duration constant but vary the frequency. |
|
2020 |
|
2021 ctr = _SpinDuration; |
|
2022 if (ctr < Knob_SpinBase) ctr = Knob_SpinBase; |
|
2023 if (ctr <= 0) return 0; |
|
2024 |
|
2025 if (Knob_SuccRestrict && _succ != NULL) return 0; |
|
2026 if (Knob_OState && NotRunnable (Self, (Thread *) _owner)) { |
|
2027 TEVENT(Spin abort - notrunnable [TOP]); |
|
2028 return 0; |
|
2029 } |
|
2030 |
|
2031 int MaxSpin = Knob_MaxSpinners; |
|
2032 if (MaxSpin >= 0) { |
|
2033 if (_Spinner > MaxSpin) { |
|
2034 TEVENT(Spin abort -- too many spinners); |
|
2035 return 0; |
|
2036 } |
|
2037 // Slightly racy, but benign ... |
|
2038 Adjust(&_Spinner, 1); |
|
2039 } |
|
2040 |
|
2041 // We're good to spin ... spin ingress. |
|
2042 // CONSIDER: use Prefetch::write() to avoid RTS->RTO upgrades |
|
2043 // when preparing to LD...CAS _owner, etc and the CAS is likely |
|
2044 // to succeed. |
|
2045 int hits = 0; |
|
2046 int msk = 0; |
|
2047 int caspty = Knob_CASPenalty; |
|
2048 int oxpty = Knob_OXPenalty; |
|
2049 int sss = Knob_SpinSetSucc; |
|
2050 if (sss && _succ == NULL) _succ = Self; |
|
2051 Thread * prv = NULL; |
|
2052 |
|
2053 // There are three ways to exit the following loop: |
|
2054 // 1. A successful spin where this thread has acquired the lock. |
|
2055 // 2. Spin failure with prejudice |
|
2056 // 3. Spin failure without prejudice |
|
2057 |
|
2058 while (--ctr >= 0) { |
|
2059 |
|
2060 // Periodic polling -- Check for pending GC |
|
2061 // Threads may spin while they're unsafe. |
|
2062 // We don't want spinning threads to delay the JVM from reaching |
|
2063 // a stop-the-world safepoint or to steal cycles from GC. |
|
2064 // If we detect a pending safepoint we abort in order that |
|
2065 // (a) this thread, if unsafe, doesn't delay the safepoint, and (b) |
|
2066 // this thread, if safe, doesn't steal cycles from GC. |
|
2067 // This is in keeping with the "no loitering in runtime" rule. |
|
2068 // We periodically check to see if there's a safepoint pending. |
|
2069 if ((ctr & 0xFF) == 0) { |
|
2070 if (SafepointSynchronize::do_call_back()) { |
|
2071 TEVENT(Spin: safepoint); |
|
2072 goto Abort; // abrupt spin egress |
|
2073 } |
|
2074 if (Knob_UsePause & 1) SpinPause(); |
|
2075 |
|
2076 int (*scb)(intptr_t,int) = SpinCallbackFunction; |
|
2077 if (hits > 50 && scb != NULL) { |
|
2078 int abend = (*scb)(SpinCallbackArgument, 0); |
|
2079 } |
|
2080 } |
|
2081 |
|
2082 if (Knob_UsePause & 2) SpinPause(); |
|
2083 |
|
2084 // Exponential back-off ... Stay off the bus to reduce coherency traffic. |
|
2085 // This is useful on classic SMP systems, but is of less utility on |
|
2086 // N1-style CMT platforms. |
|
2087 // |
|
2088 // Trade-off: lock acquisition latency vs coherency bandwidth. |
|
2089 // Lock hold times are typically short. A histogram |
|
2090 // of successful spin attempts shows that we usually acquire |
|
2091 // the lock early in the spin. That suggests we want to |
|
2092 // sample _owner frequently in the early phase of the spin, |
|
2093 // but then back-off and sample less frequently as the spin |
|
2094 // progresses. The back-off makes a good citizen on SMP big |
|
2095 // SMP systems. Oversampling _owner can consume excessive |
|
2096 // coherency bandwidth. Relatedly, if we _oversample _owner we |
|
2097 // can inadvertently interfere with the the ST m->owner=null. |
|
2098 // executed by the lock owner. |
|
2099 if (ctr & msk) continue; |
|
2100 ++hits; |
|
2101 if ((hits & 0xF) == 0) { |
|
2102 // The 0xF, above, corresponds to the exponent. |
|
2103 // Consider: (msk+1)|msk |
|
2104 msk = ((msk << 2)|3) & BackOffMask; |
|
2105 } |
|
2106 |
|
2107 // Probe _owner with TATAS |
|
2108 // If this thread observes the monitor transition or flicker |
|
2109 // from locked to unlocked to locked, then the odds that this |
|
2110 // thread will acquire the lock in this spin attempt go down |
|
2111 // considerably. The same argument applies if the CAS fails |
|
2112 // or if we observe _owner change from one non-null value to |
|
2113 // another non-null value. In such cases we might abort |
|
2114 // the spin without prejudice or apply a "penalty" to the |
|
2115 // spin count-down variable "ctr", reducing it by 100, say. |
|
2116 |
|
2117 Thread * ox = (Thread *) _owner; |
|
2118 if (ox == NULL) { |
|
2119 ox = (Thread *) Atomic::cmpxchg_ptr(Self, &_owner, NULL); |
|
2120 if (ox == NULL) { |
|
2121 // The CAS succeeded -- this thread acquired ownership |
|
2122 // Take care of some bookkeeping to exit spin state. |
|
2123 if (sss && _succ == Self) { |
|
2124 _succ = NULL; |
1988 } |
2125 } |
1989 return 0; |
2126 if (MaxSpin > 0) Adjust(&_Spinner, -1); |
1990 } |
2127 |
1991 |
2128 // Increase _SpinDuration : |
1992 for (ctr = Knob_PreSpin + 1; --ctr >= 0;) { |
2129 // The spin was successful (profitable) so we tend toward |
1993 if (TryLock(Self) > 0) { |
2130 // longer spin attempts in the future. |
1994 // Increase _SpinDuration ... |
2131 // CONSIDER: factor "ctr" into the _SpinDuration adjustment. |
|
2132 // If we acquired the lock early in the spin cycle it |
|
2133 // makes sense to increase _SpinDuration proportionally. |
1995 // Note that we don't clamp SpinDuration precisely at SpinLimit. |
2134 // Note that we don't clamp SpinDuration precisely at SpinLimit. |
1996 // Raising _SpurDuration to the poverty line is key. |
|
1997 int x = _SpinDuration; |
2135 int x = _SpinDuration; |
1998 if (x < Knob_SpinLimit) { |
2136 if (x < Knob_SpinLimit) { |
1999 if (x < Knob_Poverty) x = Knob_Poverty; |
2137 if (x < Knob_Poverty) x = Knob_Poverty; |
2000 _SpinDuration = x + Knob_BonusB; |
2138 _SpinDuration = x + Knob_Bonus; |
2001 } |
2139 } |
2002 return 1; |
2140 return 1; |
2003 } |
2141 } |
2004 SpinPause(); |
2142 |
2005 } |
2143 // The CAS failed ... we can take any of the following actions: |
2006 |
2144 // * penalize: ctr -= Knob_CASPenalty |
2007 // Admission control - verify preconditions for spinning |
2145 // * exit spin with prejudice -- goto Abort; |
2008 // |
2146 // * exit spin without prejudice. |
2009 // We always spin a little bit, just to prevent _SpinDuration == 0 from |
2147 // * Since CAS is high-latency, retry again immediately. |
2010 // becoming an absorbing state. Put another way, we spin briefly to |
2148 prv = ox; |
2011 // sample, just in case the system load, parallelism, contention, or lock |
2149 TEVENT(Spin: cas failed); |
2012 // modality changed. |
2150 if (caspty == -2) break; |
2013 // |
2151 if (caspty == -1) goto Abort; |
2014 // Consider the following alternative: |
2152 ctr -= caspty; |
2015 // Periodically set _SpinDuration = _SpinLimit and try a long/full |
2153 continue; |
2016 // spin attempt. "Periodically" might mean after a tally of |
2154 } |
2017 // the # of failed spin attempts (or iterations) reaches some threshold. |
2155 |
2018 // This takes us into the realm of 1-out-of-N spinning, where we |
2156 // Did lock ownership change hands ? |
2019 // hold the duration constant but vary the frequency. |
2157 if (ox != prv && prv != NULL) { |
2020 |
2158 TEVENT(spin: Owner changed) |
2021 ctr = _SpinDuration; |
2159 if (oxpty == -2) break; |
2022 if (ctr < Knob_SpinBase) ctr = Knob_SpinBase; |
2160 if (oxpty == -1) goto Abort; |
2023 if (ctr <= 0) return 0; |
2161 ctr -= oxpty; |
2024 |
2162 } |
2025 if (Knob_SuccRestrict && _succ != NULL) return 0; |
2163 prv = ox; |
2026 if (Knob_OState && NotRunnable (Self, (Thread *) _owner)) { |
2164 |
2027 TEVENT(Spin abort - notrunnable [TOP]); |
2165 // Abort the spin if the owner is not executing. |
2028 return 0; |
2166 // The owner must be executing in order to drop the lock. |
2029 } |
2167 // Spinning while the owner is OFFPROC is idiocy. |
2030 |
2168 // Consider: ctr -= RunnablePenalty ; |
2031 int MaxSpin = Knob_MaxSpinners; |
2169 if (Knob_OState && NotRunnable (Self, ox)) { |
2032 if (MaxSpin >= 0) { |
2170 TEVENT(Spin abort - notrunnable); |
2033 if (_Spinner > MaxSpin) { |
2171 goto Abort; |
2034 TEVENT(Spin abort -- too many spinners); |
2172 } |
2035 return 0; |
|
2036 } |
|
2037 // Slightly racy, but benign ... |
|
2038 Adjust(&_Spinner, 1); |
|
2039 } |
|
2040 |
|
2041 // We're good to spin ... spin ingress. |
|
2042 // CONSIDER: use Prefetch::write() to avoid RTS->RTO upgrades |
|
2043 // when preparing to LD...CAS _owner, etc and the CAS is likely |
|
2044 // to succeed. |
|
2045 int hits = 0; |
|
2046 int msk = 0; |
|
2047 int caspty = Knob_CASPenalty; |
|
2048 int oxpty = Knob_OXPenalty; |
|
2049 int sss = Knob_SpinSetSucc; |
|
2050 if (sss && _succ == NULL) _succ = Self; |
2173 if (sss && _succ == NULL) _succ = Self; |
2051 Thread * prv = NULL; |
2174 } |
2052 |
2175 |
2053 // There are three ways to exit the following loop: |
2176 // Spin failed with prejudice -- reduce _SpinDuration. |
2054 // 1. A successful spin where this thread has acquired the lock. |
2177 // TODO: Use an AIMD-like policy to adjust _SpinDuration. |
2055 // 2. Spin failure with prejudice |
2178 // AIMD is globally stable. |
2056 // 3. Spin failure without prejudice |
2179 TEVENT(Spin failure); |
2057 |
2180 { |
2058 while (--ctr >= 0) { |
2181 int x = _SpinDuration; |
2059 |
2182 if (x > 0) { |
2060 // Periodic polling -- Check for pending GC |
2183 // Consider an AIMD scheme like: x -= (x >> 3) + 100 |
2061 // Threads may spin while they're unsafe. |
2184 // This is globally sample and tends to damp the response. |
2062 // We don't want spinning threads to delay the JVM from reaching |
2185 x -= Knob_Penalty; |
2063 // a stop-the-world safepoint or to steal cycles from GC. |
2186 if (x < 0) x = 0; |
2064 // If we detect a pending safepoint we abort in order that |
2187 _SpinDuration = x; |
2065 // (a) this thread, if unsafe, doesn't delay the safepoint, and (b) |
2188 } |
2066 // this thread, if safe, doesn't steal cycles from GC. |
2189 } |
2067 // This is in keeping with the "no loitering in runtime" rule. |
|
2068 // We periodically check to see if there's a safepoint pending. |
|
2069 if ((ctr & 0xFF) == 0) { |
|
2070 if (SafepointSynchronize::do_call_back()) { |
|
2071 TEVENT(Spin: safepoint); |
|
2072 goto Abort; // abrupt spin egress |
|
2073 } |
|
2074 if (Knob_UsePause & 1) SpinPause(); |
|
2075 |
|
2076 int (*scb)(intptr_t,int) = SpinCallbackFunction; |
|
2077 if (hits > 50 && scb != NULL) { |
|
2078 int abend = (*scb)(SpinCallbackArgument, 0); |
|
2079 } |
|
2080 } |
|
2081 |
|
2082 if (Knob_UsePause & 2) SpinPause(); |
|
2083 |
|
2084 // Exponential back-off ... Stay off the bus to reduce coherency traffic. |
|
2085 // This is useful on classic SMP systems, but is of less utility on |
|
2086 // N1-style CMT platforms. |
|
2087 // |
|
2088 // Trade-off: lock acquisition latency vs coherency bandwidth. |
|
2089 // Lock hold times are typically short. A histogram |
|
2090 // of successful spin attempts shows that we usually acquire |
|
2091 // the lock early in the spin. That suggests we want to |
|
2092 // sample _owner frequently in the early phase of the spin, |
|
2093 // but then back-off and sample less frequently as the spin |
|
2094 // progresses. The back-off makes a good citizen on SMP big |
|
2095 // SMP systems. Oversampling _owner can consume excessive |
|
2096 // coherency bandwidth. Relatedly, if we _oversample _owner we |
|
2097 // can inadvertently interfere with the the ST m->owner=null. |
|
2098 // executed by the lock owner. |
|
2099 if (ctr & msk) continue; |
|
2100 ++hits; |
|
2101 if ((hits & 0xF) == 0) { |
|
2102 // The 0xF, above, corresponds to the exponent. |
|
2103 // Consider: (msk+1)|msk |
|
2104 msk = ((msk << 2)|3) & BackOffMask; |
|
2105 } |
|
2106 |
|
2107 // Probe _owner with TATAS |
|
2108 // If this thread observes the monitor transition or flicker |
|
2109 // from locked to unlocked to locked, then the odds that this |
|
2110 // thread will acquire the lock in this spin attempt go down |
|
2111 // considerably. The same argument applies if the CAS fails |
|
2112 // or if we observe _owner change from one non-null value to |
|
2113 // another non-null value. In such cases we might abort |
|
2114 // the spin without prejudice or apply a "penalty" to the |
|
2115 // spin count-down variable "ctr", reducing it by 100, say. |
|
2116 |
|
2117 Thread * ox = (Thread *) _owner; |
|
2118 if (ox == NULL) { |
|
2119 ox = (Thread *) Atomic::cmpxchg_ptr(Self, &_owner, NULL); |
|
2120 if (ox == NULL) { |
|
2121 // The CAS succeeded -- this thread acquired ownership |
|
2122 // Take care of some bookkeeping to exit spin state. |
|
2123 if (sss && _succ == Self) { |
|
2124 _succ = NULL; |
|
2125 } |
|
2126 if (MaxSpin > 0) Adjust(&_Spinner, -1); |
|
2127 |
|
2128 // Increase _SpinDuration : |
|
2129 // The spin was successful (profitable) so we tend toward |
|
2130 // longer spin attempts in the future. |
|
2131 // CONSIDER: factor "ctr" into the _SpinDuration adjustment. |
|
2132 // If we acquired the lock early in the spin cycle it |
|
2133 // makes sense to increase _SpinDuration proportionally. |
|
2134 // Note that we don't clamp SpinDuration precisely at SpinLimit. |
|
2135 int x = _SpinDuration; |
|
2136 if (x < Knob_SpinLimit) { |
|
2137 if (x < Knob_Poverty) x = Knob_Poverty; |
|
2138 _SpinDuration = x + Knob_Bonus; |
|
2139 } |
|
2140 return 1; |
|
2141 } |
|
2142 |
|
2143 // The CAS failed ... we can take any of the following actions: |
|
2144 // * penalize: ctr -= Knob_CASPenalty |
|
2145 // * exit spin with prejudice -- goto Abort; |
|
2146 // * exit spin without prejudice. |
|
2147 // * Since CAS is high-latency, retry again immediately. |
|
2148 prv = ox; |
|
2149 TEVENT(Spin: cas failed); |
|
2150 if (caspty == -2) break; |
|
2151 if (caspty == -1) goto Abort; |
|
2152 ctr -= caspty; |
|
2153 continue; |
|
2154 } |
|
2155 |
|
2156 // Did lock ownership change hands ? |
|
2157 if (ox != prv && prv != NULL) { |
|
2158 TEVENT(spin: Owner changed) |
|
2159 if (oxpty == -2) break; |
|
2160 if (oxpty == -1) goto Abort; |
|
2161 ctr -= oxpty; |
|
2162 } |
|
2163 prv = ox; |
|
2164 |
|
2165 // Abort the spin if the owner is not executing. |
|
2166 // The owner must be executing in order to drop the lock. |
|
2167 // Spinning while the owner is OFFPROC is idiocy. |
|
2168 // Consider: ctr -= RunnablePenalty ; |
|
2169 if (Knob_OState && NotRunnable (Self, ox)) { |
|
2170 TEVENT(Spin abort - notrunnable); |
|
2171 goto Abort; |
|
2172 } |
|
2173 if (sss && _succ == NULL) _succ = Self; |
|
2174 } |
|
2175 |
|
2176 // Spin failed with prejudice -- reduce _SpinDuration. |
|
2177 // TODO: Use an AIMD-like policy to adjust _SpinDuration. |
|
2178 // AIMD is globally stable. |
|
2179 TEVENT(Spin failure); |
|
2180 { |
|
2181 int x = _SpinDuration; |
|
2182 if (x > 0) { |
|
2183 // Consider an AIMD scheme like: x -= (x >> 3) + 100 |
|
2184 // This is globally sample and tends to damp the response. |
|
2185 x -= Knob_Penalty; |
|
2186 if (x < 0) x = 0; |
|
2187 _SpinDuration = x; |
|
2188 } |
|
2189 } |
|
2190 |
2190 |
2191 Abort: |
2191 Abort: |
2192 if (MaxSpin >= 0) Adjust(&_Spinner, -1); |
2192 if (MaxSpin >= 0) Adjust(&_Spinner, -1); |
2193 if (sss && _succ == Self) { |
2193 if (sss && _succ == Self) { |
2194 _succ = NULL; |
2194 _succ = NULL; |
2195 // Invariant: after setting succ=null a contending thread |
2195 // Invariant: after setting succ=null a contending thread |
2196 // must recheck-retry _owner before parking. This usually happens |
2196 // must recheck-retry _owner before parking. This usually happens |
2197 // in the normal usage of TrySpin(), but it's safest |
2197 // in the normal usage of TrySpin(), but it's safest |
2198 // to make TrySpin() as foolproof as possible. |
2198 // to make TrySpin() as foolproof as possible. |
2199 OrderAccess::fence(); |
2199 OrderAccess::fence(); |
2200 if (TryLock(Self) > 0) return 1; |
2200 if (TryLock(Self) > 0) return 1; |
2201 } |
2201 } |
2202 return 0; |
2202 return 0; |
2203 } |
2203 } |
2204 |
2204 |
2205 // NotRunnable() -- informed spinning |
2205 // NotRunnable() -- informed spinning |
2206 // |
2206 // |
2207 // Don't bother spinning if the owner is not eligible to drop the lock. |
2207 // Don't bother spinning if the owner is not eligible to drop the lock. |