4699 // See RTStt for details. |
4699 // See RTStt for details. |
4700 // * Consider schedctl.sc_nopreempt to cover the critical section. |
4700 // * Consider schedctl.sc_nopreempt to cover the critical section. |
4701 // |
4701 // |
4702 |
4702 |
4703 |
4703 |
4704 typedef volatile intptr_t MutexT; // Mux Lock-word |
4704 const intptr_t LOCKBIT = 1; |
4705 enum MuxBits { LOCKBIT = 1 }; |
|
4706 |
4705 |
4707 void Thread::muxAcquire(volatile intptr_t * Lock, const char * LockName) { |
4706 void Thread::muxAcquire(volatile intptr_t * Lock, const char * LockName) { |
4708 intptr_t w = Atomic::cmpxchg_ptr(LOCKBIT, Lock, 0); |
4707 intptr_t w = Atomic::cmpxchg(LOCKBIT, Lock, (intptr_t)0); |
4709 if (w == 0) return; |
4708 if (w == 0) return; |
4710 if ((w & LOCKBIT) == 0 && Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) { |
4709 if ((w & LOCKBIT) == 0 && Atomic::cmpxchg(w|LOCKBIT, Lock, w) == w) { |
4711 return; |
4710 return; |
4712 } |
4711 } |
4713 |
4712 |
4714 TEVENT(muxAcquire - Contention); |
4713 TEVENT(muxAcquire - Contention); |
4715 ParkEvent * const Self = Thread::current()->_MuxEvent; |
4714 ParkEvent * const Self = Thread::current()->_MuxEvent; |
4718 int its = (os::is_MP() ? 100 : 0) + 1; |
4717 int its = (os::is_MP() ? 100 : 0) + 1; |
4719 |
4718 |
4720 // Optional spin phase: spin-then-park strategy |
4719 // Optional spin phase: spin-then-park strategy |
4721 while (--its >= 0) { |
4720 while (--its >= 0) { |
4722 w = *Lock; |
4721 w = *Lock; |
4723 if ((w & LOCKBIT) == 0 && Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) { |
4722 if ((w & LOCKBIT) == 0 && Atomic::cmpxchg(w|LOCKBIT, Lock, w) == w) { |
4724 return; |
4723 return; |
4725 } |
4724 } |
4726 } |
4725 } |
4727 |
4726 |
4728 Self->reset(); |
4727 Self->reset(); |
4731 // CAS() both serializes execution and ratifies the fetched *Lock value. |
4730 // CAS() both serializes execution and ratifies the fetched *Lock value. |
4732 OrderAccess::fence(); |
4731 OrderAccess::fence(); |
4733 for (;;) { |
4732 for (;;) { |
4734 w = *Lock; |
4733 w = *Lock; |
4735 if ((w & LOCKBIT) == 0) { |
4734 if ((w & LOCKBIT) == 0) { |
4736 if (Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) { |
4735 if (Atomic::cmpxchg(w|LOCKBIT, Lock, w) == w) { |
4737 Self->OnList = 0; // hygiene - allows stronger asserts |
4736 Self->OnList = 0; // hygiene - allows stronger asserts |
4738 return; |
4737 return; |
4739 } |
4738 } |
4740 continue; // Interference -- *Lock changed -- Just retry |
4739 continue; // Interference -- *Lock changed -- Just retry |
4741 } |
4740 } |
4742 assert(w & LOCKBIT, "invariant"); |
4741 assert(w & LOCKBIT, "invariant"); |
4743 Self->ListNext = (ParkEvent *) (w & ~LOCKBIT); |
4742 Self->ListNext = (ParkEvent *) (w & ~LOCKBIT); |
4744 if (Atomic::cmpxchg_ptr(intptr_t(Self)|LOCKBIT, Lock, w) == w) break; |
4743 if (Atomic::cmpxchg(intptr_t(Self)|LOCKBIT, Lock, w) == w) break; |
4745 } |
4744 } |
4746 |
4745 |
4747 while (Self->OnList != 0) { |
4746 while (Self->OnList != 0) { |
4748 Self->park(); |
4747 Self->park(); |
4749 } |
4748 } |
4750 } |
4749 } |
4751 } |
4750 } |
4752 |
4751 |
4753 void Thread::muxAcquireW(volatile intptr_t * Lock, ParkEvent * ev) { |
4752 void Thread::muxAcquireW(volatile intptr_t * Lock, ParkEvent * ev) { |
4754 intptr_t w = Atomic::cmpxchg_ptr(LOCKBIT, Lock, 0); |
4753 intptr_t w = Atomic::cmpxchg(LOCKBIT, Lock, (intptr_t)0); |
4755 if (w == 0) return; |
4754 if (w == 0) return; |
4756 if ((w & LOCKBIT) == 0 && Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) { |
4755 if ((w & LOCKBIT) == 0 && Atomic::cmpxchg(w|LOCKBIT, Lock, w) == w) { |
4757 return; |
4756 return; |
4758 } |
4757 } |
4759 |
4758 |
4760 TEVENT(muxAcquire - Contention); |
4759 TEVENT(muxAcquire - Contention); |
4761 ParkEvent * ReleaseAfter = NULL; |
4760 ParkEvent * ReleaseAfter = NULL; |
4768 int its = (os::is_MP() ? 100 : 0) + 1; |
4767 int its = (os::is_MP() ? 100 : 0) + 1; |
4769 |
4768 |
4770 // Optional spin phase: spin-then-park strategy |
4769 // Optional spin phase: spin-then-park strategy |
4771 while (--its >= 0) { |
4770 while (--its >= 0) { |
4772 w = *Lock; |
4771 w = *Lock; |
4773 if ((w & LOCKBIT) == 0 && Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) { |
4772 if ((w & LOCKBIT) == 0 && Atomic::cmpxchg(w|LOCKBIT, Lock, w) == w) { |
4774 if (ReleaseAfter != NULL) { |
4773 if (ReleaseAfter != NULL) { |
4775 ParkEvent::Release(ReleaseAfter); |
4774 ParkEvent::Release(ReleaseAfter); |
4776 } |
4775 } |
4777 return; |
4776 return; |
4778 } |
4777 } |
4784 // CAS() both serializes execution and ratifies the fetched *Lock value. |
4783 // CAS() both serializes execution and ratifies the fetched *Lock value. |
4785 OrderAccess::fence(); |
4784 OrderAccess::fence(); |
4786 for (;;) { |
4785 for (;;) { |
4787 w = *Lock; |
4786 w = *Lock; |
4788 if ((w & LOCKBIT) == 0) { |
4787 if ((w & LOCKBIT) == 0) { |
4789 if (Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) { |
4788 if (Atomic::cmpxchg(w|LOCKBIT, Lock, w) == w) { |
4790 ev->OnList = 0; |
4789 ev->OnList = 0; |
4791 // We call ::Release while holding the outer lock, thus |
4790 // We call ::Release while holding the outer lock, thus |
4792 // artificially lengthening the critical section. |
4791 // artificially lengthening the critical section. |
4793 // Consider deferring the ::Release() until the subsequent unlock(), |
4792 // Consider deferring the ::Release() until the subsequent unlock(), |
4794 // after we've dropped the outer lock. |
4793 // after we've dropped the outer lock. |
4799 } |
4798 } |
4800 continue; // Interference -- *Lock changed -- Just retry |
4799 continue; // Interference -- *Lock changed -- Just retry |
4801 } |
4800 } |
4802 assert(w & LOCKBIT, "invariant"); |
4801 assert(w & LOCKBIT, "invariant"); |
4803 ev->ListNext = (ParkEvent *) (w & ~LOCKBIT); |
4802 ev->ListNext = (ParkEvent *) (w & ~LOCKBIT); |
4804 if (Atomic::cmpxchg_ptr(intptr_t(ev)|LOCKBIT, Lock, w) == w) break; |
4803 if (Atomic::cmpxchg(intptr_t(ev)|LOCKBIT, Lock, w) == w) break; |
4805 } |
4804 } |
4806 |
4805 |
4807 while (ev->OnList != 0) { |
4806 while (ev->OnList != 0) { |
4808 ev->park(); |
4807 ev->park(); |
4809 } |
4808 } |
4835 // bidirectional fence/MEMBAR semantics, ensuring that all prior memory operations |
4834 // bidirectional fence/MEMBAR semantics, ensuring that all prior memory operations |
4836 // executed within the critical section are complete and globally visible before the |
4835 // executed within the critical section are complete and globally visible before the |
4837 // store (CAS) to the lock-word that releases the lock becomes globally visible. |
4836 // store (CAS) to the lock-word that releases the lock becomes globally visible. |
4838 void Thread::muxRelease(volatile intptr_t * Lock) { |
4837 void Thread::muxRelease(volatile intptr_t * Lock) { |
4839 for (;;) { |
4838 for (;;) { |
4840 const intptr_t w = Atomic::cmpxchg_ptr(0, Lock, LOCKBIT); |
4839 const intptr_t w = Atomic::cmpxchg((intptr_t)0, Lock, LOCKBIT); |
4841 assert(w & LOCKBIT, "invariant"); |
4840 assert(w & LOCKBIT, "invariant"); |
4842 if (w == LOCKBIT) return; |
4841 if (w == LOCKBIT) return; |
4843 ParkEvent * const List = (ParkEvent *) (w & ~LOCKBIT); |
4842 ParkEvent * const List = (ParkEvent *) (w & ~LOCKBIT); |
4844 assert(List != NULL, "invariant"); |
4843 assert(List != NULL, "invariant"); |
4845 assert(List->OnList == intptr_t(Lock), "invariant"); |
4844 assert(List->OnList == intptr_t(Lock), "invariant"); |
4846 ParkEvent * const nxt = List->ListNext; |
4845 ParkEvent * const nxt = List->ListNext; |
4847 guarantee((intptr_t(nxt) & LOCKBIT) == 0, "invariant"); |
4846 guarantee((intptr_t(nxt) & LOCKBIT) == 0, "invariant"); |
4848 |
4847 |
4849 // The following CAS() releases the lock and pops the head element. |
4848 // The following CAS() releases the lock and pops the head element. |
4850 // The CAS() also ratifies the previously fetched lock-word value. |
4849 // The CAS() also ratifies the previously fetched lock-word value. |
4851 if (Atomic::cmpxchg_ptr (intptr_t(nxt), Lock, w) != w) { |
4850 if (Atomic::cmpxchg(intptr_t(nxt), Lock, w) != w) { |
4852 continue; |
4851 continue; |
4853 } |
4852 } |
4854 List->OnList = 0; |
4853 List->OnList = 0; |
4855 OrderAccess::fence(); |
4854 OrderAccess::fence(); |
4856 List->unpark(); |
4855 List->unpark(); |