859 |
859 |
860 // GC Support |
860 // GC Support |
861 bool Thread::claim_par_threads_do(uintx claim_token) { |
861 bool Thread::claim_par_threads_do(uintx claim_token) { |
862 uintx token = _threads_do_token; |
862 uintx token = _threads_do_token; |
863 if (token != claim_token) { |
863 if (token != claim_token) { |
864 uintx res = Atomic::cmpxchg(claim_token, &_threads_do_token, token); |
864 uintx res = Atomic::cmpxchg(&_threads_do_token, token, claim_token); |
865 if (res == token) { |
865 if (res == token) { |
866 return true; |
866 return true; |
867 } |
867 } |
868 guarantee(res == claim_token, "invariant"); |
868 guarantee(res == claim_token, "invariant"); |
869 } |
869 } |
4873 |
4873 |
4874 |
4874 |
4875 typedef volatile int SpinLockT; |
4875 typedef volatile int SpinLockT; |
4876 |
4876 |
4877 void Thread::SpinAcquire(volatile int * adr, const char * LockName) { |
4877 void Thread::SpinAcquire(volatile int * adr, const char * LockName) { |
4878 if (Atomic::cmpxchg (1, adr, 0) == 0) { |
4878 if (Atomic::cmpxchg(adr, 0, 1) == 0) { |
4879 return; // normal fast-path return |
4879 return; // normal fast-path return |
4880 } |
4880 } |
4881 |
4881 |
4882 // Slow-path : We've encountered contention -- Spin/Yield/Block strategy. |
4882 // Slow-path : We've encountered contention -- Spin/Yield/Block strategy. |
4883 int ctr = 0; |
4883 int ctr = 0; |
4966 |
4966 |
4967 |
4967 |
4968 const intptr_t LOCKBIT = 1; |
4968 const intptr_t LOCKBIT = 1; |
4969 |
4969 |
4970 void Thread::muxAcquire(volatile intptr_t * Lock, const char * LockName) { |
4970 void Thread::muxAcquire(volatile intptr_t * Lock, const char * LockName) { |
4971 intptr_t w = Atomic::cmpxchg(LOCKBIT, Lock, (intptr_t)0); |
4971 intptr_t w = Atomic::cmpxchg(Lock, (intptr_t)0, LOCKBIT); |
4972 if (w == 0) return; |
4972 if (w == 0) return; |
4973 if ((w & LOCKBIT) == 0 && Atomic::cmpxchg(w|LOCKBIT, Lock, w) == w) { |
4973 if ((w & LOCKBIT) == 0 && Atomic::cmpxchg(Lock, w, w|LOCKBIT) == w) { |
4974 return; |
4974 return; |
4975 } |
4975 } |
4976 |
4976 |
4977 ParkEvent * const Self = Thread::current()->_MuxEvent; |
4977 ParkEvent * const Self = Thread::current()->_MuxEvent; |
4978 assert((intptr_t(Self) & LOCKBIT) == 0, "invariant"); |
4978 assert((intptr_t(Self) & LOCKBIT) == 0, "invariant"); |
4993 // CAS() both serializes execution and ratifies the fetched *Lock value. |
4993 // CAS() both serializes execution and ratifies the fetched *Lock value. |
4994 OrderAccess::fence(); |
4994 OrderAccess::fence(); |
4995 for (;;) { |
4995 for (;;) { |
4996 w = *Lock; |
4996 w = *Lock; |
4997 if ((w & LOCKBIT) == 0) { |
4997 if ((w & LOCKBIT) == 0) { |
4998 if (Atomic::cmpxchg(w|LOCKBIT, Lock, w) == w) { |
4998 if (Atomic::cmpxchg(Lock, w, w|LOCKBIT) == w) { |
4999 Self->OnList = 0; // hygiene - allows stronger asserts |
4999 Self->OnList = 0; // hygiene - allows stronger asserts |
5000 return; |
5000 return; |
5001 } |
5001 } |
5002 continue; // Interference -- *Lock changed -- Just retry |
5002 continue; // Interference -- *Lock changed -- Just retry |
5003 } |
5003 } |
5004 assert(w & LOCKBIT, "invariant"); |
5004 assert(w & LOCKBIT, "invariant"); |
5005 Self->ListNext = (ParkEvent *) (w & ~LOCKBIT); |
5005 Self->ListNext = (ParkEvent *) (w & ~LOCKBIT); |
5006 if (Atomic::cmpxchg(intptr_t(Self)|LOCKBIT, Lock, w) == w) break; |
5006 if (Atomic::cmpxchg(Lock, w, intptr_t(Self)|LOCKBIT) == w) break; |
5007 } |
5007 } |
5008 |
5008 |
5009 while (Self->OnList != 0) { |
5009 while (Self->OnList != 0) { |
5010 Self->park(); |
5010 Self->park(); |
5011 } |
5011 } |
5037 // bidirectional fence/MEMBAR semantics, ensuring that all prior memory operations |
5037 // bidirectional fence/MEMBAR semantics, ensuring that all prior memory operations |
5038 // executed within the critical section are complete and globally visible before the |
5038 // executed within the critical section are complete and globally visible before the |
5039 // store (CAS) to the lock-word that releases the lock becomes globally visible. |
5039 // store (CAS) to the lock-word that releases the lock becomes globally visible. |
5040 void Thread::muxRelease(volatile intptr_t * Lock) { |
5040 void Thread::muxRelease(volatile intptr_t * Lock) { |
5041 for (;;) { |
5041 for (;;) { |
5042 const intptr_t w = Atomic::cmpxchg((intptr_t)0, Lock, LOCKBIT); |
5042 const intptr_t w = Atomic::cmpxchg(Lock, LOCKBIT, (intptr_t)0); |
5043 assert(w & LOCKBIT, "invariant"); |
5043 assert(w & LOCKBIT, "invariant"); |
5044 if (w == LOCKBIT) return; |
5044 if (w == LOCKBIT) return; |
5045 ParkEvent * const List = (ParkEvent *) (w & ~LOCKBIT); |
5045 ParkEvent * const List = (ParkEvent *) (w & ~LOCKBIT); |
5046 assert(List != NULL, "invariant"); |
5046 assert(List != NULL, "invariant"); |
5047 assert(List->OnList == intptr_t(Lock), "invariant"); |
5047 assert(List->OnList == intptr_t(Lock), "invariant"); |
5048 ParkEvent * const nxt = List->ListNext; |
5048 ParkEvent * const nxt = List->ListNext; |
5049 guarantee((intptr_t(nxt) & LOCKBIT) == 0, "invariant"); |
5049 guarantee((intptr_t(nxt) & LOCKBIT) == 0, "invariant"); |
5050 |
5050 |
5051 // The following CAS() releases the lock and pops the head element. |
5051 // The following CAS() releases the lock and pops the head element. |
5052 // The CAS() also ratifies the previously fetched lock-word value. |
5052 // The CAS() also ratifies the previously fetched lock-word value. |
5053 if (Atomic::cmpxchg(intptr_t(nxt), Lock, w) != w) { |
5053 if (Atomic::cmpxchg(Lock, w, intptr_t(nxt)) != w) { |
5054 continue; |
5054 continue; |
5055 } |
5055 } |
5056 List->OnList = 0; |
5056 List->OnList = 0; |
5057 OrderAccess::fence(); |
5057 OrderAccess::fence(); |
5058 List->unpark(); |
5058 List->unpark(); |