4390 OrderAccess::fence(); // guarantee at least release consistency. |
4390 OrderAccess::fence(); // guarantee at least release consistency. |
4391 // Roach-motel semantics. |
4391 // Roach-motel semantics. |
4392 // It's safe if subsequent LDs and STs float "up" into the critical section, |
4392 // It's safe if subsequent LDs and STs float "up" into the critical section, |
4393 // but prior LDs and STs within the critical section can't be allowed |
4393 // but prior LDs and STs within the critical section can't be allowed |
4394 // to reorder or float past the ST that releases the lock. |
4394 // to reorder or float past the ST that releases the lock. |
|
4395 // Loads and stores in the critical section - which appear in program |
|
4396 // order before the store that releases the lock - must also appear |
|
4397 // before the store that releases the lock in memory visibility order. |
|
4398 // Conceptually we need a #loadstore|#storestore "release" MEMBAR before |
|
4399 // the ST of 0 into the lock-word which releases the lock, so fence |
|
4400 // more than covers this on all platforms. |
4395 *adr = 0; |
4401 *adr = 0; |
4396 } |
4402 } |
4397 |
4403 |
4398 // muxAcquire and muxRelease: |
4404 // muxAcquire and muxRelease: |
4399 // |
4405 // |
4571 // Both "pop" and DMR are immune from ABA corruption -- there can be |
4577 // Both "pop" and DMR are immune from ABA corruption -- there can be |
4572 // multiple concurrent pushers, but only one popper or detacher. |
4578 // multiple concurrent pushers, but only one popper or detacher. |
4573 // This implementation pops from the head of the list. This is unfair, |
4579 // This implementation pops from the head of the list. This is unfair, |
4574 // but tends to provide excellent throughput as hot threads remain hot. |
4580 // but tends to provide excellent throughput as hot threads remain hot. |
4575 // (We wake recently run threads first). |
4581 // (We wake recently run threads first). |
4576 |
4582 // |
|
4583 // All paths through muxRelease() will execute a CAS. |
|
4584 // Release consistency -- We depend on the CAS in muxRelease() to provide full |
|
4585 // bidirectional fence/MEMBAR semantics, ensuring that all prior memory operations |
|
4586 // executed within the critical section are complete and globally visible before the |
|
4587 // store (CAS) to the lock-word that releases the lock becomes globally visible. |
4577 void Thread::muxRelease (volatile intptr_t * Lock) { |
4588 void Thread::muxRelease (volatile intptr_t * Lock) { |
4578 for (;;) { |
4589 for (;;) { |
4579 const intptr_t w = Atomic::cmpxchg_ptr(0, Lock, LOCKBIT); |
4590 const intptr_t w = Atomic::cmpxchg_ptr(0, Lock, LOCKBIT); |
4580 assert(w & LOCKBIT, "invariant"); |
4591 assert(w & LOCKBIT, "invariant"); |
4581 if (w == LOCKBIT) return; |
4592 if (w == LOCKBIT) return; |
4582 ParkEvent * List = (ParkEvent *)(w & ~LOCKBIT); |
4593 ParkEvent * const List = (ParkEvent *) (w & ~LOCKBIT); |
4583 assert(List != NULL, "invariant"); |
4594 assert(List != NULL, "invariant"); |
4584 assert(List->OnList == intptr_t(Lock), "invariant"); |
4595 assert(List->OnList == intptr_t(Lock), "invariant"); |
4585 ParkEvent * nxt = List->ListNext; |
4596 ParkEvent * const nxt = List->ListNext; |
|
4597 guarantee((intptr_t(nxt) & LOCKBIT) == 0, "invariant"); |
4586 |
4598 |
4587 // The following CAS() releases the lock and pops the head element. |
4599 // The following CAS() releases the lock and pops the head element. |
|
4600 // The CAS() also ratifies the previously fetched lock-word value. |
4588 if (Atomic::cmpxchg_ptr (intptr_t(nxt), Lock, w) != w) { |
4601 if (Atomic::cmpxchg_ptr (intptr_t(nxt), Lock, w) != w) { |
4589 continue; |
4602 continue; |
4590 } |
4603 } |
4591 List->OnList = 0; |
4604 List->OnList = 0; |
4592 OrderAccess::fence(); |
4605 OrderAccess::fence(); |