8155949: Support relaxed semantics in cmpxchg
Reviewed-by: dholmes, kbarrett, goetz, aph
Contributed-by: horii@jp.ibm.com, martin.doerr@sap.com
--- a/hotspot/src/os_cpu/aix_ppc/vm/atomic_aix_ppc.inline.hpp Tue Jun 21 18:37:04 2016 +0400
+++ b/hotspot/src/os_cpu/aix_ppc/vm/atomic_aix_ppc.inline.hpp Tue Jun 21 19:25:41 2016 -0400
@@ -291,12 +291,30 @@
return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
}
+inline void cmpxchg_pre_membar(cmpxchg_memory_order order) {
+ if (order != memory_order_relaxed) {
+ __asm__ __volatile__ (
+ /* fence */
+ strasm_sync
+ );
+ }
+}
+
+inline void cmpxchg_post_membar(cmpxchg_memory_order order) {
+ if (order != memory_order_relaxed) {
+ __asm__ __volatile__ (
+ /* fence */
+ strasm_sync
+ );
+ }
+}
+
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
-inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
// Note that cmpxchg guarantees a two-way memory barrier across
- // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
- // (see atomic.hpp).
+ // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
+ // specified otherwise (see atomic.hpp).
// Using 32 bit internally.
volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3);
@@ -312,9 +330,9 @@
unsigned int old_value, value32;
+ cmpxchg_pre_membar(order);
+
__asm__ __volatile__ (
- /* fence */
- strasm_sync
/* simple guard */
" lbz %[old_value], 0(%[dest]) \n"
" cmpw %[masked_compare_val], %[old_value] \n"
@@ -331,8 +349,6 @@
" xor %[value32], %[xor_value], %[value32] \n"
" stwcx. %[value32], 0, %[dest_base] \n"
" bne- 1b \n"
- /* acquire */
- strasm_sync
/* exit */
"2: \n"
/* out */
@@ -353,21 +369,23 @@
"memory"
);
+ cmpxchg_post_membar(order);
+
return (jbyte)(unsigned char)old_value;
}
-inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value) {
+inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
// Note that cmpxchg guarantees a two-way memory barrier across
- // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
- // (see atomic.hpp).
+ // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
+ // specified otherwise (see atomic.hpp).
unsigned int old_value;
const uint64_t zero = 0;
+ cmpxchg_pre_membar(order);
+
__asm__ __volatile__ (
- /* fence */
- strasm_sync
/* simple guard */
" lwz %[old_value], 0(%[dest]) \n"
" cmpw %[compare_value], %[old_value] \n"
@@ -379,8 +397,6 @@
" bne- 2f \n"
" stwcx. %[exchange_value], %[dest], %[zero] \n"
" bne- 1b \n"
- /* acquire */
- strasm_sync
/* exit */
"2: \n"
/* out */
@@ -397,21 +413,23 @@
"memory"
);
+ cmpxchg_post_membar(order);
+
return (jint) old_value;
}
-inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value) {
+inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
// Note that cmpxchg guarantees a two-way memory barrier across
- // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
- // (see atomic.hpp).
+ // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
+ // specified otherwise (see atomic.hpp).
long old_value;
const uint64_t zero = 0;
+ cmpxchg_pre_membar(order);
+
__asm__ __volatile__ (
- /* fence */
- strasm_sync
/* simple guard */
" ld %[old_value], 0(%[dest]) \n"
" cmpd %[compare_value], %[old_value] \n"
@@ -423,8 +441,6 @@
" bne- 2f \n"
" stdcx. %[exchange_value], %[dest], %[zero] \n"
" bne- 1b \n"
- /* acquire */
- strasm_sync
/* exit */
"2: \n"
/* out */
@@ -441,15 +457,17 @@
"memory"
);
+ cmpxchg_post_membar(order);
+
return (jlong) old_value;
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
- return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
+ return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
- return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
+ return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
#undef strasm_sync
--- a/hotspot/src/os_cpu/bsd_x86/vm/atomic_bsd_x86.inline.hpp Tue Jun 21 18:37:04 2016 +0400
+++ b/hotspot/src/os_cpu/bsd_x86/vm/atomic_bsd_x86.inline.hpp Tue Jun 21 19:25:41 2016 -0400
@@ -89,7 +89,7 @@
}
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
-inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
int mp = os::is_MP();
__asm__ volatile (LOCK_IF_MP(%4) "cmpxchgb %1,(%3)"
: "=a" (exchange_value)
@@ -98,7 +98,7 @@
return exchange_value;
}
-inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
+inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
int mp = os::is_MP();
__asm__ volatile (LOCK_IF_MP(%4) "cmpxchgl %1,(%3)"
: "=a" (exchange_value)
@@ -149,7 +149,7 @@
return exchange_value;
}
-inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
+inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
bool mp = os::is_MP();
__asm__ __volatile__ (LOCK_IF_MP(%4) "cmpxchgq %1,(%3)"
: "=a" (exchange_value)
@@ -158,12 +158,12 @@
return exchange_value;
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
- return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
+ return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
- return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
+ return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
inline jlong Atomic::load(volatile jlong* src) { return *src; }
@@ -197,16 +197,16 @@
void _Atomic_move_long(volatile jlong* src, volatile jlong* dst);
}
-inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
+inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
return _Atomic_cmpxchg_long(exchange_value, dest, compare_value, os::is_MP());
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
- return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
+ return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
- return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
+ return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
inline jlong Atomic::load(volatile jlong* src) {
--- a/hotspot/src/os_cpu/bsd_zero/vm/atomic_bsd_zero.inline.hpp Tue Jun 21 18:37:04 2016 +0400
+++ b/hotspot/src/os_cpu/bsd_zero/vm/atomic_bsd_zero.inline.hpp Tue Jun 21 19:25:41 2016 -0400
@@ -270,7 +270,8 @@
inline jint Atomic::cmpxchg(jint exchange_value,
volatile jint* dest,
- jint compare_value) {
+ jint compare_value,
+ cmpxchg_memory_order order) {
#ifdef ARM
return arm_compare_and_swap(dest, compare_value, exchange_value);
#else
@@ -284,14 +285,16 @@
inline jlong Atomic::cmpxchg(jlong exchange_value,
volatile jlong* dest,
- jlong compare_value) {
+ jlong compare_value,
+ cmpxchg_memory_order order) {
return __sync_val_compare_and_swap(dest, compare_value, exchange_value);
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value,
volatile intptr_t* dest,
- intptr_t compare_value) {
+ intptr_t compare_value,
+ cmpxchg_memory_order order) {
#ifdef ARM
return arm_compare_and_swap(dest, compare_value, exchange_value);
#else
@@ -305,11 +308,13 @@
inline void* Atomic::cmpxchg_ptr(void* exchange_value,
volatile void* dest,
- void* compare_value) {
+ void* compare_value,
+ cmpxchg_memory_order order) {
return (void *) cmpxchg_ptr((intptr_t) exchange_value,
(volatile intptr_t*) dest,
- (intptr_t) compare_value);
+ (intptr_t) compare_value,
+ order);
}
inline jlong Atomic::load(volatile jlong* src) {
--- a/hotspot/src/os_cpu/linux_aarch64/vm/atomic_linux_aarch64.inline.hpp Tue Jun 21 18:37:04 2016 +0400
+++ b/hotspot/src/os_cpu/linux_aarch64/vm/atomic_linux_aarch64.inline.hpp Tue Jun 21 19:25:41 2016 -0400
@@ -87,10 +87,28 @@
(volatile intptr_t*) dest);
}
+template <typename T> T generic_cmpxchg(T exchange_value, volatile T* dest,
+ T compare_value, cmpxchg_memory_order order)
+{
+ if (order == memory_order_relaxed) {
+ T value = compare_value;
+ __atomic_compare_exchange(dest, &value, &exchange_value, /*weak*/false,
+ __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+ return value;
+ } else {
+ return __sync_val_compare_and_swap(dest, compare_value, exchange_value);
+ }
+}
-inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value)
+#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order)
{
- return __sync_val_compare_and_swap(dest, compare_value, exchange_value);
+ return generic_cmpxchg(exchange_value, dest, compare_value, order);
+}
+
+inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order)
+{
+ return generic_cmpxchg(exchange_value, dest, compare_value, order);
}
inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; }
@@ -123,21 +141,22 @@
return res;
}
-inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value)
+inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order)
{
- return __sync_val_compare_and_swap(dest, compare_value, exchange_value);
+ return generic_cmpxchg(exchange_value, dest, compare_value, order);
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value)
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order)
{
- return __sync_val_compare_and_swap(dest, compare_value, exchange_value);
+ return generic_cmpxchg(exchange_value, dest, compare_value, order);
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value)
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order)
{
return (void *) cmpxchg_ptr((intptr_t) exchange_value,
(volatile intptr_t*) dest,
- (intptr_t) compare_value);
+ (intptr_t) compare_value,
+ order);
}
inline jlong Atomic::load(volatile jlong* src) { return *src; }
--- a/hotspot/src/os_cpu/linux_ppc/vm/atomic_linux_ppc.inline.hpp Tue Jun 21 18:37:04 2016 +0400
+++ b/hotspot/src/os_cpu/linux_ppc/vm/atomic_linux_ppc.inline.hpp Tue Jun 21 19:25:41 2016 -0400
@@ -291,12 +291,30 @@
return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
}
+inline void cmpxchg_pre_membar(cmpxchg_memory_order order) {
+ if (order != memory_order_relaxed) {
+ __asm__ __volatile__ (
+ /* fence */
+ strasm_sync
+ );
+ }
+}
+
+inline void cmpxchg_post_membar(cmpxchg_memory_order order) {
+ if (order != memory_order_relaxed) {
+ __asm__ __volatile__ (
+ /* fence */
+ strasm_sync
+ );
+ }
+}
+
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
-inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
// Note that cmpxchg guarantees a two-way memory barrier across
- // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
- // (see atomic.hpp).
+ // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
+ // specified otherwise (see atomic.hpp).
// Using 32 bit internally.
volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3);
@@ -312,9 +330,9 @@
unsigned int old_value, value32;
+ cmpxchg_pre_membar(order);
+
__asm__ __volatile__ (
- /* fence */
- strasm_sync
/* simple guard */
" lbz %[old_value], 0(%[dest]) \n"
" cmpw %[masked_compare_val], %[old_value] \n"
@@ -331,8 +349,6 @@
" xor %[value32], %[xor_value], %[value32] \n"
" stwcx. %[value32], 0, %[dest_base] \n"
" bne- 1b \n"
- /* acquire */
- strasm_sync
/* exit */
"2: \n"
/* out */
@@ -353,21 +369,23 @@
"memory"
);
+ cmpxchg_post_membar(order);
+
return (jbyte)(unsigned char)old_value;
}
-inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value) {
+inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
// Note that cmpxchg guarantees a two-way memory barrier across
- // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
- // (see atomic.hpp).
+ // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
+ // specified otherwise (see atomic.hpp).
unsigned int old_value;
const uint64_t zero = 0;
+ cmpxchg_pre_membar(order);
+
__asm__ __volatile__ (
- /* fence */
- strasm_sync
/* simple guard */
" lwz %[old_value], 0(%[dest]) \n"
" cmpw %[compare_value], %[old_value] \n"
@@ -379,8 +397,6 @@
" bne- 2f \n"
" stwcx. %[exchange_value], %[dest], %[zero] \n"
" bne- 1b \n"
- /* acquire */
- strasm_sync
/* exit */
"2: \n"
/* out */
@@ -397,21 +413,23 @@
"memory"
);
+ cmpxchg_post_membar(order);
+
return (jint) old_value;
}
-inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value) {
+inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
// Note that cmpxchg guarantees a two-way memory barrier across
- // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
- // (see atomic.hpp).
+ // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
+ // specified otherwise (see atomic.hpp).
long old_value;
const uint64_t zero = 0;
+ cmpxchg_pre_membar(order);
+
__asm__ __volatile__ (
- /* fence */
- strasm_sync
/* simple guard */
" ld %[old_value], 0(%[dest]) \n"
" cmpd %[compare_value], %[old_value] \n"
@@ -423,8 +441,6 @@
" bne- 2f \n"
" stdcx. %[exchange_value], %[dest], %[zero] \n"
" bne- 1b \n"
- /* acquire */
- strasm_sync
/* exit */
"2: \n"
/* out */
@@ -441,15 +457,17 @@
"memory"
);
+ cmpxchg_post_membar(order);
+
return (jlong) old_value;
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
- return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
+ return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
- return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
+ return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
#undef strasm_sync
--- a/hotspot/src/os_cpu/linux_sparc/vm/atomic_linux_sparc.inline.hpp Tue Jun 21 18:37:04 2016 +0400
+++ b/hotspot/src/os_cpu/linux_sparc/vm/atomic_linux_sparc.inline.hpp Tue Jun 21 19:25:41 2016 -0400
@@ -148,7 +148,7 @@
}
-inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
+inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
jint rv;
__asm__ volatile(
" cas [%2], %3, %0"
@@ -158,7 +158,7 @@
return rv;
}
-inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
+inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
#ifdef _LP64
jlong rv;
__asm__ volatile(
@@ -190,7 +190,7 @@
#endif
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
intptr_t rv;
#ifdef _LP64
__asm__ volatile(
@@ -208,8 +208,8 @@
return rv;
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
- return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value);
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
+ return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value, order);
}
#endif // OS_CPU_LINUX_SPARC_VM_ATOMIC_LINUX_SPARC_INLINE_HPP
--- a/hotspot/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp Tue Jun 21 18:37:04 2016 +0400
+++ b/hotspot/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp Tue Jun 21 19:25:41 2016 -0400
@@ -89,7 +89,7 @@
}
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
-inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
int mp = os::is_MP();
__asm__ volatile (LOCK_IF_MP(%4) "cmpxchgb %1,(%3)"
: "=a" (exchange_value)
@@ -98,7 +98,7 @@
return exchange_value;
}
-inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
+inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
int mp = os::is_MP();
__asm__ volatile (LOCK_IF_MP(%4) "cmpxchgl %1,(%3)"
: "=a" (exchange_value)
@@ -149,7 +149,7 @@
return exchange_value;
}
-inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
+inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
bool mp = os::is_MP();
__asm__ __volatile__ (LOCK_IF_MP(%4) "cmpxchgq %1,(%3)"
: "=a" (exchange_value)
@@ -158,12 +158,12 @@
return exchange_value;
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
- return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
+ return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
- return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
+ return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
inline jlong Atomic::load(volatile jlong* src) { return *src; }
@@ -197,16 +197,16 @@
void _Atomic_move_long(volatile jlong* src, volatile jlong* dst);
}
-inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
+inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
return _Atomic_cmpxchg_long(exchange_value, dest, compare_value, os::is_MP());
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
- return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
+ return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
- return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
+ return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
inline jlong Atomic::load(volatile jlong* src) {
--- a/hotspot/src/os_cpu/linux_zero/vm/atomic_linux_zero.inline.hpp Tue Jun 21 18:37:04 2016 +0400
+++ b/hotspot/src/os_cpu/linux_zero/vm/atomic_linux_zero.inline.hpp Tue Jun 21 19:25:41 2016 -0400
@@ -264,7 +264,8 @@
inline jint Atomic::cmpxchg(jint exchange_value,
volatile jint* dest,
- jint compare_value) {
+ jint compare_value,
+ cmpxchg_memory_order order) {
#ifdef ARM
return arm_compare_and_swap(dest, compare_value, exchange_value);
#else
@@ -278,14 +279,16 @@
inline jlong Atomic::cmpxchg(jlong exchange_value,
volatile jlong* dest,
- jlong compare_value) {
+ jlong compare_value,
+ cmpxchg_memory_order order) {
return __sync_val_compare_and_swap(dest, compare_value, exchange_value);
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value,
volatile intptr_t* dest,
- intptr_t compare_value) {
+ intptr_t compare_value,
+ cmpxchg_memory_order order) {
#ifdef ARM
return arm_compare_and_swap(dest, compare_value, exchange_value);
#else
@@ -299,11 +302,13 @@
inline void* Atomic::cmpxchg_ptr(void* exchange_value,
volatile void* dest,
- void* compare_value) {
+ void* compare_value,
+ cmpxchg_memory_order order) {
return (void *) cmpxchg_ptr((intptr_t) exchange_value,
(volatile intptr_t*) dest,
- (intptr_t) compare_value);
+ (intptr_t) compare_value,
+ order);
}
inline jlong Atomic::load(volatile jlong* src) {
--- a/hotspot/src/os_cpu/solaris_sparc/vm/atomic_solaris_sparc.inline.hpp Tue Jun 21 18:37:04 2016 +0400
+++ b/hotspot/src/os_cpu/solaris_sparc/vm/atomic_solaris_sparc.inline.hpp Tue Jun 21 19:25:41 2016 -0400
@@ -177,7 +177,7 @@
}
-inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
+inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
jint rv;
__asm__ volatile(
" cas [%2], %3, %0"
@@ -187,7 +187,7 @@
return rv;
}
-inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
+inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
#ifdef _LP64
jlong rv;
__asm__ volatile(
@@ -219,7 +219,7 @@
#endif //_LP64
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
intptr_t rv;
#ifdef _LP64
__asm__ volatile(
@@ -237,8 +237,8 @@
return rv;
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
- return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value);
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
+ return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value, order);
}
#else // _GNU_SOURCE
@@ -296,11 +296,11 @@
}
-inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
+inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
return _Atomic_cas32(exchange_value, dest, compare_value);
}
-inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
+inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
#ifdef _LP64
// Return 64 bit value in %o0
return _Atomic_cas64((intptr_t)exchange_value, (intptr_t *)dest, (intptr_t)compare_value);
@@ -310,7 +310,7 @@
#endif // _LP64
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
#ifdef _LP64
return _Atomic_cas64(exchange_value, dest, compare_value);
#else // _LP64
@@ -318,8 +318,8 @@
#endif // _LP64
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
- return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value);
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
+ return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value, order);
}
@@ -354,20 +354,20 @@
}
-inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
+inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
return (*os::atomic_cmpxchg_func)(exchange_value, dest, compare_value);
}
-inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
+inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
return (*os::atomic_cmpxchg_long_func)(exchange_value, dest, compare_value);
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
- return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
+ return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
- return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
+ return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
#endif // _LP64 || COMPILER2
--- a/hotspot/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp Tue Jun 21 18:37:04 2016 +0400
+++ b/hotspot/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp Tue Jun 21 19:25:41 2016 -0400
@@ -85,15 +85,15 @@
}
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
-inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
return _Atomic_cmpxchg_byte(exchange_value, dest, compare_value IS_MP_ARG());
}
-inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
+inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
return _Atomic_cmpxchg(exchange_value, dest, compare_value IS_MP_ARG());
}
-inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
+inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
return _Atomic_cmpxchg_long(exchange_value, dest, compare_value IS_MP_ARG());
}
@@ -120,11 +120,11 @@
return (void*)_Atomic_xchg_long((jlong)exchange_value, (volatile jlong*)dest);
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
return (intptr_t)_Atomic_cmpxchg_long((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)_Atomic_cmpxchg_long((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
}
@@ -148,12 +148,12 @@
return (void*)xchg((jint)exchange_value, (volatile jint*)dest);
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
- return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
+ return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
- return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
+ return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
extern "C" void _Atomic_move_long(volatile jlong* src, volatile jlong* dst);
--- a/hotspot/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp Tue Jun 21 18:37:04 2016 +0400
+++ b/hotspot/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp Tue Jun 21 19:25:41 2016 -0400
@@ -119,25 +119,25 @@
return (void *)(os::atomic_xchg_ptr_func)((intptr_t)exchange_value, (volatile intptr_t*)dest);
}
-inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
+inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
return (*os::atomic_cmpxchg_func)(exchange_value, dest, compare_value);
}
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
-inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
return (*os::atomic_cmpxchg_byte_func)(exchange_value, dest, compare_value);
}
-inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
+inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
return (*os::atomic_cmpxchg_long_func)(exchange_value, dest, compare_value);
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
- return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
+ return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
- return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
+ return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
inline jlong Atomic::load(volatile jlong* src) { return *src; }
@@ -218,7 +218,7 @@
}
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
-inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
// alternative for InterlockedCompareExchange
int mp = os::is_MP();
__asm {
@@ -230,7 +230,7 @@
}
}
-inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
+inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
// alternative for InterlockedCompareExchange
int mp = os::is_MP();
__asm {
@@ -242,7 +242,7 @@
}
}
-inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
+inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
int mp = os::is_MP();
jint ex_lo = (jint)exchange_value;
jint ex_hi = *( ((jint*)&exchange_value) + 1 );
@@ -263,12 +263,12 @@
}
}
-inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
- return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
+inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
+ return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
-inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
- return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
+inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
+ return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
inline jlong Atomic::load(volatile jlong* src) {
--- a/hotspot/src/share/vm/runtime/atomic.cpp Tue Jun 21 18:37:04 2016 +0400
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,103 +0,0 @@
-/*
- * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "runtime/atomic.inline.hpp"
-
-/*
- * This is the default implementation of byte-sized cmpxchg. It emulates jbyte-sized cmpxchg
- * in terms of jint-sized cmpxchg. Platforms may override this by defining their own inline definition
- * as well as defining VM_HAS_SPECIALIZED_CMPXCHG_BYTE. This will cause the platform specific
- * implementation to be used instead.
- */
-jbyte Atomic::cmpxchg_general(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
- assert(sizeof(jbyte) == 1, "assumption.");
- uintptr_t dest_addr = (uintptr_t)dest;
- uintptr_t offset = dest_addr % sizeof(jint);
- volatile jint* dest_int = (volatile jint*)(dest_addr - offset);
- jint cur = *dest_int;
- jbyte* cur_as_bytes = (jbyte*)(&cur);
- jint new_val = cur;
- jbyte* new_val_as_bytes = (jbyte*)(&new_val);
- new_val_as_bytes[offset] = exchange_value;
- while (cur_as_bytes[offset] == compare_value) {
- jint res = cmpxchg(new_val, dest_int, cur);
- if (res == cur) break;
- cur = res;
- new_val = cur;
- new_val_as_bytes[offset] = exchange_value;
- }
- return cur_as_bytes[offset];
-}
-
-unsigned Atomic::xchg(unsigned int exchange_value, volatile unsigned int* dest) {
- assert(sizeof(unsigned int) == sizeof(jint), "more work to do");
- return (unsigned int)Atomic::xchg((jint)exchange_value, (volatile jint*)dest);
-}
-
-unsigned Atomic::cmpxchg(unsigned int exchange_value,
- volatile unsigned int* dest, unsigned int compare_value) {
- assert(sizeof(unsigned int) == sizeof(jint), "more work to do");
- return (unsigned int)Atomic::cmpxchg((jint)exchange_value, (volatile jint*)dest,
- (jint)compare_value);
-}
-
-jlong Atomic::add(jlong add_value, volatile jlong* dest) {
- jlong old = load(dest);
- jlong new_value = old + add_value;
- while (old != cmpxchg(new_value, dest, old)) {
- old = load(dest);
- new_value = old + add_value;
- }
- return old;
-}
-
-void Atomic::inc(volatile short* dest) {
- // Most platforms do not support atomic increment on a 2-byte value. However,
- // if the value occupies the most significant 16 bits of an aligned 32-bit
- // word, then we can do this with an atomic add of 0x10000 to the 32-bit word.
- //
- // The least significant parts of this 32-bit word will never be affected, even
- // in case of overflow/underflow.
- //
- // Use the ATOMIC_SHORT_PAIR macro to get the desired alignment.
-#ifdef VM_LITTLE_ENDIAN
- assert((intx(dest) & 0x03) == 0x02, "wrong alignment");
- (void)Atomic::add(0x10000, (volatile int*)(dest-1));
-#else
- assert((intx(dest) & 0x03) == 0x00, "wrong alignment");
- (void)Atomic::add(0x10000, (volatile int*)(dest));
-#endif
-}
-
-void Atomic::dec(volatile short* dest) {
-#ifdef VM_LITTLE_ENDIAN
- assert((intx(dest) & 0x03) == 0x02, "wrong alignment");
- (void)Atomic::add(-0x10000, (volatile int*)(dest-1));
-#else
- assert((intx(dest) & 0x03) == 0x00, "wrong alignment");
- (void)Atomic::add(-0x10000, (volatile int*)(dest));
-#endif
-}
-
--- a/hotspot/src/share/vm/runtime/atomic.hpp Tue Jun 21 18:37:04 2016 +0400
+++ b/hotspot/src/share/vm/runtime/atomic.hpp Tue Jun 21 19:25:41 2016 -0400
@@ -27,10 +27,13 @@
#include "memory/allocation.hpp"
+enum cmpxchg_memory_order {
+ memory_order_relaxed,
+ // Use value which doesn't interfere with C++2011. We need to be more conservative.
+ memory_order_conservative = 8
+};
+
class Atomic : AllStatic {
- private:
- static jbyte cmpxchg_general(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value);
-
public:
// Atomic operations on jlong types are not available on all 32-bit
// platforms. If atomic ops on jlongs are defined here they must only
@@ -77,12 +80,12 @@
inline static intptr_t add_ptr(intptr_t add_value, volatile intptr_t* dest);
inline static void* add_ptr(intptr_t add_value, volatile void* dest);
// See comment above about using jlong atomics on 32-bit platforms
- static jlong add (jlong add_value, volatile jlong* dest);
+ inline static jlong add (jlong add_value, volatile jlong* dest);
// Atomically increment location. inc*() provide:
// <fence> increment-dest <membar StoreLoad|StoreStore>
inline static void inc (volatile jint* dest);
- static void inc (volatile jshort* dest);
+ inline static void inc (volatile jshort* dest);
inline static void inc (volatile size_t* dest);
inline static void inc_ptr(volatile intptr_t* dest);
inline static void inc_ptr(volatile void* dest);
@@ -90,7 +93,7 @@
// Atomically decrement a location. dec*() provide:
// <fence> decrement-dest <membar StoreLoad|StoreStore>
inline static void dec (volatile jint* dest);
- static void dec (volatile jshort* dest);
+ inline static void dec (volatile jshort* dest);
inline static void dec (volatile size_t* dest);
inline static void dec_ptr(volatile intptr_t* dest);
inline static void dec_ptr(volatile void* dest);
@@ -98,22 +101,22 @@
// Performs atomic exchange of *dest with exchange_value. Returns old
// prior value of *dest. xchg*() provide:
// <fence> exchange-value-with-dest <membar StoreLoad|StoreStore>
- inline static jint xchg (jint exchange_value, volatile jint* dest);
- static unsigned int xchg (unsigned int exchange_value, volatile unsigned int* dest);
- inline static intptr_t xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest);
- inline static void* xchg_ptr(void* exchange_value, volatile void* dest);
+ inline static jint xchg (jint exchange_value, volatile jint* dest);
+ inline static unsigned int xchg (unsigned int exchange_value, volatile unsigned int* dest);
+ inline static intptr_t xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest);
+ inline static void* xchg_ptr(void* exchange_value, volatile void* dest);
// Performs atomic compare of *dest and compare_value, and exchanges
// *dest with exchange_value if the comparison succeeded. Returns prior
// value of *dest. cmpxchg*() provide:
// <fence> compare-and-exchange <membar StoreLoad|StoreStore>
- inline static jbyte cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value);
- inline static jint cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value);
+ inline static jbyte cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order = memory_order_conservative);
+ inline static jint cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order = memory_order_conservative);
// See comment above about using jlong atomics on 32-bit platforms
- inline static jlong cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value);
- static unsigned int cmpxchg (unsigned int exchange_value, volatile unsigned int* dest, unsigned int compare_value);
- inline static intptr_t cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value);
- inline static void* cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value);
+ inline static jlong cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order = memory_order_conservative);
+ inline static unsigned int cmpxchg (unsigned int exchange_value, volatile unsigned int* dest, unsigned int compare_value, cmpxchg_memory_order order = memory_order_conservative);
+ inline static intptr_t cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order = memory_order_conservative);
+ inline static void* cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order = memory_order_conservative);
};
// To use Atomic::inc(jshort* dest) and Atomic::dec(jshort* dest), the address must be specially
--- a/hotspot/src/share/vm/runtime/atomic.inline.hpp Tue Jun 21 18:37:04 2016 +0400
+++ b/hotspot/src/share/vm/runtime/atomic.inline.hpp Tue Jun 21 19:25:41 2016 -0400
@@ -91,11 +91,83 @@
}
#ifndef VM_HAS_SPECIALIZED_CMPXCHG_BYTE
-// See comment in atomic.cpp how to override.
-inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte *dest, jbyte comparand)
+/*
+ * This is the default implementation of byte-sized cmpxchg. It emulates jbyte-sized cmpxchg
+ * in terms of jint-sized cmpxchg. Platforms may override this by defining their own inline definition
+ * as well as defining VM_HAS_SPECIALIZED_CMPXCHG_BYTE. This will cause the platform specific
+ * implementation to be used instead.
+ */
+inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte *dest, jbyte comparand, cmpxchg_memory_order order)
{
- return cmpxchg_general(exchange_value, dest, comparand);
+ assert(sizeof(jbyte) == 1, "assumption.");
+ uintptr_t dest_addr = (uintptr_t)dest;
+ uintptr_t offset = dest_addr % sizeof(jint);
+ volatile jint* dest_int = (volatile jint*)(dest_addr - offset);
+ jint cur = *dest_int;
+ jbyte* cur_as_bytes = (jbyte*)(&cur);
+ jint new_val = cur;
+ jbyte* new_val_as_bytes = (jbyte*)(&new_val);
+ new_val_as_bytes[offset] = exchange_value;
+ while (cur_as_bytes[offset] == comparand) {
+ jint res = cmpxchg(new_val, dest_int, cur, order);
+ if (res == cur) break;
+ cur = res;
+ new_val = cur;
+ new_val_as_bytes[offset] = exchange_value;
+ }
+ return cur_as_bytes[offset];
}
#endif // VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+inline unsigned Atomic::xchg(unsigned int exchange_value, volatile unsigned int* dest) {
+ assert(sizeof(unsigned int) == sizeof(jint), "more work to do");
+ return (unsigned int)Atomic::xchg((jint)exchange_value, (volatile jint*)dest);
+}
+
+inline unsigned Atomic::cmpxchg(unsigned int exchange_value,
+ volatile unsigned int* dest, unsigned int compare_value,
+ cmpxchg_memory_order order) {
+ assert(sizeof(unsigned int) == sizeof(jint), "more work to do");
+ return (unsigned int)Atomic::cmpxchg((jint)exchange_value, (volatile jint*)dest,
+ (jint)compare_value, order);
+}
+
+inline jlong Atomic::add(jlong add_value, volatile jlong* dest) {
+ jlong old = load(dest);
+ jlong new_value = old + add_value;
+ while (old != cmpxchg(new_value, dest, old)) {
+ old = load(dest);
+ new_value = old + add_value;
+ }
+ return old;
+}
+
+inline void Atomic::inc(volatile short* dest) {
+ // Most platforms do not support atomic increment on a 2-byte value. However,
+ // if the value occupies the most significant 16 bits of an aligned 32-bit
+ // word, then we can do this with an atomic add of 0x10000 to the 32-bit word.
+ //
+ // The least significant parts of this 32-bit word will never be affected, even
+ // in case of overflow/underflow.
+ //
+ // Use the ATOMIC_SHORT_PAIR macro to get the desired alignment.
+#ifdef VM_LITTLE_ENDIAN
+ assert((intx(dest) & 0x03) == 0x02, "wrong alignment");
+ (void)Atomic::add(0x10000, (volatile int*)(dest-1));
+#else
+ assert((intx(dest) & 0x03) == 0x00, "wrong alignment");
+ (void)Atomic::add(0x10000, (volatile int*)(dest));
+#endif
+}
+
+inline void Atomic::dec(volatile short* dest) {
+#ifdef VM_LITTLE_ENDIAN
+ assert((intx(dest) & 0x03) == 0x02, "wrong alignment");
+ (void)Atomic::add(-0x10000, (volatile int*)(dest-1));
+#else
+ assert((intx(dest) & 0x03) == 0x00, "wrong alignment");
+ (void)Atomic::add(-0x10000, (volatile int*)(dest));
+#endif
+}
+
#endif // SHARE_VM_RUNTIME_ATOMIC_INLINE_HPP