8058255: Native jbyte Atomic::cmpxchg for supported x86 platforms
Summary: Use the native cmpxchgb instruction on x86.
Reviewed-by: dholmes, kbarrett, phh
Contributed-by: erik.osterlund@lnu.se
--- a/hotspot/src/cpu/sparc/vm/stubGenerator_sparc.cpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/cpu/sparc/vm/stubGenerator_sparc.cpp Tue Oct 21 15:07:25 2014 +0200
@@ -4813,6 +4813,7 @@
StubRoutines::_atomic_add_entry = generate_atomic_add();
StubRoutines::_atomic_xchg_ptr_entry = StubRoutines::_atomic_xchg_entry;
StubRoutines::_atomic_cmpxchg_ptr_entry = StubRoutines::_atomic_cmpxchg_entry;
+ StubRoutines::_atomic_cmpxchg_byte_entry = ShouldNotCallThisStub();
StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long();
StubRoutines::_atomic_add_ptr_entry = StubRoutines::_atomic_add_entry;
#endif // COMPILER2 !=> _LP64
--- a/hotspot/src/cpu/x86/vm/assembler_x86.cpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/cpu/x86/vm/assembler_x86.cpp Tue Oct 21 15:07:25 2014 +0200
@@ -1297,6 +1297,17 @@
emit_operand(reg, adr);
}
+// The 8-bit cmpxchg compares the value at adr with the contents of rax,
+// and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
+// The ZF is set if the compared values were equal, and cleared otherwise.
+void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
+ InstructionMark im(this);
+ prefix(adr, reg, true);
+ emit_int8(0x0F);
+ emit_int8((unsigned char)0xB0);
+ emit_operand(reg, adr);
+}
+
void Assembler::comisd(XMMRegister dst, Address src) {
// NOTE: dbx seems to decode this as comiss even though the
// 0x66 is there. Strangly ucomisd comes out correct
--- a/hotspot/src/cpu/x86/vm/assembler_x86.hpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/cpu/x86/vm/assembler_x86.hpp Tue Oct 21 15:07:25 2014 +0200
@@ -1006,6 +1006,7 @@
void cmpxchg8 (Address adr);
+ void cmpxchgb(Register reg, Address adr);
void cmpxchgl(Register reg, Address adr);
void cmpxchgq(Register reg, Address adr);
--- a/hotspot/src/cpu/x86/vm/stubGenerator_x86_64.cpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/cpu/x86/vm/stubGenerator_x86_64.cpp Tue Oct 21 15:07:25 2014 +0200
@@ -594,9 +594,35 @@
return start;
}
- // Support for jint atomic::atomic_cmpxchg_long(jlong exchange_value,
- // volatile jlong* dest,
- // jlong compare_value)
+ // Support for jbyte atomic::atomic_cmpxchg(jbyte exchange_value, volatile jbyte* dest,
+ // jbyte compare_value)
+ //
+ // Arguments :
+ // c_rarg0: exchange_value
+ // c_rarg1: dest
+ // c_rarg2: compare_value
+ //
+ // Result:
+ // if ( compare_value == *dest ) {
+ // *dest = exchange_value
+ // return compare_value;
+ // else
+ // return *dest;
+ address generate_atomic_cmpxchg_byte() {
+ StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_byte");
+ address start = __ pc();
+
+ __ movsbq(rax, c_rarg2);
+ if ( os::is_MP() ) __ lock();
+ __ cmpxchgb(c_rarg0, Address(c_rarg1, 0));
+ __ ret(0);
+
+ return start;
+ }
+
+ // Support for jlong atomic::atomic_cmpxchg(jlong exchange_value,
+ // volatile jlong* dest,
+ // jlong compare_value)
// Arguments :
// c_rarg0: exchange_value
// c_rarg1: dest
@@ -3894,6 +3920,7 @@
StubRoutines::_atomic_xchg_entry = generate_atomic_xchg();
StubRoutines::_atomic_xchg_ptr_entry = generate_atomic_xchg_ptr();
StubRoutines::_atomic_cmpxchg_entry = generate_atomic_cmpxchg();
+ StubRoutines::_atomic_cmpxchg_byte_entry = generate_atomic_cmpxchg_byte();
StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long();
StubRoutines::_atomic_add_entry = generate_atomic_add();
StubRoutines::_atomic_add_ptr_entry = generate_atomic_add_ptr();
--- a/hotspot/src/cpu/zero/vm/stubGenerator_zero.cpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/cpu/zero/vm/stubGenerator_zero.cpp Tue Oct 21 15:07:25 2014 +0200
@@ -207,6 +207,7 @@
StubRoutines::_atomic_xchg_ptr_entry = ShouldNotCallThisStub();
StubRoutines::_atomic_cmpxchg_entry = ShouldNotCallThisStub();
StubRoutines::_atomic_cmpxchg_ptr_entry = ShouldNotCallThisStub();
+ StubRoutines::_atomic_cmpxchg_byte_entry = ShouldNotCallThisStub();
StubRoutines::_atomic_cmpxchg_long_entry = ShouldNotCallThisStub();
StubRoutines::_atomic_add_entry = ShouldNotCallThisStub();
StubRoutines::_atomic_add_ptr_entry = ShouldNotCallThisStub();
--- a/hotspot/src/os_cpu/bsd_x86/vm/atomic_bsd_x86.inline.hpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/os_cpu/bsd_x86/vm/atomic_bsd_x86.inline.hpp Tue Oct 21 15:07:25 2014 +0200
@@ -88,6 +88,15 @@
return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
}
+#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+ int mp = os::is_MP();
+ __asm__ volatile (LOCK_IF_MP(%4) "cmpxchgb %1,(%3)"
+ : "=a" (exchange_value)
+ : "q" (exchange_value), "a" (compare_value), "r" (dest), "r" (mp)
+ : "cc", "memory");
+ return exchange_value;
+}
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
int mp = os::is_MP();
--- a/hotspot/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp Tue Oct 21 15:07:25 2014 +0200
@@ -88,6 +88,15 @@
return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
}
+#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+ int mp = os::is_MP();
+ __asm__ volatile (LOCK_IF_MP(%4) "cmpxchgb %1,(%3)"
+ : "=a" (exchange_value)
+ : "q" (exchange_value), "a" (compare_value), "r" (dest), "r" (mp)
+ : "cc", "memory");
+ return exchange_value;
+}
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
int mp = os::is_MP();
--- a/hotspot/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp Tue Oct 21 15:07:25 2014 +0200
@@ -68,6 +68,8 @@
extern "C" {
jint _Atomic_add(jint add_value, volatile jint* dest IS_MP_DECL());
jint _Atomic_xchg(jint exchange_value, volatile jint* dest);
+ jbyte _Atomic_cmpxchg_byte(jbyte exchange_value, volatile jbyte* dest,
+ jbyte compare_value IS_MP_DECL());
jint _Atomic_cmpxchg(jint exchange_value, volatile jint* dest,
jint compare_value IS_MP_DECL());
jlong _Atomic_cmpxchg_long(jlong exchange_value, volatile jlong* dest,
@@ -82,6 +84,11 @@
return _Atomic_xchg(exchange_value, dest);
}
+#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+ return _Atomic_cmpxchg_byte(exchange_value, dest, compare_value IS_MP_ARG());
+}
+
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
return _Atomic_cmpxchg(exchange_value, dest, compare_value IS_MP_ARG());
}
@@ -217,6 +224,15 @@
return exchange_value;
}
+
+ inline jbyte _Atomic_cmpxchg_byte(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, int mp) {
+ __asm__ volatile (LOCK_IF_MP(%4) "cmpxchgb %1,(%3)"
+ : "=a" (exchange_value)
+ : "q" (exchange_value), "a" (compare_value), "r" (dest), "r" (mp)
+ : "cc", "memory");
+ return exchange_value;
+ }
+
// This is the interface to the atomic instruction in solaris_i486.s.
jlong _Atomic_cmpxchg_long_gcc(jlong exchange_value, volatile jlong* dest, jlong compare_value, int mp);
--- a/hotspot/src/os_cpu/solaris_x86/vm/solaris_x86_32.il Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/os_cpu/solaris_x86/vm/solaris_x86_32.il Tue Oct 21 15:07:25 2014 +0200
@@ -76,6 +76,23 @@
xchgl (%ecx), %eax
.end
+ // Support for jbyte Atomic::cmpxchg(jbyte exchange_value,
+ // volatile jbyte *dest,
+ // jbyte compare_value)
+ // An additional bool (os::is_MP()) is passed as the last argument.
+ .inline _Atomic_cmpxchg_byte,4
+ movb 8(%esp), %al // compare_value
+ movb 0(%esp), %cl // exchange_value
+ movl 4(%esp), %edx // dest
+ cmp $0, 12(%esp) // MP test
+ jne 1f
+ cmpxchgb %cl, (%edx)
+ jmp 2f
+1: lock
+ cmpxchgb %cl, (%edx)
+2:
+ .end
+
// Support for jint Atomic::cmpxchg(jint exchange_value,
// volatile jint *dest,
// jint compare_value)
--- a/hotspot/src/os_cpu/solaris_x86/vm/solaris_x86_64.il Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/os_cpu/solaris_x86/vm/solaris_x86_64.il Tue Oct 21 15:07:25 2014 +0200
@@ -77,6 +77,15 @@
movq %rdi, %rax
.end
+ // Support for jbyte Atomic::cmpxchg(jbyte exchange_value,
+ // volatile jbyte *dest,
+ // jbyte compare_value)
+ .inline _Atomic_cmpxchg_byte,3
+ movb %dl, %al // compare_value
+ lock
+ cmpxchgb %dil, (%rsi)
+ .end
+
// Support for jint Atomic::cmpxchg(jint exchange_value,
// volatile jint *dest,
// jint compare_value)
--- a/hotspot/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp Tue Oct 21 15:07:25 2014 +0200
@@ -123,6 +123,11 @@
return (*os::atomic_cmpxchg_func)(exchange_value, dest, compare_value);
}
+#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+ return (*os::atomic_cmpxchg_byte_func)(exchange_value, dest, compare_value);
+}
+
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
return (*os::atomic_cmpxchg_long_func)(exchange_value, dest, compare_value);
}
@@ -212,6 +217,19 @@
return (void*)xchg((jint)exchange_value, (volatile jint*)dest);
}
+#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+ // alternative for InterlockedCompareExchange
+ int mp = os::is_MP();
+ __asm {
+ mov edx, dest
+ mov cl, exchange_value
+ mov al, compare_value
+ LOCK_IF_MP(mp)
+ cmpxchg byte ptr [edx], cl
+ }
+}
+
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
// alternative for InterlockedCompareExchange
int mp = os::is_MP();
--- a/hotspot/src/os_cpu/windows_x86/vm/os_windows_x86.cpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/os_cpu/windows_x86/vm/os_windows_x86.cpp Tue Oct 21 15:07:25 2014 +0200
@@ -220,6 +220,7 @@
typedef jint xchg_func_t (jint, volatile jint*);
typedef intptr_t xchg_ptr_func_t (intptr_t, volatile intptr_t*);
typedef jint cmpxchg_func_t (jint, volatile jint*, jint);
+typedef jbyte cmpxchg_byte_func_t (jbyte, volatile jbyte*, jbyte);
typedef jlong cmpxchg_long_func_t (jlong, volatile jlong*, jlong);
typedef jint add_func_t (jint, volatile jint*);
typedef intptr_t add_ptr_func_t (intptr_t, volatile intptr_t*);
@@ -272,6 +273,23 @@
*dest = exchange_value;
return old_value;
}
+
+jbyte os::atomic_cmpxchg_byte_bootstrap(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+ // try to use the stub:
+ cmpxchg_byte_func_t* func = CAST_TO_FN_PTR(cmpxchg_byte_func_t*, StubRoutines::atomic_cmpxchg_byte_entry());
+
+ if (func != NULL) {
+ os::atomic_cmpxchg_byte_func = func;
+ return (*func)(exchange_value, dest, compare_value);
+ }
+ assert(Threads::number_of_threads() == 0, "for bootstrap only");
+
+ jbyte old_value = *dest;
+ if (old_value == compare_value)
+ *dest = exchange_value;
+ return old_value;
+}
+
#endif // AMD64
jlong os::atomic_cmpxchg_long_bootstrap(jlong exchange_value, volatile jlong* dest, jlong compare_value) {
@@ -321,6 +339,7 @@
xchg_func_t* os::atomic_xchg_func = os::atomic_xchg_bootstrap;
xchg_ptr_func_t* os::atomic_xchg_ptr_func = os::atomic_xchg_ptr_bootstrap;
cmpxchg_func_t* os::atomic_cmpxchg_func = os::atomic_cmpxchg_bootstrap;
+cmpxchg_byte_func_t* os::atomic_cmpxchg_byte_func = os::atomic_cmpxchg_byte_bootstrap;
add_func_t* os::atomic_add_func = os::atomic_add_bootstrap;
add_ptr_func_t* os::atomic_add_ptr_func = os::atomic_add_ptr_bootstrap;
--- a/hotspot/src/os_cpu/windows_x86/vm/os_windows_x86.hpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/os_cpu/windows_x86/vm/os_windows_x86.hpp Tue Oct 21 15:07:25 2014 +0200
@@ -33,6 +33,7 @@
static intptr_t (*atomic_xchg_ptr_func) (intptr_t, volatile intptr_t*);
static jint (*atomic_cmpxchg_func) (jint, volatile jint*, jint);
+ static jbyte (*atomic_cmpxchg_byte_func) (jbyte, volatile jbyte*, jbyte);
static jlong (*atomic_cmpxchg_long_func) (jlong, volatile jlong*, jlong);
static jint (*atomic_add_func) (jint, volatile jint*);
@@ -42,6 +43,7 @@
static intptr_t atomic_xchg_ptr_bootstrap (intptr_t, volatile intptr_t*);
static jint atomic_cmpxchg_bootstrap (jint, volatile jint*, jint);
+ static jbyte atomic_cmpxchg_byte_bootstrap(jbyte, volatile jbyte*, jbyte);
#else
static jlong (*atomic_cmpxchg_long_func) (jlong, volatile jlong*, jlong);
--- a/hotspot/src/share/vm/runtime/atomic.cpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/share/vm/runtime/atomic.cpp Tue Oct 21 15:07:25 2014 +0200
@@ -25,7 +25,13 @@
#include "precompiled.hpp"
#include "runtime/atomic.inline.hpp"
-jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
+/*
+ * This is the default implementation of byte-sized cmpxchg. It emulates jbyte-sized cmpxchg
+ * in terms of jint-sized cmpxchg. Platforms may override this by defining their own inline definition
+ * as well as defining VM_HAS_SPECIALIZED_CMPXCHG_BYTE. This will cause the platform specific
+ * implementation to be used instead.
+ */
+jbyte Atomic::cmpxchg_general(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
assert(sizeof(jbyte) == 1, "assumption.");
uintptr_t dest_addr = (uintptr_t)dest;
uintptr_t offset = dest_addr % sizeof(jint);
--- a/hotspot/src/share/vm/runtime/atomic.hpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/share/vm/runtime/atomic.hpp Tue Oct 21 15:07:25 2014 +0200
@@ -28,6 +28,9 @@
#include "memory/allocation.hpp"
class Atomic : AllStatic {
+ private:
+ static jbyte cmpxchg_general(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value);
+
public:
// Atomic operations on jlong types are not available on all 32-bit
// platforms. If atomic ops on jlongs are defined here they must only
@@ -104,7 +107,7 @@
// *dest with exchange_value if the comparison succeeded. Returns prior
// value of *dest. cmpxchg*() provide:
// <fence> compare-and-exchange <membar StoreLoad|StoreStore>
- static jbyte cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value);
+ inline static jbyte cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value);
inline static jint cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value);
// See comment above about using jlong atomics on 32-bit platforms
inline static jlong cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value);
--- a/hotspot/src/share/vm/runtime/atomic.inline.hpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/share/vm/runtime/atomic.inline.hpp Tue Oct 21 15:07:25 2014 +0200
@@ -87,4 +87,12 @@
dec_ptr((volatile intptr_t*) dest);
}
+#ifndef VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+// See comment in atomic.cpp how to override.
+inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte *dest, jbyte comparand)
+{
+ return cmpxchg_general(exchange_value, dest, comparand);
+}
+#endif // VM_HAS_SPECIALIZED_CMPXCHG_BYTE
+
#endif // SHARE_VM_RUNTIME_ATOMIC_INLINE_HPP
--- a/hotspot/src/share/vm/runtime/stubRoutines.cpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/share/vm/runtime/stubRoutines.cpp Tue Oct 21 15:07:25 2014 +0200
@@ -62,6 +62,7 @@
address StubRoutines::_atomic_store_ptr_entry = NULL;
address StubRoutines::_atomic_cmpxchg_entry = NULL;
address StubRoutines::_atomic_cmpxchg_ptr_entry = NULL;
+address StubRoutines::_atomic_cmpxchg_byte_entry = NULL;
address StubRoutines::_atomic_cmpxchg_long_entry = NULL;
address StubRoutines::_atomic_add_entry = NULL;
address StubRoutines::_atomic_add_ptr_entry = NULL;
--- a/hotspot/src/share/vm/runtime/stubRoutines.hpp Tue Nov 18 12:33:47 2014 +0000
+++ b/hotspot/src/share/vm/runtime/stubRoutines.hpp Tue Oct 21 15:07:25 2014 +0200
@@ -126,6 +126,7 @@
static address _atomic_store_ptr_entry;
static address _atomic_cmpxchg_entry;
static address _atomic_cmpxchg_ptr_entry;
+ static address _atomic_cmpxchg_byte_entry;
static address _atomic_cmpxchg_long_entry;
static address _atomic_add_entry;
static address _atomic_add_ptr_entry;
@@ -282,6 +283,7 @@
static address atomic_store_ptr_entry() { return _atomic_store_ptr_entry; }
static address atomic_cmpxchg_entry() { return _atomic_cmpxchg_entry; }
static address atomic_cmpxchg_ptr_entry() { return _atomic_cmpxchg_ptr_entry; }
+ static address atomic_cmpxchg_byte_entry() { return _atomic_cmpxchg_byte_entry; }
static address atomic_cmpxchg_long_entry() { return _atomic_cmpxchg_long_entry; }
static address atomic_add_entry() { return _atomic_add_entry; }
static address atomic_add_ptr_entry() { return _atomic_add_ptr_entry; }