--- a/hotspot/src/cpu/x86/vm/x86_64.ad Fri Feb 21 08:09:15 2014 -0800
+++ b/hotspot/src/cpu/x86/vm/x86_64.ad Mon Feb 24 15:12:26 2014 -0800
@@ -2599,231 +2599,6 @@
%}
- // obj: object to lock
- // box: box address (header location) -- killed
- // tmp: rax -- killed
- // scr: rbx -- killed
- //
- // What follows is a direct transliteration of fast_lock() and fast_unlock()
- // from i486.ad. See that file for comments.
- // TODO: where possible switch from movq (r, 0) to movl(r,0) and
- // use the shorter encoding. (Movl clears the high-order 32-bits).
-
-
- enc_class Fast_Lock(rRegP obj, rRegP box, rax_RegI tmp, rRegP scr)
- %{
- Register objReg = as_Register((int)$obj$$reg);
- Register boxReg = as_Register((int)$box$$reg);
- Register tmpReg = as_Register($tmp$$reg);
- Register scrReg = as_Register($scr$$reg);
- MacroAssembler masm(&cbuf);
-
- // Verify uniqueness of register assignments -- necessary but not sufficient
- assert (objReg != boxReg && objReg != tmpReg &&
- objReg != scrReg && tmpReg != scrReg, "invariant") ;
-
- if (_counters != NULL) {
- masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr()));
- }
- if (EmitSync & 1) {
- // Without cast to int32_t a movptr will destroy r10 which is typically obj
- masm.movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
- masm.cmpptr(rsp, (int32_t)NULL_WORD) ;
- } else
- if (EmitSync & 2) {
- Label DONE_LABEL;
- if (UseBiasedLocking) {
- // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.
- masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
- }
- // QQQ was movl...
- masm.movptr(tmpReg, 0x1);
- masm.orptr(tmpReg, Address(objReg, 0));
- masm.movptr(Address(boxReg, 0), tmpReg);
- if (os::is_MP()) {
- masm.lock();
- }
- masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
- masm.jcc(Assembler::equal, DONE_LABEL);
-
- // Recursive locking
- masm.subptr(tmpReg, rsp);
- masm.andptr(tmpReg, 7 - os::vm_page_size());
- masm.movptr(Address(boxReg, 0), tmpReg);
-
- masm.bind(DONE_LABEL);
- masm.nop(); // avoid branch to branch
- } else {
- Label DONE_LABEL, IsInflated, Egress;
-
- masm.movptr(tmpReg, Address(objReg, 0)) ;
- masm.testl (tmpReg, 0x02) ; // inflated vs stack-locked|neutral|biased
- masm.jcc (Assembler::notZero, IsInflated) ;
-
- // it's stack-locked, biased or neutral
- // TODO: optimize markword triage order to reduce the number of
- // conditional branches in the most common cases.
- // Beware -- there's a subtle invariant that fetch of the markword
- // at [FETCH], below, will never observe a biased encoding (*101b).
- // If this invariant is not held we'll suffer exclusion (safety) failure.
-
- if (UseBiasedLocking && !UseOptoBiasInlining) {
- masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, _counters);
- masm.movptr(tmpReg, Address(objReg, 0)) ; // [FETCH]
- }
-
- // was q will it destroy high?
- masm.orl (tmpReg, 1) ;
- masm.movptr(Address(boxReg, 0), tmpReg) ;
- if (os::is_MP()) { masm.lock(); }
- masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
- if (_counters != NULL) {
- masm.cond_inc32(Assembler::equal,
- ExternalAddress((address) _counters->fast_path_entry_count_addr()));
- }
- masm.jcc (Assembler::equal, DONE_LABEL);
-
- // Recursive locking
- masm.subptr(tmpReg, rsp);
- masm.andptr(tmpReg, 7 - os::vm_page_size());
- masm.movptr(Address(boxReg, 0), tmpReg);
- if (_counters != NULL) {
- masm.cond_inc32(Assembler::equal,
- ExternalAddress((address) _counters->fast_path_entry_count_addr()));
- }
- masm.jmp (DONE_LABEL) ;
-
- masm.bind (IsInflated) ;
- // It's inflated
-
- // TODO: someday avoid the ST-before-CAS penalty by
- // relocating (deferring) the following ST.
- // We should also think about trying a CAS without having
- // fetched _owner. If the CAS is successful we may
- // avoid an RTO->RTS upgrade on the $line.
- // Without cast to int32_t a movptr will destroy r10 which is typically obj
- masm.movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
-
- masm.mov (boxReg, tmpReg) ;
- masm.movptr (tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
- masm.testptr(tmpReg, tmpReg) ;
- masm.jcc (Assembler::notZero, DONE_LABEL) ;
-
- // It's inflated and appears unlocked
- if (os::is_MP()) { masm.lock(); }
- masm.cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
- // Intentional fall-through into DONE_LABEL ...
-
- masm.bind (DONE_LABEL) ;
- masm.nop () ; // avoid jmp to jmp
- }
- %}
-
- // obj: object to unlock
- // box: box address (displaced header location), killed
- // RBX: killed tmp; cannot be obj nor box
- enc_class Fast_Unlock(rRegP obj, rax_RegP box, rRegP tmp)
- %{
-
- Register objReg = as_Register($obj$$reg);
- Register boxReg = as_Register($box$$reg);
- Register tmpReg = as_Register($tmp$$reg);
- MacroAssembler masm(&cbuf);
-
- if (EmitSync & 4) {
- masm.cmpptr(rsp, 0) ;
- } else
- if (EmitSync & 8) {
- Label DONE_LABEL;
- if (UseBiasedLocking) {
- masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
- }
-
- // Check whether the displaced header is 0
- //(=> recursive unlock)
- masm.movptr(tmpReg, Address(boxReg, 0));
- masm.testptr(tmpReg, tmpReg);
- masm.jcc(Assembler::zero, DONE_LABEL);
-
- // If not recursive lock, reset the header to displaced header
- if (os::is_MP()) {
- masm.lock();
- }
- masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
- masm.bind(DONE_LABEL);
- masm.nop(); // avoid branch to branch
- } else {
- Label DONE_LABEL, Stacked, CheckSucc ;
-
- if (UseBiasedLocking && !UseOptoBiasInlining) {
- masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
- }
-
- masm.movptr(tmpReg, Address(objReg, 0)) ;
- masm.cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD) ;
- masm.jcc (Assembler::zero, DONE_LABEL) ;
- masm.testl (tmpReg, 0x02) ;
- masm.jcc (Assembler::zero, Stacked) ;
-
- // It's inflated
- masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
- masm.xorptr(boxReg, r15_thread) ;
- masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ;
- masm.jcc (Assembler::notZero, DONE_LABEL) ;
- masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ;
- masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ;
- masm.jcc (Assembler::notZero, CheckSucc) ;
- masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
- masm.jmp (DONE_LABEL) ;
-
- if ((EmitSync & 65536) == 0) {
- Label LSuccess, LGoSlowPath ;
- masm.bind (CheckSucc) ;
- masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
- masm.jcc (Assembler::zero, LGoSlowPath) ;
-
- // I'd much rather use lock:andl m->_owner, 0 as it's faster than the
- // the explicit ST;MEMBAR combination, but masm doesn't currently support
- // "ANDQ M,IMM". Don't use MFENCE here. lock:add to TOS, xchg, etc
- // are all faster when the write buffer is populated.
- masm.movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
- if (os::is_MP()) {
- masm.lock () ; masm.addl (Address(rsp, 0), 0) ;
- }
- masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
- masm.jcc (Assembler::notZero, LSuccess) ;
-
- masm.movptr (boxReg, (int32_t)NULL_WORD) ; // box is really EAX
- if (os::is_MP()) { masm.lock(); }
- masm.cmpxchgptr(r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
- masm.jcc (Assembler::notEqual, LSuccess) ;
- // Intentional fall-through into slow-path
-
- masm.bind (LGoSlowPath) ;
- masm.orl (boxReg, 1) ; // set ICC.ZF=0 to indicate failure
- masm.jmp (DONE_LABEL) ;
-
- masm.bind (LSuccess) ;
- masm.testl (boxReg, 0) ; // set ICC.ZF=1 to indicate success
- masm.jmp (DONE_LABEL) ;
- }
-
- masm.bind (Stacked) ;
- masm.movptr(tmpReg, Address (boxReg, 0)) ; // re-fetch
- if (os::is_MP()) { masm.lock(); }
- masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
-
- if (EmitSync & 65536) {
- masm.bind (CheckSucc) ;
- }
- masm.bind(DONE_LABEL);
- if (EmitSync & 32768) {
- masm.nop(); // avoid branch to branch
- }
- }
- %}
-
-
enc_class enc_rethrow()
%{
cbuf.set_insts_mark();
@@ -11453,27 +11228,25 @@
// ============================================================================
// inlined locking and unlocking
-instruct cmpFastLock(rFlagsReg cr,
- rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr)
-%{
+instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
match(Set cr (FastLock object box));
effect(TEMP tmp, TEMP scr, USE_KILL box);
-
ins_cost(300);
format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
- ins_encode(Fast_Lock(object, box, tmp, scr));
+ ins_encode %{
+ __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $scr$$Register, _counters);
+ %}
ins_pipe(pipe_slow);
%}
-instruct cmpFastUnlock(rFlagsReg cr,
- rRegP object, rax_RegP box, rRegP tmp)
-%{
+instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
match(Set cr (FastUnlock object box));
effect(TEMP tmp, USE_KILL box);
-
ins_cost(300);
format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
- ins_encode(Fast_Unlock(object, box, tmp));
+ ins_encode %{
+ __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
+ %}
ins_pipe(pipe_slow);
%}