3017 // the ST of 0 into _owner which releases the lock. This prevents loads |
3017 // the ST of 0 into _owner which releases the lock. This prevents loads |
3018 // and stores within the critical section from reordering (floating) |
3018 // and stores within the critical section from reordering (floating) |
3019 // past the store that releases the lock. But TSO is a strong memory model |
3019 // past the store that releases the lock. But TSO is a strong memory model |
3020 // and that particular flavor of barrier is a noop, so we can safely elide it. |
3020 // and that particular flavor of barrier is a noop, so we can safely elide it. |
3021 // Note that we use 1-0 locking by default for the inflated case. We |
3021 // Note that we use 1-0 locking by default for the inflated case. We |
3022 // close the resultant (and rare) race by having contented threads in |
3022 // close the resultant (and rare) race by having contended threads in |
3023 // monitorenter periodically poll _owner. |
3023 // monitorenter periodically poll _owner. |
3024 ld_ptr(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), Rscratch); |
3024 |
3025 ld_ptr(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions), Rbox); |
3025 if (EmitSync & 1024) { |
3026 xor3(Rscratch, G2_thread, Rscratch); |
3026 // Emit code to check that _owner == Self |
3027 orcc(Rbox, Rscratch, Rbox); |
3027 // We could fold the _owner test into subsequent code more efficiently |
3028 brx(Assembler::notZero, false, Assembler::pn, done); |
3028 // than using a stand-alone check, but since _owner checking is off by |
3029 delayed()-> |
3029 // default we don't bother. We also might consider predicating the |
3030 ld_ptr(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList), Rscratch); |
3030 // _owner==Self check on Xcheck:jni or running on a debug build. |
3031 ld_ptr(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq), Rbox); |
3031 ld_ptr(Address(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), Rscratch); |
3032 orcc(Rbox, Rscratch, G0); |
3032 orcc(Rscratch, G0, G0); |
3033 if (EmitSync & 65536) { |
3033 brx(Assembler::notZero, false, Assembler::pn, done); |
3034 Label LSucc ; |
3034 delayed()->nop(); |
3035 brx(Assembler::notZero, false, Assembler::pn, LSucc); |
3035 } |
3036 delayed()->nop(); |
3036 |
3037 ba(done); |
3037 if (EmitSync & 512) { |
3038 delayed()->st_ptr(G0, Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)); |
3038 // classic lock release code absent 1-0 locking |
3039 |
3039 // m->Owner = null; |
3040 bind(LSucc); |
3040 // membar #storeload |
3041 st_ptr(G0, Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)); |
3041 // if (m->cxq|m->EntryList) == null goto Success |
3042 if (os::is_MP()) { membar (StoreLoad); } |
3042 // if (m->succ != null) goto Success |
3043 ld_ptr(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ), Rscratch); |
3043 // if CAS (&m->Owner,0,Self) != 0 goto Success |
3044 andcc(Rscratch, Rscratch, G0); |
3044 // goto SlowPath |
3045 brx(Assembler::notZero, false, Assembler::pt, done); |
3045 ld_ptr(Address(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)), Rbox); |
3046 delayed()->andcc(G0, G0, G0); |
3046 orcc(Rbox, G0, G0); |
3047 add(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), Rmark); |
3047 brx(Assembler::notZero, false, Assembler::pn, done); |
3048 mov(G2_thread, Rscratch); |
3048 delayed()->nop(); |
3049 cas_ptr(Rmark, G0, Rscratch); |
3049 st_ptr(G0, Address(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
3050 // invert icc.zf and goto done |
3050 if (os::is_MP()) { membar(StoreLoad); } |
3051 br_notnull(Rscratch, false, Assembler::pt, done); |
3051 ld_ptr(Address(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)), Rscratch); |
3052 delayed()->cmp(G0, G0); |
3052 ld_ptr(Address(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)), Rbox); |
3053 ba(done); |
3053 orcc(Rbox, Rscratch, G0); |
3054 delayed()->cmp(G0, 1); |
3054 brx(Assembler::zero, false, Assembler::pt, done); |
|
3055 delayed()-> |
|
3056 ld_ptr(Address(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), Rscratch); |
|
3057 andcc(Rscratch, Rscratch, G0); |
|
3058 brx(Assembler::notZero, false, Assembler::pt, done); |
|
3059 delayed()->andcc(G0, G0, G0); |
|
3060 add(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), Rmark); |
|
3061 mov(G2_thread, Rscratch); |
|
3062 cas_ptr(Rmark, G0, Rscratch); |
|
3063 cmp(Rscratch, G0); |
|
3064 // invert icc.zf and goto done |
|
3065 brx(Assembler::notZero, false, Assembler::pt, done); |
|
3066 delayed()->cmp(G0, G0); |
|
3067 br(Assembler::always, false, Assembler::pt, done); |
|
3068 delayed()->cmp(G0, 1); |
3055 } else { |
3069 } else { |
3056 brx(Assembler::notZero, false, Assembler::pn, done); |
3070 // 1-0 form : avoids CAS and MEMBAR in the common case |
3057 delayed()->nop(); |
3071 // Do not bother to ratify that m->Owner == Self. |
3058 ba(done); |
3072 ld_ptr(Address(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)), Rbox); |
3059 delayed()->st_ptr(G0, Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)); |
3073 orcc(Rbox, G0, G0); |
|
3074 brx(Assembler::notZero, false, Assembler::pn, done); |
|
3075 delayed()-> |
|
3076 ld_ptr(Address(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)), Rscratch); |
|
3077 ld_ptr(Address(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)), Rbox); |
|
3078 orcc(Rbox, Rscratch, G0); |
|
3079 if (EmitSync & 16384) { |
|
3080 // As an optional optimization, if (EntryList|cxq) != null and _succ is null then |
|
3081 // we should transfer control directly to the slow-path. |
|
3082 // This test makes the reacquire operation below very infrequent. |
|
3083 // The logic is equivalent to : |
|
3084 // if (cxq|EntryList) == null : Owner=null; goto Success |
|
3085 // if succ == null : goto SlowPath |
|
3086 // Owner=null; membar #storeload |
|
3087 // if succ != null : goto Success |
|
3088 // if CAS(&Owner,null,Self) != null goto Success |
|
3089 // goto SlowPath |
|
3090 brx(Assembler::zero, true, Assembler::pt, done); |
|
3091 delayed()-> |
|
3092 st_ptr(G0, Address(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
|
3093 ld_ptr(Address(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), Rscratch); |
|
3094 andcc(Rscratch, Rscratch, G0) ; |
|
3095 brx(Assembler::zero, false, Assembler::pt, done); |
|
3096 delayed()->orcc(G0, 1, G0); |
|
3097 st_ptr(G0, Address(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
|
3098 } else { |
|
3099 brx(Assembler::zero, false, Assembler::pt, done); |
|
3100 delayed()-> |
|
3101 st_ptr(G0, Address(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); |
|
3102 } |
|
3103 if (os::is_MP()) { membar(StoreLoad); } |
|
3104 // Check that _succ is (or remains) non-zero |
|
3105 ld_ptr(Address(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), Rscratch); |
|
3106 andcc(Rscratch, Rscratch, G0); |
|
3107 brx(Assembler::notZero, false, Assembler::pt, done); |
|
3108 delayed()->andcc(G0, G0, G0); |
|
3109 add(Rmark, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), Rmark); |
|
3110 mov(G2_thread, Rscratch); |
|
3111 cas_ptr(Rmark, G0, Rscratch); |
|
3112 cmp(Rscratch, G0); |
|
3113 // invert icc.zf and goto done |
|
3114 // A slightly better v8+/v9 idiom would be the following: |
|
3115 // movrnz Rscratch,1,Rscratch |
|
3116 // ba done |
|
3117 // xorcc Rscratch,1,G0 |
|
3118 // In v8+ mode the idiom would be valid IFF Rscratch was a G or O register |
|
3119 brx(Assembler::notZero, false, Assembler::pt, done); |
|
3120 delayed()->cmp(G0, G0); |
|
3121 br(Assembler::always, false, Assembler::pt, done); |
|
3122 delayed()->cmp(G0, 1); |
3060 } |
3123 } |
3061 |
3124 |
3062 bind (LStacked); |
3125 bind (LStacked); |
3063 // Consider: we could replace the expensive CAS in the exit |
3126 // Consider: we could replace the expensive CAS in the exit |
3064 // path with a simple ST of the displaced mark value fetched from |
3127 // path with a simple ST of the displaced mark value fetched from |