1554 ShouldNotReachHere(); |
1554 ShouldNotReachHere(); |
1555 } |
1555 } |
1556 } |
1556 } |
1557 |
1557 |
1558 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) { |
1558 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) { |
1559 if (UseLSE) { |
1559 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, rscratch1); |
1560 __ mov(rscratch1, cmpval); |
1560 __ cset(rscratch1, Assembler::NE); |
1561 __ casal(Assembler::word, rscratch1, newval, addr); |
|
1562 __ cmpw(rscratch1, cmpval); |
|
1563 __ cset(rscratch1, Assembler::NE); |
|
1564 } else { |
|
1565 Label retry_load, nope; |
|
1566 // flush and load exclusive from the memory location |
|
1567 // and fail if it is not what we expect |
|
1568 __ prfm(Address(addr), PSTL1STRM); |
|
1569 __ bind(retry_load); |
|
1570 __ ldaxrw(rscratch1, addr); |
|
1571 __ cmpw(rscratch1, cmpval); |
|
1572 __ cset(rscratch1, Assembler::NE); |
|
1573 __ br(Assembler::NE, nope); |
|
1574 // if we store+flush with no intervening write rscratch1 wil be zero |
|
1575 __ stlxrw(rscratch1, newval, addr); |
|
1576 // retry so we only ever return after a load fails to compare |
|
1577 // ensures we don't return a stale value after a failed write. |
|
1578 __ cbnzw(rscratch1, retry_load); |
|
1579 __ bind(nope); |
|
1580 } |
|
1581 __ membar(__ AnyAny); |
1561 __ membar(__ AnyAny); |
1582 } |
1562 } |
1583 |
1563 |
1584 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) { |
1564 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) { |
1585 if (UseLSE) { |
1565 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, rscratch1); |
1586 __ mov(rscratch1, cmpval); |
1566 __ cset(rscratch1, Assembler::NE); |
1587 __ casal(Assembler::xword, rscratch1, newval, addr); |
|
1588 __ cmp(rscratch1, cmpval); |
|
1589 __ cset(rscratch1, Assembler::NE); |
|
1590 } else { |
|
1591 Label retry_load, nope; |
|
1592 // flush and load exclusive from the memory location |
|
1593 // and fail if it is not what we expect |
|
1594 __ prfm(Address(addr), PSTL1STRM); |
|
1595 __ bind(retry_load); |
|
1596 __ ldaxr(rscratch1, addr); |
|
1597 __ cmp(rscratch1, cmpval); |
|
1598 __ cset(rscratch1, Assembler::NE); |
|
1599 __ br(Assembler::NE, nope); |
|
1600 // if we store+flush with no intervening write rscratch1 wil be zero |
|
1601 __ stlxr(rscratch1, newval, addr); |
|
1602 // retry so we only ever return after a load fails to compare |
|
1603 // ensures we don't return a stale value after a failed write. |
|
1604 __ cbnz(rscratch1, retry_load); |
|
1605 __ bind(nope); |
|
1606 } |
|
1607 __ membar(__ AnyAny); |
1567 __ membar(__ AnyAny); |
1608 } |
1568 } |
1609 |
1569 |
1610 |
1570 |
1611 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { |
1571 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { |
3119 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) { |
3079 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) { |
3120 Address addr = as_Address(src->as_address_ptr(), noreg); |
3080 Address addr = as_Address(src->as_address_ptr(), noreg); |
3121 BasicType type = src->type(); |
3081 BasicType type = src->type(); |
3122 bool is_oop = type == T_OBJECT || type == T_ARRAY; |
3082 bool is_oop = type == T_OBJECT || type == T_ARRAY; |
3123 |
3083 |
3124 void (MacroAssembler::* lda)(Register Rd, Register Ra); |
3084 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr); |
3125 void (MacroAssembler::* add)(Register Rd, Register Rn, RegisterOrConstant increment); |
3085 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr); |
3126 void (MacroAssembler::* stl)(Register Rs, Register Rt, Register Rn); |
|
3127 |
3086 |
3128 switch(type) { |
3087 switch(type) { |
3129 case T_INT: |
3088 case T_INT: |
3130 lda = &MacroAssembler::ldaxrw; |
3089 xchg = &MacroAssembler::atomic_xchgalw; |
3131 add = &MacroAssembler::addw; |
3090 add = &MacroAssembler::atomic_addalw; |
3132 stl = &MacroAssembler::stlxrw; |
|
3133 break; |
3091 break; |
3134 case T_LONG: |
3092 case T_LONG: |
3135 lda = &MacroAssembler::ldaxr; |
3093 xchg = &MacroAssembler::atomic_xchgal; |
3136 add = &MacroAssembler::add; |
3094 add = &MacroAssembler::atomic_addal; |
3137 stl = &MacroAssembler::stlxr; |
|
3138 break; |
3095 break; |
3139 case T_OBJECT: |
3096 case T_OBJECT: |
3140 case T_ARRAY: |
3097 case T_ARRAY: |
3141 if (UseCompressedOops) { |
3098 if (UseCompressedOops) { |
3142 lda = &MacroAssembler::ldaxrw; |
3099 xchg = &MacroAssembler::atomic_xchgalw; |
3143 add = &MacroAssembler::addw; |
3100 add = &MacroAssembler::atomic_addalw; |
3144 stl = &MacroAssembler::stlxrw; |
|
3145 } else { |
3101 } else { |
3146 lda = &MacroAssembler::ldaxr; |
3102 xchg = &MacroAssembler::atomic_xchgal; |
3147 add = &MacroAssembler::add; |
3103 add = &MacroAssembler::atomic_addal; |
3148 stl = &MacroAssembler::stlxr; |
|
3149 } |
3104 } |
3150 break; |
3105 break; |
3151 default: |
3106 default: |
3152 ShouldNotReachHere(); |
3107 ShouldNotReachHere(); |
3153 lda = &MacroAssembler::ldaxr; |
3108 xchg = &MacroAssembler::atomic_xchgal; |
3154 add = &MacroAssembler::add; |
3109 add = &MacroAssembler::atomic_addal; // unreachable |
3155 stl = &MacroAssembler::stlxr; // unreachable |
|
3156 } |
3110 } |
3157 |
3111 |
3158 switch (code) { |
3112 switch (code) { |
3159 case lir_xadd: |
3113 case lir_xadd: |
3160 { |
3114 { |
3168 } else { |
3122 } else { |
3169 inc = RegisterOrConstant(as_reg(data)); |
3123 inc = RegisterOrConstant(as_reg(data)); |
3170 assert_different_registers(inc.as_register(), dst, addr.base(), tmp, |
3124 assert_different_registers(inc.as_register(), dst, addr.base(), tmp, |
3171 rscratch1, rscratch2); |
3125 rscratch1, rscratch2); |
3172 } |
3126 } |
3173 Label again; |
|
3174 __ lea(tmp, addr); |
3127 __ lea(tmp, addr); |
3175 __ prfm(Address(tmp), PSTL1STRM); |
3128 (_masm->*add)(dst, inc, tmp); |
3176 __ bind(again); |
|
3177 (_masm->*lda)(dst, tmp); |
|
3178 (_masm->*add)(rscratch1, dst, inc); |
|
3179 (_masm->*stl)(rscratch2, rscratch1, tmp); |
|
3180 __ cbnzw(rscratch2, again); |
|
3181 break; |
3129 break; |
3182 } |
3130 } |
3183 case lir_xchg: |
3131 case lir_xchg: |
3184 { |
3132 { |
3185 Register tmp = tmp_op->as_register(); |
3133 Register tmp = tmp_op->as_register(); |
3188 if (is_oop && UseCompressedOops) { |
3136 if (is_oop && UseCompressedOops) { |
3189 __ encode_heap_oop(rscratch1, obj); |
3137 __ encode_heap_oop(rscratch1, obj); |
3190 obj = rscratch1; |
3138 obj = rscratch1; |
3191 } |
3139 } |
3192 assert_different_registers(obj, addr.base(), tmp, rscratch2, dst); |
3140 assert_different_registers(obj, addr.base(), tmp, rscratch2, dst); |
3193 Label again; |
|
3194 __ lea(tmp, addr); |
3141 __ lea(tmp, addr); |
3195 __ prfm(Address(tmp), PSTL1STRM); |
3142 (_masm->*xchg)(dst, obj, tmp); |
3196 __ bind(again); |
|
3197 (_masm->*lda)(dst, tmp); |
|
3198 (_masm->*stl)(rscratch2, obj, tmp); |
|
3199 __ cbnzw(rscratch2, again); |
|
3200 if (is_oop && UseCompressedOops) { |
3143 if (is_oop && UseCompressedOops) { |
3201 __ decode_heap_oop(dst); |
3144 __ decode_heap_oop(dst); |
3202 } |
3145 } |
3203 } |
3146 } |
3204 break; |
3147 break; |