equal
deleted
inserted
replaced
1636 return Address(Rd); |
1636 return Address(Rd); |
1637 } |
1637 } |
1638 |
1638 |
1639 void MacroAssembler::atomic_incw(Register counter_addr, Register tmp, Register tmp2) { |
1639 void MacroAssembler::atomic_incw(Register counter_addr, Register tmp, Register tmp2) { |
1640 Label retry_load; |
1640 Label retry_load; |
|
1641 prfm(Address(counter_addr), PSTL1STRM); |
1641 bind(retry_load); |
1642 bind(retry_load); |
1642 // flush and load exclusive from the memory location |
1643 // flush and load exclusive from the memory location |
1643 ldxrw(tmp, counter_addr); |
1644 ldxrw(tmp, counter_addr); |
1644 addw(tmp, tmp, 1); |
1645 addw(tmp, tmp, 1); |
1645 // if we store+flush with no intervening write tmp wil be zero |
1646 // if we store+flush with no intervening write tmp wil be zero |
2076 cmp(tmp, oldv); |
2077 cmp(tmp, oldv); |
2077 br(Assembler::EQ, succeed); |
2078 br(Assembler::EQ, succeed); |
2078 membar(AnyAny); |
2079 membar(AnyAny); |
2079 } else { |
2080 } else { |
2080 Label retry_load, nope; |
2081 Label retry_load, nope; |
2081 |
2082 prfm(Address(addr), PSTL1STRM); |
2082 bind(retry_load); |
2083 bind(retry_load); |
2083 // flush and load exclusive from the memory location |
2084 // flush and load exclusive from the memory location |
2084 // and fail if it is not what we expect |
2085 // and fail if it is not what we expect |
2085 ldaxr(tmp, addr); |
2086 ldaxr(tmp, addr); |
2086 cmp(tmp, oldv); |
2087 cmp(tmp, oldv); |
2112 cmp(tmp, oldv); |
2113 cmp(tmp, oldv); |
2113 br(Assembler::EQ, succeed); |
2114 br(Assembler::EQ, succeed); |
2114 membar(AnyAny); |
2115 membar(AnyAny); |
2115 } else { |
2116 } else { |
2116 Label retry_load, nope; |
2117 Label retry_load, nope; |
2117 |
2118 prfm(Address(addr), PSTL1STRM); |
2118 bind(retry_load); |
2119 bind(retry_load); |
2119 // flush and load exclusive from the memory location |
2120 // flush and load exclusive from the memory location |
2120 // and fail if it is not what we expect |
2121 // and fail if it is not what we expect |
2121 ldaxrw(tmp, addr); |
2122 ldaxrw(tmp, addr); |
2122 cmp(tmp, oldv); |
2123 cmp(tmp, oldv); |
2147 lse_cas(tmp, new_val, addr, size, acquire, release, /*not_pair*/ true); |
2148 lse_cas(tmp, new_val, addr, size, acquire, release, /*not_pair*/ true); |
2148 cmp(tmp, expected); |
2149 cmp(tmp, expected); |
2149 } else { |
2150 } else { |
2150 BLOCK_COMMENT("cmpxchg {"); |
2151 BLOCK_COMMENT("cmpxchg {"); |
2151 Label retry_load, done; |
2152 Label retry_load, done; |
|
2153 prfm(Address(addr), PSTL1STRM); |
2152 bind(retry_load); |
2154 bind(retry_load); |
2153 load_exclusive(tmp, addr, size, acquire); |
2155 load_exclusive(tmp, addr, size, acquire); |
2154 if (size == xword) |
2156 if (size == xword) |
2155 cmp(tmp, expected); |
2157 cmp(tmp, expected); |
2156 else |
2158 else |
2175 Register result = rscratch2; \ |
2177 Register result = rscratch2; \ |
2176 if (prev->is_valid()) \ |
2178 if (prev->is_valid()) \ |
2177 result = different(prev, incr, addr) ? prev : rscratch2; \ |
2179 result = different(prev, incr, addr) ? prev : rscratch2; \ |
2178 \ |
2180 \ |
2179 Label retry_load; \ |
2181 Label retry_load; \ |
|
2182 prfm(Address(addr), PSTL1STRM); \ |
2180 bind(retry_load); \ |
2183 bind(retry_load); \ |
2181 LDXR(result, addr); \ |
2184 LDXR(result, addr); \ |
2182 OP(rscratch1, result, incr); \ |
2185 OP(rscratch1, result, incr); \ |
2183 STXR(rscratch2, rscratch1, addr); \ |
2186 STXR(rscratch2, rscratch1, addr); \ |
2184 cbnzw(rscratch2, retry_load); \ |
2187 cbnzw(rscratch2, retry_load); \ |
2197 Register result = rscratch2; \ |
2200 Register result = rscratch2; \ |
2198 if (prev->is_valid()) \ |
2201 if (prev->is_valid()) \ |
2199 result = different(prev, newv, addr) ? prev : rscratch2; \ |
2202 result = different(prev, newv, addr) ? prev : rscratch2; \ |
2200 \ |
2203 \ |
2201 Label retry_load; \ |
2204 Label retry_load; \ |
|
2205 prfm(Address(addr), PSTL1STRM); \ |
2202 bind(retry_load); \ |
2206 bind(retry_load); \ |
2203 LDXR(result, addr); \ |
2207 LDXR(result, addr); \ |
2204 STXR(rscratch1, newv, addr); \ |
2208 STXR(rscratch1, newv, addr); \ |
2205 cbnzw(rscratch1, retry_load); \ |
2209 cbnzw(rscratch1, retry_load); \ |
2206 if (prev->is_valid() && prev != result) \ |
2210 if (prev->is_valid() && prev != result) \ |