1563     __ cset(rscratch1, Assembler::NE);

  1563     __ cset(rscratch1, Assembler::NE);

  1564   } else {

  1564   } else {

  1565     Label retry_load, nope;

  1565     Label retry_load, nope;

  1566     // flush and load exclusive from the memory location

  1566     // flush and load exclusive from the memory location

  1567     // and fail if it is not what we expect

  1567     // and fail if it is not what we expect

  1568     __ bind(retry_load);

  1569     __ bind(retry_load);

  1569     __ ldaxrw(rscratch1, addr);

  1570     __ ldaxrw(rscratch1, addr);

  1570     __ cmpw(rscratch1, cmpval);

  1571     __ cmpw(rscratch1, cmpval);

  1571     __ cset(rscratch1, Assembler::NE);

  1572     __ cset(rscratch1, Assembler::NE);

  1572     __ br(Assembler::NE, nope);

  1573     __ br(Assembler::NE, nope);

  1588     __ cset(rscratch1, Assembler::NE);

  1589     __ cset(rscratch1, Assembler::NE);

  1589   } else {

  1590   } else {

  1590     Label retry_load, nope;

  1591     Label retry_load, nope;

  1591     // flush and load exclusive from the memory location

  1592     // flush and load exclusive from the memory location

  1592     // and fail if it is not what we expect

  1593     // and fail if it is not what we expect

  1593     __ bind(retry_load);

  1595     __ bind(retry_load);

  1594     __ ldaxr(rscratch1, addr);

  1596     __ ldaxr(rscratch1, addr);

  1595     __ cmp(rscratch1, cmpval);

  1597     __ cmp(rscratch1, cmpval);

  1596     __ cset(rscratch1, Assembler::NE);

  1598     __ cset(rscratch1, Assembler::NE);

  1597     __ br(Assembler::NE, nope);

  1599     __ br(Assembler::NE, nope);

  3168         assert_different_registers(inc.as_register(), dst, addr.base(), tmp,

  3170         assert_different_registers(inc.as_register(), dst, addr.base(), tmp,

  3169                                    rscratch1, rscratch2);

  3171                                    rscratch1, rscratch2);

  3170       }

  3172       }

  3171       Label again;

  3173       Label again;

  3172       __ lea(tmp, addr);

  3174       __ lea(tmp, addr);

  3173       __ bind(again);

  3176       __ bind(again);

  3174       (_masm->*lda)(dst, tmp);

  3177       (_masm->*lda)(dst, tmp);

  3175       (_masm->*add)(rscratch1, dst, inc);

  3178       (_masm->*add)(rscratch1, dst, inc);

  3176       (_masm->*stl)(rscratch2, rscratch1, tmp);

  3179       (_masm->*stl)(rscratch2, rscratch1, tmp);

  3177       __ cbnzw(rscratch2, again);

  3180       __ cbnzw(rscratch2, again);

  3187         obj = rscratch1;

  3190         obj = rscratch1;

  3188       }

  3191       }

  3189       assert_different_registers(obj, addr.base(), tmp, rscratch2, dst);

  3192       assert_different_registers(obj, addr.base(), tmp, rscratch2, dst);

  3190       Label again;

  3193       Label again;

  3191       __ lea(tmp, addr);

  3194       __ lea(tmp, addr);

  3192       __ bind(again);

  3196       __ bind(again);

  3193       (_masm->*lda)(dst, tmp);

  3197       (_masm->*lda)(dst, tmp);

  3194       (_masm->*stl)(rscratch2, obj, tmp);

  3198       (_masm->*stl)(rscratch2, obj, tmp);

  3195       __ cbnzw(rscratch2, again);

  3199       __ cbnzw(rscratch2, again);

  3196       if (is_oop && UseCompressedOops) {

  3200       if (is_oop && UseCompressedOops) {