3415 |
3415 |
3416 if (UseBiasedLocking && !UseOptoBiasInlining) { |
3416 if (UseBiasedLocking && !UseOptoBiasInlining) { |
3417 __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont); |
3417 __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont); |
3418 } |
3418 } |
3419 |
3419 |
3420 // Handle existing monitor |
3420 // Check for existing monitor |
3421 __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor); |
3421 __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor); |
3422 |
3422 |
3423 // Set displaced_header to be (markOop of object | UNLOCK_VALUE). |
3423 // Set tmp to be (markOop of object | UNLOCK_VALUE). |
3424 __ orr(disp_hdr, disp_hdr, markOopDesc::unlocked_value); |
3424 __ orr(tmp, disp_hdr, markOopDesc::unlocked_value); |
3425 |
|
3426 // Load Compare Value application register. |
|
3427 |
3425 |
3428 // Initialize the box. (Must happen before we update the object mark!) |
3426 // Initialize the box. (Must happen before we update the object mark!) |
3429 __ str(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes())); |
3427 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes())); |
3430 |
3428 |
3431 // Compare object markOop with mark and if equal exchange scratch1 |
3429 // Compare object markOop with an unlocked value (tmp) and if |
3432 // with object markOop. |
3430 // equal exchange the stack address of our box with object markOop. |
3433 if (UseLSE) { |
3431 // On failure disp_hdr contains the possibly locked markOop. |
3434 __ mov(tmp, disp_hdr); |
3432 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true, |
3435 __ casal(Assembler::xword, tmp, box, oop); |
3433 /*release*/ true, /*weak*/ false, disp_hdr); |
3436 __ cmp(tmp, disp_hdr); |
3434 __ br(Assembler::EQ, cont); |
3437 __ br(Assembler::EQ, cont); |
|
3438 } else { |
|
3439 Label retry_load; |
|
3440 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH)) |
|
3441 __ prfm(Address(oop), PSTL1STRM); |
|
3442 __ bind(retry_load); |
|
3443 __ ldaxr(tmp, oop); |
|
3444 __ cmp(tmp, disp_hdr); |
|
3445 __ br(Assembler::NE, cas_failed); |
|
3446 // use stlxr to ensure update is immediately visible |
|
3447 __ stlxr(tmp, box, oop); |
|
3448 __ cbzw(tmp, cont); |
|
3449 __ b(retry_load); |
|
3450 } |
|
3451 |
3435 |
3452 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); |
3436 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); |
3453 |
3437 |
3454 // If the compare-and-exchange succeeded, then we found an unlocked |
3438 // If the compare-and-exchange succeeded, then we found an unlocked |
3455 // object, will have now locked it will continue at label cont |
3439 // object, will have now locked it will continue at label cont |
3462 __ mov(rscratch1, sp); |
3446 __ mov(rscratch1, sp); |
3463 __ sub(disp_hdr, disp_hdr, rscratch1); |
3447 __ sub(disp_hdr, disp_hdr, rscratch1); |
3464 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place)); |
3448 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place)); |
3465 // If condition is true we are cont and hence we can store 0 as the |
3449 // If condition is true we are cont and hence we can store 0 as the |
3466 // displaced header in the box, which indicates that it is a recursive lock. |
3450 // displaced header in the box, which indicates that it is a recursive lock. |
3467 __ ands(tmp/*==0?*/, disp_hdr, tmp); |
3451 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result |
3468 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes())); |
3452 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes())); |
3469 |
3453 |
|
3454 __ b(cont); |
|
3455 |
3470 // Handle existing monitor. |
3456 // Handle existing monitor. |
3471 __ b(cont); |
|
3472 |
|
3473 __ bind(object_has_monitor); |
3457 __ bind(object_has_monitor); |
|
3458 |
3474 // The object's monitor m is unlocked iff m->owner == NULL, |
3459 // The object's monitor m is unlocked iff m->owner == NULL, |
3475 // otherwise m->owner may contain a thread or a stack address. |
3460 // otherwise m->owner may contain a thread or a stack address. |
3476 // |
3461 // |
3477 // Try to CAS m->owner from NULL to current thread. |
3462 // Try to CAS m->owner from NULL to current thread. |
3478 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value)); |
3463 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value)); |
3479 __ mov(disp_hdr, zr); |
3464 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true, |
3480 |
3465 /*release*/ true, /*weak*/ false, noreg); // Sets flags for result |
3481 if (UseLSE) { |
|
3482 __ mov(rscratch1, disp_hdr); |
|
3483 __ casal(Assembler::xword, rscratch1, rthread, tmp); |
|
3484 __ cmp(rscratch1, disp_hdr); |
|
3485 } else { |
|
3486 Label retry_load, fail; |
|
3487 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH)) { |
|
3488 __ prfm(Address(tmp), PSTL1STRM); |
|
3489 } |
|
3490 __ bind(retry_load); |
|
3491 __ ldaxr(rscratch1, tmp); |
|
3492 __ cmp(disp_hdr, rscratch1); |
|
3493 __ br(Assembler::NE, fail); |
|
3494 // use stlxr to ensure update is immediately visible |
|
3495 __ stlxr(rscratch1, rthread, tmp); |
|
3496 __ cbnzw(rscratch1, retry_load); |
|
3497 __ bind(fail); |
|
3498 } |
|
3499 |
3466 |
3500 // Store a non-null value into the box to avoid looking like a re-entrant |
3467 // Store a non-null value into the box to avoid looking like a re-entrant |
3501 // lock. The fast-path monitor unlock code checks for |
3468 // lock. The fast-path monitor unlock code checks for |
3502 // markOopDesc::monitor_value so use markOopDesc::unused_mark which has the |
3469 // markOopDesc::monitor_value so use markOopDesc::unused_mark which has the |
3503 // relevant bit set, and also matches ObjectSynchronizer::slow_enter. |
3470 // relevant bit set, and also matches ObjectSynchronizer::slow_enter. |
3537 |
3504 |
3538 // Check if it is still a light weight lock, this is is true if we |
3505 // Check if it is still a light weight lock, this is is true if we |
3539 // see the stack address of the basicLock in the markOop of the |
3506 // see the stack address of the basicLock in the markOop of the |
3540 // object. |
3507 // object. |
3541 |
3508 |
3542 if (UseLSE) { |
3509 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false, |
3543 __ mov(tmp, box); |
3510 /*release*/ true, /*weak*/ false, tmp); |
3544 __ casl(Assembler::xword, tmp, disp_hdr, oop); |
3511 __ b(cont); |
3545 __ cmp(tmp, box); |
|
3546 __ b(cont); |
|
3547 } else { |
|
3548 Label retry_load; |
|
3549 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH)) |
|
3550 __ prfm(Address(oop), PSTL1STRM); |
|
3551 __ bind(retry_load); |
|
3552 __ ldxr(tmp, oop); |
|
3553 __ cmp(box, tmp); |
|
3554 __ br(Assembler::NE, cont); |
|
3555 // use stlxr to ensure update is immediately visible |
|
3556 __ stlxr(tmp, disp_hdr, oop); |
|
3557 __ cbzw(tmp, cont); |
|
3558 __ b(retry_load); |
|
3559 } |
|
3560 |
3512 |
3561 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); |
3513 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); |
3562 |
3514 |
3563 // Handle existing monitor. |
3515 // Handle existing monitor. |
3564 __ bind(object_has_monitor); |
3516 __ bind(object_has_monitor); |
3565 __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor |
3517 __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor |
3566 __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes())); |
3518 __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes())); |
3567 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes())); |
3519 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes())); |
3568 __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner. |
3520 __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner. |
3569 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions |
3521 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions |
3570 __ cmp(rscratch1, zr); |
3522 __ cmp(rscratch1, zr); // Sets flags for result |
3571 __ br(Assembler::NE, cont); |
3523 __ br(Assembler::NE, cont); |
3572 |
3524 |
3573 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes())); |
3525 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes())); |
3574 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes())); |
3526 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes())); |
3575 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0. |
3527 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0. |
3576 __ cmp(rscratch1, zr); |
3528 __ cmp(rscratch1, zr); // Sets flags for result |
3577 __ cbnz(rscratch1, cont); |
3529 __ cbnz(rscratch1, cont); |
3578 // need a release store here |
3530 // need a release store here |
3579 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes())); |
3531 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes())); |
3580 __ stlr(zr, tmp); // set unowned |
3532 __ stlr(zr, tmp); // set unowned |
3581 |
3533 |