245 if(tosca_live) __ pop(rax); |
245 if(tosca_live) __ pop(rax); |
246 |
246 |
247 __ bind(done); |
247 __ bind(done); |
248 } |
248 } |
249 |
249 |
250 void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) { |
|
251 assert(ShenandoahCASBarrier, "should be enabled"); |
|
252 Label is_null; |
|
253 __ testptr(dst, dst); |
|
254 __ jcc(Assembler::zero, is_null); |
|
255 resolve_forward_pointer_not_null(masm, dst, tmp); |
|
256 __ bind(is_null); |
|
257 } |
|
258 |
|
259 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) { |
|
260 assert(ShenandoahCASBarrier || ShenandoahLoadRefBarrier, "should be enabled"); |
|
261 // The below loads the mark word, checks if the lowest two bits are |
|
262 // set, and if so, clear the lowest two bits and copy the result |
|
263 // to dst. Otherwise it leaves dst alone. |
|
264 // Implementing this is surprisingly awkward. I do it here by: |
|
265 // - Inverting the mark word |
|
266 // - Test lowest two bits == 0 |
|
267 // - If so, set the lowest two bits |
|
268 // - Invert the result back, and copy to dst |
|
269 |
|
270 bool borrow_reg = (tmp == noreg); |
|
271 if (borrow_reg) { |
|
272 // No free registers available. Make one useful. |
|
273 tmp = LP64_ONLY(rscratch1) NOT_LP64(rdx); |
|
274 if (tmp == dst) { |
|
275 tmp = LP64_ONLY(rscratch2) NOT_LP64(rcx); |
|
276 } |
|
277 __ push(tmp); |
|
278 } |
|
279 |
|
280 assert_different_registers(dst, tmp); |
|
281 |
|
282 Label done; |
|
283 __ movptr(tmp, Address(dst, oopDesc::mark_offset_in_bytes())); |
|
284 __ notptr(tmp); |
|
285 __ testb(tmp, markWord::marked_value); |
|
286 __ jccb(Assembler::notZero, done); |
|
287 __ orptr(tmp, markWord::marked_value); |
|
288 __ notptr(tmp); |
|
289 __ mov(dst, tmp); |
|
290 __ bind(done); |
|
291 |
|
292 if (borrow_reg) { |
|
293 __ pop(tmp); |
|
294 } |
|
295 } |
|
296 |
|
297 |
|
298 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) { |
250 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) { |
299 assert(ShenandoahLoadRefBarrier, "Should be enabled"); |
251 assert(ShenandoahLoadRefBarrier, "Should be enabled"); |
300 |
252 |
301 Label done; |
253 Label done; |
302 |
254 |
616 #endif |
569 #endif |
617 { |
570 { |
618 __ movptr(tmp1, oldval); |
571 __ movptr(tmp1, oldval); |
619 } |
572 } |
620 |
573 |
621 // Step 1. Try to CAS with given arguments. If successful, then we are done, |
574 // Step 1. Fast-path. |
622 // and can safely return. |
575 // |
|
576 // Try to CAS with given arguments. If successful, then we are done. |
|
577 |
623 if (os::is_MP()) __ lock(); |
578 if (os::is_MP()) __ lock(); |
624 #ifdef _LP64 |
579 #ifdef _LP64 |
625 if (UseCompressedOops) { |
580 if (UseCompressedOops) { |
626 __ cmpxchgl(newval, addr); |
581 __ cmpxchgl(newval, addr); |
627 } else |
582 } else |
628 #endif |
583 #endif |
629 { |
584 { |
630 __ cmpxchgptr(newval, addr); |
585 __ cmpxchgptr(newval, addr); |
631 } |
586 } |
632 __ jcc(Assembler::equal, done, true); |
587 __ jcc(Assembler::equal, L_success); |
633 |
588 |
634 // Step 2. CAS had failed. This may be a false negative. |
589 // Step 2. CAS had failed. This may be a false negative. |
635 // |
590 // |
636 // The trouble comes when we compare the to-space pointer with the from-space |
591 // The trouble comes when we compare the to-space pointer with the from-space |
637 // pointer to the same object. To resolve this, it will suffice to resolve both |
592 // pointer to the same object. To resolve this, it will suffice to resolve |
638 // oldval and the value from memory -- this will give both to-space pointers. |
593 // the value from memory -- this will give both to-space pointers. |
639 // If they mismatch, then it was a legitimate failure. |
594 // If they mismatch, then it was a legitimate failure. |
640 // |
595 // |
641 #ifdef _LP64 |
596 // Before reaching to resolve sequence, see if we can avoid the whole shebang |
642 if (UseCompressedOops) { |
597 // with filters. |
643 __ decode_heap_oop(tmp1); |
598 |
644 } |
599 // Filter: when offending in-memory value is NULL, the failure is definitely legitimate |
645 #endif |
600 __ testptr(oldval, oldval); |
646 resolve_forward_pointer(masm, tmp1); |
601 __ jcc(Assembler::zero, L_failure); |
|
602 |
|
603 // Filter: when heap is stable, the failure is definitely legitimate |
|
604 #ifdef _LP64 |
|
605 const Register thread = r15_thread; |
|
606 #else |
|
607 const Register thread = tmp2; |
|
608 __ get_thread(thread); |
|
609 #endif |
|
610 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); |
|
611 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED); |
|
612 __ jcc(Assembler::zero, L_failure); |
647 |
613 |
648 #ifdef _LP64 |
614 #ifdef _LP64 |
649 if (UseCompressedOops) { |
615 if (UseCompressedOops) { |
650 __ movl(tmp2, oldval); |
616 __ movl(tmp2, oldval); |
651 __ decode_heap_oop(tmp2); |
617 __ decode_heap_oop(tmp2); |
652 } else |
618 } else |
653 #endif |
619 #endif |
654 { |
620 { |
655 __ movptr(tmp2, oldval); |
621 __ movptr(tmp2, oldval); |
656 } |
622 } |
657 resolve_forward_pointer(masm, tmp2); |
623 |
658 |
624 // Decode offending in-memory value. |
|
625 // Test if-forwarded |
|
626 __ testb(Address(tmp2, oopDesc::mark_offset_in_bytes()), markWord::marked_value); |
|
627 __ jcc(Assembler::noParity, L_failure); // When odd number of bits, then not forwarded |
|
628 __ jcc(Assembler::zero, L_failure); // When it is 00, then also not forwarded |
|
629 |
|
630 // Load and mask forwarding pointer |
|
631 __ movptr(tmp2, Address(tmp2, oopDesc::mark_offset_in_bytes())); |
|
632 __ shrptr(tmp2, 2); |
|
633 __ shlptr(tmp2, 2); |
|
634 |
|
635 #ifdef _LP64 |
|
636 if (UseCompressedOops) { |
|
637 __ decode_heap_oop(tmp1); // decode for comparison |
|
638 } |
|
639 #endif |
|
640 |
|
641 // Now we have the forwarded offender in tmp2. |
|
642 // Compare and if they don't match, we have legitimate failure |
659 __ cmpptr(tmp1, tmp2); |
643 __ cmpptr(tmp1, tmp2); |
660 __ jcc(Assembler::notEqual, done, true); |
644 __ jcc(Assembler::notEqual, L_failure); |
661 |
645 |
662 // Step 3. Try to CAS again with resolved to-space pointers. |
646 // Step 3. Need to fix the memory ptr before continuing. |
663 // |
647 // |
664 // Corner case: it may happen that somebody stored the from-space pointer |
648 // At this point, we have from-space oldval in the register, and its to-space |
665 // to memory while we were preparing for retry. Therefore, we can fail again |
649 // address is in tmp2. Let's try to update it into memory. We don't care if it |
666 // on retry, and so need to do this in loop, always resolving the failure |
650 // succeeds or not. If it does, then the retrying CAS would see it and succeed. |
667 // witness. |
651 // If this fixup fails, this means somebody else beat us to it, and necessarily |
668 __ bind(retry); |
652 // with to-space ptr store. We still have to do the retry, because the GC might |
|
653 // have updated the reference for us. |
|
654 |
|
655 #ifdef _LP64 |
|
656 if (UseCompressedOops) { |
|
657 __ encode_heap_oop(tmp2); // previously decoded at step 2. |
|
658 } |
|
659 #endif |
|
660 |
|
661 if (os::is_MP()) __ lock(); |
|
662 #ifdef _LP64 |
|
663 if (UseCompressedOops) { |
|
664 __ cmpxchgl(tmp2, addr); |
|
665 } else |
|
666 #endif |
|
667 { |
|
668 __ cmpxchgptr(tmp2, addr); |
|
669 } |
|
670 |
|
671 // Step 4. Try to CAS again. |
|
672 // |
|
673 // This is guaranteed not to have false negatives, because oldval is definitely |
|
674 // to-space, and memory pointer is to-space as well. Nothing is able to store |
|
675 // from-space ptr into memory anymore. Make sure oldval is restored, after being |
|
676 // garbled during retries. |
|
677 // |
|
678 #ifdef _LP64 |
|
679 if (UseCompressedOops) { |
|
680 __ movl(oldval, tmp2); |
|
681 } else |
|
682 #endif |
|
683 { |
|
684 __ movptr(oldval, tmp2); |
|
685 } |
|
686 |
669 if (os::is_MP()) __ lock(); |
687 if (os::is_MP()) __ lock(); |
670 #ifdef _LP64 |
688 #ifdef _LP64 |
671 if (UseCompressedOops) { |
689 if (UseCompressedOops) { |
672 __ cmpxchgl(newval, addr); |
690 __ cmpxchgl(newval, addr); |
673 } else |
691 } else |
674 #endif |
692 #endif |
675 { |
693 { |
676 __ cmpxchgptr(newval, addr); |
694 __ cmpxchgptr(newval, addr); |
677 } |
695 } |
678 __ jcc(Assembler::equal, done, true); |
|
679 |
|
680 #ifdef _LP64 |
|
681 if (UseCompressedOops) { |
|
682 __ movl(tmp2, oldval); |
|
683 __ decode_heap_oop(tmp2); |
|
684 } else |
|
685 #endif |
|
686 { |
|
687 __ movptr(tmp2, oldval); |
|
688 } |
|
689 resolve_forward_pointer(masm, tmp2); |
|
690 |
|
691 __ cmpptr(tmp1, tmp2); |
|
692 __ jcc(Assembler::equal, retry, true); |
|
693 |
|
694 // Step 4. If we need a boolean result out of CAS, check the flag again, |
|
695 // and promote the result. Note that we handle the flag from both the CAS |
|
696 // itself and from the retry loop. |
|
697 __ bind(done); |
|
698 if (!exchange) { |
696 if (!exchange) { |
|
697 __ jccb(Assembler::equal, L_success); // fastpath, peeking into Step 5, no need to jump |
|
698 } |
|
699 |
|
700 // Step 5. If we need a boolean result out of CAS, set the flag appropriately. |
|
701 // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS. |
|
702 // Otherwise, failure witness for CAE is in oldval on all paths, and we can return. |
|
703 |
|
704 if (exchange) { |
|
705 __ bind(L_failure); |
|
706 __ bind(L_success); |
|
707 } else { |
699 assert(res != NULL, "need result register"); |
708 assert(res != NULL, "need result register"); |
700 #ifdef _LP64 |
709 |
701 __ setb(Assembler::equal, res); |
710 Label exit; |
702 __ movzbl(res, res); |
711 __ bind(L_failure); |
703 #else |
712 __ xorptr(res, res); |
704 // Need something else to clean the result, because some registers |
713 __ jmpb(exit); |
705 // do not have byte encoding that movzbl wants. Cannot do the xor first, |
714 |
706 // because it modifies the flags. |
715 __ bind(L_success); |
707 Label res_non_zero; |
|
708 __ movptr(res, 1); |
716 __ movptr(res, 1); |
709 __ jcc(Assembler::equal, res_non_zero, true); |
717 __ bind(exit); |
710 __ xorptr(res, res); |
|
711 __ bind(res_non_zero); |
|
712 #endif |
|
713 } |
718 } |
714 } |
719 } |
715 |
720 |
716 #undef __ |
721 #undef __ |
717 |
722 |