703 // Try to revoke bias. |
703 // Try to revoke bias. |
704 markOop header = rcvr->klass()->prototype_header(); |
704 markOop header = rcvr->klass()->prototype_header(); |
705 if (hash != markOopDesc::no_hash) { |
705 if (hash != markOopDesc::no_hash) { |
706 header = header->copy_set_hash(hash); |
706 header = header->copy_set_hash(hash); |
707 } |
707 } |
708 if (Atomic::cmpxchg_ptr(header, rcvr->mark_addr(), mark) == mark) { |
708 if (Atomic::cmpxchg(header, rcvr->mark_addr(), mark) == mark) { |
709 if (PrintBiasedLockingStatistics) |
709 if (PrintBiasedLockingStatistics) |
710 (*BiasedLocking::revoked_lock_entry_count_addr())++; |
710 (*BiasedLocking::revoked_lock_entry_count_addr())++; |
711 } |
711 } |
712 } else if ((anticipated_bias_locking_value & epoch_mask_in_place) != 0) { |
712 } else if ((anticipated_bias_locking_value & epoch_mask_in_place) != 0) { |
713 // Try to rebias. |
713 // Try to rebias. |
714 markOop new_header = (markOop) ( (intptr_t) rcvr->klass()->prototype_header() | thread_ident); |
714 markOop new_header = (markOop) ( (intptr_t) rcvr->klass()->prototype_header() | thread_ident); |
715 if (hash != markOopDesc::no_hash) { |
715 if (hash != markOopDesc::no_hash) { |
716 new_header = new_header->copy_set_hash(hash); |
716 new_header = new_header->copy_set_hash(hash); |
717 } |
717 } |
718 if (Atomic::cmpxchg_ptr((void*)new_header, rcvr->mark_addr(), mark) == mark) { |
718 if (Atomic::cmpxchg(new_header, rcvr->mark_addr(), mark) == mark) { |
719 if (PrintBiasedLockingStatistics) { |
719 if (PrintBiasedLockingStatistics) { |
720 (* BiasedLocking::rebiased_lock_entry_count_addr())++; |
720 (* BiasedLocking::rebiased_lock_entry_count_addr())++; |
721 } |
721 } |
722 } else { |
722 } else { |
723 CALL_VM(InterpreterRuntime::monitorenter(THREAD, mon), handle_exception); |
723 CALL_VM(InterpreterRuntime::monitorenter(THREAD, mon), handle_exception); |
732 header = header->copy_set_hash(hash); |
732 header = header->copy_set_hash(hash); |
733 } |
733 } |
734 markOop new_header = (markOop) ((uintptr_t) header | thread_ident); |
734 markOop new_header = (markOop) ((uintptr_t) header | thread_ident); |
735 // Debugging hint. |
735 // Debugging hint. |
736 DEBUG_ONLY(mon->lock()->set_displaced_header((markOop) (uintptr_t) 0xdeaddead);) |
736 DEBUG_ONLY(mon->lock()->set_displaced_header((markOop) (uintptr_t) 0xdeaddead);) |
737 if (Atomic::cmpxchg_ptr((void*)new_header, rcvr->mark_addr(), header) == header) { |
737 if (Atomic::cmpxchg(new_header, rcvr->mark_addr(), header) == header) { |
738 if (PrintBiasedLockingStatistics) { |
738 if (PrintBiasedLockingStatistics) { |
739 (* BiasedLocking::anonymously_biased_lock_entry_count_addr())++; |
739 (* BiasedLocking::anonymously_biased_lock_entry_count_addr())++; |
740 } |
740 } |
741 } else { |
741 } else { |
742 CALL_VM(InterpreterRuntime::monitorenter(THREAD, mon), handle_exception); |
742 CALL_VM(InterpreterRuntime::monitorenter(THREAD, mon), handle_exception); |
748 // Traditional lightweight locking. |
748 // Traditional lightweight locking. |
749 if (!success) { |
749 if (!success) { |
750 markOop displaced = rcvr->mark()->set_unlocked(); |
750 markOop displaced = rcvr->mark()->set_unlocked(); |
751 mon->lock()->set_displaced_header(displaced); |
751 mon->lock()->set_displaced_header(displaced); |
752 bool call_vm = UseHeavyMonitors; |
752 bool call_vm = UseHeavyMonitors; |
753 if (call_vm || Atomic::cmpxchg_ptr(mon, rcvr->mark_addr(), displaced) != displaced) { |
753 if (call_vm || Atomic::cmpxchg((markOop)mon, rcvr->mark_addr(), displaced) != displaced) { |
754 // Is it simple recursive case? |
754 // Is it simple recursive case? |
755 if (!call_vm && THREAD->is_lock_owned((address) displaced->clear_lock_bits())) { |
755 if (!call_vm && THREAD->is_lock_owned((address) displaced->clear_lock_bits())) { |
756 mon->lock()->set_displaced_header(NULL); |
756 mon->lock()->set_displaced_header(NULL); |
757 } else { |
757 } else { |
758 CALL_VM(InterpreterRuntime::monitorenter(THREAD, mon), handle_exception); |
758 CALL_VM(InterpreterRuntime::monitorenter(THREAD, mon), handle_exception); |
901 // try revoke bias |
901 // try revoke bias |
902 markOop header = lockee->klass()->prototype_header(); |
902 markOop header = lockee->klass()->prototype_header(); |
903 if (hash != markOopDesc::no_hash) { |
903 if (hash != markOopDesc::no_hash) { |
904 header = header->copy_set_hash(hash); |
904 header = header->copy_set_hash(hash); |
905 } |
905 } |
906 if (Atomic::cmpxchg_ptr(header, lockee->mark_addr(), mark) == mark) { |
906 if (Atomic::cmpxchg(header, lockee->mark_addr(), mark) == mark) { |
907 if (PrintBiasedLockingStatistics) { |
907 if (PrintBiasedLockingStatistics) { |
908 (*BiasedLocking::revoked_lock_entry_count_addr())++; |
908 (*BiasedLocking::revoked_lock_entry_count_addr())++; |
909 } |
909 } |
910 } |
910 } |
911 } else if ((anticipated_bias_locking_value & epoch_mask_in_place) !=0) { |
911 } else if ((anticipated_bias_locking_value & epoch_mask_in_place) !=0) { |
912 // try rebias |
912 // try rebias |
913 markOop new_header = (markOop) ( (intptr_t) lockee->klass()->prototype_header() | thread_ident); |
913 markOop new_header = (markOop) ( (intptr_t) lockee->klass()->prototype_header() | thread_ident); |
914 if (hash != markOopDesc::no_hash) { |
914 if (hash != markOopDesc::no_hash) { |
915 new_header = new_header->copy_set_hash(hash); |
915 new_header = new_header->copy_set_hash(hash); |
916 } |
916 } |
917 if (Atomic::cmpxchg_ptr((void*)new_header, lockee->mark_addr(), mark) == mark) { |
917 if (Atomic::cmpxchg(new_header, lockee->mark_addr(), mark) == mark) { |
918 if (PrintBiasedLockingStatistics) { |
918 if (PrintBiasedLockingStatistics) { |
919 (* BiasedLocking::rebiased_lock_entry_count_addr())++; |
919 (* BiasedLocking::rebiased_lock_entry_count_addr())++; |
920 } |
920 } |
921 } else { |
921 } else { |
922 CALL_VM(InterpreterRuntime::monitorenter(THREAD, entry), handle_exception); |
922 CALL_VM(InterpreterRuntime::monitorenter(THREAD, entry), handle_exception); |
930 header = header->copy_set_hash(hash); |
930 header = header->copy_set_hash(hash); |
931 } |
931 } |
932 markOop new_header = (markOop) ((uintptr_t) header | thread_ident); |
932 markOop new_header = (markOop) ((uintptr_t) header | thread_ident); |
933 // debugging hint |
933 // debugging hint |
934 DEBUG_ONLY(entry->lock()->set_displaced_header((markOop) (uintptr_t) 0xdeaddead);) |
934 DEBUG_ONLY(entry->lock()->set_displaced_header((markOop) (uintptr_t) 0xdeaddead);) |
935 if (Atomic::cmpxchg_ptr((void*)new_header, lockee->mark_addr(), header) == header) { |
935 if (Atomic::cmpxchg(new_header, lockee->mark_addr(), header) == header) { |
936 if (PrintBiasedLockingStatistics) { |
936 if (PrintBiasedLockingStatistics) { |
937 (* BiasedLocking::anonymously_biased_lock_entry_count_addr())++; |
937 (* BiasedLocking::anonymously_biased_lock_entry_count_addr())++; |
938 } |
938 } |
939 } else { |
939 } else { |
940 CALL_VM(InterpreterRuntime::monitorenter(THREAD, entry), handle_exception); |
940 CALL_VM(InterpreterRuntime::monitorenter(THREAD, entry), handle_exception); |
946 // traditional lightweight locking |
946 // traditional lightweight locking |
947 if (!success) { |
947 if (!success) { |
948 markOop displaced = lockee->mark()->set_unlocked(); |
948 markOop displaced = lockee->mark()->set_unlocked(); |
949 entry->lock()->set_displaced_header(displaced); |
949 entry->lock()->set_displaced_header(displaced); |
950 bool call_vm = UseHeavyMonitors; |
950 bool call_vm = UseHeavyMonitors; |
951 if (call_vm || Atomic::cmpxchg_ptr(entry, lockee->mark_addr(), displaced) != displaced) { |
951 if (call_vm || Atomic::cmpxchg((markOop)entry, lockee->mark_addr(), displaced) != displaced) { |
952 // Is it simple recursive case? |
952 // Is it simple recursive case? |
953 if (!call_vm && THREAD->is_lock_owned((address) displaced->clear_lock_bits())) { |
953 if (!call_vm && THREAD->is_lock_owned((address) displaced->clear_lock_bits())) { |
954 entry->lock()->set_displaced_header(NULL); |
954 entry->lock()->set_displaced_header(NULL); |
955 } else { |
955 } else { |
956 CALL_VM(InterpreterRuntime::monitorenter(THREAD, entry), handle_exception); |
956 CALL_VM(InterpreterRuntime::monitorenter(THREAD, entry), handle_exception); |
1842 // try revoke bias |
1842 // try revoke bias |
1843 markOop header = lockee->klass()->prototype_header(); |
1843 markOop header = lockee->klass()->prototype_header(); |
1844 if (hash != markOopDesc::no_hash) { |
1844 if (hash != markOopDesc::no_hash) { |
1845 header = header->copy_set_hash(hash); |
1845 header = header->copy_set_hash(hash); |
1846 } |
1846 } |
1847 if (Atomic::cmpxchg_ptr(header, lockee->mark_addr(), mark) == mark) { |
1847 if (Atomic::cmpxchg(header, lockee->mark_addr(), mark) == mark) { |
1848 if (PrintBiasedLockingStatistics) |
1848 if (PrintBiasedLockingStatistics) |
1849 (*BiasedLocking::revoked_lock_entry_count_addr())++; |
1849 (*BiasedLocking::revoked_lock_entry_count_addr())++; |
1850 } |
1850 } |
1851 } |
1851 } |
1852 else if ((anticipated_bias_locking_value & epoch_mask_in_place) !=0) { |
1852 else if ((anticipated_bias_locking_value & epoch_mask_in_place) !=0) { |
1853 // try rebias |
1853 // try rebias |
1854 markOop new_header = (markOop) ( (intptr_t) lockee->klass()->prototype_header() | thread_ident); |
1854 markOop new_header = (markOop) ( (intptr_t) lockee->klass()->prototype_header() | thread_ident); |
1855 if (hash != markOopDesc::no_hash) { |
1855 if (hash != markOopDesc::no_hash) { |
1856 new_header = new_header->copy_set_hash(hash); |
1856 new_header = new_header->copy_set_hash(hash); |
1857 } |
1857 } |
1858 if (Atomic::cmpxchg_ptr((void*)new_header, lockee->mark_addr(), mark) == mark) { |
1858 if (Atomic::cmpxchg(new_header, lockee->mark_addr(), mark) == mark) { |
1859 if (PrintBiasedLockingStatistics) |
1859 if (PrintBiasedLockingStatistics) |
1860 (* BiasedLocking::rebiased_lock_entry_count_addr())++; |
1860 (* BiasedLocking::rebiased_lock_entry_count_addr())++; |
1861 } |
1861 } |
1862 else { |
1862 else { |
1863 CALL_VM(InterpreterRuntime::monitorenter(THREAD, entry), handle_exception); |
1863 CALL_VM(InterpreterRuntime::monitorenter(THREAD, entry), handle_exception); |
1873 header = header->copy_set_hash(hash); |
1873 header = header->copy_set_hash(hash); |
1874 } |
1874 } |
1875 markOop new_header = (markOop) ((uintptr_t) header | thread_ident); |
1875 markOop new_header = (markOop) ((uintptr_t) header | thread_ident); |
1876 // debugging hint |
1876 // debugging hint |
1877 DEBUG_ONLY(entry->lock()->set_displaced_header((markOop) (uintptr_t) 0xdeaddead);) |
1877 DEBUG_ONLY(entry->lock()->set_displaced_header((markOop) (uintptr_t) 0xdeaddead);) |
1878 if (Atomic::cmpxchg_ptr((void*)new_header, lockee->mark_addr(), header) == header) { |
1878 if (Atomic::cmpxchg(new_header, lockee->mark_addr(), header) == header) { |
1879 if (PrintBiasedLockingStatistics) |
1879 if (PrintBiasedLockingStatistics) |
1880 (* BiasedLocking::anonymously_biased_lock_entry_count_addr())++; |
1880 (* BiasedLocking::anonymously_biased_lock_entry_count_addr())++; |
1881 } |
1881 } |
1882 else { |
1882 else { |
1883 CALL_VM(InterpreterRuntime::monitorenter(THREAD, entry), handle_exception); |
1883 CALL_VM(InterpreterRuntime::monitorenter(THREAD, entry), handle_exception); |
1889 // traditional lightweight locking |
1889 // traditional lightweight locking |
1890 if (!success) { |
1890 if (!success) { |
1891 markOop displaced = lockee->mark()->set_unlocked(); |
1891 markOop displaced = lockee->mark()->set_unlocked(); |
1892 entry->lock()->set_displaced_header(displaced); |
1892 entry->lock()->set_displaced_header(displaced); |
1893 bool call_vm = UseHeavyMonitors; |
1893 bool call_vm = UseHeavyMonitors; |
1894 if (call_vm || Atomic::cmpxchg_ptr(entry, lockee->mark_addr(), displaced) != displaced) { |
1894 if (call_vm || Atomic::cmpxchg((markOop)entry, lockee->mark_addr(), displaced) != displaced) { |
1895 // Is it simple recursive case? |
1895 // Is it simple recursive case? |
1896 if (!call_vm && THREAD->is_lock_owned((address) displaced->clear_lock_bits())) { |
1896 if (!call_vm && THREAD->is_lock_owned((address) displaced->clear_lock_bits())) { |
1897 entry->lock()->set_displaced_header(NULL); |
1897 entry->lock()->set_displaced_header(NULL); |
1898 } else { |
1898 } else { |
1899 CALL_VM(InterpreterRuntime::monitorenter(THREAD, entry), handle_exception); |
1899 CALL_VM(InterpreterRuntime::monitorenter(THREAD, entry), handle_exception); |
1921 most_recent->set_obj(NULL); |
1921 most_recent->set_obj(NULL); |
1922 if (!lockee->mark()->has_bias_pattern()) { |
1922 if (!lockee->mark()->has_bias_pattern()) { |
1923 bool call_vm = UseHeavyMonitors; |
1923 bool call_vm = UseHeavyMonitors; |
1924 // If it isn't recursive we either must swap old header or call the runtime |
1924 // If it isn't recursive we either must swap old header or call the runtime |
1925 if (header != NULL || call_vm) { |
1925 if (header != NULL || call_vm) { |
1926 if (call_vm || Atomic::cmpxchg_ptr(header, lockee->mark_addr(), lock) != lock) { |
1926 markOop old_header = markOopDesc::encode(lock); |
|
1927 if (call_vm || lockee->cas_set_mark(header, old_header) != old_header) { |
1927 // restore object for the slow case |
1928 // restore object for the slow case |
1928 most_recent->set_obj(lockee); |
1929 most_recent->set_obj(lockee); |
1929 CALL_VM(InterpreterRuntime::monitorexit(THREAD, most_recent), handle_exception); |
1930 CALL_VM(InterpreterRuntime::monitorexit(THREAD, most_recent), handle_exception); |
1930 } |
1931 } |
1931 } |
1932 } |
2187 // Try allocate in shared eden |
2188 // Try allocate in shared eden |
2188 retry: |
2189 retry: |
2189 HeapWord* compare_to = *Universe::heap()->top_addr(); |
2190 HeapWord* compare_to = *Universe::heap()->top_addr(); |
2190 HeapWord* new_top = compare_to + obj_size; |
2191 HeapWord* new_top = compare_to + obj_size; |
2191 if (new_top <= *Universe::heap()->end_addr()) { |
2192 if (new_top <= *Universe::heap()->end_addr()) { |
2192 if (Atomic::cmpxchg_ptr(new_top, Universe::heap()->top_addr(), compare_to) != compare_to) { |
2193 if (Atomic::cmpxchg(new_top, Universe::heap()->top_addr(), compare_to) != compare_to) { |
2193 goto retry; |
2194 goto retry; |
2194 } |
2195 } |
2195 result = (oop) compare_to; |
2196 result = (oop) compare_to; |
2196 } |
2197 } |
2197 } |
2198 } |
2973 end->set_obj(NULL); |
2974 end->set_obj(NULL); |
2974 |
2975 |
2975 if (!lockee->mark()->has_bias_pattern()) { |
2976 if (!lockee->mark()->has_bias_pattern()) { |
2976 // If it isn't recursive we either must swap old header or call the runtime |
2977 // If it isn't recursive we either must swap old header or call the runtime |
2977 if (header != NULL) { |
2978 if (header != NULL) { |
2978 if (Atomic::cmpxchg_ptr(header, lockee->mark_addr(), lock) != lock) { |
2979 markOop old_header = markOopDesc::encode(lock); |
|
2980 if (lockee->cas_set_mark(header, old_header) != old_header) { |
2979 // restore object for the slow case |
2981 // restore object for the slow case |
2980 end->set_obj(lockee); |
2982 end->set_obj(lockee); |
2981 { |
2983 { |
2982 // Prevent any HandleMarkCleaner from freeing our live handles |
2984 // Prevent any HandleMarkCleaner from freeing our live handles |
2983 HandleMark __hm(THREAD); |
2985 HandleMark __hm(THREAD); |
3048 |
3050 |
3049 if (!rcvr->mark()->has_bias_pattern()) { |
3051 if (!rcvr->mark()->has_bias_pattern()) { |
3050 base->set_obj(NULL); |
3052 base->set_obj(NULL); |
3051 // If it isn't recursive we either must swap old header or call the runtime |
3053 // If it isn't recursive we either must swap old header or call the runtime |
3052 if (header != NULL) { |
3054 if (header != NULL) { |
3053 if (Atomic::cmpxchg_ptr(header, rcvr->mark_addr(), lock) != lock) { |
3055 markOop old_header = markOopDesc::encode(lock); |
|
3056 if (rcvr->cas_set_mark(header, old_header) != old_header) { |
3054 // restore object for the slow case |
3057 // restore object for the slow case |
3055 base->set_obj(rcvr); |
3058 base->set_obj(rcvr); |
3056 { |
3059 { |
3057 // Prevent any HandleMarkCleaner from freeing our live handles |
3060 // Prevent any HandleMarkCleaner from freeing our live handles |
3058 HandleMark __hm(THREAD); |
3061 HandleMark __hm(THREAD); |