139 // Performs atomic compare of *dest and compare_value, and exchanges |
139 // Performs atomic compare of *dest and compare_value, and exchanges |
140 // *dest with exchange_value if the comparison succeeded. Returns prior |
140 // *dest with exchange_value if the comparison succeeded. Returns prior |
141 // value of *dest. cmpxchg*() provide: |
141 // value of *dest. cmpxchg*() provide: |
142 // <fence> compare-and-exchange <membar StoreLoad|StoreStore> |
142 // <fence> compare-and-exchange <membar StoreLoad|StoreStore> |
143 |
143 |
144 template<typename T, typename D, typename U> |
144 template<typename D, typename U, typename T> |
145 inline static D cmpxchg(T exchange_value, |
145 inline static D cmpxchg(D volatile* dest, |
146 D volatile* dest, |
|
147 U compare_value, |
146 U compare_value, |
|
147 T exchange_value, |
148 atomic_memory_order order = memory_order_conservative); |
148 atomic_memory_order order = memory_order_conservative); |
149 |
149 |
150 // Performs atomic compare of *dest and NULL, and replaces *dest |
150 // Performs atomic compare of *dest and NULL, and replaces *dest |
151 // with exchange_value if the comparison succeeded. Returns true if |
151 // with exchange_value if the comparison succeeded. Returns true if |
152 // the comparison succeeded and the exchange occurred. This is |
152 // the comparison succeeded and the exchange occurred. This is |
153 // often used as part of lazy initialization, as a lock-free |
153 // often used as part of lazy initialization, as a lock-free |
154 // alternative to the Double-Checked Locking Pattern. |
154 // alternative to the Double-Checked Locking Pattern. |
155 template<typename T, typename D> |
155 template<typename D, typename T> |
156 inline static bool replace_if_null(T* value, D* volatile* dest, |
156 inline static bool replace_if_null(D* volatile* dest, T* value, |
157 atomic_memory_order order = memory_order_conservative); |
157 atomic_memory_order order = memory_order_conservative); |
158 |
158 |
159 private: |
159 private: |
160 WINDOWS_ONLY(public:) // VS2017 warns (C2027) use of undefined type if IsPointerConvertible is declared private |
160 WINDOWS_ONLY(public:) // VS2017 warns (C2027) use of undefined type if IsPointerConvertible is declared private |
161 // Test whether From is implicitly convertible to To. |
161 // Test whether From is implicitly convertible to To. |
291 |
291 |
292 // Dispatch handler for cmpxchg. Provides type-based validity |
292 // Dispatch handler for cmpxchg. Provides type-based validity |
293 // checking and limited conversions around calls to the |
293 // checking and limited conversions around calls to the |
294 // platform-specific implementation layer provided by |
294 // platform-specific implementation layer provided by |
295 // PlatformCmpxchg. |
295 // PlatformCmpxchg. |
296 template<typename T, typename D, typename U, typename Enable = void> |
296 template<typename D, typename U, typename T, typename Enable = void> |
297 struct CmpxchgImpl; |
297 struct CmpxchgImpl; |
298 |
298 |
299 // Platform-specific implementation of cmpxchg. Support for sizes |
299 // Platform-specific implementation of cmpxchg. Support for sizes |
300 // of 1, 4, and 8 are required. The class is a function object that |
300 // of 1, 4, and 8 are required. The class is a function object that |
301 // must be default constructable, with these requirements: |
301 // must be default constructable, with these requirements: |
304 // - exchange_value and compare_value are of type T. |
304 // - exchange_value and compare_value are of type T. |
305 // - order is of type atomic_memory_order. |
305 // - order is of type atomic_memory_order. |
306 // - platform_cmpxchg is an object of type PlatformCmpxchg<sizeof(T)>. |
306 // - platform_cmpxchg is an object of type PlatformCmpxchg<sizeof(T)>. |
307 // |
307 // |
308 // Then |
308 // Then |
309 // platform_cmpxchg(exchange_value, dest, compare_value, order) |
309 // platform_cmpxchg(dest, compare_value, exchange_value, order) |
310 // must be a valid expression, returning a result convertible to T. |
310 // must be a valid expression, returning a result convertible to T. |
311 // |
311 // |
312 // A default definition is provided, which declares a function template |
312 // A default definition is provided, which declares a function template |
313 // T operator()(T, T volatile*, T, atomic_memory_order) const |
313 // T operator()(T volatile*, T, T, atomic_memory_order) const |
314 // |
314 // |
315 // For each required size, a platform must either provide an |
315 // For each required size, a platform must either provide an |
316 // appropriate definition of that function, or must entirely |
316 // appropriate definition of that function, or must entirely |
317 // specialize the class template for that size. |
317 // specialize the class template for that size. |
318 template<size_t byte_size> struct PlatformCmpxchg; |
318 template<size_t byte_size> struct PlatformCmpxchg; |
324 // helper invoked on the translated arguments, and the result |
324 // helper invoked on the translated arguments, and the result |
325 // translated back. Type is the parameter / return type of the |
325 // translated back. Type is the parameter / return type of the |
326 // helper function. |
326 // helper function. |
327 template<typename Type, typename Fn, typename T> |
327 template<typename Type, typename Fn, typename T> |
328 static T cmpxchg_using_helper(Fn fn, |
328 static T cmpxchg_using_helper(Fn fn, |
329 T exchange_value, |
|
330 T volatile* dest, |
329 T volatile* dest, |
331 T compare_value); |
330 T compare_value, |
|
331 T exchange_value); |
332 |
332 |
333 // Support platforms that do not provide Read-Modify-Write |
333 // Support platforms that do not provide Read-Modify-Write |
334 // byte-level atomic access. To use, derive PlatformCmpxchg<1> from |
334 // byte-level atomic access. To use, derive PlatformCmpxchg<1> from |
335 // this class. |
335 // this class. |
336 public: // Temporary, can't be private: C++03 11.4/2. Fixed by C++11. |
336 public: // Temporary, can't be private: C++03 11.4/2. Fixed by C++11. |
566 // specializations of the class. The platform file is responsible for |
566 // specializations of the class. The platform file is responsible for |
567 // providing those. |
567 // providing those. |
568 template<size_t byte_size> |
568 template<size_t byte_size> |
569 struct Atomic::PlatformCmpxchg { |
569 struct Atomic::PlatformCmpxchg { |
570 template<typename T> |
570 template<typename T> |
571 T operator()(T exchange_value, |
571 T operator()(T volatile* dest, |
572 T volatile* dest, |
|
573 T compare_value, |
572 T compare_value, |
|
573 T exchange_value, |
574 atomic_memory_order order) const; |
574 atomic_memory_order order) const; |
575 }; |
575 }; |
576 |
576 |
577 // Define the class before including platform file, which may use this |
577 // Define the class before including platform file, which may use this |
578 // as a base class, requiring it be complete. The definition is later |
578 // as a base class, requiring it be complete. The definition is later |
579 // in this file, near the other definitions related to cmpxchg. |
579 // in this file, near the other definitions related to cmpxchg. |
580 struct Atomic::CmpxchgByteUsingInt { |
580 struct Atomic::CmpxchgByteUsingInt { |
581 template<typename T> |
581 template<typename T> |
582 T operator()(T exchange_value, |
582 T operator()(T volatile* dest, |
583 T volatile* dest, |
|
584 T compare_value, |
583 T compare_value, |
|
584 T exchange_value, |
585 atomic_memory_order order) const; |
585 atomic_memory_order order) const; |
586 }; |
586 }; |
587 |
587 |
588 // Define the class before including platform file, which may specialize |
588 // Define the class before including platform file, which may specialize |
589 // the operator definition. No generic definition of specializations |
589 // the operator definition. No generic definition of specializations |
743 return PrimitiveConversions::cast<D>( |
743 return PrimitiveConversions::cast<D>( |
744 fn(PrimitiveConversions::cast<Type>(add_value), |
744 fn(PrimitiveConversions::cast<Type>(add_value), |
745 reinterpret_cast<Type volatile*>(dest))); |
745 reinterpret_cast<Type volatile*>(dest))); |
746 } |
746 } |
747 |
747 |
748 template<typename T, typename D, typename U> |
748 template<typename D, typename U, typename T> |
749 inline D Atomic::cmpxchg(T exchange_value, |
749 inline D Atomic::cmpxchg(D volatile* dest, |
750 D volatile* dest, |
|
751 U compare_value, |
750 U compare_value, |
|
751 T exchange_value, |
752 atomic_memory_order order) { |
752 atomic_memory_order order) { |
753 return CmpxchgImpl<T, D, U>()(exchange_value, dest, compare_value, order); |
753 return CmpxchgImpl<D, U, T>()(dest, compare_value, exchange_value, order); |
754 } |
754 } |
755 |
755 |
756 template<typename T, typename D> |
756 template<typename D, typename T> |
757 inline bool Atomic::replace_if_null(T* value, D* volatile* dest, |
757 inline bool Atomic::replace_if_null(D* volatile* dest, T* value, |
758 atomic_memory_order order) { |
758 atomic_memory_order order) { |
759 // Presently using a trivial implementation in terms of cmpxchg. |
759 // Presently using a trivial implementation in terms of cmpxchg. |
760 // Consider adding platform support, to permit the use of compiler |
760 // Consider adding platform support, to permit the use of compiler |
761 // intrinsics like gcc's __sync_bool_compare_and_swap. |
761 // intrinsics like gcc's __sync_bool_compare_and_swap. |
762 D* expected_null = NULL; |
762 D* expected_null = NULL; |
763 return expected_null == cmpxchg(value, dest, expected_null, order); |
763 return expected_null == cmpxchg(dest, expected_null, value, order); |
764 } |
764 } |
765 |
765 |
766 // Handle cmpxchg for integral and enum types. |
766 // Handle cmpxchg for integral and enum types. |
767 // |
767 // |
768 // All the involved types must be identical. |
768 // All the involved types must be identical. |
769 template<typename T> |
769 template<typename T> |
770 struct Atomic::CmpxchgImpl< |
770 struct Atomic::CmpxchgImpl< |
771 T, T, T, |
771 T, T, T, |
772 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type> |
772 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type> |
773 { |
773 { |
774 T operator()(T exchange_value, T volatile* dest, T compare_value, |
774 T operator()(T volatile* dest, T compare_value, T exchange_value, |
775 atomic_memory_order order) const { |
775 atomic_memory_order order) const { |
776 // Forward to the platform handler for the size of T. |
776 // Forward to the platform handler for the size of T. |
777 return PlatformCmpxchg<sizeof(T)>()(exchange_value, |
777 return PlatformCmpxchg<sizeof(T)>()(dest, |
778 dest, |
|
779 compare_value, |
778 compare_value, |
|
779 exchange_value, |
780 order); |
780 order); |
781 } |
781 } |
782 }; |
782 }; |
783 |
783 |
784 // Handle cmpxchg for pointer types. |
784 // Handle cmpxchg for pointer types. |
788 // the compare_value. |
788 // the compare_value. |
789 // |
789 // |
790 // The exchange_value must be implicitly convertible to the |
790 // The exchange_value must be implicitly convertible to the |
791 // destination's type; it must be type-correct to store the |
791 // destination's type; it must be type-correct to store the |
792 // exchange_value in the destination. |
792 // exchange_value in the destination. |
793 template<typename T, typename D, typename U> |
793 template<typename D, typename U, typename T> |
794 struct Atomic::CmpxchgImpl< |
794 struct Atomic::CmpxchgImpl< |
795 T*, D*, U*, |
795 D*, U*, T*, |
796 typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value && |
796 typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value && |
797 IsSame<typename RemoveCV<D>::type, |
797 IsSame<typename RemoveCV<D>::type, |
798 typename RemoveCV<U>::type>::value>::type> |
798 typename RemoveCV<U>::type>::value>::type> |
799 { |
799 { |
800 D* operator()(T* exchange_value, D* volatile* dest, U* compare_value, |
800 D* operator()(D* volatile* dest, U* compare_value, T* exchange_value, |
801 atomic_memory_order order) const { |
801 atomic_memory_order order) const { |
802 // Allow derived to base conversion, and adding cv-qualifiers. |
802 // Allow derived to base conversion, and adding cv-qualifiers. |
803 D* new_value = exchange_value; |
803 D* new_value = exchange_value; |
804 // Don't care what the CV qualifiers for compare_value are, |
804 // Don't care what the CV qualifiers for compare_value are, |
805 // but we need to match D* when calling platform support. |
805 // but we need to match D* when calling platform support. |
806 D* old_value = const_cast<D*>(compare_value); |
806 D* old_value = const_cast<D*>(compare_value); |
807 return PlatformCmpxchg<sizeof(D*)>()(new_value, dest, old_value, order); |
807 return PlatformCmpxchg<sizeof(D*)>()(dest, old_value, new_value, order); |
808 } |
808 } |
809 }; |
809 }; |
810 |
810 |
811 // Handle cmpxchg for types that have a translator. |
811 // Handle cmpxchg for types that have a translator. |
812 // |
812 // |
818 template<typename T> |
818 template<typename T> |
819 struct Atomic::CmpxchgImpl< |
819 struct Atomic::CmpxchgImpl< |
820 T, T, T, |
820 T, T, T, |
821 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type> |
821 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type> |
822 { |
822 { |
823 T operator()(T exchange_value, T volatile* dest, T compare_value, |
823 T operator()(T volatile* dest, T compare_value, T exchange_value, |
824 atomic_memory_order order) const { |
824 atomic_memory_order order) const { |
825 typedef PrimitiveConversions::Translate<T> Translator; |
825 typedef PrimitiveConversions::Translate<T> Translator; |
826 typedef typename Translator::Decayed Decayed; |
826 typedef typename Translator::Decayed Decayed; |
827 STATIC_ASSERT(sizeof(T) == sizeof(Decayed)); |
827 STATIC_ASSERT(sizeof(T) == sizeof(Decayed)); |
828 return Translator::recover( |
828 return Translator::recover( |
829 cmpxchg(Translator::decay(exchange_value), |
829 cmpxchg(reinterpret_cast<Decayed volatile*>(dest), |
830 reinterpret_cast<Decayed volatile*>(dest), |
|
831 Translator::decay(compare_value), |
830 Translator::decay(compare_value), |
|
831 Translator::decay(exchange_value), |
832 order)); |
832 order)); |
833 } |
833 } |
834 }; |
834 }; |
835 |
835 |
836 template<typename Type, typename Fn, typename T> |
836 template<typename Type, typename Fn, typename T> |
837 inline T Atomic::cmpxchg_using_helper(Fn fn, |
837 inline T Atomic::cmpxchg_using_helper(Fn fn, |
838 T exchange_value, |
|
839 T volatile* dest, |
838 T volatile* dest, |
840 T compare_value) { |
839 T compare_value, |
|
840 T exchange_value) { |
841 STATIC_ASSERT(sizeof(Type) == sizeof(T)); |
841 STATIC_ASSERT(sizeof(Type) == sizeof(T)); |
842 return PrimitiveConversions::cast<T>( |
842 return PrimitiveConversions::cast<T>( |
843 fn(PrimitiveConversions::cast<Type>(exchange_value), |
843 fn(PrimitiveConversions::cast<Type>(exchange_value), |
844 reinterpret_cast<Type volatile*>(dest), |
844 reinterpret_cast<Type volatile*>(dest), |
845 PrimitiveConversions::cast<Type>(compare_value))); |
845 PrimitiveConversions::cast<Type>(compare_value))); |
846 } |
846 } |
847 |
847 |
848 template<typename T> |
848 template<typename T> |
849 inline T Atomic::CmpxchgByteUsingInt::operator()(T exchange_value, |
849 inline T Atomic::CmpxchgByteUsingInt::operator()(T volatile* dest, |
850 T volatile* dest, |
|
851 T compare_value, |
850 T compare_value, |
|
851 T exchange_value, |
852 atomic_memory_order order) const { |
852 atomic_memory_order order) const { |
853 STATIC_ASSERT(sizeof(T) == sizeof(uint8_t)); |
853 STATIC_ASSERT(sizeof(T) == sizeof(uint8_t)); |
854 uint8_t canon_exchange_value = exchange_value; |
854 uint8_t canon_exchange_value = exchange_value; |
855 uint8_t canon_compare_value = compare_value; |
855 uint8_t canon_compare_value = compare_value; |
856 volatile uint32_t* aligned_dest |
856 volatile uint32_t* aligned_dest |
869 // value to swap in matches current value ... |
869 // value to swap in matches current value ... |
870 uint32_t new_value = cur; |
870 uint32_t new_value = cur; |
871 // ... except for the one byte we want to update |
871 // ... except for the one byte we want to update |
872 reinterpret_cast<uint8_t*>(&new_value)[offset] = canon_exchange_value; |
872 reinterpret_cast<uint8_t*>(&new_value)[offset] = canon_exchange_value; |
873 |
873 |
874 uint32_t res = cmpxchg(new_value, aligned_dest, cur, order); |
874 uint32_t res = cmpxchg(aligned_dest, cur, new_value, order); |
875 if (res == cur) break; // success |
875 if (res == cur) break; // success |
876 |
876 |
877 // at least one byte in the int changed value, so update |
877 // at least one byte in the int changed value, so update |
878 // our view of the current int |
878 // our view of the current int |
879 cur = res; |
879 cur = res; |