77 |
77 |
78 // Atomically store to a location |
78 // Atomically store to a location |
79 // The type T must be either a pointer type convertible to or equal |
79 // The type T must be either a pointer type convertible to or equal |
80 // to D, an integral/enum type equal to D, or a type equal to D that |
80 // to D, an integral/enum type equal to D, or a type equal to D that |
81 // is primitive convertible using PrimitiveConversions. |
81 // is primitive convertible using PrimitiveConversions. |
82 template<typename T, typename D> |
82 template<typename D, typename T> |
83 inline static void store(T store_value, volatile D* dest); |
83 inline static void store(volatile D* dest, T store_value); |
84 |
84 |
85 template <typename T, typename D> |
85 template <typename D, typename T> |
86 inline static void release_store(volatile D* dest, T store_value); |
86 inline static void release_store(volatile D* dest, T store_value); |
87 |
87 |
88 template <typename T, typename D> |
88 template <typename D, typename T> |
89 inline static void release_store_fence(volatile D* dest, T store_value); |
89 inline static void release_store_fence(volatile D* dest, T store_value); |
90 |
90 |
91 // Atomically load from a location |
91 // Atomically load from a location |
92 // The type T must be either a pointer type, an integral/enum type, |
92 // The type T must be either a pointer type, an integral/enum type, |
93 // or a type that is primitive convertible using PrimitiveConversions. |
93 // or a type that is primitive convertible using PrimitiveConversions. |
166 |
166 |
167 protected: |
167 protected: |
168 // Dispatch handler for store. Provides type-based validity |
168 // Dispatch handler for store. Provides type-based validity |
169 // checking and limited conversions around calls to the platform- |
169 // checking and limited conversions around calls to the platform- |
170 // specific implementation layer provided by PlatformOp. |
170 // specific implementation layer provided by PlatformOp. |
171 template<typename T, typename D, typename PlatformOp, typename Enable = void> |
171 template<typename D, typename T, typename PlatformOp, typename Enable = void> |
172 struct StoreImpl; |
172 struct StoreImpl; |
173 |
173 |
174 // Platform-specific implementation of store. Support for sizes |
174 // Platform-specific implementation of store. Support for sizes |
175 // of 1, 2, 4, and (if different) pointer size bytes are required. |
175 // of 1, 2, 4, and (if different) pointer size bytes are required. |
176 // The class is a function object that must be default constructable, |
176 // The class is a function object that must be default constructable, |
448 struct Atomic::StoreImpl< |
448 struct Atomic::StoreImpl< |
449 T, T, |
449 T, T, |
450 PlatformOp, |
450 PlatformOp, |
451 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type> |
451 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type> |
452 { |
452 { |
453 void operator()(T new_value, T volatile* dest) const { |
453 void operator()(T volatile* dest, T new_value) const { |
454 // Forward to the platform handler for the size of T. |
454 // Forward to the platform handler for the size of T. |
455 PlatformOp()(new_value, dest); |
455 PlatformOp()(dest, new_value); |
456 } |
456 } |
457 }; |
457 }; |
458 |
458 |
459 // Handle store for pointer types. |
459 // Handle store for pointer types. |
460 // |
460 // |
461 // The new_value must be implicitly convertible to the |
461 // The new_value must be implicitly convertible to the |
462 // destination's type; it must be type-correct to store the |
462 // destination's type; it must be type-correct to store the |
463 // new_value in the destination. |
463 // new_value in the destination. |
464 template<typename T, typename D, typename PlatformOp> |
464 template<typename D, typename T, typename PlatformOp> |
465 struct Atomic::StoreImpl< |
465 struct Atomic::StoreImpl< |
466 T*, D*, |
466 D*, T*, |
467 PlatformOp, |
467 PlatformOp, |
468 typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value>::type> |
468 typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value>::type> |
469 { |
469 { |
470 void operator()(T* new_value, D* volatile* dest) const { |
470 void operator()(D* volatile* dest, T* new_value) const { |
471 // Allow derived to base conversion, and adding cv-qualifiers. |
471 // Allow derived to base conversion, and adding cv-qualifiers. |
472 D* value = new_value; |
472 D* value = new_value; |
473 PlatformOp()(value, dest); |
473 PlatformOp()(dest, value); |
474 } |
474 } |
475 }; |
475 }; |
476 |
476 |
477 // Handle store for types that have a translator. |
477 // Handle store for types that have a translator. |
478 // |
478 // |
484 struct Atomic::StoreImpl< |
484 struct Atomic::StoreImpl< |
485 T, T, |
485 T, T, |
486 PlatformOp, |
486 PlatformOp, |
487 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type> |
487 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type> |
488 { |
488 { |
489 void operator()(T new_value, T volatile* dest) const { |
489 void operator()(T volatile* dest, T new_value) const { |
490 typedef PrimitiveConversions::Translate<T> Translator; |
490 typedef PrimitiveConversions::Translate<T> Translator; |
491 typedef typename Translator::Decayed Decayed; |
491 typedef typename Translator::Decayed Decayed; |
492 STATIC_ASSERT(sizeof(T) == sizeof(Decayed)); |
492 STATIC_ASSERT(sizeof(T) == sizeof(Decayed)); |
493 PlatformOp()(Translator::decay(new_value), |
493 PlatformOp()(reinterpret_cast<Decayed volatile*>(dest), |
494 reinterpret_cast<Decayed volatile*>(dest)); |
494 Translator::decay(new_value)); |
495 } |
495 } |
496 }; |
496 }; |
497 |
497 |
498 // Default implementation of atomic store if a specific platform |
498 // Default implementation of atomic store if a specific platform |
499 // does not provide a specialization for a certain size class. |
499 // does not provide a specialization for a certain size class. |
502 // supports wide atomics, then it has to use specialization |
502 // supports wide atomics, then it has to use specialization |
503 // of Atomic::PlatformStore for that wider size class. |
503 // of Atomic::PlatformStore for that wider size class. |
504 template<size_t byte_size> |
504 template<size_t byte_size> |
505 struct Atomic::PlatformStore { |
505 struct Atomic::PlatformStore { |
506 template<typename T> |
506 template<typename T> |
507 void operator()(T new_value, |
507 void operator()(T volatile* dest, |
508 T volatile* dest) const { |
508 T new_value) const { |
509 STATIC_ASSERT(sizeof(T) <= sizeof(void*)); // wide atomics need specialization |
509 STATIC_ASSERT(sizeof(T) <= sizeof(void*)); // wide atomics need specialization |
510 (void)const_cast<T&>(*dest = new_value); |
510 (void)const_cast<T&>(*dest = new_value); |
511 } |
511 } |
512 }; |
512 }; |
513 |
513 |
652 template <typename T> |
652 template <typename T> |
653 inline T Atomic::load_acquire(const volatile T* p) { |
653 inline T Atomic::load_acquire(const volatile T* p) { |
654 return LoadImpl<T, PlatformOrderedLoad<sizeof(T), X_ACQUIRE> >()(p); |
654 return LoadImpl<T, PlatformOrderedLoad<sizeof(T), X_ACQUIRE> >()(p); |
655 } |
655 } |
656 |
656 |
657 template<typename T, typename D> |
657 template<typename D, typename T> |
658 inline void Atomic::store(T store_value, volatile D* dest) { |
658 inline void Atomic::store(volatile D* dest, T store_value) { |
659 StoreImpl<T, D, PlatformStore<sizeof(D)> >()(store_value, dest); |
659 StoreImpl<D, T, PlatformStore<sizeof(D)> >()(dest, store_value); |
660 } |
660 } |
661 |
661 |
662 template<size_t byte_size, ScopedFenceType type> |
662 template<size_t byte_size, ScopedFenceType type> |
663 struct Atomic::PlatformOrderedStore { |
663 struct Atomic::PlatformOrderedStore { |
664 template <typename T> |
664 template <typename T> |
665 void operator()(T v, volatile T* p) const { |
665 void operator()(volatile T* p, T v) const { |
666 ScopedFence<type> f((void*)p); |
666 ScopedFence<type> f((void*)p); |
667 Atomic::store(v, p); |
667 Atomic::store(p, v); |
668 } |
668 } |
669 }; |
669 }; |
670 |
670 |
671 template <typename T, typename D> |
671 template <typename D, typename T> |
672 inline void Atomic::release_store(volatile D* p, T v) { |
672 inline void Atomic::release_store(volatile D* p, T v) { |
673 StoreImpl<T, D, PlatformOrderedStore<sizeof(D), RELEASE_X> >()(v, p); |
673 StoreImpl<D, T, PlatformOrderedStore<sizeof(D), RELEASE_X> >()(p, v); |
674 } |
674 } |
675 |
675 |
676 template <typename T, typename D> |
676 template <typename D, typename T> |
677 inline void Atomic::release_store_fence(volatile D* p, T v) { |
677 inline void Atomic::release_store_fence(volatile D* p, T v) { |
678 StoreImpl<T, D, PlatformOrderedStore<sizeof(D), RELEASE_X_FENCE> >()(v, p); |
678 StoreImpl<D, T, PlatformOrderedStore<sizeof(D), RELEASE_X_FENCE> >()(p, v); |
679 } |
679 } |
680 |
680 |
681 template<typename I, typename D> |
681 template<typename I, typename D> |
682 inline D Atomic::add(I add_value, D volatile* dest, |
682 inline D Atomic::add(I add_value, D volatile* dest, |
683 atomic_memory_order order) { |
683 atomic_memory_order order) { |