src/hotspot/share/runtime/atomic.hpp
changeset 59252 623722a6aeb9
parent 59251 4cbfa5077d68
equal deleted inserted replaced
59251:4cbfa5077d68 59252:623722a6aeb9
   139   // Performs atomic compare of *dest and compare_value, and exchanges
   139   // Performs atomic compare of *dest and compare_value, and exchanges
   140   // *dest with exchange_value if the comparison succeeded. Returns prior
   140   // *dest with exchange_value if the comparison succeeded. Returns prior
   141   // value of *dest. cmpxchg*() provide:
   141   // value of *dest. cmpxchg*() provide:
   142   // <fence> compare-and-exchange <membar StoreLoad|StoreStore>
   142   // <fence> compare-and-exchange <membar StoreLoad|StoreStore>
   143 
   143 
   144   template<typename T, typename D, typename U>
   144   template<typename D, typename U, typename T>
   145   inline static D cmpxchg(T exchange_value,
   145   inline static D cmpxchg(D volatile* dest,
   146                           D volatile* dest,
       
   147                           U compare_value,
   146                           U compare_value,
       
   147                           T exchange_value,
   148                           atomic_memory_order order = memory_order_conservative);
   148                           atomic_memory_order order = memory_order_conservative);
   149 
   149 
   150   // Performs atomic compare of *dest and NULL, and replaces *dest
   150   // Performs atomic compare of *dest and NULL, and replaces *dest
   151   // with exchange_value if the comparison succeeded.  Returns true if
   151   // with exchange_value if the comparison succeeded.  Returns true if
   152   // the comparison succeeded and the exchange occurred.  This is
   152   // the comparison succeeded and the exchange occurred.  This is
   153   // often used as part of lazy initialization, as a lock-free
   153   // often used as part of lazy initialization, as a lock-free
   154   // alternative to the Double-Checked Locking Pattern.
   154   // alternative to the Double-Checked Locking Pattern.
   155   template<typename T, typename D>
   155   template<typename D, typename T>
   156   inline static bool replace_if_null(T* value, D* volatile* dest,
   156   inline static bool replace_if_null(D* volatile* dest, T* value,
   157                                      atomic_memory_order order = memory_order_conservative);
   157                                      atomic_memory_order order = memory_order_conservative);
   158 
   158 
   159 private:
   159 private:
   160 WINDOWS_ONLY(public:) // VS2017 warns (C2027) use of undefined type if IsPointerConvertible is declared private
   160 WINDOWS_ONLY(public:) // VS2017 warns (C2027) use of undefined type if IsPointerConvertible is declared private
   161   // Test whether From is implicitly convertible to To.
   161   // Test whether From is implicitly convertible to To.
   291 
   291 
   292   // Dispatch handler for cmpxchg.  Provides type-based validity
   292   // Dispatch handler for cmpxchg.  Provides type-based validity
   293   // checking and limited conversions around calls to the
   293   // checking and limited conversions around calls to the
   294   // platform-specific implementation layer provided by
   294   // platform-specific implementation layer provided by
   295   // PlatformCmpxchg.
   295   // PlatformCmpxchg.
   296   template<typename T, typename D, typename U, typename Enable = void>
   296   template<typename D, typename U, typename T, typename Enable = void>
   297   struct CmpxchgImpl;
   297   struct CmpxchgImpl;
   298 
   298 
   299   // Platform-specific implementation of cmpxchg.  Support for sizes
   299   // Platform-specific implementation of cmpxchg.  Support for sizes
   300   // of 1, 4, and 8 are required.  The class is a function object that
   300   // of 1, 4, and 8 are required.  The class is a function object that
   301   // must be default constructable, with these requirements:
   301   // must be default constructable, with these requirements:
   304   // - exchange_value and compare_value are of type T.
   304   // - exchange_value and compare_value are of type T.
   305   // - order is of type atomic_memory_order.
   305   // - order is of type atomic_memory_order.
   306   // - platform_cmpxchg is an object of type PlatformCmpxchg<sizeof(T)>.
   306   // - platform_cmpxchg is an object of type PlatformCmpxchg<sizeof(T)>.
   307   //
   307   //
   308   // Then
   308   // Then
   309   //   platform_cmpxchg(exchange_value, dest, compare_value, order)
   309   //   platform_cmpxchg(dest, compare_value, exchange_value, order)
   310   // must be a valid expression, returning a result convertible to T.
   310   // must be a valid expression, returning a result convertible to T.
   311   //
   311   //
   312   // A default definition is provided, which declares a function template
   312   // A default definition is provided, which declares a function template
   313   //   T operator()(T, T volatile*, T, atomic_memory_order) const
   313   //   T operator()(T volatile*, T, T, atomic_memory_order) const
   314   //
   314   //
   315   // For each required size, a platform must either provide an
   315   // For each required size, a platform must either provide an
   316   // appropriate definition of that function, or must entirely
   316   // appropriate definition of that function, or must entirely
   317   // specialize the class template for that size.
   317   // specialize the class template for that size.
   318   template<size_t byte_size> struct PlatformCmpxchg;
   318   template<size_t byte_size> struct PlatformCmpxchg;
   324   // helper invoked on the translated arguments, and the result
   324   // helper invoked on the translated arguments, and the result
   325   // translated back.  Type is the parameter / return type of the
   325   // translated back.  Type is the parameter / return type of the
   326   // helper function.
   326   // helper function.
   327   template<typename Type, typename Fn, typename T>
   327   template<typename Type, typename Fn, typename T>
   328   static T cmpxchg_using_helper(Fn fn,
   328   static T cmpxchg_using_helper(Fn fn,
   329                                 T exchange_value,
       
   330                                 T volatile* dest,
   329                                 T volatile* dest,
   331                                 T compare_value);
   330                                 T compare_value,
       
   331                                 T exchange_value);
   332 
   332 
   333   // Support platforms that do not provide Read-Modify-Write
   333   // Support platforms that do not provide Read-Modify-Write
   334   // byte-level atomic access. To use, derive PlatformCmpxchg<1> from
   334   // byte-level atomic access. To use, derive PlatformCmpxchg<1> from
   335   // this class.
   335   // this class.
   336 public: // Temporary, can't be private: C++03 11.4/2. Fixed by C++11.
   336 public: // Temporary, can't be private: C++03 11.4/2. Fixed by C++11.
   566 // specializations of the class.  The platform file is responsible for
   566 // specializations of the class.  The platform file is responsible for
   567 // providing those.
   567 // providing those.
   568 template<size_t byte_size>
   568 template<size_t byte_size>
   569 struct Atomic::PlatformCmpxchg {
   569 struct Atomic::PlatformCmpxchg {
   570   template<typename T>
   570   template<typename T>
   571   T operator()(T exchange_value,
   571   T operator()(T volatile* dest,
   572                T volatile* dest,
       
   573                T compare_value,
   572                T compare_value,
       
   573                T exchange_value,
   574                atomic_memory_order order) const;
   574                atomic_memory_order order) const;
   575 };
   575 };
   576 
   576 
   577 // Define the class before including platform file, which may use this
   577 // Define the class before including platform file, which may use this
   578 // as a base class, requiring it be complete.  The definition is later
   578 // as a base class, requiring it be complete.  The definition is later
   579 // in this file, near the other definitions related to cmpxchg.
   579 // in this file, near the other definitions related to cmpxchg.
   580 struct Atomic::CmpxchgByteUsingInt {
   580 struct Atomic::CmpxchgByteUsingInt {
   581   template<typename T>
   581   template<typename T>
   582   T operator()(T exchange_value,
   582   T operator()(T volatile* dest,
   583                T volatile* dest,
       
   584                T compare_value,
   583                T compare_value,
       
   584                T exchange_value,
   585                atomic_memory_order order) const;
   585                atomic_memory_order order) const;
   586 };
   586 };
   587 
   587 
   588 // Define the class before including platform file, which may specialize
   588 // Define the class before including platform file, which may specialize
   589 // the operator definition.  No generic definition of specializations
   589 // the operator definition.  No generic definition of specializations
   743   return PrimitiveConversions::cast<D>(
   743   return PrimitiveConversions::cast<D>(
   744     fn(PrimitiveConversions::cast<Type>(add_value),
   744     fn(PrimitiveConversions::cast<Type>(add_value),
   745        reinterpret_cast<Type volatile*>(dest)));
   745        reinterpret_cast<Type volatile*>(dest)));
   746 }
   746 }
   747 
   747 
   748 template<typename T, typename D, typename U>
   748 template<typename D, typename U, typename T>
   749 inline D Atomic::cmpxchg(T exchange_value,
   749 inline D Atomic::cmpxchg(D volatile* dest,
   750                          D volatile* dest,
       
   751                          U compare_value,
   750                          U compare_value,
       
   751                          T exchange_value,
   752                          atomic_memory_order order) {
   752                          atomic_memory_order order) {
   753   return CmpxchgImpl<T, D, U>()(exchange_value, dest, compare_value, order);
   753   return CmpxchgImpl<D, U, T>()(dest, compare_value, exchange_value, order);
   754 }
   754 }
   755 
   755 
   756 template<typename T, typename D>
   756 template<typename D, typename T>
   757 inline bool Atomic::replace_if_null(T* value, D* volatile* dest,
   757 inline bool Atomic::replace_if_null(D* volatile* dest, T* value,
   758                                     atomic_memory_order order) {
   758                                     atomic_memory_order order) {
   759   // Presently using a trivial implementation in terms of cmpxchg.
   759   // Presently using a trivial implementation in terms of cmpxchg.
   760   // Consider adding platform support, to permit the use of compiler
   760   // Consider adding platform support, to permit the use of compiler
   761   // intrinsics like gcc's __sync_bool_compare_and_swap.
   761   // intrinsics like gcc's __sync_bool_compare_and_swap.
   762   D* expected_null = NULL;
   762   D* expected_null = NULL;
   763   return expected_null == cmpxchg(value, dest, expected_null, order);
   763   return expected_null == cmpxchg(dest, expected_null, value, order);
   764 }
   764 }
   765 
   765 
   766 // Handle cmpxchg for integral and enum types.
   766 // Handle cmpxchg for integral and enum types.
   767 //
   767 //
   768 // All the involved types must be identical.
   768 // All the involved types must be identical.
   769 template<typename T>
   769 template<typename T>
   770 struct Atomic::CmpxchgImpl<
   770 struct Atomic::CmpxchgImpl<
   771   T, T, T,
   771   T, T, T,
   772   typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type>
   772   typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type>
   773 {
   773 {
   774   T operator()(T exchange_value, T volatile* dest, T compare_value,
   774   T operator()(T volatile* dest, T compare_value, T exchange_value,
   775                atomic_memory_order order) const {
   775                atomic_memory_order order) const {
   776     // Forward to the platform handler for the size of T.
   776     // Forward to the platform handler for the size of T.
   777     return PlatformCmpxchg<sizeof(T)>()(exchange_value,
   777     return PlatformCmpxchg<sizeof(T)>()(dest,
   778                                         dest,
       
   779                                         compare_value,
   778                                         compare_value,
       
   779                                         exchange_value,
   780                                         order);
   780                                         order);
   781   }
   781   }
   782 };
   782 };
   783 
   783 
   784 // Handle cmpxchg for pointer types.
   784 // Handle cmpxchg for pointer types.
   788 // the compare_value.
   788 // the compare_value.
   789 //
   789 //
   790 // The exchange_value must be implicitly convertible to the
   790 // The exchange_value must be implicitly convertible to the
   791 // destination's type; it must be type-correct to store the
   791 // destination's type; it must be type-correct to store the
   792 // exchange_value in the destination.
   792 // exchange_value in the destination.
   793 template<typename T, typename D, typename U>
   793 template<typename D, typename U, typename T>
   794 struct Atomic::CmpxchgImpl<
   794 struct Atomic::CmpxchgImpl<
   795   T*, D*, U*,
   795   D*, U*, T*,
   796   typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value &&
   796   typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value &&
   797                     IsSame<typename RemoveCV<D>::type,
   797                     IsSame<typename RemoveCV<D>::type,
   798                            typename RemoveCV<U>::type>::value>::type>
   798                            typename RemoveCV<U>::type>::value>::type>
   799 {
   799 {
   800   D* operator()(T* exchange_value, D* volatile* dest, U* compare_value,
   800   D* operator()(D* volatile* dest, U* compare_value, T* exchange_value,
   801                atomic_memory_order order) const {
   801                atomic_memory_order order) const {
   802     // Allow derived to base conversion, and adding cv-qualifiers.
   802     // Allow derived to base conversion, and adding cv-qualifiers.
   803     D* new_value = exchange_value;
   803     D* new_value = exchange_value;
   804     // Don't care what the CV qualifiers for compare_value are,
   804     // Don't care what the CV qualifiers for compare_value are,
   805     // but we need to match D* when calling platform support.
   805     // but we need to match D* when calling platform support.
   806     D* old_value = const_cast<D*>(compare_value);
   806     D* old_value = const_cast<D*>(compare_value);
   807     return PlatformCmpxchg<sizeof(D*)>()(new_value, dest, old_value, order);
   807     return PlatformCmpxchg<sizeof(D*)>()(dest, old_value, new_value, order);
   808   }
   808   }
   809 };
   809 };
   810 
   810 
   811 // Handle cmpxchg for types that have a translator.
   811 // Handle cmpxchg for types that have a translator.
   812 //
   812 //
   818 template<typename T>
   818 template<typename T>
   819 struct Atomic::CmpxchgImpl<
   819 struct Atomic::CmpxchgImpl<
   820   T, T, T,
   820   T, T, T,
   821   typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
   821   typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
   822 {
   822 {
   823   T operator()(T exchange_value, T volatile* dest, T compare_value,
   823   T operator()(T volatile* dest, T compare_value, T exchange_value,
   824                atomic_memory_order order) const {
   824                atomic_memory_order order) const {
   825     typedef PrimitiveConversions::Translate<T> Translator;
   825     typedef PrimitiveConversions::Translate<T> Translator;
   826     typedef typename Translator::Decayed Decayed;
   826     typedef typename Translator::Decayed Decayed;
   827     STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
   827     STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
   828     return Translator::recover(
   828     return Translator::recover(
   829       cmpxchg(Translator::decay(exchange_value),
   829       cmpxchg(reinterpret_cast<Decayed volatile*>(dest),
   830               reinterpret_cast<Decayed volatile*>(dest),
       
   831               Translator::decay(compare_value),
   830               Translator::decay(compare_value),
       
   831               Translator::decay(exchange_value),
   832               order));
   832               order));
   833   }
   833   }
   834 };
   834 };
   835 
   835 
   836 template<typename Type, typename Fn, typename T>
   836 template<typename Type, typename Fn, typename T>
   837 inline T Atomic::cmpxchg_using_helper(Fn fn,
   837 inline T Atomic::cmpxchg_using_helper(Fn fn,
   838                                       T exchange_value,
       
   839                                       T volatile* dest,
   838                                       T volatile* dest,
   840                                       T compare_value) {
   839                                       T compare_value,
       
   840                                       T exchange_value) {
   841   STATIC_ASSERT(sizeof(Type) == sizeof(T));
   841   STATIC_ASSERT(sizeof(Type) == sizeof(T));
   842   return PrimitiveConversions::cast<T>(
   842   return PrimitiveConversions::cast<T>(
   843     fn(PrimitiveConversions::cast<Type>(exchange_value),
   843     fn(PrimitiveConversions::cast<Type>(exchange_value),
   844        reinterpret_cast<Type volatile*>(dest),
   844        reinterpret_cast<Type volatile*>(dest),
   845        PrimitiveConversions::cast<Type>(compare_value)));
   845        PrimitiveConversions::cast<Type>(compare_value)));
   846 }
   846 }
   847 
   847 
   848 template<typename T>
   848 template<typename T>
   849 inline T Atomic::CmpxchgByteUsingInt::operator()(T exchange_value,
   849 inline T Atomic::CmpxchgByteUsingInt::operator()(T volatile* dest,
   850                                                  T volatile* dest,
       
   851                                                  T compare_value,
   850                                                  T compare_value,
       
   851                                                  T exchange_value,
   852                                                  atomic_memory_order order) const {
   852                                                  atomic_memory_order order) const {
   853   STATIC_ASSERT(sizeof(T) == sizeof(uint8_t));
   853   STATIC_ASSERT(sizeof(T) == sizeof(uint8_t));
   854   uint8_t canon_exchange_value = exchange_value;
   854   uint8_t canon_exchange_value = exchange_value;
   855   uint8_t canon_compare_value = compare_value;
   855   uint8_t canon_compare_value = compare_value;
   856   volatile uint32_t* aligned_dest
   856   volatile uint32_t* aligned_dest
   869     // value to swap in matches current value ...
   869     // value to swap in matches current value ...
   870     uint32_t new_value = cur;
   870     uint32_t new_value = cur;
   871     // ... except for the one byte we want to update
   871     // ... except for the one byte we want to update
   872     reinterpret_cast<uint8_t*>(&new_value)[offset] = canon_exchange_value;
   872     reinterpret_cast<uint8_t*>(&new_value)[offset] = canon_exchange_value;
   873 
   873 
   874     uint32_t res = cmpxchg(new_value, aligned_dest, cur, order);
   874     uint32_t res = cmpxchg(aligned_dest, cur, new_value, order);
   875     if (res == cur) break;      // success
   875     if (res == cur) break;      // success
   876 
   876 
   877     // at least one byte in the int changed value, so update
   877     // at least one byte in the int changed value, so update
   878     // our view of the current int
   878     // our view of the current int
   879     cur = res;
   879     cur = res;