src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp
author stefank
Mon, 25 Nov 2019 12:22:13 +0100
changeset 59247 56bf71d64d51
parent 59122 5d73255c2d52
child 59248 e92153ed8bdc
permissions -rw-r--r--
8234562: Move OrderAccess::release_store*/load_acquire to Atomic Reviewed-by: rehn, dholmes
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
29182
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
     1
/*
53244
9807daeb47c4 8216167: Update include guards to reflect correct directories
coleenp
parents: 50029
diff changeset
     2
 * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved.
59122
5d73255c2d52 8233787: Break cycle in vm_version* includes
lucy
parents: 59102
diff changeset
     3
 * Copyright (c) 2014, 2019, Red Hat Inc. All rights reserved.
29182
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
     4
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
     5
 *
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
     6
 * This code is free software; you can redistribute it and/or modify it
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
     7
 * under the terms of the GNU General Public License version 2 only, as
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
     8
 * published by the Free Software Foundation.
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
     9
 *
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    10
 * This code is distributed in the hope that it will be useful, but WITHOUT
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    11
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    12
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    13
 * version 2 for more details (a copy is included in the LICENSE file that
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    14
 * accompanied this code).
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    15
 *
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    16
 * You should have received a copy of the GNU General Public License version
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    17
 * 2 along with this work; if not, write to the Free Software Foundation,
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    18
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    19
 *
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    20
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    21
 * or visit www.oracle.com if you need additional information or have any
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    22
 * questions.
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    23
 *
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    24
 */
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    25
53244
9807daeb47c4 8216167: Update include guards to reflect correct directories
coleenp
parents: 50029
diff changeset
    26
#ifndef OS_CPU_LINUX_AARCH64_ATOMIC_LINUX_AARCH64_HPP
9807daeb47c4 8216167: Update include guards to reflect correct directories
coleenp
parents: 50029
diff changeset
    27
#define OS_CPU_LINUX_AARCH64_ATOMIC_LINUX_AARCH64_HPP
29182
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    28
59122
5d73255c2d52 8233787: Break cycle in vm_version* includes
lucy
parents: 59102
diff changeset
    29
#include "runtime/vm_version.hpp"
29182
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    30
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    31
// Implementation of class atomic
59102
9afab9bf3e6a 8233912: aarch64: minor improvements of atomic operations
fyang
parents: 55398
diff changeset
    32
// Note that memory_order_conservative requires a full barrier after atomic stores.
9afab9bf3e6a 8233912: aarch64: minor improvements of atomic operations
fyang
parents: 55398
diff changeset
    33
// See https://patchwork.kernel.org/patch/3575821/
29182
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    34
46993
dd0f91c85ffc 8186476: Generalize Atomic::add with templates
eosterlund
parents: 46958
diff changeset
    35
template<size_t byte_size>
dd0f91c85ffc 8186476: Generalize Atomic::add with templates
eosterlund
parents: 46958
diff changeset
    36
struct Atomic::PlatformAdd
dd0f91c85ffc 8186476: Generalize Atomic::add with templates
eosterlund
parents: 46958
diff changeset
    37
  : Atomic::AddAndFetch<Atomic::PlatformAdd<byte_size> >
29182
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    38
{
46993
dd0f91c85ffc 8186476: Generalize Atomic::add with templates
eosterlund
parents: 46958
diff changeset
    39
  template<typename I, typename D>
50029
ea0a16ba6ac0 8202080: Introduce ordering semantics for Atomic::add and other RMW atomics
mdoerr
parents: 47593
diff changeset
    40
  D add_and_fetch(I add_value, D volatile* dest, atomic_memory_order order) const {
55398
e53ec3b362f4 8224851: AArch64: fix warnings and errors with Clang and GCC 8.3
ngasson
parents: 53244
diff changeset
    41
    D res = __atomic_add_fetch(dest, add_value, __ATOMIC_RELEASE);
e53ec3b362f4 8224851: AArch64: fix warnings and errors with Clang and GCC 8.3
ngasson
parents: 53244
diff changeset
    42
    FULL_MEM_BARRIER;
e53ec3b362f4 8224851: AArch64: fix warnings and errors with Clang and GCC 8.3
ngasson
parents: 53244
diff changeset
    43
    return res;
46993
dd0f91c85ffc 8186476: Generalize Atomic::add with templates
eosterlund
parents: 46958
diff changeset
    44
  }
dd0f91c85ffc 8186476: Generalize Atomic::add with templates
eosterlund
parents: 46958
diff changeset
    45
};
29182
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    46
47578
09c41c4913d9 8187977: Generalize Atomic::xchg to use templates
eosterlund
parents: 47552
diff changeset
    47
template<size_t byte_size>
09c41c4913d9 8187977: Generalize Atomic::xchg to use templates
eosterlund
parents: 47552
diff changeset
    48
template<typename T>
09c41c4913d9 8187977: Generalize Atomic::xchg to use templates
eosterlund
parents: 47552
diff changeset
    49
inline T Atomic::PlatformXchg<byte_size>::operator()(T exchange_value,
50029
ea0a16ba6ac0 8202080: Introduce ordering semantics for Atomic::add and other RMW atomics
mdoerr
parents: 47593
diff changeset
    50
                                                     T volatile* dest,
ea0a16ba6ac0 8202080: Introduce ordering semantics for Atomic::add and other RMW atomics
mdoerr
parents: 47593
diff changeset
    51
                                                     atomic_memory_order order) const {
47578
09c41c4913d9 8187977: Generalize Atomic::xchg to use templates
eosterlund
parents: 47552
diff changeset
    52
  STATIC_ASSERT(byte_size == sizeof(T));
59102
9afab9bf3e6a 8233912: aarch64: minor improvements of atomic operations
fyang
parents: 55398
diff changeset
    53
  T res = __atomic_exchange_n(dest, exchange_value, __ATOMIC_RELEASE);
29182
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    54
  FULL_MEM_BARRIER;
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    55
  return res;
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    56
}
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    57
46958
a13bd8c6b7a2 8186166: Generalize Atomic::cmpxchg with templates
eosterlund
parents: 46523
diff changeset
    58
template<size_t byte_size>
a13bd8c6b7a2 8186166: Generalize Atomic::cmpxchg with templates
eosterlund
parents: 46523
diff changeset
    59
template<typename T>
a13bd8c6b7a2 8186166: Generalize Atomic::cmpxchg with templates
eosterlund
parents: 46523
diff changeset
    60
inline T Atomic::PlatformCmpxchg<byte_size>::operator()(T exchange_value,
a13bd8c6b7a2 8186166: Generalize Atomic::cmpxchg with templates
eosterlund
parents: 46523
diff changeset
    61
                                                        T volatile* dest,
a13bd8c6b7a2 8186166: Generalize Atomic::cmpxchg with templates
eosterlund
parents: 46523
diff changeset
    62
                                                        T compare_value,
50029
ea0a16ba6ac0 8202080: Introduce ordering semantics for Atomic::add and other RMW atomics
mdoerr
parents: 47593
diff changeset
    63
                                                        atomic_memory_order order) const {
46958
a13bd8c6b7a2 8186166: Generalize Atomic::cmpxchg with templates
eosterlund
parents: 46523
diff changeset
    64
  STATIC_ASSERT(byte_size == sizeof(T));
39404
d0ad5220e91c 8155949: Support relaxed semantics in cmpxchg
mdoerr
parents: 29182
diff changeset
    65
  if (order == memory_order_relaxed) {
d0ad5220e91c 8155949: Support relaxed semantics in cmpxchg
mdoerr
parents: 29182
diff changeset
    66
    T value = compare_value;
d0ad5220e91c 8155949: Support relaxed semantics in cmpxchg
mdoerr
parents: 29182
diff changeset
    67
    __atomic_compare_exchange(dest, &value, &exchange_value, /*weak*/false,
d0ad5220e91c 8155949: Support relaxed semantics in cmpxchg
mdoerr
parents: 29182
diff changeset
    68
                              __ATOMIC_RELAXED, __ATOMIC_RELAXED);
d0ad5220e91c 8155949: Support relaxed semantics in cmpxchg
mdoerr
parents: 29182
diff changeset
    69
    return value;
d0ad5220e91c 8155949: Support relaxed semantics in cmpxchg
mdoerr
parents: 29182
diff changeset
    70
  } else {
59102
9afab9bf3e6a 8233912: aarch64: minor improvements of atomic operations
fyang
parents: 55398
diff changeset
    71
    T value = compare_value;
9afab9bf3e6a 8233912: aarch64: minor improvements of atomic operations
fyang
parents: 55398
diff changeset
    72
    FULL_MEM_BARRIER;
9afab9bf3e6a 8233912: aarch64: minor improvements of atomic operations
fyang
parents: 55398
diff changeset
    73
    __atomic_compare_exchange(dest, &value, &exchange_value, /*weak*/false,
9afab9bf3e6a 8233912: aarch64: minor improvements of atomic operations
fyang
parents: 55398
diff changeset
    74
                              __ATOMIC_RELAXED, __ATOMIC_RELAXED);
9afab9bf3e6a 8233912: aarch64: minor improvements of atomic operations
fyang
parents: 55398
diff changeset
    75
    FULL_MEM_BARRIER;
9afab9bf3e6a 8233912: aarch64: minor improvements of atomic operations
fyang
parents: 55398
diff changeset
    76
    return value;
39404
d0ad5220e91c 8155949: Support relaxed semantics in cmpxchg
mdoerr
parents: 29182
diff changeset
    77
  }
d0ad5220e91c 8155949: Support relaxed semantics in cmpxchg
mdoerr
parents: 29182
diff changeset
    78
}
29182
78af2a4d1ec3 8068055: AARCH64: os_cpu
aph
parents:
diff changeset
    79
59247
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    80
template<size_t byte_size>
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    81
struct Atomic::PlatformOrderedLoad<byte_size, X_ACQUIRE>
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    82
{
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    83
  template <typename T>
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    84
  T operator()(const volatile T* p) const { T data; __atomic_load(const_cast<T*>(p), &data, __ATOMIC_ACQUIRE); return data; }
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    85
};
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    86
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    87
template<size_t byte_size>
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    88
struct Atomic::PlatformOrderedStore<byte_size, RELEASE_X>
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    89
{
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    90
  template <typename T>
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    91
  void operator()(T v, volatile T* p) const { __atomic_store(const_cast<T*>(p), &v, __ATOMIC_RELEASE); }
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    92
};
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    93
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    94
template<size_t byte_size>
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    95
struct Atomic::PlatformOrderedStore<byte_size, RELEASE_X_FENCE>
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    96
{
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    97
  template <typename T>
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    98
  void operator()(T v, volatile T* p) const { release_store(p, v); OrderAccess::fence(); }
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
    99
};
56bf71d64d51 8234562: Move OrderAccess::release_store*/load_acquire to Atomic
stefank
parents: 59122
diff changeset
   100
53244
9807daeb47c4 8216167: Update include guards to reflect correct directories
coleenp
parents: 50029
diff changeset
   101
#endif // OS_CPU_LINUX_AARCH64_ATOMIC_LINUX_AARCH64_HPP