16namespace Synchronization {
23struct remove_volatile {
27struct remove_volatile<volatile T> {
33 return __sync_and_and_fetch(addr, val);
37ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
38 return __sync_fetch_and_add(addr, val);
41template<typename T, typename TV = typename remove_volatile<T>::type>
42ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
43 return __sync_fetch_and_add(addr, val);
46template<typename T, typename TV = typename remove_volatile<T>::type>
47ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
48 return __sync_fetch_and_sub(addr, val);
51template<typename T, typename TV = typename remove_volatile<T>::type>
52ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
53 return __sync_fetch_and_or(addr, val);
57ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
58 return __sync_add_and_fetch(addr, val);
62ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
63 return __sync_sub_and_fetch(addr, val);
66template<typename T, typename TV = typename remove_volatile<T>::type>
67ALWAYS_INLINE bool cas_strong_sequentially_consistent_helper(T *addr, TV *expected, TV *desired) {
68 TV oldval = *expected;
69 TV gotval = __sync_val_compare_and_swap(addr, oldval, *desired);
71 return oldval == gotval;
75 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
78template<typename T, typename TV = typename remove_volatile<T>::type>
79ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
80 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
84 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
88ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
89 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
93 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
97 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
102 return __sync_fetch_and_and(addr, val);
105template<typename T, typename TV = typename remove_volatile<T>::type>
106ALWAYS_INLINE T atomic_fetch_and_sequentially_consistent(T *addr, TV val) {
107 return __sync_fetch_and_and(addr, val);
117 __sync_synchronize();
124 return __sync_lock_test_and_set(addr, val);
128 return __sync_or_and_fetch(addr, val);
139 __sync_synchronize();
142template<typename T, typename TV = typename remove_volatile<T>::type>
143ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
145 __sync_synchronize();
149 __sync_synchronize();
152ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
153 __sync_synchronize();
159 return __atomic_and_fetch(addr, val, __ATOMIC_RELEASE);
163ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
164 return __atomic_fetch_add(addr, val, __ATOMIC_ACQ_REL);
167template<typename T, typename TV = typename remove_volatile<T>::type>
168ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
169 return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
172template<typename T, typename TV = typename remove_volatile<T>::type>
173ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
174 return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
177template<typename T, typename TV = typename remove_volatile<T>::type>
178ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
179 return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
183ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
184 return __atomic_add_fetch(addr, val, __ATOMIC_SEQ_CST);
188ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
189 return __atomic_sub_fetch(addr, val, __ATOMIC_SEQ_CST);
193 return __atomic_compare_exchange(addr, expected, desired,
false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
196template<typename T, typename TV = typename remove_volatile<T>::type>
197ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
198 return __atomic_compare_exchange(addr, expected, desired,
false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
202ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
203 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
207 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
211 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
215 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
220 return __atomic_fetch_and(addr, val, __ATOMIC_RELEASE);
223template<typename T, typename TV = typename remove_volatile<T>::type>
225 return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
230 __atomic_load(addr, val, __ATOMIC_RELAXED);
235 __atomic_load(addr, val, __ATOMIC_ACQUIRE);
236 __sync_synchronize();
243 __atomic_exchange(addr, &val, &result, __ATOMIC_ACQUIRE);
248 return __atomic_or_fetch(addr, val, __ATOMIC_RELAXED);
253 __atomic_store(addr, val, __ATOMIC_RELAXED);
258 __atomic_store(addr, val, __ATOMIC_RELEASE);
261template<typename T, typename TV = typename remove_volatile<T>::type>
262ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
263 __atomic_store(addr, val, __ATOMIC_SEQ_CST);
267 __atomic_thread_fence(__ATOMIC_ACQUIRE);
270ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
271 __atomic_thread_fence(__ATOMIC_SEQ_CST);