1#pragma once
2
3#ifdef NNSDK
4#include <atomic>
5#endif
6
7namespace sead
8{
9struct AtomicDirectInitTag
10{
11};
12
13template <class T>
14struct AtomicBase
15{
16public:
17 AtomicBase(T value = {}); // NOLINT(google-explicit-constructor)
18 /// Directly initialises the underlying atomic with the specified value.
19 /// Note that initialisation is not atomic.
20 AtomicBase(AtomicDirectInitTag, T value);
21 AtomicBase(const AtomicBase& rhs) { *this = rhs; }
22
23 operator T() const { return load(); }
24
25 AtomicBase& operator=(const AtomicBase& rhs)
26 {
27 store(value: rhs.load());
28 return *this;
29 }
30
31 AtomicBase& operator=(T value)
32 {
33 store(value);
34 return *this;
35 }
36
37 /// Load the current value, as if with memory_order_relaxed.
38 T load() const;
39 /// Store a new value, as if with memory_order_relaxed.
40 void store(T value);
41 /// Non-atomically store a new value.
42 void storeNonAtomic(T value);
43 /// Exchange/swap the current value, as if with memory_order_relaxed.
44 /// @return the previous value
45 T exchange(T value);
46 /// Load the current value and if it is equal to `expected`, store `desired`
47 /// as if with memory_order_relaxed.
48 /// Otherwise, this sets `original` to the current value.
49 /// @param expected The value expected to be found in the atomic object, and to be replaced.
50 /// @param desired The new value to store in the atomic object if `expected` was found.
51 /// @param original The value that was found in the atomic object if the comparison fails. May
52 /// be null. Note that this is only updated when false is returned.
53 /// @return true if and only if the value was modified
54 bool compareExchange(T expected, T desired, T* original = nullptr);
55
56protected:
57#ifdef NNSDK
58 // Nintendo appears to have manually implemented atomics with volatile and platform specific
59 // intrinsics (e.g. __builtin_arm_ldrex).
60 // For ease of implementation and portability, we will use std::atomic and cast to volatile
61 // when necessary. That is formally undefined behavior, but it should be safe because
62 // sead is built with -fno-strict-aliasing and because of the following static assertions.
63 std::atomic<T> mValue;
64 static_assert(sizeof(mValue) == sizeof(T),
65 "std::atomic<T> and T do not have the same size; unsupported case");
66 static_assert(alignof(decltype(mValue)) == alignof(volatile T),
67 "std::atomic<T> and T do not have the same alignment; unsupported case");
68 static_assert(std::atomic<T>::is_always_lock_free,
69 "std::atomic<T>::is_always_lock_free is not true; unsupported case");
70
71 const volatile T* getValuePtr() const { return reinterpret_cast<const volatile T*>(&mValue); }
72 volatile T* getValuePtr() { return reinterpret_cast<volatile T*>(&mValue); }
73
74#endif
75};
76
77template <class T>
78struct Atomic : AtomicBase<T>
79{
80 using AtomicBase<T>::AtomicBase;
81 using AtomicBase<T>::operator=;
82
83 T fetchAdd(T x);
84 T fetchSub(T x);
85 T fetchAnd(T x);
86 T fetchOr(T x);
87 T fetchXor(T x);
88 T increment() { return fetchAdd(x: 1); }
89 T decrement() { return fetchSub(x: 1); }
90
91 bool isBitOn(unsigned int bit) const;
92 /// @return whether the bit was cleared and is now set.
93 bool setBitOn(unsigned int bit);
94 /// @return whether the bit was set and is now cleared.
95 bool setBitOff(unsigned int bit);
96
97 T operator+=(T x) { return fetchAdd(x); }
98 T operator-=(T x) { return fetchSub(x); }
99 T operator&=(T x) { return fetchAnd(x); }
100 T operator|=(T x) { return fetchOr(x); }
101 T operator^=(T x) { return fetchXor(x); }
102 T operator++() { return fetchAdd(x: 1) + 1; }
103 T operator++(int) { return fetchAdd(x: 1); }
104 T operator--() { return fetchSub(x: 1) - 1; }
105 T operator--(int) { return fetchSub(x: 1); }
106};
107
108/// Specialization for pointer types.
109template <class T>
110struct Atomic<T*> : AtomicBase<T*>
111{
112 using AtomicBase<T*>::AtomicBase;
113 using AtomicBase<T*>::operator=;
114
115 T& operator*() const { return *this->load(); }
116 T* operator->() const { return this->load(); }
117};
118
119// Implementation.
120
121#ifdef NNSDK
122template <class T>
123inline AtomicBase<T>::AtomicBase(T value)
124{
125 storeNonAtomic(value);
126}
127
128template <class T>
129inline AtomicBase<T>::AtomicBase(AtomicDirectInitTag, T value) : mValue(value)
130{
131}
132
133template <class T>
134inline T AtomicBase<T>::load() const
135{
136#ifdef MATCHING_HACK_NX_CLANG
137 // Using std::atomic<T>::load prevents LLVM from folding ldr+sext into ldrsw.
138 return *getValuePtr();
139#else
140 return mValue.load(std::memory_order_relaxed);
141#endif
142}
143
144template <class T>
145inline void AtomicBase<T>::store(T value)
146{
147 mValue.store(value, std::memory_order_relaxed);
148}
149
150template <class T>
151inline void AtomicBase<T>::storeNonAtomic(T value)
152{
153 *getValuePtr() = value;
154}
155
156template <class T>
157inline T AtomicBase<T>::exchange(T value)
158{
159 return mValue.exchange(value, std::memory_order_relaxed);
160}
161
162template <class T>
163inline bool AtomicBase<T>::compareExchange(T expected, T desired, T* original)
164{
165#ifdef MATCHING_HACK_NX_CLANG
166 // Unlike Clang (https://reviews.llvm.org/D13033), Nintendo's implementation does not use clrex.
167 do
168 {
169 T value = __builtin_arm_ldrex(getValuePtr());
170 if (value != expected)
171 {
172 if (original)
173 *original = value;
174 return false;
175 }
176 } while (__builtin_arm_strex(desired, getValuePtr()));
177 return true;
178#else
179 T value = expected;
180 if (mValue.compare_exchange_strong(value, desired, std::memory_order_relaxed))
181 return true;
182 if (original)
183 *original = value;
184 return false;
185#endif
186}
187
188#ifdef MATCHING_HACK_NX_CLANG
189namespace detail
190{
191// To match Nintendo's implementation of atomics.
192template <typename T, typename F>
193inline T atomicReadModifyWrite(volatile T* value_ptr, F op)
194{
195 T value;
196 do
197 {
198 value = __builtin_arm_ldrex(value_ptr);
199 } while (__builtin_arm_strex(op(value), value_ptr));
200 return value;
201}
202} // namespace detail
203#endif
204
205template <class T>
206inline T Atomic<T>::fetchAdd(T x)
207{
208#ifdef MATCHING_HACK_NX_CLANG
209 return detail::atomicReadModifyWrite(this->getValuePtr(), [&](T val) { return val + x; });
210#else
211 return this->mValue.fetch_add(x, std::memory_order_relaxed);
212#endif
213}
214
215template <class T>
216inline T Atomic<T>::fetchSub(T x)
217{
218#ifdef MATCHING_HACK_NX_CLANG
219 return detail::atomicReadModifyWrite(this->getValuePtr(), [&](T val) { return val - x; });
220#else
221 return this->mValue.fetch_sub(x, std::memory_order_relaxed);
222#endif
223}
224
225template <class T>
226inline T Atomic<T>::fetchAnd(T x)
227{
228#ifdef MATCHING_HACK_NX_CLANG
229 return detail::atomicReadModifyWrite(this->getValuePtr(), [&](T val) { return val & x; });
230#else
231 return this->mValue.fetch_and(x, std::memory_order_relaxed);
232#endif
233}
234
235template <class T>
236inline T Atomic<T>::fetchOr(T x)
237{
238#ifdef MATCHING_HACK_NX_CLANG
239 return detail::atomicReadModifyWrite(this->getValuePtr(), [&](T val) { return val | x; });
240#else
241 return this->mValue.fetch_or(x, std::memory_order_relaxed);
242#endif
243}
244
245template <class T>
246inline T Atomic<T>::fetchXor(T x)
247{
248#ifdef MATCHING_HACK_NX_CLANG
249 return detail::atomicReadModifyWrite(this->getValuePtr(), [&](T val) { return val ^ x; });
250#else
251 return this->mValue.fetch_xor(x, std::memory_order_relaxed);
252#endif
253}
254
255template <class T>
256bool Atomic<T>::isBitOn(unsigned int bit) const
257{
258 return (this->load() & (1 << bit)) != 0;
259}
260
261template <class T>
262bool Atomic<T>::setBitOn(unsigned int bit)
263{
264#ifdef MATCHING_HACK_NX_CLANG
265 const auto old = detail::atomicReadModifyWrite(this->getValuePtr(),
266 [bit](T val) { return val | (1 << bit); });
267#else
268 const auto old = this->mValue.fetch_or(1 << bit, std::memory_order_relaxed);
269#endif
270 return (old & (1 << bit)) == 0;
271}
272
273template <class T>
274bool Atomic<T>::setBitOff(unsigned int bit)
275{
276#ifdef MATCHING_HACK_NX_CLANG
277 const auto old = detail::atomicReadModifyWrite(this->getValuePtr(),
278 [bit](T val) { return val & ~(1 << bit); });
279#else
280 const auto old = this->mValue.fetch_and(~(1 << bit), std::memory_order_relaxed);
281#endif
282 return (old & (1 << bit)) != 0;
283}
284#else // NNSDK
285#error "Unknown platform"
286#endif
287} // namespace sead
288