Skip to content

Commit 96faffb

Browse files
nabijaczlewelinabijaczleweli
committed
libspl: implement atomics in terms of atomics
On default-configured amd64 with clang 13.0.0-++20210405110107+e2a0f512eaca-1~exp1~20210405210820.1423 this now generates equivalent code (though with depeer prologues) to the assembly implementation (though, notably, the _nv variants are branchless, but atomic_{clear,set}_long_excl are pretty bad (still infinitely better than the garbage pthread mutexes generate)) Signed-off-by: Ahelenia Ziemiańska <[email protected]>
1 parent 3e66053 commit 96faffb

File tree

1 file changed

+39
-140
lines changed

1 file changed

+39
-140
lines changed

‎lib/libspl/asm-generic/atomic.c

Lines changed: 39 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -25,16 +25,6 @@
2525
*/
2626

2727
#include <atomic.h>
28-
#include <assert.h>
29-
#include <pthread.h>
30-
31-
/*
32-
* All operations are implemented by serializing them through a global
33-
* pthread mutex. This provides a correct generic implementation.
34-
* However all supported architectures are encouraged to provide a
35-
* native implementation is assembly for performance reasons.
36-
*/
37-
pthread_mutex_t atomic_lock = PTHREAD_MUTEX_INITIALIZER;
3828

3929
/*
4030
* These are the void returning variants
@@ -43,9 +33,7 @@ pthread_mutex_t atomic_lock = PTHREAD_MUTEX_INITIALIZER;
4333
#define ATOMIC_INC(name, type) \
4434
void atomic_inc_##name(volatile type *target) \
4535
{ \
46-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
47-
(*target)++; \
48-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
36+
__atomic_add_fetch(target, 1, __ATOMIC_SEQ_CST); \
4937
}
5038

5139
ATOMIC_INC(8, uint8_t)
@@ -61,9 +49,7 @@ ATOMIC_INC(64, uint64_t)
6149
#define ATOMIC_DEC(name, type) \
6250
void atomic_dec_##name(volatile type *target) \
6351
{ \
64-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
65-
(*target)--; \
66-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
52+
__atomic_sub_fetch(target, 1, __ATOMIC_SEQ_CST); \
6753
}
6854

6955
ATOMIC_DEC(8, uint8_t)
@@ -79,9 +65,7 @@ ATOMIC_DEC(64, uint64_t)
7965
#define ATOMIC_ADD(name, type1, type2) \
8066
void atomic_add_##name(volatile type1 *target, type2 bits) \
8167
{ \
82-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
83-
*target += bits; \
84-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
68+
__atomic_add_fetch(target, bits, __ATOMIC_SEQ_CST); \
8569
}
8670

8771
ATOMIC_ADD(8, uint8_t, int8_t)
@@ -96,18 +80,14 @@ ATOMIC_ADD(64, uint64_t, int64_t)
9680
void
9781
atomic_add_ptr(volatile void *target, ssize_t bits)
9882
{
99-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
100-
*(caddr_t *)target += bits;
101-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
83+
__atomic_add_fetch((void **)target, bits, __ATOMIC_SEQ_CST);
10284
}
10385

10486

10587
#define ATOMIC_SUB(name, type1, type2) \
10688
void atomic_sub_##name(volatile type1 *target, type2 bits) \
10789
{ \
108-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
109-
*target -= bits; \
110-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
90+
__atomic_sub_fetch(target, bits, __ATOMIC_SEQ_CST); \
11191
}
11292

11393
ATOMIC_SUB(8, uint8_t, int8_t)
@@ -122,18 +102,14 @@ ATOMIC_SUB(64, uint64_t, int64_t)
122102
void
123103
atomic_sub_ptr(volatile void *target, ssize_t bits)
124104
{
125-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
126-
*(caddr_t *)target -= bits;
127-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
105+
__atomic_sub_fetch((void **)target, bits, __ATOMIC_SEQ_CST);
128106
}
129107

130108

131109
#define ATOMIC_OR(name, type) \
132110
void atomic_or_##name(volatile type *target, type bits) \
133111
{ \
134-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
135-
*target |= bits; \
136-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
112+
__atomic_or_fetch(target, bits, __ATOMIC_SEQ_CST); \
137113
}
138114

139115
ATOMIC_OR(8, uint8_t)
@@ -149,9 +125,7 @@ ATOMIC_OR(64, uint64_t)
149125
#define ATOMIC_AND(name, type) \
150126
void atomic_and_##name(volatile type *target, type bits) \
151127
{ \
152-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
153-
*target &= bits; \
154-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
128+
__atomic_and_fetch(target, bits, __ATOMIC_SEQ_CST); \
155129
}
156130

157131
ATOMIC_AND(8, uint8_t)
@@ -171,11 +145,7 @@ ATOMIC_AND(64, uint64_t)
171145
#define ATOMIC_INC_NV(name, type) \
172146
type atomic_inc_##name##_nv(volatile type *target) \
173147
{ \
174-
type rc; \
175-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
176-
rc = (++(*target)); \
177-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
178-
return (rc); \
148+
return (__atomic_add_fetch(target, 1, __ATOMIC_SEQ_CST)); \
179149
}
180150

181151
ATOMIC_INC_NV(8, uint8_t)
@@ -191,11 +161,7 @@ ATOMIC_INC_NV(64, uint64_t)
191161
#define ATOMIC_DEC_NV(name, type) \
192162
type atomic_dec_##name##_nv(volatile type *target) \
193163
{ \
194-
type rc; \
195-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
196-
rc = (--(*target)); \
197-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
198-
return (rc); \
164+
return (__atomic_sub_fetch(target, 1, __ATOMIC_SEQ_CST)); \
199165
}
200166

201167
ATOMIC_DEC_NV(8, uint8_t)
@@ -209,13 +175,9 @@ ATOMIC_DEC_NV(64, uint64_t)
209175

210176

211177
#define ATOMIC_ADD_NV(name, type1, type2) \
212-
type1 atomic_add_##name##_nv(volatile type1 *target, type2 bits)\
178+
type1 atomic_add_##name##_nv(volatile type1 *target, type2 bits) \
213179
{ \
214-
type1 rc; \
215-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
216-
rc = (*target += bits); \
217-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
218-
return (rc); \
180+
return (__atomic_add_fetch(target, bits, __ATOMIC_SEQ_CST)); \
219181
}
220182

221183
ATOMIC_ADD_NV(8, uint8_t, int8_t)
@@ -230,24 +192,14 @@ ATOMIC_ADD_NV(64, uint64_t, int64_t)
230192
void *
231193
atomic_add_ptr_nv(volatile void *target, ssize_t bits)
232194
{
233-
void *ptr;
234-
235-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
236-
ptr = (*(caddr_t *)target += bits);
237-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
238-
239-
return (ptr);
195+
return __atomic_add_fetch((void **)target, bits, __ATOMIC_SEQ_CST);
240196
}
241197

242198

243199
#define ATOMIC_SUB_NV(name, type1, type2) \
244-
type1 atomic_sub_##name##_nv(volatile type1 *target, type2 bits)\
200+
type1 atomic_sub_##name##_nv(volatile type1 *target, type2 bits) \
245201
{ \
246-
type1 rc; \
247-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
248-
rc = (*target -= bits); \
249-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
250-
return (rc); \
202+
return (__atomic_sub_fetch(target, bits, __ATOMIC_SEQ_CST)); \
251203
}
252204

253205
ATOMIC_SUB_NV(8, uint8_t, int8_t)
@@ -262,24 +214,14 @@ ATOMIC_SUB_NV(64, uint64_t, int64_t)
262214
void *
263215
atomic_sub_ptr_nv(volatile void *target, ssize_t bits)
264216
{
265-
void *ptr;
266-
267-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
268-
ptr = (*(caddr_t *)target -= bits);
269-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
270-
271-
return (ptr);
217+
return __atomic_sub_fetch((void **)target, bits, __ATOMIC_SEQ_CST);
272218
}
273219

274220

275221
#define ATOMIC_OR_NV(name, type) \
276222
type atomic_or_##name##_nv(volatile type *target, type bits) \
277223
{ \
278-
type rc; \
279-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
280-
rc = (*target |= bits); \
281-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
282-
return (rc); \
224+
return (__atomic_or_fetch(target, bits, __ATOMIC_SEQ_CST)); \
283225
}
284226

285227
ATOMIC_OR_NV(8, uint8_t)
@@ -295,11 +237,7 @@ ATOMIC_OR_NV(64, uint64_t)
295237
#define ATOMIC_AND_NV(name, type) \
296238
type atomic_and_##name##_nv(volatile type *target, type bits) \
297239
{ \
298-
type rc; \
299-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
300-
rc = (*target &= bits); \
301-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
302-
return (rc); \
240+
return (__atomic_and_fetch(target, bits, __ATOMIC_SEQ_CST)); \
303241
}
304242

305243
ATOMIC_AND_NV(8, uint8_t)
@@ -313,19 +251,15 @@ ATOMIC_AND_NV(64, uint64_t)
313251

314252

315253
/*
316-
* If *arg1 == arg2, set *arg1 = arg3; return old value
254+
* If *tgt == exp, set *tgt = des; return old value
317255
*/
318256

319257
#define ATOMIC_CAS(name, type) \
320-
type atomic_cas_##name(volatile type *target, type arg1, type arg2) \
258+
type atomic_cas_##name(volatile type *target, type exp, type des) \
321259
{ \
322-
type old; \
323-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
324-
old = *target; \
325-
if (old == arg1) \
326-
*target = arg2; \
327-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
328-
return (old); \
260+
__atomic_compare_exchange_n(target, &exp, des, B_FALSE, \
261+
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); \
262+
return (exp); \
329263
}
330264

331265
ATOMIC_CAS(8, uint8_t)
@@ -338,17 +272,12 @@ ATOMIC_CAS(ulong, ulong_t)
338272
ATOMIC_CAS(64, uint64_t)
339273

340274
void *
341-
atomic_cas_ptr(volatile void *target, void *arg1, void *arg2)
275+
atomic_cas_ptr(volatile void *target, void *exp, void *des)
342276
{
343-
void *old;
344-
345-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
346-
old = *(void **)target;
347-
if (old == arg1)
348-
*(void **)target = arg2;
349-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
350277

351-
return (old);
278+
__atomic_compare_exchange_n((void **)target, &exp, des, B_FALSE,
279+
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
280+
return (exp);
352281
}
353282

354283

@@ -359,12 +288,7 @@ atomic_cas_ptr(volatile void *target, void *arg1, void *arg2)
359288
#define ATOMIC_SWAP(name, type) \
360289
type atomic_swap_##name(volatile type *target, type bits) \
361290
{ \
362-
type old; \
363-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
364-
old = *target; \
365-
*target = bits; \
366-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
367-
return (old); \
291+
return (__atomic_exchange_n(target, bits, __ATOMIC_SEQ_CST)); \
368292
}
369293

370294
ATOMIC_SWAP(8, uint8_t)
@@ -380,71 +304,46 @@ ATOMIC_SWAP(64, uint64_t)
380304
void *
381305
atomic_swap_ptr(volatile void *target, void *bits)
382306
{
383-
void *old;
384-
385-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
386-
old = *(void **)target;
387-
*(void **)target = bits;
388-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
389-
390-
return (old);
307+
return (__atomic_exchange_n((void **)target, bits, __ATOMIC_SEQ_CST));
391308
}
392309

393310

394311
int
395312
atomic_set_long_excl(volatile ulong_t *target, uint_t value)
396313
{
397-
ulong_t bit;
398-
399-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
400-
bit = (1UL << value);
401-
if ((*target & bit) != 0) {
402-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
403-
return (-1);
404-
}
405-
*target |= bit;
406-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
407-
408-
return (0);
314+
ulong_t bit = 1UL << value;
315+
ulong_t old = __atomic_fetch_or(target, bit, __ATOMIC_SEQ_CST);
316+
return (old & bit) ? -1 : 0;
409317
}
410318

411319
int
412320
atomic_clear_long_excl(volatile ulong_t *target, uint_t value)
413321
{
414-
ulong_t bit;
415-
416-
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
417-
bit = (1UL << value);
418-
if ((*target & bit) == 0) {
419-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
420-
return (-1);
421-
}
422-
*target &= ~bit;
423-
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
424-
425-
return (0);
322+
ulong_t bit = 1UL << value;
323+
ulong_t old = __atomic_fetch_and(target, ~bit, __ATOMIC_SEQ_CST);
324+
return ((old & bit) ? 0 : -1);
426325
}
427326

428327
void
429328
membar_enter(void)
430329
{
431-
/* XXX - Implement me */
330+
__atomic_thread_fence(__ATOMIC_SEQ_CST);
432331
}
433332

434333
void
435334
membar_exit(void)
436335
{
437-
/* XXX - Implement me */
336+
__atomic_thread_fence(__ATOMIC_SEQ_CST);
438337
}
439338

440339
void
441340
membar_producer(void)
442341
{
443-
/* XXX - Implement me */
342+
__atomic_thread_fence(__ATOMIC_RELEASE);
444343
}
445344

446345
void
447346
membar_consumer(void)
448347
{
449-
/* XXX - Implement me */
348+
__atomic_thread_fence(__ATOMIC_ACQUIRE);
450349
}

0 commit comments

Comments
 (0)