atomic.h 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536
  1. /* Internal macros for atomic operations for GNU C Library.
  2. Copyright (C) 2002-2006, 2009 Free Software Foundation, Inc.
  3. This file is part of the GNU C Library.
  4. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
  5. The GNU C Library is free software; you can redistribute it and/or
  6. modify it under the terms of the GNU Lesser General Public
  7. License as published by the Free Software Foundation; either
  8. version 2.1 of the License, or (at your option) any later version.
  9. The GNU C Library is distributed in the hope that it will be useful,
  10. but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  12. Lesser General Public License for more details.
  13. You should have received a copy of the GNU Lesser General Public
  14. License along with the GNU C Library; if not, see
  15. <http://www.gnu.org/licenses/>. */
  16. #ifndef _ATOMIC_H
  17. #define _ATOMIC_H 1
  18. /* This header defines three types of macros:
  19. - atomic arithmetic and logic operation on memory. They all
  20. have the prefix "atomic_".
  21. - conditionally atomic operations of the same kinds. These
  22. always behave identical but can be faster when atomicity
  23. is not really needed since only one thread has access to
  24. the memory location. In that case the code is slower in
  25. the multi-thread case. The interfaces have the prefix
  26. "catomic_".
  27. - support functions like barriers. They also have the preifx
  28. "atomic_".
  29. Architectures must provide a few lowlevel macros (the compare
  30. and exchange definitions). All others are optional. They
  31. should only be provided if the architecture has specific
  32. support for the operation.
  33. As <atomic.h> macros are usually heavily nested and often use local
  34. variables to make sure side-effects are evaluated properly, use for
  35. macro local variables a per-macro unique prefix. This file uses
  36. __atgN_ prefix where N is different in each macro. */
  37. #include <stdlib.h>
  38. #include <bits/atomic.h>
  39. /* Wrapper macros to call pre_NN_post (mem, ...) where NN is the
  40. bit width of *MEM. The calling macro puts parens around MEM
  41. and following args. */
  42. #define __atomic_val_bysize(pre, post, mem, ...) \
  43. ({ \
  44. __typeof (*mem) __atg1_result; \
  45. if (sizeof (*mem) == 1) \
  46. __atg1_result = pre##_8_##post (mem, __VA_ARGS__); \
  47. else if (sizeof (*mem) == 2) \
  48. __atg1_result = pre##_16_##post (mem, __VA_ARGS__); \
  49. else if (sizeof (*mem) == 4) \
  50. __atg1_result = pre##_32_##post (mem, __VA_ARGS__); \
  51. else if (sizeof (*mem) == 8) \
  52. __atg1_result = pre##_64_##post (mem, __VA_ARGS__); \
  53. else \
  54. abort (); \
  55. __atg1_result; \
  56. })
  57. #define __atomic_bool_bysize(pre, post, mem, ...) \
  58. ({ \
  59. int __atg2_result; \
  60. if (sizeof (*mem) == 1) \
  61. __atg2_result = pre##_8_##post (mem, __VA_ARGS__); \
  62. else if (sizeof (*mem) == 2) \
  63. __atg2_result = pre##_16_##post (mem, __VA_ARGS__); \
  64. else if (sizeof (*mem) == 4) \
  65. __atg2_result = pre##_32_##post (mem, __VA_ARGS__); \
  66. else if (sizeof (*mem) == 8) \
  67. __atg2_result = pre##_64_##post (mem, __VA_ARGS__); \
  68. else \
  69. abort (); \
  70. __atg2_result; \
  71. })
  72. /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
  73. Return the old *MEM value. */
  74. #if !defined atomic_compare_and_exchange_val_acq \
  75. && defined __arch_compare_and_exchange_val_32_acq
  76. # define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  77. __atomic_val_bysize (__arch_compare_and_exchange_val,acq, \
  78. mem, newval, oldval)
  79. #endif
  80. #ifndef catomic_compare_and_exchange_val_acq
  81. # ifdef __arch_c_compare_and_exchange_val_32_acq
  82. # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  83. __atomic_val_bysize (__arch_c_compare_and_exchange_val,acq, \
  84. mem, newval, oldval)
  85. # else
  86. # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  87. atomic_compare_and_exchange_val_acq (mem, newval, oldval)
  88. # endif
  89. #endif
  90. #ifndef catomic_compare_and_exchange_val_rel
  91. # ifndef atomic_compare_and_exchange_val_rel
  92. # define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \
  93. catomic_compare_and_exchange_val_acq (mem, newval, oldval)
  94. # else
  95. # define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \
  96. atomic_compare_and_exchange_val_rel (mem, newval, oldval)
  97. # endif
  98. #endif
  99. #ifndef atomic_compare_and_exchange_val_rel
  100. # define atomic_compare_and_exchange_val_rel(mem, newval, oldval) \
  101. atomic_compare_and_exchange_val_acq (mem, newval, oldval)
  102. #endif
  103. /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
  104. Return zero if *MEM was changed or non-zero if no exchange happened. */
  105. #ifndef atomic_compare_and_exchange_bool_acq
  106. # ifdef __arch_compare_and_exchange_bool_32_acq
  107. # define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
  108. __atomic_bool_bysize (__arch_compare_and_exchange_bool,acq, \
  109. mem, newval, oldval)
  110. # else
  111. # define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
  112. ({ /* Cannot use __oldval here, because macros later in this file might \
  113. call this macro with __oldval argument. */ \
  114. __typeof (oldval) __atg3_old = (oldval); \
  115. atomic_compare_and_exchange_val_acq (mem, newval, __atg3_old) \
  116. != __atg3_old; \
  117. })
  118. # endif
  119. #endif
  120. #ifndef catomic_compare_and_exchange_bool_acq
  121. # ifdef __arch_c_compare_and_exchange_bool_32_acq
  122. # define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
  123. __atomic_bool_bysize (__arch_c_compare_and_exchange_bool,acq, \
  124. mem, newval, oldval)
  125. # else
  126. # define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
  127. ({ /* Cannot use __oldval here, because macros later in this file might \
  128. call this macro with __oldval argument. */ \
  129. __typeof (oldval) __atg4_old = (oldval); \
  130. catomic_compare_and_exchange_val_acq (mem, newval, __atg4_old) \
  131. != __atg4_old; \
  132. })
  133. # endif
  134. #endif
  135. #ifndef catomic_compare_and_exchange_bool_rel
  136. # ifndef atomic_compare_and_exchange_bool_rel
  137. # define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
  138. catomic_compare_and_exchange_bool_acq (mem, newval, oldval)
  139. # else
  140. # define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
  141. atomic_compare_and_exchange_bool_rel (mem, newval, oldval)
  142. # endif
  143. #endif
  144. #ifndef atomic_compare_and_exchange_bool_rel
  145. # define atomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
  146. atomic_compare_and_exchange_bool_acq (mem, newval, oldval)
  147. #endif
  148. /* Store NEWVALUE in *MEM and return the old value. */
  149. #ifndef atomic_exchange_acq
  150. # define atomic_exchange_acq(mem, newvalue) \
  151. ({ __typeof (*(mem)) __atg5_oldval; \
  152. __typeof (mem) __atg5_memp = (mem); \
  153. __typeof (*(mem)) __atg5_value = (newvalue); \
  154. \
  155. do \
  156. __atg5_oldval = *__atg5_memp; \
  157. while (__builtin_expect \
  158. (atomic_compare_and_exchange_bool_acq (__atg5_memp, __atg5_value, \
  159. __atg5_oldval), 0)); \
  160. \
  161. __atg5_oldval; })
  162. #endif
  163. #ifndef atomic_exchange_rel
  164. # define atomic_exchange_rel(mem, newvalue) atomic_exchange_acq (mem, newvalue)
  165. #endif
  166. /* Add VALUE to *MEM and return the old value of *MEM. */
  167. #ifndef atomic_exchange_and_add
  168. # define atomic_exchange_and_add(mem, value) \
  169. ({ __typeof (*(mem)) __atg6_oldval; \
  170. __typeof (mem) __atg6_memp = (mem); \
  171. __typeof (*(mem)) __atg6_value = (value); \
  172. \
  173. do \
  174. __atg6_oldval = *__atg6_memp; \
  175. while (__builtin_expect \
  176. (atomic_compare_and_exchange_bool_acq (__atg6_memp, \
  177. __atg6_oldval \
  178. + __atg6_value, \
  179. __atg6_oldval), 0)); \
  180. \
  181. __atg6_oldval; })
  182. #endif
  183. #ifndef catomic_exchange_and_add
  184. # define catomic_exchange_and_add(mem, value) \
  185. ({ __typeof (*(mem)) __atg7_oldv; \
  186. __typeof (mem) __atg7_memp = (mem); \
  187. __typeof (*(mem)) __atg7_value = (value); \
  188. \
  189. do \
  190. __atg7_oldv = *__atg7_memp; \
  191. while (__builtin_expect \
  192. (catomic_compare_and_exchange_bool_acq (__atg7_memp, \
  193. __atg7_oldv \
  194. + __atg7_value, \
  195. __atg7_oldv), 0)); \
  196. \
  197. __atg7_oldv; })
  198. #endif
  199. #ifndef atomic_max
  200. # define atomic_max(mem, value) \
  201. do { \
  202. __typeof (*(mem)) __atg8_oldval; \
  203. __typeof (mem) __atg8_memp = (mem); \
  204. __typeof (*(mem)) __atg8_value = (value); \
  205. do { \
  206. __atg8_oldval = *__atg8_memp; \
  207. if (__atg8_oldval >= __atg8_value) \
  208. break; \
  209. } while (__builtin_expect \
  210. (atomic_compare_and_exchange_bool_acq (__atg8_memp, __atg8_value,\
  211. __atg8_oldval), 0)); \
  212. } while (0)
  213. #endif
  214. #ifndef catomic_max
  215. # define catomic_max(mem, value) \
  216. do { \
  217. __typeof (*(mem)) __atg9_oldv; \
  218. __typeof (mem) __atg9_memp = (mem); \
  219. __typeof (*(mem)) __atg9_value = (value); \
  220. do { \
  221. __atg9_oldv = *__atg9_memp; \
  222. if (__atg9_oldv >= __atg9_value) \
  223. break; \
  224. } while (__builtin_expect \
  225. (catomic_compare_and_exchange_bool_acq (__atg9_memp, \
  226. __atg9_value, \
  227. __atg9_oldv), 0)); \
  228. } while (0)
  229. #endif
  230. #ifndef atomic_min
  231. # define atomic_min(mem, value) \
  232. do { \
  233. __typeof (*(mem)) __atg10_oldval; \
  234. __typeof (mem) __atg10_memp = (mem); \
  235. __typeof (*(mem)) __atg10_value = (value); \
  236. do { \
  237. __atg10_oldval = *__atg10_memp; \
  238. if (__atg10_oldval <= __atg10_value) \
  239. break; \
  240. } while (__builtin_expect \
  241. (atomic_compare_and_exchange_bool_acq (__atg10_memp, \
  242. __atg10_value, \
  243. __atg10_oldval), 0)); \
  244. } while (0)
  245. #endif
  246. #ifndef atomic_add
  247. # define atomic_add(mem, value) (void) atomic_exchange_and_add ((mem), (value))
  248. #endif
  249. #ifndef catomic_add
  250. # define catomic_add(mem, value) \
  251. (void) catomic_exchange_and_add ((mem), (value))
  252. #endif
  253. #ifndef atomic_increment
  254. # define atomic_increment(mem) atomic_add ((mem), 1)
  255. #endif
  256. #ifndef catomic_increment
  257. # define catomic_increment(mem) catomic_add ((mem), 1)
  258. #endif
  259. #ifndef atomic_increment_val
  260. # define atomic_increment_val(mem) (atomic_exchange_and_add ((mem), 1) + 1)
  261. #endif
  262. #ifndef catomic_increment_val
  263. # define catomic_increment_val(mem) (catomic_exchange_and_add ((mem), 1) + 1)
  264. #endif
  265. /* Add one to *MEM and return true iff it's now zero. */
  266. #ifndef atomic_increment_and_test
  267. # define atomic_increment_and_test(mem) \
  268. (atomic_exchange_and_add ((mem), 1) + 1 == 0)
  269. #endif
  270. #ifndef atomic_decrement
  271. # define atomic_decrement(mem) atomic_add ((mem), -1)
  272. #endif
  273. #ifndef catomic_decrement
  274. # define catomic_decrement(mem) catomic_add ((mem), -1)
  275. #endif
  276. #ifndef atomic_decrement_val
  277. # define atomic_decrement_val(mem) (atomic_exchange_and_add ((mem), -1) - 1)
  278. #endif
  279. #ifndef catomic_decrement_val
  280. # define catomic_decrement_val(mem) (catomic_exchange_and_add ((mem), -1) - 1)
  281. #endif
  282. /* Subtract 1 from *MEM and return true iff it's now zero. */
  283. #ifndef atomic_decrement_and_test
  284. # define atomic_decrement_and_test(mem) \
  285. (atomic_exchange_and_add ((mem), -1) == 1)
  286. #endif
  287. /* Decrement *MEM if it is > 0, and return the old value. */
  288. #ifndef atomic_decrement_if_positive
  289. # define atomic_decrement_if_positive(mem) \
  290. ({ __typeof (*(mem)) __atg11_oldval; \
  291. __typeof (mem) __atg11_memp = (mem); \
  292. \
  293. do \
  294. { \
  295. __atg11_oldval = *__atg11_memp; \
  296. if (__builtin_expect (__atg11_oldval <= 0, 0)) \
  297. break; \
  298. } \
  299. while (__builtin_expect \
  300. (atomic_compare_and_exchange_bool_acq (__atg11_memp, \
  301. __atg11_oldval - 1, \
  302. __atg11_oldval), 0)); \
  303. __atg11_oldval; })
  304. #endif
  305. #ifndef atomic_add_negative
  306. # define atomic_add_negative(mem, value) \
  307. ({ __typeof (value) __atg12_value = (value); \
  308. atomic_exchange_and_add (mem, __atg12_value) < -__atg12_value; })
  309. #endif
  310. #ifndef atomic_add_zero
  311. # define atomic_add_zero(mem, value) \
  312. ({ __typeof (value) __atg13_value = (value); \
  313. atomic_exchange_and_add (mem, __atg13_value) == -__atg13_value; })
  314. #endif
  315. #ifndef atomic_bit_set
  316. # define atomic_bit_set(mem, bit) \
  317. (void) atomic_bit_test_set(mem, bit)
  318. #endif
  319. #ifndef atomic_bit_test_set
  320. # define atomic_bit_test_set(mem, bit) \
  321. ({ __typeof (*(mem)) __atg14_old; \
  322. __typeof (mem) __atg14_memp = (mem); \
  323. __typeof (*(mem)) __atg14_mask = ((__typeof (*(mem))) 1 << (bit)); \
  324. \
  325. do \
  326. __atg14_old = (*__atg14_memp); \
  327. while (__builtin_expect \
  328. (atomic_compare_and_exchange_bool_acq (__atg14_memp, \
  329. __atg14_old | __atg14_mask,\
  330. __atg14_old), 0)); \
  331. \
  332. __atg14_old & __atg14_mask; })
  333. #endif
  334. /* Atomically *mem &= mask. */
  335. #ifndef atomic_and
  336. # define atomic_and(mem, mask) \
  337. do { \
  338. __typeof (*(mem)) __atg15_old; \
  339. __typeof (mem) __atg15_memp = (mem); \
  340. __typeof (*(mem)) __atg15_mask = (mask); \
  341. \
  342. do \
  343. __atg15_old = (*__atg15_memp); \
  344. while (__builtin_expect \
  345. (atomic_compare_and_exchange_bool_acq (__atg15_memp, \
  346. __atg15_old & __atg15_mask, \
  347. __atg15_old), 0)); \
  348. } while (0)
  349. #endif
  350. #ifndef catomic_and
  351. # define catomic_and(mem, mask) \
  352. do { \
  353. __typeof (*(mem)) __atg20_old; \
  354. __typeof (mem) __atg20_memp = (mem); \
  355. __typeof (*(mem)) __atg20_mask = (mask); \
  356. \
  357. do \
  358. __atg20_old = (*__atg20_memp); \
  359. while (__builtin_expect \
  360. (catomic_compare_and_exchange_bool_acq (__atg20_memp, \
  361. __atg20_old & __atg20_mask,\
  362. __atg20_old), 0)); \
  363. } while (0)
  364. #endif
  365. /* Atomically *mem &= mask and return the old value of *mem. */
  366. #ifndef atomic_and_val
  367. # define atomic_and_val(mem, mask) \
  368. ({ __typeof (*(mem)) __atg16_old; \
  369. __typeof (mem) __atg16_memp = (mem); \
  370. __typeof (*(mem)) __atg16_mask = (mask); \
  371. \
  372. do \
  373. __atg16_old = (*__atg16_memp); \
  374. while (__builtin_expect \
  375. (atomic_compare_and_exchange_bool_acq (__atg16_memp, \
  376. __atg16_old & __atg16_mask,\
  377. __atg16_old), 0)); \
  378. \
  379. __atg16_old; })
  380. #endif
  381. /* Atomically *mem |= mask and return the old value of *mem. */
  382. #ifndef atomic_or
  383. # define atomic_or(mem, mask) \
  384. do { \
  385. __typeof (*(mem)) __atg17_old; \
  386. __typeof (mem) __atg17_memp = (mem); \
  387. __typeof (*(mem)) __atg17_mask = (mask); \
  388. \
  389. do \
  390. __atg17_old = (*__atg17_memp); \
  391. while (__builtin_expect \
  392. (atomic_compare_and_exchange_bool_acq (__atg17_memp, \
  393. __atg17_old | __atg17_mask, \
  394. __atg17_old), 0)); \
  395. } while (0)
  396. #endif
  397. #ifndef catomic_or
  398. # define catomic_or(mem, mask) \
  399. do { \
  400. __typeof (*(mem)) __atg18_old; \
  401. __typeof (mem) __atg18_memp = (mem); \
  402. __typeof (*(mem)) __atg18_mask = (mask); \
  403. \
  404. do \
  405. __atg18_old = (*__atg18_memp); \
  406. while (__builtin_expect \
  407. (catomic_compare_and_exchange_bool_acq (__atg18_memp, \
  408. __atg18_old | __atg18_mask,\
  409. __atg18_old), 0)); \
  410. } while (0)
  411. #endif
  412. /* Atomically *mem |= mask and return the old value of *mem. */
  413. #ifndef atomic_or_val
  414. # define atomic_or_val(mem, mask) \
  415. ({ __typeof (*(mem)) __atg19_old; \
  416. __typeof (mem) __atg19_memp = (mem); \
  417. __typeof (*(mem)) __atg19_mask = (mask); \
  418. \
  419. do \
  420. __atg19_old = (*__atg19_memp); \
  421. while (__builtin_expect \
  422. (atomic_compare_and_exchange_bool_acq (__atg19_memp, \
  423. __atg19_old | __atg19_mask,\
  424. __atg19_old), 0)); \
  425. \
  426. __atg19_old; })
  427. #endif
  428. #ifndef atomic_full_barrier
  429. # define atomic_full_barrier() __asm__ ("" ::: "memory")
  430. #endif
  431. #ifndef atomic_read_barrier
  432. # define atomic_read_barrier() atomic_full_barrier ()
  433. #endif
  434. #ifndef atomic_write_barrier
  435. # define atomic_write_barrier() atomic_full_barrier ()
  436. #endif
  437. #ifndef atomic_forced_read
  438. # define atomic_forced_read(x) \
  439. ({ __typeof (x) __x; __asm__ ("" : "=r" (__x) : "0" (x)); __x; })
  440. #endif
  441. #ifndef atomic_delay
  442. # define atomic_delay() do { /* nothing */ } while (0)
  443. #endif
  444. #endif /* atomic.h */