memset.S 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400
  1. /* Optimized version of the standard memset() function.
  2. This file is part of the GNU C Library.
  3. Copyright (C) 2000, 2001, 2002, 2003 Free Software Foundation, Inc.
  4. Contributed by Dan Pop for Itanium <Dan.Pop@cern.ch>.
  5. Rewritten for McKinley by Sverre Jarp, HP Labs/CERN <Sverre.Jarp@cern.ch>
  6. The GNU C Library is free software; you can redistribute it and/or
  7. modify it under the terms of the GNU Lesser General Public
  8. License as published by the Free Software Foundation; either
  9. version 2.1 of the License, or (at your option) any later version.
  10. The GNU C Library is distributed in the hope that it will be useful,
  11. but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  13. Lesser General Public License for more details.
  14. You should have received a copy of the GNU Lesser General Public
  15. License along with the GNU C Library; if not, write to the Free
  16. Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  17. 02111-1307 USA. */
  18. /* Return: dest
  19. Inputs:
  20. in0: dest
  21. in1: value
  22. in2: count
  23. The algorithm is fairly straightforward: set byte by byte until we
  24. we get to a 16B-aligned address, then loop on 128 B chunks using an
  25. early store as prefetching, then loop on 32B chucks, then clear remaining
  26. words, finally clear remaining bytes.
  27. Since a stf.spill f0 can store 16B in one go, we use this instruction
  28. to get peak speed when value = 0. */
  29. #include "sysdep.h"
  30. #undef ret
  31. #define dest in0
  32. #define value in1
  33. #define cnt in2
  34. #define tmp r31
  35. #define save_lc r30
  36. #define ptr0 r29
  37. #define ptr1 r28
  38. #define ptr2 r27
  39. #define ptr3 r26
  40. #define ptr9 r24
  41. #define loopcnt r23
  42. #define linecnt r22
  43. #define bytecnt r21
  44. #define fvalue f6
  45. // This routine uses only scratch predicate registers (p6 - p15)
  46. #define p_scr p6 // default register for same-cycle branches
  47. #define p_nz p7
  48. #define p_zr p8
  49. #define p_unalgn p9
  50. #define p_y p11
  51. #define p_n p12
  52. #define p_yy p13
  53. #define p_nn p14
  54. #define movi0 mov
  55. #define MIN1 15
  56. #define MIN1P1HALF 8
  57. #define LINE_SIZE 128
  58. #define LSIZE_SH 7 // shift amount
  59. #define PREF_AHEAD 8
  60. #define USE_FLP
  61. #if defined(USE_INT)
  62. #define store st8
  63. #define myval value
  64. #elif defined(USE_FLP)
  65. #define store stf8
  66. #define myval fvalue
  67. #endif
  68. .align 64
  69. ENTRY(memset)
  70. { .mmi
  71. .prologue
  72. alloc tmp = ar.pfs, 3, 0, 0, 0
  73. lfetch.nt1 [dest]
  74. .save ar.lc, save_lc
  75. movi0 save_lc = ar.lc
  76. } { .mmi
  77. .body
  78. mov ret0 = dest // return value
  79. cmp.ne p_nz, p_zr = value, r0 // use stf.spill if value is zero
  80. cmp.eq p_scr, p0 = cnt, r0
  81. ;; }
  82. { .mmi
  83. and ptr2 = -(MIN1+1), dest // aligned address
  84. and tmp = MIN1, dest // prepare to check for alignment
  85. tbit.nz p_y, p_n = dest, 0 // Do we have an odd address? (M_B_U)
  86. } { .mib
  87. mov ptr1 = dest
  88. mux1 value = value, @brcst // create 8 identical bytes in word
  89. (p_scr) br.ret.dpnt.many rp // return immediately if count = 0
  90. ;; }
  91. { .mib
  92. cmp.ne p_unalgn, p0 = tmp, r0
  93. } { .mib // NB: # of bytes to move is 1 higher
  94. sub bytecnt = (MIN1+1), tmp // than loopcnt
  95. cmp.gt p_scr, p0 = 16, cnt // is it a minimalistic task?
  96. (p_scr) br.cond.dptk.many .move_bytes_unaligned // go move just a few (M_B_U)
  97. ;; }
  98. { .mmi
  99. (p_unalgn) add ptr1 = (MIN1+1), ptr2 // after alignment
  100. (p_unalgn) add ptr2 = MIN1P1HALF, ptr2 // after alignment
  101. (p_unalgn) tbit.nz.unc p_y, p_n = bytecnt, 3 // should we do a st8 ?
  102. ;; }
  103. { .mib
  104. (p_y) add cnt = -8, cnt
  105. (p_unalgn) tbit.nz.unc p_yy, p_nn = bytecnt, 2 // should we do a st4 ?
  106. } { .mib
  107. (p_y) st8 [ptr2] = value, -4
  108. (p_n) add ptr2 = 4, ptr2
  109. ;; }
  110. { .mib
  111. (p_yy) add cnt = -4, cnt
  112. (p_unalgn) tbit.nz.unc p_y, p_n = bytecnt, 1 // should we do a st2 ?
  113. } { .mib
  114. (p_yy) st4 [ptr2] = value, -2
  115. (p_nn) add ptr2 = 2, ptr2
  116. ;; }
  117. { .mmi
  118. mov tmp = LINE_SIZE+1 // for compare
  119. (p_y) add cnt = -2, cnt
  120. (p_unalgn) tbit.nz.unc p_yy, p_nn = bytecnt, 0 // should we do a st1 ?
  121. } { .mmi
  122. setf.sig fvalue=value // transfer value to FLP side
  123. (p_y) st2 [ptr2] = value, -1
  124. (p_n) add ptr2 = 1, ptr2
  125. ;; }
  126. { .mmi
  127. (p_yy) st1 [ptr2] = value
  128. cmp.gt p_scr, p0 = tmp, cnt // is it a minimalistic task?
  129. } { .mbb
  130. (p_yy) add cnt = -1, cnt
  131. (p_scr) br.cond.dpnt.many .fraction_of_line // go move just a few
  132. ;; }
  133. { .mib
  134. nop.m 0
  135. shr.u linecnt = cnt, LSIZE_SH
  136. (p_zr) br.cond.dptk.many .l1b // Jump to use stf.spill
  137. ;; }
  138. #ifndef GAS_ALIGN_BREAKS_UNWIND_INFO
  139. .align 32 // -------- // L1A: store ahead into cache lines; fill later
  140. #endif
  141. { .mmi
  142. and tmp = -(LINE_SIZE), cnt // compute end of range
  143. mov ptr9 = ptr1 // used for prefetching
  144. and cnt = (LINE_SIZE-1), cnt // remainder
  145. } { .mmi
  146. mov loopcnt = PREF_AHEAD-1 // default prefetch loop
  147. cmp.gt p_scr, p0 = PREF_AHEAD, linecnt // check against actual value
  148. ;; }
  149. { .mmi
  150. (p_scr) add loopcnt = -1, linecnt // start of stores
  151. add ptr2 = 8, ptr1 // (beyond prefetch stores)
  152. add ptr1 = tmp, ptr1 // first address beyond total
  153. ;; } // range
  154. { .mmi
  155. add tmp = -1, linecnt // next loop count
  156. movi0 ar.lc = loopcnt
  157. ;; }
  158. .pref_l1a:
  159. { .mib
  160. store [ptr9] = myval, 128 // Do stores one cache line apart
  161. nop.i 0
  162. br.cloop.dptk.few .pref_l1a
  163. ;; }
  164. { .mmi
  165. add ptr0 = 16, ptr2 // Two stores in parallel
  166. movi0 ar.lc = tmp
  167. ;; }
  168. .l1ax:
  169. { .mmi
  170. store [ptr2] = myval, 8
  171. store [ptr0] = myval, 8
  172. ;; }
  173. { .mmi
  174. store [ptr2] = myval, 24
  175. store [ptr0] = myval, 24
  176. ;; }
  177. { .mmi
  178. store [ptr2] = myval, 8
  179. store [ptr0] = myval, 8
  180. ;; }
  181. { .mmi
  182. store [ptr2] = myval, 24
  183. store [ptr0] = myval, 24
  184. ;; }
  185. { .mmi
  186. store [ptr2] = myval, 8
  187. store [ptr0] = myval, 8
  188. ;; }
  189. { .mmi
  190. store [ptr2] = myval, 24
  191. store [ptr0] = myval, 24
  192. ;; }
  193. { .mmi
  194. store [ptr2] = myval, 8
  195. store [ptr0] = myval, 32
  196. cmp.lt p_scr, p0 = ptr9, ptr1 // do we need more prefetching?
  197. ;; }
  198. { .mmb
  199. store [ptr2] = myval, 24
  200. (p_scr) store [ptr9] = myval, 128
  201. br.cloop.dptk.few .l1ax
  202. ;; }
  203. { .mbb
  204. cmp.le p_scr, p0 = 8, cnt // just a few bytes left ?
  205. (p_scr) br.cond.dpnt.many .fraction_of_line // Branch no. 2
  206. br.cond.dpnt.many .move_bytes_from_alignment // Branch no. 3
  207. ;; }
  208. #ifdef GAS_ALIGN_BREAKS_UNWIND_INFO
  209. { nop 0 }
  210. #else
  211. .align 32
  212. #endif
  213. .l1b: // ------------------ // L1B: store ahead into cache lines; fill later
  214. { .mmi
  215. and tmp = -(LINE_SIZE), cnt // compute end of range
  216. mov ptr9 = ptr1 // used for prefetching
  217. and cnt = (LINE_SIZE-1), cnt // remainder
  218. } { .mmi
  219. mov loopcnt = PREF_AHEAD-1 // default prefetch loop
  220. cmp.gt p_scr, p0 = PREF_AHEAD, linecnt // check against actual value
  221. ;; }
  222. { .mmi
  223. (p_scr) add loopcnt = -1, linecnt
  224. add ptr2 = 16, ptr1 // start of stores (beyond prefetch stores)
  225. add ptr1 = tmp, ptr1 // first address beyond total range
  226. ;; }
  227. { .mmi
  228. add tmp = -1, linecnt // next loop count
  229. movi0 ar.lc = loopcnt
  230. ;; }
  231. .pref_l1b:
  232. { .mib
  233. stf.spill [ptr9] = f0, 128 // Do stores one cache line apart
  234. nop.i 0
  235. br.cloop.dptk.few .pref_l1b
  236. ;; }
  237. { .mmi
  238. add ptr0 = 16, ptr2 // Two stores in parallel
  239. movi0 ar.lc = tmp
  240. ;; }
  241. .l1bx:
  242. { .mmi
  243. stf.spill [ptr2] = f0, 32
  244. stf.spill [ptr0] = f0, 32
  245. ;; }
  246. { .mmi
  247. stf.spill [ptr2] = f0, 32
  248. stf.spill [ptr0] = f0, 32
  249. ;; }
  250. { .mmi
  251. stf.spill [ptr2] = f0, 32
  252. stf.spill [ptr0] = f0, 64
  253. cmp.lt p_scr, p0 = ptr9, ptr1 // do we need more prefetching?
  254. ;; }
  255. { .mmb
  256. stf.spill [ptr2] = f0, 32
  257. (p_scr) stf.spill [ptr9] = f0, 128
  258. br.cloop.dptk.few .l1bx
  259. ;; }
  260. { .mib
  261. cmp.gt p_scr, p0 = 8, cnt // just a few bytes left ?
  262. (p_scr) br.cond.dpnt.many .move_bytes_from_alignment
  263. ;; }
  264. .fraction_of_line:
  265. { .mib
  266. add ptr2 = 16, ptr1
  267. shr.u loopcnt = cnt, 5 // loopcnt = cnt / 32
  268. ;; }
  269. { .mib
  270. cmp.eq p_scr, p0 = loopcnt, r0
  271. add loopcnt = -1, loopcnt
  272. (p_scr) br.cond.dpnt.many store_words
  273. ;; }
  274. { .mib
  275. and cnt = 0x1f, cnt // compute the remaining cnt
  276. movi0 ar.lc = loopcnt
  277. ;; }
  278. #ifndef GAS_ALIGN_BREAKS_UNWIND_INFO
  279. .align 32
  280. #endif
  281. .l2: // ---------------------------- // L2A: store 32B in 2 cycles
  282. { .mmb
  283. store [ptr1] = myval, 8
  284. store [ptr2] = myval, 8
  285. ;; } { .mmb
  286. store [ptr1] = myval, 24
  287. store [ptr2] = myval, 24
  288. br.cloop.dptk.many .l2
  289. ;; }
  290. store_words:
  291. { .mib
  292. cmp.gt p_scr, p0 = 8, cnt // just a few bytes left ?
  293. (p_scr) br.cond.dpnt.many .move_bytes_from_alignment // Branch
  294. ;; }
  295. { .mmi
  296. store [ptr1] = myval, 8 // store
  297. cmp.le p_y, p_n = 16, cnt //
  298. add cnt = -8, cnt // subtract
  299. ;; }
  300. { .mmi
  301. (p_y) store [ptr1] = myval, 8 // store
  302. (p_y) cmp.le.unc p_yy, p_nn = 16, cnt //
  303. (p_y) add cnt = -8, cnt // subtract
  304. ;; }
  305. { .mmi // store
  306. (p_yy) store [ptr1] = myval, 8 //
  307. (p_yy) add cnt = -8, cnt // subtract
  308. ;; }
  309. .move_bytes_from_alignment:
  310. { .mib
  311. cmp.eq p_scr, p0 = cnt, r0
  312. tbit.nz.unc p_y, p0 = cnt, 2 // should we terminate with a st4 ?
  313. (p_scr) br.cond.dpnt.few .restore_and_exit
  314. ;; }
  315. { .mib
  316. (p_y) st4 [ptr1] = value, 4
  317. tbit.nz.unc p_yy, p0 = cnt, 1 // should we terminate with a st2 ?
  318. ;; }
  319. { .mib
  320. (p_yy) st2 [ptr1] = value, 2
  321. tbit.nz.unc p_y, p0 = cnt, 0
  322. ;; }
  323. { .mib
  324. (p_y) st1 [ptr1] = value
  325. ;; }
  326. .restore_and_exit:
  327. { .mib
  328. nop.m 0
  329. movi0 ar.lc = save_lc
  330. br.ret.sptk.many rp
  331. ;; }
  332. .move_bytes_unaligned:
  333. { .mmi
  334. .pred.rel "mutex",p_y, p_n
  335. .pred.rel "mutex",p_yy, p_nn
  336. (p_n) cmp.le p_yy, p_nn = 4, cnt
  337. (p_y) cmp.le p_yy, p_nn = 5, cnt
  338. (p_n) add ptr2 = 2, ptr1
  339. } { .mmi
  340. (p_y) add ptr2 = 3, ptr1
  341. (p_y) st1 [ptr1] = value, 1 // fill 1 (odd-aligned) byte
  342. (p_y) add cnt = -1, cnt // [15, 14 (or less) left]
  343. ;; }
  344. { .mmi
  345. (p_yy) cmp.le.unc p_y, p0 = 8, cnt
  346. add ptr3 = ptr1, cnt // prepare last store
  347. movi0 ar.lc = save_lc
  348. } { .mmi
  349. (p_yy) st2 [ptr1] = value, 4 // fill 2 (aligned) bytes
  350. (p_yy) st2 [ptr2] = value, 4 // fill 2 (aligned) bytes
  351. (p_yy) add cnt = -4, cnt // [11, 10 (o less) left]
  352. ;; }
  353. { .mmi
  354. (p_y) cmp.le.unc p_yy, p0 = 8, cnt
  355. add ptr3 = -1, ptr3 // last store
  356. tbit.nz p_scr, p0 = cnt, 1 // will there be a st2 at the end ?
  357. } { .mmi
  358. (p_y) st2 [ptr1] = value, 4 // fill 2 (aligned) bytes
  359. (p_y) st2 [ptr2] = value, 4 // fill 2 (aligned) bytes
  360. (p_y) add cnt = -4, cnt // [7, 6 (or less) left]
  361. ;; }
  362. { .mmi
  363. (p_yy) st2 [ptr1] = value, 4 // fill 2 (aligned) bytes
  364. (p_yy) st2 [ptr2] = value, 4 // fill 2 (aligned) bytes
  365. // [3, 2 (or less) left]
  366. tbit.nz p_y, p0 = cnt, 0 // will there be a st1 at the end ?
  367. } { .mmi
  368. (p_yy) add cnt = -4, cnt
  369. ;; }
  370. { .mmb
  371. (p_scr) st2 [ptr1] = value // fill 2 (aligned) bytes
  372. (p_y) st1 [ptr3] = value // fill last byte (using ptr3)
  373. br.ret.sptk.many rp
  374. ;; }
  375. END(memset)
  376. libc_hidden_def (memset)