bzero.S 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320
  1. /* Optimized version of the standard bzero() function.
  2. This file is part of the GNU C Library.
  3. Copyright (C) 2000, 2001, 2002 Free Software Foundation, Inc.
  4. Contributed by Dan Pop for Itanium <Dan.Pop@cern.ch>.
  5. Rewritten for McKinley by Sverre Jarp, HP Labs/CERN <Sverre.Jarp@cern.ch>
  6. The GNU C Library is free software; you can redistribute it and/or
  7. modify it under the terms of the GNU Lesser General Public
  8. License as published by the Free Software Foundation; either
  9. version 2.1 of the License, or (at your option) any later version.
  10. The GNU C Library is distributed in the hope that it will be useful,
  11. but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  13. Lesser General Public License for more details.
  14. You should have received a copy of the GNU Lesser General Public
  15. License along with the GNU C Library; if not, write to the Free
  16. Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  17. 02111-1307 USA. */
  18. /* Return: dest
  19. Inputs:
  20. in0: dest
  21. in1: count
  22. The algorithm is fairly straightforward: set byte by byte until we
  23. we get to a 16B-aligned address, then loop on 128 B chunks using an
  24. early store as prefetching, then loop on 32B chucks, then clear remaining
  25. words, finally clear remaining bytes.
  26. Since a stf.spill f0 can store 16B in one go, we use this instruction
  27. to get peak speed. */
  28. #include "sysdep.h"
  29. #ifdef __UCLIBC_SUSV3_LEGACY__
  30. #undef ret
  31. #define dest in0
  32. #define cnt in1
  33. #define tmp r31
  34. #define save_lc r30
  35. #define ptr0 r29
  36. #define ptr1 r28
  37. #define ptr2 r27
  38. #define ptr3 r26
  39. #define ptr9 r24
  40. #define loopcnt r23
  41. #define linecnt r22
  42. #define bytecnt r21
  43. // This routine uses only scratch predicate registers (p6 - p15)
  44. #define p_scr p6 // default register for same-cycle branches
  45. #define p_unalgn p9
  46. #define p_y p11
  47. #define p_n p12
  48. #define p_yy p13
  49. #define p_nn p14
  50. #define movi0 mov
  51. #define MIN1 15
  52. #define MIN1P1HALF 8
  53. #define LINE_SIZE 128
  54. #define LSIZE_SH 7 // shift amount
  55. #define PREF_AHEAD 8
  56. #define USE_FLP
  57. #if defined(USE_INT)
  58. #define store st8
  59. #define myval r0
  60. #elif defined(USE_FLP)
  61. #define store stf8
  62. #define myval f0
  63. #endif
  64. .align 64
  65. ENTRY(bzero)
  66. { .mmi
  67. .prologue
  68. alloc tmp = ar.pfs, 2, 0, 0, 0
  69. lfetch.nt1 [dest]
  70. .save ar.lc, save_lc
  71. movi0 save_lc = ar.lc
  72. } { .mmi
  73. .body
  74. mov ret0 = dest // return value
  75. nop.m 0
  76. cmp.eq p_scr, p0 = cnt, r0
  77. ;; }
  78. { .mmi
  79. and ptr2 = -(MIN1+1), dest // aligned address
  80. and tmp = MIN1, dest // prepare to check for alignment
  81. tbit.nz p_y, p_n = dest, 0 // Do we have an odd address? (M_B_U)
  82. } { .mib
  83. mov ptr1 = dest
  84. nop.i 0
  85. (p_scr) br.ret.dpnt.many rp // return immediately if count = 0
  86. ;; }
  87. { .mib
  88. cmp.ne p_unalgn, p0 = tmp, r0
  89. } { .mib // NB: # of bytes to move is 1
  90. sub bytecnt = (MIN1+1), tmp // higher than loopcnt
  91. cmp.gt p_scr, p0 = 16, cnt // is it a minimalistic task?
  92. (p_scr) br.cond.dptk.many .move_bytes_unaligned // go move just a few (M_B_U)
  93. ;; }
  94. { .mmi
  95. (p_unalgn) add ptr1 = (MIN1+1), ptr2 // after alignment
  96. (p_unalgn) add ptr2 = MIN1P1HALF, ptr2 // after alignment
  97. (p_unalgn) tbit.nz.unc p_y, p_n = bytecnt, 3 // should we do a st8 ?
  98. ;; }
  99. { .mib
  100. (p_y) add cnt = -8, cnt
  101. (p_unalgn) tbit.nz.unc p_yy, p_nn = bytecnt, 2 // should we do a st4 ?
  102. } { .mib
  103. (p_y) st8 [ptr2] = r0,-4
  104. (p_n) add ptr2 = 4, ptr2
  105. ;; }
  106. { .mib
  107. (p_yy) add cnt = -4, cnt
  108. (p_unalgn) tbit.nz.unc p_y, p_n = bytecnt, 1 // should we do a st2 ?
  109. } { .mib
  110. (p_yy) st4 [ptr2] = r0,-2
  111. (p_nn) add ptr2 = 2, ptr2
  112. ;; }
  113. { .mmi
  114. mov tmp = LINE_SIZE+1 // for compare
  115. (p_y) add cnt = -2, cnt
  116. (p_unalgn) tbit.nz.unc p_yy, p_nn = bytecnt, 0 // should we do a st1 ?
  117. } { .mmi
  118. nop.m 0
  119. (p_y) st2 [ptr2] = r0,-1
  120. (p_n) add ptr2 = 1, ptr2
  121. ;; }
  122. { .mmi
  123. (p_yy) st1 [ptr2] = r0
  124. cmp.gt p_scr, p0 = tmp, cnt // is it a minimalistic task?
  125. } { .mbb
  126. (p_yy) add cnt = -1, cnt
  127. (p_scr) br.cond.dpnt.many .fraction_of_line // go move just a few
  128. ;; }
  129. { .mib
  130. nop.m 0
  131. shr.u linecnt = cnt, LSIZE_SH
  132. nop.b 0
  133. ;; }
  134. .align 32
  135. .l1b: // ------------------// L1B: store ahead into cache lines; fill later
  136. { .mmi
  137. and tmp = -(LINE_SIZE), cnt // compute end of range
  138. mov ptr9 = ptr1 // used for prefetching
  139. and cnt = (LINE_SIZE-1), cnt // remainder
  140. } { .mmi
  141. mov loopcnt = PREF_AHEAD-1 // default prefetch loop
  142. cmp.gt p_scr, p0 = PREF_AHEAD, linecnt // check against actual value
  143. ;; }
  144. { .mmi
  145. (p_scr) add loopcnt = -1, linecnt
  146. add ptr2 = 16, ptr1 // start of stores (beyond prefetch stores)
  147. add ptr1 = tmp, ptr1 // first address beyond total range
  148. ;; }
  149. { .mmi
  150. add tmp = -1, linecnt // next loop count
  151. movi0 ar.lc = loopcnt
  152. ;; }
  153. .pref_l1b:
  154. { .mib
  155. stf.spill [ptr9] = f0, 128 // Do stores one cache line apart
  156. nop.i 0
  157. br.cloop.dptk.few .pref_l1b
  158. ;; }
  159. { .mmi
  160. add ptr0 = 16, ptr2 // Two stores in parallel
  161. movi0 ar.lc = tmp
  162. ;; }
  163. .l1bx:
  164. { .mmi
  165. stf.spill [ptr2] = f0, 32
  166. stf.spill [ptr0] = f0, 32
  167. ;; }
  168. { .mmi
  169. stf.spill [ptr2] = f0, 32
  170. stf.spill [ptr0] = f0, 32
  171. ;; }
  172. { .mmi
  173. stf.spill [ptr2] = f0, 32
  174. stf.spill [ptr0] = f0, 64
  175. cmp.lt p_scr, p0 = ptr9, ptr1 // do we need more prefetching?
  176. ;; }
  177. { .mmb
  178. stf.spill [ptr2] = f0, 32
  179. (p_scr) stf.spill [ptr9] = f0, 128
  180. br.cloop.dptk.few .l1bx
  181. ;; }
  182. { .mib
  183. cmp.gt p_scr, p0 = 8, cnt // just a few bytes left ?
  184. (p_scr) br.cond.dpnt.many .move_bytes_from_alignment
  185. ;; }
  186. .fraction_of_line:
  187. { .mib
  188. add ptr2 = 16, ptr1
  189. shr.u loopcnt = cnt, 5 // loopcnt = cnt / 32
  190. ;; }
  191. { .mib
  192. cmp.eq p_scr, p0 = loopcnt, r0
  193. add loopcnt = -1, loopcnt
  194. (p_scr) br.cond.dpnt.many .store_words
  195. ;; }
  196. { .mib
  197. and cnt = 0x1f, cnt // compute the remaining cnt
  198. movi0 ar.lc = loopcnt
  199. ;; }
  200. .align 32
  201. .l2: // -----------------------------// L2A: store 32B in 2 cycles
  202. { .mmb
  203. store [ptr1] = myval, 8
  204. store [ptr2] = myval, 8
  205. ;; } { .mmb
  206. store [ptr1] = myval, 24
  207. store [ptr2] = myval, 24
  208. br.cloop.dptk.many .l2
  209. ;; }
  210. .store_words:
  211. { .mib
  212. cmp.gt p_scr, p0 = 8, cnt // just a few bytes left ?
  213. (p_scr) br.cond.dpnt.many .move_bytes_from_alignment // Branch
  214. ;; }
  215. { .mmi
  216. store [ptr1] = myval, 8 // store
  217. cmp.le p_y, p_n = 16, cnt //
  218. add cnt = -8, cnt // subtract
  219. ;; }
  220. { .mmi
  221. (p_y) store [ptr1] = myval, 8 // store
  222. (p_y) cmp.le.unc p_yy, p_nn = 16, cnt
  223. (p_y) add cnt = -8, cnt // subtract
  224. ;; }
  225. { .mmi // store
  226. (p_yy) store [ptr1] = myval, 8
  227. (p_yy) add cnt = -8, cnt // subtract
  228. ;; }
  229. .move_bytes_from_alignment:
  230. { .mib
  231. cmp.eq p_scr, p0 = cnt, r0
  232. tbit.nz.unc p_y, p0 = cnt, 2 // should we terminate with a st4 ?
  233. (p_scr) br.cond.dpnt.few .restore_and_exit
  234. ;; }
  235. { .mib
  236. (p_y) st4 [ptr1] = r0,4
  237. tbit.nz.unc p_yy, p0 = cnt, 1 // should we terminate with a st2 ?
  238. ;; }
  239. { .mib
  240. (p_yy) st2 [ptr1] = r0,2
  241. tbit.nz.unc p_y, p0 = cnt, 0 // should we terminate with a st1 ?
  242. ;; }
  243. { .mib
  244. (p_y) st1 [ptr1] = r0
  245. ;; }
  246. .restore_and_exit:
  247. { .mib
  248. nop.m 0
  249. movi0 ar.lc = save_lc
  250. br.ret.sptk.many rp
  251. ;; }
  252. .move_bytes_unaligned:
  253. { .mmi
  254. .pred.rel "mutex",p_y, p_n
  255. .pred.rel "mutex",p_yy, p_nn
  256. (p_n) cmp.le p_yy, p_nn = 4, cnt
  257. (p_y) cmp.le p_yy, p_nn = 5, cnt
  258. (p_n) add ptr2 = 2, ptr1
  259. } { .mmi
  260. (p_y) add ptr2 = 3, ptr1
  261. (p_y) st1 [ptr1] = r0, 1 // fill 1 (odd-aligned) byte
  262. (p_y) add cnt = -1, cnt // [15, 14 (or less) left]
  263. ;; }
  264. { .mmi
  265. (p_yy) cmp.le.unc p_y, p0 = 8, cnt
  266. add ptr3 = ptr1, cnt // prepare last store
  267. movi0 ar.lc = save_lc
  268. } { .mmi
  269. (p_yy) st2 [ptr1] = r0, 4 // fill 2 (aligned) bytes
  270. (p_yy) st2 [ptr2] = r0, 4 // fill 2 (aligned) bytes
  271. (p_yy) add cnt = -4, cnt // [11, 10 (o less) left]
  272. ;; }
  273. { .mmi
  274. (p_y) cmp.le.unc p_yy, p0 = 8, cnt
  275. add ptr3 = -1, ptr3 // last store
  276. tbit.nz p_scr, p0 = cnt, 1 // will there be a st2 at the end ?
  277. } { .mmi
  278. (p_y) st2 [ptr1] = r0, 4 // fill 2 (aligned) bytes
  279. (p_y) st2 [ptr2] = r0, 4 // fill 2 (aligned) bytes
  280. (p_y) add cnt = -4, cnt // [7, 6 (or less) left]
  281. ;; }
  282. { .mmi
  283. (p_yy) st2 [ptr1] = r0, 4 // fill 2 (aligned) bytes
  284. (p_yy) st2 [ptr2] = r0, 4 // fill 2 (aligned) bytes
  285. // [3, 2 (or less) left]
  286. tbit.nz p_y, p0 = cnt, 0 // will there be a st1 at the end ?
  287. } { .mmi
  288. (p_yy) add cnt = -4, cnt
  289. ;; }
  290. { .mmb
  291. (p_scr) st2 [ptr1] = r0 // fill 2 (aligned) bytes
  292. (p_y) st1 [ptr3] = r0 // fill last byte (using ptr3)
  293. br.ret.sptk.many rp
  294. ;; }
  295. END(bzero)
  296. #endif