memcpy.S 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921
  1. /* Copy SIZE bytes from SRC to DEST.
  2. For UltraSPARC.
  3. Copyright (C) 1996, 97, 98, 99, 2003 Free Software Foundation, Inc.
  4. This file is part of the GNU C Library.
  5. Contributed by David S. Miller (davem@caip.rutgers.edu) and
  6. Jakub Jelinek (jakub@redhat.com).
  7. The GNU C Library is free software; you can redistribute it and/or
  8. modify it under the terms of the GNU Lesser General Public
  9. License as published by the Free Software Foundation; either
  10. version 2.1 of the License, or (at your option) any later version.
  11. The GNU C Library is distributed in the hope that it will be useful,
  12. but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. Lesser General Public License for more details.
  15. You should have received a copy of the GNU Lesser General Public
  16. License along with the GNU C Library; if not, write to the Free
  17. Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18. 02111-1307 USA. */
  19. #include <asm/asi.h>
  20. #ifndef XCC
  21. #define USE_BPR
  22. .register %g2, #scratch
  23. .register %g3, #scratch
  24. .register %g6, #scratch
  25. #define XCC xcc
  26. #endif
  27. #define FPRS_FEF 4
  28. #define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9) \
  29. faligndata %f1, %f2, %f48; \
  30. faligndata %f2, %f3, %f50; \
  31. faligndata %f3, %f4, %f52; \
  32. faligndata %f4, %f5, %f54; \
  33. faligndata %f5, %f6, %f56; \
  34. faligndata %f6, %f7, %f58; \
  35. faligndata %f7, %f8, %f60; \
  36. faligndata %f8, %f9, %f62;
  37. #define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \
  38. ldda [%src] %asi, %fdest; \
  39. add %src, 0x40, %src; \
  40. add %dest, 0x40, %dest; \
  41. subcc %len, 0x40, %len; \
  42. be,pn %xcc, jmptgt; \
  43. stda %fsrc, [%dest - 0x40] %asi;
  44. #define LOOP_CHUNK1(src, dest, len, branch_dest) \
  45. MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest)
  46. #define LOOP_CHUNK2(src, dest, len, branch_dest) \
  47. MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest)
  48. #define LOOP_CHUNK3(src, dest, len, branch_dest) \
  49. MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
  50. #define STORE_SYNC(dest, fsrc) \
  51. stda %fsrc, [%dest] %asi; \
  52. add %dest, 0x40, %dest;
  53. #define STORE_JUMP(dest, fsrc, target) \
  54. stda %fsrc, [%dest] %asi; \
  55. add %dest, 0x40, %dest; \
  56. ba,pt %xcc, target;
  57. #define VISLOOP_PAD nop; nop; nop; nop; \
  58. nop; nop; nop; nop; \
  59. nop; nop; nop; nop; \
  60. nop; nop; nop;
  61. #define FINISH_VISCHUNK(dest, f0, f1, left) \
  62. subcc %left, 8, %left; \
  63. bl,pn %xcc, 205f; \
  64. faligndata %f0, %f1, %f48; \
  65. std %f48, [%dest]; \
  66. add %dest, 8, %dest;
  67. #define UNEVEN_VISCHUNK(dest, f0, f1, left) \
  68. subcc %left, 8, %left; \
  69. bl,pn %xcc, 205f; \
  70. fsrc1 %f0, %f1; \
  71. ba,a,pt %xcc, 204f;
  72. /* Macros for non-VIS memcpy code. */
  73. #define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3) \
  74. ldx [%src + offset + 0x00], %t0; \
  75. ldx [%src + offset + 0x08], %t1; \
  76. ldx [%src + offset + 0x10], %t2; \
  77. ldx [%src + offset + 0x18], %t3; \
  78. stw %t0, [%dst + offset + 0x04]; \
  79. srlx %t0, 32, %t0; \
  80. stw %t0, [%dst + offset + 0x00]; \
  81. stw %t1, [%dst + offset + 0x0c]; \
  82. srlx %t1, 32, %t1; \
  83. stw %t1, [%dst + offset + 0x08]; \
  84. stw %t2, [%dst + offset + 0x14]; \
  85. srlx %t2, 32, %t2; \
  86. stw %t2, [%dst + offset + 0x10]; \
  87. stw %t3, [%dst + offset + 0x1c]; \
  88. srlx %t3, 32, %t3; \
  89. stw %t3, [%dst + offset + 0x18];
  90. #define MOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3) \
  91. ldx [%src + offset + 0x00], %t0; \
  92. ldx [%src + offset + 0x08], %t1; \
  93. ldx [%src + offset + 0x10], %t2; \
  94. ldx [%src + offset + 0x18], %t3; \
  95. stx %t0, [%dst + offset + 0x00]; \
  96. stx %t1, [%dst + offset + 0x08]; \
  97. stx %t2, [%dst + offset + 0x10]; \
  98. stx %t3, [%dst + offset + 0x18]; \
  99. ldx [%src + offset + 0x20], %t0; \
  100. ldx [%src + offset + 0x28], %t1; \
  101. ldx [%src + offset + 0x30], %t2; \
  102. ldx [%src + offset + 0x38], %t3; \
  103. stx %t0, [%dst + offset + 0x20]; \
  104. stx %t1, [%dst + offset + 0x28]; \
  105. stx %t2, [%dst + offset + 0x30]; \
  106. stx %t3, [%dst + offset + 0x38];
  107. #define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \
  108. ldx [%src - offset - 0x10], %t0; \
  109. ldx [%src - offset - 0x08], %t1; \
  110. stw %t0, [%dst - offset - 0x0c]; \
  111. srlx %t0, 32, %t2; \
  112. stw %t2, [%dst - offset - 0x10]; \
  113. stw %t1, [%dst - offset - 0x04]; \
  114. srlx %t1, 32, %t3; \
  115. stw %t3, [%dst - offset - 0x08];
  116. #define MOVE_LASTALIGNCHUNK(src, dst, offset, t0, t1) \
  117. ldx [%src - offset - 0x10], %t0; \
  118. ldx [%src - offset - 0x08], %t1; \
  119. stx %t0, [%dst - offset - 0x10]; \
  120. stx %t1, [%dst - offset - 0x08];
  121. /* Macros for non-VIS memmove code. */
  122. #define RMOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3) \
  123. ldx [%src - offset - 0x20], %t0; \
  124. ldx [%src - offset - 0x18], %t1; \
  125. ldx [%src - offset - 0x10], %t2; \
  126. ldx [%src - offset - 0x08], %t3; \
  127. stw %t0, [%dst - offset - 0x1c]; \
  128. srlx %t0, 32, %t0; \
  129. stw %t0, [%dst - offset - 0x20]; \
  130. stw %t1, [%dst - offset - 0x14]; \
  131. srlx %t1, 32, %t1; \
  132. stw %t1, [%dst - offset - 0x18]; \
  133. stw %t2, [%dst - offset - 0x0c]; \
  134. srlx %t2, 32, %t2; \
  135. stw %t2, [%dst - offset - 0x10]; \
  136. stw %t3, [%dst - offset - 0x04]; \
  137. srlx %t3, 32, %t3; \
  138. stw %t3, [%dst - offset - 0x08];
  139. #define RMOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3) \
  140. ldx [%src - offset - 0x20], %t0; \
  141. ldx [%src - offset - 0x18], %t1; \
  142. ldx [%src - offset - 0x10], %t2; \
  143. ldx [%src - offset - 0x08], %t3; \
  144. stx %t0, [%dst - offset - 0x20]; \
  145. stx %t1, [%dst - offset - 0x18]; \
  146. stx %t2, [%dst - offset - 0x10]; \
  147. stx %t3, [%dst - offset - 0x08]; \
  148. ldx [%src - offset - 0x40], %t0; \
  149. ldx [%src - offset - 0x38], %t1; \
  150. ldx [%src - offset - 0x30], %t2; \
  151. ldx [%src - offset - 0x28], %t3; \
  152. stx %t0, [%dst - offset - 0x40]; \
  153. stx %t1, [%dst - offset - 0x38]; \
  154. stx %t2, [%dst - offset - 0x30]; \
  155. stx %t3, [%dst - offset - 0x28];
  156. #define RMOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \
  157. ldx [%src + offset + 0x00], %t0; \
  158. ldx [%src + offset + 0x08], %t1; \
  159. stw %t0, [%dst + offset + 0x04]; \
  160. srlx %t0, 32, %t2; \
  161. stw %t2, [%dst + offset + 0x00]; \
  162. stw %t1, [%dst + offset + 0x0c]; \
  163. srlx %t1, 32, %t3; \
  164. stw %t3, [%dst + offset + 0x08];
  165. #define RMOVE_LASTALIGNCHUNK(src, dst, offset, t0, t1) \
  166. ldx [%src + offset + 0x00], %t0; \
  167. ldx [%src + offset + 0x08], %t1; \
  168. stx %t0, [%dst + offset + 0x00]; \
  169. stx %t1, [%dst + offset + 0x08];
  170. .text
  171. .align 32
  172. ENTRY(bcopy)
  173. sub %o1, %o0, %o4 /* IEU0 Group */
  174. mov %o0, %g3 /* IEU1 */
  175. cmp %o4, %o2 /* IEU1 Group */
  176. mov %o1, %o0 /* IEU0 */
  177. bgeu,pt %XCC, 210f /* CTI */
  178. mov %g3, %o1 /* IEU0 Group */
  179. #ifndef USE_BPR
  180. srl %o2, 0, %o2 /* IEU1 */
  181. #endif
  182. brnz,pn %o2, 220f /* CTI Group */
  183. add %o0, %o2, %o0 /* IEU0 */
  184. retl
  185. nop
  186. END(bcopy)
  187. libc_hidden_def(bcopy)
  188. .align 32
  189. 200: be,pt %xcc, 201f /* CTI */
  190. andcc %o0, 0x38, %g5 /* IEU1 Group */
  191. mov 8, %g1 /* IEU0 */
  192. sub %g1, %g2, %g2 /* IEU0 Group */
  193. andcc %o0, 1, %g0 /* IEU1 */
  194. be,pt %icc, 2f /* CTI */
  195. sub %o2, %g2, %o2 /* IEU0 Group */
  196. 1: ldub [%o1], %o5 /* Load Group */
  197. add %o1, 1, %o1 /* IEU0 */
  198. add %o0, 1, %o0 /* IEU1 */
  199. subcc %g2, 1, %g2 /* IEU1 Group */
  200. be,pn %xcc, 3f /* CTI */
  201. stb %o5, [%o0 - 1] /* Store */
  202. 2: ldub [%o1], %o5 /* Load Group */
  203. add %o0, 2, %o0 /* IEU0 */
  204. ldub [%o1 + 1], %g3 /* Load Group */
  205. subcc %g2, 2, %g2 /* IEU1 Group */
  206. stb %o5, [%o0 - 2] /* Store */
  207. add %o1, 2, %o1 /* IEU0 */
  208. bne,pt %xcc, 2b /* CTI Group */
  209. stb %g3, [%o0 - 1] /* Store */
  210. 3: andcc %o0, 0x38, %g5 /* IEU1 Group */
  211. 201: be,pt %icc, 202f /* CTI */
  212. mov 64, %g1 /* IEU0 */
  213. fmovd %f0, %f2 /* FPU */
  214. sub %g1, %g5, %g5 /* IEU0 Group */
  215. alignaddr %o1, %g0, %g1 /* GRU Group */
  216. ldd [%g1], %f4 /* Load Group */
  217. sub %o2, %g5, %o2 /* IEU0 */
  218. 1: ldd [%g1 + 0x8], %f6 /* Load Group */
  219. add %g1, 0x8, %g1 /* IEU0 Group */
  220. subcc %g5, 8, %g5 /* IEU1 */
  221. faligndata %f4, %f6, %f0 /* GRU Group */
  222. std %f0, [%o0] /* Store */
  223. add %o1, 8, %o1 /* IEU0 Group */
  224. be,pn %xcc, 202f /* CTI */
  225. add %o0, 8, %o0 /* IEU1 */
  226. ldd [%g1 + 0x8], %f4 /* Load Group */
  227. add %g1, 8, %g1 /* IEU0 */
  228. subcc %g5, 8, %g5 /* IEU1 */
  229. faligndata %f6, %f4, %f0 /* GRU Group */
  230. std %f0, [%o0] /* Store */
  231. add %o1, 8, %o1 /* IEU0 */
  232. bne,pt %xcc, 1b /* CTI Group */
  233. add %o0, 8, %o0 /* IEU0 */
  234. 202: membar #LoadStore | #StoreStore | #StoreLoad /* LSU Group */
  235. wr %g0, ASI_BLK_P, %asi /* LSU Group */
  236. subcc %o2, 0x40, %g6 /* IEU1 Group */
  237. mov %o1, %g1 /* IEU0 */
  238. andncc %g6, (0x40 - 1), %g6 /* IEU1 Group */
  239. srl %g1, 3, %g2 /* IEU0 */
  240. sub %o2, %g6, %g3 /* IEU0 Group */
  241. andn %o1, (0x40 - 1), %o1 /* IEU1 */
  242. and %g2, 7, %g2 /* IEU0 Group */
  243. andncc %g3, 0x7, %g3 /* IEU1 */
  244. fmovd %f0, %f2 /* FPU */
  245. sub %g3, 0x10, %g3 /* IEU0 Group */
  246. sub %o2, %g6, %o2 /* IEU1 */
  247. alignaddr %g1, %g0, %g0 /* GRU Group */
  248. add %g1, %g6, %g1 /* IEU0 Group */
  249. subcc %o2, %g3, %o2 /* IEU1 */
  250. ldda [%o1 + 0x00] %asi, %f0 /* LSU Group */
  251. add %g1, %g3, %g1 /* IEU0 */
  252. ldda [%o1 + 0x40] %asi, %f16 /* LSU Group */
  253. sub %g6, 0x80, %g6 /* IEU0 */
  254. ldda [%o1 + 0x80] %asi, %f32 /* LSU Group */
  255. /* Clk1 Group 8-( */
  256. /* Clk2 Group 8-( */
  257. /* Clk3 Group 8-( */
  258. /* Clk4 Group 8-( */
  259. 203: rd %pc, %g5 /* PDU Group 8-( */
  260. addcc %g5, %lo(300f - 203b), %g5 /* IEU1 Group */
  261. sll %g2, 9, %g2 /* IEU0 */
  262. jmpl %g5 + %g2, %g0 /* CTI Group brk forced*/
  263. addcc %o1, 0xc0, %o1 /* IEU1 Group */
  264. .align 512 /* OK, here comes the fun part... */
  265. 300: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) LOOP_CHUNK1(o1, o0, g6, 301f)
  266. FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) LOOP_CHUNK2(o1, o0, g6, 302f)
  267. FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) LOOP_CHUNK3(o1, o0, g6, 303f)
  268. b,pt %xcc, 300b+4; faligndata %f0, %f2, %f48
  269. 301: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) STORE_SYNC(o0, f48) membar #Sync
  270. FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) STORE_JUMP(o0, f48, 400f) membar #Sync
  271. 302: FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) STORE_SYNC(o0, f48) membar #Sync
  272. FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) STORE_JUMP(o0, f48, 416f) membar #Sync
  273. 303: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) STORE_SYNC(o0, f48) membar #Sync
  274. FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) STORE_JUMP(o0, f48, 432f) membar #Sync
  275. VISLOOP_PAD
  276. 310: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) LOOP_CHUNK1(o1, o0, g6, 311f)
  277. FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) LOOP_CHUNK2(o1, o0, g6, 312f)
  278. FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) LOOP_CHUNK3(o1, o0, g6, 313f)
  279. b,pt %xcc, 310b+4; faligndata %f2, %f4, %f48
  280. 311: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) STORE_SYNC(o0, f48) membar #Sync
  281. FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) STORE_JUMP(o0, f48, 402f) membar #Sync
  282. 312: FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) STORE_SYNC(o0, f48) membar #Sync
  283. FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) STORE_JUMP(o0, f48, 418f) membar #Sync
  284. 313: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) STORE_SYNC(o0, f48) membar #Sync
  285. FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) STORE_JUMP(o0, f48, 434f) membar #Sync
  286. VISLOOP_PAD
  287. 320: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) LOOP_CHUNK1(o1, o0, g6, 321f)
  288. FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) LOOP_CHUNK2(o1, o0, g6, 322f)
  289. FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) LOOP_CHUNK3(o1, o0, g6, 323f)
  290. b,pt %xcc, 320b+4; faligndata %f4, %f6, %f48
  291. 321: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) STORE_SYNC(o0, f48) membar #Sync
  292. FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) STORE_JUMP(o0, f48, 404f) membar #Sync
  293. 322: FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) STORE_SYNC(o0, f48) membar #Sync
  294. FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) STORE_JUMP(o0, f48, 420f) membar #Sync
  295. 323: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) STORE_SYNC(o0, f48) membar #Sync
  296. FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) STORE_JUMP(o0, f48, 436f) membar #Sync
  297. VISLOOP_PAD
  298. 330: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) LOOP_CHUNK1(o1, o0, g6, 331f)
  299. FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) LOOP_CHUNK2(o1, o0, g6, 332f)
  300. FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) LOOP_CHUNK3(o1, o0, g6, 333f)
  301. b,pt %xcc, 330b+4; faligndata %f6, %f8, %f48
  302. 331: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) STORE_SYNC(o0, f48) membar #Sync
  303. FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) STORE_JUMP(o0, f48, 406f) membar #Sync
  304. 332: FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) STORE_SYNC(o0, f48) membar #Sync
  305. FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) STORE_JUMP(o0, f48, 422f) membar #Sync
  306. 333: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) STORE_SYNC(o0, f48) membar #Sync
  307. FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) STORE_JUMP(o0, f48, 438f) membar #Sync
  308. VISLOOP_PAD
  309. 340: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) LOOP_CHUNK1(o1, o0, g6, 341f)
  310. FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) LOOP_CHUNK2(o1, o0, g6, 342f)
  311. FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) LOOP_CHUNK3(o1, o0, g6, 343f)
  312. b,pt %xcc, 340b+4; faligndata %f8, %f10, %f48
  313. 341: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) STORE_SYNC(o0, f48) membar #Sync
  314. FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) STORE_JUMP(o0, f48, 408f) membar #Sync
  315. 342: FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) STORE_SYNC(o0, f48) membar #Sync
  316. FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) STORE_JUMP(o0, f48, 424f) membar #Sync
  317. 343: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) STORE_SYNC(o0, f48) membar #Sync
  318. FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) STORE_JUMP(o0, f48, 440f) membar #Sync
  319. VISLOOP_PAD
  320. 350: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) LOOP_CHUNK1(o1, o0, g6, 351f)
  321. FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) LOOP_CHUNK2(o1, o0, g6, 352f)
  322. FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) LOOP_CHUNK3(o1, o0, g6, 353f)
  323. b,pt %xcc, 350b+4; faligndata %f10, %f12, %f48
  324. 351: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) STORE_SYNC(o0, f48) membar #Sync
  325. FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) STORE_JUMP(o0, f48, 410f) membar #Sync
  326. 352: FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) STORE_SYNC(o0, f48) membar #Sync
  327. FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) STORE_JUMP(o0, f48, 426f) membar #Sync
  328. 353: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) STORE_SYNC(o0, f48) membar #Sync
  329. FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) STORE_JUMP(o0, f48, 442f) membar #Sync
  330. VISLOOP_PAD
  331. 360: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) LOOP_CHUNK1(o1, o0, g6, 361f)
  332. FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) LOOP_CHUNK2(o1, o0, g6, 362f)
  333. FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) LOOP_CHUNK3(o1, o0, g6, 363f)
  334. b,pt %xcc, 360b+4; faligndata %f12, %f14, %f48
  335. 361: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) STORE_SYNC(o0, f48) membar #Sync
  336. FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) STORE_JUMP(o0, f48, 412f) membar #Sync
  337. 362: FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) STORE_SYNC(o0, f48) membar #Sync
  338. FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) STORE_JUMP(o0, f48, 428f) membar #Sync
  339. 363: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) STORE_SYNC(o0, f48) membar #Sync
  340. FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) STORE_JUMP(o0, f48, 444f) membar #Sync
  341. VISLOOP_PAD
  342. 370: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) LOOP_CHUNK1(o1, o0, g6, 371f)
  343. FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) LOOP_CHUNK2(o1, o0, g6, 372f)
  344. FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) LOOP_CHUNK3(o1, o0, g6, 373f)
  345. b,pt %xcc, 370b+4; faligndata %f14, %f16, %f48
  346. 371: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) STORE_SYNC(o0, f48) membar #Sync
  347. FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) STORE_JUMP(o0, f48, 414f) membar #Sync
  348. 372: FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) STORE_SYNC(o0, f48) membar #Sync
  349. FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) STORE_JUMP(o0, f48, 430f) membar #Sync
  350. 373: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) STORE_SYNC(o0, f48) membar #Sync
  351. FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) STORE_JUMP(o0, f48, 446f) membar #Sync
  352. VISLOOP_PAD
  353. 400: FINISH_VISCHUNK(o0, f0, f2, g3)
  354. 402: FINISH_VISCHUNK(o0, f2, f4, g3)
  355. 404: FINISH_VISCHUNK(o0, f4, f6, g3)
  356. 406: FINISH_VISCHUNK(o0, f6, f8, g3)
  357. 408: FINISH_VISCHUNK(o0, f8, f10, g3)
  358. 410: FINISH_VISCHUNK(o0, f10, f12, g3)
  359. 412: FINISH_VISCHUNK(o0, f12, f14, g3)
  360. 414: UNEVEN_VISCHUNK(o0, f14, f0, g3)
  361. 416: FINISH_VISCHUNK(o0, f16, f18, g3)
  362. 418: FINISH_VISCHUNK(o0, f18, f20, g3)
  363. 420: FINISH_VISCHUNK(o0, f20, f22, g3)
  364. 422: FINISH_VISCHUNK(o0, f22, f24, g3)
  365. 424: FINISH_VISCHUNK(o0, f24, f26, g3)
  366. 426: FINISH_VISCHUNK(o0, f26, f28, g3)
  367. 428: FINISH_VISCHUNK(o0, f28, f30, g3)
  368. 430: UNEVEN_VISCHUNK(o0, f30, f0, g3)
  369. 432: FINISH_VISCHUNK(o0, f32, f34, g3)
  370. 434: FINISH_VISCHUNK(o0, f34, f36, g3)
  371. 436: FINISH_VISCHUNK(o0, f36, f38, g3)
  372. 438: FINISH_VISCHUNK(o0, f38, f40, g3)
  373. 440: FINISH_VISCHUNK(o0, f40, f42, g3)
  374. 442: FINISH_VISCHUNK(o0, f42, f44, g3)
  375. 444: FINISH_VISCHUNK(o0, f44, f46, g3)
  376. 446: UNEVEN_VISCHUNK(o0, f46, f0, g3)
  377. 204: ldd [%o1], %f2 /* Load Group */
  378. add %o1, 8, %o1 /* IEU0 */
  379. subcc %g3, 8, %g3 /* IEU1 */
  380. faligndata %f0, %f2, %f8 /* GRU Group */
  381. std %f8, [%o0] /* Store */
  382. bl,pn %xcc, 205f /* CTI */
  383. add %o0, 8, %o0 /* IEU0 Group */
  384. ldd [%o1], %f0 /* Load Group */
  385. add %o1, 8, %o1 /* IEU0 */
  386. subcc %g3, 8, %g3 /* IEU1 */
  387. faligndata %f2, %f0, %f8 /* GRU Group */
  388. std %f8, [%o0] /* Store */
  389. bge,pt %xcc, 204b /* CTI */
  390. add %o0, 8, %o0 /* IEU0 Group */
  391. 205: brz,pt %o2, 207f /* CTI Group */
  392. mov %g1, %o1 /* IEU0 */
  393. 206: ldub [%o1], %g5 /* LOAD */
  394. add %o1, 1, %o1 /* IEU0 */
  395. add %o0, 1, %o0 /* IEU1 */
  396. subcc %o2, 1, %o2 /* IEU1 */
  397. bne,pt %xcc, 206b /* CTI */
  398. stb %g5, [%o0 - 1] /* Store Group */
  399. 207: membar #StoreLoad | #StoreStore /* LSU Group */
  400. wr %g0, FPRS_FEF, %fprs
  401. retl
  402. mov %g4, %o0
  403. 208: andcc %o2, 1, %g0 /* IEU1 Group */
  404. be,pt %icc, 2f+4 /* CTI */
  405. 1: ldub [%o1], %g5 /* LOAD Group */
  406. add %o1, 1, %o1 /* IEU0 */
  407. add %o0, 1, %o0 /* IEU1 */
  408. subcc %o2, 1, %o2 /* IEU1 Group */
  409. be,pn %xcc, 209f /* CTI */
  410. stb %g5, [%o0 - 1] /* Store */
  411. 2: ldub [%o1], %g5 /* LOAD Group */
  412. add %o0, 2, %o0 /* IEU0 */
  413. ldub [%o1 + 1], %o5 /* LOAD Group */
  414. add %o1, 2, %o1 /* IEU0 */
  415. subcc %o2, 2, %o2 /* IEU1 Group */
  416. stb %g5, [%o0 - 2] /* Store */
  417. bne,pt %xcc, 2b /* CTI */
  418. stb %o5, [%o0 - 1] /* Store */
  419. 209: retl
  420. mov %g4, %o0
  421. #ifdef USE_BPR
  422. /* void *__align_cpy_4(void *dest, void *src, size_t n)
  423. * SPARC v9 SYSV ABI
  424. * Like memcpy, but results are undefined if (!n || ((dest | src | n) & 3))
  425. */
  426. .align 32
  427. ENTRY(__align_cpy_4)
  428. mov %o0, %g4 /* IEU0 Group */
  429. cmp %o2, 15 /* IEU1 */
  430. bleu,pn %xcc, 208b /* CTI */
  431. cmp %o2, (64 * 6) /* IEU1 Group */
  432. bgeu,pn %xcc, 200b /* CTI */
  433. andcc %o0, 7, %g2 /* IEU1 Group */
  434. ba,pt %xcc, 216f /* CTI */
  435. andcc %o1, 4, %g0 /* IEU1 Group */
  436. END(__align_cpy_4)
  437. /* void *__align_cpy_8(void *dest, void *src, size_t n)
  438. * SPARC v9 SYSV ABI
  439. * Like memcpy, but results are undefined if (!n || ((dest | src | n) & 7))
  440. */
  441. .align 32
  442. ENTRY(__align_cpy_8)
  443. mov %o0, %g4 /* IEU0 Group */
  444. cmp %o2, 15 /* IEU1 */
  445. bleu,pn %xcc, 208b /* CTI */
  446. cmp %o2, (64 * 6) /* IEU1 Group */
  447. bgeu,pn %xcc, 201b /* CTI */
  448. andcc %o0, 0x38, %g5 /* IEU1 Group */
  449. andcc %o2, -128, %g6 /* IEU1 Group */
  450. bne,a,pt %xcc, 82f + 4 /* CTI */
  451. ldx [%o1], %g1 /* Load */
  452. ba,pt %xcc, 41f /* CTI Group */
  453. andcc %o2, 0x70, %g6 /* IEU1 */
  454. END(__align_cpy_8)
  455. /* void *__align_cpy_16(void *dest, void *src, size_t n)
  456. * SPARC v9 SYSV ABI
  457. * Like memcpy, but results are undefined if (!n || ((dest | src | n) & 15))
  458. */
  459. .align 32
  460. ENTRY(__align_cpy_16)
  461. mov %o0, %g4 /* IEU0 Group */
  462. cmp %o2, (64 * 6) /* IEU1 */
  463. bgeu,pn %xcc, 201b /* CTI */
  464. andcc %o0, 0x38, %g5 /* IEU1 Group */
  465. andcc %o2, -128, %g6 /* IEU1 Group */
  466. bne,a,pt %xcc, 82f + 4 /* CTI */
  467. ldx [%o1], %g1 /* Load */
  468. ba,pt %xcc, 41f /* CTI Group */
  469. andcc %o2, 0x70, %g6 /* IEU1 */
  470. END(__align_cpy_16)
  471. #endif
  472. .align 32
  473. ENTRY(memcpy)
  474. 210:
  475. #ifndef USE_BPR
  476. srl %o2, 0, %o2 /* IEU1 Group */
  477. #endif
  478. brz,pn %o2, 209b /* CTI Group */
  479. mov %o0, %g4 /* IEU0 */
  480. 218: cmp %o2, 15 /* IEU1 Group */
  481. bleu,pn %xcc, 208b /* CTI */
  482. cmp %o2, (64 * 6) /* IEU1 Group */
  483. bgeu,pn %xcc, 200b /* CTI */
  484. andcc %o0, 7, %g2 /* IEU1 Group */
  485. sub %o0, %o1, %g5 /* IEU0 */
  486. andcc %g5, 3, %o5 /* IEU1 Group */
  487. bne,pn %xcc, 212f /* CTI */
  488. andcc %o1, 3, %g0 /* IEU1 Group */
  489. be,a,pt %xcc, 216f /* CTI */
  490. andcc %o1, 4, %g0 /* IEU1 Group */
  491. andcc %o1, 1, %g0 /* IEU1 Group */
  492. be,pn %xcc, 4f /* CTI */
  493. andcc %o1, 2, %g0 /* IEU1 Group */
  494. ldub [%o1], %g2 /* Load Group */
  495. add %o1, 1, %o1 /* IEU0 */
  496. add %o0, 1, %o0 /* IEU1 */
  497. sub %o2, 1, %o2 /* IEU0 Group */
  498. bne,pn %xcc, 5f /* CTI Group */
  499. stb %g2, [%o0 - 1] /* Store */
  500. 4: lduh [%o1], %g2 /* Load Group */
  501. add %o1, 2, %o1 /* IEU0 */
  502. add %o0, 2, %o0 /* IEU1 */
  503. sub %o2, 2, %o2 /* IEU0 */
  504. sth %g2, [%o0 - 2] /* Store Group + bubble */
  505. 5: andcc %o1, 4, %g0 /* IEU1 */
  506. 216: be,a,pn %xcc, 2f /* CTI */
  507. andcc %o2, -128, %g6 /* IEU1 Group */
  508. lduw [%o1], %g5 /* Load Group */
  509. add %o1, 4, %o1 /* IEU0 */
  510. add %o0, 4, %o0 /* IEU1 */
  511. sub %o2, 4, %o2 /* IEU0 Group */
  512. stw %g5, [%o0 - 4] /* Store */
  513. andcc %o2, -128, %g6 /* IEU1 Group */
  514. 2: be,pn %xcc, 215f /* CTI */
  515. andcc %o0, 4, %g0 /* IEU1 Group */
  516. be,pn %xcc, 82f + 4 /* CTI Group */
  517. 5: MOVE_BIGCHUNK(o1, o0, 0x00, g1, g3, g5, o5)
  518. MOVE_BIGCHUNK(o1, o0, 0x20, g1, g3, g5, o5)
  519. MOVE_BIGCHUNK(o1, o0, 0x40, g1, g3, g5, o5)
  520. MOVE_BIGCHUNK(o1, o0, 0x60, g1, g3, g5, o5)
  521. 35: subcc %g6, 128, %g6 /* IEU1 Group */
  522. add %o1, 128, %o1 /* IEU0 */
  523. bne,pt %xcc, 5b /* CTI */
  524. add %o0, 128, %o0 /* IEU0 Group */
  525. 215: andcc %o2, 0x70, %g6 /* IEU1 Group */
  526. 41: be,pn %xcc, 80f /* CTI */
  527. andcc %o2, 8, %g0 /* IEU1 Group */
  528. /* Clk1 8-( */
  529. /* Clk2 8-( */
  530. /* Clk3 8-( */
  531. /* Clk4 8-( */
  532. 79: rd %pc, %o5 /* PDU Group */
  533. sll %g6, 1, %g5 /* IEU0 Group */
  534. add %o1, %g6, %o1 /* IEU1 */
  535. sub %o5, %g5, %o5 /* IEU0 Group */
  536. jmpl %o5 + %lo(80f - 79b), %g0 /* CTI Group brk forced*/
  537. add %o0, %g6, %o0 /* IEU0 Group */
  538. 36: MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g5, o5)
  539. MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g5, o5)
  540. MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g5, o5)
  541. MOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g5, o5)
  542. MOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g5, o5)
  543. MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g5, o5)
  544. MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g5, o5)
  545. 80: be,pt %xcc, 81f /* CTI */
  546. andcc %o2, 4, %g0 /* IEU1 */
  547. ldx [%o1], %g2 /* Load Group */
  548. add %o0, 8, %o0 /* IEU0 */
  549. stw %g2, [%o0 - 0x4] /* Store Group */
  550. add %o1, 8, %o1 /* IEU1 */
  551. srlx %g2, 32, %g2 /* IEU0 Group */
  552. stw %g2, [%o0 - 0x8] /* Store */
  553. 81: be,pt %xcc, 1f /* CTI */
  554. andcc %o2, 2, %g0 /* IEU1 Group */
  555. lduw [%o1], %g2 /* Load Group */
  556. add %o1, 4, %o1 /* IEU0 */
  557. stw %g2, [%o0] /* Store Group */
  558. add %o0, 4, %o0 /* IEU0 */
  559. 1: be,pt %xcc, 1f /* CTI */
  560. andcc %o2, 1, %g0 /* IEU1 Group */
  561. lduh [%o1], %g2 /* Load Group */
  562. add %o1, 2, %o1 /* IEU0 */
  563. sth %g2, [%o0] /* Store Group */
  564. add %o0, 2, %o0 /* IEU0 */
  565. 1: be,pt %xcc, 211f /* CTI */
  566. nop /* IEU1 */
  567. ldub [%o1], %g2 /* Load Group */
  568. stb %g2, [%o0] /* Store Group + bubble */
  569. 211: retl
  570. mov %g4, %o0
  571. 82: MOVE_BIGALIGNCHUNK(o1, o0, 0x00, g1, g3, g5, o5)
  572. MOVE_BIGALIGNCHUNK(o1, o0, 0x40, g1, g3, g5, o5)
  573. 37: subcc %g6, 128, %g6 /* IEU1 Group */
  574. add %o1, 128, %o1 /* IEU0 */
  575. bne,pt %xcc, 82b /* CTI */
  576. add %o0, 128, %o0 /* IEU0 Group */
  577. andcc %o2, 0x70, %g6 /* IEU1 */
  578. be,pn %xcc, 84f /* CTI */
  579. andcc %o2, 8, %g0 /* IEU1 Group */
  580. /* Clk1 8-( */
  581. /* Clk2 8-( */
  582. /* Clk3 8-( */
  583. /* Clk4 8-( */
  584. 83: rd %pc, %o5 /* PDU Group */
  585. add %o1, %g6, %o1 /* IEU0 Group */
  586. sub %o5, %g6, %o5 /* IEU1 */
  587. jmpl %o5 + %lo(84f - 83b), %g0 /* CTI Group brk forced*/
  588. add %o0, %g6, %o0 /* IEU0 Group */
  589. 38: MOVE_LASTALIGNCHUNK(o1, o0, 0x60, g2, g3)
  590. MOVE_LASTALIGNCHUNK(o1, o0, 0x50, g2, g3)
  591. MOVE_LASTALIGNCHUNK(o1, o0, 0x40, g2, g3)
  592. MOVE_LASTALIGNCHUNK(o1, o0, 0x30, g2, g3)
  593. MOVE_LASTALIGNCHUNK(o1, o0, 0x20, g2, g3)
  594. MOVE_LASTALIGNCHUNK(o1, o0, 0x10, g2, g3)
  595. MOVE_LASTALIGNCHUNK(o1, o0, 0x00, g2, g3)
  596. 84: be,pt %xcc, 85f /* CTI Group */
  597. andcc %o2, 4, %g0 /* IEU1 */
  598. ldx [%o1], %g2 /* Load Group */
  599. add %o0, 8, %o0 /* IEU0 */
  600. add %o1, 8, %o1 /* IEU0 Group */
  601. stx %g2, [%o0 - 0x8] /* Store */
  602. 85: be,pt %xcc, 1f /* CTI */
  603. andcc %o2, 2, %g0 /* IEU1 Group */
  604. lduw [%o1], %g2 /* Load Group */
  605. add %o0, 4, %o0 /* IEU0 */
  606. add %o1, 4, %o1 /* IEU0 Group */
  607. stw %g2, [%o0 - 0x4] /* Store */
  608. 1: be,pt %xcc, 1f /* CTI */
  609. andcc %o2, 1, %g0 /* IEU1 Group */
  610. lduh [%o1], %g2 /* Load Group */
  611. add %o0, 2, %o0 /* IEU0 */
  612. add %o1, 2, %o1 /* IEU0 Group */
  613. sth %g2, [%o0 - 0x2] /* Store */
  614. 1: be,pt %xcc, 1f /* CTI */
  615. nop /* IEU0 Group */
  616. ldub [%o1], %g2 /* Load Group */
  617. stb %g2, [%o0] /* Store Group + bubble */
  618. 1: retl
  619. mov %g4, %o0
  620. 212: brz,pt %g2, 2f /* CTI Group */
  621. mov 8, %g1 /* IEU0 */
  622. sub %g1, %g2, %g2 /* IEU0 Group */
  623. sub %o2, %g2, %o2 /* IEU0 Group */
  624. 1: ldub [%o1], %g5 /* Load Group */
  625. add %o1, 1, %o1 /* IEU0 */
  626. add %o0, 1, %o0 /* IEU1 */
  627. subcc %g2, 1, %g2 /* IEU1 Group */
  628. bne,pt %xcc, 1b /* CTI */
  629. stb %g5, [%o0 - 1] /* Store */
  630. 2: andn %o2, 7, %g5 /* IEU0 Group */
  631. and %o2, 7, %o2 /* IEU1 */
  632. fmovd %f0, %f2 /* FPU */
  633. alignaddr %o1, %g0, %g1 /* GRU Group */
  634. ldd [%g1], %f4 /* Load Group */
  635. 1: ldd [%g1 + 0x8], %f6 /* Load Group */
  636. add %g1, 0x8, %g1 /* IEU0 Group */
  637. subcc %g5, 8, %g5 /* IEU1 */
  638. faligndata %f4, %f6, %f0 /* GRU Group */
  639. std %f0, [%o0] /* Store */
  640. add %o1, 8, %o1 /* IEU0 Group */
  641. be,pn %xcc, 213f /* CTI */
  642. add %o0, 8, %o0 /* IEU1 */
  643. ldd [%g1 + 0x8], %f4 /* Load Group */
  644. add %g1, 8, %g1 /* IEU0 */
  645. subcc %g5, 8, %g5 /* IEU1 */
  646. faligndata %f6, %f4, %f0 /* GRU Group */
  647. std %f0, [%o0] /* Store */
  648. add %o1, 8, %o1 /* IEU0 */
  649. bne,pn %xcc, 1b /* CTI Group */
  650. add %o0, 8, %o0 /* IEU0 */
  651. 213: brz,pn %o2, 214f /* CTI Group */
  652. nop /* IEU0 */
  653. ldub [%o1], %g5 /* LOAD */
  654. add %o1, 1, %o1 /* IEU0 */
  655. add %o0, 1, %o0 /* IEU1 */
  656. subcc %o2, 1, %o2 /* IEU1 */
  657. bne,pt %xcc, 206b /* CTI */
  658. stb %g5, [%o0 - 1] /* Store Group */
  659. 214: wr %g0, FPRS_FEF, %fprs
  660. retl
  661. mov %g4, %o0
  662. END(memcpy)
  663. libc_hidden_def(memcpy)
  664. .align 32
  665. 228: andcc %o2, 1, %g0 /* IEU1 Group */
  666. be,pt %icc, 2f+4 /* CTI */
  667. 1: ldub [%o1 - 1], %o5 /* LOAD Group */
  668. sub %o1, 1, %o1 /* IEU0 */
  669. sub %o0, 1, %o0 /* IEU1 */
  670. subcc %o2, 1, %o2 /* IEU1 Group */
  671. be,pn %xcc, 229f /* CTI */
  672. stb %o5, [%o0] /* Store */
  673. 2: ldub [%o1 - 1], %o5 /* LOAD Group */
  674. sub %o0, 2, %o0 /* IEU0 */
  675. ldub [%o1 - 2], %g5 /* LOAD Group */
  676. sub %o1, 2, %o1 /* IEU0 */
  677. subcc %o2, 2, %o2 /* IEU1 Group */
  678. stb %o5, [%o0 + 1] /* Store */
  679. bne,pt %xcc, 2b /* CTI */
  680. stb %g5, [%o0] /* Store */
  681. 229: retl
  682. mov %g4, %o0
  683. 219: retl
  684. nop
  685. .align 32
  686. ENTRY(memmove)
  687. #ifndef USE_BPR
  688. srl %o2, 0, %o2 /* IEU1 Group */
  689. #endif
  690. brz,pn %o2, 219b /* CTI Group */
  691. sub %o0, %o1, %o4 /* IEU0 */
  692. cmp %o4, %o2 /* IEU1 Group */
  693. bgeu,pt %XCC, 218b /* CTI */
  694. mov %o0, %g4 /* IEU0 */
  695. add %o0, %o2, %o0 /* IEU0 Group */
  696. 220: add %o1, %o2, %o1 /* IEU1 */
  697. cmp %o2, 15 /* IEU1 Group */
  698. bleu,pn %xcc, 228b /* CTI */
  699. andcc %o0, 7, %g2 /* IEU1 Group */
  700. sub %o0, %o1, %g5 /* IEU0 */
  701. andcc %g5, 3, %o5 /* IEU1 Group */
  702. bne,pn %xcc, 232f /* CTI */
  703. andcc %o1, 3, %g0 /* IEU1 Group */
  704. be,a,pt %xcc, 236f /* CTI */
  705. andcc %o1, 4, %g0 /* IEU1 Group */
  706. andcc %o1, 1, %g0 /* IEU1 Group */
  707. be,pn %xcc, 4f /* CTI */
  708. andcc %o1, 2, %g0 /* IEU1 Group */
  709. ldub [%o1 - 1], %g2 /* Load Group */
  710. sub %o1, 1, %o1 /* IEU0 */
  711. sub %o0, 1, %o0 /* IEU1 */
  712. sub %o2, 1, %o2 /* IEU0 Group */
  713. be,pn %xcc, 5f /* CTI Group */
  714. stb %g2, [%o0] /* Store */
  715. 4: lduh [%o1 - 2], %g2 /* Load Group */
  716. sub %o1, 2, %o1 /* IEU0 */
  717. sub %o0, 2, %o0 /* IEU1 */
  718. sub %o2, 2, %o2 /* IEU0 */
  719. sth %g2, [%o0] /* Store Group + bubble */
  720. 5: andcc %o1, 4, %g0 /* IEU1 */
  721. 236: be,a,pn %xcc, 2f /* CTI */
  722. andcc %o2, -128, %g6 /* IEU1 Group */
  723. lduw [%o1 - 4], %g5 /* Load Group */
  724. sub %o1, 4, %o1 /* IEU0 */
  725. sub %o0, 4, %o0 /* IEU1 */
  726. sub %o2, 4, %o2 /* IEU0 Group */
  727. stw %g5, [%o0] /* Store */
  728. andcc %o2, -128, %g6 /* IEU1 Group */
  729. 2: be,pn %xcc, 235f /* CTI */
  730. andcc %o0, 4, %g0 /* IEU1 Group */
  731. be,pn %xcc, 282f + 4 /* CTI Group */
  732. 5: RMOVE_BIGCHUNK(o1, o0, 0x00, g1, g3, g5, o5)
  733. RMOVE_BIGCHUNK(o1, o0, 0x20, g1, g3, g5, o5)
  734. RMOVE_BIGCHUNK(o1, o0, 0x40, g1, g3, g5, o5)
  735. RMOVE_BIGCHUNK(o1, o0, 0x60, g1, g3, g5, o5)
  736. subcc %g6, 128, %g6 /* IEU1 Group */
  737. sub %o1, 128, %o1 /* IEU0 */
  738. bne,pt %xcc, 5b /* CTI */
  739. sub %o0, 128, %o0 /* IEU0 Group */
  740. 235: andcc %o2, 0x70, %g6 /* IEU1 Group */
  741. 41: be,pn %xcc, 280f /* CTI */
  742. andcc %o2, 8, %g0 /* IEU1 Group */
  743. /* Clk1 8-( */
  744. /* Clk2 8-( */
  745. /* Clk3 8-( */
  746. /* Clk4 8-( */
  747. 279: rd %pc, %o5 /* PDU Group */
  748. sll %g6, 1, %g5 /* IEU0 Group */
  749. sub %o1, %g6, %o1 /* IEU1 */
  750. sub %o5, %g5, %o5 /* IEU0 Group */
  751. jmpl %o5 + %lo(280f - 279b), %g0 /* CTI Group brk forced*/
  752. sub %o0, %g6, %o0 /* IEU0 Group */
  753. RMOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g5, o5)
  754. RMOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g5, o5)
  755. RMOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g5, o5)
  756. RMOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g5, o5)
  757. RMOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g5, o5)
  758. RMOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g5, o5)
  759. RMOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g5, o5)
  760. 280: be,pt %xcc, 281f /* CTI */
  761. andcc %o2, 4, %g0 /* IEU1 */
  762. ldx [%o1 - 8], %g2 /* Load Group */
  763. sub %o0, 8, %o0 /* IEU0 */
  764. stw %g2, [%o0 + 4] /* Store Group */
  765. sub %o1, 8, %o1 /* IEU1 */
  766. srlx %g2, 32, %g2 /* IEU0 Group */
  767. stw %g2, [%o0] /* Store */
  768. 281: be,pt %xcc, 1f /* CTI */
  769. andcc %o2, 2, %g0 /* IEU1 Group */
  770. lduw [%o1 - 4], %g2 /* Load Group */
  771. sub %o1, 4, %o1 /* IEU0 */
  772. stw %g2, [%o0 - 4] /* Store Group */
  773. sub %o0, 4, %o0 /* IEU0 */
  774. 1: be,pt %xcc, 1f /* CTI */
  775. andcc %o2, 1, %g0 /* IEU1 Group */
  776. lduh [%o1 - 2], %g2 /* Load Group */
  777. sub %o1, 2, %o1 /* IEU0 */
  778. sth %g2, [%o0 - 2] /* Store Group */
  779. sub %o0, 2, %o0 /* IEU0 */
  780. 1: be,pt %xcc, 211f /* CTI */
  781. nop /* IEU1 */
  782. ldub [%o1 - 1], %g2 /* Load Group */
  783. stb %g2, [%o0 - 1] /* Store Group + bubble */
  784. 211: retl
  785. mov %g4, %o0
  786. 282: RMOVE_BIGALIGNCHUNK(o1, o0, 0x00, g1, g3, g5, o5)
  787. RMOVE_BIGALIGNCHUNK(o1, o0, 0x40, g1, g3, g5, o5)
  788. subcc %g6, 128, %g6 /* IEU1 Group */
  789. sub %o1, 128, %o1 /* IEU0 */
  790. bne,pt %xcc, 282b /* CTI */
  791. sub %o0, 128, %o0 /* IEU0 Group */
  792. andcc %o2, 0x70, %g6 /* IEU1 */
  793. be,pn %xcc, 284f /* CTI */
  794. andcc %o2, 8, %g0 /* IEU1 Group */
  795. /* Clk1 8-( */
  796. /* Clk2 8-( */
  797. /* Clk3 8-( */
  798. /* Clk4 8-( */
  799. 283: rd %pc, %o5 /* PDU Group */
  800. sub %o1, %g6, %o1 /* IEU0 Group */
  801. sub %o5, %g6, %o5 /* IEU1 */
  802. jmpl %o5 + %lo(284f - 283b), %g0 /* CTI Group brk forced*/
  803. sub %o0, %g6, %o0 /* IEU0 Group */
  804. RMOVE_LASTALIGNCHUNK(o1, o0, 0x60, g2, g3)
  805. RMOVE_LASTALIGNCHUNK(o1, o0, 0x50, g2, g3)
  806. RMOVE_LASTALIGNCHUNK(o1, o0, 0x40, g2, g3)
  807. RMOVE_LASTALIGNCHUNK(o1, o0, 0x30, g2, g3)
  808. RMOVE_LASTALIGNCHUNK(o1, o0, 0x20, g2, g3)
  809. RMOVE_LASTALIGNCHUNK(o1, o0, 0x10, g2, g3)
  810. RMOVE_LASTALIGNCHUNK(o1, o0, 0x00, g2, g3)
  811. 284: be,pt %xcc, 285f /* CTI Group */
  812. andcc %o2, 4, %g0 /* IEU1 */
  813. ldx [%o1 - 8], %g2 /* Load Group */
  814. sub %o0, 8, %o0 /* IEU0 */
  815. sub %o1, 8, %o1 /* IEU0 Group */
  816. stx %g2, [%o0] /* Store */
  817. 285: be,pt %xcc, 1f /* CTI */
  818. andcc %o2, 2, %g0 /* IEU1 Group */
  819. lduw [%o1 - 4], %g2 /* Load Group */
  820. sub %o0, 4, %o0 /* IEU0 */
  821. sub %o1, 4, %o1 /* IEU0 Group */
  822. stw %g2, [%o0] /* Store */
  823. 1: be,pt %xcc, 1f /* CTI */
  824. andcc %o2, 1, %g0 /* IEU1 Group */
  825. lduh [%o1 - 2], %g2 /* Load Group */
  826. sub %o0, 2, %o0 /* IEU0 */
  827. sub %o1, 2, %o1 /* IEU0 Group */
  828. sth %g2, [%o0] /* Store */
  829. 1: be,pt %xcc, 1f /* CTI */
  830. nop /* IEU0 Group */
  831. ldub [%o1 - 1], %g2 /* Load Group */
  832. stb %g2, [%o0 - 1] /* Store Group + bubble */
  833. 1: retl
  834. mov %g4, %o0
  835. 232: brz,pt %g2, 2f /* CTI Group */
  836. sub %o2, %g2, %o2 /* IEU0 Group */
  837. 1: ldub [%o1 - 1], %g5 /* Load Group */
  838. sub %o1, 1, %o1 /* IEU0 */
  839. sub %o0, 1, %o0 /* IEU1 */
  840. subcc %g2, 1, %g2 /* IEU1 Group */
  841. bne,pt %xcc, 1b /* CTI */
  842. stb %g5, [%o0] /* Store */
  843. 2: andn %o2, 7, %g5 /* IEU0 Group */
  844. and %o2, 7, %o2 /* IEU1 */
  845. fmovd %f0, %f2 /* FPU */
  846. alignaddr %o1, %g0, %g1 /* GRU Group */
  847. ldd [%g1], %f4 /* Load Group */
  848. 1: ldd [%g1 - 8], %f6 /* Load Group */
  849. sub %g1, 8, %g1 /* IEU0 Group */
  850. subcc %g5, 8, %g5 /* IEU1 */
  851. faligndata %f6, %f4, %f0 /* GRU Group */
  852. std %f0, [%o0 - 8] /* Store */
  853. sub %o1, 8, %o1 /* IEU0 Group */
  854. be,pn %xcc, 233f /* CTI */
  855. sub %o0, 8, %o0 /* IEU1 */
  856. ldd [%g1 - 8], %f4 /* Load Group */
  857. sub %g1, 8, %g1 /* IEU0 */
  858. subcc %g5, 8, %g5 /* IEU1 */
  859. faligndata %f4, %f6, %f0 /* GRU Group */
  860. std %f0, [%o0 - 8] /* Store */
  861. sub %o1, 8, %o1 /* IEU0 */
  862. bne,pn %xcc, 1b /* CTI Group */
  863. sub %o0, 8, %o0 /* IEU0 */
  864. 233: brz,pn %o2, 234f /* CTI Group */
  865. nop /* IEU0 */
  866. 237: ldub [%o1 - 1], %g5 /* LOAD */
  867. sub %o1, 1, %o1 /* IEU0 */
  868. sub %o0, 1, %o0 /* IEU1 */
  869. subcc %o2, 1, %o2 /* IEU1 */
  870. bne,pt %xcc, 237b /* CTI */
  871. stb %g5, [%o0] /* Store Group */
  872. 234: wr %g0, FPRS_FEF, %fprs
  873. retl
  874. mov %g4, %o0
  875. END(memmove)
  876. libc_hidden_def(memmove)
  877. #ifdef USE_BPR
  878. weak_alias(memcpy,__align_cpy_1)
  879. weak_alias(memcpy,__align_cpy_2)
  880. #endif