memcpy.S 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908
  1. /*
  2. * "memcpy" implementation of SuperH
  3. *
  4. * Copyright (C) 1999 Niibe Yutaka
  5. * Copyright (c) 2002 STMicroelectronics Ltd
  6. * Modified from memcpy.S and micro-optimised for SH4
  7. * Stuart Menefy (stuart.menefy@st.com)
  8. *
  9. * Copyright (c) 2009 STMicroelectronics Ltd
  10. * Optimised using prefetching and 64bit data transfer via FPU
  11. * Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
  12. */
  13. /*
  14. * void *memcpy(void *dst, const void *src, size_t n);
  15. *
  16. * It is assumed that there is no overlap between src and dst.
  17. * If there is an overlap, then the results are undefined.
  18. */
  19. #include <sysdep.h>
  20. #include <endian.h>
  21. #if defined (__LITTLE_ENDIAN__) && defined (__SH_FPU_ANY__)
  22. #define MEMCPY_USES_FPU
  23. /* Use paired single precision load or store mode for 64-bit tranfering.
  24. * FPSCR.SZ=1,FPSCR.SZ=0 is well defined on both SH4-200 and SH4-300.
  25. * Currenlty it has been only implemented and tested for little endian mode. */
  26. .macro FPU_SET_PAIRED_PREC
  27. sts fpscr, r7
  28. mov #0x10, r6 ! PR=0 SZ=1
  29. shll16 r6
  30. lds r6, fpscr
  31. .endm
  32. .macro RESTORE_FPSCR
  33. lds r7, fpscr
  34. .endm
  35. #endif
  36. !
  37. ! GHIJ KLMN OPQR --> ...G HIJK LMNO PQR.
  38. !
  39. ! Size is 16 or greater, and may have trailing bytes
  40. .balign 32
  41. .Lcase1:
  42. ! Read a long word and write a long word at once
  43. ! At the start of each iteration, r7 contains last long load
  44. add #-1,r5 ! 79 EX
  45. mov r4,r2 ! 5 MT (0 cycles latency)
  46. mov.l @(r0,r5),r7 ! 21 LS (2 cycles latency)
  47. add #-4,r5 ! 50 EX
  48. add #7,r2 ! 79 EX
  49. !
  50. #ifdef __LITTLE_ENDIAN__
  51. ! 6 cycles, 4 bytes per iteration
  52. 3: mov.l @(r0,r5),r1 ! 21 LS (latency=2) ! NMLK
  53. mov r7, r3 ! 5 MT (latency=0) ! RQPO
  54. cmp/hi r2,r0 ! 57 MT
  55. shll16 r3 ! 103 EX
  56. mov r1,r6 ! 5 MT (latency=0)
  57. shll8 r3 ! 102 EX ! Oxxx
  58. shlr8 r6 ! 106 EX ! xNML
  59. mov r1, r7 ! 5 MT (latency=0)
  60. or r6,r3 ! 82 EX ! ONML
  61. bt/s 3b ! 109 BR
  62. mov.l r3,@-r0 ! 30 LS
  63. #else
  64. 3: mov.l @(r0,r5),r1 ! 21 LS (latency=2) ! KLMN
  65. mov r7,r3 ! 5 MT (latency=0) ! OPQR
  66. cmp/hi r2,r0 ! 57 MT
  67. shlr16 r3 ! 107 EX
  68. shlr8 r3 ! 106 EX ! xxxO
  69. mov r1,r6 ! 5 MT (latency=0)
  70. shll8 r6 ! 102 EX ! LMNx
  71. mov r1,r7 ! 5 MT (latency=0)
  72. or r6,r3 ! 82 EX ! LMNO
  73. bt/s 3b ! 109 BR
  74. mov.l r3,@-r0 ! 30 LS
  75. #endif
  76. ! Finally, copy a byte at once, if necessary
  77. add #4,r5 ! 50 EX
  78. cmp/eq r4,r0 ! 54 MT
  79. add #-6,r2 ! 50 EX
  80. bt 9f ! 109 BR
  81. 8: cmp/hi r2,r0 ! 57 MT
  82. mov.b @(r0,r5),r1 ! 20 LS (latency=2)
  83. bt/s 8b ! 109 BR
  84. mov.b r1,@-r0 ! 29 LS
  85. 9: rts
  86. nop
  87. !
  88. ! GHIJ KLMN OPQR --> .GHI JKLM NOPQ R...
  89. !
  90. ! Size is 16 or greater, and may have trailing bytes
  91. .balign 32
  92. .Lcase3:
  93. ! Read a long word and write a long word at once
  94. ! At the start of each iteration, r7 contains last long load
  95. add #-3,r5 ! 79 EX
  96. mov r4,r2 ! 5 MT (0 cycles latency)
  97. mov.l @(r0,r5),r7 ! 21 LS (2 cycles latency)
  98. add #-4,r5 ! 50 EX
  99. add #7,r2 ! 79 EX
  100. !
  101. #ifdef __LITTLE_ENDIAN__
  102. ! 6 cycles, 4 bytes per iteration
  103. 3: mov.l @(r0,r5),r1 ! 21 LS (latency=2) ! NMLK
  104. mov r7, r3 ! 5 MT (latency=0) ! RQPO
  105. cmp/hi r2,r0 ! 57 MT
  106. shll8 r3 ! 102 EX ! QPOx
  107. mov r1,r6 ! 5 MT (latency=0)
  108. shlr16 r6 ! 107 EX
  109. shlr8 r6 ! 106 EX ! xxxN
  110. mov r1, r7 ! 5 MT (latency=0)
  111. or r6,r3 ! 82 EX ! QPON
  112. bt/s 3b ! 109 BR
  113. mov.l r3,@-r0 ! 30 LS
  114. #else
  115. 3: mov r7,r3 ! OPQR
  116. shlr8 r3 ! xOPQ
  117. mov.l @(r0,r5),r7 ! KLMN
  118. mov r7,r6
  119. shll16 r6
  120. shll8 r6 ! Nxxx
  121. or r6,r3 ! NOPQ
  122. cmp/hi r2,r0
  123. bt/s 3b
  124. mov.l r3,@-r0
  125. #endif
  126. ! Finally, copy a byte at once, if necessary
  127. add #6,r5 ! 50 EX
  128. cmp/eq r4,r0 ! 54 MT
  129. add #-6,r2 ! 50 EX
  130. bt 9f ! 109 BR
  131. 8: cmp/hi r2,r0 ! 57 MT
  132. mov.b @(r0,r5),r1 ! 20 LS (latency=2)
  133. bt/s 8b ! 109 BR
  134. mov.b r1,@-r0 ! 29 LS
  135. 9: rts
  136. nop
  137. ENTRY(memcpy)
  138. ! Calculate the invariants which will be used in the remainder
  139. ! of the code:
  140. !
  141. ! r4 --> [ ... ] DST [ ... ] SRC
  142. ! [ ... ] [ ... ]
  143. ! : :
  144. ! r0 --> [ ... ] r0+r5 --> [ ... ]
  145. !
  146. !
  147. ! Short circuit the common case of src, dst and len being 32 bit aligned
  148. ! and test for zero length move
  149. mov r6, r0 ! 5 MT (0 cycle latency)
  150. or r4, r0 ! 82 EX
  151. or r5, r0 ! 82 EX
  152. tst r6, r6 ! 86 MT
  153. bt/s 99f ! 111 BR (zero len)
  154. tst #3, r0 ! 87 MT
  155. mov r4, r0 ! 5 MT (0 cycle latency)
  156. add r6, r0 ! 49 EX
  157. bt/s .Lcase00 ! 111 BR (aligned)
  158. sub r4, r5 ! 75 EX
  159. ! Arguments are not nicely long word aligned or zero len.
  160. ! Check for small copies, and if so do a simple byte at a time copy.
  161. !
  162. ! Deciding on an exact value of 'small' is not easy, as the point at which
  163. ! using the optimised routines become worthwhile varies (these are the
  164. ! cycle counts for differnet sizes using byte-at-a-time vs. optimised):
  165. ! size byte-at-time long word byte
  166. ! 16 42 39-40 46-50 50-55
  167. ! 24 58 43-44 54-58 62-67
  168. ! 36 82 49-50 66-70 80-85
  169. ! However the penalty for getting it 'wrong' is much higher for long word
  170. ! aligned data (and this is more common), so use a value of 16.
  171. mov #16, r1 ! 6 EX
  172. cmp/gt r6,r1 ! 56 MT
  173. add #-1,r5 ! 50 EX
  174. bf/s 6f ! 108 BR (not small)
  175. mov r5, r3 ! 5 MT (latency=0)
  176. shlr r6 ! 104 EX
  177. mov.b @(r0,r5),r1 ! 20 LS (latency=2)
  178. bf/s 4f ! 111 BR
  179. add #-1,r3 ! 50 EX
  180. tst r6, r6 ! 86 MT
  181. bt/s 98f ! 110 BR
  182. mov.b r1,@-r0 ! 29 LS
  183. ! 4 cycles, 2 bytes per iteration
  184. 3: mov.b @(r0,r5),r1 ! 20 LS (latency=2)
  185. 4: mov.b @(r0,r3),r2 ! 20 LS (latency=2)
  186. dt r6 ! 67 EX
  187. mov.b r1,@-r0 ! 29 LS
  188. bf/s 3b ! 111 BR
  189. mov.b r2,@-r0 ! 29 LS
  190. 98:
  191. rts
  192. nop
  193. 99: rts
  194. mov r4, r0
  195. ! Size is not small, so its worthwhile looking for optimisations.
  196. ! First align destination to a long word boundary.
  197. !
  198. ! r5 = normal value -1
  199. 6: tst #3, r0 ! 87 MT
  200. mov #3, r3 ! 6 EX
  201. bt/s 2f ! 111 BR
  202. and r0,r3 ! 78 EX
  203. ! 3 cycles, 1 byte per iteration
  204. 1: dt r3 ! 67 EX
  205. mov.b @(r0,r5),r1 ! 19 LS (latency=2)
  206. add #-1, r6 ! 79 EX
  207. bf/s 1b ! 109 BR
  208. mov.b r1,@-r0 ! 28 LS
  209. 2: add #1, r5 ! 79 EX
  210. ! Now select the appropriate bulk transfer code based on relative
  211. ! alignment of src and dst.
  212. mov r0, r3 ! 5 MT (latency=0)
  213. mov r5, r0 ! 5 MT (latency=0)
  214. tst #1, r0 ! 87 MT
  215. bf/s 1f ! 111 BR
  216. mov #64, r7 ! 6 EX
  217. ! bit 0 clear
  218. cmp/ge r7, r6 ! 55 MT
  219. bt/s 2f ! 111 BR
  220. tst #2, r0 ! 87 MT
  221. ! small
  222. bt/s .Lcase0
  223. mov r3, r0
  224. bra .Lcase2
  225. nop
  226. ! big
  227. 2: bt/s .Lcase0b
  228. mov r3, r0
  229. bra .Lcase2b
  230. nop
  231. ! bit 0 set
  232. 1: tst #2, r0 ! 87 MT
  233. bt/s .Lcase1
  234. mov r3, r0
  235. bra .Lcase3
  236. nop
  237. !
  238. ! GHIJ KLMN OPQR --> GHIJ KLMN OPQR
  239. !
  240. ! src, dst and size are all long word aligned
  241. ! size is non-zero
  242. .balign 32
  243. .Lcase00:
  244. mov #64, r1 ! 6 EX
  245. mov r5, r3 ! 5 MT (latency=0)
  246. cmp/gt r6, r1 ! 56 MT
  247. add #-4, r5 ! 50 EX
  248. bf .Lcase00b ! 108 BR (big loop)
  249. shlr2 r6 ! 105 EX
  250. shlr r6 ! 104 EX
  251. mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  252. bf/s 4f ! 111 BR
  253. add #-8, r3 ! 50 EX
  254. tst r6, r6 ! 86 MT
  255. bt/s 5f ! 110 BR
  256. mov.l r1,@-r0 ! 30 LS
  257. ! 4 cycles, 2 long words per iteration
  258. 3: mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  259. 4: mov.l @(r0, r3), r2 ! 21 LS (latency=2)
  260. dt r6 ! 67 EX
  261. mov.l r1, @-r0 ! 30 LS
  262. bf/s 3b ! 109 BR
  263. mov.l r2, @-r0 ! 30 LS
  264. 5: rts
  265. nop
  266. ! Size is 16 or greater and less than 64, but may have trailing bytes
  267. .balign 32
  268. .Lcase0:
  269. add #-4, r5 ! 50 EX
  270. mov r4, r7 ! 5 MT (latency=0)
  271. mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  272. mov #4, r2 ! 6 EX
  273. add #11, r7 ! 50 EX
  274. tst r2, r6 ! 86 MT
  275. mov r5, r3 ! 5 MT (latency=0)
  276. bt/s 4f ! 111 BR
  277. add #-4, r3 ! 50 EX
  278. mov.l r1,@-r0 ! 30 LS
  279. ! 4 cycles, 2 long words per iteration
  280. 3: mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  281. 4: mov.l @(r0, r3), r2 ! 21 LS (latency=2)
  282. cmp/hi r7, r0
  283. mov.l r1, @-r0 ! 30 LS
  284. bt/s 3b ! 109 BR
  285. mov.l r2, @-r0 ! 30 LS
  286. ! Copy the final 0-3 bytes
  287. add #3,r5 ! 50 EX
  288. cmp/eq r0, r4 ! 54 MT
  289. add #-10, r7 ! 50 EX
  290. bt 9f ! 110 BR
  291. ! 3 cycles, 1 byte per iteration
  292. 1: mov.b @(r0,r5),r1 ! 19 LS
  293. cmp/hi r7,r0 ! 57 MT
  294. bt/s 1b ! 111 BR
  295. mov.b r1,@-r0 ! 28 LS
  296. 9: rts
  297. nop
  298. ! Size is at least 64 bytes, so will be going round the big loop at least once.
  299. !
  300. ! r2 = rounded up r4
  301. ! r3 = rounded down r0
  302. .balign 32
  303. .Lcase0b:
  304. add #-4, r5 ! 50 EX
  305. .Lcase00b:
  306. mov r0, r3 ! 5 MT (latency=0)
  307. mov #(~0x1f), r1 ! 6 EX
  308. and r1, r3 ! 78 EX
  309. mov r4, r2 ! 5 MT (latency=0)
  310. cmp/eq r3, r0 ! 54 MT
  311. add #0x1f, r2 ! 50 EX
  312. bt/s 1f ! 110 BR
  313. and r1, r2 ! 78 EX
  314. ! copy initial words until cache line aligned
  315. mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  316. tst #4, r0 ! 87 MT
  317. mov r5, r6 ! 5 MT (latency=0)
  318. add #-4, r6 ! 50 EX
  319. bt/s 4f ! 111 BR
  320. add #8, r3 ! 50 EX
  321. tst #0x18, r0 ! 87 MT
  322. bt/s 1f ! 109 BR
  323. mov.l r1,@-r0 ! 30 LS
  324. ! 4 cycles, 2 long words per iteration
  325. 3: mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  326. 4: mov.l @(r0, r6), r7 ! 21 LS (latency=2)
  327. cmp/eq r3, r0 ! 54 MT
  328. mov.l r1, @-r0 ! 30 LS
  329. bf/s 3b ! 109 BR
  330. mov.l r7, @-r0 ! 30 LS
  331. #ifdef MEMCPY_USES_FPU
  332. ! Copy the cache line aligned blocks by using the FPU registers.
  333. ! If src and dst are well aligned adopt 64-bit data transfer.
  334. ! We also need r0 as a temporary (for movca), so 'undo' the invariant:
  335. ! r5: src (was r0+r5)
  336. ! r1: dest (was r0)
  337. 1:
  338. add r0, r5
  339. mov r0, r1
  340. add #-0x1c, r5
  341. mov r5, r0
  342. tst #7, r0 ! src is 8byte aligned
  343. mov r5, r3
  344. add #-64, r3 ! To pefetch head
  345. bt/s 3f
  346. pref @r3
  347. 2: fmov.s @r5+, fr0
  348. mov r1, r6
  349. fmov.s @r5+, fr1
  350. add #-32, r6
  351. fmov.s @r5+, fr2
  352. fmov.s @r5+, fr3
  353. fmov.s @r5+, fr4
  354. fmov.s @r5+, fr5
  355. fmov.s @r5+, fr6
  356. fmov.s @r5+, fr7
  357. add #-0x40, r5
  358. movca.l r0, @r6 ! Cache allocate + store on dst-32.
  359. fmov.s fr7, @-r1
  360. fmov.s fr6, @-r1
  361. fmov.s fr5, @-r1
  362. fmov.s fr4, @-r1
  363. fmov.s fr3, @-r1
  364. fmov.s fr2, @-r1
  365. fmov.s fr1, @-r1
  366. fmov.s fr0, @-r1
  367. add #-32, r3
  368. cmp/eq r2,r1
  369. bf/s 2b
  370. pref @r3 ! Prefetch the next cache line.
  371. bra 5f
  372. 3: FPU_SET_PAIRED_PREC
  373. 4: fmov @r5+, dr0
  374. mov r1, r6
  375. fmov @r5+, dr2
  376. add #-32, r6
  377. fmov @r5+, dr4
  378. fmov @r5+, dr6
  379. add #-0x40, r5
  380. movca.l r0, @r6
  381. fmov dr6, @-r1
  382. fmov dr4, @-r1
  383. fmov dr2, @-r1
  384. fmov dr0, @-r1
  385. add #-32, r3
  386. cmp/eq r2,r1
  387. bf/s 4b
  388. pref @r3
  389. RESTORE_FPSCR
  390. 5: mov r1, r0
  391. cmp/eq r4, r0 ! 54 MT
  392. bf/s 1f ! 109 BR
  393. sub r1, r5 ! 75 EX
  394. rts
  395. nop
  396. 1:
  397. #else
  398. ! Copy the cache line aligned blocks
  399. !
  400. ! In use: r0, r2, r4, r5
  401. ! Scratch: r1, r3, r6, r7
  402. !
  403. ! We could do this with the four scratch registers, but if src
  404. ! and dest hit the same cache line, this will thrash, so make
  405. ! use of additional registers.
  406. !
  407. ! We also need r0 as a temporary (for movca), so 'undo' the invariant:
  408. ! r5: src (was r0+r5)
  409. ! r1: dest (was r0)
  410. ! this can be reversed at the end, so we don't need to save any extra
  411. ! state.
  412. !
  413. 1: mov.l r8, @-r15 ! 30 LS
  414. add r0, r5 ! 49 EX
  415. mov.l r9, @-r15 ! 30 LS
  416. mov r0, r1 ! 5 MT (latency=0)
  417. mov.l r10, @-r15 ! 30 LS
  418. add #-0x1c, r5 ! 50 EX
  419. mov.l r11, @-r15 ! 30 LS
  420. ! 16 cycles, 32 bytes per iteration
  421. 2: mov.l @(0x00,r5),r0 ! 18 LS (latency=2)
  422. add #-0x20, r1 ! 50 EX
  423. mov.l @(0x04,r5),r3 ! 18 LS (latency=2)
  424. mov.l @(0x08,r5),r6 ! 18 LS (latency=2)
  425. mov.l @(0x0c,r5),r7 ! 18 LS (latency=2)
  426. mov.l @(0x10,r5),r8 ! 18 LS (latency=2)
  427. mov.l @(0x14,r5),r9 ! 18 LS (latency=2)
  428. mov.l @(0x18,r5),r10 ! 18 LS (latency=2)
  429. mov.l @(0x1c,r5),r11 ! 18 LS (latency=2)
  430. movca.l r0,@r1 ! 40 LS (latency=3-7)
  431. mov.l r3,@(0x04,r1) ! 33 LS
  432. mov.l r6,@(0x08,r1) ! 33 LS
  433. mov.l r7,@(0x0c,r1) ! 33 LS
  434. mov.l r8,@(0x10,r1) ! 33 LS
  435. add #-0x20, r5 ! 50 EX
  436. mov.l r9,@(0x14,r1) ! 33 LS
  437. cmp/eq r2,r1 ! 54 MT
  438. mov.l r10,@(0x18,r1) ! 33 LS
  439. bf/s 2b ! 109 BR
  440. mov.l r11,@(0x1c,r1) ! 33 LS
  441. mov r1, r0 ! 5 MT (latency=0)
  442. mov.l @r15+, r11 ! 15 LS
  443. sub r1, r5 ! 75 EX
  444. mov.l @r15+, r10 ! 15 LS
  445. cmp/eq r4, r0 ! 54 MT
  446. bf/s 1f ! 109 BR
  447. mov.l @r15+, r9 ! 15 LS
  448. rts
  449. 1: mov.l @r15+, r8 ! 15 LS
  450. #endif
  451. sub r4, r1 ! 75 EX (len remaining)
  452. ! number of trailing bytes is non-zero
  453. !
  454. ! invariants restored (r5 already decremented by 4)
  455. ! also r1=num bytes remaining
  456. mov #4, r2 ! 6 EX
  457. mov r4, r7 ! 5 MT (latency=0)
  458. add #0x1c, r5 ! 50 EX (back to -4)
  459. cmp/hs r2, r1 ! 58 MT
  460. bf/s 5f ! 108 BR
  461. add #11, r7 ! 50 EX
  462. mov.l @(r0, r5), r6 ! 21 LS (latency=2)
  463. tst r2, r1 ! 86 MT
  464. mov r5, r3 ! 5 MT (latency=0)
  465. bt/s 4f ! 111 BR
  466. add #-4, r3 ! 50 EX
  467. cmp/hs r2, r1 ! 58 MT
  468. bt/s 5f ! 111 BR
  469. mov.l r6,@-r0 ! 30 LS
  470. ! 4 cycles, 2 long words per iteration
  471. 3: mov.l @(r0, r5), r6 ! 21 LS (latency=2)
  472. 4: mov.l @(r0, r3), r2 ! 21 LS (latency=2)
  473. cmp/hi r7, r0
  474. mov.l r6, @-r0 ! 30 LS
  475. bt/s 3b ! 109 BR
  476. mov.l r2, @-r0 ! 30 LS
  477. ! Copy the final 0-3 bytes
  478. 5: cmp/eq r0, r4 ! 54 MT
  479. add #-10, r7 ! 50 EX
  480. bt 9f ! 110 BR
  481. add #3,r5 ! 50 EX
  482. ! 3 cycles, 1 byte per iteration
  483. 1: mov.b @(r0,r5),r1 ! 19 LS
  484. cmp/hi r7,r0 ! 57 MT
  485. bt/s 1b ! 111 BR
  486. mov.b r1,@-r0 ! 28 LS
  487. 9: rts
  488. nop
  489. !
  490. ! GHIJ KLMN OPQR --> ..GH IJKL MNOP QR..
  491. !
  492. .balign 32
  493. .Lcase2:
  494. ! Size is 16 or greater and less then 64, but may have trailing bytes
  495. 2: mov r5, r6 ! 5 MT (latency=0)
  496. add #-2,r5 ! 50 EX
  497. mov r4,r2 ! 5 MT (latency=0)
  498. add #-4,r6 ! 50 EX
  499. add #7,r2 ! 50 EX
  500. 3: mov.w @(r0,r5),r1 ! 20 LS (latency=2)
  501. mov.w @(r0,r6),r3 ! 20 LS (latency=2)
  502. cmp/hi r2,r0 ! 57 MT
  503. mov.w r1,@-r0 ! 29 LS
  504. bt/s 3b ! 111 BR
  505. mov.w r3,@-r0 ! 29 LS
  506. bra 10f
  507. nop
  508. .balign 32
  509. .Lcase2b:
  510. ! Size is at least 64 bytes, so will be going round the big loop at least once.
  511. !
  512. ! r2 = rounded up r4
  513. ! r3 = rounded down r0
  514. mov r0, r3 ! 5 MT (latency=0)
  515. mov #(~0x1f), r1 ! 6 EX
  516. and r1, r3 ! 78 EX
  517. mov r4, r2 ! 5 MT (latency=0)
  518. cmp/eq r3, r0 ! 54 MT
  519. add #0x1f, r2 ! 50 EX
  520. add #-2, r5 ! 50 EX
  521. bt/s 1f ! 110 BR
  522. and r1, r2 ! 78 EX
  523. ! Copy a short word one at a time until we are cache line aligned
  524. ! Normal values: r0, r2, r3, r4
  525. ! Unused: r1, r6, r7
  526. ! Mod: r5 (=r5-2)
  527. !
  528. add #2, r3 ! 50 EX
  529. 2: mov.w @(r0,r5),r1 ! 20 LS (latency=2)
  530. cmp/eq r3,r0 ! 54 MT
  531. bf/s 2b ! 111 BR
  532. mov.w r1,@-r0 ! 29 LS
  533. ! Copy the cache line aligned blocks
  534. !
  535. ! In use: r0, r2, r4, r5 (=r5-2)
  536. ! Scratch: r1, r3, r6, r7
  537. !
  538. ! We could do this with the four scratch registers, but if src
  539. ! and dest hit the same cache line, this will thrash, so make
  540. ! use of additional registers.
  541. !
  542. ! We also need r0 as a temporary (for movca), so 'undo' the invariant:
  543. ! r5: src (was r0+r5)
  544. ! r1: dest (was r0)
  545. ! this can be reversed at the end, so we don't need to save any extra
  546. ! state.
  547. !
  548. 1: mov.l r8, @-r15 ! 30 LS
  549. add r0, r5 ! 49 EX
  550. mov.l r9, @-r15 ! 30 LS
  551. mov r0, r1 ! 5 MT (latency=0)
  552. mov.l r10, @-r15 ! 30 LS
  553. add #-0x1e, r5 ! 50 EX
  554. mov.l r11, @-r15 ! 30 LS
  555. mov.l r12, @-r15 ! 30 LS
  556. ! 17 cycles, 32 bytes per iteration
  557. #ifdef __LITTLE_ENDIAN__
  558. 2: mov.w @r5+, r0 ! 14 LS (latency=2) ..JI
  559. add #-0x20, r1 ! 50 EX
  560. mov.l @r5+, r3 ! 15 LS (latency=2) NMLK
  561. mov.l @r5+, r6 ! 15 LS (latency=2) RQPO
  562. shll16 r0 ! 103 EX JI..
  563. mov.l @r5+, r7 ! 15 LS (latency=2)
  564. xtrct r3, r0 ! 48 EX LKJI
  565. mov.l @r5+, r8 ! 15 LS (latency=2)
  566. xtrct r6, r3 ! 48 EX PONM
  567. mov.l @r5+, r9 ! 15 LS (latency=2)
  568. xtrct r7, r6 ! 48 EX
  569. mov.l @r5+, r10 ! 15 LS (latency=2)
  570. xtrct r8, r7 ! 48 EX
  571. mov.l @r5+, r11 ! 15 LS (latency=2)
  572. xtrct r9, r8 ! 48 EX
  573. mov.w @r5+, r12 ! 15 LS (latency=2)
  574. xtrct r10, r9 ! 48 EX
  575. movca.l r0,@r1 ! 40 LS (latency=3-7)
  576. xtrct r11, r10 ! 48 EX
  577. mov.l r3, @(0x04,r1) ! 33 LS
  578. xtrct r12, r11 ! 48 EX
  579. mov.l r6, @(0x08,r1) ! 33 LS
  580. mov.l r7, @(0x0c,r1) ! 33 LS
  581. mov.l r8, @(0x10,r1) ! 33 LS
  582. add #-0x40, r5 ! 50 EX
  583. mov.l r9, @(0x14,r1) ! 33 LS
  584. cmp/eq r2,r1 ! 54 MT
  585. mov.l r10, @(0x18,r1) ! 33 LS
  586. bf/s 2b ! 109 BR
  587. mov.l r11, @(0x1c,r1) ! 33 LS
  588. #else
  589. 2: mov.w @(0x1e,r5), r0 ! 17 LS (latency=2)
  590. add #-2, r5 ! 50 EX
  591. mov.l @(0x1c,r5), r3 ! 18 LS (latency=2)
  592. add #-4, r1 ! 50 EX
  593. mov.l @(0x18,r5), r6 ! 18 LS (latency=2)
  594. shll16 r0 ! 103 EX
  595. mov.l @(0x14,r5), r7 ! 18 LS (latency=2)
  596. xtrct r3, r0 ! 48 EX
  597. mov.l @(0x10,r5), r8 ! 18 LS (latency=2)
  598. xtrct r6, r3 ! 48 EX
  599. mov.l @(0x0c,r5), r9 ! 18 LS (latency=2)
  600. xtrct r7, r6 ! 48 EX
  601. mov.l @(0x08,r5), r10 ! 18 LS (latency=2)
  602. xtrct r8, r7 ! 48 EX
  603. mov.l @(0x04,r5), r11 ! 18 LS (latency=2)
  604. xtrct r9, r8 ! 48 EX
  605. mov.l @(0x00,r5), r12 ! 18 LS (latency=2)
  606. xtrct r10, r9 ! 48 EX
  607. movca.l r0,@r1 ! 40 LS (latency=3-7)
  608. add #-0x1c, r1 ! 50 EX
  609. mov.l r3, @(0x18,r1) ! 33 LS
  610. xtrct r11, r10 ! 48 EX
  611. mov.l r6, @(0x14,r1) ! 33 LS
  612. xtrct r12, r11 ! 48 EX
  613. mov.l r7, @(0x10,r1) ! 33 LS
  614. mov.l r8, @(0x0c,r1) ! 33 LS
  615. add #-0x1e, r5 ! 50 EX
  616. mov.l r9, @(0x08,r1) ! 33 LS
  617. cmp/eq r2,r1 ! 54 MT
  618. mov.l r10, @(0x04,r1) ! 33 LS
  619. bf/s 2b ! 109 BR
  620. mov.l r11, @(0x00,r1) ! 33 LS
  621. #endif
  622. mov.l @r15+, r12
  623. mov r1, r0 ! 5 MT (latency=0)
  624. mov.l @r15+, r11 ! 15 LS
  625. sub r1, r5 ! 75 EX
  626. mov.l @r15+, r10 ! 15 LS
  627. cmp/eq r4, r0 ! 54 MT
  628. bf/s 1f ! 109 BR
  629. mov.l @r15+, r9 ! 15 LS
  630. rts
  631. 1: mov.l @r15+, r8 ! 15 LS
  632. add #0x1e, r5 ! 50 EX
  633. ! Finish off a short word at a time
  634. ! r5 must be invariant - 2
  635. 10: mov r4,r2 ! 5 MT (latency=0)
  636. add #1,r2 ! 50 EX
  637. cmp/hi r2, r0 ! 57 MT
  638. bf/s 1f ! 109 BR
  639. add #2, r2 ! 50 EX
  640. 3: mov.w @(r0,r5),r1 ! 20 LS
  641. cmp/hi r2,r0 ! 57 MT
  642. bt/s 3b ! 109 BR
  643. mov.w r1,@-r0 ! 29 LS
  644. 1:
  645. !
  646. ! Finally, copy the last byte if necessary
  647. cmp/eq r4,r0 ! 54 MT
  648. bt/s 9b
  649. add #1,r5
  650. mov.b @(r0,r5),r1
  651. rts
  652. mov.b r1,@-r0
  653. END(memcpy)
  654. libc_hidden_def (memcpy)