projects / openwrt / openwrt.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
brcm2708: update linux 4.4 patches to latest version
[openwrt/openwrt.git] / target / linux / brcm2708 / patches-4.4 / 0079-Improve-__copy_to_user-and-__copy_from_user-performa.patch
1 From ffdb5092c571c88a6d924c2c3dbf8bec9d70a3e6 Mon Sep 17 00:00:00 2001
2 From: Phil Elwell <phil@raspberrypi.org>
3 Date: Mon, 13 Oct 2014 11:47:53 +0100
4 Subject: [PATCH] Improve __copy_to_user and __copy_from_user performance
5
6 Provide a __copy_from_user that uses memcpy. On BCM2708, use
7 optimised memcpy/memmove/memcmp/memset implementations.
8
9 arch/arm: Add mmiocpy/set aliases for memcpy/set
10
11 See: https://github.com/raspberrypi/linux/issues/1082
12 ---
13 arch/arm/include/asm/string.h | 5 +
14 arch/arm/include/asm/uaccess.h | 3 +
15 arch/arm/lib/Makefile | 15 +-
16 arch/arm/lib/arm-mem.h | 159 ++++++++++++
17 arch/arm/lib/copy_from_user.S | 4 +-
18 arch/arm/lib/exports_rpi.c | 37 +++
19 arch/arm/lib/memcmp_rpi.S | 285 +++++++++++++++++++++
20 arch/arm/lib/memcpy_rpi.S | 61 +++++
21 arch/arm/lib/memcpymove.h | 506 +++++++++++++++++++++++++++++++++++++
22 arch/arm/lib/memmove_rpi.S | 61 +++++
23 arch/arm/lib/memset_rpi.S | 123 +++++++++
24 arch/arm/lib/uaccess_with_memcpy.c | 112 +++++++-
25 12 files changed, 1365 insertions(+), 6 deletions(-)
26 create mode 100644 arch/arm/lib/arm-mem.h
27 create mode 100644 arch/arm/lib/exports_rpi.c
28 create mode 100644 arch/arm/lib/memcmp_rpi.S
29 create mode 100644 arch/arm/lib/memcpy_rpi.S
30 create mode 100644 arch/arm/lib/memcpymove.h
31 create mode 100644 arch/arm/lib/memmove_rpi.S
32 create mode 100644 arch/arm/lib/memset_rpi.S
33
34 --- a/arch/arm/include/asm/string.h
35 +++ b/arch/arm/include/asm/string.h
36 @@ -24,6 +24,11 @@ extern void * memchr(const void *, int,
37 #define __HAVE_ARCH_MEMSET
38 extern void * memset(void *, int, __kernel_size_t);
39
40 +#ifdef CONFIG_MACH_BCM2708
41 +#define __HAVE_ARCH_MEMCMP
42 +extern int memcmp(const void *, const void *, size_t);
43 +#endif
44 +
45 extern void __memzero(void *ptr, __kernel_size_t n);
46
47 #define memset(p,v,n) \
48 --- a/arch/arm/include/asm/uaccess.h
49 +++ b/arch/arm/include/asm/uaccess.h
50 @@ -493,6 +493,9 @@ do { \
51 extern unsigned long __must_check
52 arm_copy_from_user(void *to, const void __user *from, unsigned long n);
53
54 +extern unsigned long __must_check
55 +__copy_from_user_std(void *to, const void __user *from, unsigned long n);
56 +
57 static inline unsigned long __must_check
58 __copy_from_user(void *to, const void __user *from, unsigned long n)
59 {
60 --- a/arch/arm/lib/Makefile
61 +++ b/arch/arm/lib/Makefile
62 @@ -6,9 +6,8 @@
63
64 lib-y := backtrace.o changebit.o csumipv6.o csumpartial.o \
65 csumpartialcopy.o csumpartialcopyuser.o clearbit.o \
66 - delay.o delay-loop.o findbit.o memchr.o memcpy.o \
67 - memmove.o memset.o memzero.o setbit.o \
68 - strchr.o strrchr.o \
69 + delay.o delay-loop.o findbit.o memchr.o memzero.o \
70 + setbit.o strchr.o strrchr.o \
71 testchangebit.o testclearbit.o testsetbit.o \
72 ashldi3.o ashrdi3.o lshrdi3.o muldi3.o \
73 ucmpdi2.o lib1funcs.o div64.o \
74 @@ -18,6 +17,16 @@ lib-y := backtrace.o changebit.o csumip
75 mmu-y := clear_user.o copy_page.o getuser.o putuser.o \
76 copy_from_user.o copy_to_user.o
77
78 +# Choose optimised implementations for Raspberry Pi
79 +ifeq ($(CONFIG_MACH_BCM2708),y)
80 + CFLAGS_uaccess_with_memcpy.o += -DCOPY_FROM_USER_THRESHOLD=1600
81 + CFLAGS_uaccess_with_memcpy.o += -DCOPY_TO_USER_THRESHOLD=672
82 + obj-$(CONFIG_MODULES) += exports_rpi.o
83 + lib-y += memcpy_rpi.o memmove_rpi.o memset_rpi.o memcmp_rpi.o
84 +else
85 + lib-y += memcpy.o memmove.o memset.o
86 +endif
87 +
88 # using lib_ here won't override already available weak symbols
89 obj-$(CONFIG_UACCESS_WITH_MEMCPY) += uaccess_with_memcpy.o
90
91 --- /dev/null
92 +++ b/arch/arm/lib/arm-mem.h
93 @@ -0,0 +1,159 @@
94 +/*
95 +Copyright (c) 2013, Raspberry Pi Foundation
96 +Copyright (c) 2013, RISC OS Open Ltd
97 +All rights reserved.
98 +
99 +Redistribution and use in source and binary forms, with or without
100 +modification, are permitted provided that the following conditions are met:
101 + * Redistributions of source code must retain the above copyright
102 + notice, this list of conditions and the following disclaimer.
103 + * Redistributions in binary form must reproduce the above copyright
104 + notice, this list of conditions and the following disclaimer in the
105 + documentation and/or other materials provided with the distribution.
106 + * Neither the name of the copyright holder nor the
107 + names of its contributors may be used to endorse or promote products
108 + derived from this software without specific prior written permission.
109 +
110 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
111 +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
112 +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
113 +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
114 +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
115 +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
116 +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
117 +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
118 +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
119 +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
120 +*/
121 +
122 +.macro myfunc fname
123 + .func fname
124 + .global fname
125 +fname:
126 +.endm
127 +
128 +.macro preload_leading_step1 backwards, ptr, base
129 +/* If the destination is already 16-byte aligned, then we need to preload
130 + * between 0 and prefetch_distance (inclusive) cache lines ahead so there
131 + * are no gaps when the inner loop starts.
132 + */
133 + .if backwards
134 + sub ptr, base, #1
135 + bic ptr, ptr, #31
136 + .else
137 + bic ptr, base, #31
138 + .endif
139 + .set OFFSET, 0
140 + .rept prefetch_distance+1
141 + pld [ptr, #OFFSET]
142 + .if backwards
143 + .set OFFSET, OFFSET-32
144 + .else
145 + .set OFFSET, OFFSET+32
146 + .endif
147 + .endr
148 +.endm
149 +
150 +.macro preload_leading_step2 backwards, ptr, base, leading_bytes, tmp
151 +/* However, if the destination is not 16-byte aligned, we may need to
152 + * preload one more cache line than that. The question we need to ask is:
153 + * are the leading bytes more than the amount by which the source
154 + * pointer will be rounded down for preloading, and if so, by how many
155 + * cache lines?
156 + */
157 + .if backwards
158 +/* Here we compare against how many bytes we are into the
159 + * cache line, counting down from the highest such address.
160 + * Effectively, we want to calculate
161 + * leading_bytes = dst&15
162 + * cacheline_offset = 31-((src-leading_bytes-1)&31)
163 + * extra_needed = leading_bytes - cacheline_offset
164 + * and test if extra_needed is <= 0, or rearranging:
165 + * leading_bytes + (src-leading_bytes-1)&31 <= 31
166 + */
167 + mov tmp, base, lsl #32-5
168 + sbc tmp, tmp, leading_bytes, lsl #32-5
169 + adds tmp, tmp, leading_bytes, lsl #32-5
170 + bcc 61f
171 + pld [ptr, #-32*(prefetch_distance+1)]
172 + .else
173 +/* Effectively, we want to calculate
174 + * leading_bytes = (-dst)&15
175 + * cacheline_offset = (src+leading_bytes)&31
176 + * extra_needed = leading_bytes - cacheline_offset
177 + * and test if extra_needed is <= 0.
178 + */
179 + mov tmp, base, lsl #32-5
180 + add tmp, tmp, leading_bytes, lsl #32-5
181 + rsbs tmp, tmp, leading_bytes, lsl #32-5
182 + bls 61f
183 + pld [ptr, #32*(prefetch_distance+1)]
184 + .endif
185 +61:
186 +.endm
187 +
188 +.macro preload_trailing backwards, base, remain, tmp
189 + /* We need either 0, 1 or 2 extra preloads */
190 + .if backwards
191 + rsb tmp, base, #0
192 + mov tmp, tmp, lsl #32-5
193 + .else
194 + mov tmp, base, lsl #32-5
195 + .endif
196 + adds tmp, tmp, remain, lsl #32-5
197 + adceqs tmp, tmp, #0
198 + /* The instruction above has two effects: ensures Z is only
199 + * set if C was clear (so Z indicates that both shifted quantities
200 + * were 0), and clears C if Z was set (so C indicates that the sum
201 + * of the shifted quantities was greater and not equal to 32) */
202 + beq 82f
203 + .if backwards
204 + sub tmp, base, #1
205 + bic tmp, tmp, #31
206 + .else
207 + bic tmp, base, #31
208 + .endif
209 + bcc 81f
210 + .if backwards
211 + pld [tmp, #-32*(prefetch_distance+1)]
212 +81:
213 + pld [tmp, #-32*prefetch_distance]
214 + .else
215 + pld [tmp, #32*(prefetch_distance+2)]
216 +81:
217 + pld [tmp, #32*(prefetch_distance+1)]
218 + .endif
219 +82:
220 +.endm
221 +
222 +.macro preload_all backwards, narrow_case, shift, base, remain, tmp0, tmp1
223 + .if backwards
224 + sub tmp0, base, #1
225 + bic tmp0, tmp0, #31
226 + pld [tmp0]
227 + sub tmp1, base, remain, lsl #shift
228 + .else
229 + bic tmp0, base, #31
230 + pld [tmp0]
231 + add tmp1, base, remain, lsl #shift
232 + sub tmp1, tmp1, #1
233 + .endif
234 + bic tmp1, tmp1, #31
235 + cmp tmp1, tmp0
236 + beq 92f
237 + .if narrow_case
238 + /* In this case, all the data fits in either 1 or 2 cache lines */
239 + pld [tmp1]
240 + .else
241 +91:
242 + .if backwards
243 + sub tmp0, tmp0, #32
244 + .else
245 + add tmp0, tmp0, #32
246 + .endif
247 + cmp tmp0, tmp1
248 + pld [tmp0]
249 + bne 91b
250 + .endif
251 +92:
252 +.endm
253 --- a/arch/arm/lib/copy_from_user.S
254 +++ b/arch/arm/lib/copy_from_user.S
255 @@ -89,11 +89,13 @@
256
257 .text
258
259 -ENTRY(arm_copy_from_user)
260 +ENTRY(__copy_from_user_std)
261 +WEAK(arm_copy_from_user)
262
263 #include "copy_template.S"
264
265 ENDPROC(arm_copy_from_user)
266 +ENDPROC(__copy_from_user_std)
267
268 .pushsection .fixup,"ax"
269 .align 0
270 --- /dev/null
271 +++ b/arch/arm/lib/exports_rpi.c
272 @@ -0,0 +1,37 @@
273 +/**
274 + * Copyright (c) 2014, Raspberry Pi (Trading) Ltd.
275 + *
276 + * Redistribution and use in source and binary forms, with or without
277 + * modification, are permitted provided that the following conditions
278 + * are met:
279 + * 1. Redistributions of source code must retain the above copyright
280 + * notice, this list of conditions, and the following disclaimer,
281 + * without modification.
282 + * 2. Redistributions in binary form must reproduce the above copyright
283 + * notice, this list of conditions and the following disclaimer in the
284 + * documentation and/or other materials provided with the distribution.
285 + * 3. The names of the above-listed copyright holders may not be used
286 + * to endorse or promote products derived from this software without
287 + * specific prior written permission.
288 + *
289 + * ALTERNATIVELY, this software may be distributed under the terms of the
290 + * GNU General Public License ("GPL") version 2, as published by the Free
291 + * Software Foundation.
292 + *
293 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
294 + * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
295 + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
296 + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
297 + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
298 + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
299 + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
300 + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
301 + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
302 + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
303 + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
304 + */
305 +
306 +#include <linux/kernel.h>
307 +#include <linux/module.h>
308 +
309 +EXPORT_SYMBOL(memcmp);
310 --- /dev/null
311 +++ b/arch/arm/lib/memcmp_rpi.S
312 @@ -0,0 +1,285 @@
313 +/*
314 +Copyright (c) 2013, Raspberry Pi Foundation
315 +Copyright (c) 2013, RISC OS Open Ltd
316 +All rights reserved.
317 +
318 +Redistribution and use in source and binary forms, with or without
319 +modification, are permitted provided that the following conditions are met:
320 + * Redistributions of source code must retain the above copyright
321 + notice, this list of conditions and the following disclaimer.
322 + * Redistributions in binary form must reproduce the above copyright
323 + notice, this list of conditions and the following disclaimer in the
324 + documentation and/or other materials provided with the distribution.
325 + * Neither the name of the copyright holder nor the
326 + names of its contributors may be used to endorse or promote products
327 + derived from this software without specific prior written permission.
328 +
329 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
330 +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
331 +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
332 +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
333 +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
334 +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
335 +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
336 +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
337 +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
338 +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
339 +*/
340 +
341 +#include <linux/linkage.h>
342 +#include "arm-mem.h"
343 +
344 +/* Prevent the stack from becoming executable */
345 +#if defined(__linux__) && defined(__ELF__)
346 +.section .note.GNU-stack,"",%progbits
347 +#endif
348 +
349 + .text
350 + .arch armv6
351 + .object_arch armv4
352 + .arm
353 + .altmacro
354 + .p2align 2
355 +
356 +.macro memcmp_process_head unaligned
357 + .if unaligned
358 + ldr DAT0, [S_1], #4
359 + ldr DAT1, [S_1], #4
360 + ldr DAT2, [S_1], #4
361 + ldr DAT3, [S_1], #4
362 + .else
363 + ldmia S_1!, {DAT0, DAT1, DAT2, DAT3}
364 + .endif
365 + ldmia S_2!, {DAT4, DAT5, DAT6, DAT7}
366 +.endm
367 +
368 +.macro memcmp_process_tail
369 + cmp DAT0, DAT4
370 + cmpeq DAT1, DAT5
371 + cmpeq DAT2, DAT6
372 + cmpeq DAT3, DAT7
373 + bne 200f
374 +.endm
375 +
376 +.macro memcmp_leading_31bytes
377 + movs DAT0, OFF, lsl #31
378 + ldrmib DAT0, [S_1], #1
379 + ldrcsh DAT1, [S_1], #2
380 + ldrmib DAT4, [S_2], #1
381 + ldrcsh DAT5, [S_2], #2
382 + movpl DAT0, #0
383 + movcc DAT1, #0
384 + movpl DAT4, #0
385 + movcc DAT5, #0
386 + submi N, N, #1
387 + subcs N, N, #2
388 + cmp DAT0, DAT4
389 + cmpeq DAT1, DAT5
390 + bne 200f
391 + movs DAT0, OFF, lsl #29
392 + ldrmi DAT0, [S_1], #4
393 + ldrcs DAT1, [S_1], #4
394 + ldrcs DAT2, [S_1], #4
395 + ldrmi DAT4, [S_2], #4
396 + ldmcsia S_2!, {DAT5, DAT6}
397 + movpl DAT0, #0
398 + movcc DAT1, #0
399 + movcc DAT2, #0
400 + movpl DAT4, #0
401 + movcc DAT5, #0
402 + movcc DAT6, #0
403 + submi N, N, #4
404 + subcs N, N, #8
405 + cmp DAT0, DAT4
406 + cmpeq DAT1, DAT5
407 + cmpeq DAT2, DAT6
408 + bne 200f
409 + tst OFF, #16
410 + beq 105f
411 + memcmp_process_head 1
412 + sub N, N, #16
413 + memcmp_process_tail
414 +105:
415 +.endm
416 +
417 +.macro memcmp_trailing_15bytes unaligned
418 + movs N, N, lsl #29
419 + .if unaligned
420 + ldrcs DAT0, [S_1], #4
421 + ldrcs DAT1, [S_1], #4
422 + .else
423 + ldmcsia S_1!, {DAT0, DAT1}
424 + .endif
425 + ldrmi DAT2, [S_1], #4
426 + ldmcsia S_2!, {DAT4, DAT5}
427 + ldrmi DAT6, [S_2], #4
428 + movcc DAT0, #0
429 + movcc DAT1, #0
430 + movpl DAT2, #0
431 + movcc DAT4, #0
432 + movcc DAT5, #0
433 + movpl DAT6, #0
434 + cmp DAT0, DAT4
435 + cmpeq DAT1, DAT5
436 + cmpeq DAT2, DAT6
437 + bne 200f
438 + movs N, N, lsl #2
439 + ldrcsh DAT0, [S_1], #2
440 + ldrmib DAT1, [S_1]
441 + ldrcsh DAT4, [S_2], #2
442 + ldrmib DAT5, [S_2]
443 + movcc DAT0, #0
444 + movpl DAT1, #0
445 + movcc DAT4, #0
446 + movpl DAT5, #0
447 + cmp DAT0, DAT4
448 + cmpeq DAT1, DAT5
449 + bne 200f
450 +.endm
451 +
452 +.macro memcmp_long_inner_loop unaligned
453 +110:
454 + memcmp_process_head unaligned
455 + pld [S_2, #prefetch_distance*32 + 16]
456 + memcmp_process_tail
457 + memcmp_process_head unaligned
458 + pld [S_1, OFF]
459 + memcmp_process_tail
460 + subs N, N, #32
461 + bhs 110b
462 + /* Just before the final (prefetch_distance+1) 32-byte blocks,
463 + * deal with final preloads */
464 + preload_trailing 0, S_1, N, DAT0
465 + preload_trailing 0, S_2, N, DAT0
466 + add N, N, #(prefetch_distance+2)*32 - 16
467 +120:
468 + memcmp_process_head unaligned
469 + memcmp_process_tail
470 + subs N, N, #16
471 + bhs 120b
472 + /* Trailing words and bytes */
473 + tst N, #15
474 + beq 199f
475 + memcmp_trailing_15bytes unaligned
476 +199: /* Reached end without detecting a difference */
477 + mov a1, #0
478 + setend le
479 + pop {DAT1-DAT6, pc}
480 +.endm
481 +
482 +.macro memcmp_short_inner_loop unaligned
483 + subs N, N, #16 /* simplifies inner loop termination */
484 + blo 122f
485 +120:
486 + memcmp_process_head unaligned
487 + memcmp_process_tail
488 + subs N, N, #16
489 + bhs 120b
490 +122: /* Trailing words and bytes */
491 + tst N, #15
492 + beq 199f
493 + memcmp_trailing_15bytes unaligned
494 +199: /* Reached end without detecting a difference */
495 + mov a1, #0
496 + setend le
497 + pop {DAT1-DAT6, pc}
498 +.endm
499 +
500 +/*
501 + * int memcmp(const void *s1, const void *s2, size_t n);
502 + * On entry:
503 + * a1 = pointer to buffer 1
504 + * a2 = pointer to buffer 2
505 + * a3 = number of bytes to compare (as unsigned chars)
506 + * On exit:
507 + * a1 = >0/=0/<0 if s1 >/=/< s2
508 + */
509 +
510 +.set prefetch_distance, 2
511 +
512 +ENTRY(memcmp)
513 + S_1 .req a1
514 + S_2 .req a2
515 + N .req a3
516 + DAT0 .req a4
517 + DAT1 .req v1
518 + DAT2 .req v2
519 + DAT3 .req v3
520 + DAT4 .req v4
521 + DAT5 .req v5
522 + DAT6 .req v6
523 + DAT7 .req ip
524 + OFF .req lr
525 +
526 + push {DAT1-DAT6, lr}
527 + setend be /* lowest-addressed bytes are most significant */
528 +
529 + /* To preload ahead as we go, we need at least (prefetch_distance+2) 32-byte blocks */
530 + cmp N, #(prefetch_distance+3)*32 - 1
531 + blo 170f
532 +
533 + /* Long case */
534 + /* Adjust N so that the decrement instruction can also test for
535 + * inner loop termination. We want it to stop when there are
536 + * (prefetch_distance+1) complete blocks to go. */
537 + sub N, N, #(prefetch_distance+2)*32
538 + preload_leading_step1 0, DAT0, S_1
539 + preload_leading_step1 0, DAT1, S_2
540 + tst S_2, #31
541 + beq 154f
542 + rsb OFF, S_2, #0 /* no need to AND with 15 here */
543 + preload_leading_step2 0, DAT0, S_1, OFF, DAT2
544 + preload_leading_step2 0, DAT1, S_2, OFF, DAT2
545 + memcmp_leading_31bytes
546 +154: /* Second source now cacheline (32-byte) aligned; we have at
547 + * least one prefetch to go. */
548 + /* Prefetch offset is best selected such that it lies in the
549 + * first 8 of each 32 bytes - but it's just as easy to aim for
550 + * the first one */
551 + and OFF, S_1, #31
552 + rsb OFF, OFF, #32*prefetch_distance
553 + tst S_1, #3
554 + bne 140f
555 + memcmp_long_inner_loop 0
556 +140: memcmp_long_inner_loop 1
557 +
558 +170: /* Short case */
559 + teq N, #0
560 + beq 199f
561 + preload_all 0, 0, 0, S_1, N, DAT0, DAT1
562 + preload_all 0, 0, 0, S_2, N, DAT0, DAT1
563 + tst S_2, #3
564 + beq 174f
565 +172: subs N, N, #1
566 + blo 199f
567 + ldrb DAT0, [S_1], #1
568 + ldrb DAT4, [S_2], #1
569 + cmp DAT0, DAT4
570 + bne 200f
571 + tst S_2, #3
572 + bne 172b
573 +174: /* Second source now 4-byte aligned; we have 0 or more bytes to go */
574 + tst S_1, #3
575 + bne 140f
576 + memcmp_short_inner_loop 0
577 +140: memcmp_short_inner_loop 1
578 +
579 +200: /* Difference found: determine sign. */
580 + movhi a1, #1
581 + movlo a1, #-1
582 + setend le
583 + pop {DAT1-DAT6, pc}
584 +
585 + .unreq S_1
586 + .unreq S_2
587 + .unreq N
588 + .unreq DAT0
589 + .unreq DAT1
590 + .unreq DAT2
591 + .unreq DAT3
592 + .unreq DAT4
593 + .unreq DAT5
594 + .unreq DAT6
595 + .unreq DAT7
596 + .unreq OFF
597 +ENDPROC(memcmp)
598 --- /dev/null
599 +++ b/arch/arm/lib/memcpy_rpi.S
600 @@ -0,0 +1,61 @@
601 +/*
602 +Copyright (c) 2013, Raspberry Pi Foundation
603 +Copyright (c) 2013, RISC OS Open Ltd
604 +All rights reserved.
605 +
606 +Redistribution and use in source and binary forms, with or without
607 +modification, are permitted provided that the following conditions are met:
608 + * Redistributions of source code must retain the above copyright
609 + notice, this list of conditions and the following disclaimer.
610 + * Redistributions in binary form must reproduce the above copyright
611 + notice, this list of conditions and the following disclaimer in the
612 + documentation and/or other materials provided with the distribution.
613 + * Neither the name of the copyright holder nor the
614 + names of its contributors may be used to endorse or promote products
615 + derived from this software without specific prior written permission.
616 +
617 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
618 +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
619 +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
620 +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
621 +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
622 +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
623 +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
624 +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
625 +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
626 +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
627 +*/
628 +
629 +#include <linux/linkage.h>
630 +#include "arm-mem.h"
631 +#include "memcpymove.h"
632 +
633 +/* Prevent the stack from becoming executable */
634 +#if defined(__linux__) && defined(__ELF__)
635 +.section .note.GNU-stack,"",%progbits
636 +#endif
637 +
638 + .text
639 + .arch armv6
640 + .object_arch armv4
641 + .arm
642 + .altmacro
643 + .p2align 2
644 +
645 +/*
646 + * void *memcpy(void * restrict s1, const void * restrict s2, size_t n);
647 + * On entry:
648 + * a1 = pointer to destination
649 + * a2 = pointer to source
650 + * a3 = number of bytes to copy
651 + * On exit:
652 + * a1 preserved
653 + */
654 +
655 +.set prefetch_distance, 3
656 +
657 +ENTRY(mmiocpy)
658 +ENTRY(memcpy)
659 + memcpy 0
660 +ENDPROC(memcpy)
661 +ENDPROC(mmiocpy)
662 --- /dev/null
663 +++ b/arch/arm/lib/memcpymove.h
664 @@ -0,0 +1,506 @@
665 +/*
666 +Copyright (c) 2013, Raspberry Pi Foundation
667 +Copyright (c) 2013, RISC OS Open Ltd
668 +All rights reserved.
669 +
670 +Redistribution and use in source and binary forms, with or without
671 +modification, are permitted provided that the following conditions are met:
672 + * Redistributions of source code must retain the above copyright
673 + notice, this list of conditions and the following disclaimer.
674 + * Redistributions in binary form must reproduce the above copyright
675 + notice, this list of conditions and the following disclaimer in the
676 + documentation and/or other materials provided with the distribution.
677 + * Neither the name of the copyright holder nor the
678 + names of its contributors may be used to endorse or promote products
679 + derived from this software without specific prior written permission.
680 +
681 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
682 +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
683 +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
684 +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
685 +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
686 +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
687 +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
688 +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
689 +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
690 +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
691 +*/
692 +
693 +.macro unaligned_words backwards, align, use_pld, words, r0, r1, r2, r3, r4, r5, r6, r7, r8
694 + .if words == 1
695 + .if backwards
696 + mov r1, r0, lsl #32-align*8
697 + ldr r0, [S, #-4]!
698 + orr r1, r1, r0, lsr #align*8
699 + str r1, [D, #-4]!
700 + .else
701 + mov r0, r1, lsr #align*8
702 + ldr r1, [S, #4]!
703 + orr r0, r0, r1, lsl #32-align*8
704 + str r0, [D], #4
705 + .endif
706 + .elseif words == 2
707 + .if backwards
708 + ldr r1, [S, #-4]!
709 + mov r2, r0, lsl #32-align*8
710 + ldr r0, [S, #-4]!
711 + orr r2, r2, r1, lsr #align*8
712 + mov r1, r1, lsl #32-align*8
713 + orr r1, r1, r0, lsr #align*8
714 + stmdb D!, {r1, r2}
715 + .else
716 + ldr r1, [S, #4]!
717 + mov r0, r2, lsr #align*8
718 + ldr r2, [S, #4]!
719 + orr r0, r0, r1, lsl #32-align*8
720 + mov r1, r1, lsr #align*8
721 + orr r1, r1, r2, lsl #32-align*8
722 + stmia D!, {r0, r1}
723 + .endif
724 + .elseif words == 4
725 + .if backwards
726 + ldmdb S!, {r2, r3}
727 + mov r4, r0, lsl #32-align*8
728 + ldmdb S!, {r0, r1}
729 + orr r4, r4, r3, lsr #align*8
730 + mov r3, r3, lsl #32-align*8
731 + orr r3, r3, r2, lsr #align*8
732 + mov r2, r2, lsl #32-align*8
733 + orr r2, r2, r1, lsr #align*8
734 + mov r1, r1, lsl #32-align*8
735 + orr r1, r1, r0, lsr #align*8
736 + stmdb D!, {r1, r2, r3, r4}
737 + .else
738 + ldmib S!, {r1, r2}
739 + mov r0, r4, lsr #align*8
740 + ldmib S!, {r3, r4}
741 + orr r0, r0, r1, lsl #32-align*8
742 + mov r1, r1, lsr #align*8
743 + orr r1, r1, r2, lsl #32-align*8
744 + mov r2, r2, lsr #align*8
745 + orr r2, r2, r3, lsl #32-align*8
746 + mov r3, r3, lsr #align*8
747 + orr r3, r3, r4, lsl #32-align*8
748 + stmia D!, {r0, r1, r2, r3}
749 + .endif
750 + .elseif words == 8
751 + .if backwards
752 + ldmdb S!, {r4, r5, r6, r7}
753 + mov r8, r0, lsl #32-align*8
754 + ldmdb S!, {r0, r1, r2, r3}
755 + .if use_pld
756 + pld [S, OFF]
757 + .endif
758 + orr r8, r8, r7, lsr #align*8
759 + mov r7, r7, lsl #32-align*8
760 + orr r7, r7, r6, lsr #align*8
761 + mov r6, r6, lsl #32-align*8
762 + orr r6, r6, r5, lsr #align*8
763 + mov r5, r5, lsl #32-align*8
764 + orr r5, r5, r4, lsr #align*8
765 + mov r4, r4, lsl #32-align*8
766 + orr r4, r4, r3, lsr #align*8
767 + mov r3, r3, lsl #32-align*8
768 + orr r3, r3, r2, lsr #align*8
769 + mov r2, r2, lsl #32-align*8
770 + orr r2, r2, r1, lsr #align*8
771 + mov r1, r1, lsl #32-align*8
772 + orr r1, r1, r0, lsr #align*8
773 + stmdb D!, {r5, r6, r7, r8}
774 + stmdb D!, {r1, r2, r3, r4}
775 + .else
776 + ldmib S!, {r1, r2, r3, r4}
777 + mov r0, r8, lsr #align*8
778 + ldmib S!, {r5, r6, r7, r8}
779 + .if use_pld
780 + pld [S, OFF]
781 + .endif
782 + orr r0, r0, r1, lsl #32-align*8
783 + mov r1, r1, lsr #align*8
784 + orr r1, r1, r2, lsl #32-align*8
785 + mov r2, r2, lsr #align*8
786 + orr r2, r2, r3, lsl #32-align*8
787 + mov r3, r3, lsr #align*8
788 + orr r3, r3, r4, lsl #32-align*8
789 + mov r4, r4, lsr #align*8
790 + orr r4, r4, r5, lsl #32-align*8
791 + mov r5, r5, lsr #align*8
792 + orr r5, r5, r6, lsl #32-align*8
793 + mov r6, r6, lsr #align*8
794 + orr r6, r6, r7, lsl #32-align*8
795 + mov r7, r7, lsr #align*8
796 + orr r7, r7, r8, lsl #32-align*8
797 + stmia D!, {r0, r1, r2, r3}
798 + stmia D!, {r4, r5, r6, r7}
799 + .endif
800 + .endif
801 +.endm
802 +
803 +.macro memcpy_leading_15bytes backwards, align
804 + movs DAT1, DAT2, lsl #31
805 + sub N, N, DAT2
806 + .if backwards
807 + ldrmib DAT0, [S, #-1]!
808 + ldrcsh DAT1, [S, #-2]!
809 + strmib DAT0, [D, #-1]!
810 + strcsh DAT1, [D, #-2]!
811 + .else
812 + ldrmib DAT0, [S], #1
813 + ldrcsh DAT1, [S], #2
814 + strmib DAT0, [D], #1
815 + strcsh DAT1, [D], #2
816 + .endif
817 + movs DAT1, DAT2, lsl #29
818 + .if backwards
819 + ldrmi DAT0, [S, #-4]!
820 + .if align == 0
821 + ldmcsdb S!, {DAT1, DAT2}
822 + .else
823 + ldrcs DAT2, [S, #-4]!
824 + ldrcs DAT1, [S, #-4]!
825 + .endif
826 + strmi DAT0, [D, #-4]!
827 + stmcsdb D!, {DAT1, DAT2}
828 + .else
829 + ldrmi DAT0, [S], #4
830 + .if align == 0
831 + ldmcsia S!, {DAT1, DAT2}
832 + .else
833 + ldrcs DAT1, [S], #4
834 + ldrcs DAT2, [S], #4
835 + .endif
836 + strmi DAT0, [D], #4
837 + stmcsia D!, {DAT1, DAT2}
838 + .endif
839 +.endm
840 +
841 +.macro memcpy_trailing_15bytes backwards, align
842 + movs N, N, lsl #29
843 + .if backwards
844 + .if align == 0
845 + ldmcsdb S!, {DAT0, DAT1}
846 + .else
847 + ldrcs DAT1, [S, #-4]!
848 + ldrcs DAT0, [S, #-4]!
849 + .endif
850 + ldrmi DAT2, [S, #-4]!
851 + stmcsdb D!, {DAT0, DAT1}
852 + strmi DAT2, [D, #-4]!
853 + .else
854 + .if align == 0
855 + ldmcsia S!, {DAT0, DAT1}
856 + .else
857 + ldrcs DAT0, [S], #4
858 + ldrcs DAT1, [S], #4
859 + .endif
860 + ldrmi DAT2, [S], #4
861 + stmcsia D!, {DAT0, DAT1}
862 + strmi DAT2, [D], #4
863 + .endif
864 + movs N, N, lsl #2
865 + .if backwards
866 + ldrcsh DAT0, [S, #-2]!
867 + ldrmib DAT1, [S, #-1]
868 + strcsh DAT0, [D, #-2]!
869 + strmib DAT1, [D, #-1]
870 + .else
871 + ldrcsh DAT0, [S], #2
872 + ldrmib DAT1, [S]
873 + strcsh DAT0, [D], #2
874 + strmib DAT1, [D]
875 + .endif
876 +.endm
877 +
878 +.macro memcpy_long_inner_loop backwards, align
879 + .if align != 0
880 + .if backwards
881 + ldr DAT0, [S, #-align]!
882 + .else
883 + ldr LAST, [S, #-align]!
884 + .endif
885 + .endif
886 +110:
887 + .if align == 0
888 + .if backwards
889 + ldmdb S!, {DAT0, DAT1, DAT2, DAT3, DAT4, DAT5, DAT6, LAST}
890 + pld [S, OFF]
891 + stmdb D!, {DAT4, DAT5, DAT6, LAST}
892 + stmdb D!, {DAT0, DAT1, DAT2, DAT3}
893 + .else
894 + ldmia S!, {DAT0, DAT1, DAT2, DAT3, DAT4, DAT5, DAT6, LAST}
895 + pld [S, OFF]
896 + stmia D!, {DAT0, DAT1, DAT2, DAT3}
897 + stmia D!, {DAT4, DAT5, DAT6, LAST}
898 + .endif
899 + .else
900 + unaligned_words backwards, align, 1, 8, DAT0, DAT1, DAT2, DAT3, DAT4, DAT5, DAT6, DAT7, LAST
901 + .endif
902 + subs N, N, #32
903 + bhs 110b
904 + /* Just before the final (prefetch_distance+1) 32-byte blocks, deal with final preloads */
905 + preload_trailing backwards, S, N, OFF
906 + add N, N, #(prefetch_distance+2)*32 - 32
907 +120:
908 + .if align == 0
909 + .if backwards
910 + ldmdb S!, {DAT0, DAT1, DAT2, DAT3, DAT4, DAT5, DAT6, LAST}
911 + stmdb D!, {DAT4, DAT5, DAT6, LAST}
912 + stmdb D!, {DAT0, DAT1, DAT2, DAT3}
913 + .else
914 + ldmia S!, {DAT0, DAT1, DAT2, DAT3, DAT4, DAT5, DAT6, LAST}
915 + stmia D!, {DAT0, DAT1, DAT2, DAT3}
916 + stmia D!, {DAT4, DAT5, DAT6, LAST}
917 + .endif
918 + .else
919 + unaligned_words backwards, align, 0, 8, DAT0, DAT1, DAT2, DAT3, DAT4, DAT5, DAT6, DAT7, LAST
920 + .endif
921 + subs N, N, #32
922 + bhs 120b
923 + tst N, #16
924 + .if align == 0
925 + .if backwards
926 + ldmnedb S!, {DAT0, DAT1, DAT2, LAST}
927 + stmnedb D!, {DAT0, DAT1, DAT2, LAST}
928 + .else
929 + ldmneia S!, {DAT0, DAT1, DAT2, LAST}
930 + stmneia D!, {DAT0, DAT1, DAT2, LAST}
931 + .endif
932 + .else
933 + beq 130f
934 + unaligned_words backwards, align, 0, 4, DAT0, DAT1, DAT2, DAT3, LAST
935 +130:
936 + .endif
937 + /* Trailing words and bytes */
938 + tst N, #15
939 + beq 199f
940 + .if align != 0
941 + add S, S, #align
942 + .endif
943 + memcpy_trailing_15bytes backwards, align
944 +199:
945 + pop {DAT3, DAT4, DAT5, DAT6, DAT7}
946 + pop {D, DAT1, DAT2, pc}
947 +.endm
948 +
949 +.macro memcpy_medium_inner_loop backwards, align
950 +120:
951 + .if backwards
952 + .if align == 0
953 + ldmdb S!, {DAT0, DAT1, DAT2, LAST}
954 + .else
955 + ldr LAST, [S, #-4]!
956 + ldr DAT2, [S, #-4]!
957 + ldr DAT1, [S, #-4]!
958 + ldr DAT0, [S, #-4]!
959 + .endif
960 + stmdb D!, {DAT0, DAT1, DAT2, LAST}
961 + .else
962 + .if align == 0
963 + ldmia S!, {DAT0, DAT1, DAT2, LAST}
964 + .else
965 + ldr DAT0, [S], #4
966 + ldr DAT1, [S], #4
967 + ldr DAT2, [S], #4
968 + ldr LAST, [S], #4
969 + .endif
970 + stmia D!, {DAT0, DAT1, DAT2, LAST}
971 + .endif
972 + subs N, N, #16
973 + bhs 120b
974 + /* Trailing words and bytes */
975 + tst N, #15
976 + beq 199f
977 + memcpy_trailing_15bytes backwards, align
978 +199:
979 + pop {D, DAT1, DAT2, pc}
980 +.endm
981 +
982 +.macro memcpy_short_inner_loop backwards, align
983 + tst N, #16
984 + .if backwards
985 + .if align == 0
986 + ldmnedb S!, {DAT0, DAT1, DAT2, LAST}
987 + .else
988 + ldrne LAST, [S, #-4]!
989 + ldrne DAT2, [S, #-4]!
990 + ldrne DAT1, [S, #-4]!
991 + ldrne DAT0, [S, #-4]!
992 + .endif
993 + stmnedb D!, {DAT0, DAT1, DAT2, LAST}
994 + .else
995 + .if align == 0
996 + ldmneia S!, {DAT0, DAT1, DAT2, LAST}
997 + .else
998 + ldrne DAT0, [S], #4
999 + ldrne DAT1, [S], #4
1000 + ldrne DAT2, [S], #4
1001 + ldrne LAST, [S], #4
1002 + .endif
1003 + stmneia D!, {DAT0, DAT1, DAT2, LAST}
1004 + .endif
1005 + memcpy_trailing_15bytes backwards, align
1006 +199:
1007 + pop {D, DAT1, DAT2, pc}
1008 +.endm
1009 +
1010 +.macro memcpy backwards
1011 + D .req a1
1012 + S .req a2
1013 + N .req a3
1014 + DAT0 .req a4
1015 + DAT1 .req v1
1016 + DAT2 .req v2
1017 + DAT3 .req v3
1018 + DAT4 .req v4
1019 + DAT5 .req v5
1020 + DAT6 .req v6
1021 + DAT7 .req sl
1022 + LAST .req ip
1023 + OFF .req lr
1024 +
1025 + .cfi_startproc
1026 +
1027 + push {D, DAT1, DAT2, lr}
1028 +
1029 + .cfi_def_cfa_offset 16
1030 + .cfi_rel_offset D, 0
1031 + .cfi_undefined S
1032 + .cfi_undefined N
1033 + .cfi_undefined DAT0
1034 + .cfi_rel_offset DAT1, 4
1035 + .cfi_rel_offset DAT2, 8
1036 + .cfi_undefined LAST
1037 + .cfi_rel_offset lr, 12
1038 +
1039 + .if backwards
1040 + add D, D, N
1041 + add S, S, N
1042 + .endif
1043 +
1044 + /* See if we're guaranteed to have at least one 16-byte aligned 16-byte write */
1045 + cmp N, #31
1046 + blo 170f
1047 + /* To preload ahead as we go, we need at least (prefetch_distance+2) 32-byte blocks */
1048 + cmp N, #(prefetch_distance+3)*32 - 1
1049 + blo 160f
1050 +
1051 + /* Long case */
1052 + push {DAT3, DAT4, DAT5, DAT6, DAT7}
1053 +
1054 + .cfi_def_cfa_offset 36
1055 + .cfi_rel_offset D, 20
1056 + .cfi_rel_offset DAT1, 24
1057 + .cfi_rel_offset DAT2, 28
1058 + .cfi_rel_offset DAT3, 0
1059 + .cfi_rel_offset DAT4, 4
1060 + .cfi_rel_offset DAT5, 8
1061 + .cfi_rel_offset DAT6, 12
1062 + .cfi_rel_offset DAT7, 16
1063 + .cfi_rel_offset lr, 32
1064 +
1065 + /* Adjust N so that the decrement instruction can also test for
1066 + * inner loop termination. We want it to stop when there are
1067 + * (prefetch_distance+1) complete blocks to go. */
1068 + sub N, N, #(prefetch_distance+2)*32
1069 + preload_leading_step1 backwards, DAT0, S
1070 + .if backwards
1071 + /* Bug in GAS: it accepts, but mis-assembles the instruction
1072 + * ands DAT2, D, #60, 2
1073 + * which sets DAT2 to the number of leading bytes until destination is aligned and also clears C (sets borrow)
1074 + */
1075 + .word 0xE210513C
1076 + beq 154f
1077 + .else
1078 + ands DAT2, D, #15
1079 + beq 154f
1080 + rsb DAT2, DAT2, #16 /* number of leading bytes until destination aligned */
1081 + .endif
1082 + preload_leading_step2 backwards, DAT0, S, DAT2, OFF
1083 + memcpy_leading_15bytes backwards, 1
1084 +154: /* Destination now 16-byte aligned; we have at least one prefetch as well as at least one 16-byte output block */
1085 + /* Prefetch offset is best selected such that it lies in the first 8 of each 32 bytes - but it's just as easy to aim for the first one */
1086 + .if backwards
1087 + rsb OFF, S, #3
1088 + and OFF, OFF, #28
1089 + sub OFF, OFF, #32*(prefetch_distance+1)
1090 + .else
1091 + and OFF, S, #28
1092 + rsb OFF, OFF, #32*prefetch_distance
1093 + .endif
1094 + movs DAT0, S, lsl #31
1095 + bhi 157f
1096 + bcs 156f
1097 + bmi 155f
1098 + memcpy_long_inner_loop backwards, 0
1099 +155: memcpy_long_inner_loop backwards, 1
1100 +156: memcpy_long_inner_loop backwards, 2
1101 +157: memcpy_long_inner_loop backwards, 3
1102 +
1103 + .cfi_def_cfa_offset 16
1104 + .cfi_rel_offset D, 0
1105 + .cfi_rel_offset DAT1, 4
1106 + .cfi_rel_offset DAT2, 8
1107 + .cfi_same_value DAT3
1108 + .cfi_same_value DAT4
1109 + .cfi_same_value DAT5
1110 + .cfi_same_value DAT6
1111 + .cfi_same_value DAT7
1112 + .cfi_rel_offset lr, 12
1113 +
1114 +160: /* Medium case */
1115 + preload_all backwards, 0, 0, S, N, DAT2, OFF
1116 + sub N, N, #16 /* simplifies inner loop termination */
1117 + .if backwards
1118 + ands DAT2, D, #15
1119 + beq 164f
1120 + .else
1121 + ands DAT2, D, #15
1122 + beq 164f
1123 + rsb DAT2, DAT2, #16
1124 + .endif
1125 + memcpy_leading_15bytes backwards, align
1126 +164: /* Destination now 16-byte aligned; we have at least one 16-byte output block */
1127 + tst S, #3
1128 + bne 140f
1129 + memcpy_medium_inner_loop backwards, 0
1130 +140: memcpy_medium_inner_loop backwards, 1
1131 +
1132 +170: /* Short case, less than 31 bytes, so no guarantee of at least one 16-byte block */
1133 + teq N, #0
1134 + beq 199f
1135 + preload_all backwards, 1, 0, S, N, DAT2, LAST
1136 + tst D, #3
1137 + beq 174f
1138 +172: subs N, N, #1
1139 + blo 199f
1140 + .if backwards
1141 + ldrb DAT0, [S, #-1]!
1142 + strb DAT0, [D, #-1]!
1143 + .else
1144 + ldrb DAT0, [S], #1
1145 + strb DAT0, [D], #1
1146 + .endif
1147 + tst D, #3
1148 + bne 172b
1149 +174: /* Destination now 4-byte aligned; we have 0 or more output bytes to go */
1150 + tst S, #3
1151 + bne 140f
1152 + memcpy_short_inner_loop backwards, 0
1153 +140: memcpy_short_inner_loop backwards, 1
1154 +
1155 + .cfi_endproc
1156 +
1157 + .unreq D
1158 + .unreq S
1159 + .unreq N
1160 + .unreq DAT0
1161 + .unreq DAT1
1162 + .unreq DAT2
1163 + .unreq DAT3
1164 + .unreq DAT4
1165 + .unreq DAT5
1166 + .unreq DAT6
1167 + .unreq DAT7
1168 + .unreq LAST
1169 + .unreq OFF
1170 +.endm
1171 --- /dev/null
1172 +++ b/arch/arm/lib/memmove_rpi.S
1173 @@ -0,0 +1,61 @@
1174 +/*
1175 +Copyright (c) 2013, Raspberry Pi Foundation
1176 +Copyright (c) 2013, RISC OS Open Ltd
1177 +All rights reserved.
1178 +
1179 +Redistribution and use in source and binary forms, with or without
1180 +modification, are permitted provided that the following conditions are met:
1181 + * Redistributions of source code must retain the above copyright
1182 + notice, this list of conditions and the following disclaimer.
1183 + * Redistributions in binary form must reproduce the above copyright
1184 + notice, this list of conditions and the following disclaimer in the
1185 + documentation and/or other materials provided with the distribution.
1186 + * Neither the name of the copyright holder nor the
1187 + names of its contributors may be used to endorse or promote products
1188 + derived from this software without specific prior written permission.
1189 +
1190 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
1191 +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
1192 +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
1193 +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
1194 +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
1195 +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
1196 +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
1197 +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
1198 +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
1199 +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1200 +*/
1201 +
1202 +#include <linux/linkage.h>
1203 +#include "arm-mem.h"
1204 +#include "memcpymove.h"
1205 +
1206 +/* Prevent the stack from becoming executable */
1207 +#if defined(__linux__) && defined(__ELF__)
1208 +.section .note.GNU-stack,"",%progbits
1209 +#endif
1210 +
1211 + .text
1212 + .arch armv6
1213 + .object_arch armv4
1214 + .arm
1215 + .altmacro
1216 + .p2align 2
1217 +
1218 +/*
1219 + * void *memmove(void *s1, const void *s2, size_t n);
1220 + * On entry:
1221 + * a1 = pointer to destination
1222 + * a2 = pointer to source
1223 + * a3 = number of bytes to copy
1224 + * On exit:
1225 + * a1 preserved
1226 + */
1227 +
1228 +.set prefetch_distance, 3
1229 +
1230 +ENTRY(memmove)
1231 + cmp a2, a1
1232 + bpl memcpy /* pl works even over -1 - 0 and 0x7fffffff - 0x80000000 boundaries */
1233 + memcpy 1
1234 +ENDPROC(memmove)
1235 --- /dev/null
1236 +++ b/arch/arm/lib/memset_rpi.S
1237 @@ -0,0 +1,123 @@
1238 +/*
1239 +Copyright (c) 2013, Raspberry Pi Foundation
1240 +Copyright (c) 2013, RISC OS Open Ltd
1241 +All rights reserved.
1242 +
1243 +Redistribution and use in source and binary forms, with or without
1244 +modification, are permitted provided that the following conditions are met:
1245 + * Redistributions of source code must retain the above copyright
1246 + notice, this list of conditions and the following disclaimer.
1247 + * Redistributions in binary form must reproduce the above copyright
1248 + notice, this list of conditions and the following disclaimer in the
1249 + documentation and/or other materials provided with the distribution.
1250 + * Neither the name of the copyright holder nor the
1251 + names of its contributors may be used to endorse or promote products
1252 + derived from this software without specific prior written permission.
1253 +
1254 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
1255 +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
1256 +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
1257 +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
1258 +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
1259 +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
1260 +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
1261 +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
1262 +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
1263 +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1264 +*/
1265 +
1266 +#include <linux/linkage.h>
1267 +#include "arm-mem.h"
1268 +
1269 +/* Prevent the stack from becoming executable */
1270 +#if defined(__linux__) && defined(__ELF__)
1271 +.section .note.GNU-stack,"",%progbits
1272 +#endif
1273 +
1274 + .text
1275 + .arch armv6
1276 + .object_arch armv4
1277 + .arm
1278 + .altmacro
1279 + .p2align 2
1280 +
1281 +/*
1282 + * void *memset(void *s, int c, size_t n);
1283 + * On entry:
1284 + * a1 = pointer to buffer to fill
1285 + * a2 = byte pattern to fill with (caller-narrowed)
1286 + * a3 = number of bytes to fill
1287 + * On exit:
1288 + * a1 preserved
1289 + */
1290 +ENTRY(mmioset)
1291 +ENTRY(memset)
1292 + S .req a1
1293 + DAT0 .req a2
1294 + N .req a3
1295 + DAT1 .req a4
1296 + DAT2 .req ip
1297 + DAT3 .req lr
1298 +
1299 + orr DAT0, DAT0, lsl #8
1300 + push {S, lr}
1301 + orr DAT0, DAT0, lsl #16
1302 + mov DAT1, DAT0
1303 +
1304 + /* See if we're guaranteed to have at least one 16-byte aligned 16-byte write */
1305 + cmp N, #31
1306 + blo 170f
1307 +
1308 +161: sub N, N, #16 /* simplifies inner loop termination */
1309 + /* Leading words and bytes */
1310 + tst S, #15
1311 + beq 164f
1312 + rsb DAT3, S, #0 /* bits 0-3 = number of leading bytes until aligned */
1313 + movs DAT2, DAT3, lsl #31
1314 + submi N, N, #1
1315 + strmib DAT0, [S], #1
1316 + subcs N, N, #2
1317 + strcsh DAT0, [S], #2
1318 + movs DAT2, DAT3, lsl #29
1319 + submi N, N, #4
1320 + strmi DAT0, [S], #4
1321 + subcs N, N, #8
1322 + stmcsia S!, {DAT0, DAT1}
1323 +164: /* Delayed set up of DAT2 and DAT3 so we could use them as scratch registers above */
1324 + mov DAT2, DAT0
1325 + mov DAT3, DAT0
1326 + /* Now the inner loop of 16-byte stores */
1327 +165: stmia S!, {DAT0, DAT1, DAT2, DAT3}
1328 + subs N, N, #16
1329 + bhs 165b
1330 +166: /* Trailing words and bytes */
1331 + movs N, N, lsl #29
1332 + stmcsia S!, {DAT0, DAT1}
1333 + strmi DAT0, [S], #4
1334 + movs N, N, lsl #2
1335 + strcsh DAT0, [S], #2
1336 + strmib DAT0, [S]
1337 +199: pop {S, pc}
1338 +
1339 +170: /* Short case */
1340 + mov DAT2, DAT0
1341 + mov DAT3, DAT0
1342 + tst S, #3
1343 + beq 174f
1344 +172: subs N, N, #1
1345 + blo 199b
1346 + strb DAT0, [S], #1
1347 + tst S, #3
1348 + bne 172b
1349 +174: tst N, #16
1350 + stmneia S!, {DAT0, DAT1, DAT2, DAT3}
1351 + b 166b
1352 +
1353 + .unreq S
1354 + .unreq DAT0
1355 + .unreq N
1356 + .unreq DAT1
1357 + .unreq DAT2
1358 + .unreq DAT3
1359 +ENDPROC(memset)
1360 +ENDPROC(mmioset)
1361 --- a/arch/arm/lib/uaccess_with_memcpy.c
1362 +++ b/arch/arm/lib/uaccess_with_memcpy.c
1363 @@ -22,6 +22,14 @@
1364 #include <asm/current.h>
1365 #include <asm/page.h>
1366
1367 +#ifndef COPY_FROM_USER_THRESHOLD
1368 +#define COPY_FROM_USER_THRESHOLD 64
1369 +#endif
1370 +
1371 +#ifndef COPY_TO_USER_THRESHOLD
1372 +#define COPY_TO_USER_THRESHOLD 64
1373 +#endif
1374 +
1375 static int
1376 pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
1377 {
1378 @@ -85,7 +93,44 @@ pin_page_for_write(const void __user *_a
1379 return 1;
1380 }
1381
1382 -static unsigned long noinline
1383 +static int
1384 +pin_page_for_read(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
1385 +{
1386 + unsigned long addr = (unsigned long)_addr;
1387 + pgd_t *pgd;
1388 + pmd_t *pmd;
1389 + pte_t *pte;
1390 + pud_t *pud;
1391 + spinlock_t *ptl;
1392 +
1393 + pgd = pgd_offset(current->mm, addr);
1394 + if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd)))
1395 + {
1396 + return 0;
1397 + }
1398 + pud = pud_offset(pgd, addr);
1399 + if (unlikely(pud_none(*pud) || pud_bad(*pud)))
1400 + {
1401 + return 0;
1402 + }
1403 +
1404 + pmd = pmd_offset(pud, addr);
1405 + if (unlikely(pmd_none(*pmd) || pmd_bad(*pmd)))
1406 + return 0;
1407 +
1408 + pte = pte_offset_map_lock(current->mm, pmd, addr, &ptl);
1409 + if (unlikely(!pte_present(*pte) || !pte_young(*pte))) {
1410 + pte_unmap_unlock(pte, ptl);
1411 + return 0;
1412 + }
1413 +
1414 + *ptep = pte;
1415 + *ptlp = ptl;
1416 +
1417 + return 1;
1418 +}
1419 +
1420 +unsigned long noinline
1421 __copy_to_user_memcpy(void __user *to, const void *from, unsigned long n)
1422 {
1423 unsigned long ua_flags;
1424 @@ -138,6 +183,54 @@ out:
1425 return n;
1426 }
1427
1428 +unsigned long noinline
1429 +__copy_from_user_memcpy(void *to, const void __user *from, unsigned long n)
1430 +{
1431 + int atomic;
1432 +
1433 + if (unlikely(segment_eq(get_fs(), KERNEL_DS))) {
1434 + memcpy(to, (const void *)from, n);
1435 + return 0;
1436 + }
1437 +
1438 + /* the mmap semaphore is taken only if not in an atomic context */
1439 + atomic = in_atomic();
1440 +
1441 + if (!atomic)
1442 + down_read(&current->mm->mmap_sem);
1443 + while (n) {
1444 + pte_t *pte;
1445 + spinlock_t *ptl;
1446 + int tocopy;
1447 +
1448 + while (!pin_page_for_read(from, &pte, &ptl)) {
1449 + char temp;
1450 + if (!atomic)
1451 + up_read(&current->mm->mmap_sem);
1452 + if (__get_user(temp, (char __user *)from))
1453 + goto out;
1454 + if (!atomic)
1455 + down_read(&current->mm->mmap_sem);
1456 + }
1457 +
1458 + tocopy = (~(unsigned long)from & ~PAGE_MASK) + 1;
1459 + if (tocopy > n)
1460 + tocopy = n;
1461 +
1462 + memcpy(to, (const void *)from, tocopy);
1463 + to += tocopy;
1464 + from += tocopy;
1465 + n -= tocopy;
1466 +
1467 + pte_unmap_unlock(pte, ptl);
1468 + }
1469 + if (!atomic)
1470 + up_read(&current->mm->mmap_sem);
1471 +
1472 +out:
1473 + return n;
1474 +}
1475 +
1476 unsigned long
1477 arm_copy_to_user(void __user *to, const void *from, unsigned long n)
1478 {
1479 @@ -148,7 +241,7 @@ arm_copy_to_user(void __user *to, const
1480 * With frame pointer disabled, tail call optimization kicks in
1481 * as well making this test almost invisible.
1482 */
1483 - if (n < 64) {
1484 + if (n < COPY_TO_USER_THRESHOLD) {
1485 unsigned long ua_flags = uaccess_save_and_enable();
1486 n = __copy_to_user_std(to, from, n);
1487 uaccess_restore(ua_flags);
1488 @@ -157,6 +250,21 @@ arm_copy_to_user(void __user *to, const
1489 }
1490 return n;
1491 }
1492 +
1493 +unsigned long __must_check
1494 +arm_copy_from_user(void *to, const void __user *from, unsigned long n)
1495 +{
1496 + /*
1497 + * This test is stubbed out of the main function above to keep
1498 + * the overhead for small copies low by avoiding a large
1499 + * register dump on the stack just to reload them right away.
1500 + * With frame pointer disabled, tail call optimization kicks in
1501 + * as well making this test almost invisible.
1502 + */
1503 + if (n < COPY_FROM_USER_THRESHOLD)
1504 + return __copy_from_user_std(to, from, n);
1505 + return __copy_from_user_memcpy(to, from, n);
1506 +}
1507
1508 static unsigned long noinline
1509 __clear_user_memset(void __user *addr, unsigned long n)
OpenWrt Source Repository