projects / openwrt / staging / wigyori.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
brcm2708: update to v3.18
[openwrt/staging/wigyori.git] / target / linux / brcm2708 / patches-3.18 / 0062-Improve-__copy_to_user-and-__copy_from_user-performa.patch
1 From 49e011c979aee23801198617a0052b0b087583a6 Mon Sep 17 00:00:00 2001
2 From: Phil Elwell <phil@raspberrypi.org>
3 Date: Mon, 13 Oct 2014 11:47:53 +0100
4 Subject: [PATCH 062/114] Improve __copy_to_user and __copy_from_user
5 performance
6
7 Provide a __copy_from_user that uses memcpy. On BCM2708, use
8 optimised memcpy/memmove/memcmp/memset implementations.
9 ---
10 arch/arm/include/asm/string.h | 5 +
11 arch/arm/include/asm/uaccess.h | 1 +
12 arch/arm/lib/Makefile | 15 +-
13 arch/arm/lib/arm-mem.h | 159 ++++++++++++
14 arch/arm/lib/copy_from_user.S | 4 +-
15 arch/arm/lib/exports_rpi.c | 37 +++
16 arch/arm/lib/memcmp_rpi.S | 285 +++++++++++++++++++++
17 arch/arm/lib/memcpy_rpi.S | 59 +++++
18 arch/arm/lib/memcpymove.h | 506 +++++++++++++++++++++++++++++++++++++
19 arch/arm/lib/memmove_rpi.S | 61 +++++
20 arch/arm/lib/memset_rpi.S | 121 +++++++++
21 arch/arm/lib/uaccess_with_memcpy.c | 112 +++++++-
22 12 files changed, 1359 insertions(+), 6 deletions(-)
23 create mode 100644 arch/arm/lib/arm-mem.h
24 create mode 100644 arch/arm/lib/exports_rpi.c
25 create mode 100644 arch/arm/lib/memcmp_rpi.S
26 create mode 100644 arch/arm/lib/memcpy_rpi.S
27 create mode 100644 arch/arm/lib/memcpymove.h
28 create mode 100644 arch/arm/lib/memmove_rpi.S
29 create mode 100644 arch/arm/lib/memset_rpi.S
30
31 diff --git a/arch/arm/include/asm/string.h b/arch/arm/include/asm/string.h
32 index cf4f3aa..9fe7780 100644
33 --- a/arch/arm/include/asm/string.h
34 +++ b/arch/arm/include/asm/string.h
35 @@ -24,6 +24,11 @@ extern void * memchr(const void *, int, __kernel_size_t);
36 #define __HAVE_ARCH_MEMSET
37 extern void * memset(void *, int, __kernel_size_t);
38
39 +#ifdef CONFIG_MACH_BCM2708
40 +#define __HAVE_ARCH_MEMCMP
41 +extern int memcmp(const void *, const void *, size_t);
42 +#endif
43 +
44 extern void __memzero(void *ptr, __kernel_size_t n);
45
46 #define memset(p,v,n) \
47 diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h
48 index 4767eb9..bcd43de 100644
49 --- a/arch/arm/include/asm/uaccess.h
50 +++ b/arch/arm/include/asm/uaccess.h
51 @@ -475,6 +475,7 @@ do { \
52
53 #ifdef CONFIG_MMU
54 extern unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n);
55 +extern unsigned long __must_check __copy_from_user_std(void *to, const void __user *from, unsigned long n);
56 extern unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n);
57 extern unsigned long __must_check __copy_to_user_std(void __user *to, const void *from, unsigned long n);
58 extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n);
59 diff --git a/arch/arm/lib/Makefile b/arch/arm/lib/Makefile
60 index 0573faa..498f5dd 100644
61 --- a/arch/arm/lib/Makefile
62 +++ b/arch/arm/lib/Makefile
63 @@ -6,15 +6,24 @@
64
65 lib-y := backtrace.o changebit.o csumipv6.o csumpartial.o \
66 csumpartialcopy.o csumpartialcopyuser.o clearbit.o \
67 - delay.o delay-loop.o findbit.o memchr.o memcpy.o \
68 - memmove.o memset.o memzero.o setbit.o \
69 - strchr.o strrchr.o \
70 + delay.o delay-loop.o findbit.o memchr.o memzero.o \
71 + setbit.o strchr.o strrchr.o \
72 testchangebit.o testclearbit.o testsetbit.o \
73 ashldi3.o ashrdi3.o lshrdi3.o muldi3.o \
74 ucmpdi2.o lib1funcs.o div64.o \
75 io-readsb.o io-writesb.o io-readsl.o io-writesl.o \
76 call_with_stack.o bswapsdi2.o
77
78 +# Choose optimised implementations for Raspberry Pi
79 +ifeq ($(CONFIG_MACH_BCM2708),y)
80 + CFLAGS_uaccess_with_memcpy.o += -DCOPY_FROM_USER_THRESHOLD=1600
81 + CFLAGS_uaccess_with_memcpy.o += -DCOPY_TO_USER_THRESHOLD=672
82 + obj-$(CONFIG_MODULES) += exports_rpi.o
83 + lib-y += memcpy_rpi.o memmove_rpi.o memset_rpi.o memcmp_rpi.o
84 +else
85 + lib-y += memcpy.o memmove.o memset.o
86 +endif
87 +
88 mmu-y := clear_user.o copy_page.o getuser.o putuser.o
89
90 # the code in uaccess.S is not preemption safe and
91 diff --git a/arch/arm/lib/arm-mem.h b/arch/arm/lib/arm-mem.h
92 new file mode 100644
93 index 0000000..5d4bda1
94 --- /dev/null
95 +++ b/arch/arm/lib/arm-mem.h
96 @@ -0,0 +1,159 @@
97 +/*
98 +Copyright (c) 2013, Raspberry Pi Foundation
99 +Copyright (c) 2013, RISC OS Open Ltd
100 +All rights reserved.
101 +
102 +Redistribution and use in source and binary forms, with or without
103 +modification, are permitted provided that the following conditions are met:
104 + * Redistributions of source code must retain the above copyright
105 + notice, this list of conditions and the following disclaimer.
106 + * Redistributions in binary form must reproduce the above copyright
107 + notice, this list of conditions and the following disclaimer in the
108 + documentation and/or other materials provided with the distribution.
109 + * Neither the name of the copyright holder nor the
110 + names of its contributors may be used to endorse or promote products
111 + derived from this software without specific prior written permission.
112 +
113 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
114 +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
115 +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
116 +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
117 +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
118 +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
119 +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
120 +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
121 +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
122 +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
123 +*/
124 +
125 +.macro myfunc fname
126 + .func fname
127 + .global fname
128 +fname:
129 +.endm
130 +
131 +.macro preload_leading_step1 backwards, ptr, base
132 +/* If the destination is already 16-byte aligned, then we need to preload
133 + * between 0 and prefetch_distance (inclusive) cache lines ahead so there
134 + * are no gaps when the inner loop starts.
135 + */
136 + .if backwards
137 + sub ptr, base, #1
138 + bic ptr, ptr, #31
139 + .else
140 + bic ptr, base, #31
141 + .endif
142 + .set OFFSET, 0
143 + .rept prefetch_distance+1
144 + pld [ptr, #OFFSET]
145 + .if backwards
146 + .set OFFSET, OFFSET-32
147 + .else
148 + .set OFFSET, OFFSET+32
149 + .endif
150 + .endr
151 +.endm
152 +
153 +.macro preload_leading_step2 backwards, ptr, base, leading_bytes, tmp
154 +/* However, if the destination is not 16-byte aligned, we may need to
155 + * preload one more cache line than that. The question we need to ask is:
156 + * are the leading bytes more than the amount by which the source
157 + * pointer will be rounded down for preloading, and if so, by how many
158 + * cache lines?
159 + */
160 + .if backwards
161 +/* Here we compare against how many bytes we are into the
162 + * cache line, counting down from the highest such address.
163 + * Effectively, we want to calculate
164 + * leading_bytes = dst&15
165 + * cacheline_offset = 31-((src-leading_bytes-1)&31)
166 + * extra_needed = leading_bytes - cacheline_offset
167 + * and test if extra_needed is <= 0, or rearranging:
168 + * leading_bytes + (src-leading_bytes-1)&31 <= 31
169 + */
170 + mov tmp, base, lsl #32-5
171 + sbc tmp, tmp, leading_bytes, lsl #32-5
172 + adds tmp, tmp, leading_bytes, lsl #32-5
173 + bcc 61f
174 + pld [ptr, #-32*(prefetch_distance+1)]
175 + .else
176 +/* Effectively, we want to calculate
177 + * leading_bytes = (-dst)&15
178 + * cacheline_offset = (src+leading_bytes)&31
179 + * extra_needed = leading_bytes - cacheline_offset
180 + * and test if extra_needed is <= 0.
181 + */
182 + mov tmp, base, lsl #32-5
183 + add tmp, tmp, leading_bytes, lsl #32-5
184 + rsbs tmp, tmp, leading_bytes, lsl #32-5
185 + bls 61f
186 + pld [ptr, #32*(prefetch_distance+1)]
187 + .endif
188 +61:
189 +.endm
190 +
191 +.macro preload_trailing backwards, base, remain, tmp
192 + /* We need either 0, 1 or 2 extra preloads */
193 + .if backwards
194 + rsb tmp, base, #0
195 + mov tmp, tmp, lsl #32-5
196 + .else
197 + mov tmp, base, lsl #32-5
198 + .endif
199 + adds tmp, tmp, remain, lsl #32-5
200 + adceqs tmp, tmp, #0
201 + /* The instruction above has two effects: ensures Z is only
202 + * set if C was clear (so Z indicates that both shifted quantities
203 + * were 0), and clears C if Z was set (so C indicates that the sum
204 + * of the shifted quantities was greater and not equal to 32) */
205 + beq 82f
206 + .if backwards
207 + sub tmp, base, #1
208 + bic tmp, tmp, #31
209 + .else
210 + bic tmp, base, #31
211 + .endif
212 + bcc 81f
213 + .if backwards
214 + pld [tmp, #-32*(prefetch_distance+1)]
215 +81:
216 + pld [tmp, #-32*prefetch_distance]
217 + .else
218 + pld [tmp, #32*(prefetch_distance+2)]
219 +81:
220 + pld [tmp, #32*(prefetch_distance+1)]
221 + .endif
222 +82:
223 +.endm
224 +
225 +.macro preload_all backwards, narrow_case, shift, base, remain, tmp0, tmp1
226 + .if backwards
227 + sub tmp0, base, #1
228 + bic tmp0, tmp0, #31
229 + pld [tmp0]
230 + sub tmp1, base, remain, lsl #shift
231 + .else
232 + bic tmp0, base, #31
233 + pld [tmp0]
234 + add tmp1, base, remain, lsl #shift
235 + sub tmp1, tmp1, #1
236 + .endif
237 + bic tmp1, tmp1, #31
238 + cmp tmp1, tmp0
239 + beq 92f
240 + .if narrow_case
241 + /* In this case, all the data fits in either 1 or 2 cache lines */
242 + pld [tmp1]
243 + .else
244 +91:
245 + .if backwards
246 + sub tmp0, tmp0, #32
247 + .else
248 + add tmp0, tmp0, #32
249 + .endif
250 + cmp tmp0, tmp1
251 + pld [tmp0]
252 + bne 91b
253 + .endif
254 +92:
255 +.endm
256 diff --git a/arch/arm/lib/copy_from_user.S b/arch/arm/lib/copy_from_user.S
257 index 66a477a..3faddf7 100644
258 --- a/arch/arm/lib/copy_from_user.S
259 +++ b/arch/arm/lib/copy_from_user.S
260 @@ -84,11 +84,13 @@
261
262 .text
263
264 -ENTRY(__copy_from_user)
265 +ENTRY(__copy_from_user_std)
266 +WEAK(__copy_from_user)
267
268 #include "copy_template.S"
269
270 ENDPROC(__copy_from_user)
271 +ENDPROC(__copy_from_user_std)
272
273 .pushsection .fixup,"ax"
274 .align 0
275 diff --git a/arch/arm/lib/exports_rpi.c b/arch/arm/lib/exports_rpi.c
276 new file mode 100644
277 index 0000000..1f82604
278 --- /dev/null
279 +++ b/arch/arm/lib/exports_rpi.c
280 @@ -0,0 +1,37 @@
281 +/**
282 + * Copyright (c) 2014, Raspberry Pi (Trading) Ltd.
283 + *
284 + * Redistribution and use in source and binary forms, with or without
285 + * modification, are permitted provided that the following conditions
286 + * are met:
287 + * 1. Redistributions of source code must retain the above copyright
288 + * notice, this list of conditions, and the following disclaimer,
289 + * without modification.
290 + * 2. Redistributions in binary form must reproduce the above copyright
291 + * notice, this list of conditions and the following disclaimer in the
292 + * documentation and/or other materials provided with the distribution.
293 + * 3. The names of the above-listed copyright holders may not be used
294 + * to endorse or promote products derived from this software without
295 + * specific prior written permission.
296 + *
297 + * ALTERNATIVELY, this software may be distributed under the terms of the
298 + * GNU General Public License ("GPL") version 2, as published by the Free
299 + * Software Foundation.
300 + *
301 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
302 + * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
303 + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
304 + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
305 + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
306 + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
307 + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
308 + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
309 + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
310 + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
311 + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
312 + */
313 +
314 +#include <linux/kernel.h>
315 +#include <linux/module.h>
316 +
317 +EXPORT_SYMBOL(memcmp);
318 diff --git a/arch/arm/lib/memcmp_rpi.S b/arch/arm/lib/memcmp_rpi.S
319 new file mode 100644
320 index 0000000..bf6e4ed
321 --- /dev/null
322 +++ b/arch/arm/lib/memcmp_rpi.S
323 @@ -0,0 +1,285 @@
324 +/*
325 +Copyright (c) 2013, Raspberry Pi Foundation
326 +Copyright (c) 2013, RISC OS Open Ltd
327 +All rights reserved.
328 +
329 +Redistribution and use in source and binary forms, with or without
330 +modification, are permitted provided that the following conditions are met:
331 + * Redistributions of source code must retain the above copyright
332 + notice, this list of conditions and the following disclaimer.
333 + * Redistributions in binary form must reproduce the above copyright
334 + notice, this list of conditions and the following disclaimer in the
335 + documentation and/or other materials provided with the distribution.
336 + * Neither the name of the copyright holder nor the
337 + names of its contributors may be used to endorse or promote products
338 + derived from this software without specific prior written permission.
339 +
340 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
341 +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
342 +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
343 +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
344 +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
345 +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
346 +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
347 +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
348 +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
349 +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
350 +*/
351 +
352 +#include <linux/linkage.h>
353 +#include "arm-mem.h"
354 +
355 +/* Prevent the stack from becoming executable */
356 +#if defined(__linux__) && defined(__ELF__)
357 +.section .note.GNU-stack,"",%progbits
358 +#endif
359 +
360 + .text
361 + .arch armv6
362 + .object_arch armv4
363 + .arm
364 + .altmacro
365 + .p2align 2
366 +
367 +.macro memcmp_process_head unaligned
368 + .if unaligned
369 + ldr DAT0, [S_1], #4
370 + ldr DAT1, [S_1], #4
371 + ldr DAT2, [S_1], #4
372 + ldr DAT3, [S_1], #4
373 + .else
374 + ldmia S_1!, {DAT0, DAT1, DAT2, DAT3}
375 + .endif
376 + ldmia S_2!, {DAT4, DAT5, DAT6, DAT7}
377 +.endm
378 +
379 +.macro memcmp_process_tail
380 + cmp DAT0, DAT4
381 + cmpeq DAT1, DAT5
382 + cmpeq DAT2, DAT6
383 + cmpeq DAT3, DAT7
384 + bne 200f
385 +.endm
386 +
387 +.macro memcmp_leading_31bytes
388 + movs DAT0, OFF, lsl #31
389 + ldrmib DAT0, [S_1], #1
390 + ldrcsh DAT1, [S_1], #2
391 + ldrmib DAT4, [S_2], #1
392 + ldrcsh DAT5, [S_2], #2
393 + movpl DAT0, #0
394 + movcc DAT1, #0
395 + movpl DAT4, #0
396 + movcc DAT5, #0
397 + submi N, N, #1
398 + subcs N, N, #2
399 + cmp DAT0, DAT4
400 + cmpeq DAT1, DAT5
401 + bne 200f
402 + movs DAT0, OFF, lsl #29
403 + ldrmi DAT0, [S_1], #4
404 + ldrcs DAT1, [S_1], #4
405 + ldrcs DAT2, [S_1], #4
406 + ldrmi DAT4, [S_2], #4
407 + ldmcsia S_2!, {DAT5, DAT6}
408 + movpl DAT0, #0
409 + movcc DAT1, #0
410 + movcc DAT2, #0
411 + movpl DAT4, #0
412 + movcc DAT5, #0
413 + movcc DAT6, #0
414 + submi N, N, #4
415 + subcs N, N, #8
416 + cmp DAT0, DAT4
417 + cmpeq DAT1, DAT5
418 + cmpeq DAT2, DAT6
419 + bne 200f
420 + tst OFF, #16
421 + beq 105f
422 + memcmp_process_head 1
423 + sub N, N, #16
424 + memcmp_process_tail
425 +105:
426 +.endm
427 +
428 +.macro memcmp_trailing_15bytes unaligned
429 + movs N, N, lsl #29
430 + .if unaligned
431 + ldrcs DAT0, [S_1], #4
432 + ldrcs DAT1, [S_1], #4
433 + .else
434 + ldmcsia S_1!, {DAT0, DAT1}
435 + .endif
436 + ldrmi DAT2, [S_1], #4
437 + ldmcsia S_2!, {DAT4, DAT5}
438 + ldrmi DAT6, [S_2], #4
439 + movcc DAT0, #0
440 + movcc DAT1, #0
441 + movpl DAT2, #0
442 + movcc DAT4, #0
443 + movcc DAT5, #0
444 + movpl DAT6, #0
445 + cmp DAT0, DAT4
446 + cmpeq DAT1, DAT5
447 + cmpeq DAT2, DAT6
448 + bne 200f
449 + movs N, N, lsl #2
450 + ldrcsh DAT0, [S_1], #2
451 + ldrmib DAT1, [S_1]
452 + ldrcsh DAT4, [S_2], #2
453 + ldrmib DAT5, [S_2]
454 + movcc DAT0, #0
455 + movpl DAT1, #0
456 + movcc DAT4, #0
457 + movpl DAT5, #0
458 + cmp DAT0, DAT4
459 + cmpeq DAT1, DAT5
460 + bne 200f
461 +.endm
462 +
463 +.macro memcmp_long_inner_loop unaligned
464 +110:
465 + memcmp_process_head unaligned
466 + pld [S_2, #prefetch_distance*32 + 16]
467 + memcmp_process_tail
468 + memcmp_process_head unaligned
469 + pld [S_1, OFF]
470 + memcmp_process_tail
471 + subs N, N, #32
472 + bhs 110b
473 + /* Just before the final (prefetch_distance+1) 32-byte blocks,
474 + * deal with final preloads */
475 + preload_trailing 0, S_1, N, DAT0
476 + preload_trailing 0, S_2, N, DAT0
477 + add N, N, #(prefetch_distance+2)*32 - 16
478 +120:
479 + memcmp_process_head unaligned
480 + memcmp_process_tail
481 + subs N, N, #16
482 + bhs 120b
483 + /* Trailing words and bytes */
484 + tst N, #15
485 + beq 199f
486 + memcmp_trailing_15bytes unaligned
487 +199: /* Reached end without detecting a difference */
488 + mov a1, #0
489 + setend le
490 + pop {DAT1-DAT6, pc}
491 +.endm
492 +
493 +.macro memcmp_short_inner_loop unaligned
494 + subs N, N, #16 /* simplifies inner loop termination */
495 + blo 122f
496 +120:
497 + memcmp_process_head unaligned
498 + memcmp_process_tail
499 + subs N, N, #16
500 + bhs 120b
501 +122: /* Trailing words and bytes */
502 + tst N, #15
503 + beq 199f
504 + memcmp_trailing_15bytes unaligned
505 +199: /* Reached end without detecting a difference */
506 + mov a1, #0
507 + setend le
508 + pop {DAT1-DAT6, pc}
509 +.endm
510 +
511 +/*
512 + * int memcmp(const void *s1, const void *s2, size_t n);
513 + * On entry:
514 + * a1 = pointer to buffer 1
515 + * a2 = pointer to buffer 2
516 + * a3 = number of bytes to compare (as unsigned chars)
517 + * On exit:
518 + * a1 = >0/=0/<0 if s1 >/=/< s2
519 + */
520 +
521 +.set prefetch_distance, 2
522 +
523 +ENTRY(memcmp)
524 + S_1 .req a1
525 + S_2 .req a2
526 + N .req a3
527 + DAT0 .req a4
528 + DAT1 .req v1
529 + DAT2 .req v2
530 + DAT3 .req v3
531 + DAT4 .req v4
532 + DAT5 .req v5
533 + DAT6 .req v6
534 + DAT7 .req ip
535 + OFF .req lr
536 +
537 + push {DAT1-DAT6, lr}
538 + setend be /* lowest-addressed bytes are most significant */
539 +
540 + /* To preload ahead as we go, we need at least (prefetch_distance+2) 32-byte blocks */
541 + cmp N, #(prefetch_distance+3)*32 - 1
542 + blo 170f
543 +
544 + /* Long case */
545 + /* Adjust N so that the decrement instruction can also test for
546 + * inner loop termination. We want it to stop when there are
547 + * (prefetch_distance+1) complete blocks to go. */
548 + sub N, N, #(prefetch_distance+2)*32
549 + preload_leading_step1 0, DAT0, S_1
550 + preload_leading_step1 0, DAT1, S_2
551 + tst S_2, #31
552 + beq 154f
553 + rsb OFF, S_2, #0 /* no need to AND with 15 here */
554 + preload_leading_step2 0, DAT0, S_1, OFF, DAT2
555 + preload_leading_step2 0, DAT1, S_2, OFF, DAT2
556 + memcmp_leading_31bytes
557 +154: /* Second source now cacheline (32-byte) aligned; we have at
558 + * least one prefetch to go. */
559 + /* Prefetch offset is best selected such that it lies in the
560 + * first 8 of each 32 bytes - but it's just as easy to aim for
561 + * the first one */
562 + and OFF, S_1, #31
563 + rsb OFF, OFF, #32*prefetch_distance
564 + tst S_1, #3
565 + bne 140f
566 + memcmp_long_inner_loop 0
567 +140: memcmp_long_inner_loop 1
568 +
569 +170: /* Short case */
570 + teq N, #0
571 + beq 199f
572 + preload_all 0, 0, 0, S_1, N, DAT0, DAT1
573 + preload_all 0, 0, 0, S_2, N, DAT0, DAT1
574 + tst S_2, #3
575 + beq 174f
576 +172: subs N, N, #1
577 + blo 199f
578 + ldrb DAT0, [S_1], #1
579 + ldrb DAT4, [S_2], #1
580 + cmp DAT0, DAT4
581 + bne 200f
582 + tst S_2, #3
583 + bne 172b
584 +174: /* Second source now 4-byte aligned; we have 0 or more bytes to go */
585 + tst S_1, #3
586 + bne 140f
587 + memcmp_short_inner_loop 0
588 +140: memcmp_short_inner_loop 1
589 +
590 +200: /* Difference found: determine sign. */
591 + movhi a1, #1
592 + movlo a1, #-1
593 + setend le
594 + pop {DAT1-DAT6, pc}
595 +
596 + .unreq S_1
597 + .unreq S_2
598 + .unreq N
599 + .unreq DAT0
600 + .unreq DAT1
601 + .unreq DAT2
602 + .unreq DAT3
603 + .unreq DAT4
604 + .unreq DAT5
605 + .unreq DAT6
606 + .unreq DAT7
607 + .unreq OFF
608 +ENDPROC(memcmp)
609 diff --git a/arch/arm/lib/memcpy_rpi.S b/arch/arm/lib/memcpy_rpi.S
610 new file mode 100644
611 index 0000000..15ff2bd
612 --- /dev/null
613 +++ b/arch/arm/lib/memcpy_rpi.S
614 @@ -0,0 +1,59 @@
615 +/*
616 +Copyright (c) 2013, Raspberry Pi Foundation
617 +Copyright (c) 2013, RISC OS Open Ltd
618 +All rights reserved.
619 +
620 +Redistribution and use in source and binary forms, with or without
621 +modification, are permitted provided that the following conditions are met:
622 + * Redistributions of source code must retain the above copyright
623 + notice, this list of conditions and the following disclaimer.
624 + * Redistributions in binary form must reproduce the above copyright
625 + notice, this list of conditions and the following disclaimer in the
626 + documentation and/or other materials provided with the distribution.
627 + * Neither the name of the copyright holder nor the
628 + names of its contributors may be used to endorse or promote products
629 + derived from this software without specific prior written permission.
630 +
631 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
632 +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
633 +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
634 +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
635 +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
636 +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
637 +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
638 +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
639 +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
640 +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
641 +*/
642 +
643 +#include <linux/linkage.h>
644 +#include "arm-mem.h"
645 +#include "memcpymove.h"
646 +
647 +/* Prevent the stack from becoming executable */
648 +#if defined(__linux__) && defined(__ELF__)
649 +.section .note.GNU-stack,"",%progbits
650 +#endif
651 +
652 + .text
653 + .arch armv6
654 + .object_arch armv4
655 + .arm
656 + .altmacro
657 + .p2align 2
658 +
659 +/*
660 + * void *memcpy(void * restrict s1, const void * restrict s2, size_t n);
661 + * On entry:
662 + * a1 = pointer to destination
663 + * a2 = pointer to source
664 + * a3 = number of bytes to copy
665 + * On exit:
666 + * a1 preserved
667 + */
668 +
669 +.set prefetch_distance, 3
670 +
671 +ENTRY(memcpy)
672 + memcpy 0
673 +ENDPROC(memcpy)
674 diff --git a/arch/arm/lib/memcpymove.h b/arch/arm/lib/memcpymove.h
675 new file mode 100644
676 index 0000000..d8be584
677 --- /dev/null
678 +++ b/arch/arm/lib/memcpymove.h
679 @@ -0,0 +1,506 @@
680 +/*
681 +Copyright (c) 2013, Raspberry Pi Foundation
682 +Copyright (c) 2013, RISC OS Open Ltd
683 +All rights reserved.
684 +
685 +Redistribution and use in source and binary forms, with or without
686 +modification, are permitted provided that the following conditions are met:
687 + * Redistributions of source code must retain the above copyright
688 + notice, this list of conditions and the following disclaimer.
689 + * Redistributions in binary form must reproduce the above copyright
690 + notice, this list of conditions and the following disclaimer in the
691 + documentation and/or other materials provided with the distribution.
692 + * Neither the name of the copyright holder nor the
693 + names of its contributors may be used to endorse or promote products
694 + derived from this software without specific prior written permission.
695 +
696 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
697 +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
698 +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
699 +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
700 +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
701 +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
702 +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
703 +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
704 +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
705 +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
706 +*/
707 +
708 +.macro unaligned_words backwards, align, use_pld, words, r0, r1, r2, r3, r4, r5, r6, r7, r8
709 + .if words == 1
710 + .if backwards
711 + mov r1, r0, lsl #32-align*8
712 + ldr r0, [S, #-4]!
713 + orr r1, r1, r0, lsr #align*8
714 + str r1, [D, #-4]!
715 + .else
716 + mov r0, r1, lsr #align*8
717 + ldr r1, [S, #4]!
718 + orr r0, r0, r1, lsl #32-align*8
719 + str r0, [D], #4
720 + .endif
721 + .elseif words == 2
722 + .if backwards
723 + ldr r1, [S, #-4]!
724 + mov r2, r0, lsl #32-align*8
725 + ldr r0, [S, #-4]!
726 + orr r2, r2, r1, lsr #align*8
727 + mov r1, r1, lsl #32-align*8
728 + orr r1, r1, r0, lsr #align*8
729 + stmdb D!, {r1, r2}
730 + .else
731 + ldr r1, [S, #4]!
732 + mov r0, r2, lsr #align*8
733 + ldr r2, [S, #4]!
734 + orr r0, r0, r1, lsl #32-align*8
735 + mov r1, r1, lsr #align*8
736 + orr r1, r1, r2, lsl #32-align*8
737 + stmia D!, {r0, r1}
738 + .endif
739 + .elseif words == 4
740 + .if backwards
741 + ldmdb S!, {r2, r3}
742 + mov r4, r0, lsl #32-align*8
743 + ldmdb S!, {r0, r1}
744 + orr r4, r4, r3, lsr #align*8
745 + mov r3, r3, lsl #32-align*8
746 + orr r3, r3, r2, lsr #align*8
747 + mov r2, r2, lsl #32-align*8
748 + orr r2, r2, r1, lsr #align*8
749 + mov r1, r1, lsl #32-align*8
750 + orr r1, r1, r0, lsr #align*8
751 + stmdb D!, {r1, r2, r3, r4}
752 + .else
753 + ldmib S!, {r1, r2}
754 + mov r0, r4, lsr #align*8
755 + ldmib S!, {r3, r4}
756 + orr r0, r0, r1, lsl #32-align*8
757 + mov r1, r1, lsr #align*8
758 + orr r1, r1, r2, lsl #32-align*8
759 + mov r2, r2, lsr #align*8
760 + orr r2, r2, r3, lsl #32-align*8
761 + mov r3, r3, lsr #align*8
762 + orr r3, r3, r4, lsl #32-align*8
763 + stmia D!, {r0, r1, r2, r3}
764 + .endif
765 + .elseif words == 8
766 + .if backwards
767 + ldmdb S!, {r4, r5, r6, r7}
768 + mov r8, r0, lsl #32-align*8
769 + ldmdb S!, {r0, r1, r2, r3}
770 + .if use_pld
771 + pld [S, OFF]
772 + .endif
773 + orr r8, r8, r7, lsr #align*8
774 + mov r7, r7, lsl #32-align*8
775 + orr r7, r7, r6, lsr #align*8
776 + mov r6, r6, lsl #32-align*8
777 + orr r6, r6, r5, lsr #align*8
778 + mov r5, r5, lsl #32-align*8
779 + orr r5, r5, r4, lsr #align*8
780 + mov r4, r4, lsl #32-align*8
781 + orr r4, r4, r3, lsr #align*8
782 + mov r3, r3, lsl #32-align*8
783 + orr r3, r3, r2, lsr #align*8
784 + mov r2, r2, lsl #32-align*8
785 + orr r2, r2, r1, lsr #align*8
786 + mov r1, r1, lsl #32-align*8
787 + orr r1, r1, r0, lsr #align*8
788 + stmdb D!, {r5, r6, r7, r8}
789 + stmdb D!, {r1, r2, r3, r4}
790 + .else
791 + ldmib S!, {r1, r2, r3, r4}
792 + mov r0, r8, lsr #align*8
793 + ldmib S!, {r5, r6, r7, r8}
794 + .if use_pld
795 + pld [S, OFF]
796 + .endif
797 + orr r0, r0, r1, lsl #32-align*8
798 + mov r1, r1, lsr #align*8
799 + orr r1, r1, r2, lsl #32-align*8
800 + mov r2, r2, lsr #align*8
801 + orr r2, r2, r3, lsl #32-align*8
802 + mov r3, r3, lsr #align*8
803 + orr r3, r3, r4, lsl #32-align*8
804 + mov r4, r4, lsr #align*8
805 + orr r4, r4, r5, lsl #32-align*8
806 + mov r5, r5, lsr #align*8
807 + orr r5, r5, r6, lsl #32-align*8
808 + mov r6, r6, lsr #align*8
809 + orr r6, r6, r7, lsl #32-align*8
810 + mov r7, r7, lsr #align*8
811 + orr r7, r7, r8, lsl #32-align*8
812 + stmia D!, {r0, r1, r2, r3}
813 + stmia D!, {r4, r5, r6, r7}
814 + .endif
815 + .endif
816 +.endm
817 +
818 +.macro memcpy_leading_15bytes backwards, align
819 + movs DAT1, DAT2, lsl #31
820 + sub N, N, DAT2
821 + .if backwards
822 + ldrmib DAT0, [S, #-1]!
823 + ldrcsh DAT1, [S, #-2]!
824 + strmib DAT0, [D, #-1]!
825 + strcsh DAT1, [D, #-2]!
826 + .else
827 + ldrmib DAT0, [S], #1
828 + ldrcsh DAT1, [S], #2
829 + strmib DAT0, [D], #1
830 + strcsh DAT1, [D], #2
831 + .endif
832 + movs DAT1, DAT2, lsl #29
833 + .if backwards
834 + ldrmi DAT0, [S, #-4]!
835 + .if align == 0
836 + ldmcsdb S!, {DAT1, DAT2}
837 + .else
838 + ldrcs DAT2, [S, #-4]!
839 + ldrcs DAT1, [S, #-4]!
840 + .endif
841 + strmi DAT0, [D, #-4]!
842 + stmcsdb D!, {DAT1, DAT2}
843 + .else
844 + ldrmi DAT0, [S], #4
845 + .if align == 0
846 + ldmcsia S!, {DAT1, DAT2}
847 + .else
848 + ldrcs DAT1, [S], #4
849 + ldrcs DAT2, [S], #4
850 + .endif
851 + strmi DAT0, [D], #4
852 + stmcsia D!, {DAT1, DAT2}
853 + .endif
854 +.endm
855 +
856 +.macro memcpy_trailing_15bytes backwards, align
857 + movs N, N, lsl #29
858 + .if backwards
859 + .if align == 0
860 + ldmcsdb S!, {DAT0, DAT1}
861 + .else
862 + ldrcs DAT1, [S, #-4]!
863 + ldrcs DAT0, [S, #-4]!
864 + .endif
865 + ldrmi DAT2, [S, #-4]!
866 + stmcsdb D!, {DAT0, DAT1}
867 + strmi DAT2, [D, #-4]!
868 + .else
869 + .if align == 0
870 + ldmcsia S!, {DAT0, DAT1}
871 + .else
872 + ldrcs DAT0, [S], #4
873 + ldrcs DAT1, [S], #4
874 + .endif
875 + ldrmi DAT2, [S], #4
876 + stmcsia D!, {DAT0, DAT1}
877 + strmi DAT2, [D], #4
878 + .endif
879 + movs N, N, lsl #2
880 + .if backwards
881 + ldrcsh DAT0, [S, #-2]!
882 + ldrmib DAT1, [S, #-1]
883 + strcsh DAT0, [D, #-2]!
884 + strmib DAT1, [D, #-1]
885 + .else
886 + ldrcsh DAT0, [S], #2
887 + ldrmib DAT1, [S]
888 + strcsh DAT0, [D], #2
889 + strmib DAT1, [D]
890 + .endif
891 +.endm
892 +
893 +.macro memcpy_long_inner_loop backwards, align
894 + .if align != 0
895 + .if backwards
896 + ldr DAT0, [S, #-align]!
897 + .else
898 + ldr LAST, [S, #-align]!
899 + .endif
900 + .endif
901 +110:
902 + .if align == 0
903 + .if backwards
904 + ldmdb S!, {DAT0, DAT1, DAT2, DAT3, DAT4, DAT5, DAT6, LAST}
905 + pld [S, OFF]
906 + stmdb D!, {DAT4, DAT5, DAT6, LAST}
907 + stmdb D!, {DAT0, DAT1, DAT2, DAT3}
908 + .else
909 + ldmia S!, {DAT0, DAT1, DAT2, DAT3, DAT4, DAT5, DAT6, LAST}
910 + pld [S, OFF]
911 + stmia D!, {DAT0, DAT1, DAT2, DAT3}
912 + stmia D!, {DAT4, DAT5, DAT6, LAST}
913 + .endif
914 + .else
915 + unaligned_words backwards, align, 1, 8, DAT0, DAT1, DAT2, DAT3, DAT4, DAT5, DAT6, DAT7, LAST
916 + .endif
917 + subs N, N, #32
918 + bhs 110b
919 + /* Just before the final (prefetch_distance+1) 32-byte blocks, deal with final preloads */
920 + preload_trailing backwards, S, N, OFF
921 + add N, N, #(prefetch_distance+2)*32 - 32
922 +120:
923 + .if align == 0
924 + .if backwards
925 + ldmdb S!, {DAT0, DAT1, DAT2, DAT3, DAT4, DAT5, DAT6, LAST}
926 + stmdb D!, {DAT4, DAT5, DAT6, LAST}
927 + stmdb D!, {DAT0, DAT1, DAT2, DAT3}
928 + .else
929 + ldmia S!, {DAT0, DAT1, DAT2, DAT3, DAT4, DAT5, DAT6, LAST}
930 + stmia D!, {DAT0, DAT1, DAT2, DAT3}
931 + stmia D!, {DAT4, DAT5, DAT6, LAST}
932 + .endif
933 + .else
934 + unaligned_words backwards, align, 0, 8, DAT0, DAT1, DAT2, DAT3, DAT4, DAT5, DAT6, DAT7, LAST
935 + .endif
936 + subs N, N, #32
937 + bhs 120b
938 + tst N, #16
939 + .if align == 0
940 + .if backwards
941 + ldmnedb S!, {DAT0, DAT1, DAT2, LAST}
942 + stmnedb D!, {DAT0, DAT1, DAT2, LAST}
943 + .else
944 + ldmneia S!, {DAT0, DAT1, DAT2, LAST}
945 + stmneia D!, {DAT0, DAT1, DAT2, LAST}
946 + .endif
947 + .else
948 + beq 130f
949 + unaligned_words backwards, align, 0, 4, DAT0, DAT1, DAT2, DAT3, LAST
950 +130:
951 + .endif
952 + /* Trailing words and bytes */
953 + tst N, #15
954 + beq 199f
955 + .if align != 0
956 + add S, S, #align
957 + .endif
958 + memcpy_trailing_15bytes backwards, align
959 +199:
960 + pop {DAT3, DAT4, DAT5, DAT6, DAT7}
961 + pop {D, DAT1, DAT2, pc}
962 +.endm
963 +
964 +.macro memcpy_medium_inner_loop backwards, align
965 +120:
966 + .if backwards
967 + .if align == 0
968 + ldmdb S!, {DAT0, DAT1, DAT2, LAST}
969 + .else
970 + ldr LAST, [S, #-4]!
971 + ldr DAT2, [S, #-4]!
972 + ldr DAT1, [S, #-4]!
973 + ldr DAT0, [S, #-4]!
974 + .endif
975 + stmdb D!, {DAT0, DAT1, DAT2, LAST}
976 + .else
977 + .if align == 0
978 + ldmia S!, {DAT0, DAT1, DAT2, LAST}
979 + .else
980 + ldr DAT0, [S], #4
981 + ldr DAT1, [S], #4
982 + ldr DAT2, [S], #4
983 + ldr LAST, [S], #4
984 + .endif
985 + stmia D!, {DAT0, DAT1, DAT2, LAST}
986 + .endif
987 + subs N, N, #16
988 + bhs 120b
989 + /* Trailing words and bytes */
990 + tst N, #15
991 + beq 199f
992 + memcpy_trailing_15bytes backwards, align
993 +199:
994 + pop {D, DAT1, DAT2, pc}
995 +.endm
996 +
997 +.macro memcpy_short_inner_loop backwards, align
998 + tst N, #16
999 + .if backwards
1000 + .if align == 0
1001 + ldmnedb S!, {DAT0, DAT1, DAT2, LAST}
1002 + .else
1003 + ldrne LAST, [S, #-4]!
1004 + ldrne DAT2, [S, #-4]!
1005 + ldrne DAT1, [S, #-4]!
1006 + ldrne DAT0, [S, #-4]!
1007 + .endif
1008 + stmnedb D!, {DAT0, DAT1, DAT2, LAST}
1009 + .else
1010 + .if align == 0
1011 + ldmneia S!, {DAT0, DAT1, DAT2, LAST}
1012 + .else
1013 + ldrne DAT0, [S], #4
1014 + ldrne DAT1, [S], #4
1015 + ldrne DAT2, [S], #4
1016 + ldrne LAST, [S], #4
1017 + .endif
1018 + stmneia D!, {DAT0, DAT1, DAT2, LAST}
1019 + .endif
1020 + memcpy_trailing_15bytes backwards, align
1021 +199:
1022 + pop {D, DAT1, DAT2, pc}
1023 +.endm
1024 +
1025 +.macro memcpy backwards
1026 + D .req a1
1027 + S .req a2
1028 + N .req a3
1029 + DAT0 .req a4
1030 + DAT1 .req v1
1031 + DAT2 .req v2
1032 + DAT3 .req v3
1033 + DAT4 .req v4
1034 + DAT5 .req v5
1035 + DAT6 .req v6
1036 + DAT7 .req sl
1037 + LAST .req ip
1038 + OFF .req lr
1039 +
1040 + .cfi_startproc
1041 +
1042 + push {D, DAT1, DAT2, lr}
1043 +
1044 + .cfi_def_cfa_offset 16
1045 + .cfi_rel_offset D, 0
1046 + .cfi_undefined S
1047 + .cfi_undefined N
1048 + .cfi_undefined DAT0
1049 + .cfi_rel_offset DAT1, 4
1050 + .cfi_rel_offset DAT2, 8
1051 + .cfi_undefined LAST
1052 + .cfi_rel_offset lr, 12
1053 +
1054 + .if backwards
1055 + add D, D, N
1056 + add S, S, N
1057 + .endif
1058 +
1059 + /* See if we're guaranteed to have at least one 16-byte aligned 16-byte write */
1060 + cmp N, #31
1061 + blo 170f
1062 + /* To preload ahead as we go, we need at least (prefetch_distance+2) 32-byte blocks */
1063 + cmp N, #(prefetch_distance+3)*32 - 1
1064 + blo 160f
1065 +
1066 + /* Long case */
1067 + push {DAT3, DAT4, DAT5, DAT6, DAT7}
1068 +
1069 + .cfi_def_cfa_offset 36
1070 + .cfi_rel_offset D, 20
1071 + .cfi_rel_offset DAT1, 24
1072 + .cfi_rel_offset DAT2, 28
1073 + .cfi_rel_offset DAT3, 0
1074 + .cfi_rel_offset DAT4, 4
1075 + .cfi_rel_offset DAT5, 8
1076 + .cfi_rel_offset DAT6, 12
1077 + .cfi_rel_offset DAT7, 16
1078 + .cfi_rel_offset lr, 32
1079 +
1080 + /* Adjust N so that the decrement instruction can also test for
1081 + * inner loop termination. We want it to stop when there are
1082 + * (prefetch_distance+1) complete blocks to go. */
1083 + sub N, N, #(prefetch_distance+2)*32
1084 + preload_leading_step1 backwards, DAT0, S
1085 + .if backwards
1086 + /* Bug in GAS: it accepts, but mis-assembles the instruction
1087 + * ands DAT2, D, #60, 2
1088 + * which sets DAT2 to the number of leading bytes until destination is aligned and also clears C (sets borrow)
1089 + */
1090 + .word 0xE210513C
1091 + beq 154f
1092 + .else
1093 + ands DAT2, D, #15
1094 + beq 154f
1095 + rsb DAT2, DAT2, #16 /* number of leading bytes until destination aligned */
1096 + .endif
1097 + preload_leading_step2 backwards, DAT0, S, DAT2, OFF
1098 + memcpy_leading_15bytes backwards, 1
1099 +154: /* Destination now 16-byte aligned; we have at least one prefetch as well as at least one 16-byte output block */
1100 + /* Prefetch offset is best selected such that it lies in the first 8 of each 32 bytes - but it's just as easy to aim for the first one */
1101 + .if backwards
1102 + rsb OFF, S, #3
1103 + and OFF, OFF, #28
1104 + sub OFF, OFF, #32*(prefetch_distance+1)
1105 + .else
1106 + and OFF, S, #28
1107 + rsb OFF, OFF, #32*prefetch_distance
1108 + .endif
1109 + movs DAT0, S, lsl #31
1110 + bhi 157f
1111 + bcs 156f
1112 + bmi 155f
1113 + memcpy_long_inner_loop backwards, 0
1114 +155: memcpy_long_inner_loop backwards, 1
1115 +156: memcpy_long_inner_loop backwards, 2
1116 +157: memcpy_long_inner_loop backwards, 3
1117 +
1118 + .cfi_def_cfa_offset 16
1119 + .cfi_rel_offset D, 0
1120 + .cfi_rel_offset DAT1, 4
1121 + .cfi_rel_offset DAT2, 8
1122 + .cfi_same_value DAT3
1123 + .cfi_same_value DAT4
1124 + .cfi_same_value DAT5
1125 + .cfi_same_value DAT6
1126 + .cfi_same_value DAT7
1127 + .cfi_rel_offset lr, 12
1128 +
1129 +160: /* Medium case */
1130 + preload_all backwards, 0, 0, S, N, DAT2, OFF
1131 + sub N, N, #16 /* simplifies inner loop termination */
1132 + .if backwards
1133 + ands DAT2, D, #15
1134 + beq 164f
1135 + .else
1136 + ands DAT2, D, #15
1137 + beq 164f
1138 + rsb DAT2, DAT2, #16
1139 + .endif
1140 + memcpy_leading_15bytes backwards, align
1141 +164: /* Destination now 16-byte aligned; we have at least one 16-byte output block */
1142 + tst S, #3
1143 + bne 140f
1144 + memcpy_medium_inner_loop backwards, 0
1145 +140: memcpy_medium_inner_loop backwards, 1
1146 +
1147 +170: /* Short case, less than 31 bytes, so no guarantee of at least one 16-byte block */
1148 + teq N, #0
1149 + beq 199f
1150 + preload_all backwards, 1, 0, S, N, DAT2, LAST
1151 + tst D, #3
1152 + beq 174f
1153 +172: subs N, N, #1
1154 + blo 199f
1155 + .if backwards
1156 + ldrb DAT0, [S, #-1]!
1157 + strb DAT0, [D, #-1]!
1158 + .else
1159 + ldrb DAT0, [S], #1
1160 + strb DAT0, [D], #1
1161 + .endif
1162 + tst D, #3
1163 + bne 172b
1164 +174: /* Destination now 4-byte aligned; we have 0 or more output bytes to go */
1165 + tst S, #3
1166 + bne 140f
1167 + memcpy_short_inner_loop backwards, 0
1168 +140: memcpy_short_inner_loop backwards, 1
1169 +
1170 + .cfi_endproc
1171 +
1172 + .unreq D
1173 + .unreq S
1174 + .unreq N
1175 + .unreq DAT0
1176 + .unreq DAT1
1177 + .unreq DAT2
1178 + .unreq DAT3
1179 + .unreq DAT4
1180 + .unreq DAT5
1181 + .unreq DAT6
1182 + .unreq DAT7
1183 + .unreq LAST
1184 + .unreq OFF
1185 +.endm
1186 diff --git a/arch/arm/lib/memmove_rpi.S b/arch/arm/lib/memmove_rpi.S
1187 new file mode 100644
1188 index 0000000..8b0760c
1189 --- /dev/null
1190 +++ b/arch/arm/lib/memmove_rpi.S
1191 @@ -0,0 +1,61 @@
1192 +/*
1193 +Copyright (c) 2013, Raspberry Pi Foundation
1194 +Copyright (c) 2013, RISC OS Open Ltd
1195 +All rights reserved.
1196 +
1197 +Redistribution and use in source and binary forms, with or without
1198 +modification, are permitted provided that the following conditions are met:
1199 + * Redistributions of source code must retain the above copyright
1200 + notice, this list of conditions and the following disclaimer.
1201 + * Redistributions in binary form must reproduce the above copyright
1202 + notice, this list of conditions and the following disclaimer in the
1203 + documentation and/or other materials provided with the distribution.
1204 + * Neither the name of the copyright holder nor the
1205 + names of its contributors may be used to endorse or promote products
1206 + derived from this software without specific prior written permission.
1207 +
1208 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
1209 +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
1210 +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
1211 +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
1212 +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
1213 +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
1214 +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
1215 +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
1216 +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
1217 +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1218 +*/
1219 +
1220 +#include <linux/linkage.h>
1221 +#include "arm-mem.h"
1222 +#include "memcpymove.h"
1223 +
1224 +/* Prevent the stack from becoming executable */
1225 +#if defined(__linux__) && defined(__ELF__)
1226 +.section .note.GNU-stack,"",%progbits
1227 +#endif
1228 +
1229 + .text
1230 + .arch armv6
1231 + .object_arch armv4
1232 + .arm
1233 + .altmacro
1234 + .p2align 2
1235 +
1236 +/*
1237 + * void *memmove(void *s1, const void *s2, size_t n);
1238 + * On entry:
1239 + * a1 = pointer to destination
1240 + * a2 = pointer to source
1241 + * a3 = number of bytes to copy
1242 + * On exit:
1243 + * a1 preserved
1244 + */
1245 +
1246 +.set prefetch_distance, 3
1247 +
1248 +ENTRY(memmove)
1249 + cmp a2, a1
1250 + bpl memcpy /* pl works even over -1 - 0 and 0x7fffffff - 0x80000000 boundaries */
1251 + memcpy 1
1252 +ENDPROC(memmove)
1253 diff --git a/arch/arm/lib/memset_rpi.S b/arch/arm/lib/memset_rpi.S
1254 new file mode 100644
1255 index 0000000..2cde883
1256 --- /dev/null
1257 +++ b/arch/arm/lib/memset_rpi.S
1258 @@ -0,0 +1,121 @@
1259 +/*
1260 +Copyright (c) 2013, Raspberry Pi Foundation
1261 +Copyright (c) 2013, RISC OS Open Ltd
1262 +All rights reserved.
1263 +
1264 +Redistribution and use in source and binary forms, with or without
1265 +modification, are permitted provided that the following conditions are met:
1266 + * Redistributions of source code must retain the above copyright
1267 + notice, this list of conditions and the following disclaimer.
1268 + * Redistributions in binary form must reproduce the above copyright
1269 + notice, this list of conditions and the following disclaimer in the
1270 + documentation and/or other materials provided with the distribution.
1271 + * Neither the name of the copyright holder nor the
1272 + names of its contributors may be used to endorse or promote products
1273 + derived from this software without specific prior written permission.
1274 +
1275 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
1276 +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
1277 +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
1278 +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
1279 +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
1280 +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
1281 +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
1282 +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
1283 +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
1284 +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1285 +*/
1286 +
1287 +#include <linux/linkage.h>
1288 +#include "arm-mem.h"
1289 +
1290 +/* Prevent the stack from becoming executable */
1291 +#if defined(__linux__) && defined(__ELF__)
1292 +.section .note.GNU-stack,"",%progbits
1293 +#endif
1294 +
1295 + .text
1296 + .arch armv6
1297 + .object_arch armv4
1298 + .arm
1299 + .altmacro
1300 + .p2align 2
1301 +
1302 +/*
1303 + * void *memset(void *s, int c, size_t n);
1304 + * On entry:
1305 + * a1 = pointer to buffer to fill
1306 + * a2 = byte pattern to fill with (caller-narrowed)
1307 + * a3 = number of bytes to fill
1308 + * On exit:
1309 + * a1 preserved
1310 + */
1311 +ENTRY(memset)
1312 + S .req a1
1313 + DAT0 .req a2
1314 + N .req a3
1315 + DAT1 .req a4
1316 + DAT2 .req ip
1317 + DAT3 .req lr
1318 +
1319 + orr DAT0, DAT0, lsl #8
1320 + push {S, lr}
1321 + orr DAT0, DAT0, lsl #16
1322 + mov DAT1, DAT0
1323 +
1324 + /* See if we're guaranteed to have at least one 16-byte aligned 16-byte write */
1325 + cmp N, #31
1326 + blo 170f
1327 +
1328 +161: sub N, N, #16 /* simplifies inner loop termination */
1329 + /* Leading words and bytes */
1330 + tst S, #15
1331 + beq 164f
1332 + rsb DAT3, S, #0 /* bits 0-3 = number of leading bytes until aligned */
1333 + movs DAT2, DAT3, lsl #31
1334 + submi N, N, #1
1335 + strmib DAT0, [S], #1
1336 + subcs N, N, #2
1337 + strcsh DAT0, [S], #2
1338 + movs DAT2, DAT3, lsl #29
1339 + submi N, N, #4
1340 + strmi DAT0, [S], #4
1341 + subcs N, N, #8
1342 + stmcsia S!, {DAT0, DAT1}
1343 +164: /* Delayed set up of DAT2 and DAT3 so we could use them as scratch registers above */
1344 + mov DAT2, DAT0
1345 + mov DAT3, DAT0
1346 + /* Now the inner loop of 16-byte stores */
1347 +165: stmia S!, {DAT0, DAT1, DAT2, DAT3}
1348 + subs N, N, #16
1349 + bhs 165b
1350 +166: /* Trailing words and bytes */
1351 + movs N, N, lsl #29
1352 + stmcsia S!, {DAT0, DAT1}
1353 + strmi DAT0, [S], #4
1354 + movs N, N, lsl #2
1355 + strcsh DAT0, [S], #2
1356 + strmib DAT0, [S]
1357 +199: pop {S, pc}
1358 +
1359 +170: /* Short case */
1360 + mov DAT2, DAT0
1361 + mov DAT3, DAT0
1362 + tst S, #3
1363 + beq 174f
1364 +172: subs N, N, #1
1365 + blo 199b
1366 + strb DAT0, [S], #1
1367 + tst S, #3
1368 + bne 172b
1369 +174: tst N, #16
1370 + stmneia S!, {DAT0, DAT1, DAT2, DAT3}
1371 + b 166b
1372 +
1373 + .unreq S
1374 + .unreq DAT0
1375 + .unreq N
1376 + .unreq DAT1
1377 + .unreq DAT2
1378 + .unreq DAT3
1379 +ENDPROC(memset)
1380 diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c
1381 index 3e58d71..0622891 100644
1382 --- a/arch/arm/lib/uaccess_with_memcpy.c
1383 +++ b/arch/arm/lib/uaccess_with_memcpy.c
1384 @@ -22,6 +22,14 @@
1385 #include <asm/current.h>
1386 #include <asm/page.h>
1387
1388 +#ifndef COPY_FROM_USER_THRESHOLD
1389 +#define COPY_FROM_USER_THRESHOLD 64
1390 +#endif
1391 +
1392 +#ifndef COPY_TO_USER_THRESHOLD
1393 +#define COPY_TO_USER_THRESHOLD 64
1394 +#endif
1395 +
1396 static int
1397 pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
1398 {
1399 @@ -85,7 +93,44 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
1400 return 1;
1401 }
1402
1403 -static unsigned long noinline
1404 +static int
1405 +pin_page_for_read(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
1406 +{
1407 + unsigned long addr = (unsigned long)_addr;
1408 + pgd_t *pgd;
1409 + pmd_t *pmd;
1410 + pte_t *pte;
1411 + pud_t *pud;
1412 + spinlock_t *ptl;
1413 +
1414 + pgd = pgd_offset(current->mm, addr);
1415 + if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd)))
1416 + {
1417 + return 0;
1418 + }
1419 + pud = pud_offset(pgd, addr);
1420 + if (unlikely(pud_none(*pud) || pud_bad(*pud)))
1421 + {
1422 + return 0;
1423 + }
1424 +
1425 + pmd = pmd_offset(pud, addr);
1426 + if (unlikely(pmd_none(*pmd) || pmd_bad(*pmd)))
1427 + return 0;
1428 +
1429 + pte = pte_offset_map_lock(current->mm, pmd, addr, &ptl);
1430 + if (unlikely(!pte_present(*pte) || !pte_young(*pte))) {
1431 + pte_unmap_unlock(pte, ptl);
1432 + return 0;
1433 + }
1434 +
1435 + *ptep = pte;
1436 + *ptlp = ptl;
1437 +
1438 + return 1;
1439 +}
1440 +
1441 +unsigned long noinline
1442 __copy_to_user_memcpy(void __user *to, const void *from, unsigned long n)
1443 {
1444 int atomic;
1445 @@ -135,6 +180,54 @@ out:
1446 return n;
1447 }
1448
1449 +unsigned long noinline
1450 +__copy_from_user_memcpy(void *to, const void __user *from, unsigned long n)
1451 +{
1452 + int atomic;
1453 +
1454 + if (unlikely(segment_eq(get_fs(), KERNEL_DS))) {
1455 + memcpy(to, (const void *)from, n);
1456 + return 0;
1457 + }
1458 +
1459 + /* the mmap semaphore is taken only if not in an atomic context */
1460 + atomic = in_atomic();
1461 +
1462 + if (!atomic)
1463 + down_read(&current->mm->mmap_sem);
1464 + while (n) {
1465 + pte_t *pte;
1466 + spinlock_t *ptl;
1467 + int tocopy;
1468 +
1469 + while (!pin_page_for_read(from, &pte, &ptl)) {
1470 + char temp;
1471 + if (!atomic)
1472 + up_read(&current->mm->mmap_sem);
1473 + if (__get_user(temp, (char __user *)from))
1474 + goto out;
1475 + if (!atomic)
1476 + down_read(&current->mm->mmap_sem);
1477 + }
1478 +
1479 + tocopy = (~(unsigned long)from & ~PAGE_MASK) + 1;
1480 + if (tocopy > n)
1481 + tocopy = n;
1482 +
1483 + memcpy(to, (const void *)from, tocopy);
1484 + to += tocopy;
1485 + from += tocopy;
1486 + n -= tocopy;
1487 +
1488 + pte_unmap_unlock(pte, ptl);
1489 + }
1490 + if (!atomic)
1491 + up_read(&current->mm->mmap_sem);
1492 +
1493 +out:
1494 + return n;
1495 +}
1496 +
1497 unsigned long
1498 __copy_to_user(void __user *to, const void *from, unsigned long n)
1499 {
1500 @@ -145,10 +238,25 @@ __copy_to_user(void __user *to, const void *from, unsigned long n)
1501 * With frame pointer disabled, tail call optimization kicks in
1502 * as well making this test almost invisible.
1503 */
1504 - if (n < 64)
1505 + if (n < COPY_TO_USER_THRESHOLD)
1506 return __copy_to_user_std(to, from, n);
1507 return __copy_to_user_memcpy(to, from, n);
1508 }
1509 +
1510 +unsigned long
1511 +__copy_from_user(void *to, const void __user *from, unsigned long n)
1512 +{
1513 + /*
1514 + * This test is stubbed out of the main function above to keep
1515 + * the overhead for small copies low by avoiding a large
1516 + * register dump on the stack just to reload them right away.
1517 + * With frame pointer disabled, tail call optimization kicks in
1518 + * as well making this test almost invisible.
1519 + */
1520 + if (n < COPY_FROM_USER_THRESHOLD)
1521 + return __copy_from_user_std(to, from, n);
1522 + return __copy_from_user_memcpy(to, from, n);
1523 +}
1524
1525 static unsigned long noinline
1526 __clear_user_memset(void __user *addr, unsigned long n)
1527 --
1528 1.8.3.2
1529
Staging tree of Zoltan Herpai