kevin/linux
1
Fork 0
Code
73196cd364
linux/arch/sparc/lib/NGbzero.S

161 lines
3.3 KiB
ArmAsm
Raw Normal View History

[SPARC64]: Niagara optimized memset/bzero/clear_user. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-21 15:29:42 -07:00
/* NGbzero.S: Niagara optimized memset/clear_user.
*
* Copyright (C) 2006 David S. Miller (davem@davemloft.net)
*/
#include <asm/asi.h>
#define EX_ST(x,y) \
98: x,y; \
[SPARC64]: Fix inconsistent .section usage in lib/ A few places missed the "a" specifier for the __ex_table section. Add these so we avoid generation an additional section at link time. Latest modpost would otherwise complain like this: WARNING: vmlinux.o (__ex_table.2): section name inconsistency. (.[number]+) following section name. Did you forget to use "ax"/"aw" in a .S file? Note that for example <linux/init.h> contains section definitions for use in .S files. WARNING: vmlinux.o (__ex_table.4): section name inconsistency. (.[number]+) following section name. Did you forget to use "ax"/"aw" in a .S file? Note that for example <linux/init.h> contains section definitions for use in .S files. Signed-off-by: Sam Ravnborg <sam@ravnborg.org> Signed-off-by: David S. Miller <davem@davemloft.net>
2008-01-31 17:48:49 -07:00
.section __ex_table,"a";\
[SPARC64]: Niagara optimized memset/bzero/clear_user. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-21 15:29:42 -07:00
.align 4; \
sparc64: Kill .fixup section bloat. This is an implementation of a suggestion made by Chris Torek: -------------------- Something else I noticed in passing: the EX and EX_LD/EX_ST macros scattered throughout the various .S files make a fair bit of .fixup code, all of which does the same thing. At the cost of one symbol in copy_in_user.S, you could just have one common two-instruction retl-and-mov-1 fixup that they all share. -------------------- The following is with a defconfig build: text data bss dec hex filename 3972767 344024 584449 4901240 4ac978 vmlinux.orig 3968887 344024 584449 4897360 4aba50 vmlinux Signed-off-by: David S. Miller <davem@davemloft.net>
2009-02-08 23:00:55 -07:00
.word 98b, __retl_o1; \
[SPARC64]: Niagara optimized memset/bzero/clear_user. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-21 15:29:42 -07:00
.text; \
.align 4;
.text
.globl NGmemset
.type NGmemset, #function
NGmemset: /* %o0=buf, %o1=pat, %o2=len */
and %o1, 0xff, %o3
mov %o2, %o1
sllx %o3, 8, %g1
or %g1, %o3, %o2
sllx %o2, 16, %g1
or %g1, %o2, %o2
sllx %o2, 32, %g1
ba,pt %xcc, 1f
or %g1, %o2, %o2
.globl NGbzero
.type NGbzero, #function
NGbzero:
clr %o2
1: brz,pn %o1, NGbzero_return
mov %o0, %o3
/* %o5: saved %asi, restored at NGbzero_done
* %g7: store-init %asi to use
* %o4: non-store-init %asi to use
*/
rd %asi, %o5
mov ASI_BLK_INIT_QUAD_LDD_P, %g7
mov ASI_P, %o4
wr %o4, 0x0, %asi
NGbzero_from_clear_user:
cmp %o1, 15
bl,pn %icc, NGbzero_tiny
andcc %o0, 0x7, %g1
be,pt %xcc, 2f
mov 8, %g2
sub %g2, %g1, %g1
sub %o1, %g1, %o1
1: EX_ST(stba %o2, [%o0 + 0x00] %asi)
subcc %g1, 1, %g1
bne,pt %xcc, 1b
add %o0, 1, %o0
2: cmp %o1, 128
bl,pn %icc, NGbzero_medium
andcc %o0, (64 - 1), %g1
be,pt %xcc, NGbzero_pre_loop
mov 64, %g2
sub %g2, %g1, %g1
sub %o1, %g1, %o1
1: EX_ST(stxa %o2, [%o0 + 0x00] %asi)
subcc %g1, 8, %g1
bne,pt %xcc, 1b
add %o0, 8, %o0
NGbzero_pre_loop:
wr %g7, 0x0, %asi
andn %o1, (64 - 1), %g1
sub %o1, %g1, %o1
NGbzero_loop:
EX_ST(stxa %o2, [%o0 + 0x00] %asi)
EX_ST(stxa %o2, [%o0 + 0x08] %asi)
EX_ST(stxa %o2, [%o0 + 0x10] %asi)
EX_ST(stxa %o2, [%o0 + 0x18] %asi)
EX_ST(stxa %o2, [%o0 + 0x20] %asi)
EX_ST(stxa %o2, [%o0 + 0x28] %asi)
EX_ST(stxa %o2, [%o0 + 0x30] %asi)
EX_ST(stxa %o2, [%o0 + 0x38] %asi)
subcc %g1, 64, %g1
bne,pt %xcc, NGbzero_loop
add %o0, 64, %o0
[SPARC64]: store-init needs trailing membar. The manual says that it is required and we actually have crash reports where loads see stale data due to not having membars here. In one case the networking does: memset(skb, 0, offsetof(struct sk_buff, truesize)); and then some code later checks skb->nohdr for zero, but it's still the value that was there before the memset(). Note that arch/sparc64/lib/xor.S already got this right. Signed-off-by: David S. Miller <davem@davemloft.net>
2007-03-19 13:27:33 -07:00
membar #Sync
[SPARC64]: Niagara optimized memset/bzero/clear_user. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-21 15:29:42 -07:00
wr %o4, 0x0, %asi
brz,pn %o1, NGbzero_done
NGbzero_medium:
andncc %o1, 0x7, %g1
be,pn %xcc, 2f
sub %o1, %g1, %o1
1: EX_ST(stxa %o2, [%o0 + 0x00] %asi)
subcc %g1, 8, %g1
bne,pt %xcc, 1b
add %o0, 8, %o0
2: brz,pt %o1, NGbzero_done
nop
NGbzero_tiny:
1: EX_ST(stba %o2, [%o0 + 0x00] %asi)
subcc %o1, 1, %o1
bne,pt %icc, 1b
add %o0, 1, %o0
/* fallthrough */
NGbzero_done:
wr %o5, 0x0, %asi
NGbzero_return:
retl
mov %o3, %o0
.size NGbzero, .-NGbzero
.size NGmemset, .-NGmemset
.globl NGclear_user
.type NGclear_user, #function
NGclear_user: /* %o0=buf, %o1=len */
rd %asi, %o5
brz,pn %o1, NGbzero_done
clr %o3
cmp %o5, ASI_AIUS
bne,pn %icc, NGbzero
clr %o2
mov ASI_BLK_INIT_QUAD_LDD_AIUS, %g7
ba,pt %xcc, NGbzero_from_clear_user
mov ASI_AIUS, %o4
.size NGclear_user, .-NGclear_user
#define BRANCH_ALWAYS 0x10680000
#define NOP 0x01000000
#define NG_DO_PATCH(OLD, NEW) \
sethi %hi(NEW), %g1; \
or %g1, %lo(NEW), %g1; \
sethi %hi(OLD), %g2; \
or %g2, %lo(OLD), %g2; \
sub %g1, %g2, %g1; \
sethi %hi(BRANCH_ALWAYS), %g3; \
sll %g1, 11, %g1; \
srl %g1, 11 + 2, %g1; \
or %g3, %lo(BRANCH_ALWAYS), %g3; \
or %g3, %g1, %g3; \
stw %g3, [%g2]; \
sethi %hi(NOP), %g3; \
or %g3, %lo(NOP), %g3; \
stw %g3, [%g2 + 0x4]; \
flush %g2;
.globl niagara_patch_bzero
.type niagara_patch_bzero,#function
niagara_patch_bzero:
NG_DO_PATCH(memset, NGmemset)
NG_DO_PATCH(__bzero, NGbzero)
NG_DO_PATCH(__clear_user, NGclear_user)
[SPARC64]: Optimized TSB table initialization. We only need to write an invalid tag every 16 bytes, so taking advantage of this can save many instructions compared to the simple memset() call we make now. A prefetching implementation is implemented for sun4u and a block-init store version if implemented for Niagara. The next trick is to be able to perform an init and a copy_tsb() in parallel when growing a TSB table. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-19 00:55:11 -07:00
NG_DO_PATCH(tsb_init, NGtsb_init)
[SPARC64]: Niagara optimized memset/bzero/clear_user. Signed-off-by: David S. Miller <davem@davemloft.net>
2006-02-21 15:29:42 -07:00
retl
nop
.size niagara_patch_bzero,.-niagara_patch_bzero
Copy Permalink