1 From: Felix Fietkau <nbd@nbd.name>
2 Subject: mips: replace -mlong-calls with -mno-long-calls to make function calls faster in kernel modules to achieve this, try to
4 lede-commit: 3b3d64743ba2a874df9d70cd19e242205b0a788c
5 Signed-off-by: Felix Fietkau <nbd@nbd.name>
7 arch/mips/Makefile | 5 +
8 arch/mips/include/asm/module.h | 5 +
9 arch/mips/kernel/module.c | 279 ++++++++++++++++++++++++++++++++++++++++-
10 3 files changed, 284 insertions(+), 5 deletions(-)
12 diff --git a/arch/mips/Makefile b/arch/mips/Makefile
13 index 48dc1a9c3e42..77bf5db20d65 100644
14 --- a/arch/mips/Makefile
15 +++ b/arch/mips/Makefile
16 @@ -93,8 +93,13 @@ all-$(CONFIG_SYS_SUPPORTS_ZBOOT)+= vmlinuz
17 cflags-y += -G 0 -mno-abicalls -fno-pic -pipe -mno-branch-likely
18 cflags-y += -msoft-float
19 LDFLAGS_vmlinux += -G 0 -static -n -nostdlib
21 KBUILD_AFLAGS_MODULE += -mlong-calls
22 KBUILD_CFLAGS_MODULE += -mlong-calls
24 +KBUILD_AFLAGS_MODULE += -mno-long-calls
25 +KBUILD_CFLAGS_MODULE += -mno-long-calls
28 ifeq ($(CONFIG_RELOCATABLE),y)
29 LDFLAGS_vmlinux += --emit-relocs
30 diff --git a/arch/mips/include/asm/module.h b/arch/mips/include/asm/module.h
31 index 702c273e67a9..1d4f3b37cefe 100644
32 --- a/arch/mips/include/asm/module.h
33 +++ b/arch/mips/include/asm/module.h
34 @@ -11,6 +11,11 @@ struct mod_arch_specific {
35 const struct exception_table_entry *dbe_start;
36 const struct exception_table_entry *dbe_end;
37 struct mips_hi16 *r_mips_hi16_list;
41 + unsigned int phys_plt_offset;
42 + unsigned int virt_plt_offset;
45 typedef uint8_t Elf64_Byte; /* Type for a 8-bit quantity. */
46 diff --git a/arch/mips/kernel/module.c b/arch/mips/kernel/module.c
47 index 94627a3a6a0d..947981a9aa72 100644
48 --- a/arch/mips/kernel/module.c
49 +++ b/arch/mips/kernel/module.c
50 @@ -44,14 +44,221 @@ struct mips_hi16 {
51 static LIST_HEAD(dbe_list);
52 static DEFINE_SPINLOCK(dbe_lock);
56 + * Get the potential max trampolines size required of the init and
57 + * non-init sections. Only used if we cannot find enough contiguous
58 + * physically mapped memory to put the module into.
61 +get_plt_size(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
62 + const char *secstrings, unsigned int symindex, bool is_init)
64 + unsigned long ret = 0;
68 + /* Everything marked ALLOC (this includes the exported symbols) */
69 + for (i = 1; i < hdr->e_shnum; ++i) {
70 + unsigned int info = sechdrs[i].sh_info;
72 + if (sechdrs[i].sh_type != SHT_REL
73 + && sechdrs[i].sh_type != SHT_RELA)
76 + /* Not a valid relocation section? */
77 + if (info >= hdr->e_shnum)
80 + /* Don't bother with non-allocated sections */
81 + if (!(sechdrs[info].sh_flags & SHF_ALLOC))
84 + /* If it's called *.init*, and we're not init, we're
86 + if ((strstr(secstrings + sechdrs[i].sh_name, ".init") != 0)
90 + syms = (Elf_Sym *) sechdrs[symindex].sh_addr;
91 + if (sechdrs[i].sh_type == SHT_REL) {
92 + Elf_Mips_Rel *rel = (void *) sechdrs[i].sh_addr;
93 + unsigned int size = sechdrs[i].sh_size / sizeof(*rel);
95 + for (j = 0; j < size; ++j) {
98 + if (ELF_MIPS_R_TYPE(rel[j]) != R_MIPS_26)
101 + sym = syms + ELF_MIPS_R_SYM(rel[j]);
102 + if (!is_init && sym->st_shndx != SHN_UNDEF)
105 + ret += 4 * sizeof(int);
108 + Elf_Mips_Rela *rela = (void *) sechdrs[i].sh_addr;
109 + unsigned int size = sechdrs[i].sh_size / sizeof(*rela);
111 + for (j = 0; j < size; ++j) {
114 + if (ELF_MIPS_R_TYPE(rela[j]) != R_MIPS_26)
117 + sym = syms + ELF_MIPS_R_SYM(rela[j]);
118 + if (!is_init && sym->st_shndx != SHN_UNDEF)
121 + ret += 4 * sizeof(int);
129 +#ifndef MODULE_START
130 +static void *alloc_phys(unsigned long size)
136 + size = PAGE_ALIGN(size);
137 + order = get_order(size);
139 + page = alloc_pages(GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN |
140 + __GFP_THISNODE, order);
144 + split_page(page, order);
146 + /* mark all pages except for the last one */
147 + for (p = page; p + 1 < page + (size >> PAGE_SHIFT); ++p)
148 + set_bit(PG_owner_priv_1, &p->flags);
150 + for (p = page + (size >> PAGE_SHIFT); p < page + (1 << order); ++p)
153 + return page_address(page);
157 +static void free_phys(void *ptr)
162 + page = virt_to_page(ptr);
164 + free = test_and_clear_bit(PG_owner_priv_1, &page->flags);
171 void *module_alloc(unsigned long size)
174 return __vmalloc_node_range(size, 1, MODULE_START, MODULE_END,
175 GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE,
176 __builtin_return_address(0));
183 + ptr = alloc_phys(size);
185 + /* If we failed to allocate physically contiguous memory,
186 + * fall back to regular vmalloc. The module loader code will
187 + * create jump tables to handle long jumps */
189 + return vmalloc(size);
195 +static inline bool is_phys_addr(void *ptr)
198 + return (KSEGX((unsigned long)ptr) == CKSEG0);
200 + return (KSEGX(ptr) == KSEG0);
204 +/* Free memory returned from module_alloc */
205 +void module_memfree(void *module_region)
207 + if (is_phys_addr(module_region))
208 + free_phys(module_region);
210 + vfree(module_region);
213 +static void *__module_alloc(int size, bool phys)
218 + ptr = kmalloc(size, GFP_KERNEL);
220 + ptr = vmalloc(size);
224 +static void __module_free(void *ptr)
226 + if (is_phys_addr(ptr))
232 +int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
233 + char *secstrings, struct module *mod)
235 + unsigned int symindex = 0;
236 + unsigned int core_size, init_size;
239 + mod->arch.phys_plt_offset = 0;
240 + mod->arch.virt_plt_offset = 0;
241 + mod->arch.phys_plt_tbl = NULL;
242 + mod->arch.virt_plt_tbl = NULL;
244 + if (IS_ENABLED(CONFIG_64BIT))
247 + for (i = 1; i < hdr->e_shnum; i++)
248 + if (sechdrs[i].sh_type == SHT_SYMTAB)
251 + core_size = get_plt_size(hdr, sechdrs, secstrings, symindex, false);
252 + init_size = get_plt_size(hdr, sechdrs, secstrings, symindex, true);
254 + if ((core_size + init_size) == 0)
257 + mod->arch.phys_plt_tbl = __module_alloc(core_size + init_size, 1);
258 + if (!mod->arch.phys_plt_tbl)
261 + mod->arch.virt_plt_tbl = __module_alloc(core_size + init_size, 0);
262 + if (!mod->arch.virt_plt_tbl) {
263 + __module_free(mod->arch.phys_plt_tbl);
264 + mod->arch.phys_plt_tbl = NULL;
271 int apply_r_mips_none(struct module *me, u32 *location, Elf_Addr v)
273 @@ -65,8 +272,39 @@ static int apply_r_mips_32_rel(struct module *me, u32 *location, Elf_Addr v)
277 +static Elf_Addr add_plt_entry_to(unsigned *plt_offset,
278 + void *start, Elf_Addr v)
280 + unsigned *tramp = start + *plt_offset;
281 + *plt_offset += 4 * sizeof(int);
283 + /* adjust carry for addiu */
284 + if (v & 0x00008000)
287 + tramp[0] = 0x3c190000 | (v >> 16); /* lui t9, hi16 */
288 + tramp[1] = 0x27390000 | (v & 0xffff); /* addiu t9, t9, lo16 */
289 + tramp[2] = 0x03200008; /* jr t9 */
290 + tramp[3] = 0x00000000; /* nop */
292 + return (Elf_Addr) tramp;
295 +static Elf_Addr add_plt_entry(struct module *me, void *location, Elf_Addr v)
297 + if (is_phys_addr(location))
298 + return add_plt_entry_to(&me->arch.phys_plt_offset,
299 + me->arch.phys_plt_tbl, v);
301 + return add_plt_entry_to(&me->arch.virt_plt_offset,
302 + me->arch.virt_plt_tbl, v);
306 static int apply_r_mips_26_rel(struct module *me, u32 *location, Elf_Addr v)
308 + u32 ofs = *location & 0x03ffffff;
311 pr_err("module %s: dangerous R_MIPS_26 REL relocation\n",
313 @@ -74,13 +312,17 @@ static int apply_r_mips_26_rel(struct module *me, u32 *location, Elf_Addr v)
316 if ((v & 0xf0000000) != (((unsigned long)location + 4) & 0xf0000000)) {
317 - pr_err("module %s: relocation overflow\n",
320 + v = add_plt_entry(me, location, v + (ofs << 2));
322 + pr_err("module %s: relocation overflow\n",
329 *location = (*location & ~0x03ffffff) |
330 - ((*location + (v >> 2)) & 0x03ffffff);
331 + ((ofs + (v >> 2)) & 0x03ffffff);
335 @@ -349,9 +591,36 @@ int module_finalize(const Elf_Ehdr *hdr,
336 list_add(&me->arch.dbe_list, &dbe_list);
337 spin_unlock_irq(&dbe_lock);
340 + /* Get rid of the fixup trampoline if we're running the module
341 + * from physically mapped address space */
342 + if (me->arch.phys_plt_offset == 0) {
343 + __module_free(me->arch.phys_plt_tbl);
344 + me->arch.phys_plt_tbl = NULL;
346 + if (me->arch.virt_plt_offset == 0) {
347 + __module_free(me->arch.virt_plt_tbl);
348 + me->arch.virt_plt_tbl = NULL;
354 +void module_arch_freeing_init(struct module *mod)
356 + if (mod->state == MODULE_STATE_LIVE)
359 + if (mod->arch.phys_plt_tbl) {
360 + __module_free(mod->arch.phys_plt_tbl);
361 + mod->arch.phys_plt_tbl = NULL;
363 + if (mod->arch.virt_plt_tbl) {
364 + __module_free(mod->arch.virt_plt_tbl);
365 + mod->arch.virt_plt_tbl = NULL;
369 void module_arch_cleanup(struct module *mod)
371 spin_lock_irq(&dbe_lock);