LoongArch: modules/ftrace: Initialize PLT at load time

This patch implements ftrace trampolines through plt entry.

Tested by forcing ftrace_make_call() to use the module PLT, and then
loading up a module after setting up ftrace with:

| echo ":mod:<module-name>" > set_ftrace_filter;
| echo function > current_tracer;
| modprobe <module-name>

Since FTRACE_ADDR/FTRACE_REGS_ADDR is only defined when CONFIG_DYNAMIC_
FTRACE is selected, we wrap their usage in module_init_ftrace_plt() with
ifdeffery rather than using IS_ENABLED().

Signed-off-by: Qing Zhang <zhangqing@loongson.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>

diff --git a/arch/loongarch/include/asm/ftrace.h b/arch/loongarch/include/asm/ftrace.h
index 8c7d137..90f9d33 100644 (file)
--- a/arch/loongarch/include/asm/ftrace.h
+++ b/arch/loongarch/include/asm/ftrace.h
@@ -6,6 +6,10 @@
 #ifndef _ASM_LOONGARCH_FTRACE_H
 #define _ASM_LOONGARCH_FTRACE_H
 
+#define FTRACE_PLT_IDX         0
+#define FTRACE_REGS_PLT_IDX    1
+#define NR_FTRACE_PLTS         2
+
 #define GRAPH_FAKE_OFFSET (sizeof(struct pt_regs) - offsetof(struct pt_regs, regs[1]))
 
 #ifdef CONFIG_FUNCTION_TRACER

diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h
index 88e1673..c00e151 100644 (file)
--- a/arch/loongarch/include/asm/inst.h
+++ b/arch/loongarch/include/asm/inst.h
@@ -13,10 +13,12 @@
 
 #define ADDR_IMMMASK_LU52ID    0xFFF0000000000000
 #define ADDR_IMMMASK_LU32ID    0x000FFFFF00000000
+#define ADDR_IMMMASK_LU12IW    0x00000000FFFFF000
 #define ADDR_IMMMASK_ADDU16ID  0x00000000FFFF0000
 
 #define ADDR_IMMSHIFT_LU52ID   52
 #define ADDR_IMMSHIFT_LU32ID   32
+#define ADDR_IMMSHIFT_LU12IW   12
 #define ADDR_IMMSHIFT_ADDU16ID 16
 
 #define ADDR_IMM(addr, INSN)   ((addr & ADDR_IMMMASK_##INSN) >> ADDR_IMMSHIFT_##INSN)
@@ -360,6 +362,7 @@ u32 larch_insn_gen_bl(unsigned long pc, unsigned long dest);
 u32 larch_insn_gen_or(enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk);
 u32 larch_insn_gen_move(enum loongarch_gpr rd, enum loongarch_gpr rj);
 
+u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm);
 u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm);
 u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
 u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, unsigned long pc, unsigned long dest);

diff --git a/arch/loongarch/include/asm/module.h b/arch/loongarch/include/asm/module.h
index 60dc62a..12a0f1e 100644 (file)
--- a/arch/loongarch/include/asm/module.h
+++ b/arch/loongarch/include/asm/module.h
@@ -20,6 +20,9 @@ struct mod_arch_specific {
        struct mod_section got;
        struct mod_section plt;
        struct mod_section plt_idx;
+
+       /* For CONFIG_DYNAMIC_FTRACE */
+       struct plt_entry *ftrace_trampolines;
 };
 
 struct got_entry {
@@ -49,7 +52,7 @@ static inline struct plt_entry emit_plt_entry(unsigned long val)
 {
        u32 lu12iw, lu32id, lu52id, jirl;
 
-       lu12iw = (lu12iw_op << 25 | (((val >> 12) & 0xfffff) << 5) | LOONGARCH_GPR_T1);
+       lu12iw = larch_insn_gen_lu12iw(LOONGARCH_GPR_T1, ADDR_IMM(val, LU12IW));
        lu32id = larch_insn_gen_lu32id(LOONGARCH_GPR_T1, ADDR_IMM(val, LU32ID));
        lu52id = larch_insn_gen_lu52id(LOONGARCH_GPR_T1, LOONGARCH_GPR_T1, ADDR_IMM(val, LU52ID));
        jirl = larch_insn_gen_jirl(0, LOONGARCH_GPR_T1, 0, (val & 0xfff));

diff --git a/arch/loongarch/include/asm/module.lds.h b/arch/loongarch/include/asm/module.lds.h
index a3d1bc0..438f09d 100644 (file)
--- a/arch/loongarch/include/asm/module.lds.h
+++ b/arch/loongarch/include/asm/module.lds.h
@@ -5,4 +5,5 @@ SECTIONS {
        .got : { BYTE(0) }
        .plt : { BYTE(0) }
        .plt.idx : { BYTE(0) }
+       .ftrace_trampoline : { BYTE(0) }
 }

diff --git a/arch/loongarch/kernel/ftrace_dyn.c b/arch/loongarch/kernel/ftrace_dyn.c
index e23c3be..0f07591 100644 (file)
--- a/arch/loongarch/kernel/ftrace_dyn.c
+++ b/arch/loongarch/kernel/ftrace_dyn.c
@@ -9,6 +9,7 @@
 #include <linux/uaccess.h>
 
 #include <asm/inst.h>
+#include <asm/module.h>
 
 static int ftrace_modify_code(unsigned long pc, u32 old, u32 new, bool validate)
 {
@@ -29,18 +30,78 @@ static int ftrace_modify_code(unsigned long pc, u32 old, u32 new, bool validate)
 }
 
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
+
+#ifdef CONFIG_MODULES
+static inline int __get_mod(struct module **mod, unsigned long addr)
+{
+       preempt_disable();
+       *mod = __module_text_address(addr);
+       preempt_enable();
+
+       if (WARN_ON(!(*mod)))
+               return -EINVAL;
+
+       return 0;
+}
+
+static struct plt_entry *get_ftrace_plt(struct module *mod, unsigned long addr)
+{
+       struct plt_entry *plt = mod->arch.ftrace_trampolines;
+
+       if (addr == FTRACE_ADDR)
+               return &plt[FTRACE_PLT_IDX];
+       if (addr == FTRACE_REGS_ADDR &&
+                       IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
+               return &plt[FTRACE_REGS_PLT_IDX];
+
+       return NULL;
+}
+
+static unsigned long get_plt_addr(struct module *mod, unsigned long addr)
+{
+       struct plt_entry *plt;
+
+       plt = get_ftrace_plt(mod, addr);
+       if (!plt) {
+               pr_err("ftrace: no module PLT for %ps\n", (void *)addr);
+               return -EINVAL;
+       }
+
+       return (unsigned long)plt;
+}
+#endif
+
 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr)
 {
        u32 old, new;
        unsigned long pc;
+       long offset __maybe_unused;
 
        pc = rec->ip + LOONGARCH_INSN_SIZE;
 
+#ifdef CONFIG_MODULES
+       offset = (long)pc - (long)addr;
+
+       if (offset < -SZ_128M || offset >= SZ_128M) {
+               int ret;
+               struct module *mod;
+
+               ret = __get_mod(&mod, pc);
+               if (ret)
+                       return ret;
+
+               addr = get_plt_addr(mod, addr);
+
+               old_addr = get_plt_addr(mod, old_addr);
+       }
+#endif
+
        new = larch_insn_gen_bl(pc, addr);
        old = larch_insn_gen_bl(pc, old_addr);
 
        return ftrace_modify_code(pc, old, new, true);
 }
+
 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
 
 int ftrace_update_ftrace_func(ftrace_func_t func)
@@ -91,9 +152,25 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 {
        u32 old, new;
        unsigned long pc;
+       long offset __maybe_unused;
 
        pc = rec->ip + LOONGARCH_INSN_SIZE;
 
+#ifdef CONFIG_MODULES
+       offset = (long)pc - (long)addr;
+
+       if (offset < -SZ_128M || offset >= SZ_128M) {
+               int ret;
+               struct module *mod;
+
+               ret = __get_mod(&mod, pc);
+               if (ret)
+                       return ret;
+
+               addr = get_plt_addr(mod, addr);
+       }
+#endif
+
        old = larch_insn_gen_nop();
        new = larch_insn_gen_bl(pc, addr);
 
@@ -104,9 +181,25 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long ad
 {
        u32 old, new;
        unsigned long pc;
+       long offset __maybe_unused;
 
        pc = rec->ip + LOONGARCH_INSN_SIZE;
 
+#ifdef CONFIG_MODULES
+       offset = (long)pc - (long)addr;
+
+       if (offset < -SZ_128M || offset >= SZ_128M) {
+               int ret;
+               struct module *mod;
+
+               ret = __get_mod(&mod, pc);
+               if (ret)
+                       return ret;
+
+               addr = get_plt_addr(mod, addr);
+       }
+#endif
+
        new = larch_insn_gen_nop();
        old = larch_insn_gen_bl(pc, addr);
 

diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c
index 39671e8..512579d 100644 (file)
--- a/arch/loongarch/kernel/inst.c
+++ b/arch/loongarch/kernel/inst.c
@@ -120,6 +120,17 @@ u32 larch_insn_gen_move(enum loongarch_gpr rd, enum loongarch_gpr rj)
        return larch_insn_gen_or(rd, rj, 0);
 }
 
+u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm)
+{
+       union loongarch_instruction insn;
+
+       insn.reg1i20_format.opcode = lu12iw_op;
+       insn.reg1i20_format.rd = rd;
+       insn.reg1i20_format.immediate = imm;
+
+       return insn.word;
+}
+
 u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm)
 {
        union loongarch_instruction insn;

diff --git a/arch/loongarch/kernel/module-sections.c b/arch/loongarch/kernel/module-sections.c
index 13d9a42..d4dbcda 100644 (file)
--- a/arch/loongarch/kernel/module-sections.c
+++ b/arch/loongarch/kernel/module-sections.c
@@ -6,6 +6,7 @@
 #include <linux/elf.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/ftrace.h>
 
 Elf_Addr module_emit_got_entry(struct module *mod, Elf_Shdr *sechdrs, Elf_Addr val)
 {
@@ -103,7 +104,7 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
                              char *secstrings, struct module *mod)
 {
        unsigned int i, num_plts = 0, num_gots = 0;
-       Elf_Shdr *got_sec, *plt_sec, *plt_idx_sec;
+       Elf_Shdr *got_sec, *plt_sec, *plt_idx_sec, *tramp = NULL;
 
        /*
         * Find the empty .plt sections.
@@ -115,6 +116,8 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
                        mod->arch.plt.shndx = i;
                else if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt.idx"))
                        mod->arch.plt_idx.shndx = i;
+               else if (!strcmp(secstrings + sechdrs[i].sh_name, ".ftrace_trampoline"))
+                       tramp = sechdrs + i;
        }
 
        if (!mod->arch.got.shndx) {
@@ -170,5 +173,12 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
        mod->arch.plt_idx.num_entries = 0;
        mod->arch.plt_idx.max_entries = num_plts;
 
+       if (tramp) {
+               tramp->sh_type = SHT_NOBITS;
+               tramp->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
+               tramp->sh_addralign = __alignof__(struct plt_entry);
+               tramp->sh_size = NR_FTRACE_PLTS * sizeof(struct plt_entry);
+       }
+
        return 0;
 }

diff --git a/arch/loongarch/kernel/module.c b/arch/loongarch/kernel/module.c
index 899dc67..b8b8608 100644 (file)
--- a/arch/loongarch/kernel/module.c
+++ b/arch/loongarch/kernel/module.c
@@ -15,9 +15,11 @@
 #include <linux/vmalloc.h>
 #include <linux/slab.h>
 #include <linux/fs.h>
+#include <linux/ftrace.h>
 #include <linux/string.h>
 #include <linux/kernel.h>
 #include <asm/alternative.h>
+#include <asm/inst.h>
 
 static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top)
 {
@@ -473,6 +475,23 @@ void *module_alloc(unsigned long size)
                        GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0));
 }
 
+static void module_init_ftrace_plt(const Elf_Ehdr *hdr,
+                                  const Elf_Shdr *sechdrs, struct module *mod)
+{
+#ifdef CONFIG_DYNAMIC_FTRACE
+       struct plt_entry *ftrace_plts;
+
+       ftrace_plts = (void *)sechdrs->sh_addr;
+
+       ftrace_plts[FTRACE_PLT_IDX] = emit_plt_entry(FTRACE_ADDR);
+
+       if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
+               ftrace_plts[FTRACE_REGS_PLT_IDX] = emit_plt_entry(FTRACE_REGS_ADDR);
+
+       mod->arch.ftrace_trampolines = ftrace_plts;
+#endif
+}
+
 int module_finalize(const Elf_Ehdr *hdr,
                    const Elf_Shdr *sechdrs, struct module *mod)
 {
@@ -482,6 +501,8 @@ int module_finalize(const Elf_Ehdr *hdr,
        for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) {
                if (!strcmp(".altinstructions", secstrs + s->sh_name))
                        apply_alternatives((void *)s->sh_addr, (void *)s->sh_addr + s->sh_size);
+               if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name))
+                       module_init_ftrace_plt(hdr, s, mod);
        }
 
        return 0;