2 * Kprobes-based tracing events
4 * Created by Masami Hiramatsu <mhiramat@redhat.com>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #include <linux/module.h>
21 #include <linux/uaccess.h>
23 #include "trace_probe.h"
25 #define KPROBE_EVENT_SYSTEM "kprobes"
28 * Kprobe event core functions
31 struct list_head list;
32 struct kretprobe rp; /* Use rp.kp for kprobe use */
34 const char *symbol; /* symbol name */
35 struct trace_probe tp;
38 struct event_file_link {
39 struct ftrace_event_file *file;
40 struct list_head list;
43 #define SIZEOF_TRACE_KPROBE(n) \
44 (offsetof(struct trace_kprobe, tp.args) + \
45 (sizeof(struct probe_arg) * (n)))
48 static __kprobes bool trace_kprobe_is_return(struct trace_kprobe *tk)
50 return tk->rp.handler != NULL;
53 static __kprobes const char *trace_kprobe_symbol(struct trace_kprobe *tk)
55 return tk->symbol ? tk->symbol : "unknown";
58 static __kprobes unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
60 return tk->rp.kp.offset;
63 static __kprobes bool trace_kprobe_has_gone(struct trace_kprobe *tk)
65 return !!(kprobe_gone(&tk->rp.kp));
68 static __kprobes bool trace_kprobe_within_module(struct trace_kprobe *tk,
71 int len = strlen(mod->name);
72 const char *name = trace_kprobe_symbol(tk);
73 return strncmp(mod->name, name, len) == 0 && name[len] == ':';
76 static __kprobes bool trace_kprobe_is_on_module(struct trace_kprobe *tk)
78 return !!strchr(trace_kprobe_symbol(tk), ':');
81 static int register_kprobe_event(struct trace_kprobe *tk);
82 static int unregister_kprobe_event(struct trace_kprobe *tk);
84 static DEFINE_MUTEX(probe_lock);
85 static LIST_HEAD(probe_list);
87 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
88 static int kretprobe_dispatcher(struct kretprobe_instance *ri,
89 struct pt_regs *regs);
92 * Allocate new trace_probe and initialize it (including kprobes).
94 static struct trace_kprobe *alloc_trace_kprobe(const char *group,
99 int nargs, bool is_return)
101 struct trace_kprobe *tk;
104 tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL);
109 tk->symbol = kstrdup(symbol, GFP_KERNEL);
112 tk->rp.kp.symbol_name = tk->symbol;
113 tk->rp.kp.offset = offs;
115 tk->rp.kp.addr = addr;
118 tk->rp.handler = kretprobe_dispatcher;
120 tk->rp.kp.pre_handler = kprobe_dispatcher;
122 if (!event || !is_good_name(event)) {
127 tk->tp.call.class = &tk->tp.class;
128 tk->tp.call.name = kstrdup(event, GFP_KERNEL);
129 if (!tk->tp.call.name)
132 if (!group || !is_good_name(group)) {
137 tk->tp.class.system = kstrdup(group, GFP_KERNEL);
138 if (!tk->tp.class.system)
141 INIT_LIST_HEAD(&tk->list);
142 INIT_LIST_HEAD(&tk->tp.files);
145 kfree(tk->tp.call.name);
151 static void free_trace_kprobe(struct trace_kprobe *tk)
155 for (i = 0; i < tk->tp.nr_args; i++)
156 traceprobe_free_probe_arg(&tk->tp.args[i]);
158 kfree(tk->tp.call.class->system);
159 kfree(tk->tp.call.name);
164 static struct trace_kprobe *find_trace_kprobe(const char *event,
167 struct trace_kprobe *tk;
169 list_for_each_entry(tk, &probe_list, list)
170 if (strcmp(tk->tp.call.name, event) == 0 &&
171 strcmp(tk->tp.call.class->system, group) == 0)
178 * if the file is NULL, enable "perf" handler, or enable "trace" handler.
181 enable_trace_kprobe(struct trace_kprobe *tk, struct ftrace_event_file *file)
186 struct event_file_link *link;
188 link = kmalloc(sizeof(*link), GFP_KERNEL);
195 list_add_tail_rcu(&link->list, &tk->tp.files);
197 tk->tp.flags |= TP_FLAG_TRACE;
199 tk->tp.flags |= TP_FLAG_PROFILE;
201 if (trace_probe_is_registered(&tk->tp) && !trace_kprobe_has_gone(tk)) {
202 if (trace_kprobe_is_return(tk))
203 ret = enable_kretprobe(&tk->rp);
205 ret = enable_kprobe(&tk->rp.kp);
211 static struct event_file_link *
212 find_event_file_link(struct trace_probe *tp, struct ftrace_event_file *file)
214 struct event_file_link *link;
216 list_for_each_entry(link, &tp->files, list)
217 if (link->file == file)
224 * Disable trace_probe
225 * if the file is NULL, disable "perf" handler, or disable "trace" handler.
228 disable_trace_kprobe(struct trace_kprobe *tk, struct ftrace_event_file *file)
230 struct event_file_link *link = NULL;
235 link = find_event_file_link(&tk->tp, file);
241 list_del_rcu(&link->list);
243 if (!list_empty(&tk->tp.files))
246 tk->tp.flags &= ~TP_FLAG_TRACE;
248 tk->tp.flags &= ~TP_FLAG_PROFILE;
250 if (!trace_probe_is_enabled(&tk->tp) && trace_probe_is_registered(&tk->tp)) {
251 if (trace_kprobe_is_return(tk))
252 disable_kretprobe(&tk->rp);
254 disable_kprobe(&tk->rp.kp);
260 * Synchronize with kprobe_trace_func/kretprobe_trace_func
261 * to ensure disabled (all running handlers are finished).
262 * This is not only for kfree(), but also the caller,
263 * trace_remove_event_call() supposes it for releasing
264 * event_call related objects, which will be accessed in
265 * the kprobe_trace_func/kretprobe_trace_func.
268 kfree(link); /* Ignored if link == NULL */
274 /* Internal register function - just handle k*probes and flags */
275 static int __register_trace_kprobe(struct trace_kprobe *tk)
279 if (trace_probe_is_registered(&tk->tp))
282 for (i = 0; i < tk->tp.nr_args; i++)
283 traceprobe_update_arg(&tk->tp.args[i]);
285 /* Set/clear disabled flag according to tp->flag */
286 if (trace_probe_is_enabled(&tk->tp))
287 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
289 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;
291 if (trace_kprobe_is_return(tk))
292 ret = register_kretprobe(&tk->rp);
294 ret = register_kprobe(&tk->rp.kp);
297 tk->tp.flags |= TP_FLAG_REGISTERED;
299 pr_warning("Could not insert probe at %s+%lu: %d\n",
300 trace_kprobe_symbol(tk), trace_kprobe_offset(tk), ret);
301 if (ret == -ENOENT && trace_kprobe_is_on_module(tk)) {
302 pr_warning("This probe might be able to register after"
303 "target module is loaded. Continue.\n");
305 } else if (ret == -EILSEQ) {
306 pr_warning("Probing address(0x%p) is not an "
307 "instruction boundary.\n",
316 /* Internal unregister function - just handle k*probes and flags */
317 static void __unregister_trace_kprobe(struct trace_kprobe *tk)
319 if (trace_probe_is_registered(&tk->tp)) {
320 if (trace_kprobe_is_return(tk))
321 unregister_kretprobe(&tk->rp);
323 unregister_kprobe(&tk->rp.kp);
324 tk->tp.flags &= ~TP_FLAG_REGISTERED;
325 /* Cleanup kprobe for reuse */
326 if (tk->rp.kp.symbol_name)
327 tk->rp.kp.addr = NULL;
331 /* Unregister a trace_probe and probe_event: call with locking probe_lock */
332 static int unregister_trace_kprobe(struct trace_kprobe *tk)
334 /* Enabled event can not be unregistered */
335 if (trace_probe_is_enabled(&tk->tp))
338 /* Will fail if probe is being used by ftrace or perf */
339 if (unregister_kprobe_event(tk))
342 __unregister_trace_kprobe(tk);
348 /* Register a trace_probe and probe_event */
349 static int register_trace_kprobe(struct trace_kprobe *tk)
351 struct trace_kprobe *old_tk;
354 mutex_lock(&probe_lock);
356 /* Delete old (same name) event if exist */
357 old_tk = find_trace_kprobe(tk->tp.call.name, tk->tp.call.class->system);
359 ret = unregister_trace_kprobe(old_tk);
362 free_trace_kprobe(old_tk);
365 /* Register new event */
366 ret = register_kprobe_event(tk);
368 pr_warning("Failed to register probe event(%d)\n", ret);
372 /* Register k*probe */
373 ret = __register_trace_kprobe(tk);
375 unregister_kprobe_event(tk);
377 list_add_tail(&tk->list, &probe_list);
380 mutex_unlock(&probe_lock);
384 /* Module notifier call back, checking event on the module */
385 static int trace_kprobe_module_callback(struct notifier_block *nb,
386 unsigned long val, void *data)
388 struct module *mod = data;
389 struct trace_kprobe *tk;
392 if (val != MODULE_STATE_COMING)
395 /* Update probes on coming module */
396 mutex_lock(&probe_lock);
397 list_for_each_entry(tk, &probe_list, list) {
398 if (trace_kprobe_within_module(tk, mod)) {
399 /* Don't need to check busy - this should have gone. */
400 __unregister_trace_kprobe(tk);
401 ret = __register_trace_kprobe(tk);
403 pr_warning("Failed to re-register probe %s on"
405 tk->tp.call.name, mod->name, ret);
408 mutex_unlock(&probe_lock);
413 static struct notifier_block trace_kprobe_module_nb = {
414 .notifier_call = trace_kprobe_module_callback,
415 .priority = 1 /* Invoked after kprobe module callback */
418 static int create_trace_kprobe(int argc, char **argv)
422 * - Add kprobe: p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
423 * - Add kretprobe: r[:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS]
425 * $retval : fetch return value
426 * $stack : fetch stack address
427 * $stackN : fetch Nth of stack (N:0-)
428 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
429 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
430 * %REG : fetch register REG
431 * Dereferencing memory fetch:
432 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
433 * Alias name of args:
434 * NAME=FETCHARG : set NAME as alias of FETCHARG.
436 * FETCHARG:TYPE : use TYPE instead of unsigned long.
438 struct trace_kprobe *tk;
440 bool is_return = false, is_delete = false;
441 char *symbol = NULL, *event = NULL, *group = NULL;
443 unsigned long offset = 0;
445 char buf[MAX_EVENT_NAME_LEN];
447 /* argc must be >= 1 */
448 if (argv[0][0] == 'p')
450 else if (argv[0][0] == 'r')
452 else if (argv[0][0] == '-')
455 pr_info("Probe definition must be started with 'p', 'r' or"
460 if (argv[0][1] == ':') {
462 if (strchr(event, '/')) {
464 event = strchr(group, '/') + 1;
466 if (strlen(group) == 0) {
467 pr_info("Group name is not specified\n");
471 if (strlen(event) == 0) {
472 pr_info("Event name is not specified\n");
477 group = KPROBE_EVENT_SYSTEM;
481 pr_info("Delete command needs an event name.\n");
484 mutex_lock(&probe_lock);
485 tk = find_trace_kprobe(event, group);
487 mutex_unlock(&probe_lock);
488 pr_info("Event %s/%s doesn't exist.\n", group, event);
491 /* delete an event */
492 ret = unregister_trace_kprobe(tk);
494 free_trace_kprobe(tk);
495 mutex_unlock(&probe_lock);
500 pr_info("Probe point is not specified.\n");
503 if (isdigit(argv[1][0])) {
505 pr_info("Return probe point must be a symbol.\n");
508 /* an address specified */
509 ret = kstrtoul(&argv[1][0], 0, (unsigned long *)&addr);
511 pr_info("Failed to parse address.\n");
515 /* a symbol specified */
517 /* TODO: support .init module functions */
518 ret = traceprobe_split_symbol_offset(symbol, &offset);
520 pr_info("Failed to parse symbol.\n");
523 if (offset && is_return) {
524 pr_info("Return probe must be used without offset.\n");
528 argc -= 2; argv += 2;
532 /* Make a new event name */
534 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
535 is_return ? 'r' : 'p', symbol, offset);
537 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p",
538 is_return ? 'r' : 'p', addr);
541 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, argc,
544 pr_info("Failed to allocate trace_probe.(%d)\n",
549 /* parse arguments */
551 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
552 struct probe_arg *parg = &tk->tp.args[i];
554 /* Increment count for freeing args in error case */
557 /* Parse argument name */
558 arg = strchr(argv[i], '=');
561 parg->name = kstrdup(argv[i], GFP_KERNEL);
564 /* If argument name is omitted, set "argN" */
565 snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
566 parg->name = kstrdup(buf, GFP_KERNEL);
570 pr_info("Failed to allocate argument[%d] name.\n", i);
575 if (!is_good_name(parg->name)) {
576 pr_info("Invalid argument[%d] name: %s\n",
582 if (traceprobe_conflict_field_name(parg->name,
584 pr_info("Argument[%d] name '%s' conflicts with "
585 "another field.\n", i, argv[i]);
590 /* Parse fetch argument */
591 ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,
594 pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
599 ret = register_trace_kprobe(tk);
605 free_trace_kprobe(tk);
609 static int release_all_trace_kprobes(void)
611 struct trace_kprobe *tk;
614 mutex_lock(&probe_lock);
615 /* Ensure no probe is in use. */
616 list_for_each_entry(tk, &probe_list, list)
617 if (trace_probe_is_enabled(&tk->tp)) {
621 /* TODO: Use batch unregistration */
622 while (!list_empty(&probe_list)) {
623 tk = list_entry(probe_list.next, struct trace_kprobe, list);
624 ret = unregister_trace_kprobe(tk);
627 free_trace_kprobe(tk);
631 mutex_unlock(&probe_lock);
636 /* Probes listing interfaces */
637 static void *probes_seq_start(struct seq_file *m, loff_t *pos)
639 mutex_lock(&probe_lock);
640 return seq_list_start(&probe_list, *pos);
643 static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos)
645 return seq_list_next(v, &probe_list, pos);
648 static void probes_seq_stop(struct seq_file *m, void *v)
650 mutex_unlock(&probe_lock);
653 static int probes_seq_show(struct seq_file *m, void *v)
655 struct trace_kprobe *tk = v;
658 seq_printf(m, "%c", trace_kprobe_is_return(tk) ? 'r' : 'p');
659 seq_printf(m, ":%s/%s", tk->tp.call.class->system, tk->tp.call.name);
662 seq_printf(m, " 0x%p", tk->rp.kp.addr);
663 else if (tk->rp.kp.offset)
664 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk),
667 seq_printf(m, " %s", trace_kprobe_symbol(tk));
669 for (i = 0; i < tk->tp.nr_args; i++)
670 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
676 static const struct seq_operations probes_seq_op = {
677 .start = probes_seq_start,
678 .next = probes_seq_next,
679 .stop = probes_seq_stop,
680 .show = probes_seq_show
683 static int probes_open(struct inode *inode, struct file *file)
687 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
688 ret = release_all_trace_kprobes();
693 return seq_open(file, &probes_seq_op);
696 static ssize_t probes_write(struct file *file, const char __user *buffer,
697 size_t count, loff_t *ppos)
699 return traceprobe_probes_write(file, buffer, count, ppos,
700 create_trace_kprobe);
703 static const struct file_operations kprobe_events_ops = {
704 .owner = THIS_MODULE,
708 .release = seq_release,
709 .write = probes_write,
712 /* Probes profiling interfaces */
713 static int probes_profile_seq_show(struct seq_file *m, void *v)
715 struct trace_kprobe *tk = v;
717 seq_printf(m, " %-44s %15lu %15lu\n", tk->tp.call.name, tk->nhit,
723 static const struct seq_operations profile_seq_op = {
724 .start = probes_seq_start,
725 .next = probes_seq_next,
726 .stop = probes_seq_stop,
727 .show = probes_profile_seq_show
730 static int profile_open(struct inode *inode, struct file *file)
732 return seq_open(file, &profile_seq_op);
735 static const struct file_operations kprobe_profile_ops = {
736 .owner = THIS_MODULE,
737 .open = profile_open,
740 .release = seq_release,
744 static __kprobes void
745 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
746 struct ftrace_event_file *ftrace_file)
748 struct kprobe_trace_entry_head *entry;
749 struct ring_buffer_event *event;
750 struct ring_buffer *buffer;
752 unsigned long irq_flags;
753 struct ftrace_event_call *call = &tk->tp.call;
755 WARN_ON(call != ftrace_file->event_call);
757 if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags))
760 local_save_flags(irq_flags);
761 pc = preempt_count();
763 dsize = __get_data_size(&tk->tp, regs);
764 size = sizeof(*entry) + tk->tp.size + dsize;
766 event = trace_event_buffer_lock_reserve(&buffer, ftrace_file,
768 size, irq_flags, pc);
772 entry = ring_buffer_event_data(event);
773 entry->ip = (unsigned long)tk->rp.kp.addr;
774 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
776 if (!filter_check_discard(ftrace_file, entry, buffer, event))
777 trace_buffer_unlock_commit_regs(buffer, event,
778 irq_flags, pc, regs);
781 static __kprobes void
782 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
784 struct event_file_link *link;
786 list_for_each_entry_rcu(link, &tk->tp.files, list)
787 __kprobe_trace_func(tk, regs, link->file);
790 /* Kretprobe handler */
791 static __kprobes void
792 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
793 struct pt_regs *regs,
794 struct ftrace_event_file *ftrace_file)
796 struct kretprobe_trace_entry_head *entry;
797 struct ring_buffer_event *event;
798 struct ring_buffer *buffer;
800 unsigned long irq_flags;
801 struct ftrace_event_call *call = &tk->tp.call;
803 WARN_ON(call != ftrace_file->event_call);
805 if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags))
808 local_save_flags(irq_flags);
809 pc = preempt_count();
811 dsize = __get_data_size(&tk->tp, regs);
812 size = sizeof(*entry) + tk->tp.size + dsize;
814 event = trace_event_buffer_lock_reserve(&buffer, ftrace_file,
816 size, irq_flags, pc);
820 entry = ring_buffer_event_data(event);
821 entry->func = (unsigned long)tk->rp.kp.addr;
822 entry->ret_ip = (unsigned long)ri->ret_addr;
823 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
825 if (!filter_check_discard(ftrace_file, entry, buffer, event))
826 trace_buffer_unlock_commit_regs(buffer, event,
827 irq_flags, pc, regs);
830 static __kprobes void
831 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
832 struct pt_regs *regs)
834 struct event_file_link *link;
836 list_for_each_entry_rcu(link, &tk->tp.files, list)
837 __kretprobe_trace_func(tk, ri, regs, link->file);
840 /* Event entry printers */
841 static enum print_line_t
842 print_kprobe_event(struct trace_iterator *iter, int flags,
843 struct trace_event *event)
845 struct kprobe_trace_entry_head *field;
846 struct trace_seq *s = &iter->seq;
847 struct trace_probe *tp;
851 field = (struct kprobe_trace_entry_head *)iter->ent;
852 tp = container_of(event, struct trace_probe, call.event);
854 if (!trace_seq_printf(s, "%s: (", tp->call.name))
857 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
860 if (!trace_seq_puts(s, ")"))
863 data = (u8 *)&field[1];
864 for (i = 0; i < tp->nr_args; i++)
865 if (!tp->args[i].type->print(s, tp->args[i].name,
866 data + tp->args[i].offset, field))
869 if (!trace_seq_puts(s, "\n"))
872 return TRACE_TYPE_HANDLED;
874 return TRACE_TYPE_PARTIAL_LINE;
877 static enum print_line_t
878 print_kretprobe_event(struct trace_iterator *iter, int flags,
879 struct trace_event *event)
881 struct kretprobe_trace_entry_head *field;
882 struct trace_seq *s = &iter->seq;
883 struct trace_probe *tp;
887 field = (struct kretprobe_trace_entry_head *)iter->ent;
888 tp = container_of(event, struct trace_probe, call.event);
890 if (!trace_seq_printf(s, "%s: (", tp->call.name))
893 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
896 if (!trace_seq_puts(s, " <- "))
899 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
902 if (!trace_seq_puts(s, ")"))
905 data = (u8 *)&field[1];
906 for (i = 0; i < tp->nr_args; i++)
907 if (!tp->args[i].type->print(s, tp->args[i].name,
908 data + tp->args[i].offset, field))
911 if (!trace_seq_puts(s, "\n"))
914 return TRACE_TYPE_HANDLED;
916 return TRACE_TYPE_PARTIAL_LINE;
920 static int kprobe_event_define_fields(struct ftrace_event_call *event_call)
923 struct kprobe_trace_entry_head field;
924 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
926 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
927 /* Set argument names as fields */
928 for (i = 0; i < tk->tp.nr_args; i++) {
929 struct probe_arg *parg = &tk->tp.args[i];
931 ret = trace_define_field(event_call, parg->type->fmttype,
933 sizeof(field) + parg->offset,
935 parg->type->is_signed,
943 static int kretprobe_event_define_fields(struct ftrace_event_call *event_call)
946 struct kretprobe_trace_entry_head field;
947 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
949 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
950 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
951 /* Set argument names as fields */
952 for (i = 0; i < tk->tp.nr_args; i++) {
953 struct probe_arg *parg = &tk->tp.args[i];
955 ret = trace_define_field(event_call, parg->type->fmttype,
957 sizeof(field) + parg->offset,
959 parg->type->is_signed,
967 #ifdef CONFIG_PERF_EVENTS
969 /* Kprobe profile handler */
970 static __kprobes void
971 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
973 struct ftrace_event_call *call = &tk->tp.call;
974 struct kprobe_trace_entry_head *entry;
975 struct hlist_head *head;
976 int size, __size, dsize;
979 head = this_cpu_ptr(call->perf_events);
980 if (hlist_empty(head))
983 dsize = __get_data_size(&tk->tp, regs);
984 __size = sizeof(*entry) + tk->tp.size + dsize;
985 size = ALIGN(__size + sizeof(u32), sizeof(u64));
988 entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx);
992 entry->ip = (unsigned long)tk->rp.kp.addr;
993 memset(&entry[1], 0, dsize);
994 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
995 perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
998 /* Kretprobe profile handler */
999 static __kprobes void
1000 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1001 struct pt_regs *regs)
1003 struct ftrace_event_call *call = &tk->tp.call;
1004 struct kretprobe_trace_entry_head *entry;
1005 struct hlist_head *head;
1006 int size, __size, dsize;
1009 head = this_cpu_ptr(call->perf_events);
1010 if (hlist_empty(head))
1013 dsize = __get_data_size(&tk->tp, regs);
1014 __size = sizeof(*entry) + tk->tp.size + dsize;
1015 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1016 size -= sizeof(u32);
1018 entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx);
1022 entry->func = (unsigned long)tk->rp.kp.addr;
1023 entry->ret_ip = (unsigned long)ri->ret_addr;
1024 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1025 perf_trace_buf_submit(entry, size, rctx, 0, 1, regs, head, NULL);
1027 #endif /* CONFIG_PERF_EVENTS */
1030 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1032 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
1033 * lockless, but we can't race with this __init function.
1036 int kprobe_register(struct ftrace_event_call *event,
1037 enum trace_reg type, void *data)
1039 struct trace_kprobe *tk = (struct trace_kprobe *)event->data;
1040 struct ftrace_event_file *file = data;
1043 case TRACE_REG_REGISTER:
1044 return enable_trace_kprobe(tk, file);
1045 case TRACE_REG_UNREGISTER:
1046 return disable_trace_kprobe(tk, file);
1048 #ifdef CONFIG_PERF_EVENTS
1049 case TRACE_REG_PERF_REGISTER:
1050 return enable_trace_kprobe(tk, NULL);
1051 case TRACE_REG_PERF_UNREGISTER:
1052 return disable_trace_kprobe(tk, NULL);
1053 case TRACE_REG_PERF_OPEN:
1054 case TRACE_REG_PERF_CLOSE:
1055 case TRACE_REG_PERF_ADD:
1056 case TRACE_REG_PERF_DEL:
1064 int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
1066 struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
1070 if (tk->tp.flags & TP_FLAG_TRACE)
1071 kprobe_trace_func(tk, regs);
1072 #ifdef CONFIG_PERF_EVENTS
1073 if (tk->tp.flags & TP_FLAG_PROFILE)
1074 kprobe_perf_func(tk, regs);
1076 return 0; /* We don't tweek kernel, so just return 0 */
1080 int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
1082 struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp);
1086 if (tk->tp.flags & TP_FLAG_TRACE)
1087 kretprobe_trace_func(tk, ri, regs);
1088 #ifdef CONFIG_PERF_EVENTS
1089 if (tk->tp.flags & TP_FLAG_PROFILE)
1090 kretprobe_perf_func(tk, ri, regs);
1092 return 0; /* We don't tweek kernel, so just return 0 */
1095 static struct trace_event_functions kretprobe_funcs = {
1096 .trace = print_kretprobe_event
1099 static struct trace_event_functions kprobe_funcs = {
1100 .trace = print_kprobe_event
1103 static int register_kprobe_event(struct trace_kprobe *tk)
1105 struct ftrace_event_call *call = &tk->tp.call;
1108 /* Initialize ftrace_event_call */
1109 INIT_LIST_HEAD(&call->class->fields);
1110 if (trace_kprobe_is_return(tk)) {
1111 call->event.funcs = &kretprobe_funcs;
1112 call->class->define_fields = kretprobe_event_define_fields;
1114 call->event.funcs = &kprobe_funcs;
1115 call->class->define_fields = kprobe_event_define_fields;
1117 if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0)
1119 ret = register_ftrace_event(&call->event);
1121 kfree(call->print_fmt);
1125 call->class->reg = kprobe_register;
1127 ret = trace_add_event_call(call);
1129 pr_info("Failed to register kprobe event: %s\n", call->name);
1130 kfree(call->print_fmt);
1131 unregister_ftrace_event(&call->event);
1136 static int unregister_kprobe_event(struct trace_kprobe *tk)
1140 /* tp->event is unregistered in trace_remove_event_call() */
1141 ret = trace_remove_event_call(&tk->tp.call);
1143 kfree(tk->tp.call.print_fmt);
1147 /* Make a debugfs interface for controlling probe points */
1148 static __init int init_kprobe_trace(void)
1150 struct dentry *d_tracer;
1151 struct dentry *entry;
1153 if (register_module_notifier(&trace_kprobe_module_nb))
1156 d_tracer = tracing_init_dentry();
1160 entry = debugfs_create_file("kprobe_events", 0644, d_tracer,
1161 NULL, &kprobe_events_ops);
1163 /* Event list interface */
1165 pr_warning("Could not create debugfs "
1166 "'kprobe_events' entry\n");
1168 /* Profile interface */
1169 entry = debugfs_create_file("kprobe_profile", 0444, d_tracer,
1170 NULL, &kprobe_profile_ops);
1173 pr_warning("Could not create debugfs "
1174 "'kprobe_profile' entry\n");
1177 fs_initcall(init_kprobe_trace);
1180 #ifdef CONFIG_FTRACE_STARTUP_TEST
1183 * The "__used" keeps gcc from removing the function symbol
1184 * from the kallsyms table.
1186 static __used int kprobe_trace_selftest_target(int a1, int a2, int a3,
1187 int a4, int a5, int a6)
1189 return a1 + a2 + a3 + a4 + a5 + a6;
1192 static struct ftrace_event_file *
1193 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)
1195 struct ftrace_event_file *file;
1197 list_for_each_entry(file, &tr->events, list)
1198 if (file->event_call == &tk->tp.call)
1205 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this
1206 * stage, we can do this lockless.
1208 static __init int kprobe_trace_self_tests_init(void)
1211 int (*target)(int, int, int, int, int, int);
1212 struct trace_kprobe *tk;
1213 struct ftrace_event_file *file;
1215 target = kprobe_trace_selftest_target;
1217 pr_info("Testing kprobe tracing: ");
1219 ret = traceprobe_command("p:testprobe kprobe_trace_selftest_target "
1220 "$stack $stack0 +0($stack)",
1221 create_trace_kprobe);
1222 if (WARN_ON_ONCE(ret)) {
1223 pr_warn("error on probing function entry.\n");
1226 /* Enable trace point */
1227 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1228 if (WARN_ON_ONCE(tk == NULL)) {
1229 pr_warn("error on getting new probe.\n");
1232 file = find_trace_probe_file(tk, top_trace_array());
1233 if (WARN_ON_ONCE(file == NULL)) {
1234 pr_warn("error on getting probe file.\n");
1237 enable_trace_kprobe(tk, file);
1241 ret = traceprobe_command("r:testprobe2 kprobe_trace_selftest_target "
1242 "$retval", create_trace_kprobe);
1243 if (WARN_ON_ONCE(ret)) {
1244 pr_warn("error on probing function return.\n");
1247 /* Enable trace point */
1248 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1249 if (WARN_ON_ONCE(tk == NULL)) {
1250 pr_warn("error on getting 2nd new probe.\n");
1253 file = find_trace_probe_file(tk, top_trace_array());
1254 if (WARN_ON_ONCE(file == NULL)) {
1255 pr_warn("error on getting probe file.\n");
1258 enable_trace_kprobe(tk, file);
1265 ret = target(1, 2, 3, 4, 5, 6);
1267 /* Disable trace points before removing it */
1268 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1269 if (WARN_ON_ONCE(tk == NULL)) {
1270 pr_warn("error on getting test probe.\n");
1273 file = find_trace_probe_file(tk, top_trace_array());
1274 if (WARN_ON_ONCE(file == NULL)) {
1275 pr_warn("error on getting probe file.\n");
1278 disable_trace_kprobe(tk, file);
1281 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1282 if (WARN_ON_ONCE(tk == NULL)) {
1283 pr_warn("error on getting 2nd test probe.\n");
1286 file = find_trace_probe_file(tk, top_trace_array());
1287 if (WARN_ON_ONCE(file == NULL)) {
1288 pr_warn("error on getting probe file.\n");
1291 disable_trace_kprobe(tk, file);
1294 ret = traceprobe_command("-:testprobe", create_trace_kprobe);
1295 if (WARN_ON_ONCE(ret)) {
1296 pr_warn("error on deleting a probe.\n");
1300 ret = traceprobe_command("-:testprobe2", create_trace_kprobe);
1301 if (WARN_ON_ONCE(ret)) {
1302 pr_warn("error on deleting a probe.\n");
1307 release_all_trace_kprobes();
1309 pr_cont("NG: Some tests are failed. Please check them.\n");
1315 late_initcall(kprobe_trace_self_tests_init);