1 #include <trace/syscall.h>
2 #include <trace/events/syscalls.h>
3 #include <linux/syscalls.h>
4 #include <linux/slab.h>
5 #include <linux/kernel.h>
6 #include <linux/module.h> /* for MODULE_NAME_LEN via KSYM_SYMBOL_LEN */
7 #include <linux/ftrace.h>
8 #include <linux/perf_event.h>
9 #include <asm/syscall.h>
11 #include "trace_output.h"
14 static DEFINE_MUTEX(syscall_trace_lock);
16 static int syscall_enter_register(struct ftrace_event_call *event,
17 enum trace_reg type, void *data);
18 static int syscall_exit_register(struct ftrace_event_call *event,
19 enum trace_reg type, void *data);
21 static struct list_head *
22 syscall_get_enter_fields(struct ftrace_event_call *call)
24 struct syscall_metadata *entry = call->data;
26 return &entry->enter_fields;
29 extern struct syscall_metadata *__start_syscalls_metadata[];
30 extern struct syscall_metadata *__stop_syscalls_metadata[];
32 static struct syscall_metadata **syscalls_metadata;
34 #ifndef ARCH_HAS_SYSCALL_MATCH_SYM_NAME
35 static inline bool arch_syscall_match_sym_name(const char *sym, const char *name)
38 * Only compare after the "sys" prefix. Archs that use
39 * syscall wrappers may have syscalls symbols aliases prefixed
40 * with ".SyS" or ".sys" instead of "sys", leading to an unwanted
43 return !strcmp(sym + 3, name + 3);
47 #ifdef ARCH_TRACE_IGNORE_COMPAT_SYSCALLS
49 * Some architectures that allow for 32bit applications
50 * to run on a 64bit kernel, do not map the syscalls for
51 * the 32bit tasks the same as they do for 64bit tasks.
55 * In such a case, instead of reporting the wrong syscalls,
58 * For an arch to ignore the compat syscalls it needs to
59 * define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS as well as
60 * define the function arch_trace_is_compat_syscall() to let
61 * the tracing system know that it should ignore it.
64 trace_get_syscall_nr(struct task_struct *task, struct pt_regs *regs)
66 if (unlikely(arch_trace_is_compat_syscall(regs)))
69 return syscall_get_nr(task, regs);
73 trace_get_syscall_nr(struct task_struct *task, struct pt_regs *regs)
75 return syscall_get_nr(task, regs);
77 #endif /* ARCH_TRACE_IGNORE_COMPAT_SYSCALLS */
79 static __init struct syscall_metadata *
80 find_syscall_meta(unsigned long syscall)
82 struct syscall_metadata **start;
83 struct syscall_metadata **stop;
84 char str[KSYM_SYMBOL_LEN];
87 start = __start_syscalls_metadata;
88 stop = __stop_syscalls_metadata;
89 kallsyms_lookup(syscall, NULL, NULL, NULL, str);
91 if (arch_syscall_match_sym_name(str, "sys_ni_syscall"))
94 for ( ; start < stop; start++) {
95 if ((*start)->name && arch_syscall_match_sym_name(str, (*start)->name))
101 static struct syscall_metadata *syscall_nr_to_meta(int nr)
103 if (!syscalls_metadata || nr >= NR_syscalls || nr < 0)
106 return syscalls_metadata[nr];
109 static enum print_line_t
110 print_syscall_enter(struct trace_iterator *iter, int flags,
111 struct trace_event *event)
113 struct trace_seq *s = &iter->seq;
114 struct trace_entry *ent = iter->ent;
115 struct syscall_trace_enter *trace;
116 struct syscall_metadata *entry;
119 trace = (typeof(trace))ent;
121 entry = syscall_nr_to_meta(syscall);
126 if (entry->enter_event->event.type != ent->type) {
131 ret = trace_seq_printf(s, "%s(", entry->name);
133 return TRACE_TYPE_PARTIAL_LINE;
135 for (i = 0; i < entry->nb_args; i++) {
136 /* parameter types */
137 if (trace_flags & TRACE_ITER_VERBOSE) {
138 ret = trace_seq_printf(s, "%s ", entry->types[i]);
140 return TRACE_TYPE_PARTIAL_LINE;
142 /* parameter values */
143 ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
145 i == entry->nb_args - 1 ? "" : ", ");
147 return TRACE_TYPE_PARTIAL_LINE;
150 ret = trace_seq_putc(s, ')');
152 return TRACE_TYPE_PARTIAL_LINE;
155 ret = trace_seq_putc(s, '\n');
157 return TRACE_TYPE_PARTIAL_LINE;
159 return TRACE_TYPE_HANDLED;
162 static enum print_line_t
163 print_syscall_exit(struct trace_iterator *iter, int flags,
164 struct trace_event *event)
166 struct trace_seq *s = &iter->seq;
167 struct trace_entry *ent = iter->ent;
168 struct syscall_trace_exit *trace;
170 struct syscall_metadata *entry;
173 trace = (typeof(trace))ent;
175 entry = syscall_nr_to_meta(syscall);
178 trace_seq_putc(s, '\n');
179 return TRACE_TYPE_HANDLED;
182 if (entry->exit_event->event.type != ent->type) {
184 return TRACE_TYPE_UNHANDLED;
187 ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
190 return TRACE_TYPE_PARTIAL_LINE;
192 return TRACE_TYPE_HANDLED;
195 extern char *__bad_type_size(void);
197 #define SYSCALL_FIELD(type, name) \
198 sizeof(type) != sizeof(trace.name) ? \
199 __bad_type_size() : \
200 #type, #name, offsetof(typeof(trace), name), \
201 sizeof(trace.name), is_signed_type(type)
204 __set_enter_print_fmt(struct syscall_metadata *entry, char *buf, int len)
209 /* When len=0, we just calculate the needed length */
210 #define LEN_OR_ZERO (len ? len - pos : 0)
212 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
213 for (i = 0; i < entry->nb_args; i++) {
214 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
215 entry->args[i], sizeof(unsigned long),
216 i == entry->nb_args - 1 ? "" : ", ");
218 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
220 for (i = 0; i < entry->nb_args; i++) {
221 pos += snprintf(buf + pos, LEN_OR_ZERO,
222 ", ((unsigned long)(REC->%s))", entry->args[i]);
227 /* return the length of print_fmt */
231 static int __init set_syscall_print_fmt(struct ftrace_event_call *call)
235 struct syscall_metadata *entry = call->data;
237 if (entry->enter_event != call) {
238 call->print_fmt = "\"0x%lx\", REC->ret";
242 /* First: called with 0 length to calculate the needed length */
243 len = __set_enter_print_fmt(entry, NULL, 0);
245 print_fmt = kmalloc(len + 1, GFP_KERNEL);
249 /* Second: actually write the @print_fmt */
250 __set_enter_print_fmt(entry, print_fmt, len + 1);
251 call->print_fmt = print_fmt;
256 static void __init free_syscall_print_fmt(struct ftrace_event_call *call)
258 struct syscall_metadata *entry = call->data;
260 if (entry->enter_event == call)
261 kfree(call->print_fmt);
264 static int __init syscall_enter_define_fields(struct ftrace_event_call *call)
266 struct syscall_trace_enter trace;
267 struct syscall_metadata *meta = call->data;
270 int offset = offsetof(typeof(trace), args);
272 ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
276 for (i = 0; i < meta->nb_args; i++) {
277 ret = trace_define_field(call, meta->types[i],
278 meta->args[i], offset,
279 sizeof(unsigned long), 0,
281 offset += sizeof(unsigned long);
287 static int __init syscall_exit_define_fields(struct ftrace_event_call *call)
289 struct syscall_trace_exit trace;
292 ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
296 ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
302 static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id)
304 struct trace_array *tr = data;
305 struct ftrace_event_file *ftrace_file;
306 struct syscall_trace_enter *entry;
307 struct syscall_metadata *sys_data;
308 struct ring_buffer_event *event;
309 struct ring_buffer *buffer;
310 unsigned long irq_flags;
315 syscall_nr = trace_get_syscall_nr(current, regs);
319 /* Here we're inside tp handler's rcu_read_lock_sched (__DO_TRACE) */
320 ftrace_file = rcu_dereference_sched(tr->enter_syscall_files[syscall_nr]);
324 if (ftrace_trigger_soft_disabled(ftrace_file))
327 sys_data = syscall_nr_to_meta(syscall_nr);
331 size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;
333 local_save_flags(irq_flags);
334 pc = preempt_count();
336 buffer = tr->trace_buffer.buffer;
337 event = trace_buffer_lock_reserve(buffer,
338 sys_data->enter_event->event.type, size, irq_flags, pc);
342 entry = ring_buffer_event_data(event);
343 entry->nr = syscall_nr;
344 syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);
346 event_trigger_unlock_commit(ftrace_file, buffer, event, entry,
350 static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret)
352 struct trace_array *tr = data;
353 struct ftrace_event_file *ftrace_file;
354 struct syscall_trace_exit *entry;
355 struct syscall_metadata *sys_data;
356 struct ring_buffer_event *event;
357 struct ring_buffer *buffer;
358 unsigned long irq_flags;
362 syscall_nr = trace_get_syscall_nr(current, regs);
366 /* Here we're inside tp handler's rcu_read_lock_sched (__DO_TRACE()) */
367 ftrace_file = rcu_dereference_sched(tr->exit_syscall_files[syscall_nr]);
371 if (ftrace_trigger_soft_disabled(ftrace_file))
374 sys_data = syscall_nr_to_meta(syscall_nr);
378 local_save_flags(irq_flags);
379 pc = preempt_count();
381 buffer = tr->trace_buffer.buffer;
382 event = trace_buffer_lock_reserve(buffer,
383 sys_data->exit_event->event.type, sizeof(*entry),
388 entry = ring_buffer_event_data(event);
389 entry->nr = syscall_nr;
390 entry->ret = syscall_get_return_value(current, regs);
392 event_trigger_unlock_commit(ftrace_file, buffer, event, entry,
396 static int reg_event_syscall_enter(struct ftrace_event_file *file,
397 struct ftrace_event_call *call)
399 struct trace_array *tr = file->tr;
403 num = ((struct syscall_metadata *)call->data)->syscall_nr;
404 if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
406 mutex_lock(&syscall_trace_lock);
407 if (!tr->sys_refcount_enter)
408 ret = register_trace_sys_enter(ftrace_syscall_enter, tr);
410 rcu_assign_pointer(tr->enter_syscall_files[num], file);
411 tr->sys_refcount_enter++;
413 mutex_unlock(&syscall_trace_lock);
417 static void unreg_event_syscall_enter(struct ftrace_event_file *file,
418 struct ftrace_event_call *call)
420 struct trace_array *tr = file->tr;
423 num = ((struct syscall_metadata *)call->data)->syscall_nr;
424 if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
426 mutex_lock(&syscall_trace_lock);
427 tr->sys_refcount_enter--;
428 rcu_assign_pointer(tr->enter_syscall_files[num], NULL);
429 if (!tr->sys_refcount_enter)
430 unregister_trace_sys_enter(ftrace_syscall_enter, tr);
431 mutex_unlock(&syscall_trace_lock);
434 static int reg_event_syscall_exit(struct ftrace_event_file *file,
435 struct ftrace_event_call *call)
437 struct trace_array *tr = file->tr;
441 num = ((struct syscall_metadata *)call->data)->syscall_nr;
442 if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
444 mutex_lock(&syscall_trace_lock);
445 if (!tr->sys_refcount_exit)
446 ret = register_trace_sys_exit(ftrace_syscall_exit, tr);
448 rcu_assign_pointer(tr->exit_syscall_files[num], file);
449 tr->sys_refcount_exit++;
451 mutex_unlock(&syscall_trace_lock);
455 static void unreg_event_syscall_exit(struct ftrace_event_file *file,
456 struct ftrace_event_call *call)
458 struct trace_array *tr = file->tr;
461 num = ((struct syscall_metadata *)call->data)->syscall_nr;
462 if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
464 mutex_lock(&syscall_trace_lock);
465 tr->sys_refcount_exit--;
466 rcu_assign_pointer(tr->exit_syscall_files[num], NULL);
467 if (!tr->sys_refcount_exit)
468 unregister_trace_sys_exit(ftrace_syscall_exit, tr);
469 mutex_unlock(&syscall_trace_lock);
472 static int __init init_syscall_trace(struct ftrace_event_call *call)
477 num = ((struct syscall_metadata *)call->data)->syscall_nr;
478 if (num < 0 || num >= NR_syscalls) {
479 pr_debug("syscall %s metadata not mapped, disabling ftrace event\n",
480 ((struct syscall_metadata *)call->data)->name);
484 if (set_syscall_print_fmt(call) < 0)
487 id = trace_event_raw_init(call);
490 free_syscall_print_fmt(call);
497 struct trace_event_functions enter_syscall_print_funcs = {
498 .trace = print_syscall_enter,
501 struct trace_event_functions exit_syscall_print_funcs = {
502 .trace = print_syscall_exit,
505 struct ftrace_event_class __refdata event_class_syscall_enter = {
506 .system = "syscalls",
507 .reg = syscall_enter_register,
508 .define_fields = syscall_enter_define_fields,
509 .get_fields = syscall_get_enter_fields,
510 .raw_init = init_syscall_trace,
513 struct ftrace_event_class __refdata event_class_syscall_exit = {
514 .system = "syscalls",
515 .reg = syscall_exit_register,
516 .define_fields = syscall_exit_define_fields,
517 .fields = LIST_HEAD_INIT(event_class_syscall_exit.fields),
518 .raw_init = init_syscall_trace,
521 unsigned long __init __weak arch_syscall_addr(int nr)
523 return (unsigned long)sys_call_table[nr];
526 static int __init init_ftrace_syscalls(void)
528 struct syscall_metadata *meta;
532 syscalls_metadata = kcalloc(NR_syscalls, sizeof(*syscalls_metadata),
534 if (!syscalls_metadata) {
539 for (i = 0; i < NR_syscalls; i++) {
540 addr = arch_syscall_addr(i);
541 meta = find_syscall_meta(addr);
545 meta->syscall_nr = i;
546 syscalls_metadata[i] = meta;
551 early_initcall(init_ftrace_syscalls);
553 #ifdef CONFIG_PERF_EVENTS
555 static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls);
556 static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
557 static int sys_perf_refcount_enter;
558 static int sys_perf_refcount_exit;
560 static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
562 struct syscall_metadata *sys_data;
563 struct syscall_trace_enter *rec;
564 struct hlist_head *head;
569 syscall_nr = trace_get_syscall_nr(current, regs);
572 if (!test_bit(syscall_nr, enabled_perf_enter_syscalls))
575 sys_data = syscall_nr_to_meta(syscall_nr);
579 head = this_cpu_ptr(sys_data->enter_event->perf_events);
580 if (hlist_empty(head))
583 /* get the size after alignment with the u32 buffer size field */
584 size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
585 size = ALIGN(size + sizeof(u32), sizeof(u64));
588 rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
589 sys_data->enter_event->event.type, regs, &rctx);
593 rec->nr = syscall_nr;
594 syscall_get_arguments(current, regs, 0, sys_data->nb_args,
595 (unsigned long *)&rec->args);
596 perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
599 static int perf_sysenter_enable(struct ftrace_event_call *call)
604 num = ((struct syscall_metadata *)call->data)->syscall_nr;
606 mutex_lock(&syscall_trace_lock);
607 if (!sys_perf_refcount_enter)
608 ret = register_trace_sys_enter(perf_syscall_enter, NULL);
610 pr_info("event trace: Could not activate"
611 "syscall entry trace point");
613 set_bit(num, enabled_perf_enter_syscalls);
614 sys_perf_refcount_enter++;
616 mutex_unlock(&syscall_trace_lock);
620 static void perf_sysenter_disable(struct ftrace_event_call *call)
624 num = ((struct syscall_metadata *)call->data)->syscall_nr;
626 mutex_lock(&syscall_trace_lock);
627 sys_perf_refcount_enter--;
628 clear_bit(num, enabled_perf_enter_syscalls);
629 if (!sys_perf_refcount_enter)
630 unregister_trace_sys_enter(perf_syscall_enter, NULL);
631 mutex_unlock(&syscall_trace_lock);
634 static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
636 struct syscall_metadata *sys_data;
637 struct syscall_trace_exit *rec;
638 struct hlist_head *head;
643 syscall_nr = trace_get_syscall_nr(current, regs);
646 if (!test_bit(syscall_nr, enabled_perf_exit_syscalls))
649 sys_data = syscall_nr_to_meta(syscall_nr);
653 head = this_cpu_ptr(sys_data->exit_event->perf_events);
654 if (hlist_empty(head))
657 /* We can probably do that at build time */
658 size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
661 rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
662 sys_data->exit_event->event.type, regs, &rctx);
666 rec->nr = syscall_nr;
667 rec->ret = syscall_get_return_value(current, regs);
668 perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
671 static int perf_sysexit_enable(struct ftrace_event_call *call)
676 num = ((struct syscall_metadata *)call->data)->syscall_nr;
678 mutex_lock(&syscall_trace_lock);
679 if (!sys_perf_refcount_exit)
680 ret = register_trace_sys_exit(perf_syscall_exit, NULL);
682 pr_info("event trace: Could not activate"
683 "syscall exit trace point");
685 set_bit(num, enabled_perf_exit_syscalls);
686 sys_perf_refcount_exit++;
688 mutex_unlock(&syscall_trace_lock);
692 static void perf_sysexit_disable(struct ftrace_event_call *call)
696 num = ((struct syscall_metadata *)call->data)->syscall_nr;
698 mutex_lock(&syscall_trace_lock);
699 sys_perf_refcount_exit--;
700 clear_bit(num, enabled_perf_exit_syscalls);
701 if (!sys_perf_refcount_exit)
702 unregister_trace_sys_exit(perf_syscall_exit, NULL);
703 mutex_unlock(&syscall_trace_lock);
706 #endif /* CONFIG_PERF_EVENTS */
708 static int syscall_enter_register(struct ftrace_event_call *event,
709 enum trace_reg type, void *data)
711 struct ftrace_event_file *file = data;
714 case TRACE_REG_REGISTER:
715 return reg_event_syscall_enter(file, event);
716 case TRACE_REG_UNREGISTER:
717 unreg_event_syscall_enter(file, event);
720 #ifdef CONFIG_PERF_EVENTS
721 case TRACE_REG_PERF_REGISTER:
722 return perf_sysenter_enable(event);
723 case TRACE_REG_PERF_UNREGISTER:
724 perf_sysenter_disable(event);
726 case TRACE_REG_PERF_OPEN:
727 case TRACE_REG_PERF_CLOSE:
728 case TRACE_REG_PERF_ADD:
729 case TRACE_REG_PERF_DEL:
736 static int syscall_exit_register(struct ftrace_event_call *event,
737 enum trace_reg type, void *data)
739 struct ftrace_event_file *file = data;
742 case TRACE_REG_REGISTER:
743 return reg_event_syscall_exit(file, event);
744 case TRACE_REG_UNREGISTER:
745 unreg_event_syscall_exit(file, event);
748 #ifdef CONFIG_PERF_EVENTS
749 case TRACE_REG_PERF_REGISTER:
750 return perf_sysexit_enable(event);
751 case TRACE_REG_PERF_UNREGISTER:
752 perf_sysexit_disable(event);
754 case TRACE_REG_PERF_OPEN:
755 case TRACE_REG_PERF_CLOSE:
756 case TRACE_REG_PERF_ADD:
757 case TRACE_REG_PERF_DEL: