2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
41 #include <linux/sched/rt.h>
44 #include "trace_output.h"
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
50 bool ring_buffer_expanded;
53 * We need to change this state when a selftest is running.
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
56 * insertions into the ring-buffer such as trace_printk could occurred
57 * at the same time, giving false positive or negative results.
59 static bool __read_mostly tracing_selftest_running;
62 * If a tracer is running, we do not want to run SELFTEST.
64 bool __read_mostly tracing_selftest_disabled;
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
71 static struct tracer_flags dummy_tracer_flags = {
73 .opts = dummy_tracer_opt
76 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
82 * To prevent the comm cache from being overwritten when no
83 * tracing is active, only save the comm when a trace event
86 static DEFINE_PER_CPU(bool, trace_cmdline_save);
89 * Kill all tracing for good (never come back).
90 * It is initialized to 1 but will turn to zero if the initialization
91 * of the tracer is successful. But that is the only place that sets
94 static int tracing_disabled = 1;
96 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
98 cpumask_var_t __read_mostly tracing_buffer_mask;
101 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
103 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104 * is set, then ftrace_dump is called. This will output the contents
105 * of the ftrace buffers to the console. This is very useful for
106 * capturing traces that lead to crashes and outputing it to a
109 * It is default off, but you can enable it with either specifying
110 * "ftrace_dump_on_oops" in the kernel command line, or setting
111 * /proc/sys/kernel/ftrace_dump_on_oops
112 * Set 1 if you want to dump buffers of all CPUs
113 * Set 2 if you want to dump the buffer of the CPU that triggered oops
116 enum ftrace_dump_mode ftrace_dump_on_oops;
118 /* When set, tracing will stop when a WARN*() is hit */
119 int __disable_trace_on_warning;
121 static int tracing_set_tracer(const char *buf);
123 #define MAX_TRACER_SIZE 100
124 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
125 static char *default_bootup_tracer;
127 static bool allocate_snapshot;
129 static int __init set_cmdline_ftrace(char *str)
131 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
132 default_bootup_tracer = bootup_tracer_buf;
133 /* We are using ftrace early, expand it */
134 ring_buffer_expanded = true;
137 __setup("ftrace=", set_cmdline_ftrace);
139 static int __init set_ftrace_dump_on_oops(char *str)
141 if (*str++ != '=' || !*str) {
142 ftrace_dump_on_oops = DUMP_ALL;
146 if (!strcmp("orig_cpu", str)) {
147 ftrace_dump_on_oops = DUMP_ORIG;
153 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
155 static int __init stop_trace_on_warning(char *str)
157 __disable_trace_on_warning = 1;
160 __setup("traceoff_on_warning=", stop_trace_on_warning);
162 static int __init boot_alloc_snapshot(char *str)
164 allocate_snapshot = true;
165 /* We also need the main ring buffer expanded */
166 ring_buffer_expanded = true;
169 __setup("alloc_snapshot", boot_alloc_snapshot);
172 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
173 static char *trace_boot_options __initdata;
175 static int __init set_trace_boot_options(char *str)
177 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
178 trace_boot_options = trace_boot_options_buf;
181 __setup("trace_options=", set_trace_boot_options);
184 unsigned long long ns2usecs(cycle_t nsec)
192 * The global_trace is the descriptor that holds the tracing
193 * buffers for the live tracing. For each CPU, it contains
194 * a link list of pages that will store trace entries. The
195 * page descriptor of the pages in the memory is used to hold
196 * the link list by linking the lru item in the page descriptor
197 * to each of the pages in the buffer per CPU.
199 * For each active CPU there is a data field that holds the
200 * pages for the buffer for that CPU. Each CPU has the same number
201 * of pages allocated for its buffer.
203 static struct trace_array global_trace;
205 LIST_HEAD(ftrace_trace_arrays);
207 int trace_array_get(struct trace_array *this_tr)
209 struct trace_array *tr;
212 mutex_lock(&trace_types_lock);
213 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
220 mutex_unlock(&trace_types_lock);
225 static void __trace_array_put(struct trace_array *this_tr)
227 WARN_ON(!this_tr->ref);
231 void trace_array_put(struct trace_array *this_tr)
233 mutex_lock(&trace_types_lock);
234 __trace_array_put(this_tr);
235 mutex_unlock(&trace_types_lock);
238 int filter_check_discard(struct ftrace_event_file *file, void *rec,
239 struct ring_buffer *buffer,
240 struct ring_buffer_event *event)
242 if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&
243 !filter_match_preds(file->filter, rec)) {
244 ring_buffer_discard_commit(buffer, event);
250 EXPORT_SYMBOL_GPL(filter_check_discard);
252 int call_filter_check_discard(struct ftrace_event_call *call, void *rec,
253 struct ring_buffer *buffer,
254 struct ring_buffer_event *event)
256 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
257 !filter_match_preds(call->filter, rec)) {
258 ring_buffer_discard_commit(buffer, event);
264 EXPORT_SYMBOL_GPL(call_filter_check_discard);
266 cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
270 /* Early boot up does not have a buffer yet */
272 return trace_clock_local();
274 ts = ring_buffer_time_stamp(buf->buffer, cpu);
275 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
280 cycle_t ftrace_now(int cpu)
282 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
286 * tracing_is_enabled - Show if global_trace has been disabled
288 * Shows if the global trace has been enabled or not. It uses the
289 * mirror flag "buffer_disabled" to be used in fast paths such as for
290 * the irqsoff tracer. But it may be inaccurate due to races. If you
291 * need to know the accurate state, use tracing_is_on() which is a little
292 * slower, but accurate.
294 int tracing_is_enabled(void)
297 * For quick access (irqsoff uses this in fast path), just
298 * return the mirror variable of the state of the ring buffer.
299 * It's a little racy, but we don't really care.
302 return !global_trace.buffer_disabled;
306 * trace_buf_size is the size in bytes that is allocated
307 * for a buffer. Note, the number of bytes is always rounded
310 * This number is purposely set to a low number of 16384.
311 * If the dump on oops happens, it will be much appreciated
312 * to not have to wait for all that output. Anyway this can be
313 * boot time and run time configurable.
315 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
317 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
319 /* trace_types holds a link list of available tracers. */
320 static struct tracer *trace_types __read_mostly;
323 * trace_types_lock is used to protect the trace_types list.
325 DEFINE_MUTEX(trace_types_lock);
328 * serialize the access of the ring buffer
330 * ring buffer serializes readers, but it is low level protection.
331 * The validity of the events (which returns by ring_buffer_peek() ..etc)
332 * are not protected by ring buffer.
334 * The content of events may become garbage if we allow other process consumes
335 * these events concurrently:
336 * A) the page of the consumed events may become a normal page
337 * (not reader page) in ring buffer, and this page will be rewrited
338 * by events producer.
339 * B) The page of the consumed events may become a page for splice_read,
340 * and this page will be returned to system.
342 * These primitives allow multi process access to different cpu ring buffer
345 * These primitives don't distinguish read-only and read-consume access.
346 * Multi read-only access are also serialized.
350 static DECLARE_RWSEM(all_cpu_access_lock);
351 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
353 static inline void trace_access_lock(int cpu)
355 if (cpu == RING_BUFFER_ALL_CPUS) {
356 /* gain it for accessing the whole ring buffer. */
357 down_write(&all_cpu_access_lock);
359 /* gain it for accessing a cpu ring buffer. */
361 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
362 down_read(&all_cpu_access_lock);
364 /* Secondly block other access to this @cpu ring buffer. */
365 mutex_lock(&per_cpu(cpu_access_lock, cpu));
369 static inline void trace_access_unlock(int cpu)
371 if (cpu == RING_BUFFER_ALL_CPUS) {
372 up_write(&all_cpu_access_lock);
374 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
375 up_read(&all_cpu_access_lock);
379 static inline void trace_access_lock_init(void)
383 for_each_possible_cpu(cpu)
384 mutex_init(&per_cpu(cpu_access_lock, cpu));
389 static DEFINE_MUTEX(access_lock);
391 static inline void trace_access_lock(int cpu)
394 mutex_lock(&access_lock);
397 static inline void trace_access_unlock(int cpu)
400 mutex_unlock(&access_lock);
403 static inline void trace_access_lock_init(void)
409 /* trace_flags holds trace_options default values */
410 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
411 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
412 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
413 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
415 static void tracer_tracing_on(struct trace_array *tr)
417 if (tr->trace_buffer.buffer)
418 ring_buffer_record_on(tr->trace_buffer.buffer);
420 * This flag is looked at when buffers haven't been allocated
421 * yet, or by some tracers (like irqsoff), that just want to
422 * know if the ring buffer has been disabled, but it can handle
423 * races of where it gets disabled but we still do a record.
424 * As the check is in the fast path of the tracers, it is more
425 * important to be fast than accurate.
427 tr->buffer_disabled = 0;
428 /* Make the flag seen by readers */
433 * tracing_on - enable tracing buffers
435 * This function enables tracing buffers that may have been
436 * disabled with tracing_off.
438 void tracing_on(void)
440 tracer_tracing_on(&global_trace);
442 EXPORT_SYMBOL_GPL(tracing_on);
445 * __trace_puts - write a constant string into the trace buffer.
446 * @ip: The address of the caller
447 * @str: The constant string to write
448 * @size: The size of the string.
450 int __trace_puts(unsigned long ip, const char *str, int size)
452 struct ring_buffer_event *event;
453 struct ring_buffer *buffer;
454 struct print_entry *entry;
455 unsigned long irq_flags;
458 if (unlikely(tracing_selftest_running || tracing_disabled))
461 alloc = sizeof(*entry) + size + 2; /* possible \n added */
463 local_save_flags(irq_flags);
464 buffer = global_trace.trace_buffer.buffer;
465 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
466 irq_flags, preempt_count());
470 entry = ring_buffer_event_data(event);
473 memcpy(&entry->buf, str, size);
475 /* Add a newline if necessary */
476 if (entry->buf[size - 1] != '\n') {
477 entry->buf[size] = '\n';
478 entry->buf[size + 1] = '\0';
480 entry->buf[size] = '\0';
482 __buffer_unlock_commit(buffer, event);
486 EXPORT_SYMBOL_GPL(__trace_puts);
489 * __trace_bputs - write the pointer to a constant string into trace buffer
490 * @ip: The address of the caller
491 * @str: The constant string to write to the buffer to
493 int __trace_bputs(unsigned long ip, const char *str)
495 struct ring_buffer_event *event;
496 struct ring_buffer *buffer;
497 struct bputs_entry *entry;
498 unsigned long irq_flags;
499 int size = sizeof(struct bputs_entry);
501 if (unlikely(tracing_selftest_running || tracing_disabled))
504 local_save_flags(irq_flags);
505 buffer = global_trace.trace_buffer.buffer;
506 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
507 irq_flags, preempt_count());
511 entry = ring_buffer_event_data(event);
515 __buffer_unlock_commit(buffer, event);
519 EXPORT_SYMBOL_GPL(__trace_bputs);
521 #ifdef CONFIG_TRACER_SNAPSHOT
523 * trace_snapshot - take a snapshot of the current buffer.
525 * This causes a swap between the snapshot buffer and the current live
526 * tracing buffer. You can use this to take snapshots of the live
527 * trace when some condition is triggered, but continue to trace.
529 * Note, make sure to allocate the snapshot with either
530 * a tracing_snapshot_alloc(), or by doing it manually
531 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
533 * If the snapshot buffer is not allocated, it will stop tracing.
534 * Basically making a permanent snapshot.
536 void tracing_snapshot(void)
538 struct trace_array *tr = &global_trace;
539 struct tracer *tracer = tr->current_trace;
543 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
544 internal_trace_puts("*** snapshot is being ignored ***\n");
548 if (!tr->allocated_snapshot) {
549 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
550 internal_trace_puts("*** stopping trace here! ***\n");
555 /* Note, snapshot can not be used when the tracer uses it */
556 if (tracer->use_max_tr) {
557 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
558 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
562 local_irq_save(flags);
563 update_max_tr(tr, current, smp_processor_id());
564 local_irq_restore(flags);
566 EXPORT_SYMBOL_GPL(tracing_snapshot);
568 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
569 struct trace_buffer *size_buf, int cpu_id);
570 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
572 static int alloc_snapshot(struct trace_array *tr)
576 if (!tr->allocated_snapshot) {
578 /* allocate spare buffer */
579 ret = resize_buffer_duplicate_size(&tr->max_buffer,
580 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
584 tr->allocated_snapshot = true;
590 void free_snapshot(struct trace_array *tr)
593 * We don't free the ring buffer. instead, resize it because
594 * The max_tr ring buffer has some state (e.g. ring->clock) and
595 * we want preserve it.
597 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
598 set_buffer_entries(&tr->max_buffer, 1);
599 tracing_reset_online_cpus(&tr->max_buffer);
600 tr->allocated_snapshot = false;
604 * tracing_alloc_snapshot - allocate snapshot buffer.
606 * This only allocates the snapshot buffer if it isn't already
607 * allocated - it doesn't also take a snapshot.
609 * This is meant to be used in cases where the snapshot buffer needs
610 * to be set up for events that can't sleep but need to be able to
611 * trigger a snapshot.
613 int tracing_alloc_snapshot(void)
615 struct trace_array *tr = &global_trace;
618 ret = alloc_snapshot(tr);
623 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
626 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
628 * This is similar to trace_snapshot(), but it will allocate the
629 * snapshot buffer if it isn't already allocated. Use this only
630 * where it is safe to sleep, as the allocation may sleep.
632 * This causes a swap between the snapshot buffer and the current live
633 * tracing buffer. You can use this to take snapshots of the live
634 * trace when some condition is triggered, but continue to trace.
636 void tracing_snapshot_alloc(void)
640 ret = tracing_alloc_snapshot();
646 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
648 void tracing_snapshot(void)
650 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
652 EXPORT_SYMBOL_GPL(tracing_snapshot);
653 int tracing_alloc_snapshot(void)
655 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
658 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
659 void tracing_snapshot_alloc(void)
664 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
665 #endif /* CONFIG_TRACER_SNAPSHOT */
667 static void tracer_tracing_off(struct trace_array *tr)
669 if (tr->trace_buffer.buffer)
670 ring_buffer_record_off(tr->trace_buffer.buffer);
672 * This flag is looked at when buffers haven't been allocated
673 * yet, or by some tracers (like irqsoff), that just want to
674 * know if the ring buffer has been disabled, but it can handle
675 * races of where it gets disabled but we still do a record.
676 * As the check is in the fast path of the tracers, it is more
677 * important to be fast than accurate.
679 tr->buffer_disabled = 1;
680 /* Make the flag seen by readers */
685 * tracing_off - turn off tracing buffers
687 * This function stops the tracing buffers from recording data.
688 * It does not disable any overhead the tracers themselves may
689 * be causing. This function simply causes all recording to
690 * the ring buffers to fail.
692 void tracing_off(void)
694 tracer_tracing_off(&global_trace);
696 EXPORT_SYMBOL_GPL(tracing_off);
698 void disable_trace_on_warning(void)
700 if (__disable_trace_on_warning)
705 * tracer_tracing_is_on - show real state of ring buffer enabled
706 * @tr : the trace array to know if ring buffer is enabled
708 * Shows real state of the ring buffer if it is enabled or not.
710 static int tracer_tracing_is_on(struct trace_array *tr)
712 if (tr->trace_buffer.buffer)
713 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
714 return !tr->buffer_disabled;
718 * tracing_is_on - show state of ring buffers enabled
720 int tracing_is_on(void)
722 return tracer_tracing_is_on(&global_trace);
724 EXPORT_SYMBOL_GPL(tracing_is_on);
726 static int __init set_buf_size(char *str)
728 unsigned long buf_size;
732 buf_size = memparse(str, &str);
733 /* nr_entries can not be zero */
736 trace_buf_size = buf_size;
739 __setup("trace_buf_size=", set_buf_size);
741 static int __init set_tracing_thresh(char *str)
743 unsigned long threshold;
748 ret = kstrtoul(str, 0, &threshold);
751 tracing_thresh = threshold * 1000;
754 __setup("tracing_thresh=", set_tracing_thresh);
756 unsigned long nsecs_to_usecs(unsigned long nsecs)
761 /* These must match the bit postions in trace_iterator_flags */
762 static const char *trace_options[] = {
795 int in_ns; /* is this clock in nanoseconds? */
797 { trace_clock_local, "local", 1 },
798 { trace_clock_global, "global", 1 },
799 { trace_clock_counter, "counter", 0 },
800 { trace_clock_jiffies, "uptime", 1 },
801 { trace_clock, "perf", 1 },
806 * trace_parser_get_init - gets the buffer for trace parser
808 int trace_parser_get_init(struct trace_parser *parser, int size)
810 memset(parser, 0, sizeof(*parser));
812 parser->buffer = kmalloc(size, GFP_KERNEL);
821 * trace_parser_put - frees the buffer for trace parser
823 void trace_parser_put(struct trace_parser *parser)
825 kfree(parser->buffer);
829 * trace_get_user - reads the user input string separated by space
830 * (matched by isspace(ch))
832 * For each string found the 'struct trace_parser' is updated,
833 * and the function returns.
835 * Returns number of bytes read.
837 * See kernel/trace/trace.h for 'struct trace_parser' details.
839 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
840 size_t cnt, loff_t *ppos)
847 trace_parser_clear(parser);
849 ret = get_user(ch, ubuf++);
857 * The parser is not finished with the last write,
858 * continue reading the user input without skipping spaces.
861 /* skip white space */
862 while (cnt && isspace(ch)) {
863 ret = get_user(ch, ubuf++);
870 /* only spaces were written */
880 /* read the non-space input */
881 while (cnt && !isspace(ch)) {
882 if (parser->idx < parser->size - 1)
883 parser->buffer[parser->idx++] = ch;
888 ret = get_user(ch, ubuf++);
895 /* We either got finished input or we have to wait for another call. */
897 parser->buffer[parser->idx] = 0;
898 parser->cont = false;
899 } else if (parser->idx < parser->size - 1) {
901 parser->buffer[parser->idx++] = ch;
914 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
922 if (s->len <= s->readpos)
925 len = s->len - s->readpos;
928 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
938 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
942 if (s->len <= s->readpos)
945 len = s->len - s->readpos;
948 memcpy(buf, s->buffer + s->readpos, cnt);
955 * ftrace_max_lock is used to protect the swapping of buffers
956 * when taking a max snapshot. The buffers themselves are
957 * protected by per_cpu spinlocks. But the action of the swap
958 * needs its own lock.
960 * This is defined as a arch_spinlock_t in order to help
961 * with performance when lockdep debugging is enabled.
963 * It is also used in other places outside the update_max_tr
964 * so it needs to be defined outside of the
965 * CONFIG_TRACER_MAX_TRACE.
967 static arch_spinlock_t ftrace_max_lock =
968 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
970 unsigned long __read_mostly tracing_thresh;
972 #ifdef CONFIG_TRACER_MAX_TRACE
973 unsigned long __read_mostly tracing_max_latency;
976 * Copy the new maximum trace into the separate maximum-trace
977 * structure. (this way the maximum trace is permanently saved,
978 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
981 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
983 struct trace_buffer *trace_buf = &tr->trace_buffer;
984 struct trace_buffer *max_buf = &tr->max_buffer;
985 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
986 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
989 max_buf->time_start = data->preempt_timestamp;
991 max_data->saved_latency = tracing_max_latency;
992 max_data->critical_start = data->critical_start;
993 max_data->critical_end = data->critical_end;
995 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
996 max_data->pid = tsk->pid;
998 * If tsk == current, then use current_uid(), as that does not use
999 * RCU. The irq tracer can be called out of RCU scope.
1002 max_data->uid = current_uid();
1004 max_data->uid = task_uid(tsk);
1006 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1007 max_data->policy = tsk->policy;
1008 max_data->rt_priority = tsk->rt_priority;
1010 /* record this tasks comm */
1011 tracing_record_cmdline(tsk);
1015 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1017 * @tsk: the task with the latency
1018 * @cpu: The cpu that initiated the trace.
1020 * Flip the buffers between the @tr and the max_tr and record information
1021 * about which task was the cause of this latency.
1024 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1026 struct ring_buffer *buf;
1031 WARN_ON_ONCE(!irqs_disabled());
1033 if (!tr->allocated_snapshot) {
1034 /* Only the nop tracer should hit this when disabling */
1035 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1039 arch_spin_lock(&ftrace_max_lock);
1041 buf = tr->trace_buffer.buffer;
1042 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1043 tr->max_buffer.buffer = buf;
1045 __update_max_tr(tr, tsk, cpu);
1046 arch_spin_unlock(&ftrace_max_lock);
1050 * update_max_tr_single - only copy one trace over, and reset the rest
1052 * @tsk - task with the latency
1053 * @cpu - the cpu of the buffer to copy.
1055 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1058 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1065 WARN_ON_ONCE(!irqs_disabled());
1066 if (!tr->allocated_snapshot) {
1067 /* Only the nop tracer should hit this when disabling */
1068 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1072 arch_spin_lock(&ftrace_max_lock);
1074 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1076 if (ret == -EBUSY) {
1078 * We failed to swap the buffer due to a commit taking
1079 * place on this CPU. We fail to record, but we reset
1080 * the max trace buffer (no one writes directly to it)
1081 * and flag that it failed.
1083 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1084 "Failed to swap buffers due to commit in progress\n");
1087 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1089 __update_max_tr(tr, tsk, cpu);
1090 arch_spin_unlock(&ftrace_max_lock);
1092 #endif /* CONFIG_TRACER_MAX_TRACE */
1094 static int default_wait_pipe(struct trace_iterator *iter)
1096 /* Iterators are static, they should be filled or empty */
1097 if (trace_buffer_iter(iter, iter->cpu_file))
1100 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
1103 #ifdef CONFIG_FTRACE_STARTUP_TEST
1104 static int run_tracer_selftest(struct tracer *type)
1106 struct trace_array *tr = &global_trace;
1107 struct tracer *saved_tracer = tr->current_trace;
1110 if (!type->selftest || tracing_selftest_disabled)
1114 * Run a selftest on this tracer.
1115 * Here we reset the trace buffer, and set the current
1116 * tracer to be this tracer. The tracer can then run some
1117 * internal tracing to verify that everything is in order.
1118 * If we fail, we do not register this tracer.
1120 tracing_reset_online_cpus(&tr->trace_buffer);
1122 tr->current_trace = type;
1124 #ifdef CONFIG_TRACER_MAX_TRACE
1125 if (type->use_max_tr) {
1126 /* If we expanded the buffers, make sure the max is expanded too */
1127 if (ring_buffer_expanded)
1128 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1129 RING_BUFFER_ALL_CPUS);
1130 tr->allocated_snapshot = true;
1134 /* the test is responsible for initializing and enabling */
1135 pr_info("Testing tracer %s: ", type->name);
1136 ret = type->selftest(type, tr);
1137 /* the test is responsible for resetting too */
1138 tr->current_trace = saved_tracer;
1140 printk(KERN_CONT "FAILED!\n");
1141 /* Add the warning after printing 'FAILED' */
1145 /* Only reset on passing, to avoid touching corrupted buffers */
1146 tracing_reset_online_cpus(&tr->trace_buffer);
1148 #ifdef CONFIG_TRACER_MAX_TRACE
1149 if (type->use_max_tr) {
1150 tr->allocated_snapshot = false;
1152 /* Shrink the max buffer again */
1153 if (ring_buffer_expanded)
1154 ring_buffer_resize(tr->max_buffer.buffer, 1,
1155 RING_BUFFER_ALL_CPUS);
1159 printk(KERN_CONT "PASSED\n");
1163 static inline int run_tracer_selftest(struct tracer *type)
1167 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1170 * register_tracer - register a tracer with the ftrace system.
1171 * @type - the plugin for the tracer
1173 * Register a new plugin tracer.
1175 int register_tracer(struct tracer *type)
1181 pr_info("Tracer must have a name\n");
1185 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1186 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1190 mutex_lock(&trace_types_lock);
1192 tracing_selftest_running = true;
1194 for (t = trace_types; t; t = t->next) {
1195 if (strcmp(type->name, t->name) == 0) {
1197 pr_info("Tracer %s already registered\n",
1204 if (!type->set_flag)
1205 type->set_flag = &dummy_set_flag;
1207 type->flags = &dummy_tracer_flags;
1209 if (!type->flags->opts)
1210 type->flags->opts = dummy_tracer_opt;
1211 if (!type->wait_pipe)
1212 type->wait_pipe = default_wait_pipe;
1214 ret = run_tracer_selftest(type);
1218 type->next = trace_types;
1222 tracing_selftest_running = false;
1223 mutex_unlock(&trace_types_lock);
1225 if (ret || !default_bootup_tracer)
1228 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1231 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1232 /* Do we want this tracer to start on bootup? */
1233 tracing_set_tracer(type->name);
1234 default_bootup_tracer = NULL;
1235 /* disable other selftests, since this will break it. */
1236 tracing_selftest_disabled = true;
1237 #ifdef CONFIG_FTRACE_STARTUP_TEST
1238 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1246 void tracing_reset(struct trace_buffer *buf, int cpu)
1248 struct ring_buffer *buffer = buf->buffer;
1253 ring_buffer_record_disable(buffer);
1255 /* Make sure all commits have finished */
1256 synchronize_sched();
1257 ring_buffer_reset_cpu(buffer, cpu);
1259 ring_buffer_record_enable(buffer);
1262 void tracing_reset_online_cpus(struct trace_buffer *buf)
1264 struct ring_buffer *buffer = buf->buffer;
1270 ring_buffer_record_disable(buffer);
1272 /* Make sure all commits have finished */
1273 synchronize_sched();
1275 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1277 for_each_online_cpu(cpu)
1278 ring_buffer_reset_cpu(buffer, cpu);
1280 ring_buffer_record_enable(buffer);
1283 /* Must have trace_types_lock held */
1284 void tracing_reset_all_online_cpus(void)
1286 struct trace_array *tr;
1288 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1289 tracing_reset_online_cpus(&tr->trace_buffer);
1290 #ifdef CONFIG_TRACER_MAX_TRACE
1291 tracing_reset_online_cpus(&tr->max_buffer);
1296 #define SAVED_CMDLINES 128
1297 #define NO_CMDLINE_MAP UINT_MAX
1298 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1299 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
1300 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
1301 static int cmdline_idx;
1302 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1304 /* temporary disable recording */
1305 static atomic_t trace_record_cmdline_disabled __read_mostly;
1307 static void trace_init_cmdlines(void)
1309 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
1310 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
1314 int is_tracing_stopped(void)
1316 return global_trace.stop_count;
1320 * tracing_start - quick start of the tracer
1322 * If tracing is enabled but was stopped by tracing_stop,
1323 * this will start the tracer back up.
1325 void tracing_start(void)
1327 struct ring_buffer *buffer;
1328 unsigned long flags;
1330 if (tracing_disabled)
1333 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1334 if (--global_trace.stop_count) {
1335 if (global_trace.stop_count < 0) {
1336 /* Someone screwed up their debugging */
1338 global_trace.stop_count = 0;
1343 /* Prevent the buffers from switching */
1344 arch_spin_lock(&ftrace_max_lock);
1346 buffer = global_trace.trace_buffer.buffer;
1348 ring_buffer_record_enable(buffer);
1350 #ifdef CONFIG_TRACER_MAX_TRACE
1351 buffer = global_trace.max_buffer.buffer;
1353 ring_buffer_record_enable(buffer);
1356 arch_spin_unlock(&ftrace_max_lock);
1359 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1362 static void tracing_start_tr(struct trace_array *tr)
1364 struct ring_buffer *buffer;
1365 unsigned long flags;
1367 if (tracing_disabled)
1370 /* If global, we need to also start the max tracer */
1371 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1372 return tracing_start();
1374 raw_spin_lock_irqsave(&tr->start_lock, flags);
1376 if (--tr->stop_count) {
1377 if (tr->stop_count < 0) {
1378 /* Someone screwed up their debugging */
1385 buffer = tr->trace_buffer.buffer;
1387 ring_buffer_record_enable(buffer);
1390 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1394 * tracing_stop - quick stop of the tracer
1396 * Light weight way to stop tracing. Use in conjunction with
1399 void tracing_stop(void)
1401 struct ring_buffer *buffer;
1402 unsigned long flags;
1404 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1405 if (global_trace.stop_count++)
1408 /* Prevent the buffers from switching */
1409 arch_spin_lock(&ftrace_max_lock);
1411 buffer = global_trace.trace_buffer.buffer;
1413 ring_buffer_record_disable(buffer);
1415 #ifdef CONFIG_TRACER_MAX_TRACE
1416 buffer = global_trace.max_buffer.buffer;
1418 ring_buffer_record_disable(buffer);
1421 arch_spin_unlock(&ftrace_max_lock);
1424 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1427 static void tracing_stop_tr(struct trace_array *tr)
1429 struct ring_buffer *buffer;
1430 unsigned long flags;
1432 /* If global, we need to also stop the max tracer */
1433 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1434 return tracing_stop();
1436 raw_spin_lock_irqsave(&tr->start_lock, flags);
1437 if (tr->stop_count++)
1440 buffer = tr->trace_buffer.buffer;
1442 ring_buffer_record_disable(buffer);
1445 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1448 void trace_stop_cmdline_recording(void);
1450 static int trace_save_cmdline(struct task_struct *tsk)
1454 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1458 * It's not the end of the world if we don't get
1459 * the lock, but we also don't want to spin
1460 * nor do we want to disable interrupts,
1461 * so if we miss here, then better luck next time.
1463 if (!arch_spin_trylock(&trace_cmdline_lock))
1466 idx = map_pid_to_cmdline[tsk->pid];
1467 if (idx == NO_CMDLINE_MAP) {
1468 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1471 * Check whether the cmdline buffer at idx has a pid
1472 * mapped. We are going to overwrite that entry so we
1473 * need to clear the map_pid_to_cmdline. Otherwise we
1474 * would read the new comm for the old pid.
1476 pid = map_cmdline_to_pid[idx];
1477 if (pid != NO_CMDLINE_MAP)
1478 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1480 map_cmdline_to_pid[idx] = tsk->pid;
1481 map_pid_to_cmdline[tsk->pid] = idx;
1486 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1488 arch_spin_unlock(&trace_cmdline_lock);
1493 void trace_find_cmdline(int pid, char comm[])
1498 strcpy(comm, "<idle>");
1502 if (WARN_ON_ONCE(pid < 0)) {
1503 strcpy(comm, "<XXX>");
1507 if (pid > PID_MAX_DEFAULT) {
1508 strcpy(comm, "<...>");
1513 arch_spin_lock(&trace_cmdline_lock);
1514 map = map_pid_to_cmdline[pid];
1515 if (map != NO_CMDLINE_MAP)
1516 strcpy(comm, saved_cmdlines[map]);
1518 strcpy(comm, "<...>");
1520 arch_spin_unlock(&trace_cmdline_lock);
1524 void tracing_record_cmdline(struct task_struct *tsk)
1526 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1529 if (!__this_cpu_read(trace_cmdline_save))
1532 if (trace_save_cmdline(tsk))
1533 __this_cpu_write(trace_cmdline_save, false);
1537 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1540 struct task_struct *tsk = current;
1542 entry->preempt_count = pc & 0xff;
1543 entry->pid = (tsk) ? tsk->pid : 0;
1545 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1546 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1548 TRACE_FLAG_IRQS_NOSUPPORT |
1550 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1551 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1552 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1553 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1555 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1557 struct ring_buffer_event *
1558 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1561 unsigned long flags, int pc)
1563 struct ring_buffer_event *event;
1565 event = ring_buffer_lock_reserve(buffer, len);
1566 if (event != NULL) {
1567 struct trace_entry *ent = ring_buffer_event_data(event);
1569 tracing_generic_entry_update(ent, flags, pc);
1577 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1579 __this_cpu_write(trace_cmdline_save, true);
1580 ring_buffer_unlock_commit(buffer, event);
1584 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1585 struct ring_buffer_event *event,
1586 unsigned long flags, int pc)
1588 __buffer_unlock_commit(buffer, event);
1590 ftrace_trace_stack(buffer, flags, 6, pc);
1591 ftrace_trace_userstack(buffer, flags, pc);
1594 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1595 struct ring_buffer_event *event,
1596 unsigned long flags, int pc)
1598 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1600 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1602 static struct ring_buffer *temp_buffer;
1604 struct ring_buffer_event *
1605 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1606 struct ftrace_event_file *ftrace_file,
1607 int type, unsigned long len,
1608 unsigned long flags, int pc)
1610 struct ring_buffer_event *entry;
1612 *current_rb = ftrace_file->tr->trace_buffer.buffer;
1613 entry = trace_buffer_lock_reserve(*current_rb,
1614 type, len, flags, pc);
1616 * If tracing is off, but we have triggers enabled
1617 * we still need to look at the event data. Use the temp_buffer
1618 * to store the trace event for the tigger to use. It's recusive
1619 * safe and will not be recorded anywhere.
1621 if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) {
1622 *current_rb = temp_buffer;
1623 entry = trace_buffer_lock_reserve(*current_rb,
1624 type, len, flags, pc);
1628 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1630 struct ring_buffer_event *
1631 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1632 int type, unsigned long len,
1633 unsigned long flags, int pc)
1635 *current_rb = global_trace.trace_buffer.buffer;
1636 return trace_buffer_lock_reserve(*current_rb,
1637 type, len, flags, pc);
1639 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1641 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1642 struct ring_buffer_event *event,
1643 unsigned long flags, int pc)
1645 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1647 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1649 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1650 struct ring_buffer_event *event,
1651 unsigned long flags, int pc,
1652 struct pt_regs *regs)
1654 __buffer_unlock_commit(buffer, event);
1656 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1657 ftrace_trace_userstack(buffer, flags, pc);
1659 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1661 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1662 struct ring_buffer_event *event)
1664 ring_buffer_discard_commit(buffer, event);
1666 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1669 trace_function(struct trace_array *tr,
1670 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1673 struct ftrace_event_call *call = &event_function;
1674 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1675 struct ring_buffer_event *event;
1676 struct ftrace_entry *entry;
1678 /* If we are reading the ring buffer, don't trace */
1679 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1682 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1686 entry = ring_buffer_event_data(event);
1688 entry->parent_ip = parent_ip;
1690 if (!call_filter_check_discard(call, entry, buffer, event))
1691 __buffer_unlock_commit(buffer, event);
1694 #ifdef CONFIG_STACKTRACE
1696 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1697 struct ftrace_stack {
1698 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1701 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1702 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1704 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1705 unsigned long flags,
1706 int skip, int pc, struct pt_regs *regs)
1708 struct ftrace_event_call *call = &event_kernel_stack;
1709 struct ring_buffer_event *event;
1710 struct stack_entry *entry;
1711 struct stack_trace trace;
1713 int size = FTRACE_STACK_ENTRIES;
1715 trace.nr_entries = 0;
1719 * Since events can happen in NMIs there's no safe way to
1720 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1721 * or NMI comes in, it will just have to use the default
1722 * FTRACE_STACK_SIZE.
1724 preempt_disable_notrace();
1726 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1728 * We don't need any atomic variables, just a barrier.
1729 * If an interrupt comes in, we don't care, because it would
1730 * have exited and put the counter back to what we want.
1731 * We just need a barrier to keep gcc from moving things
1735 if (use_stack == 1) {
1736 trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1737 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1740 save_stack_trace_regs(regs, &trace);
1742 save_stack_trace(&trace);
1744 if (trace.nr_entries > size)
1745 size = trace.nr_entries;
1747 /* From now on, use_stack is a boolean */
1750 size *= sizeof(unsigned long);
1752 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1753 sizeof(*entry) + size, flags, pc);
1756 entry = ring_buffer_event_data(event);
1758 memset(&entry->caller, 0, size);
1761 memcpy(&entry->caller, trace.entries,
1762 trace.nr_entries * sizeof(unsigned long));
1764 trace.max_entries = FTRACE_STACK_ENTRIES;
1765 trace.entries = entry->caller;
1767 save_stack_trace_regs(regs, &trace);
1769 save_stack_trace(&trace);
1772 entry->size = trace.nr_entries;
1774 if (!call_filter_check_discard(call, entry, buffer, event))
1775 __buffer_unlock_commit(buffer, event);
1778 /* Again, don't let gcc optimize things here */
1780 __this_cpu_dec(ftrace_stack_reserve);
1781 preempt_enable_notrace();
1785 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1786 int skip, int pc, struct pt_regs *regs)
1788 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1791 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1794 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1797 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1800 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1803 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1806 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1810 * trace_dump_stack - record a stack back trace in the trace buffer
1811 * @skip: Number of functions to skip (helper handlers)
1813 void trace_dump_stack(int skip)
1815 unsigned long flags;
1817 if (tracing_disabled || tracing_selftest_running)
1820 local_save_flags(flags);
1823 * Skip 3 more, seems to get us at the caller of
1827 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1828 flags, skip, preempt_count(), NULL);
1831 static DEFINE_PER_CPU(int, user_stack_count);
1834 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1836 struct ftrace_event_call *call = &event_user_stack;
1837 struct ring_buffer_event *event;
1838 struct userstack_entry *entry;
1839 struct stack_trace trace;
1841 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1845 * NMIs can not handle page faults, even with fix ups.
1846 * The save user stack can (and often does) fault.
1848 if (unlikely(in_nmi()))
1852 * prevent recursion, since the user stack tracing may
1853 * trigger other kernel events.
1856 if (__this_cpu_read(user_stack_count))
1859 __this_cpu_inc(user_stack_count);
1861 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1862 sizeof(*entry), flags, pc);
1864 goto out_drop_count;
1865 entry = ring_buffer_event_data(event);
1867 entry->tgid = current->tgid;
1868 memset(&entry->caller, 0, sizeof(entry->caller));
1870 trace.nr_entries = 0;
1871 trace.max_entries = FTRACE_STACK_ENTRIES;
1873 trace.entries = entry->caller;
1875 save_stack_trace_user(&trace);
1876 if (!call_filter_check_discard(call, entry, buffer, event))
1877 __buffer_unlock_commit(buffer, event);
1880 __this_cpu_dec(user_stack_count);
1886 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1888 ftrace_trace_userstack(tr, flags, preempt_count());
1892 #endif /* CONFIG_STACKTRACE */
1894 /* created for use with alloc_percpu */
1895 struct trace_buffer_struct {
1896 char buffer[TRACE_BUF_SIZE];
1899 static struct trace_buffer_struct *trace_percpu_buffer;
1900 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1901 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1902 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1905 * The buffer used is dependent on the context. There is a per cpu
1906 * buffer for normal context, softirq contex, hard irq context and
1907 * for NMI context. Thise allows for lockless recording.
1909 * Note, if the buffers failed to be allocated, then this returns NULL
1911 static char *get_trace_buf(void)
1913 struct trace_buffer_struct *percpu_buffer;
1916 * If we have allocated per cpu buffers, then we do not
1917 * need to do any locking.
1920 percpu_buffer = trace_percpu_nmi_buffer;
1922 percpu_buffer = trace_percpu_irq_buffer;
1923 else if (in_softirq())
1924 percpu_buffer = trace_percpu_sirq_buffer;
1926 percpu_buffer = trace_percpu_buffer;
1931 return this_cpu_ptr(&percpu_buffer->buffer[0]);
1934 static int alloc_percpu_trace_buffer(void)
1936 struct trace_buffer_struct *buffers;
1937 struct trace_buffer_struct *sirq_buffers;
1938 struct trace_buffer_struct *irq_buffers;
1939 struct trace_buffer_struct *nmi_buffers;
1941 buffers = alloc_percpu(struct trace_buffer_struct);
1945 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1949 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1953 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1957 trace_percpu_buffer = buffers;
1958 trace_percpu_sirq_buffer = sirq_buffers;
1959 trace_percpu_irq_buffer = irq_buffers;
1960 trace_percpu_nmi_buffer = nmi_buffers;
1965 free_percpu(irq_buffers);
1967 free_percpu(sirq_buffers);
1969 free_percpu(buffers);
1971 WARN(1, "Could not allocate percpu trace_printk buffer");
1975 static int buffers_allocated;
1977 void trace_printk_init_buffers(void)
1979 if (buffers_allocated)
1982 if (alloc_percpu_trace_buffer())
1985 pr_info("ftrace: Allocated trace_printk buffers\n");
1987 /* Expand the buffers to set size */
1988 tracing_update_buffers();
1990 buffers_allocated = 1;
1993 * trace_printk_init_buffers() can be called by modules.
1994 * If that happens, then we need to start cmdline recording
1995 * directly here. If the global_trace.buffer is already
1996 * allocated here, then this was called by module code.
1998 if (global_trace.trace_buffer.buffer)
1999 tracing_start_cmdline_record();
2002 void trace_printk_start_comm(void)
2004 /* Start tracing comms if trace printk is set */
2005 if (!buffers_allocated)
2007 tracing_start_cmdline_record();
2010 static void trace_printk_start_stop_comm(int enabled)
2012 if (!buffers_allocated)
2016 tracing_start_cmdline_record();
2018 tracing_stop_cmdline_record();
2022 * trace_vbprintk - write binary msg to tracing buffer
2025 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2027 struct ftrace_event_call *call = &event_bprint;
2028 struct ring_buffer_event *event;
2029 struct ring_buffer *buffer;
2030 struct trace_array *tr = &global_trace;
2031 struct bprint_entry *entry;
2032 unsigned long flags;
2034 int len = 0, size, pc;
2036 if (unlikely(tracing_selftest_running || tracing_disabled))
2039 /* Don't pollute graph traces with trace_vprintk internals */
2040 pause_graph_tracing();
2042 pc = preempt_count();
2043 preempt_disable_notrace();
2045 tbuffer = get_trace_buf();
2051 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2053 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2056 local_save_flags(flags);
2057 size = sizeof(*entry) + sizeof(u32) * len;
2058 buffer = tr->trace_buffer.buffer;
2059 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2063 entry = ring_buffer_event_data(event);
2067 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2068 if (!call_filter_check_discard(call, entry, buffer, event)) {
2069 __buffer_unlock_commit(buffer, event);
2070 ftrace_trace_stack(buffer, flags, 6, pc);
2074 preempt_enable_notrace();
2075 unpause_graph_tracing();
2079 EXPORT_SYMBOL_GPL(trace_vbprintk);
2082 __trace_array_vprintk(struct ring_buffer *buffer,
2083 unsigned long ip, const char *fmt, va_list args)
2085 struct ftrace_event_call *call = &event_print;
2086 struct ring_buffer_event *event;
2087 int len = 0, size, pc;
2088 struct print_entry *entry;
2089 unsigned long flags;
2092 if (tracing_disabled || tracing_selftest_running)
2095 /* Don't pollute graph traces with trace_vprintk internals */
2096 pause_graph_tracing();
2098 pc = preempt_count();
2099 preempt_disable_notrace();
2102 tbuffer = get_trace_buf();
2108 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2109 if (len > TRACE_BUF_SIZE)
2112 local_save_flags(flags);
2113 size = sizeof(*entry) + len + 1;
2114 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2118 entry = ring_buffer_event_data(event);
2121 memcpy(&entry->buf, tbuffer, len);
2122 entry->buf[len] = '\0';
2123 if (!call_filter_check_discard(call, entry, buffer, event)) {
2124 __buffer_unlock_commit(buffer, event);
2125 ftrace_trace_stack(buffer, flags, 6, pc);
2128 preempt_enable_notrace();
2129 unpause_graph_tracing();
2134 int trace_array_vprintk(struct trace_array *tr,
2135 unsigned long ip, const char *fmt, va_list args)
2137 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2140 int trace_array_printk(struct trace_array *tr,
2141 unsigned long ip, const char *fmt, ...)
2146 if (!(trace_flags & TRACE_ITER_PRINTK))
2150 ret = trace_array_vprintk(tr, ip, fmt, ap);
2155 int trace_array_printk_buf(struct ring_buffer *buffer,
2156 unsigned long ip, const char *fmt, ...)
2161 if (!(trace_flags & TRACE_ITER_PRINTK))
2165 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2170 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2172 return trace_array_vprintk(&global_trace, ip, fmt, args);
2174 EXPORT_SYMBOL_GPL(trace_vprintk);
2176 static void trace_iterator_increment(struct trace_iterator *iter)
2178 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2182 ring_buffer_read(buf_iter, NULL);
2185 static struct trace_entry *
2186 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2187 unsigned long *lost_events)
2189 struct ring_buffer_event *event;
2190 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2193 event = ring_buffer_iter_peek(buf_iter, ts);
2195 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2199 iter->ent_size = ring_buffer_event_length(event);
2200 return ring_buffer_event_data(event);
2206 static struct trace_entry *
2207 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2208 unsigned long *missing_events, u64 *ent_ts)
2210 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2211 struct trace_entry *ent, *next = NULL;
2212 unsigned long lost_events = 0, next_lost = 0;
2213 int cpu_file = iter->cpu_file;
2214 u64 next_ts = 0, ts;
2220 * If we are in a per_cpu trace file, don't bother by iterating over
2221 * all cpu and peek directly.
2223 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2224 if (ring_buffer_empty_cpu(buffer, cpu_file))
2226 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2228 *ent_cpu = cpu_file;
2233 for_each_tracing_cpu(cpu) {
2235 if (ring_buffer_empty_cpu(buffer, cpu))
2238 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2241 * Pick the entry with the smallest timestamp:
2243 if (ent && (!next || ts < next_ts)) {
2247 next_lost = lost_events;
2248 next_size = iter->ent_size;
2252 iter->ent_size = next_size;
2255 *ent_cpu = next_cpu;
2261 *missing_events = next_lost;
2266 /* Find the next real entry, without updating the iterator itself */
2267 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2268 int *ent_cpu, u64 *ent_ts)
2270 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2273 /* Find the next real entry, and increment the iterator to the next entry */
2274 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2276 iter->ent = __find_next_entry(iter, &iter->cpu,
2277 &iter->lost_events, &iter->ts);
2280 trace_iterator_increment(iter);
2282 return iter->ent ? iter : NULL;
2285 static void trace_consume(struct trace_iterator *iter)
2287 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2288 &iter->lost_events);
2291 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2293 struct trace_iterator *iter = m->private;
2297 WARN_ON_ONCE(iter->leftover);
2301 /* can't go backwards */
2306 ent = trace_find_next_entry_inc(iter);
2310 while (ent && iter->idx < i)
2311 ent = trace_find_next_entry_inc(iter);
2318 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2320 struct ring_buffer_event *event;
2321 struct ring_buffer_iter *buf_iter;
2322 unsigned long entries = 0;
2325 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2327 buf_iter = trace_buffer_iter(iter, cpu);
2331 ring_buffer_iter_reset(buf_iter);
2334 * We could have the case with the max latency tracers
2335 * that a reset never took place on a cpu. This is evident
2336 * by the timestamp being before the start of the buffer.
2338 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2339 if (ts >= iter->trace_buffer->time_start)
2342 ring_buffer_read(buf_iter, NULL);
2345 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2349 * The current tracer is copied to avoid a global locking
2352 static void *s_start(struct seq_file *m, loff_t *pos)
2354 struct trace_iterator *iter = m->private;
2355 struct trace_array *tr = iter->tr;
2356 int cpu_file = iter->cpu_file;
2362 * copy the tracer to avoid using a global lock all around.
2363 * iter->trace is a copy of current_trace, the pointer to the
2364 * name may be used instead of a strcmp(), as iter->trace->name
2365 * will point to the same string as current_trace->name.
2367 mutex_lock(&trace_types_lock);
2368 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2369 *iter->trace = *tr->current_trace;
2370 mutex_unlock(&trace_types_lock);
2372 #ifdef CONFIG_TRACER_MAX_TRACE
2373 if (iter->snapshot && iter->trace->use_max_tr)
2374 return ERR_PTR(-EBUSY);
2377 if (!iter->snapshot)
2378 atomic_inc(&trace_record_cmdline_disabled);
2380 if (*pos != iter->pos) {
2385 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2386 for_each_tracing_cpu(cpu)
2387 tracing_iter_reset(iter, cpu);
2389 tracing_iter_reset(iter, cpu_file);
2392 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2397 * If we overflowed the seq_file before, then we want
2398 * to just reuse the trace_seq buffer again.
2404 p = s_next(m, p, &l);
2408 trace_event_read_lock();
2409 trace_access_lock(cpu_file);
2413 static void s_stop(struct seq_file *m, void *p)
2415 struct trace_iterator *iter = m->private;
2417 #ifdef CONFIG_TRACER_MAX_TRACE
2418 if (iter->snapshot && iter->trace->use_max_tr)
2422 if (!iter->snapshot)
2423 atomic_dec(&trace_record_cmdline_disabled);
2425 trace_access_unlock(iter->cpu_file);
2426 trace_event_read_unlock();
2430 get_total_entries(struct trace_buffer *buf,
2431 unsigned long *total, unsigned long *entries)
2433 unsigned long count;
2439 for_each_tracing_cpu(cpu) {
2440 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2442 * If this buffer has skipped entries, then we hold all
2443 * entries for the trace and we need to ignore the
2444 * ones before the time stamp.
2446 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2447 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2448 /* total is the same as the entries */
2452 ring_buffer_overrun_cpu(buf->buffer, cpu);
2457 static void print_lat_help_header(struct seq_file *m)
2459 seq_puts(m, "# _------=> CPU# \n");
2460 seq_puts(m, "# / _-----=> irqs-off \n");
2461 seq_puts(m, "# | / _----=> need-resched \n");
2462 seq_puts(m, "# || / _---=> hardirq/softirq \n");
2463 seq_puts(m, "# ||| / _--=> preempt-depth \n");
2464 seq_puts(m, "# |||| / delay \n");
2465 seq_puts(m, "# cmd pid ||||| time | caller \n");
2466 seq_puts(m, "# \\ / ||||| \\ | / \n");
2469 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2471 unsigned long total;
2472 unsigned long entries;
2474 get_total_entries(buf, &total, &entries);
2475 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2476 entries, total, num_online_cpus());
2480 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2482 print_event_info(buf, m);
2483 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
2484 seq_puts(m, "# | | | | |\n");
2487 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2489 print_event_info(buf, m);
2490 seq_puts(m, "# _-----=> irqs-off\n");
2491 seq_puts(m, "# / _----=> need-resched\n");
2492 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2493 seq_puts(m, "# || / _--=> preempt-depth\n");
2494 seq_puts(m, "# ||| / delay\n");
2495 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2496 seq_puts(m, "# | | | |||| | |\n");
2500 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2502 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2503 struct trace_buffer *buf = iter->trace_buffer;
2504 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2505 struct tracer *type = iter->trace;
2506 unsigned long entries;
2507 unsigned long total;
2508 const char *name = "preemption";
2512 get_total_entries(buf, &total, &entries);
2514 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2516 seq_puts(m, "# -----------------------------------"
2517 "---------------------------------\n");
2518 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2519 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2520 nsecs_to_usecs(data->saved_latency),
2524 #if defined(CONFIG_PREEMPT_NONE)
2526 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2528 #elif defined(CONFIG_PREEMPT)
2533 /* These are reserved for later use */
2536 seq_printf(m, " #P:%d)\n", num_online_cpus());
2540 seq_puts(m, "# -----------------\n");
2541 seq_printf(m, "# | task: %.16s-%d "
2542 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2543 data->comm, data->pid,
2544 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2545 data->policy, data->rt_priority);
2546 seq_puts(m, "# -----------------\n");
2548 if (data->critical_start) {
2549 seq_puts(m, "# => started at: ");
2550 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2551 trace_print_seq(m, &iter->seq);
2552 seq_puts(m, "\n# => ended at: ");
2553 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2554 trace_print_seq(m, &iter->seq);
2555 seq_puts(m, "\n#\n");
2561 static void test_cpu_buff_start(struct trace_iterator *iter)
2563 struct trace_seq *s = &iter->seq;
2565 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2568 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2571 if (cpumask_test_cpu(iter->cpu, iter->started))
2574 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2577 cpumask_set_cpu(iter->cpu, iter->started);
2579 /* Don't print started cpu buffer for the first entry of the trace */
2581 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2585 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2587 struct trace_seq *s = &iter->seq;
2588 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2589 struct trace_entry *entry;
2590 struct trace_event *event;
2594 test_cpu_buff_start(iter);
2596 event = ftrace_find_event(entry->type);
2598 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2599 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2600 if (!trace_print_lat_context(iter))
2603 if (!trace_print_context(iter))
2609 return event->funcs->trace(iter, sym_flags, event);
2611 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2614 return TRACE_TYPE_HANDLED;
2616 return TRACE_TYPE_PARTIAL_LINE;
2619 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2621 struct trace_seq *s = &iter->seq;
2622 struct trace_entry *entry;
2623 struct trace_event *event;
2627 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2628 if (!trace_seq_printf(s, "%d %d %llu ",
2629 entry->pid, iter->cpu, iter->ts))
2633 event = ftrace_find_event(entry->type);
2635 return event->funcs->raw(iter, 0, event);
2637 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2640 return TRACE_TYPE_HANDLED;
2642 return TRACE_TYPE_PARTIAL_LINE;
2645 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2647 struct trace_seq *s = &iter->seq;
2648 unsigned char newline = '\n';
2649 struct trace_entry *entry;
2650 struct trace_event *event;
2654 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2655 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2656 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2657 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2660 event = ftrace_find_event(entry->type);
2662 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2663 if (ret != TRACE_TYPE_HANDLED)
2667 SEQ_PUT_FIELD_RET(s, newline);
2669 return TRACE_TYPE_HANDLED;
2672 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2674 struct trace_seq *s = &iter->seq;
2675 struct trace_entry *entry;
2676 struct trace_event *event;
2680 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2681 SEQ_PUT_FIELD_RET(s, entry->pid);
2682 SEQ_PUT_FIELD_RET(s, iter->cpu);
2683 SEQ_PUT_FIELD_RET(s, iter->ts);
2686 event = ftrace_find_event(entry->type);
2687 return event ? event->funcs->binary(iter, 0, event) :
2691 int trace_empty(struct trace_iterator *iter)
2693 struct ring_buffer_iter *buf_iter;
2696 /* If we are looking at one CPU buffer, only check that one */
2697 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2698 cpu = iter->cpu_file;
2699 buf_iter = trace_buffer_iter(iter, cpu);
2701 if (!ring_buffer_iter_empty(buf_iter))
2704 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2710 for_each_tracing_cpu(cpu) {
2711 buf_iter = trace_buffer_iter(iter, cpu);
2713 if (!ring_buffer_iter_empty(buf_iter))
2716 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2724 /* Called with trace_event_read_lock() held. */
2725 enum print_line_t print_trace_line(struct trace_iterator *iter)
2727 enum print_line_t ret;
2729 if (iter->lost_events &&
2730 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2731 iter->cpu, iter->lost_events))
2732 return TRACE_TYPE_PARTIAL_LINE;
2734 if (iter->trace && iter->trace->print_line) {
2735 ret = iter->trace->print_line(iter);
2736 if (ret != TRACE_TYPE_UNHANDLED)
2740 if (iter->ent->type == TRACE_BPUTS &&
2741 trace_flags & TRACE_ITER_PRINTK &&
2742 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2743 return trace_print_bputs_msg_only(iter);
2745 if (iter->ent->type == TRACE_BPRINT &&
2746 trace_flags & TRACE_ITER_PRINTK &&
2747 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2748 return trace_print_bprintk_msg_only(iter);
2750 if (iter->ent->type == TRACE_PRINT &&
2751 trace_flags & TRACE_ITER_PRINTK &&
2752 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2753 return trace_print_printk_msg_only(iter);
2755 if (trace_flags & TRACE_ITER_BIN)
2756 return print_bin_fmt(iter);
2758 if (trace_flags & TRACE_ITER_HEX)
2759 return print_hex_fmt(iter);
2761 if (trace_flags & TRACE_ITER_RAW)
2762 return print_raw_fmt(iter);
2764 return print_trace_fmt(iter);
2767 void trace_latency_header(struct seq_file *m)
2769 struct trace_iterator *iter = m->private;
2771 /* print nothing if the buffers are empty */
2772 if (trace_empty(iter))
2775 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2776 print_trace_header(m, iter);
2778 if (!(trace_flags & TRACE_ITER_VERBOSE))
2779 print_lat_help_header(m);
2782 void trace_default_header(struct seq_file *m)
2784 struct trace_iterator *iter = m->private;
2786 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2789 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2790 /* print nothing if the buffers are empty */
2791 if (trace_empty(iter))
2793 print_trace_header(m, iter);
2794 if (!(trace_flags & TRACE_ITER_VERBOSE))
2795 print_lat_help_header(m);
2797 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2798 if (trace_flags & TRACE_ITER_IRQ_INFO)
2799 print_func_help_header_irq(iter->trace_buffer, m);
2801 print_func_help_header(iter->trace_buffer, m);
2806 static void test_ftrace_alive(struct seq_file *m)
2808 if (!ftrace_is_dead())
2810 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2811 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2814 #ifdef CONFIG_TRACER_MAX_TRACE
2815 static void show_snapshot_main_help(struct seq_file *m)
2817 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2818 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2819 seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2820 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n");
2821 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2822 seq_printf(m, "# is not a '0' or '1')\n");
2825 static void show_snapshot_percpu_help(struct seq_file *m)
2827 seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2828 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2829 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2830 seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n");
2832 seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2833 seq_printf(m, "# Must use main snapshot file to allocate.\n");
2835 seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2836 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2837 seq_printf(m, "# is not a '0' or '1')\n");
2840 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2842 if (iter->tr->allocated_snapshot)
2843 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2845 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2847 seq_printf(m, "# Snapshot commands:\n");
2848 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2849 show_snapshot_main_help(m);
2851 show_snapshot_percpu_help(m);
2854 /* Should never be called */
2855 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2858 static int s_show(struct seq_file *m, void *v)
2860 struct trace_iterator *iter = v;
2863 if (iter->ent == NULL) {
2865 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2867 test_ftrace_alive(m);
2869 if (iter->snapshot && trace_empty(iter))
2870 print_snapshot_help(m, iter);
2871 else if (iter->trace && iter->trace->print_header)
2872 iter->trace->print_header(m);
2874 trace_default_header(m);
2876 } else if (iter->leftover) {
2878 * If we filled the seq_file buffer earlier, we
2879 * want to just show it now.
2881 ret = trace_print_seq(m, &iter->seq);
2883 /* ret should this time be zero, but you never know */
2884 iter->leftover = ret;
2887 print_trace_line(iter);
2888 ret = trace_print_seq(m, &iter->seq);
2890 * If we overflow the seq_file buffer, then it will
2891 * ask us for this data again at start up.
2893 * ret is 0 if seq_file write succeeded.
2896 iter->leftover = ret;
2903 * Should be used after trace_array_get(), trace_types_lock
2904 * ensures that i_cdev was already initialized.
2906 static inline int tracing_get_cpu(struct inode *inode)
2908 if (inode->i_cdev) /* See trace_create_cpu_file() */
2909 return (long)inode->i_cdev - 1;
2910 return RING_BUFFER_ALL_CPUS;
2913 static const struct seq_operations tracer_seq_ops = {
2920 static struct trace_iterator *
2921 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2923 struct trace_array *tr = inode->i_private;
2924 struct trace_iterator *iter;
2927 if (tracing_disabled)
2928 return ERR_PTR(-ENODEV);
2930 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2932 return ERR_PTR(-ENOMEM);
2934 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2936 if (!iter->buffer_iter)
2940 * We make a copy of the current tracer to avoid concurrent
2941 * changes on it while we are reading.
2943 mutex_lock(&trace_types_lock);
2944 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2948 *iter->trace = *tr->current_trace;
2950 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2955 #ifdef CONFIG_TRACER_MAX_TRACE
2956 /* Currently only the top directory has a snapshot */
2957 if (tr->current_trace->print_max || snapshot)
2958 iter->trace_buffer = &tr->max_buffer;
2961 iter->trace_buffer = &tr->trace_buffer;
2962 iter->snapshot = snapshot;
2964 iter->cpu_file = tracing_get_cpu(inode);
2965 mutex_init(&iter->mutex);
2967 /* Notify the tracer early; before we stop tracing. */
2968 if (iter->trace && iter->trace->open)
2969 iter->trace->open(iter);
2971 /* Annotate start of buffers if we had overruns */
2972 if (ring_buffer_overruns(iter->trace_buffer->buffer))
2973 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2975 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2976 if (trace_clocks[tr->clock_id].in_ns)
2977 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2979 /* stop the trace while dumping if we are not opening "snapshot" */
2980 if (!iter->snapshot)
2981 tracing_stop_tr(tr);
2983 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
2984 for_each_tracing_cpu(cpu) {
2985 iter->buffer_iter[cpu] =
2986 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2988 ring_buffer_read_prepare_sync();
2989 for_each_tracing_cpu(cpu) {
2990 ring_buffer_read_start(iter->buffer_iter[cpu]);
2991 tracing_iter_reset(iter, cpu);
2994 cpu = iter->cpu_file;
2995 iter->buffer_iter[cpu] =
2996 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2997 ring_buffer_read_prepare_sync();
2998 ring_buffer_read_start(iter->buffer_iter[cpu]);
2999 tracing_iter_reset(iter, cpu);
3002 mutex_unlock(&trace_types_lock);
3007 mutex_unlock(&trace_types_lock);
3009 kfree(iter->buffer_iter);
3011 seq_release_private(inode, file);
3012 return ERR_PTR(-ENOMEM);
3015 int tracing_open_generic(struct inode *inode, struct file *filp)
3017 if (tracing_disabled)
3020 filp->private_data = inode->i_private;
3024 bool tracing_is_disabled(void)
3026 return (tracing_disabled) ? true: false;
3030 * Open and update trace_array ref count.
3031 * Must have the current trace_array passed to it.
3033 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3035 struct trace_array *tr = inode->i_private;
3037 if (tracing_disabled)
3040 if (trace_array_get(tr) < 0)
3043 filp->private_data = inode->i_private;
3048 static int tracing_release(struct inode *inode, struct file *file)
3050 struct trace_array *tr = inode->i_private;
3051 struct seq_file *m = file->private_data;
3052 struct trace_iterator *iter;
3055 if (!(file->f_mode & FMODE_READ)) {
3056 trace_array_put(tr);
3060 /* Writes do not use seq_file */
3062 mutex_lock(&trace_types_lock);
3064 for_each_tracing_cpu(cpu) {
3065 if (iter->buffer_iter[cpu])
3066 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3069 if (iter->trace && iter->trace->close)
3070 iter->trace->close(iter);
3072 if (!iter->snapshot)
3073 /* reenable tracing if it was previously enabled */
3074 tracing_start_tr(tr);
3076 __trace_array_put(tr);
3078 mutex_unlock(&trace_types_lock);
3080 mutex_destroy(&iter->mutex);
3081 free_cpumask_var(iter->started);
3083 kfree(iter->buffer_iter);
3084 seq_release_private(inode, file);
3089 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3091 struct trace_array *tr = inode->i_private;
3093 trace_array_put(tr);
3097 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3099 struct trace_array *tr = inode->i_private;
3101 trace_array_put(tr);
3103 return single_release(inode, file);
3106 static int tracing_open(struct inode *inode, struct file *file)
3108 struct trace_array *tr = inode->i_private;
3109 struct trace_iterator *iter;
3112 if (trace_array_get(tr) < 0)
3115 /* If this file was open for write, then erase contents */
3116 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3117 int cpu = tracing_get_cpu(inode);
3119 if (cpu == RING_BUFFER_ALL_CPUS)
3120 tracing_reset_online_cpus(&tr->trace_buffer);
3122 tracing_reset(&tr->trace_buffer, cpu);
3125 if (file->f_mode & FMODE_READ) {
3126 iter = __tracing_open(inode, file, false);
3128 ret = PTR_ERR(iter);
3129 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3130 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3134 trace_array_put(tr);
3140 t_next(struct seq_file *m, void *v, loff_t *pos)
3142 struct tracer *t = v;
3152 static void *t_start(struct seq_file *m, loff_t *pos)
3157 mutex_lock(&trace_types_lock);
3158 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
3164 static void t_stop(struct seq_file *m, void *p)
3166 mutex_unlock(&trace_types_lock);
3169 static int t_show(struct seq_file *m, void *v)
3171 struct tracer *t = v;
3176 seq_printf(m, "%s", t->name);
3185 static const struct seq_operations show_traces_seq_ops = {
3192 static int show_traces_open(struct inode *inode, struct file *file)
3194 if (tracing_disabled)
3197 return seq_open(file, &show_traces_seq_ops);
3201 tracing_write_stub(struct file *filp, const char __user *ubuf,
3202 size_t count, loff_t *ppos)
3207 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3211 if (file->f_mode & FMODE_READ)
3212 ret = seq_lseek(file, offset, whence);
3214 file->f_pos = ret = 0;
3219 static const struct file_operations tracing_fops = {
3220 .open = tracing_open,
3222 .write = tracing_write_stub,
3223 .llseek = tracing_lseek,
3224 .release = tracing_release,
3227 static const struct file_operations show_traces_fops = {
3228 .open = show_traces_open,
3230 .release = seq_release,
3231 .llseek = seq_lseek,
3235 * The tracer itself will not take this lock, but still we want
3236 * to provide a consistent cpumask to user-space:
3238 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3241 * Temporary storage for the character representation of the
3242 * CPU bitmask (and one more byte for the newline):
3244 static char mask_str[NR_CPUS + 1];
3247 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3248 size_t count, loff_t *ppos)
3250 struct trace_array *tr = file_inode(filp)->i_private;
3253 mutex_lock(&tracing_cpumask_update_lock);
3255 len = cpumask_scnprintf(mask_str, count, tr->tracing_cpumask);
3256 if (count - len < 2) {
3260 len += sprintf(mask_str + len, "\n");
3261 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3264 mutex_unlock(&tracing_cpumask_update_lock);
3270 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3271 size_t count, loff_t *ppos)
3273 struct trace_array *tr = file_inode(filp)->i_private;
3274 cpumask_var_t tracing_cpumask_new;
3277 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3280 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3284 mutex_lock(&tracing_cpumask_update_lock);
3286 local_irq_disable();
3287 arch_spin_lock(&ftrace_max_lock);
3288 for_each_tracing_cpu(cpu) {
3290 * Increase/decrease the disabled counter if we are
3291 * about to flip a bit in the cpumask:
3293 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3294 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3295 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3296 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3298 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3299 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3300 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3301 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3304 arch_spin_unlock(&ftrace_max_lock);
3307 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3309 mutex_unlock(&tracing_cpumask_update_lock);
3310 free_cpumask_var(tracing_cpumask_new);
3315 free_cpumask_var(tracing_cpumask_new);
3320 static const struct file_operations tracing_cpumask_fops = {
3321 .open = tracing_open_generic_tr,
3322 .read = tracing_cpumask_read,
3323 .write = tracing_cpumask_write,
3324 .release = tracing_release_generic_tr,
3325 .llseek = generic_file_llseek,
3328 static int tracing_trace_options_show(struct seq_file *m, void *v)
3330 struct tracer_opt *trace_opts;
3331 struct trace_array *tr = m->private;
3335 mutex_lock(&trace_types_lock);
3336 tracer_flags = tr->current_trace->flags->val;
3337 trace_opts = tr->current_trace->flags->opts;
3339 for (i = 0; trace_options[i]; i++) {
3340 if (trace_flags & (1 << i))
3341 seq_printf(m, "%s\n", trace_options[i]);
3343 seq_printf(m, "no%s\n", trace_options[i]);
3346 for (i = 0; trace_opts[i].name; i++) {
3347 if (tracer_flags & trace_opts[i].bit)
3348 seq_printf(m, "%s\n", trace_opts[i].name);
3350 seq_printf(m, "no%s\n", trace_opts[i].name);
3352 mutex_unlock(&trace_types_lock);
3357 static int __set_tracer_option(struct tracer *trace,
3358 struct tracer_flags *tracer_flags,
3359 struct tracer_opt *opts, int neg)
3363 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
3368 tracer_flags->val &= ~opts->bit;
3370 tracer_flags->val |= opts->bit;
3374 /* Try to assign a tracer specific option */
3375 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
3377 struct tracer_flags *tracer_flags = trace->flags;
3378 struct tracer_opt *opts = NULL;
3381 for (i = 0; tracer_flags->opts[i].name; i++) {
3382 opts = &tracer_flags->opts[i];
3384 if (strcmp(cmp, opts->name) == 0)
3385 return __set_tracer_option(trace, trace->flags,
3392 /* Some tracers require overwrite to stay enabled */
3393 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3395 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3401 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3403 /* do nothing if flag is already set */
3404 if (!!(trace_flags & mask) == !!enabled)
3407 /* Give the tracer a chance to approve the change */
3408 if (tr->current_trace->flag_changed)
3409 if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
3413 trace_flags |= mask;
3415 trace_flags &= ~mask;
3417 if (mask == TRACE_ITER_RECORD_CMD)
3418 trace_event_enable_cmd_record(enabled);
3420 if (mask == TRACE_ITER_OVERWRITE) {
3421 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3422 #ifdef CONFIG_TRACER_MAX_TRACE
3423 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3427 if (mask == TRACE_ITER_PRINTK)
3428 trace_printk_start_stop_comm(enabled);
3433 static int trace_set_options(struct trace_array *tr, char *option)
3440 cmp = strstrip(option);
3442 if (strncmp(cmp, "no", 2) == 0) {
3447 mutex_lock(&trace_types_lock);
3449 for (i = 0; trace_options[i]; i++) {
3450 if (strcmp(cmp, trace_options[i]) == 0) {
3451 ret = set_tracer_flag(tr, 1 << i, !neg);
3456 /* If no option could be set, test the specific tracer options */
3457 if (!trace_options[i])
3458 ret = set_tracer_option(tr->current_trace, cmp, neg);
3460 mutex_unlock(&trace_types_lock);
3466 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3467 size_t cnt, loff_t *ppos)
3469 struct seq_file *m = filp->private_data;
3470 struct trace_array *tr = m->private;
3474 if (cnt >= sizeof(buf))
3477 if (copy_from_user(&buf, ubuf, cnt))
3482 ret = trace_set_options(tr, buf);
3491 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3493 struct trace_array *tr = inode->i_private;
3496 if (tracing_disabled)
3499 if (trace_array_get(tr) < 0)
3502 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3504 trace_array_put(tr);
3509 static const struct file_operations tracing_iter_fops = {
3510 .open = tracing_trace_options_open,
3512 .llseek = seq_lseek,
3513 .release = tracing_single_release_tr,
3514 .write = tracing_trace_options_write,
3517 static const char readme_msg[] =
3518 "tracing mini-HOWTO:\n\n"
3519 "# echo 0 > tracing_on : quick way to disable tracing\n"
3520 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3521 " Important files:\n"
3522 " trace\t\t\t- The static contents of the buffer\n"
3523 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3524 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3525 " current_tracer\t- function and latency tracers\n"
3526 " available_tracers\t- list of configured tracers for current_tracer\n"
3527 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3528 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3529 " trace_clock\t\t-change the clock used to order events\n"
3530 " local: Per cpu clock but may not be synced across CPUs\n"
3531 " global: Synced across CPUs but slows tracing down.\n"
3532 " counter: Not a clock, but just an increment\n"
3533 " uptime: Jiffy counter from time of boot\n"
3534 " perf: Same clock that perf events use\n"
3535 #ifdef CONFIG_X86_64
3536 " x86-tsc: TSC cycle counter\n"
3538 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3539 " tracing_cpumask\t- Limit which CPUs to trace\n"
3540 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3541 "\t\t\t Remove sub-buffer with rmdir\n"
3542 " trace_options\t\t- Set format or modify how tracing happens\n"
3543 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3544 "\t\t\t option name\n"
3545 #ifdef CONFIG_DYNAMIC_FTRACE
3546 "\n available_filter_functions - list of functions that can be filtered on\n"
3547 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3548 "\t\t\t functions\n"
3549 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3550 "\t modules: Can select a group via module\n"
3551 "\t Format: :mod:<module-name>\n"
3552 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3553 "\t triggers: a command to perform when function is hit\n"
3554 "\t Format: <function>:<trigger>[:count]\n"
3555 "\t trigger: traceon, traceoff\n"
3556 "\t\t enable_event:<system>:<event>\n"
3557 "\t\t disable_event:<system>:<event>\n"
3558 #ifdef CONFIG_STACKTRACE
3561 #ifdef CONFIG_TRACER_SNAPSHOT
3564 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3565 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3566 "\t The first one will disable tracing every time do_fault is hit\n"
3567 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3568 "\t The first time do trap is hit and it disables tracing, the\n"
3569 "\t counter will decrement to 2. If tracing is already disabled,\n"
3570 "\t the counter will not decrement. It only decrements when the\n"
3571 "\t trigger did work\n"
3572 "\t To remove trigger without count:\n"
3573 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3574 "\t To remove trigger with a count:\n"
3575 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3576 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3577 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3578 "\t modules: Can select a group via module command :mod:\n"
3579 "\t Does not accept triggers\n"
3580 #endif /* CONFIG_DYNAMIC_FTRACE */
3581 #ifdef CONFIG_FUNCTION_TRACER
3582 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3585 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3586 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3587 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3589 #ifdef CONFIG_TRACER_SNAPSHOT
3590 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3591 "\t\t\t snapshot buffer. Read the contents for more\n"
3592 "\t\t\t information\n"
3594 #ifdef CONFIG_STACK_TRACER
3595 " stack_trace\t\t- Shows the max stack trace when active\n"
3596 " stack_max_size\t- Shows current max stack size that was traced\n"
3597 "\t\t\t Write into this file to reset the max size (trigger a\n"
3598 "\t\t\t new trace)\n"
3599 #ifdef CONFIG_DYNAMIC_FTRACE
3600 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3603 #endif /* CONFIG_STACK_TRACER */
3604 " events/\t\t- Directory containing all trace event subsystems:\n"
3605 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3606 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3607 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3609 " filter\t\t- If set, only events passing filter are traced\n"
3610 " events/<system>/<event>/\t- Directory containing control files for\n"
3612 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3613 " filter\t\t- If set, only events passing filter are traced\n"
3614 " trigger\t\t- If set, a command to perform when event is hit\n"
3615 "\t Format: <trigger>[:count][if <filter>]\n"
3616 "\t trigger: traceon, traceoff\n"
3617 "\t enable_event:<system>:<event>\n"
3618 "\t disable_event:<system>:<event>\n"
3619 #ifdef CONFIG_STACKTRACE
3622 #ifdef CONFIG_TRACER_SNAPSHOT
3625 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3626 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3627 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3628 "\t events/block/block_unplug/trigger\n"
3629 "\t The first disables tracing every time block_unplug is hit.\n"
3630 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3631 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3632 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3633 "\t Like function triggers, the counter is only decremented if it\n"
3634 "\t enabled or disabled tracing.\n"
3635 "\t To remove a trigger without a count:\n"
3636 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3637 "\t To remove a trigger with a count:\n"
3638 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3639 "\t Filters can be ignored when removing a trigger.\n"
3643 tracing_readme_read(struct file *filp, char __user *ubuf,
3644 size_t cnt, loff_t *ppos)
3646 return simple_read_from_buffer(ubuf, cnt, ppos,
3647 readme_msg, strlen(readme_msg));
3650 static const struct file_operations tracing_readme_fops = {
3651 .open = tracing_open_generic,
3652 .read = tracing_readme_read,
3653 .llseek = generic_file_llseek,
3657 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3658 size_t cnt, loff_t *ppos)
3667 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3671 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3679 for (i = 0; i < SAVED_CMDLINES; i++) {
3682 pid = map_cmdline_to_pid[i];
3683 if (pid == -1 || pid == NO_CMDLINE_MAP)
3686 trace_find_cmdline(pid, buf_comm);
3687 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3692 len = simple_read_from_buffer(ubuf, cnt, ppos,
3701 static const struct file_operations tracing_saved_cmdlines_fops = {
3702 .open = tracing_open_generic,
3703 .read = tracing_saved_cmdlines_read,
3704 .llseek = generic_file_llseek,
3708 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3709 size_t cnt, loff_t *ppos)
3711 struct trace_array *tr = filp->private_data;
3712 char buf[MAX_TRACER_SIZE+2];
3715 mutex_lock(&trace_types_lock);
3716 r = sprintf(buf, "%s\n", tr->current_trace->name);
3717 mutex_unlock(&trace_types_lock);
3719 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3722 int tracer_init(struct tracer *t, struct trace_array *tr)
3724 tracing_reset_online_cpus(&tr->trace_buffer);
3728 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3732 for_each_tracing_cpu(cpu)
3733 per_cpu_ptr(buf->data, cpu)->entries = val;
3736 #ifdef CONFIG_TRACER_MAX_TRACE
3737 /* resize @tr's buffer to the size of @size_tr's entries */
3738 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3739 struct trace_buffer *size_buf, int cpu_id)
3743 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3744 for_each_tracing_cpu(cpu) {
3745 ret = ring_buffer_resize(trace_buf->buffer,
3746 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3749 per_cpu_ptr(trace_buf->data, cpu)->entries =
3750 per_cpu_ptr(size_buf->data, cpu)->entries;
3753 ret = ring_buffer_resize(trace_buf->buffer,
3754 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3756 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3757 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3762 #endif /* CONFIG_TRACER_MAX_TRACE */
3764 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3765 unsigned long size, int cpu)
3770 * If kernel or user changes the size of the ring buffer
3771 * we use the size that was given, and we can forget about
3772 * expanding it later.
3774 ring_buffer_expanded = true;
3776 /* May be called before buffers are initialized */
3777 if (!tr->trace_buffer.buffer)
3780 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3784 #ifdef CONFIG_TRACER_MAX_TRACE
3785 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3786 !tr->current_trace->use_max_tr)
3789 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3791 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3792 &tr->trace_buffer, cpu);
3795 * AARGH! We are left with different
3796 * size max buffer!!!!
3797 * The max buffer is our "snapshot" buffer.
3798 * When a tracer needs a snapshot (one of the
3799 * latency tracers), it swaps the max buffer
3800 * with the saved snap shot. We succeeded to
3801 * update the size of the main buffer, but failed to
3802 * update the size of the max buffer. But when we tried
3803 * to reset the main buffer to the original size, we
3804 * failed there too. This is very unlikely to
3805 * happen, but if it does, warn and kill all
3809 tracing_disabled = 1;
3814 if (cpu == RING_BUFFER_ALL_CPUS)
3815 set_buffer_entries(&tr->max_buffer, size);
3817 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
3820 #endif /* CONFIG_TRACER_MAX_TRACE */
3822 if (cpu == RING_BUFFER_ALL_CPUS)
3823 set_buffer_entries(&tr->trace_buffer, size);
3825 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
3830 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3831 unsigned long size, int cpu_id)
3835 mutex_lock(&trace_types_lock);
3837 if (cpu_id != RING_BUFFER_ALL_CPUS) {
3838 /* make sure, this cpu is enabled in the mask */
3839 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3845 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
3850 mutex_unlock(&trace_types_lock);
3857 * tracing_update_buffers - used by tracing facility to expand ring buffers
3859 * To save on memory when the tracing is never used on a system with it
3860 * configured in. The ring buffers are set to a minimum size. But once
3861 * a user starts to use the tracing facility, then they need to grow
3862 * to their default size.
3864 * This function is to be called when a tracer is about to be used.
3866 int tracing_update_buffers(void)
3870 mutex_lock(&trace_types_lock);
3871 if (!ring_buffer_expanded)
3872 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
3873 RING_BUFFER_ALL_CPUS);
3874 mutex_unlock(&trace_types_lock);
3879 struct trace_option_dentry;
3881 static struct trace_option_dentry *
3882 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
3885 destroy_trace_option_files(struct trace_option_dentry *topts);
3887 static int tracing_set_tracer(const char *buf)
3889 static struct trace_option_dentry *topts;
3890 struct trace_array *tr = &global_trace;
3892 #ifdef CONFIG_TRACER_MAX_TRACE
3897 mutex_lock(&trace_types_lock);
3899 if (!ring_buffer_expanded) {
3900 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
3901 RING_BUFFER_ALL_CPUS);
3907 for (t = trace_types; t; t = t->next) {
3908 if (strcmp(t->name, buf) == 0)
3915 if (t == tr->current_trace)
3918 trace_branch_disable();
3920 tr->current_trace->enabled = false;
3922 if (tr->current_trace->reset)
3923 tr->current_trace->reset(tr);
3925 /* Current trace needs to be nop_trace before synchronize_sched */
3926 tr->current_trace = &nop_trace;
3928 #ifdef CONFIG_TRACER_MAX_TRACE
3929 had_max_tr = tr->allocated_snapshot;
3931 if (had_max_tr && !t->use_max_tr) {
3933 * We need to make sure that the update_max_tr sees that
3934 * current_trace changed to nop_trace to keep it from
3935 * swapping the buffers after we resize it.
3936 * The update_max_tr is called from interrupts disabled
3937 * so a synchronized_sched() is sufficient.
3939 synchronize_sched();
3943 destroy_trace_option_files(topts);
3945 topts = create_trace_option_files(tr, t);
3947 #ifdef CONFIG_TRACER_MAX_TRACE
3948 if (t->use_max_tr && !had_max_tr) {
3949 ret = alloc_snapshot(tr);
3956 ret = tracer_init(t, tr);
3961 tr->current_trace = t;
3962 tr->current_trace->enabled = true;
3963 trace_branch_enable(tr);
3965 mutex_unlock(&trace_types_lock);
3971 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3972 size_t cnt, loff_t *ppos)
3974 char buf[MAX_TRACER_SIZE+1];
3981 if (cnt > MAX_TRACER_SIZE)
3982 cnt = MAX_TRACER_SIZE;
3984 if (copy_from_user(&buf, ubuf, cnt))
3989 /* strip ending whitespace. */
3990 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3993 err = tracing_set_tracer(buf);
4003 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4004 size_t cnt, loff_t *ppos)
4006 unsigned long *ptr = filp->private_data;
4010 r = snprintf(buf, sizeof(buf), "%ld\n",
4011 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4012 if (r > sizeof(buf))
4014 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4018 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4019 size_t cnt, loff_t *ppos)
4021 unsigned long *ptr = filp->private_data;
4025 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4034 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4036 struct trace_array *tr = inode->i_private;
4037 struct trace_iterator *iter;
4040 if (tracing_disabled)
4043 if (trace_array_get(tr) < 0)
4046 mutex_lock(&trace_types_lock);
4048 /* create a buffer to store the information to pass to userspace */
4049 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4052 __trace_array_put(tr);
4057 * We make a copy of the current tracer to avoid concurrent
4058 * changes on it while we are reading.
4060 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
4065 *iter->trace = *tr->current_trace;
4067 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4072 /* trace pipe does not show start of buffer */
4073 cpumask_setall(iter->started);
4075 if (trace_flags & TRACE_ITER_LATENCY_FMT)
4076 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4078 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4079 if (trace_clocks[tr->clock_id].in_ns)
4080 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4083 iter->trace_buffer = &tr->trace_buffer;
4084 iter->cpu_file = tracing_get_cpu(inode);
4085 mutex_init(&iter->mutex);
4086 filp->private_data = iter;
4088 if (iter->trace->pipe_open)
4089 iter->trace->pipe_open(iter);
4091 nonseekable_open(inode, filp);
4093 mutex_unlock(&trace_types_lock);
4099 __trace_array_put(tr);
4100 mutex_unlock(&trace_types_lock);
4104 static int tracing_release_pipe(struct inode *inode, struct file *file)
4106 struct trace_iterator *iter = file->private_data;
4107 struct trace_array *tr = inode->i_private;
4109 mutex_lock(&trace_types_lock);
4111 if (iter->trace->pipe_close)
4112 iter->trace->pipe_close(iter);
4114 mutex_unlock(&trace_types_lock);
4116 free_cpumask_var(iter->started);
4117 mutex_destroy(&iter->mutex);
4121 trace_array_put(tr);
4127 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4129 /* Iterators are static, they should be filled or empty */
4130 if (trace_buffer_iter(iter, iter->cpu_file))
4131 return POLLIN | POLLRDNORM;
4133 if (trace_flags & TRACE_ITER_BLOCK)
4135 * Always select as readable when in blocking mode
4137 return POLLIN | POLLRDNORM;
4139 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4144 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4146 struct trace_iterator *iter = filp->private_data;
4148 return trace_poll(iter, filp, poll_table);
4152 * This is a make-shift waitqueue.
4153 * A tracer might use this callback on some rare cases:
4155 * 1) the current tracer might hold the runqueue lock when it wakes up
4156 * a reader, hence a deadlock (sched, function, and function graph tracers)
4157 * 2) the function tracers, trace all functions, we don't want
4158 * the overhead of calling wake_up and friends
4159 * (and tracing them too)
4161 * Anyway, this is really very primitive wakeup.
4163 int poll_wait_pipe(struct trace_iterator *iter)
4165 set_current_state(TASK_INTERRUPTIBLE);
4166 /* sleep for 100 msecs, and try again. */
4167 schedule_timeout(HZ / 10);
4171 /* Must be called with trace_types_lock mutex held. */
4172 static int tracing_wait_pipe(struct file *filp)
4174 struct trace_iterator *iter = filp->private_data;
4177 while (trace_empty(iter)) {
4179 if ((filp->f_flags & O_NONBLOCK)) {
4183 mutex_unlock(&iter->mutex);
4185 ret = iter->trace->wait_pipe(iter);
4187 mutex_lock(&iter->mutex);
4192 if (signal_pending(current))
4196 * We block until we read something and tracing is disabled.
4197 * We still block if tracing is disabled, but we have never
4198 * read anything. This allows a user to cat this file, and
4199 * then enable tracing. But after we have read something,
4200 * we give an EOF when tracing is again disabled.
4202 * iter->pos will be 0 if we haven't read anything.
4204 if (!tracing_is_on() && iter->pos)
4215 tracing_read_pipe(struct file *filp, char __user *ubuf,
4216 size_t cnt, loff_t *ppos)
4218 struct trace_iterator *iter = filp->private_data;
4219 struct trace_array *tr = iter->tr;
4222 /* return any leftover data */
4223 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4227 trace_seq_init(&iter->seq);
4229 /* copy the tracer to avoid using a global lock all around */
4230 mutex_lock(&trace_types_lock);
4231 if (unlikely(iter->trace->name != tr->current_trace->name))
4232 *iter->trace = *tr->current_trace;
4233 mutex_unlock(&trace_types_lock);
4236 * Avoid more than one consumer on a single file descriptor
4237 * This is just a matter of traces coherency, the ring buffer itself
4240 mutex_lock(&iter->mutex);
4241 if (iter->trace->read) {
4242 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4248 sret = tracing_wait_pipe(filp);
4252 /* stop when tracing is finished */
4253 if (trace_empty(iter)) {
4258 if (cnt >= PAGE_SIZE)
4259 cnt = PAGE_SIZE - 1;
4261 /* reset all but tr, trace, and overruns */
4262 memset(&iter->seq, 0,
4263 sizeof(struct trace_iterator) -
4264 offsetof(struct trace_iterator, seq));
4265 cpumask_clear(iter->started);
4268 trace_event_read_lock();
4269 trace_access_lock(iter->cpu_file);
4270 while (trace_find_next_entry_inc(iter) != NULL) {
4271 enum print_line_t ret;
4272 int len = iter->seq.len;
4274 ret = print_trace_line(iter);
4275 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4276 /* don't print partial lines */
4277 iter->seq.len = len;
4280 if (ret != TRACE_TYPE_NO_CONSUME)
4281 trace_consume(iter);
4283 if (iter->seq.len >= cnt)
4287 * Setting the full flag means we reached the trace_seq buffer
4288 * size and we should leave by partial output condition above.
4289 * One of the trace_seq_* functions is not used properly.
4291 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4294 trace_access_unlock(iter->cpu_file);
4295 trace_event_read_unlock();
4297 /* Now copy what we have to the user */
4298 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4299 if (iter->seq.readpos >= iter->seq.len)
4300 trace_seq_init(&iter->seq);
4303 * If there was nothing to send to user, in spite of consuming trace
4304 * entries, go back to wait for more entries.
4310 mutex_unlock(&iter->mutex);
4315 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4318 __free_page(spd->pages[idx]);
4321 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4323 .map = generic_pipe_buf_map,
4324 .unmap = generic_pipe_buf_unmap,
4325 .confirm = generic_pipe_buf_confirm,
4326 .release = generic_pipe_buf_release,
4327 .steal = generic_pipe_buf_steal,
4328 .get = generic_pipe_buf_get,
4332 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4337 /* Seq buffer is page-sized, exactly what we need. */
4339 count = iter->seq.len;
4340 ret = print_trace_line(iter);
4341 count = iter->seq.len - count;
4344 iter->seq.len -= count;
4347 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4348 iter->seq.len -= count;
4352 if (ret != TRACE_TYPE_NO_CONSUME)
4353 trace_consume(iter);
4355 if (!trace_find_next_entry_inc(iter)) {
4365 static ssize_t tracing_splice_read_pipe(struct file *filp,
4367 struct pipe_inode_info *pipe,
4371 struct page *pages_def[PIPE_DEF_BUFFERS];
4372 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4373 struct trace_iterator *iter = filp->private_data;
4374 struct splice_pipe_desc spd = {
4376 .partial = partial_def,
4377 .nr_pages = 0, /* This gets updated below. */
4378 .nr_pages_max = PIPE_DEF_BUFFERS,
4380 .ops = &tracing_pipe_buf_ops,
4381 .spd_release = tracing_spd_release_pipe,
4383 struct trace_array *tr = iter->tr;
4388 if (splice_grow_spd(pipe, &spd))
4391 /* copy the tracer to avoid using a global lock all around */
4392 mutex_lock(&trace_types_lock);
4393 if (unlikely(iter->trace->name != tr->current_trace->name))
4394 *iter->trace = *tr->current_trace;
4395 mutex_unlock(&trace_types_lock);
4397 mutex_lock(&iter->mutex);
4399 if (iter->trace->splice_read) {
4400 ret = iter->trace->splice_read(iter, filp,
4401 ppos, pipe, len, flags);
4406 ret = tracing_wait_pipe(filp);
4410 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4415 trace_event_read_lock();
4416 trace_access_lock(iter->cpu_file);
4418 /* Fill as many pages as possible. */
4419 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
4420 spd.pages[i] = alloc_page(GFP_KERNEL);
4424 rem = tracing_fill_pipe_page(rem, iter);
4426 /* Copy the data into the page, so we can start over. */
4427 ret = trace_seq_to_buffer(&iter->seq,
4428 page_address(spd.pages[i]),
4431 __free_page(spd.pages[i]);
4434 spd.partial[i].offset = 0;
4435 spd.partial[i].len = iter->seq.len;
4437 trace_seq_init(&iter->seq);
4440 trace_access_unlock(iter->cpu_file);
4441 trace_event_read_unlock();
4442 mutex_unlock(&iter->mutex);
4446 ret = splice_to_pipe(pipe, &spd);
4448 splice_shrink_spd(&spd);
4452 mutex_unlock(&iter->mutex);
4457 tracing_entries_read(struct file *filp, char __user *ubuf,
4458 size_t cnt, loff_t *ppos)
4460 struct inode *inode = file_inode(filp);
4461 struct trace_array *tr = inode->i_private;
4462 int cpu = tracing_get_cpu(inode);
4467 mutex_lock(&trace_types_lock);
4469 if (cpu == RING_BUFFER_ALL_CPUS) {
4470 int cpu, buf_size_same;
4475 /* check if all cpu sizes are same */
4476 for_each_tracing_cpu(cpu) {
4477 /* fill in the size from first enabled cpu */
4479 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4480 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4486 if (buf_size_same) {
4487 if (!ring_buffer_expanded)
4488 r = sprintf(buf, "%lu (expanded: %lu)\n",
4490 trace_buf_size >> 10);
4492 r = sprintf(buf, "%lu\n", size >> 10);
4494 r = sprintf(buf, "X\n");
4496 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4498 mutex_unlock(&trace_types_lock);
4500 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4505 tracing_entries_write(struct file *filp, const char __user *ubuf,
4506 size_t cnt, loff_t *ppos)
4508 struct inode *inode = file_inode(filp);
4509 struct trace_array *tr = inode->i_private;
4513 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4517 /* must have at least 1 entry */
4521 /* value is in KB */
4523 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4533 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4534 size_t cnt, loff_t *ppos)
4536 struct trace_array *tr = filp->private_data;
4539 unsigned long size = 0, expanded_size = 0;
4541 mutex_lock(&trace_types_lock);
4542 for_each_tracing_cpu(cpu) {
4543 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4544 if (!ring_buffer_expanded)
4545 expanded_size += trace_buf_size >> 10;
4547 if (ring_buffer_expanded)
4548 r = sprintf(buf, "%lu\n", size);
4550 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4551 mutex_unlock(&trace_types_lock);
4553 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4557 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4558 size_t cnt, loff_t *ppos)
4561 * There is no need to read what the user has written, this function
4562 * is just to make sure that there is no error when "echo" is used
4571 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4573 struct trace_array *tr = inode->i_private;
4575 /* disable tracing ? */
4576 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4577 tracer_tracing_off(tr);
4578 /* resize the ring buffer to 0 */
4579 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4581 trace_array_put(tr);
4587 tracing_mark_write(struct file *filp, const char __user *ubuf,
4588 size_t cnt, loff_t *fpos)
4590 unsigned long addr = (unsigned long)ubuf;
4591 struct trace_array *tr = filp->private_data;
4592 struct ring_buffer_event *event;
4593 struct ring_buffer *buffer;
4594 struct print_entry *entry;
4595 unsigned long irq_flags;
4596 struct page *pages[2];
4606 if (tracing_disabled)
4609 if (!(trace_flags & TRACE_ITER_MARKERS))
4612 if (cnt > TRACE_BUF_SIZE)
4613 cnt = TRACE_BUF_SIZE;
4616 * Userspace is injecting traces into the kernel trace buffer.
4617 * We want to be as non intrusive as possible.
4618 * To do so, we do not want to allocate any special buffers
4619 * or take any locks, but instead write the userspace data
4620 * straight into the ring buffer.
4622 * First we need to pin the userspace buffer into memory,
4623 * which, most likely it is, because it just referenced it.
4624 * But there's no guarantee that it is. By using get_user_pages_fast()
4625 * and kmap_atomic/kunmap_atomic() we can get access to the
4626 * pages directly. We then write the data directly into the
4629 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4631 /* check if we cross pages */
4632 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4635 offset = addr & (PAGE_SIZE - 1);
4638 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4639 if (ret < nr_pages) {
4641 put_page(pages[ret]);
4646 for (i = 0; i < nr_pages; i++)
4647 map_page[i] = kmap_atomic(pages[i]);
4649 local_save_flags(irq_flags);
4650 size = sizeof(*entry) + cnt + 2; /* possible \n added */
4651 buffer = tr->trace_buffer.buffer;
4652 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4653 irq_flags, preempt_count());
4655 /* Ring buffer disabled, return as if not open for write */
4660 entry = ring_buffer_event_data(event);
4661 entry->ip = _THIS_IP_;
4663 if (nr_pages == 2) {
4664 len = PAGE_SIZE - offset;
4665 memcpy(&entry->buf, map_page[0] + offset, len);
4666 memcpy(&entry->buf[len], map_page[1], cnt - len);
4668 memcpy(&entry->buf, map_page[0] + offset, cnt);
4670 if (entry->buf[cnt - 1] != '\n') {
4671 entry->buf[cnt] = '\n';
4672 entry->buf[cnt + 1] = '\0';
4674 entry->buf[cnt] = '\0';
4676 __buffer_unlock_commit(buffer, event);
4683 for (i = 0; i < nr_pages; i++){
4684 kunmap_atomic(map_page[i]);
4691 static int tracing_clock_show(struct seq_file *m, void *v)
4693 struct trace_array *tr = m->private;
4696 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4698 "%s%s%s%s", i ? " " : "",
4699 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4700 i == tr->clock_id ? "]" : "");
4706 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4707 size_t cnt, loff_t *fpos)
4709 struct seq_file *m = filp->private_data;
4710 struct trace_array *tr = m->private;
4712 const char *clockstr;
4715 if (cnt >= sizeof(buf))
4718 if (copy_from_user(&buf, ubuf, cnt))
4723 clockstr = strstrip(buf);
4725 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4726 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4729 if (i == ARRAY_SIZE(trace_clocks))
4732 mutex_lock(&trace_types_lock);
4736 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4739 * New clock may not be consistent with the previous clock.
4740 * Reset the buffer so that it doesn't have incomparable timestamps.
4742 tracing_reset_online_cpus(&tr->trace_buffer);
4744 #ifdef CONFIG_TRACER_MAX_TRACE
4745 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4746 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4747 tracing_reset_online_cpus(&tr->max_buffer);
4750 mutex_unlock(&trace_types_lock);
4757 static int tracing_clock_open(struct inode *inode, struct file *file)
4759 struct trace_array *tr = inode->i_private;
4762 if (tracing_disabled)
4765 if (trace_array_get(tr))
4768 ret = single_open(file, tracing_clock_show, inode->i_private);
4770 trace_array_put(tr);
4775 struct ftrace_buffer_info {
4776 struct trace_iterator iter;
4781 #ifdef CONFIG_TRACER_SNAPSHOT
4782 static int tracing_snapshot_open(struct inode *inode, struct file *file)
4784 struct trace_array *tr = inode->i_private;
4785 struct trace_iterator *iter;
4789 if (trace_array_get(tr) < 0)
4792 if (file->f_mode & FMODE_READ) {
4793 iter = __tracing_open(inode, file, true);
4795 ret = PTR_ERR(iter);
4797 /* Writes still need the seq_file to hold the private data */
4799 m = kzalloc(sizeof(*m), GFP_KERNEL);
4802 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4810 iter->trace_buffer = &tr->max_buffer;
4811 iter->cpu_file = tracing_get_cpu(inode);
4813 file->private_data = m;
4817 trace_array_put(tr);
4823 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4826 struct seq_file *m = filp->private_data;
4827 struct trace_iterator *iter = m->private;
4828 struct trace_array *tr = iter->tr;
4832 ret = tracing_update_buffers();
4836 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4840 mutex_lock(&trace_types_lock);
4842 if (tr->current_trace->use_max_tr) {
4849 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4853 if (tr->allocated_snapshot)
4857 /* Only allow per-cpu swap if the ring buffer supports it */
4858 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4859 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4864 if (!tr->allocated_snapshot) {
4865 ret = alloc_snapshot(tr);
4869 local_irq_disable();
4870 /* Now, we're going to swap */
4871 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4872 update_max_tr(tr, current, smp_processor_id());
4874 update_max_tr_single(tr, current, iter->cpu_file);
4878 if (tr->allocated_snapshot) {
4879 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4880 tracing_reset_online_cpus(&tr->max_buffer);
4882 tracing_reset(&tr->max_buffer, iter->cpu_file);
4892 mutex_unlock(&trace_types_lock);
4896 static int tracing_snapshot_release(struct inode *inode, struct file *file)
4898 struct seq_file *m = file->private_data;
4901 ret = tracing_release(inode, file);
4903 if (file->f_mode & FMODE_READ)
4906 /* If write only, the seq_file is just a stub */
4914 static int tracing_buffers_open(struct inode *inode, struct file *filp);
4915 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
4916 size_t count, loff_t *ppos);
4917 static int tracing_buffers_release(struct inode *inode, struct file *file);
4918 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4919 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
4921 static int snapshot_raw_open(struct inode *inode, struct file *filp)
4923 struct ftrace_buffer_info *info;
4926 ret = tracing_buffers_open(inode, filp);
4930 info = filp->private_data;
4932 if (info->iter.trace->use_max_tr) {
4933 tracing_buffers_release(inode, filp);
4937 info->iter.snapshot = true;
4938 info->iter.trace_buffer = &info->iter.tr->max_buffer;
4943 #endif /* CONFIG_TRACER_SNAPSHOT */
4946 static const struct file_operations tracing_max_lat_fops = {
4947 .open = tracing_open_generic,
4948 .read = tracing_max_lat_read,
4949 .write = tracing_max_lat_write,
4950 .llseek = generic_file_llseek,
4953 static const struct file_operations set_tracer_fops = {
4954 .open = tracing_open_generic,
4955 .read = tracing_set_trace_read,
4956 .write = tracing_set_trace_write,
4957 .llseek = generic_file_llseek,
4960 static const struct file_operations tracing_pipe_fops = {
4961 .open = tracing_open_pipe,
4962 .poll = tracing_poll_pipe,
4963 .read = tracing_read_pipe,
4964 .splice_read = tracing_splice_read_pipe,
4965 .release = tracing_release_pipe,
4966 .llseek = no_llseek,
4969 static const struct file_operations tracing_entries_fops = {
4970 .open = tracing_open_generic_tr,
4971 .read = tracing_entries_read,
4972 .write = tracing_entries_write,
4973 .llseek = generic_file_llseek,
4974 .release = tracing_release_generic_tr,
4977 static const struct file_operations tracing_total_entries_fops = {
4978 .open = tracing_open_generic_tr,
4979 .read = tracing_total_entries_read,
4980 .llseek = generic_file_llseek,
4981 .release = tracing_release_generic_tr,
4984 static const struct file_operations tracing_free_buffer_fops = {
4985 .open = tracing_open_generic_tr,
4986 .write = tracing_free_buffer_write,
4987 .release = tracing_free_buffer_release,
4990 static const struct file_operations tracing_mark_fops = {
4991 .open = tracing_open_generic_tr,
4992 .write = tracing_mark_write,
4993 .llseek = generic_file_llseek,
4994 .release = tracing_release_generic_tr,
4997 static const struct file_operations trace_clock_fops = {
4998 .open = tracing_clock_open,
5000 .llseek = seq_lseek,
5001 .release = tracing_single_release_tr,
5002 .write = tracing_clock_write,
5005 #ifdef CONFIG_TRACER_SNAPSHOT
5006 static const struct file_operations snapshot_fops = {
5007 .open = tracing_snapshot_open,
5009 .write = tracing_snapshot_write,
5010 .llseek = tracing_lseek,
5011 .release = tracing_snapshot_release,
5014 static const struct file_operations snapshot_raw_fops = {
5015 .open = snapshot_raw_open,
5016 .read = tracing_buffers_read,
5017 .release = tracing_buffers_release,
5018 .splice_read = tracing_buffers_splice_read,
5019 .llseek = no_llseek,
5022 #endif /* CONFIG_TRACER_SNAPSHOT */
5024 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5026 struct trace_array *tr = inode->i_private;
5027 struct ftrace_buffer_info *info;
5030 if (tracing_disabled)
5033 if (trace_array_get(tr) < 0)
5036 info = kzalloc(sizeof(*info), GFP_KERNEL);
5038 trace_array_put(tr);
5042 mutex_lock(&trace_types_lock);
5045 info->iter.cpu_file = tracing_get_cpu(inode);
5046 info->iter.trace = tr->current_trace;
5047 info->iter.trace_buffer = &tr->trace_buffer;
5049 /* Force reading ring buffer for first read */
5050 info->read = (unsigned int)-1;
5052 filp->private_data = info;
5054 mutex_unlock(&trace_types_lock);
5056 ret = nonseekable_open(inode, filp);
5058 trace_array_put(tr);
5064 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5066 struct ftrace_buffer_info *info = filp->private_data;
5067 struct trace_iterator *iter = &info->iter;
5069 return trace_poll(iter, filp, poll_table);
5073 tracing_buffers_read(struct file *filp, char __user *ubuf,
5074 size_t count, loff_t *ppos)
5076 struct ftrace_buffer_info *info = filp->private_data;
5077 struct trace_iterator *iter = &info->iter;
5084 mutex_lock(&trace_types_lock);
5086 #ifdef CONFIG_TRACER_MAX_TRACE
5087 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5094 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5100 /* Do we have previous read data to read? */
5101 if (info->read < PAGE_SIZE)
5105 trace_access_lock(iter->cpu_file);
5106 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5110 trace_access_unlock(iter->cpu_file);
5113 if (trace_empty(iter)) {
5114 if ((filp->f_flags & O_NONBLOCK)) {
5118 mutex_unlock(&trace_types_lock);
5119 ret = iter->trace->wait_pipe(iter);
5120 mutex_lock(&trace_types_lock);
5125 if (signal_pending(current)) {
5137 size = PAGE_SIZE - info->read;
5141 ret = copy_to_user(ubuf, info->spare + info->read, size);
5152 mutex_unlock(&trace_types_lock);
5157 static int tracing_buffers_release(struct inode *inode, struct file *file)
5159 struct ftrace_buffer_info *info = file->private_data;
5160 struct trace_iterator *iter = &info->iter;
5162 mutex_lock(&trace_types_lock);
5164 __trace_array_put(iter->tr);
5167 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5170 mutex_unlock(&trace_types_lock);
5176 struct ring_buffer *buffer;
5181 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5182 struct pipe_buffer *buf)
5184 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5189 ring_buffer_free_read_page(ref->buffer, ref->page);
5194 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5195 struct pipe_buffer *buf)
5197 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5202 /* Pipe buffer operations for a buffer. */
5203 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5205 .map = generic_pipe_buf_map,
5206 .unmap = generic_pipe_buf_unmap,
5207 .confirm = generic_pipe_buf_confirm,
5208 .release = buffer_pipe_buf_release,
5209 .steal = generic_pipe_buf_steal,
5210 .get = buffer_pipe_buf_get,
5214 * Callback from splice_to_pipe(), if we need to release some pages
5215 * at the end of the spd in case we error'ed out in filling the pipe.
5217 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5219 struct buffer_ref *ref =
5220 (struct buffer_ref *)spd->partial[i].private;
5225 ring_buffer_free_read_page(ref->buffer, ref->page);
5227 spd->partial[i].private = 0;
5231 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5232 struct pipe_inode_info *pipe, size_t len,
5235 struct ftrace_buffer_info *info = file->private_data;
5236 struct trace_iterator *iter = &info->iter;
5237 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5238 struct page *pages_def[PIPE_DEF_BUFFERS];
5239 struct splice_pipe_desc spd = {
5241 .partial = partial_def,
5242 .nr_pages_max = PIPE_DEF_BUFFERS,
5244 .ops = &buffer_pipe_buf_ops,
5245 .spd_release = buffer_spd_release,
5247 struct buffer_ref *ref;
5248 int entries, size, i;
5251 mutex_lock(&trace_types_lock);
5253 #ifdef CONFIG_TRACER_MAX_TRACE
5254 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5260 if (splice_grow_spd(pipe, &spd)) {
5265 if (*ppos & (PAGE_SIZE - 1)) {
5270 if (len & (PAGE_SIZE - 1)) {
5271 if (len < PAGE_SIZE) {
5279 trace_access_lock(iter->cpu_file);
5280 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5282 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
5286 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5291 ref->buffer = iter->trace_buffer->buffer;
5292 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5298 r = ring_buffer_read_page(ref->buffer, &ref->page,
5299 len, iter->cpu_file, 1);
5301 ring_buffer_free_read_page(ref->buffer, ref->page);
5307 * zero out any left over data, this is going to
5310 size = ring_buffer_page_len(ref->page);
5311 if (size < PAGE_SIZE)
5312 memset(ref->page + size, 0, PAGE_SIZE - size);
5314 page = virt_to_page(ref->page);
5316 spd.pages[i] = page;
5317 spd.partial[i].len = PAGE_SIZE;
5318 spd.partial[i].offset = 0;
5319 spd.partial[i].private = (unsigned long)ref;
5323 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5326 trace_access_unlock(iter->cpu_file);
5329 /* did we read anything? */
5330 if (!spd.nr_pages) {
5331 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5335 mutex_unlock(&trace_types_lock);
5336 ret = iter->trace->wait_pipe(iter);
5337 mutex_lock(&trace_types_lock);
5340 if (signal_pending(current)) {
5347 ret = splice_to_pipe(pipe, &spd);
5348 splice_shrink_spd(&spd);
5350 mutex_unlock(&trace_types_lock);
5355 static const struct file_operations tracing_buffers_fops = {
5356 .open = tracing_buffers_open,
5357 .read = tracing_buffers_read,
5358 .poll = tracing_buffers_poll,
5359 .release = tracing_buffers_release,
5360 .splice_read = tracing_buffers_splice_read,
5361 .llseek = no_llseek,
5365 tracing_stats_read(struct file *filp, char __user *ubuf,
5366 size_t count, loff_t *ppos)
5368 struct inode *inode = file_inode(filp);
5369 struct trace_array *tr = inode->i_private;
5370 struct trace_buffer *trace_buf = &tr->trace_buffer;
5371 int cpu = tracing_get_cpu(inode);
5372 struct trace_seq *s;
5374 unsigned long long t;
5375 unsigned long usec_rem;
5377 s = kmalloc(sizeof(*s), GFP_KERNEL);
5383 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5384 trace_seq_printf(s, "entries: %ld\n", cnt);
5386 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5387 trace_seq_printf(s, "overrun: %ld\n", cnt);
5389 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5390 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5392 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5393 trace_seq_printf(s, "bytes: %ld\n", cnt);
5395 if (trace_clocks[tr->clock_id].in_ns) {
5396 /* local or global for trace_clock */
5397 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5398 usec_rem = do_div(t, USEC_PER_SEC);
5399 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5402 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5403 usec_rem = do_div(t, USEC_PER_SEC);
5404 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5406 /* counter or tsc mode for trace_clock */
5407 trace_seq_printf(s, "oldest event ts: %llu\n",
5408 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5410 trace_seq_printf(s, "now ts: %llu\n",
5411 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5414 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5415 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5417 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5418 trace_seq_printf(s, "read events: %ld\n", cnt);
5420 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5427 static const struct file_operations tracing_stats_fops = {
5428 .open = tracing_open_generic_tr,
5429 .read = tracing_stats_read,
5430 .llseek = generic_file_llseek,
5431 .release = tracing_release_generic_tr,
5434 #ifdef CONFIG_DYNAMIC_FTRACE
5436 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5442 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5443 size_t cnt, loff_t *ppos)
5445 static char ftrace_dyn_info_buffer[1024];
5446 static DEFINE_MUTEX(dyn_info_mutex);
5447 unsigned long *p = filp->private_data;
5448 char *buf = ftrace_dyn_info_buffer;
5449 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5452 mutex_lock(&dyn_info_mutex);
5453 r = sprintf(buf, "%ld ", *p);
5455 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5458 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5460 mutex_unlock(&dyn_info_mutex);
5465 static const struct file_operations tracing_dyn_info_fops = {
5466 .open = tracing_open_generic,
5467 .read = tracing_read_dyn_info,
5468 .llseek = generic_file_llseek,
5470 #endif /* CONFIG_DYNAMIC_FTRACE */
5472 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5474 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5480 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5482 unsigned long *count = (long *)data;
5494 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5495 struct ftrace_probe_ops *ops, void *data)
5497 long count = (long)data;
5499 seq_printf(m, "%ps:", (void *)ip);
5501 seq_printf(m, "snapshot");
5504 seq_printf(m, ":unlimited\n");
5506 seq_printf(m, ":count=%ld\n", count);
5511 static struct ftrace_probe_ops snapshot_probe_ops = {
5512 .func = ftrace_snapshot,
5513 .print = ftrace_snapshot_print,
5516 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5517 .func = ftrace_count_snapshot,
5518 .print = ftrace_snapshot_print,
5522 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5523 char *glob, char *cmd, char *param, int enable)
5525 struct ftrace_probe_ops *ops;
5526 void *count = (void *)-1;
5530 /* hash funcs only work with set_ftrace_filter */
5534 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5536 if (glob[0] == '!') {
5537 unregister_ftrace_function_probe_func(glob+1, ops);
5544 number = strsep(¶m, ":");
5546 if (!strlen(number))
5550 * We use the callback data field (which is a pointer)
5553 ret = kstrtoul(number, 0, (unsigned long *)&count);
5558 ret = register_ftrace_function_probe(glob, ops, count);
5561 alloc_snapshot(&global_trace);
5563 return ret < 0 ? ret : 0;
5566 static struct ftrace_func_command ftrace_snapshot_cmd = {
5568 .func = ftrace_trace_snapshot_callback,
5571 static __init int register_snapshot_cmd(void)
5573 return register_ftrace_command(&ftrace_snapshot_cmd);
5576 static inline __init int register_snapshot_cmd(void) { return 0; }
5577 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5579 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5584 if (!debugfs_initialized())
5587 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5588 tr->dir = debugfs_create_dir("tracing", NULL);
5591 pr_warn_once("Could not create debugfs directory 'tracing'\n");
5596 struct dentry *tracing_init_dentry(void)
5598 return tracing_init_dentry_tr(&global_trace);
5601 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5603 struct dentry *d_tracer;
5606 return tr->percpu_dir;
5608 d_tracer = tracing_init_dentry_tr(tr);
5612 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5614 WARN_ONCE(!tr->percpu_dir,
5615 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5617 return tr->percpu_dir;
5620 static struct dentry *
5621 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
5622 void *data, long cpu, const struct file_operations *fops)
5624 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5626 if (ret) /* See tracing_get_cpu() */
5627 ret->d_inode->i_cdev = (void *)(cpu + 1);
5632 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5634 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5635 struct dentry *d_cpu;
5636 char cpu_dir[30]; /* 30 characters should be more than enough */
5641 snprintf(cpu_dir, 30, "cpu%ld", cpu);
5642 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5644 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5648 /* per cpu trace_pipe */
5649 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5650 tr, cpu, &tracing_pipe_fops);
5653 trace_create_cpu_file("trace", 0644, d_cpu,
5654 tr, cpu, &tracing_fops);
5656 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5657 tr, cpu, &tracing_buffers_fops);
5659 trace_create_cpu_file("stats", 0444, d_cpu,
5660 tr, cpu, &tracing_stats_fops);
5662 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5663 tr, cpu, &tracing_entries_fops);
5665 #ifdef CONFIG_TRACER_SNAPSHOT
5666 trace_create_cpu_file("snapshot", 0644, d_cpu,
5667 tr, cpu, &snapshot_fops);
5669 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5670 tr, cpu, &snapshot_raw_fops);
5674 #ifdef CONFIG_FTRACE_SELFTEST
5675 /* Let selftest have access to static functions in this file */
5676 #include "trace_selftest.c"
5679 struct trace_option_dentry {
5680 struct tracer_opt *opt;
5681 struct tracer_flags *flags;
5682 struct trace_array *tr;
5683 struct dentry *entry;
5687 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5690 struct trace_option_dentry *topt = filp->private_data;
5693 if (topt->flags->val & topt->opt->bit)
5698 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5702 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5705 struct trace_option_dentry *topt = filp->private_data;
5709 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5713 if (val != 0 && val != 1)
5716 if (!!(topt->flags->val & topt->opt->bit) != val) {
5717 mutex_lock(&trace_types_lock);
5718 ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
5720 mutex_unlock(&trace_types_lock);
5731 static const struct file_operations trace_options_fops = {
5732 .open = tracing_open_generic,
5733 .read = trace_options_read,
5734 .write = trace_options_write,
5735 .llseek = generic_file_llseek,
5739 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5742 long index = (long)filp->private_data;
5745 if (trace_flags & (1 << index))
5750 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5754 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5757 struct trace_array *tr = &global_trace;
5758 long index = (long)filp->private_data;
5762 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5766 if (val != 0 && val != 1)
5769 mutex_lock(&trace_types_lock);
5770 ret = set_tracer_flag(tr, 1 << index, val);
5771 mutex_unlock(&trace_types_lock);
5781 static const struct file_operations trace_options_core_fops = {
5782 .open = tracing_open_generic,
5783 .read = trace_options_core_read,
5784 .write = trace_options_core_write,
5785 .llseek = generic_file_llseek,
5788 struct dentry *trace_create_file(const char *name,
5790 struct dentry *parent,
5792 const struct file_operations *fops)
5796 ret = debugfs_create_file(name, mode, parent, data, fops);
5798 pr_warning("Could not create debugfs '%s' entry\n", name);
5804 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
5806 struct dentry *d_tracer;
5811 d_tracer = tracing_init_dentry_tr(tr);
5815 tr->options = debugfs_create_dir("options", d_tracer);
5817 pr_warning("Could not create debugfs directory 'options'\n");
5825 create_trace_option_file(struct trace_array *tr,
5826 struct trace_option_dentry *topt,
5827 struct tracer_flags *flags,
5828 struct tracer_opt *opt)
5830 struct dentry *t_options;
5832 t_options = trace_options_init_dentry(tr);
5836 topt->flags = flags;
5840 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
5841 &trace_options_fops);
5845 static struct trace_option_dentry *
5846 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
5848 struct trace_option_dentry *topts;
5849 struct tracer_flags *flags;
5850 struct tracer_opt *opts;
5856 flags = tracer->flags;
5858 if (!flags || !flags->opts)
5863 for (cnt = 0; opts[cnt].name; cnt++)
5866 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
5870 for (cnt = 0; opts[cnt].name; cnt++)
5871 create_trace_option_file(tr, &topts[cnt], flags,
5878 destroy_trace_option_files(struct trace_option_dentry *topts)
5885 for (cnt = 0; topts[cnt].opt; cnt++) {
5886 if (topts[cnt].entry)
5887 debugfs_remove(topts[cnt].entry);
5893 static struct dentry *
5894 create_trace_option_core_file(struct trace_array *tr,
5895 const char *option, long index)
5897 struct dentry *t_options;
5899 t_options = trace_options_init_dentry(tr);
5903 return trace_create_file(option, 0644, t_options, (void *)index,
5904 &trace_options_core_fops);
5907 static __init void create_trace_options_dir(struct trace_array *tr)
5909 struct dentry *t_options;
5912 t_options = trace_options_init_dentry(tr);
5916 for (i = 0; trace_options[i]; i++)
5917 create_trace_option_core_file(tr, trace_options[i], i);
5921 rb_simple_read(struct file *filp, char __user *ubuf,
5922 size_t cnt, loff_t *ppos)
5924 struct trace_array *tr = filp->private_data;
5928 r = tracer_tracing_is_on(tr);
5929 r = sprintf(buf, "%d\n", r);
5931 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5935 rb_simple_write(struct file *filp, const char __user *ubuf,
5936 size_t cnt, loff_t *ppos)
5938 struct trace_array *tr = filp->private_data;
5939 struct ring_buffer *buffer = tr->trace_buffer.buffer;
5943 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5948 mutex_lock(&trace_types_lock);
5950 tracer_tracing_on(tr);
5951 if (tr->current_trace->start)
5952 tr->current_trace->start(tr);
5954 tracer_tracing_off(tr);
5955 if (tr->current_trace->stop)
5956 tr->current_trace->stop(tr);
5958 mutex_unlock(&trace_types_lock);
5966 static const struct file_operations rb_simple_fops = {
5967 .open = tracing_open_generic_tr,
5968 .read = rb_simple_read,
5969 .write = rb_simple_write,
5970 .release = tracing_release_generic_tr,
5971 .llseek = default_llseek,
5974 struct dentry *trace_instance_dir;
5977 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
5980 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
5982 enum ring_buffer_flags rb_flags;
5984 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5988 buf->buffer = ring_buffer_alloc(size, rb_flags);
5992 buf->data = alloc_percpu(struct trace_array_cpu);
5994 ring_buffer_free(buf->buffer);
5998 /* Allocate the first page for all buffers */
5999 set_buffer_entries(&tr->trace_buffer,
6000 ring_buffer_size(tr->trace_buffer.buffer, 0));
6005 static int allocate_trace_buffers(struct trace_array *tr, int size)
6009 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6013 #ifdef CONFIG_TRACER_MAX_TRACE
6014 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6015 allocate_snapshot ? size : 1);
6017 ring_buffer_free(tr->trace_buffer.buffer);
6018 free_percpu(tr->trace_buffer.data);
6021 tr->allocated_snapshot = allocate_snapshot;
6024 * Only the top level trace array gets its snapshot allocated
6025 * from the kernel command line.
6027 allocate_snapshot = false;
6032 static int new_instance_create(const char *name)
6034 struct trace_array *tr;
6037 mutex_lock(&trace_types_lock);
6040 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6041 if (tr->name && strcmp(tr->name, name) == 0)
6046 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6050 tr->name = kstrdup(name, GFP_KERNEL);
6054 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6057 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6059 raw_spin_lock_init(&tr->start_lock);
6061 tr->current_trace = &nop_trace;
6063 INIT_LIST_HEAD(&tr->systems);
6064 INIT_LIST_HEAD(&tr->events);
6066 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6069 tr->dir = debugfs_create_dir(name, trace_instance_dir);
6073 ret = event_trace_add_tracer(tr->dir, tr);
6075 debugfs_remove_recursive(tr->dir);
6079 init_tracer_debugfs(tr, tr->dir);
6081 list_add(&tr->list, &ftrace_trace_arrays);
6083 mutex_unlock(&trace_types_lock);
6088 if (tr->trace_buffer.buffer)
6089 ring_buffer_free(tr->trace_buffer.buffer);
6090 free_cpumask_var(tr->tracing_cpumask);
6095 mutex_unlock(&trace_types_lock);
6101 static int instance_delete(const char *name)
6103 struct trace_array *tr;
6107 mutex_lock(&trace_types_lock);
6110 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6111 if (tr->name && strcmp(tr->name, name) == 0) {
6123 list_del(&tr->list);
6125 event_trace_del_tracer(tr);
6126 debugfs_remove_recursive(tr->dir);
6127 free_percpu(tr->trace_buffer.data);
6128 ring_buffer_free(tr->trace_buffer.buffer);
6136 mutex_unlock(&trace_types_lock);
6141 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
6143 struct dentry *parent;
6146 /* Paranoid: Make sure the parent is the "instances" directory */
6147 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6148 if (WARN_ON_ONCE(parent != trace_instance_dir))
6152 * The inode mutex is locked, but debugfs_create_dir() will also
6153 * take the mutex. As the instances directory can not be destroyed
6154 * or changed in any other way, it is safe to unlock it, and
6155 * let the dentry try. If two users try to make the same dir at
6156 * the same time, then the new_instance_create() will determine the
6159 mutex_unlock(&inode->i_mutex);
6161 ret = new_instance_create(dentry->d_iname);
6163 mutex_lock(&inode->i_mutex);
6168 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
6170 struct dentry *parent;
6173 /* Paranoid: Make sure the parent is the "instances" directory */
6174 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6175 if (WARN_ON_ONCE(parent != trace_instance_dir))
6178 /* The caller did a dget() on dentry */
6179 mutex_unlock(&dentry->d_inode->i_mutex);
6182 * The inode mutex is locked, but debugfs_create_dir() will also
6183 * take the mutex. As the instances directory can not be destroyed
6184 * or changed in any other way, it is safe to unlock it, and
6185 * let the dentry try. If two users try to make the same dir at
6186 * the same time, then the instance_delete() will determine the
6189 mutex_unlock(&inode->i_mutex);
6191 ret = instance_delete(dentry->d_iname);
6193 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6194 mutex_lock(&dentry->d_inode->i_mutex);
6199 static const struct inode_operations instance_dir_inode_operations = {
6200 .lookup = simple_lookup,
6201 .mkdir = instance_mkdir,
6202 .rmdir = instance_rmdir,
6205 static __init void create_trace_instances(struct dentry *d_tracer)
6207 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6208 if (WARN_ON(!trace_instance_dir))
6211 /* Hijack the dir inode operations, to allow mkdir */
6212 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6216 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
6220 trace_create_file("tracing_cpumask", 0644, d_tracer,
6221 tr, &tracing_cpumask_fops);
6223 trace_create_file("trace_options", 0644, d_tracer,
6224 tr, &tracing_iter_fops);
6226 trace_create_file("trace", 0644, d_tracer,
6229 trace_create_file("trace_pipe", 0444, d_tracer,
6230 tr, &tracing_pipe_fops);
6232 trace_create_file("buffer_size_kb", 0644, d_tracer,
6233 tr, &tracing_entries_fops);
6235 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6236 tr, &tracing_total_entries_fops);
6238 trace_create_file("free_buffer", 0200, d_tracer,
6239 tr, &tracing_free_buffer_fops);
6241 trace_create_file("trace_marker", 0220, d_tracer,
6242 tr, &tracing_mark_fops);
6244 trace_create_file("trace_clock", 0644, d_tracer, tr,
6247 trace_create_file("tracing_on", 0644, d_tracer,
6248 tr, &rb_simple_fops);
6250 #ifdef CONFIG_TRACER_SNAPSHOT
6251 trace_create_file("snapshot", 0644, d_tracer,
6252 tr, &snapshot_fops);
6255 for_each_tracing_cpu(cpu)
6256 tracing_init_debugfs_percpu(tr, cpu);
6260 static __init int tracer_init_debugfs(void)
6262 struct dentry *d_tracer;
6264 trace_access_lock_init();
6266 d_tracer = tracing_init_dentry();
6270 init_tracer_debugfs(&global_trace, d_tracer);
6272 trace_create_file("available_tracers", 0444, d_tracer,
6273 &global_trace, &show_traces_fops);
6275 trace_create_file("current_tracer", 0644, d_tracer,
6276 &global_trace, &set_tracer_fops);
6278 #ifdef CONFIG_TRACER_MAX_TRACE
6279 trace_create_file("tracing_max_latency", 0644, d_tracer,
6280 &tracing_max_latency, &tracing_max_lat_fops);
6283 trace_create_file("tracing_thresh", 0644, d_tracer,
6284 &tracing_thresh, &tracing_max_lat_fops);
6286 trace_create_file("README", 0444, d_tracer,
6287 NULL, &tracing_readme_fops);
6289 trace_create_file("saved_cmdlines", 0444, d_tracer,
6290 NULL, &tracing_saved_cmdlines_fops);
6292 #ifdef CONFIG_DYNAMIC_FTRACE
6293 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6294 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6297 create_trace_instances(d_tracer);
6299 create_trace_options_dir(&global_trace);
6304 static int trace_panic_handler(struct notifier_block *this,
6305 unsigned long event, void *unused)
6307 if (ftrace_dump_on_oops)
6308 ftrace_dump(ftrace_dump_on_oops);
6312 static struct notifier_block trace_panic_notifier = {
6313 .notifier_call = trace_panic_handler,
6315 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6318 static int trace_die_handler(struct notifier_block *self,
6324 if (ftrace_dump_on_oops)
6325 ftrace_dump(ftrace_dump_on_oops);
6333 static struct notifier_block trace_die_notifier = {
6334 .notifier_call = trace_die_handler,
6339 * printk is set to max of 1024, we really don't need it that big.
6340 * Nothing should be printing 1000 characters anyway.
6342 #define TRACE_MAX_PRINT 1000
6345 * Define here KERN_TRACE so that we have one place to modify
6346 * it if we decide to change what log level the ftrace dump
6349 #define KERN_TRACE KERN_EMERG
6352 trace_printk_seq(struct trace_seq *s)
6354 /* Probably should print a warning here. */
6355 if (s->len >= TRACE_MAX_PRINT)
6356 s->len = TRACE_MAX_PRINT;
6358 /* should be zero ended, but we are paranoid. */
6359 s->buffer[s->len] = 0;
6361 printk(KERN_TRACE "%s", s->buffer);
6366 void trace_init_global_iter(struct trace_iterator *iter)
6368 iter->tr = &global_trace;
6369 iter->trace = iter->tr->current_trace;
6370 iter->cpu_file = RING_BUFFER_ALL_CPUS;
6371 iter->trace_buffer = &global_trace.trace_buffer;
6373 if (iter->trace && iter->trace->open)
6374 iter->trace->open(iter);
6376 /* Annotate start of buffers if we had overruns */
6377 if (ring_buffer_overruns(iter->trace_buffer->buffer))
6378 iter->iter_flags |= TRACE_FILE_ANNOTATE;
6380 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6381 if (trace_clocks[iter->tr->clock_id].in_ns)
6382 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6385 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6387 /* use static because iter can be a bit big for the stack */
6388 static struct trace_iterator iter;
6389 static atomic_t dump_running;
6390 unsigned int old_userobj;
6391 unsigned long flags;
6394 /* Only allow one dump user at a time. */
6395 if (atomic_inc_return(&dump_running) != 1) {
6396 atomic_dec(&dump_running);
6401 * Always turn off tracing when we dump.
6402 * We don't need to show trace output of what happens
6403 * between multiple crashes.
6405 * If the user does a sysrq-z, then they can re-enable
6406 * tracing with echo 1 > tracing_on.
6410 local_irq_save(flags);
6412 /* Simulate the iterator */
6413 trace_init_global_iter(&iter);
6415 for_each_tracing_cpu(cpu) {
6416 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6419 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6421 /* don't look at user memory in panic mode */
6422 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6424 switch (oops_dump_mode) {
6426 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6429 iter.cpu_file = raw_smp_processor_id();
6434 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6435 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6438 printk(KERN_TRACE "Dumping ftrace buffer:\n");
6440 /* Did function tracer already get disabled? */
6441 if (ftrace_is_dead()) {
6442 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6443 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6447 * We need to stop all tracing on all CPUS to read the
6448 * the next buffer. This is a bit expensive, but is
6449 * not done often. We fill all what we can read,
6450 * and then release the locks again.
6453 while (!trace_empty(&iter)) {
6456 printk(KERN_TRACE "---------------------------------\n");
6460 /* reset all but tr, trace, and overruns */
6461 memset(&iter.seq, 0,
6462 sizeof(struct trace_iterator) -
6463 offsetof(struct trace_iterator, seq));
6464 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6467 if (trace_find_next_entry_inc(&iter) != NULL) {
6470 ret = print_trace_line(&iter);
6471 if (ret != TRACE_TYPE_NO_CONSUME)
6472 trace_consume(&iter);
6474 touch_nmi_watchdog();
6476 trace_printk_seq(&iter.seq);
6480 printk(KERN_TRACE " (ftrace buffer empty)\n");
6482 printk(KERN_TRACE "---------------------------------\n");
6485 trace_flags |= old_userobj;
6487 for_each_tracing_cpu(cpu) {
6488 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6490 atomic_dec(&dump_running);
6491 local_irq_restore(flags);
6493 EXPORT_SYMBOL_GPL(ftrace_dump);
6495 __init static int tracer_alloc_buffers(void)
6501 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6504 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
6505 goto out_free_buffer_mask;
6507 /* Only allocate trace_printk buffers if a trace_printk exists */
6508 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6509 /* Must be called before global_trace.buffer is allocated */
6510 trace_printk_init_buffers();
6512 /* To save memory, keep the ring buffer size to its minimum */
6513 if (ring_buffer_expanded)
6514 ring_buf_size = trace_buf_size;
6518 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6519 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
6521 raw_spin_lock_init(&global_trace.start_lock);
6523 /* Used for event triggers */
6524 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
6526 goto out_free_cpumask;
6528 /* TODO: make the number of buffers hot pluggable with CPUS */
6529 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6530 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6532 goto out_free_temp_buffer;
6535 if (global_trace.buffer_disabled)
6538 trace_init_cmdlines();
6541 * register_tracer() might reference current_trace, so it
6542 * needs to be set before we register anything. This is
6543 * just a bootstrap of current_trace anyway.
6545 global_trace.current_trace = &nop_trace;
6547 register_tracer(&nop_trace);
6549 /* All seems OK, enable tracing */
6550 tracing_disabled = 0;
6552 atomic_notifier_chain_register(&panic_notifier_list,
6553 &trace_panic_notifier);
6555 register_die_notifier(&trace_die_notifier);
6557 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6559 INIT_LIST_HEAD(&global_trace.systems);
6560 INIT_LIST_HEAD(&global_trace.events);
6561 list_add(&global_trace.list, &ftrace_trace_arrays);
6563 while (trace_boot_options) {
6566 option = strsep(&trace_boot_options, ",");
6567 trace_set_options(&global_trace, option);
6570 register_snapshot_cmd();
6574 out_free_temp_buffer:
6575 ring_buffer_free(temp_buffer);
6577 free_percpu(global_trace.trace_buffer.data);
6578 #ifdef CONFIG_TRACER_MAX_TRACE
6579 free_percpu(global_trace.max_buffer.data);
6581 free_cpumask_var(global_trace.tracing_cpumask);
6582 out_free_buffer_mask:
6583 free_cpumask_var(tracing_buffer_mask);
6588 __init static int clear_boot_tracer(void)
6591 * The default tracer at boot buffer is an init section.
6592 * This function is called in lateinit. If we did not
6593 * find the boot tracer, then clear it out, to prevent
6594 * later registration from accessing the buffer that is
6595 * about to be freed.
6597 if (!default_bootup_tracer)
6600 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6601 default_bootup_tracer);
6602 default_bootup_tracer = NULL;
6607 early_initcall(tracer_alloc_buffers);
6608 fs_initcall(tracer_init_debugfs);
6609 late_initcall(clear_boot_tracer);