2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
41 #include <linux/sched/rt.h>
44 #include "trace_output.h"
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
50 bool ring_buffer_expanded;
53 * We need to change this state when a selftest is running.
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
56 * insertions into the ring-buffer such as trace_printk could occurred
57 * at the same time, giving false positive or negative results.
59 static bool __read_mostly tracing_selftest_running;
62 * If a tracer is running, we do not want to run SELFTEST.
64 bool __read_mostly tracing_selftest_disabled;
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
71 static struct tracer_flags dummy_tracer_flags = {
73 .opts = dummy_tracer_opt
76 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
82 * To prevent the comm cache from being overwritten when no
83 * tracing is active, only save the comm when a trace event
86 static DEFINE_PER_CPU(bool, trace_cmdline_save);
89 * Kill all tracing for good (never come back).
90 * It is initialized to 1 but will turn to zero if the initialization
91 * of the tracer is successful. But that is the only place that sets
94 static int tracing_disabled = 1;
96 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
98 cpumask_var_t __read_mostly tracing_buffer_mask;
101 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
103 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104 * is set, then ftrace_dump is called. This will output the contents
105 * of the ftrace buffers to the console. This is very useful for
106 * capturing traces that lead to crashes and outputing it to a
109 * It is default off, but you can enable it with either specifying
110 * "ftrace_dump_on_oops" in the kernel command line, or setting
111 * /proc/sys/kernel/ftrace_dump_on_oops
112 * Set 1 if you want to dump buffers of all CPUs
113 * Set 2 if you want to dump the buffer of the CPU that triggered oops
116 enum ftrace_dump_mode ftrace_dump_on_oops;
118 /* When set, tracing will stop when a WARN*() is hit */
119 int __disable_trace_on_warning;
121 static int tracing_set_tracer(const char *buf);
123 #define MAX_TRACER_SIZE 100
124 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
125 static char *default_bootup_tracer;
127 static bool allocate_snapshot;
129 static int __init set_cmdline_ftrace(char *str)
131 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
132 default_bootup_tracer = bootup_tracer_buf;
133 /* We are using ftrace early, expand it */
134 ring_buffer_expanded = true;
137 __setup("ftrace=", set_cmdline_ftrace);
139 static int __init set_ftrace_dump_on_oops(char *str)
141 if (*str++ != '=' || !*str) {
142 ftrace_dump_on_oops = DUMP_ALL;
146 if (!strcmp("orig_cpu", str)) {
147 ftrace_dump_on_oops = DUMP_ORIG;
153 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
155 static int __init stop_trace_on_warning(char *str)
157 __disable_trace_on_warning = 1;
160 __setup("traceoff_on_warning=", stop_trace_on_warning);
162 static int __init boot_alloc_snapshot(char *str)
164 allocate_snapshot = true;
165 /* We also need the main ring buffer expanded */
166 ring_buffer_expanded = true;
169 __setup("alloc_snapshot", boot_alloc_snapshot);
172 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
173 static char *trace_boot_options __initdata;
175 static int __init set_trace_boot_options(char *str)
177 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
178 trace_boot_options = trace_boot_options_buf;
181 __setup("trace_options=", set_trace_boot_options);
184 unsigned long long ns2usecs(cycle_t nsec)
192 * The global_trace is the descriptor that holds the tracing
193 * buffers for the live tracing. For each CPU, it contains
194 * a link list of pages that will store trace entries. The
195 * page descriptor of the pages in the memory is used to hold
196 * the link list by linking the lru item in the page descriptor
197 * to each of the pages in the buffer per CPU.
199 * For each active CPU there is a data field that holds the
200 * pages for the buffer for that CPU. Each CPU has the same number
201 * of pages allocated for its buffer.
203 static struct trace_array global_trace;
205 LIST_HEAD(ftrace_trace_arrays);
207 int trace_array_get(struct trace_array *this_tr)
209 struct trace_array *tr;
212 mutex_lock(&trace_types_lock);
213 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
220 mutex_unlock(&trace_types_lock);
225 static void __trace_array_put(struct trace_array *this_tr)
227 WARN_ON(!this_tr->ref);
231 void trace_array_put(struct trace_array *this_tr)
233 mutex_lock(&trace_types_lock);
234 __trace_array_put(this_tr);
235 mutex_unlock(&trace_types_lock);
238 int filter_check_discard(struct ftrace_event_file *file, void *rec,
239 struct ring_buffer *buffer,
240 struct ring_buffer_event *event)
242 if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&
243 !filter_match_preds(file->filter, rec)) {
244 ring_buffer_discard_commit(buffer, event);
250 EXPORT_SYMBOL_GPL(filter_check_discard);
252 int call_filter_check_discard(struct ftrace_event_call *call, void *rec,
253 struct ring_buffer *buffer,
254 struct ring_buffer_event *event)
256 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
257 !filter_match_preds(call->filter, rec)) {
258 ring_buffer_discard_commit(buffer, event);
264 EXPORT_SYMBOL_GPL(call_filter_check_discard);
266 cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
270 /* Early boot up does not have a buffer yet */
272 return trace_clock_local();
274 ts = ring_buffer_time_stamp(buf->buffer, cpu);
275 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
280 cycle_t ftrace_now(int cpu)
282 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
286 * tracing_is_enabled - Show if global_trace has been disabled
288 * Shows if the global trace has been enabled or not. It uses the
289 * mirror flag "buffer_disabled" to be used in fast paths such as for
290 * the irqsoff tracer. But it may be inaccurate due to races. If you
291 * need to know the accurate state, use tracing_is_on() which is a little
292 * slower, but accurate.
294 int tracing_is_enabled(void)
297 * For quick access (irqsoff uses this in fast path), just
298 * return the mirror variable of the state of the ring buffer.
299 * It's a little racy, but we don't really care.
302 return !global_trace.buffer_disabled;
306 * trace_buf_size is the size in bytes that is allocated
307 * for a buffer. Note, the number of bytes is always rounded
310 * This number is purposely set to a low number of 16384.
311 * If the dump on oops happens, it will be much appreciated
312 * to not have to wait for all that output. Anyway this can be
313 * boot time and run time configurable.
315 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
317 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
319 /* trace_types holds a link list of available tracers. */
320 static struct tracer *trace_types __read_mostly;
323 * trace_types_lock is used to protect the trace_types list.
325 DEFINE_MUTEX(trace_types_lock);
328 * serialize the access of the ring buffer
330 * ring buffer serializes readers, but it is low level protection.
331 * The validity of the events (which returns by ring_buffer_peek() ..etc)
332 * are not protected by ring buffer.
334 * The content of events may become garbage if we allow other process consumes
335 * these events concurrently:
336 * A) the page of the consumed events may become a normal page
337 * (not reader page) in ring buffer, and this page will be rewrited
338 * by events producer.
339 * B) The page of the consumed events may become a page for splice_read,
340 * and this page will be returned to system.
342 * These primitives allow multi process access to different cpu ring buffer
345 * These primitives don't distinguish read-only and read-consume access.
346 * Multi read-only access are also serialized.
350 static DECLARE_RWSEM(all_cpu_access_lock);
351 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
353 static inline void trace_access_lock(int cpu)
355 if (cpu == RING_BUFFER_ALL_CPUS) {
356 /* gain it for accessing the whole ring buffer. */
357 down_write(&all_cpu_access_lock);
359 /* gain it for accessing a cpu ring buffer. */
361 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
362 down_read(&all_cpu_access_lock);
364 /* Secondly block other access to this @cpu ring buffer. */
365 mutex_lock(&per_cpu(cpu_access_lock, cpu));
369 static inline void trace_access_unlock(int cpu)
371 if (cpu == RING_BUFFER_ALL_CPUS) {
372 up_write(&all_cpu_access_lock);
374 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
375 up_read(&all_cpu_access_lock);
379 static inline void trace_access_lock_init(void)
383 for_each_possible_cpu(cpu)
384 mutex_init(&per_cpu(cpu_access_lock, cpu));
389 static DEFINE_MUTEX(access_lock);
391 static inline void trace_access_lock(int cpu)
394 mutex_lock(&access_lock);
397 static inline void trace_access_unlock(int cpu)
400 mutex_unlock(&access_lock);
403 static inline void trace_access_lock_init(void)
409 /* trace_flags holds trace_options default values */
410 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
411 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
412 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
413 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
415 static void tracer_tracing_on(struct trace_array *tr)
417 if (tr->trace_buffer.buffer)
418 ring_buffer_record_on(tr->trace_buffer.buffer);
420 * This flag is looked at when buffers haven't been allocated
421 * yet, or by some tracers (like irqsoff), that just want to
422 * know if the ring buffer has been disabled, but it can handle
423 * races of where it gets disabled but we still do a record.
424 * As the check is in the fast path of the tracers, it is more
425 * important to be fast than accurate.
427 tr->buffer_disabled = 0;
428 /* Make the flag seen by readers */
433 * tracing_on - enable tracing buffers
435 * This function enables tracing buffers that may have been
436 * disabled with tracing_off.
438 void tracing_on(void)
440 tracer_tracing_on(&global_trace);
442 EXPORT_SYMBOL_GPL(tracing_on);
445 * __trace_puts - write a constant string into the trace buffer.
446 * @ip: The address of the caller
447 * @str: The constant string to write
448 * @size: The size of the string.
450 int __trace_puts(unsigned long ip, const char *str, int size)
452 struct ring_buffer_event *event;
453 struct ring_buffer *buffer;
454 struct print_entry *entry;
455 unsigned long irq_flags;
458 if (unlikely(tracing_selftest_running || tracing_disabled))
461 alloc = sizeof(*entry) + size + 2; /* possible \n added */
463 local_save_flags(irq_flags);
464 buffer = global_trace.trace_buffer.buffer;
465 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
466 irq_flags, preempt_count());
470 entry = ring_buffer_event_data(event);
473 memcpy(&entry->buf, str, size);
475 /* Add a newline if necessary */
476 if (entry->buf[size - 1] != '\n') {
477 entry->buf[size] = '\n';
478 entry->buf[size + 1] = '\0';
480 entry->buf[size] = '\0';
482 __buffer_unlock_commit(buffer, event);
486 EXPORT_SYMBOL_GPL(__trace_puts);
489 * __trace_bputs - write the pointer to a constant string into trace buffer
490 * @ip: The address of the caller
491 * @str: The constant string to write to the buffer to
493 int __trace_bputs(unsigned long ip, const char *str)
495 struct ring_buffer_event *event;
496 struct ring_buffer *buffer;
497 struct bputs_entry *entry;
498 unsigned long irq_flags;
499 int size = sizeof(struct bputs_entry);
501 if (unlikely(tracing_selftest_running || tracing_disabled))
504 local_save_flags(irq_flags);
505 buffer = global_trace.trace_buffer.buffer;
506 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
507 irq_flags, preempt_count());
511 entry = ring_buffer_event_data(event);
515 __buffer_unlock_commit(buffer, event);
519 EXPORT_SYMBOL_GPL(__trace_bputs);
521 #ifdef CONFIG_TRACER_SNAPSHOT
523 * trace_snapshot - take a snapshot of the current buffer.
525 * This causes a swap between the snapshot buffer and the current live
526 * tracing buffer. You can use this to take snapshots of the live
527 * trace when some condition is triggered, but continue to trace.
529 * Note, make sure to allocate the snapshot with either
530 * a tracing_snapshot_alloc(), or by doing it manually
531 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
533 * If the snapshot buffer is not allocated, it will stop tracing.
534 * Basically making a permanent snapshot.
536 void tracing_snapshot(void)
538 struct trace_array *tr = &global_trace;
539 struct tracer *tracer = tr->current_trace;
543 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
544 internal_trace_puts("*** snapshot is being ignored ***\n");
548 if (!tr->allocated_snapshot) {
549 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
550 internal_trace_puts("*** stopping trace here! ***\n");
555 /* Note, snapshot can not be used when the tracer uses it */
556 if (tracer->use_max_tr) {
557 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
558 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
562 local_irq_save(flags);
563 update_max_tr(tr, current, smp_processor_id());
564 local_irq_restore(flags);
566 EXPORT_SYMBOL_GPL(tracing_snapshot);
568 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
569 struct trace_buffer *size_buf, int cpu_id);
570 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
572 static int alloc_snapshot(struct trace_array *tr)
576 if (!tr->allocated_snapshot) {
578 /* allocate spare buffer */
579 ret = resize_buffer_duplicate_size(&tr->max_buffer,
580 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
584 tr->allocated_snapshot = true;
590 void free_snapshot(struct trace_array *tr)
593 * We don't free the ring buffer. instead, resize it because
594 * The max_tr ring buffer has some state (e.g. ring->clock) and
595 * we want preserve it.
597 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
598 set_buffer_entries(&tr->max_buffer, 1);
599 tracing_reset_online_cpus(&tr->max_buffer);
600 tr->allocated_snapshot = false;
604 * tracing_alloc_snapshot - allocate snapshot buffer.
606 * This only allocates the snapshot buffer if it isn't already
607 * allocated - it doesn't also take a snapshot.
609 * This is meant to be used in cases where the snapshot buffer needs
610 * to be set up for events that can't sleep but need to be able to
611 * trigger a snapshot.
613 int tracing_alloc_snapshot(void)
615 struct trace_array *tr = &global_trace;
618 ret = alloc_snapshot(tr);
623 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
626 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
628 * This is similar to trace_snapshot(), but it will allocate the
629 * snapshot buffer if it isn't already allocated. Use this only
630 * where it is safe to sleep, as the allocation may sleep.
632 * This causes a swap between the snapshot buffer and the current live
633 * tracing buffer. You can use this to take snapshots of the live
634 * trace when some condition is triggered, but continue to trace.
636 void tracing_snapshot_alloc(void)
640 ret = tracing_alloc_snapshot();
646 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
648 void tracing_snapshot(void)
650 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
652 EXPORT_SYMBOL_GPL(tracing_snapshot);
653 int tracing_alloc_snapshot(void)
655 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
658 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
659 void tracing_snapshot_alloc(void)
664 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
665 #endif /* CONFIG_TRACER_SNAPSHOT */
667 static void tracer_tracing_off(struct trace_array *tr)
669 if (tr->trace_buffer.buffer)
670 ring_buffer_record_off(tr->trace_buffer.buffer);
672 * This flag is looked at when buffers haven't been allocated
673 * yet, or by some tracers (like irqsoff), that just want to
674 * know if the ring buffer has been disabled, but it can handle
675 * races of where it gets disabled but we still do a record.
676 * As the check is in the fast path of the tracers, it is more
677 * important to be fast than accurate.
679 tr->buffer_disabled = 1;
680 /* Make the flag seen by readers */
685 * tracing_off - turn off tracing buffers
687 * This function stops the tracing buffers from recording data.
688 * It does not disable any overhead the tracers themselves may
689 * be causing. This function simply causes all recording to
690 * the ring buffers to fail.
692 void tracing_off(void)
694 tracer_tracing_off(&global_trace);
696 EXPORT_SYMBOL_GPL(tracing_off);
698 void disable_trace_on_warning(void)
700 if (__disable_trace_on_warning)
705 * tracer_tracing_is_on - show real state of ring buffer enabled
706 * @tr : the trace array to know if ring buffer is enabled
708 * Shows real state of the ring buffer if it is enabled or not.
710 static int tracer_tracing_is_on(struct trace_array *tr)
712 if (tr->trace_buffer.buffer)
713 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
714 return !tr->buffer_disabled;
718 * tracing_is_on - show state of ring buffers enabled
720 int tracing_is_on(void)
722 return tracer_tracing_is_on(&global_trace);
724 EXPORT_SYMBOL_GPL(tracing_is_on);
726 static int __init set_buf_size(char *str)
728 unsigned long buf_size;
732 buf_size = memparse(str, &str);
733 /* nr_entries can not be zero */
736 trace_buf_size = buf_size;
739 __setup("trace_buf_size=", set_buf_size);
741 static int __init set_tracing_thresh(char *str)
743 unsigned long threshold;
748 ret = kstrtoul(str, 0, &threshold);
751 tracing_thresh = threshold * 1000;
754 __setup("tracing_thresh=", set_tracing_thresh);
756 unsigned long nsecs_to_usecs(unsigned long nsecs)
761 /* These must match the bit postions in trace_iterator_flags */
762 static const char *trace_options[] = {
795 int in_ns; /* is this clock in nanoseconds? */
797 { trace_clock_local, "local", 1 },
798 { trace_clock_global, "global", 1 },
799 { trace_clock_counter, "counter", 0 },
800 { trace_clock_jiffies, "uptime", 1 },
801 { trace_clock, "perf", 1 },
806 * trace_parser_get_init - gets the buffer for trace parser
808 int trace_parser_get_init(struct trace_parser *parser, int size)
810 memset(parser, 0, sizeof(*parser));
812 parser->buffer = kmalloc(size, GFP_KERNEL);
821 * trace_parser_put - frees the buffer for trace parser
823 void trace_parser_put(struct trace_parser *parser)
825 kfree(parser->buffer);
829 * trace_get_user - reads the user input string separated by space
830 * (matched by isspace(ch))
832 * For each string found the 'struct trace_parser' is updated,
833 * and the function returns.
835 * Returns number of bytes read.
837 * See kernel/trace/trace.h for 'struct trace_parser' details.
839 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
840 size_t cnt, loff_t *ppos)
847 trace_parser_clear(parser);
849 ret = get_user(ch, ubuf++);
857 * The parser is not finished with the last write,
858 * continue reading the user input without skipping spaces.
861 /* skip white space */
862 while (cnt && isspace(ch)) {
863 ret = get_user(ch, ubuf++);
870 /* only spaces were written */
880 /* read the non-space input */
881 while (cnt && !isspace(ch)) {
882 if (parser->idx < parser->size - 1)
883 parser->buffer[parser->idx++] = ch;
888 ret = get_user(ch, ubuf++);
895 /* We either got finished input or we have to wait for another call. */
897 parser->buffer[parser->idx] = 0;
898 parser->cont = false;
899 } else if (parser->idx < parser->size - 1) {
901 parser->buffer[parser->idx++] = ch;
914 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
922 if (s->len <= s->readpos)
925 len = s->len - s->readpos;
928 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
938 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
942 if (s->len <= s->readpos)
945 len = s->len - s->readpos;
948 memcpy(buf, s->buffer + s->readpos, cnt);
955 * ftrace_max_lock is used to protect the swapping of buffers
956 * when taking a max snapshot. The buffers themselves are
957 * protected by per_cpu spinlocks. But the action of the swap
958 * needs its own lock.
960 * This is defined as a arch_spinlock_t in order to help
961 * with performance when lockdep debugging is enabled.
963 * It is also used in other places outside the update_max_tr
964 * so it needs to be defined outside of the
965 * CONFIG_TRACER_MAX_TRACE.
967 static arch_spinlock_t ftrace_max_lock =
968 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
970 unsigned long __read_mostly tracing_thresh;
972 #ifdef CONFIG_TRACER_MAX_TRACE
973 unsigned long __read_mostly tracing_max_latency;
976 * Copy the new maximum trace into the separate maximum-trace
977 * structure. (this way the maximum trace is permanently saved,
978 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
981 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
983 struct trace_buffer *trace_buf = &tr->trace_buffer;
984 struct trace_buffer *max_buf = &tr->max_buffer;
985 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
986 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
989 max_buf->time_start = data->preempt_timestamp;
991 max_data->saved_latency = tracing_max_latency;
992 max_data->critical_start = data->critical_start;
993 max_data->critical_end = data->critical_end;
995 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
996 max_data->pid = tsk->pid;
998 * If tsk == current, then use current_uid(), as that does not use
999 * RCU. The irq tracer can be called out of RCU scope.
1002 max_data->uid = current_uid();
1004 max_data->uid = task_uid(tsk);
1006 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1007 max_data->policy = tsk->policy;
1008 max_data->rt_priority = tsk->rt_priority;
1010 /* record this tasks comm */
1011 tracing_record_cmdline(tsk);
1015 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1017 * @tsk: the task with the latency
1018 * @cpu: The cpu that initiated the trace.
1020 * Flip the buffers between the @tr and the max_tr and record information
1021 * about which task was the cause of this latency.
1024 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1026 struct ring_buffer *buf;
1031 WARN_ON_ONCE(!irqs_disabled());
1033 if (!tr->allocated_snapshot) {
1034 /* Only the nop tracer should hit this when disabling */
1035 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1039 arch_spin_lock(&ftrace_max_lock);
1041 buf = tr->trace_buffer.buffer;
1042 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1043 tr->max_buffer.buffer = buf;
1045 __update_max_tr(tr, tsk, cpu);
1046 arch_spin_unlock(&ftrace_max_lock);
1050 * update_max_tr_single - only copy one trace over, and reset the rest
1052 * @tsk - task with the latency
1053 * @cpu - the cpu of the buffer to copy.
1055 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1058 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1065 WARN_ON_ONCE(!irqs_disabled());
1066 if (!tr->allocated_snapshot) {
1067 /* Only the nop tracer should hit this when disabling */
1068 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1072 arch_spin_lock(&ftrace_max_lock);
1074 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1076 if (ret == -EBUSY) {
1078 * We failed to swap the buffer due to a commit taking
1079 * place on this CPU. We fail to record, but we reset
1080 * the max trace buffer (no one writes directly to it)
1081 * and flag that it failed.
1083 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1084 "Failed to swap buffers due to commit in progress\n");
1087 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1089 __update_max_tr(tr, tsk, cpu);
1090 arch_spin_unlock(&ftrace_max_lock);
1092 #endif /* CONFIG_TRACER_MAX_TRACE */
1094 static void default_wait_pipe(struct trace_iterator *iter)
1096 /* Iterators are static, they should be filled or empty */
1097 if (trace_buffer_iter(iter, iter->cpu_file))
1100 ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
1103 #ifdef CONFIG_FTRACE_STARTUP_TEST
1104 static int run_tracer_selftest(struct tracer *type)
1106 struct trace_array *tr = &global_trace;
1107 struct tracer *saved_tracer = tr->current_trace;
1110 if (!type->selftest || tracing_selftest_disabled)
1114 * Run a selftest on this tracer.
1115 * Here we reset the trace buffer, and set the current
1116 * tracer to be this tracer. The tracer can then run some
1117 * internal tracing to verify that everything is in order.
1118 * If we fail, we do not register this tracer.
1120 tracing_reset_online_cpus(&tr->trace_buffer);
1122 tr->current_trace = type;
1124 #ifdef CONFIG_TRACER_MAX_TRACE
1125 if (type->use_max_tr) {
1126 /* If we expanded the buffers, make sure the max is expanded too */
1127 if (ring_buffer_expanded)
1128 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1129 RING_BUFFER_ALL_CPUS);
1130 tr->allocated_snapshot = true;
1134 /* the test is responsible for initializing and enabling */
1135 pr_info("Testing tracer %s: ", type->name);
1136 ret = type->selftest(type, tr);
1137 /* the test is responsible for resetting too */
1138 tr->current_trace = saved_tracer;
1140 printk(KERN_CONT "FAILED!\n");
1141 /* Add the warning after printing 'FAILED' */
1145 /* Only reset on passing, to avoid touching corrupted buffers */
1146 tracing_reset_online_cpus(&tr->trace_buffer);
1148 #ifdef CONFIG_TRACER_MAX_TRACE
1149 if (type->use_max_tr) {
1150 tr->allocated_snapshot = false;
1152 /* Shrink the max buffer again */
1153 if (ring_buffer_expanded)
1154 ring_buffer_resize(tr->max_buffer.buffer, 1,
1155 RING_BUFFER_ALL_CPUS);
1159 printk(KERN_CONT "PASSED\n");
1163 static inline int run_tracer_selftest(struct tracer *type)
1167 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1170 * register_tracer - register a tracer with the ftrace system.
1171 * @type - the plugin for the tracer
1173 * Register a new plugin tracer.
1175 int register_tracer(struct tracer *type)
1181 pr_info("Tracer must have a name\n");
1185 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1186 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1190 mutex_lock(&trace_types_lock);
1192 tracing_selftest_running = true;
1194 for (t = trace_types; t; t = t->next) {
1195 if (strcmp(type->name, t->name) == 0) {
1197 pr_info("Tracer %s already registered\n",
1204 if (!type->set_flag)
1205 type->set_flag = &dummy_set_flag;
1207 type->flags = &dummy_tracer_flags;
1209 if (!type->flags->opts)
1210 type->flags->opts = dummy_tracer_opt;
1211 if (!type->wait_pipe)
1212 type->wait_pipe = default_wait_pipe;
1214 ret = run_tracer_selftest(type);
1218 type->next = trace_types;
1222 tracing_selftest_running = false;
1223 mutex_unlock(&trace_types_lock);
1225 if (ret || !default_bootup_tracer)
1228 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1231 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1232 /* Do we want this tracer to start on bootup? */
1233 tracing_set_tracer(type->name);
1234 default_bootup_tracer = NULL;
1235 /* disable other selftests, since this will break it. */
1236 tracing_selftest_disabled = true;
1237 #ifdef CONFIG_FTRACE_STARTUP_TEST
1238 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1246 void tracing_reset(struct trace_buffer *buf, int cpu)
1248 struct ring_buffer *buffer = buf->buffer;
1253 ring_buffer_record_disable(buffer);
1255 /* Make sure all commits have finished */
1256 synchronize_sched();
1257 ring_buffer_reset_cpu(buffer, cpu);
1259 ring_buffer_record_enable(buffer);
1262 void tracing_reset_online_cpus(struct trace_buffer *buf)
1264 struct ring_buffer *buffer = buf->buffer;
1270 ring_buffer_record_disable(buffer);
1272 /* Make sure all commits have finished */
1273 synchronize_sched();
1275 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1277 for_each_online_cpu(cpu)
1278 ring_buffer_reset_cpu(buffer, cpu);
1280 ring_buffer_record_enable(buffer);
1283 /* Must have trace_types_lock held */
1284 void tracing_reset_all_online_cpus(void)
1286 struct trace_array *tr;
1288 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1289 tracing_reset_online_cpus(&tr->trace_buffer);
1290 #ifdef CONFIG_TRACER_MAX_TRACE
1291 tracing_reset_online_cpus(&tr->max_buffer);
1296 #define SAVED_CMDLINES 128
1297 #define NO_CMDLINE_MAP UINT_MAX
1298 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1299 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
1300 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
1301 static int cmdline_idx;
1302 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1304 /* temporary disable recording */
1305 static atomic_t trace_record_cmdline_disabled __read_mostly;
1307 static void trace_init_cmdlines(void)
1309 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
1310 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
1314 int is_tracing_stopped(void)
1316 return global_trace.stop_count;
1320 * tracing_start - quick start of the tracer
1322 * If tracing is enabled but was stopped by tracing_stop,
1323 * this will start the tracer back up.
1325 void tracing_start(void)
1327 struct ring_buffer *buffer;
1328 unsigned long flags;
1330 if (tracing_disabled)
1333 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1334 if (--global_trace.stop_count) {
1335 if (global_trace.stop_count < 0) {
1336 /* Someone screwed up their debugging */
1338 global_trace.stop_count = 0;
1343 /* Prevent the buffers from switching */
1344 arch_spin_lock(&ftrace_max_lock);
1346 buffer = global_trace.trace_buffer.buffer;
1348 ring_buffer_record_enable(buffer);
1350 #ifdef CONFIG_TRACER_MAX_TRACE
1351 buffer = global_trace.max_buffer.buffer;
1353 ring_buffer_record_enable(buffer);
1356 arch_spin_unlock(&ftrace_max_lock);
1360 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1363 static void tracing_start_tr(struct trace_array *tr)
1365 struct ring_buffer *buffer;
1366 unsigned long flags;
1368 if (tracing_disabled)
1371 /* If global, we need to also start the max tracer */
1372 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1373 return tracing_start();
1375 raw_spin_lock_irqsave(&tr->start_lock, flags);
1377 if (--tr->stop_count) {
1378 if (tr->stop_count < 0) {
1379 /* Someone screwed up their debugging */
1386 buffer = tr->trace_buffer.buffer;
1388 ring_buffer_record_enable(buffer);
1391 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1395 * tracing_stop - quick stop of the tracer
1397 * Light weight way to stop tracing. Use in conjunction with
1400 void tracing_stop(void)
1402 struct ring_buffer *buffer;
1403 unsigned long flags;
1406 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1407 if (global_trace.stop_count++)
1410 /* Prevent the buffers from switching */
1411 arch_spin_lock(&ftrace_max_lock);
1413 buffer = global_trace.trace_buffer.buffer;
1415 ring_buffer_record_disable(buffer);
1417 #ifdef CONFIG_TRACER_MAX_TRACE
1418 buffer = global_trace.max_buffer.buffer;
1420 ring_buffer_record_disable(buffer);
1423 arch_spin_unlock(&ftrace_max_lock);
1426 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1429 static void tracing_stop_tr(struct trace_array *tr)
1431 struct ring_buffer *buffer;
1432 unsigned long flags;
1434 /* If global, we need to also stop the max tracer */
1435 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1436 return tracing_stop();
1438 raw_spin_lock_irqsave(&tr->start_lock, flags);
1439 if (tr->stop_count++)
1442 buffer = tr->trace_buffer.buffer;
1444 ring_buffer_record_disable(buffer);
1447 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1450 void trace_stop_cmdline_recording(void);
1452 static int trace_save_cmdline(struct task_struct *tsk)
1456 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1460 * It's not the end of the world if we don't get
1461 * the lock, but we also don't want to spin
1462 * nor do we want to disable interrupts,
1463 * so if we miss here, then better luck next time.
1465 if (!arch_spin_trylock(&trace_cmdline_lock))
1468 idx = map_pid_to_cmdline[tsk->pid];
1469 if (idx == NO_CMDLINE_MAP) {
1470 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1473 * Check whether the cmdline buffer at idx has a pid
1474 * mapped. We are going to overwrite that entry so we
1475 * need to clear the map_pid_to_cmdline. Otherwise we
1476 * would read the new comm for the old pid.
1478 pid = map_cmdline_to_pid[idx];
1479 if (pid != NO_CMDLINE_MAP)
1480 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1482 map_cmdline_to_pid[idx] = tsk->pid;
1483 map_pid_to_cmdline[tsk->pid] = idx;
1488 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1490 arch_spin_unlock(&trace_cmdline_lock);
1495 void trace_find_cmdline(int pid, char comm[])
1500 strcpy(comm, "<idle>");
1504 if (WARN_ON_ONCE(pid < 0)) {
1505 strcpy(comm, "<XXX>");
1509 if (pid > PID_MAX_DEFAULT) {
1510 strcpy(comm, "<...>");
1515 arch_spin_lock(&trace_cmdline_lock);
1516 map = map_pid_to_cmdline[pid];
1517 if (map != NO_CMDLINE_MAP)
1518 strcpy(comm, saved_cmdlines[map]);
1520 strcpy(comm, "<...>");
1522 arch_spin_unlock(&trace_cmdline_lock);
1526 void tracing_record_cmdline(struct task_struct *tsk)
1528 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1531 if (!__this_cpu_read(trace_cmdline_save))
1534 if (trace_save_cmdline(tsk))
1535 __this_cpu_write(trace_cmdline_save, false);
1539 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1542 struct task_struct *tsk = current;
1544 entry->preempt_count = pc & 0xff;
1545 entry->pid = (tsk) ? tsk->pid : 0;
1547 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1548 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1550 TRACE_FLAG_IRQS_NOSUPPORT |
1552 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1553 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1554 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1555 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1557 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1559 struct ring_buffer_event *
1560 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1563 unsigned long flags, int pc)
1565 struct ring_buffer_event *event;
1567 event = ring_buffer_lock_reserve(buffer, len);
1568 if (event != NULL) {
1569 struct trace_entry *ent = ring_buffer_event_data(event);
1571 tracing_generic_entry_update(ent, flags, pc);
1579 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1581 __this_cpu_write(trace_cmdline_save, true);
1582 ring_buffer_unlock_commit(buffer, event);
1586 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1587 struct ring_buffer_event *event,
1588 unsigned long flags, int pc)
1590 __buffer_unlock_commit(buffer, event);
1592 ftrace_trace_stack(buffer, flags, 6, pc);
1593 ftrace_trace_userstack(buffer, flags, pc);
1596 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1597 struct ring_buffer_event *event,
1598 unsigned long flags, int pc)
1600 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1602 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1604 static struct ring_buffer *temp_buffer;
1606 struct ring_buffer_event *
1607 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1608 struct ftrace_event_file *ftrace_file,
1609 int type, unsigned long len,
1610 unsigned long flags, int pc)
1612 struct ring_buffer_event *entry;
1614 *current_rb = ftrace_file->tr->trace_buffer.buffer;
1615 entry = trace_buffer_lock_reserve(*current_rb,
1616 type, len, flags, pc);
1618 * If tracing is off, but we have triggers enabled
1619 * we still need to look at the event data. Use the temp_buffer
1620 * to store the trace event for the tigger to use. It's recusive
1621 * safe and will not be recorded anywhere.
1623 if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) {
1624 *current_rb = temp_buffer;
1625 entry = trace_buffer_lock_reserve(*current_rb,
1626 type, len, flags, pc);
1630 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1632 struct ring_buffer_event *
1633 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1634 int type, unsigned long len,
1635 unsigned long flags, int pc)
1637 *current_rb = global_trace.trace_buffer.buffer;
1638 return trace_buffer_lock_reserve(*current_rb,
1639 type, len, flags, pc);
1641 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1643 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1644 struct ring_buffer_event *event,
1645 unsigned long flags, int pc)
1647 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1649 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1651 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1652 struct ring_buffer_event *event,
1653 unsigned long flags, int pc,
1654 struct pt_regs *regs)
1656 __buffer_unlock_commit(buffer, event);
1658 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1659 ftrace_trace_userstack(buffer, flags, pc);
1661 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1663 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1664 struct ring_buffer_event *event)
1666 ring_buffer_discard_commit(buffer, event);
1668 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1671 trace_function(struct trace_array *tr,
1672 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1675 struct ftrace_event_call *call = &event_function;
1676 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1677 struct ring_buffer_event *event;
1678 struct ftrace_entry *entry;
1680 /* If we are reading the ring buffer, don't trace */
1681 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1684 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1688 entry = ring_buffer_event_data(event);
1690 entry->parent_ip = parent_ip;
1692 if (!call_filter_check_discard(call, entry, buffer, event))
1693 __buffer_unlock_commit(buffer, event);
1696 #ifdef CONFIG_STACKTRACE
1698 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1699 struct ftrace_stack {
1700 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1703 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1704 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1706 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1707 unsigned long flags,
1708 int skip, int pc, struct pt_regs *regs)
1710 struct ftrace_event_call *call = &event_kernel_stack;
1711 struct ring_buffer_event *event;
1712 struct stack_entry *entry;
1713 struct stack_trace trace;
1715 int size = FTRACE_STACK_ENTRIES;
1717 trace.nr_entries = 0;
1721 * Since events can happen in NMIs there's no safe way to
1722 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1723 * or NMI comes in, it will just have to use the default
1724 * FTRACE_STACK_SIZE.
1726 preempt_disable_notrace();
1728 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1730 * We don't need any atomic variables, just a barrier.
1731 * If an interrupt comes in, we don't care, because it would
1732 * have exited and put the counter back to what we want.
1733 * We just need a barrier to keep gcc from moving things
1737 if (use_stack == 1) {
1738 trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1739 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1742 save_stack_trace_regs(regs, &trace);
1744 save_stack_trace(&trace);
1746 if (trace.nr_entries > size)
1747 size = trace.nr_entries;
1749 /* From now on, use_stack is a boolean */
1752 size *= sizeof(unsigned long);
1754 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1755 sizeof(*entry) + size, flags, pc);
1758 entry = ring_buffer_event_data(event);
1760 memset(&entry->caller, 0, size);
1763 memcpy(&entry->caller, trace.entries,
1764 trace.nr_entries * sizeof(unsigned long));
1766 trace.max_entries = FTRACE_STACK_ENTRIES;
1767 trace.entries = entry->caller;
1769 save_stack_trace_regs(regs, &trace);
1771 save_stack_trace(&trace);
1774 entry->size = trace.nr_entries;
1776 if (!call_filter_check_discard(call, entry, buffer, event))
1777 __buffer_unlock_commit(buffer, event);
1780 /* Again, don't let gcc optimize things here */
1782 __this_cpu_dec(ftrace_stack_reserve);
1783 preempt_enable_notrace();
1787 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1788 int skip, int pc, struct pt_regs *regs)
1790 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1793 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1796 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1799 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1802 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1805 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1808 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1812 * trace_dump_stack - record a stack back trace in the trace buffer
1813 * @skip: Number of functions to skip (helper handlers)
1815 void trace_dump_stack(int skip)
1817 unsigned long flags;
1819 if (tracing_disabled || tracing_selftest_running)
1822 local_save_flags(flags);
1825 * Skip 3 more, seems to get us at the caller of
1829 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1830 flags, skip, preempt_count(), NULL);
1833 static DEFINE_PER_CPU(int, user_stack_count);
1836 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1838 struct ftrace_event_call *call = &event_user_stack;
1839 struct ring_buffer_event *event;
1840 struct userstack_entry *entry;
1841 struct stack_trace trace;
1843 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1847 * NMIs can not handle page faults, even with fix ups.
1848 * The save user stack can (and often does) fault.
1850 if (unlikely(in_nmi()))
1854 * prevent recursion, since the user stack tracing may
1855 * trigger other kernel events.
1858 if (__this_cpu_read(user_stack_count))
1861 __this_cpu_inc(user_stack_count);
1863 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1864 sizeof(*entry), flags, pc);
1866 goto out_drop_count;
1867 entry = ring_buffer_event_data(event);
1869 entry->tgid = current->tgid;
1870 memset(&entry->caller, 0, sizeof(entry->caller));
1872 trace.nr_entries = 0;
1873 trace.max_entries = FTRACE_STACK_ENTRIES;
1875 trace.entries = entry->caller;
1877 save_stack_trace_user(&trace);
1878 if (!call_filter_check_discard(call, entry, buffer, event))
1879 __buffer_unlock_commit(buffer, event);
1882 __this_cpu_dec(user_stack_count);
1888 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1890 ftrace_trace_userstack(tr, flags, preempt_count());
1894 #endif /* CONFIG_STACKTRACE */
1896 /* created for use with alloc_percpu */
1897 struct trace_buffer_struct {
1898 char buffer[TRACE_BUF_SIZE];
1901 static struct trace_buffer_struct *trace_percpu_buffer;
1902 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1903 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1904 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1907 * The buffer used is dependent on the context. There is a per cpu
1908 * buffer for normal context, softirq contex, hard irq context and
1909 * for NMI context. Thise allows for lockless recording.
1911 * Note, if the buffers failed to be allocated, then this returns NULL
1913 static char *get_trace_buf(void)
1915 struct trace_buffer_struct *percpu_buffer;
1918 * If we have allocated per cpu buffers, then we do not
1919 * need to do any locking.
1922 percpu_buffer = trace_percpu_nmi_buffer;
1924 percpu_buffer = trace_percpu_irq_buffer;
1925 else if (in_softirq())
1926 percpu_buffer = trace_percpu_sirq_buffer;
1928 percpu_buffer = trace_percpu_buffer;
1933 return this_cpu_ptr(&percpu_buffer->buffer[0]);
1936 static int alloc_percpu_trace_buffer(void)
1938 struct trace_buffer_struct *buffers;
1939 struct trace_buffer_struct *sirq_buffers;
1940 struct trace_buffer_struct *irq_buffers;
1941 struct trace_buffer_struct *nmi_buffers;
1943 buffers = alloc_percpu(struct trace_buffer_struct);
1947 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1951 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1955 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1959 trace_percpu_buffer = buffers;
1960 trace_percpu_sirq_buffer = sirq_buffers;
1961 trace_percpu_irq_buffer = irq_buffers;
1962 trace_percpu_nmi_buffer = nmi_buffers;
1967 free_percpu(irq_buffers);
1969 free_percpu(sirq_buffers);
1971 free_percpu(buffers);
1973 WARN(1, "Could not allocate percpu trace_printk buffer");
1977 static int buffers_allocated;
1979 void trace_printk_init_buffers(void)
1981 if (buffers_allocated)
1984 if (alloc_percpu_trace_buffer())
1987 pr_info("ftrace: Allocated trace_printk buffers\n");
1989 /* Expand the buffers to set size */
1990 tracing_update_buffers();
1992 buffers_allocated = 1;
1995 * trace_printk_init_buffers() can be called by modules.
1996 * If that happens, then we need to start cmdline recording
1997 * directly here. If the global_trace.buffer is already
1998 * allocated here, then this was called by module code.
2000 if (global_trace.trace_buffer.buffer)
2001 tracing_start_cmdline_record();
2004 void trace_printk_start_comm(void)
2006 /* Start tracing comms if trace printk is set */
2007 if (!buffers_allocated)
2009 tracing_start_cmdline_record();
2012 static void trace_printk_start_stop_comm(int enabled)
2014 if (!buffers_allocated)
2018 tracing_start_cmdline_record();
2020 tracing_stop_cmdline_record();
2024 * trace_vbprintk - write binary msg to tracing buffer
2027 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2029 struct ftrace_event_call *call = &event_bprint;
2030 struct ring_buffer_event *event;
2031 struct ring_buffer *buffer;
2032 struct trace_array *tr = &global_trace;
2033 struct bprint_entry *entry;
2034 unsigned long flags;
2036 int len = 0, size, pc;
2038 if (unlikely(tracing_selftest_running || tracing_disabled))
2041 /* Don't pollute graph traces with trace_vprintk internals */
2042 pause_graph_tracing();
2044 pc = preempt_count();
2045 preempt_disable_notrace();
2047 tbuffer = get_trace_buf();
2053 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2055 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2058 local_save_flags(flags);
2059 size = sizeof(*entry) + sizeof(u32) * len;
2060 buffer = tr->trace_buffer.buffer;
2061 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2065 entry = ring_buffer_event_data(event);
2069 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2070 if (!call_filter_check_discard(call, entry, buffer, event)) {
2071 __buffer_unlock_commit(buffer, event);
2072 ftrace_trace_stack(buffer, flags, 6, pc);
2076 preempt_enable_notrace();
2077 unpause_graph_tracing();
2081 EXPORT_SYMBOL_GPL(trace_vbprintk);
2084 __trace_array_vprintk(struct ring_buffer *buffer,
2085 unsigned long ip, const char *fmt, va_list args)
2087 struct ftrace_event_call *call = &event_print;
2088 struct ring_buffer_event *event;
2089 int len = 0, size, pc;
2090 struct print_entry *entry;
2091 unsigned long flags;
2094 if (tracing_disabled || tracing_selftest_running)
2097 /* Don't pollute graph traces with trace_vprintk internals */
2098 pause_graph_tracing();
2100 pc = preempt_count();
2101 preempt_disable_notrace();
2104 tbuffer = get_trace_buf();
2110 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2111 if (len > TRACE_BUF_SIZE)
2114 local_save_flags(flags);
2115 size = sizeof(*entry) + len + 1;
2116 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2120 entry = ring_buffer_event_data(event);
2123 memcpy(&entry->buf, tbuffer, len);
2124 entry->buf[len] = '\0';
2125 if (!call_filter_check_discard(call, entry, buffer, event)) {
2126 __buffer_unlock_commit(buffer, event);
2127 ftrace_trace_stack(buffer, flags, 6, pc);
2130 preempt_enable_notrace();
2131 unpause_graph_tracing();
2136 int trace_array_vprintk(struct trace_array *tr,
2137 unsigned long ip, const char *fmt, va_list args)
2139 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2142 int trace_array_printk(struct trace_array *tr,
2143 unsigned long ip, const char *fmt, ...)
2148 if (!(trace_flags & TRACE_ITER_PRINTK))
2152 ret = trace_array_vprintk(tr, ip, fmt, ap);
2157 int trace_array_printk_buf(struct ring_buffer *buffer,
2158 unsigned long ip, const char *fmt, ...)
2163 if (!(trace_flags & TRACE_ITER_PRINTK))
2167 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2172 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2174 return trace_array_vprintk(&global_trace, ip, fmt, args);
2176 EXPORT_SYMBOL_GPL(trace_vprintk);
2178 static void trace_iterator_increment(struct trace_iterator *iter)
2180 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2184 ring_buffer_read(buf_iter, NULL);
2187 static struct trace_entry *
2188 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2189 unsigned long *lost_events)
2191 struct ring_buffer_event *event;
2192 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2195 event = ring_buffer_iter_peek(buf_iter, ts);
2197 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2201 iter->ent_size = ring_buffer_event_length(event);
2202 return ring_buffer_event_data(event);
2208 static struct trace_entry *
2209 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2210 unsigned long *missing_events, u64 *ent_ts)
2212 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2213 struct trace_entry *ent, *next = NULL;
2214 unsigned long lost_events = 0, next_lost = 0;
2215 int cpu_file = iter->cpu_file;
2216 u64 next_ts = 0, ts;
2222 * If we are in a per_cpu trace file, don't bother by iterating over
2223 * all cpu and peek directly.
2225 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2226 if (ring_buffer_empty_cpu(buffer, cpu_file))
2228 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2230 *ent_cpu = cpu_file;
2235 for_each_tracing_cpu(cpu) {
2237 if (ring_buffer_empty_cpu(buffer, cpu))
2240 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2243 * Pick the entry with the smallest timestamp:
2245 if (ent && (!next || ts < next_ts)) {
2249 next_lost = lost_events;
2250 next_size = iter->ent_size;
2254 iter->ent_size = next_size;
2257 *ent_cpu = next_cpu;
2263 *missing_events = next_lost;
2268 /* Find the next real entry, without updating the iterator itself */
2269 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2270 int *ent_cpu, u64 *ent_ts)
2272 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2275 /* Find the next real entry, and increment the iterator to the next entry */
2276 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2278 iter->ent = __find_next_entry(iter, &iter->cpu,
2279 &iter->lost_events, &iter->ts);
2282 trace_iterator_increment(iter);
2284 return iter->ent ? iter : NULL;
2287 static void trace_consume(struct trace_iterator *iter)
2289 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2290 &iter->lost_events);
2293 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2295 struct trace_iterator *iter = m->private;
2299 WARN_ON_ONCE(iter->leftover);
2303 /* can't go backwards */
2308 ent = trace_find_next_entry_inc(iter);
2312 while (ent && iter->idx < i)
2313 ent = trace_find_next_entry_inc(iter);
2320 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2322 struct ring_buffer_event *event;
2323 struct ring_buffer_iter *buf_iter;
2324 unsigned long entries = 0;
2327 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2329 buf_iter = trace_buffer_iter(iter, cpu);
2333 ring_buffer_iter_reset(buf_iter);
2336 * We could have the case with the max latency tracers
2337 * that a reset never took place on a cpu. This is evident
2338 * by the timestamp being before the start of the buffer.
2340 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2341 if (ts >= iter->trace_buffer->time_start)
2344 ring_buffer_read(buf_iter, NULL);
2347 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2351 * The current tracer is copied to avoid a global locking
2354 static void *s_start(struct seq_file *m, loff_t *pos)
2356 struct trace_iterator *iter = m->private;
2357 struct trace_array *tr = iter->tr;
2358 int cpu_file = iter->cpu_file;
2364 * copy the tracer to avoid using a global lock all around.
2365 * iter->trace is a copy of current_trace, the pointer to the
2366 * name may be used instead of a strcmp(), as iter->trace->name
2367 * will point to the same string as current_trace->name.
2369 mutex_lock(&trace_types_lock);
2370 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2371 *iter->trace = *tr->current_trace;
2372 mutex_unlock(&trace_types_lock);
2374 #ifdef CONFIG_TRACER_MAX_TRACE
2375 if (iter->snapshot && iter->trace->use_max_tr)
2376 return ERR_PTR(-EBUSY);
2379 if (!iter->snapshot)
2380 atomic_inc(&trace_record_cmdline_disabled);
2382 if (*pos != iter->pos) {
2387 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2388 for_each_tracing_cpu(cpu)
2389 tracing_iter_reset(iter, cpu);
2391 tracing_iter_reset(iter, cpu_file);
2394 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2399 * If we overflowed the seq_file before, then we want
2400 * to just reuse the trace_seq buffer again.
2406 p = s_next(m, p, &l);
2410 trace_event_read_lock();
2411 trace_access_lock(cpu_file);
2415 static void s_stop(struct seq_file *m, void *p)
2417 struct trace_iterator *iter = m->private;
2419 #ifdef CONFIG_TRACER_MAX_TRACE
2420 if (iter->snapshot && iter->trace->use_max_tr)
2424 if (!iter->snapshot)
2425 atomic_dec(&trace_record_cmdline_disabled);
2427 trace_access_unlock(iter->cpu_file);
2428 trace_event_read_unlock();
2432 get_total_entries(struct trace_buffer *buf,
2433 unsigned long *total, unsigned long *entries)
2435 unsigned long count;
2441 for_each_tracing_cpu(cpu) {
2442 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2444 * If this buffer has skipped entries, then we hold all
2445 * entries for the trace and we need to ignore the
2446 * ones before the time stamp.
2448 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2449 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2450 /* total is the same as the entries */
2454 ring_buffer_overrun_cpu(buf->buffer, cpu);
2459 static void print_lat_help_header(struct seq_file *m)
2461 seq_puts(m, "# _------=> CPU# \n");
2462 seq_puts(m, "# / _-----=> irqs-off \n");
2463 seq_puts(m, "# | / _----=> need-resched \n");
2464 seq_puts(m, "# || / _---=> hardirq/softirq \n");
2465 seq_puts(m, "# ||| / _--=> preempt-depth \n");
2466 seq_puts(m, "# |||| / delay \n");
2467 seq_puts(m, "# cmd pid ||||| time | caller \n");
2468 seq_puts(m, "# \\ / ||||| \\ | / \n");
2471 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2473 unsigned long total;
2474 unsigned long entries;
2476 get_total_entries(buf, &total, &entries);
2477 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2478 entries, total, num_online_cpus());
2482 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2484 print_event_info(buf, m);
2485 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
2486 seq_puts(m, "# | | | | |\n");
2489 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2491 print_event_info(buf, m);
2492 seq_puts(m, "# _-----=> irqs-off\n");
2493 seq_puts(m, "# / _----=> need-resched\n");
2494 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2495 seq_puts(m, "# || / _--=> preempt-depth\n");
2496 seq_puts(m, "# ||| / delay\n");
2497 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2498 seq_puts(m, "# | | | |||| | |\n");
2502 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2504 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2505 struct trace_buffer *buf = iter->trace_buffer;
2506 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2507 struct tracer *type = iter->trace;
2508 unsigned long entries;
2509 unsigned long total;
2510 const char *name = "preemption";
2514 get_total_entries(buf, &total, &entries);
2516 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2518 seq_puts(m, "# -----------------------------------"
2519 "---------------------------------\n");
2520 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2521 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2522 nsecs_to_usecs(data->saved_latency),
2526 #if defined(CONFIG_PREEMPT_NONE)
2528 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2530 #elif defined(CONFIG_PREEMPT)
2535 /* These are reserved for later use */
2538 seq_printf(m, " #P:%d)\n", num_online_cpus());
2542 seq_puts(m, "# -----------------\n");
2543 seq_printf(m, "# | task: %.16s-%d "
2544 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2545 data->comm, data->pid,
2546 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2547 data->policy, data->rt_priority);
2548 seq_puts(m, "# -----------------\n");
2550 if (data->critical_start) {
2551 seq_puts(m, "# => started at: ");
2552 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2553 trace_print_seq(m, &iter->seq);
2554 seq_puts(m, "\n# => ended at: ");
2555 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2556 trace_print_seq(m, &iter->seq);
2557 seq_puts(m, "\n#\n");
2563 static void test_cpu_buff_start(struct trace_iterator *iter)
2565 struct trace_seq *s = &iter->seq;
2567 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2570 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2573 if (cpumask_test_cpu(iter->cpu, iter->started))
2576 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2579 cpumask_set_cpu(iter->cpu, iter->started);
2581 /* Don't print started cpu buffer for the first entry of the trace */
2583 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2587 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2589 struct trace_seq *s = &iter->seq;
2590 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2591 struct trace_entry *entry;
2592 struct trace_event *event;
2596 test_cpu_buff_start(iter);
2598 event = ftrace_find_event(entry->type);
2600 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2601 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2602 if (!trace_print_lat_context(iter))
2605 if (!trace_print_context(iter))
2611 return event->funcs->trace(iter, sym_flags, event);
2613 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2616 return TRACE_TYPE_HANDLED;
2618 return TRACE_TYPE_PARTIAL_LINE;
2621 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2623 struct trace_seq *s = &iter->seq;
2624 struct trace_entry *entry;
2625 struct trace_event *event;
2629 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2630 if (!trace_seq_printf(s, "%d %d %llu ",
2631 entry->pid, iter->cpu, iter->ts))
2635 event = ftrace_find_event(entry->type);
2637 return event->funcs->raw(iter, 0, event);
2639 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2642 return TRACE_TYPE_HANDLED;
2644 return TRACE_TYPE_PARTIAL_LINE;
2647 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2649 struct trace_seq *s = &iter->seq;
2650 unsigned char newline = '\n';
2651 struct trace_entry *entry;
2652 struct trace_event *event;
2656 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2657 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2658 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2659 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2662 event = ftrace_find_event(entry->type);
2664 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2665 if (ret != TRACE_TYPE_HANDLED)
2669 SEQ_PUT_FIELD_RET(s, newline);
2671 return TRACE_TYPE_HANDLED;
2674 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2676 struct trace_seq *s = &iter->seq;
2677 struct trace_entry *entry;
2678 struct trace_event *event;
2682 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2683 SEQ_PUT_FIELD_RET(s, entry->pid);
2684 SEQ_PUT_FIELD_RET(s, iter->cpu);
2685 SEQ_PUT_FIELD_RET(s, iter->ts);
2688 event = ftrace_find_event(entry->type);
2689 return event ? event->funcs->binary(iter, 0, event) :
2693 int trace_empty(struct trace_iterator *iter)
2695 struct ring_buffer_iter *buf_iter;
2698 /* If we are looking at one CPU buffer, only check that one */
2699 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2700 cpu = iter->cpu_file;
2701 buf_iter = trace_buffer_iter(iter, cpu);
2703 if (!ring_buffer_iter_empty(buf_iter))
2706 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2712 for_each_tracing_cpu(cpu) {
2713 buf_iter = trace_buffer_iter(iter, cpu);
2715 if (!ring_buffer_iter_empty(buf_iter))
2718 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2726 /* Called with trace_event_read_lock() held. */
2727 enum print_line_t print_trace_line(struct trace_iterator *iter)
2729 enum print_line_t ret;
2731 if (iter->lost_events &&
2732 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2733 iter->cpu, iter->lost_events))
2734 return TRACE_TYPE_PARTIAL_LINE;
2736 if (iter->trace && iter->trace->print_line) {
2737 ret = iter->trace->print_line(iter);
2738 if (ret != TRACE_TYPE_UNHANDLED)
2742 if (iter->ent->type == TRACE_BPUTS &&
2743 trace_flags & TRACE_ITER_PRINTK &&
2744 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2745 return trace_print_bputs_msg_only(iter);
2747 if (iter->ent->type == TRACE_BPRINT &&
2748 trace_flags & TRACE_ITER_PRINTK &&
2749 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2750 return trace_print_bprintk_msg_only(iter);
2752 if (iter->ent->type == TRACE_PRINT &&
2753 trace_flags & TRACE_ITER_PRINTK &&
2754 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2755 return trace_print_printk_msg_only(iter);
2757 if (trace_flags & TRACE_ITER_BIN)
2758 return print_bin_fmt(iter);
2760 if (trace_flags & TRACE_ITER_HEX)
2761 return print_hex_fmt(iter);
2763 if (trace_flags & TRACE_ITER_RAW)
2764 return print_raw_fmt(iter);
2766 return print_trace_fmt(iter);
2769 void trace_latency_header(struct seq_file *m)
2771 struct trace_iterator *iter = m->private;
2773 /* print nothing if the buffers are empty */
2774 if (trace_empty(iter))
2777 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2778 print_trace_header(m, iter);
2780 if (!(trace_flags & TRACE_ITER_VERBOSE))
2781 print_lat_help_header(m);
2784 void trace_default_header(struct seq_file *m)
2786 struct trace_iterator *iter = m->private;
2788 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2791 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2792 /* print nothing if the buffers are empty */
2793 if (trace_empty(iter))
2795 print_trace_header(m, iter);
2796 if (!(trace_flags & TRACE_ITER_VERBOSE))
2797 print_lat_help_header(m);
2799 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2800 if (trace_flags & TRACE_ITER_IRQ_INFO)
2801 print_func_help_header_irq(iter->trace_buffer, m);
2803 print_func_help_header(iter->trace_buffer, m);
2808 static void test_ftrace_alive(struct seq_file *m)
2810 if (!ftrace_is_dead())
2812 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2813 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2816 #ifdef CONFIG_TRACER_MAX_TRACE
2817 static void show_snapshot_main_help(struct seq_file *m)
2819 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2820 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2821 seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2822 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n");
2823 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2824 seq_printf(m, "# is not a '0' or '1')\n");
2827 static void show_snapshot_percpu_help(struct seq_file *m)
2829 seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2830 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2831 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2832 seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n");
2834 seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2835 seq_printf(m, "# Must use main snapshot file to allocate.\n");
2837 seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2838 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2839 seq_printf(m, "# is not a '0' or '1')\n");
2842 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2844 if (iter->tr->allocated_snapshot)
2845 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2847 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2849 seq_printf(m, "# Snapshot commands:\n");
2850 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2851 show_snapshot_main_help(m);
2853 show_snapshot_percpu_help(m);
2856 /* Should never be called */
2857 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2860 static int s_show(struct seq_file *m, void *v)
2862 struct trace_iterator *iter = v;
2865 if (iter->ent == NULL) {
2867 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2869 test_ftrace_alive(m);
2871 if (iter->snapshot && trace_empty(iter))
2872 print_snapshot_help(m, iter);
2873 else if (iter->trace && iter->trace->print_header)
2874 iter->trace->print_header(m);
2876 trace_default_header(m);
2878 } else if (iter->leftover) {
2880 * If we filled the seq_file buffer earlier, we
2881 * want to just show it now.
2883 ret = trace_print_seq(m, &iter->seq);
2885 /* ret should this time be zero, but you never know */
2886 iter->leftover = ret;
2889 print_trace_line(iter);
2890 ret = trace_print_seq(m, &iter->seq);
2892 * If we overflow the seq_file buffer, then it will
2893 * ask us for this data again at start up.
2895 * ret is 0 if seq_file write succeeded.
2898 iter->leftover = ret;
2905 * Should be used after trace_array_get(), trace_types_lock
2906 * ensures that i_cdev was already initialized.
2908 static inline int tracing_get_cpu(struct inode *inode)
2910 if (inode->i_cdev) /* See trace_create_cpu_file() */
2911 return (long)inode->i_cdev - 1;
2912 return RING_BUFFER_ALL_CPUS;
2915 static const struct seq_operations tracer_seq_ops = {
2922 static struct trace_iterator *
2923 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2925 struct trace_array *tr = inode->i_private;
2926 struct trace_iterator *iter;
2929 if (tracing_disabled)
2930 return ERR_PTR(-ENODEV);
2932 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2934 return ERR_PTR(-ENOMEM);
2936 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2938 if (!iter->buffer_iter)
2942 * We make a copy of the current tracer to avoid concurrent
2943 * changes on it while we are reading.
2945 mutex_lock(&trace_types_lock);
2946 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2950 *iter->trace = *tr->current_trace;
2952 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2957 #ifdef CONFIG_TRACER_MAX_TRACE
2958 /* Currently only the top directory has a snapshot */
2959 if (tr->current_trace->print_max || snapshot)
2960 iter->trace_buffer = &tr->max_buffer;
2963 iter->trace_buffer = &tr->trace_buffer;
2964 iter->snapshot = snapshot;
2966 iter->cpu_file = tracing_get_cpu(inode);
2967 mutex_init(&iter->mutex);
2969 /* Notify the tracer early; before we stop tracing. */
2970 if (iter->trace && iter->trace->open)
2971 iter->trace->open(iter);
2973 /* Annotate start of buffers if we had overruns */
2974 if (ring_buffer_overruns(iter->trace_buffer->buffer))
2975 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2977 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2978 if (trace_clocks[tr->clock_id].in_ns)
2979 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2981 /* stop the trace while dumping if we are not opening "snapshot" */
2982 if (!iter->snapshot)
2983 tracing_stop_tr(tr);
2985 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
2986 for_each_tracing_cpu(cpu) {
2987 iter->buffer_iter[cpu] =
2988 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2990 ring_buffer_read_prepare_sync();
2991 for_each_tracing_cpu(cpu) {
2992 ring_buffer_read_start(iter->buffer_iter[cpu]);
2993 tracing_iter_reset(iter, cpu);
2996 cpu = iter->cpu_file;
2997 iter->buffer_iter[cpu] =
2998 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2999 ring_buffer_read_prepare_sync();
3000 ring_buffer_read_start(iter->buffer_iter[cpu]);
3001 tracing_iter_reset(iter, cpu);
3004 mutex_unlock(&trace_types_lock);
3009 mutex_unlock(&trace_types_lock);
3011 kfree(iter->buffer_iter);
3013 seq_release_private(inode, file);
3014 return ERR_PTR(-ENOMEM);
3017 int tracing_open_generic(struct inode *inode, struct file *filp)
3019 if (tracing_disabled)
3022 filp->private_data = inode->i_private;
3026 bool tracing_is_disabled(void)
3028 return (tracing_disabled) ? true: false;
3032 * Open and update trace_array ref count.
3033 * Must have the current trace_array passed to it.
3035 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3037 struct trace_array *tr = inode->i_private;
3039 if (tracing_disabled)
3042 if (trace_array_get(tr) < 0)
3045 filp->private_data = inode->i_private;
3050 static int tracing_release(struct inode *inode, struct file *file)
3052 struct trace_array *tr = inode->i_private;
3053 struct seq_file *m = file->private_data;
3054 struct trace_iterator *iter;
3057 if (!(file->f_mode & FMODE_READ)) {
3058 trace_array_put(tr);
3062 /* Writes do not use seq_file */
3064 mutex_lock(&trace_types_lock);
3066 for_each_tracing_cpu(cpu) {
3067 if (iter->buffer_iter[cpu])
3068 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3071 if (iter->trace && iter->trace->close)
3072 iter->trace->close(iter);
3074 if (!iter->snapshot)
3075 /* reenable tracing if it was previously enabled */
3076 tracing_start_tr(tr);
3078 __trace_array_put(tr);
3080 mutex_unlock(&trace_types_lock);
3082 mutex_destroy(&iter->mutex);
3083 free_cpumask_var(iter->started);
3085 kfree(iter->buffer_iter);
3086 seq_release_private(inode, file);
3091 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3093 struct trace_array *tr = inode->i_private;
3095 trace_array_put(tr);
3099 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3101 struct trace_array *tr = inode->i_private;
3103 trace_array_put(tr);
3105 return single_release(inode, file);
3108 static int tracing_open(struct inode *inode, struct file *file)
3110 struct trace_array *tr = inode->i_private;
3111 struct trace_iterator *iter;
3114 if (trace_array_get(tr) < 0)
3117 /* If this file was open for write, then erase contents */
3118 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3119 int cpu = tracing_get_cpu(inode);
3121 if (cpu == RING_BUFFER_ALL_CPUS)
3122 tracing_reset_online_cpus(&tr->trace_buffer);
3124 tracing_reset(&tr->trace_buffer, cpu);
3127 if (file->f_mode & FMODE_READ) {
3128 iter = __tracing_open(inode, file, false);
3130 ret = PTR_ERR(iter);
3131 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3132 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3136 trace_array_put(tr);
3142 t_next(struct seq_file *m, void *v, loff_t *pos)
3144 struct tracer *t = v;
3154 static void *t_start(struct seq_file *m, loff_t *pos)
3159 mutex_lock(&trace_types_lock);
3160 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
3166 static void t_stop(struct seq_file *m, void *p)
3168 mutex_unlock(&trace_types_lock);
3171 static int t_show(struct seq_file *m, void *v)
3173 struct tracer *t = v;
3178 seq_printf(m, "%s", t->name);
3187 static const struct seq_operations show_traces_seq_ops = {
3194 static int show_traces_open(struct inode *inode, struct file *file)
3196 if (tracing_disabled)
3199 return seq_open(file, &show_traces_seq_ops);
3203 tracing_write_stub(struct file *filp, const char __user *ubuf,
3204 size_t count, loff_t *ppos)
3209 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3213 if (file->f_mode & FMODE_READ)
3214 ret = seq_lseek(file, offset, whence);
3216 file->f_pos = ret = 0;
3221 static const struct file_operations tracing_fops = {
3222 .open = tracing_open,
3224 .write = tracing_write_stub,
3225 .llseek = tracing_lseek,
3226 .release = tracing_release,
3229 static const struct file_operations show_traces_fops = {
3230 .open = show_traces_open,
3232 .release = seq_release,
3233 .llseek = seq_lseek,
3237 * The tracer itself will not take this lock, but still we want
3238 * to provide a consistent cpumask to user-space:
3240 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3243 * Temporary storage for the character representation of the
3244 * CPU bitmask (and one more byte for the newline):
3246 static char mask_str[NR_CPUS + 1];
3249 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3250 size_t count, loff_t *ppos)
3252 struct trace_array *tr = file_inode(filp)->i_private;
3255 mutex_lock(&tracing_cpumask_update_lock);
3257 len = cpumask_scnprintf(mask_str, count, tr->tracing_cpumask);
3258 if (count - len < 2) {
3262 len += sprintf(mask_str + len, "\n");
3263 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3266 mutex_unlock(&tracing_cpumask_update_lock);
3272 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3273 size_t count, loff_t *ppos)
3275 struct trace_array *tr = file_inode(filp)->i_private;
3276 cpumask_var_t tracing_cpumask_new;
3279 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3282 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3286 mutex_lock(&tracing_cpumask_update_lock);
3288 local_irq_disable();
3289 arch_spin_lock(&ftrace_max_lock);
3290 for_each_tracing_cpu(cpu) {
3292 * Increase/decrease the disabled counter if we are
3293 * about to flip a bit in the cpumask:
3295 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3296 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3297 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3298 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3300 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3301 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3302 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3303 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3306 arch_spin_unlock(&ftrace_max_lock);
3309 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3311 mutex_unlock(&tracing_cpumask_update_lock);
3312 free_cpumask_var(tracing_cpumask_new);
3317 free_cpumask_var(tracing_cpumask_new);
3322 static const struct file_operations tracing_cpumask_fops = {
3323 .open = tracing_open_generic_tr,
3324 .read = tracing_cpumask_read,
3325 .write = tracing_cpumask_write,
3326 .release = tracing_release_generic_tr,
3327 .llseek = generic_file_llseek,
3330 static int tracing_trace_options_show(struct seq_file *m, void *v)
3332 struct tracer_opt *trace_opts;
3333 struct trace_array *tr = m->private;
3337 mutex_lock(&trace_types_lock);
3338 tracer_flags = tr->current_trace->flags->val;
3339 trace_opts = tr->current_trace->flags->opts;
3341 for (i = 0; trace_options[i]; i++) {
3342 if (trace_flags & (1 << i))
3343 seq_printf(m, "%s\n", trace_options[i]);
3345 seq_printf(m, "no%s\n", trace_options[i]);
3348 for (i = 0; trace_opts[i].name; i++) {
3349 if (tracer_flags & trace_opts[i].bit)
3350 seq_printf(m, "%s\n", trace_opts[i].name);
3352 seq_printf(m, "no%s\n", trace_opts[i].name);
3354 mutex_unlock(&trace_types_lock);
3359 static int __set_tracer_option(struct tracer *trace,
3360 struct tracer_flags *tracer_flags,
3361 struct tracer_opt *opts, int neg)
3365 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
3370 tracer_flags->val &= ~opts->bit;
3372 tracer_flags->val |= opts->bit;
3376 /* Try to assign a tracer specific option */
3377 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
3379 struct tracer_flags *tracer_flags = trace->flags;
3380 struct tracer_opt *opts = NULL;
3383 for (i = 0; tracer_flags->opts[i].name; i++) {
3384 opts = &tracer_flags->opts[i];
3386 if (strcmp(cmp, opts->name) == 0)
3387 return __set_tracer_option(trace, trace->flags,
3394 /* Some tracers require overwrite to stay enabled */
3395 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3397 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3403 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3405 /* do nothing if flag is already set */
3406 if (!!(trace_flags & mask) == !!enabled)
3409 /* Give the tracer a chance to approve the change */
3410 if (tr->current_trace->flag_changed)
3411 if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
3415 trace_flags |= mask;
3417 trace_flags &= ~mask;
3419 if (mask == TRACE_ITER_RECORD_CMD)
3420 trace_event_enable_cmd_record(enabled);
3422 if (mask == TRACE_ITER_OVERWRITE) {
3423 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3424 #ifdef CONFIG_TRACER_MAX_TRACE
3425 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3429 if (mask == TRACE_ITER_PRINTK)
3430 trace_printk_start_stop_comm(enabled);
3435 static int trace_set_options(struct trace_array *tr, char *option)
3442 cmp = strstrip(option);
3444 if (strncmp(cmp, "no", 2) == 0) {
3449 mutex_lock(&trace_types_lock);
3451 for (i = 0; trace_options[i]; i++) {
3452 if (strcmp(cmp, trace_options[i]) == 0) {
3453 ret = set_tracer_flag(tr, 1 << i, !neg);
3458 /* If no option could be set, test the specific tracer options */
3459 if (!trace_options[i])
3460 ret = set_tracer_option(tr->current_trace, cmp, neg);
3462 mutex_unlock(&trace_types_lock);
3468 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3469 size_t cnt, loff_t *ppos)
3471 struct seq_file *m = filp->private_data;
3472 struct trace_array *tr = m->private;
3476 if (cnt >= sizeof(buf))
3479 if (copy_from_user(&buf, ubuf, cnt))
3484 ret = trace_set_options(tr, buf);
3493 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3495 struct trace_array *tr = inode->i_private;
3498 if (tracing_disabled)
3501 if (trace_array_get(tr) < 0)
3504 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3506 trace_array_put(tr);
3511 static const struct file_operations tracing_iter_fops = {
3512 .open = tracing_trace_options_open,
3514 .llseek = seq_lseek,
3515 .release = tracing_single_release_tr,
3516 .write = tracing_trace_options_write,
3519 static const char readme_msg[] =
3520 "tracing mini-HOWTO:\n\n"
3521 "# echo 0 > tracing_on : quick way to disable tracing\n"
3522 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3523 " Important files:\n"
3524 " trace\t\t\t- The static contents of the buffer\n"
3525 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3526 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3527 " current_tracer\t- function and latency tracers\n"
3528 " available_tracers\t- list of configured tracers for current_tracer\n"
3529 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3530 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3531 " trace_clock\t\t-change the clock used to order events\n"
3532 " local: Per cpu clock but may not be synced across CPUs\n"
3533 " global: Synced across CPUs but slows tracing down.\n"
3534 " counter: Not a clock, but just an increment\n"
3535 " uptime: Jiffy counter from time of boot\n"
3536 " perf: Same clock that perf events use\n"
3537 #ifdef CONFIG_X86_64
3538 " x86-tsc: TSC cycle counter\n"
3540 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3541 " tracing_cpumask\t- Limit which CPUs to trace\n"
3542 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3543 "\t\t\t Remove sub-buffer with rmdir\n"
3544 " trace_options\t\t- Set format or modify how tracing happens\n"
3545 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3546 "\t\t\t option name\n"
3547 #ifdef CONFIG_DYNAMIC_FTRACE
3548 "\n available_filter_functions - list of functions that can be filtered on\n"
3549 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3550 "\t\t\t functions\n"
3551 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3552 "\t modules: Can select a group via module\n"
3553 "\t Format: :mod:<module-name>\n"
3554 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3555 "\t triggers: a command to perform when function is hit\n"
3556 "\t Format: <function>:<trigger>[:count]\n"
3557 "\t trigger: traceon, traceoff\n"
3558 "\t\t enable_event:<system>:<event>\n"
3559 "\t\t disable_event:<system>:<event>\n"
3560 #ifdef CONFIG_STACKTRACE
3563 #ifdef CONFIG_TRACER_SNAPSHOT
3566 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3567 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3568 "\t The first one will disable tracing every time do_fault is hit\n"
3569 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3570 "\t The first time do trap is hit and it disables tracing, the\n"
3571 "\t counter will decrement to 2. If tracing is already disabled,\n"
3572 "\t the counter will not decrement. It only decrements when the\n"
3573 "\t trigger did work\n"
3574 "\t To remove trigger without count:\n"
3575 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3576 "\t To remove trigger with a count:\n"
3577 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3578 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3579 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3580 "\t modules: Can select a group via module command :mod:\n"
3581 "\t Does not accept triggers\n"
3582 #endif /* CONFIG_DYNAMIC_FTRACE */
3583 #ifdef CONFIG_FUNCTION_TRACER
3584 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3587 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3588 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3589 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3591 #ifdef CONFIG_TRACER_SNAPSHOT
3592 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3593 "\t\t\t snapshot buffer. Read the contents for more\n"
3594 "\t\t\t information\n"
3596 #ifdef CONFIG_STACK_TRACER
3597 " stack_trace\t\t- Shows the max stack trace when active\n"
3598 " stack_max_size\t- Shows current max stack size that was traced\n"
3599 "\t\t\t Write into this file to reset the max size (trigger a\n"
3600 "\t\t\t new trace)\n"
3601 #ifdef CONFIG_DYNAMIC_FTRACE
3602 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3605 #endif /* CONFIG_STACK_TRACER */
3606 " events/\t\t- Directory containing all trace event subsystems:\n"
3607 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3608 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3609 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3611 " filter\t\t- If set, only events passing filter are traced\n"
3612 " events/<system>/<event>/\t- Directory containing control files for\n"
3614 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3615 " filter\t\t- If set, only events passing filter are traced\n"
3616 " trigger\t\t- If set, a command to perform when event is hit\n"
3617 "\t Format: <trigger>[:count][if <filter>]\n"
3618 "\t trigger: traceon, traceoff\n"
3619 "\t enable_event:<system>:<event>\n"
3620 "\t disable_event:<system>:<event>\n"
3621 #ifdef CONFIG_STACKTRACE
3624 #ifdef CONFIG_TRACER_SNAPSHOT
3627 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3628 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3629 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3630 "\t events/block/block_unplug/trigger\n"
3631 "\t The first disables tracing every time block_unplug is hit.\n"
3632 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3633 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3634 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3635 "\t Like function triggers, the counter is only decremented if it\n"
3636 "\t enabled or disabled tracing.\n"
3637 "\t To remove a trigger without a count:\n"
3638 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3639 "\t To remove a trigger with a count:\n"
3640 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3641 "\t Filters can be ignored when removing a trigger.\n"
3645 tracing_readme_read(struct file *filp, char __user *ubuf,
3646 size_t cnt, loff_t *ppos)
3648 return simple_read_from_buffer(ubuf, cnt, ppos,
3649 readme_msg, strlen(readme_msg));
3652 static const struct file_operations tracing_readme_fops = {
3653 .open = tracing_open_generic,
3654 .read = tracing_readme_read,
3655 .llseek = generic_file_llseek,
3659 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3660 size_t cnt, loff_t *ppos)
3669 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3673 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3681 for (i = 0; i < SAVED_CMDLINES; i++) {
3684 pid = map_cmdline_to_pid[i];
3685 if (pid == -1 || pid == NO_CMDLINE_MAP)
3688 trace_find_cmdline(pid, buf_comm);
3689 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3694 len = simple_read_from_buffer(ubuf, cnt, ppos,
3703 static const struct file_operations tracing_saved_cmdlines_fops = {
3704 .open = tracing_open_generic,
3705 .read = tracing_saved_cmdlines_read,
3706 .llseek = generic_file_llseek,
3710 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3711 size_t cnt, loff_t *ppos)
3713 struct trace_array *tr = filp->private_data;
3714 char buf[MAX_TRACER_SIZE+2];
3717 mutex_lock(&trace_types_lock);
3718 r = sprintf(buf, "%s\n", tr->current_trace->name);
3719 mutex_unlock(&trace_types_lock);
3721 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3724 int tracer_init(struct tracer *t, struct trace_array *tr)
3726 tracing_reset_online_cpus(&tr->trace_buffer);
3730 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3734 for_each_tracing_cpu(cpu)
3735 per_cpu_ptr(buf->data, cpu)->entries = val;
3738 #ifdef CONFIG_TRACER_MAX_TRACE
3739 /* resize @tr's buffer to the size of @size_tr's entries */
3740 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3741 struct trace_buffer *size_buf, int cpu_id)
3745 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3746 for_each_tracing_cpu(cpu) {
3747 ret = ring_buffer_resize(trace_buf->buffer,
3748 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3751 per_cpu_ptr(trace_buf->data, cpu)->entries =
3752 per_cpu_ptr(size_buf->data, cpu)->entries;
3755 ret = ring_buffer_resize(trace_buf->buffer,
3756 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3758 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3759 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3764 #endif /* CONFIG_TRACER_MAX_TRACE */
3766 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3767 unsigned long size, int cpu)
3772 * If kernel or user changes the size of the ring buffer
3773 * we use the size that was given, and we can forget about
3774 * expanding it later.
3776 ring_buffer_expanded = true;
3778 /* May be called before buffers are initialized */
3779 if (!tr->trace_buffer.buffer)
3782 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3786 #ifdef CONFIG_TRACER_MAX_TRACE
3787 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3788 !tr->current_trace->use_max_tr)
3791 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3793 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3794 &tr->trace_buffer, cpu);
3797 * AARGH! We are left with different
3798 * size max buffer!!!!
3799 * The max buffer is our "snapshot" buffer.
3800 * When a tracer needs a snapshot (one of the
3801 * latency tracers), it swaps the max buffer
3802 * with the saved snap shot. We succeeded to
3803 * update the size of the main buffer, but failed to
3804 * update the size of the max buffer. But when we tried
3805 * to reset the main buffer to the original size, we
3806 * failed there too. This is very unlikely to
3807 * happen, but if it does, warn and kill all
3811 tracing_disabled = 1;
3816 if (cpu == RING_BUFFER_ALL_CPUS)
3817 set_buffer_entries(&tr->max_buffer, size);
3819 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
3822 #endif /* CONFIG_TRACER_MAX_TRACE */
3824 if (cpu == RING_BUFFER_ALL_CPUS)
3825 set_buffer_entries(&tr->trace_buffer, size);
3827 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
3832 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3833 unsigned long size, int cpu_id)
3837 mutex_lock(&trace_types_lock);
3839 if (cpu_id != RING_BUFFER_ALL_CPUS) {
3840 /* make sure, this cpu is enabled in the mask */
3841 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3847 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
3852 mutex_unlock(&trace_types_lock);
3859 * tracing_update_buffers - used by tracing facility to expand ring buffers
3861 * To save on memory when the tracing is never used on a system with it
3862 * configured in. The ring buffers are set to a minimum size. But once
3863 * a user starts to use the tracing facility, then they need to grow
3864 * to their default size.
3866 * This function is to be called when a tracer is about to be used.
3868 int tracing_update_buffers(void)
3872 mutex_lock(&trace_types_lock);
3873 if (!ring_buffer_expanded)
3874 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
3875 RING_BUFFER_ALL_CPUS);
3876 mutex_unlock(&trace_types_lock);
3881 struct trace_option_dentry;
3883 static struct trace_option_dentry *
3884 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
3887 destroy_trace_option_files(struct trace_option_dentry *topts);
3889 static int tracing_set_tracer(const char *buf)
3891 static struct trace_option_dentry *topts;
3892 struct trace_array *tr = &global_trace;
3894 #ifdef CONFIG_TRACER_MAX_TRACE
3899 mutex_lock(&trace_types_lock);
3901 if (!ring_buffer_expanded) {
3902 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
3903 RING_BUFFER_ALL_CPUS);
3909 for (t = trace_types; t; t = t->next) {
3910 if (strcmp(t->name, buf) == 0)
3917 if (t == tr->current_trace)
3920 trace_branch_disable();
3922 tr->current_trace->enabled = false;
3924 if (tr->current_trace->reset)
3925 tr->current_trace->reset(tr);
3927 /* Current trace needs to be nop_trace before synchronize_sched */
3928 tr->current_trace = &nop_trace;
3930 #ifdef CONFIG_TRACER_MAX_TRACE
3931 had_max_tr = tr->allocated_snapshot;
3933 if (had_max_tr && !t->use_max_tr) {
3935 * We need to make sure that the update_max_tr sees that
3936 * current_trace changed to nop_trace to keep it from
3937 * swapping the buffers after we resize it.
3938 * The update_max_tr is called from interrupts disabled
3939 * so a synchronized_sched() is sufficient.
3941 synchronize_sched();
3945 destroy_trace_option_files(topts);
3947 topts = create_trace_option_files(tr, t);
3949 #ifdef CONFIG_TRACER_MAX_TRACE
3950 if (t->use_max_tr && !had_max_tr) {
3951 ret = alloc_snapshot(tr);
3958 ret = tracer_init(t, tr);
3963 tr->current_trace = t;
3964 tr->current_trace->enabled = true;
3965 trace_branch_enable(tr);
3967 mutex_unlock(&trace_types_lock);
3973 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3974 size_t cnt, loff_t *ppos)
3976 char buf[MAX_TRACER_SIZE+1];
3983 if (cnt > MAX_TRACER_SIZE)
3984 cnt = MAX_TRACER_SIZE;
3986 if (copy_from_user(&buf, ubuf, cnt))
3991 /* strip ending whitespace. */
3992 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3995 err = tracing_set_tracer(buf);
4005 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4006 size_t cnt, loff_t *ppos)
4008 unsigned long *ptr = filp->private_data;
4012 r = snprintf(buf, sizeof(buf), "%ld\n",
4013 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4014 if (r > sizeof(buf))
4016 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4020 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4021 size_t cnt, loff_t *ppos)
4023 unsigned long *ptr = filp->private_data;
4027 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4036 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4038 struct trace_array *tr = inode->i_private;
4039 struct trace_iterator *iter;
4042 if (tracing_disabled)
4045 if (trace_array_get(tr) < 0)
4048 mutex_lock(&trace_types_lock);
4050 /* create a buffer to store the information to pass to userspace */
4051 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4054 __trace_array_put(tr);
4059 * We make a copy of the current tracer to avoid concurrent
4060 * changes on it while we are reading.
4062 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
4067 *iter->trace = *tr->current_trace;
4069 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4074 /* trace pipe does not show start of buffer */
4075 cpumask_setall(iter->started);
4077 if (trace_flags & TRACE_ITER_LATENCY_FMT)
4078 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4080 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4081 if (trace_clocks[tr->clock_id].in_ns)
4082 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4085 iter->trace_buffer = &tr->trace_buffer;
4086 iter->cpu_file = tracing_get_cpu(inode);
4087 mutex_init(&iter->mutex);
4088 filp->private_data = iter;
4090 if (iter->trace->pipe_open)
4091 iter->trace->pipe_open(iter);
4093 nonseekable_open(inode, filp);
4095 mutex_unlock(&trace_types_lock);
4101 __trace_array_put(tr);
4102 mutex_unlock(&trace_types_lock);
4106 static int tracing_release_pipe(struct inode *inode, struct file *file)
4108 struct trace_iterator *iter = file->private_data;
4109 struct trace_array *tr = inode->i_private;
4111 mutex_lock(&trace_types_lock);
4113 if (iter->trace->pipe_close)
4114 iter->trace->pipe_close(iter);
4116 mutex_unlock(&trace_types_lock);
4118 free_cpumask_var(iter->started);
4119 mutex_destroy(&iter->mutex);
4123 trace_array_put(tr);
4129 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4131 /* Iterators are static, they should be filled or empty */
4132 if (trace_buffer_iter(iter, iter->cpu_file))
4133 return POLLIN | POLLRDNORM;
4135 if (trace_flags & TRACE_ITER_BLOCK)
4137 * Always select as readable when in blocking mode
4139 return POLLIN | POLLRDNORM;
4141 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4146 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4148 struct trace_iterator *iter = filp->private_data;
4150 return trace_poll(iter, filp, poll_table);
4154 * This is a make-shift waitqueue.
4155 * A tracer might use this callback on some rare cases:
4157 * 1) the current tracer might hold the runqueue lock when it wakes up
4158 * a reader, hence a deadlock (sched, function, and function graph tracers)
4159 * 2) the function tracers, trace all functions, we don't want
4160 * the overhead of calling wake_up and friends
4161 * (and tracing them too)
4163 * Anyway, this is really very primitive wakeup.
4165 void poll_wait_pipe(struct trace_iterator *iter)
4167 set_current_state(TASK_INTERRUPTIBLE);
4168 /* sleep for 100 msecs, and try again. */
4169 schedule_timeout(HZ / 10);
4172 /* Must be called with trace_types_lock mutex held. */
4173 static int tracing_wait_pipe(struct file *filp)
4175 struct trace_iterator *iter = filp->private_data;
4177 while (trace_empty(iter)) {
4179 if ((filp->f_flags & O_NONBLOCK)) {
4183 mutex_unlock(&iter->mutex);
4185 iter->trace->wait_pipe(iter);
4187 mutex_lock(&iter->mutex);
4189 if (signal_pending(current))
4193 * We block until we read something and tracing is disabled.
4194 * We still block if tracing is disabled, but we have never
4195 * read anything. This allows a user to cat this file, and
4196 * then enable tracing. But after we have read something,
4197 * we give an EOF when tracing is again disabled.
4199 * iter->pos will be 0 if we haven't read anything.
4201 if (!tracing_is_on() && iter->pos)
4212 tracing_read_pipe(struct file *filp, char __user *ubuf,
4213 size_t cnt, loff_t *ppos)
4215 struct trace_iterator *iter = filp->private_data;
4216 struct trace_array *tr = iter->tr;
4219 /* return any leftover data */
4220 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4224 trace_seq_init(&iter->seq);
4226 /* copy the tracer to avoid using a global lock all around */
4227 mutex_lock(&trace_types_lock);
4228 if (unlikely(iter->trace->name != tr->current_trace->name))
4229 *iter->trace = *tr->current_trace;
4230 mutex_unlock(&trace_types_lock);
4233 * Avoid more than one consumer on a single file descriptor
4234 * This is just a matter of traces coherency, the ring buffer itself
4237 mutex_lock(&iter->mutex);
4238 if (iter->trace->read) {
4239 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4245 sret = tracing_wait_pipe(filp);
4249 /* stop when tracing is finished */
4250 if (trace_empty(iter)) {
4255 if (cnt >= PAGE_SIZE)
4256 cnt = PAGE_SIZE - 1;
4258 /* reset all but tr, trace, and overruns */
4259 memset(&iter->seq, 0,
4260 sizeof(struct trace_iterator) -
4261 offsetof(struct trace_iterator, seq));
4262 cpumask_clear(iter->started);
4265 trace_event_read_lock();
4266 trace_access_lock(iter->cpu_file);
4267 while (trace_find_next_entry_inc(iter) != NULL) {
4268 enum print_line_t ret;
4269 int len = iter->seq.len;
4271 ret = print_trace_line(iter);
4272 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4273 /* don't print partial lines */
4274 iter->seq.len = len;
4277 if (ret != TRACE_TYPE_NO_CONSUME)
4278 trace_consume(iter);
4280 if (iter->seq.len >= cnt)
4284 * Setting the full flag means we reached the trace_seq buffer
4285 * size and we should leave by partial output condition above.
4286 * One of the trace_seq_* functions is not used properly.
4288 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4291 trace_access_unlock(iter->cpu_file);
4292 trace_event_read_unlock();
4294 /* Now copy what we have to the user */
4295 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4296 if (iter->seq.readpos >= iter->seq.len)
4297 trace_seq_init(&iter->seq);
4300 * If there was nothing to send to user, in spite of consuming trace
4301 * entries, go back to wait for more entries.
4307 mutex_unlock(&iter->mutex);
4312 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4315 __free_page(spd->pages[idx]);
4318 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4320 .map = generic_pipe_buf_map,
4321 .unmap = generic_pipe_buf_unmap,
4322 .confirm = generic_pipe_buf_confirm,
4323 .release = generic_pipe_buf_release,
4324 .steal = generic_pipe_buf_steal,
4325 .get = generic_pipe_buf_get,
4329 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4334 /* Seq buffer is page-sized, exactly what we need. */
4336 count = iter->seq.len;
4337 ret = print_trace_line(iter);
4338 count = iter->seq.len - count;
4341 iter->seq.len -= count;
4344 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4345 iter->seq.len -= count;
4349 if (ret != TRACE_TYPE_NO_CONSUME)
4350 trace_consume(iter);
4352 if (!trace_find_next_entry_inc(iter)) {
4362 static ssize_t tracing_splice_read_pipe(struct file *filp,
4364 struct pipe_inode_info *pipe,
4368 struct page *pages_def[PIPE_DEF_BUFFERS];
4369 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4370 struct trace_iterator *iter = filp->private_data;
4371 struct splice_pipe_desc spd = {
4373 .partial = partial_def,
4374 .nr_pages = 0, /* This gets updated below. */
4375 .nr_pages_max = PIPE_DEF_BUFFERS,
4377 .ops = &tracing_pipe_buf_ops,
4378 .spd_release = tracing_spd_release_pipe,
4380 struct trace_array *tr = iter->tr;
4385 if (splice_grow_spd(pipe, &spd))
4388 /* copy the tracer to avoid using a global lock all around */
4389 mutex_lock(&trace_types_lock);
4390 if (unlikely(iter->trace->name != tr->current_trace->name))
4391 *iter->trace = *tr->current_trace;
4392 mutex_unlock(&trace_types_lock);
4394 mutex_lock(&iter->mutex);
4396 if (iter->trace->splice_read) {
4397 ret = iter->trace->splice_read(iter, filp,
4398 ppos, pipe, len, flags);
4403 ret = tracing_wait_pipe(filp);
4407 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4412 trace_event_read_lock();
4413 trace_access_lock(iter->cpu_file);
4415 /* Fill as many pages as possible. */
4416 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
4417 spd.pages[i] = alloc_page(GFP_KERNEL);
4421 rem = tracing_fill_pipe_page(rem, iter);
4423 /* Copy the data into the page, so we can start over. */
4424 ret = trace_seq_to_buffer(&iter->seq,
4425 page_address(spd.pages[i]),
4428 __free_page(spd.pages[i]);
4431 spd.partial[i].offset = 0;
4432 spd.partial[i].len = iter->seq.len;
4434 trace_seq_init(&iter->seq);
4437 trace_access_unlock(iter->cpu_file);
4438 trace_event_read_unlock();
4439 mutex_unlock(&iter->mutex);
4443 ret = splice_to_pipe(pipe, &spd);
4445 splice_shrink_spd(&spd);
4449 mutex_unlock(&iter->mutex);
4454 tracing_entries_read(struct file *filp, char __user *ubuf,
4455 size_t cnt, loff_t *ppos)
4457 struct inode *inode = file_inode(filp);
4458 struct trace_array *tr = inode->i_private;
4459 int cpu = tracing_get_cpu(inode);
4464 mutex_lock(&trace_types_lock);
4466 if (cpu == RING_BUFFER_ALL_CPUS) {
4467 int cpu, buf_size_same;
4472 /* check if all cpu sizes are same */
4473 for_each_tracing_cpu(cpu) {
4474 /* fill in the size from first enabled cpu */
4476 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4477 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4483 if (buf_size_same) {
4484 if (!ring_buffer_expanded)
4485 r = sprintf(buf, "%lu (expanded: %lu)\n",
4487 trace_buf_size >> 10);
4489 r = sprintf(buf, "%lu\n", size >> 10);
4491 r = sprintf(buf, "X\n");
4493 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4495 mutex_unlock(&trace_types_lock);
4497 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4502 tracing_entries_write(struct file *filp, const char __user *ubuf,
4503 size_t cnt, loff_t *ppos)
4505 struct inode *inode = file_inode(filp);
4506 struct trace_array *tr = inode->i_private;
4510 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4514 /* must have at least 1 entry */
4518 /* value is in KB */
4520 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4530 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4531 size_t cnt, loff_t *ppos)
4533 struct trace_array *tr = filp->private_data;
4536 unsigned long size = 0, expanded_size = 0;
4538 mutex_lock(&trace_types_lock);
4539 for_each_tracing_cpu(cpu) {
4540 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4541 if (!ring_buffer_expanded)
4542 expanded_size += trace_buf_size >> 10;
4544 if (ring_buffer_expanded)
4545 r = sprintf(buf, "%lu\n", size);
4547 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4548 mutex_unlock(&trace_types_lock);
4550 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4554 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4555 size_t cnt, loff_t *ppos)
4558 * There is no need to read what the user has written, this function
4559 * is just to make sure that there is no error when "echo" is used
4568 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4570 struct trace_array *tr = inode->i_private;
4572 /* disable tracing ? */
4573 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4574 tracer_tracing_off(tr);
4575 /* resize the ring buffer to 0 */
4576 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4578 trace_array_put(tr);
4584 tracing_mark_write(struct file *filp, const char __user *ubuf,
4585 size_t cnt, loff_t *fpos)
4587 unsigned long addr = (unsigned long)ubuf;
4588 struct trace_array *tr = filp->private_data;
4589 struct ring_buffer_event *event;
4590 struct ring_buffer *buffer;
4591 struct print_entry *entry;
4592 unsigned long irq_flags;
4593 struct page *pages[2];
4603 if (tracing_disabled)
4606 if (!(trace_flags & TRACE_ITER_MARKERS))
4609 if (cnt > TRACE_BUF_SIZE)
4610 cnt = TRACE_BUF_SIZE;
4613 * Userspace is injecting traces into the kernel trace buffer.
4614 * We want to be as non intrusive as possible.
4615 * To do so, we do not want to allocate any special buffers
4616 * or take any locks, but instead write the userspace data
4617 * straight into the ring buffer.
4619 * First we need to pin the userspace buffer into memory,
4620 * which, most likely it is, because it just referenced it.
4621 * But there's no guarantee that it is. By using get_user_pages_fast()
4622 * and kmap_atomic/kunmap_atomic() we can get access to the
4623 * pages directly. We then write the data directly into the
4626 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4628 /* check if we cross pages */
4629 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4632 offset = addr & (PAGE_SIZE - 1);
4635 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4636 if (ret < nr_pages) {
4638 put_page(pages[ret]);
4643 for (i = 0; i < nr_pages; i++)
4644 map_page[i] = kmap_atomic(pages[i]);
4646 local_save_flags(irq_flags);
4647 size = sizeof(*entry) + cnt + 2; /* possible \n added */
4648 buffer = tr->trace_buffer.buffer;
4649 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4650 irq_flags, preempt_count());
4652 /* Ring buffer disabled, return as if not open for write */
4657 entry = ring_buffer_event_data(event);
4658 entry->ip = _THIS_IP_;
4660 if (nr_pages == 2) {
4661 len = PAGE_SIZE - offset;
4662 memcpy(&entry->buf, map_page[0] + offset, len);
4663 memcpy(&entry->buf[len], map_page[1], cnt - len);
4665 memcpy(&entry->buf, map_page[0] + offset, cnt);
4667 if (entry->buf[cnt - 1] != '\n') {
4668 entry->buf[cnt] = '\n';
4669 entry->buf[cnt + 1] = '\0';
4671 entry->buf[cnt] = '\0';
4673 __buffer_unlock_commit(buffer, event);
4680 for (i = 0; i < nr_pages; i++){
4681 kunmap_atomic(map_page[i]);
4688 static int tracing_clock_show(struct seq_file *m, void *v)
4690 struct trace_array *tr = m->private;
4693 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4695 "%s%s%s%s", i ? " " : "",
4696 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4697 i == tr->clock_id ? "]" : "");
4703 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4704 size_t cnt, loff_t *fpos)
4706 struct seq_file *m = filp->private_data;
4707 struct trace_array *tr = m->private;
4709 const char *clockstr;
4712 if (cnt >= sizeof(buf))
4715 if (copy_from_user(&buf, ubuf, cnt))
4720 clockstr = strstrip(buf);
4722 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4723 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4726 if (i == ARRAY_SIZE(trace_clocks))
4729 mutex_lock(&trace_types_lock);
4733 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4736 * New clock may not be consistent with the previous clock.
4737 * Reset the buffer so that it doesn't have incomparable timestamps.
4739 tracing_reset_online_cpus(&tr->trace_buffer);
4741 #ifdef CONFIG_TRACER_MAX_TRACE
4742 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4743 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4744 tracing_reset_online_cpus(&tr->max_buffer);
4747 mutex_unlock(&trace_types_lock);
4754 static int tracing_clock_open(struct inode *inode, struct file *file)
4756 struct trace_array *tr = inode->i_private;
4759 if (tracing_disabled)
4762 if (trace_array_get(tr))
4765 ret = single_open(file, tracing_clock_show, inode->i_private);
4767 trace_array_put(tr);
4772 struct ftrace_buffer_info {
4773 struct trace_iterator iter;
4778 #ifdef CONFIG_TRACER_SNAPSHOT
4779 static int tracing_snapshot_open(struct inode *inode, struct file *file)
4781 struct trace_array *tr = inode->i_private;
4782 struct trace_iterator *iter;
4786 if (trace_array_get(tr) < 0)
4789 if (file->f_mode & FMODE_READ) {
4790 iter = __tracing_open(inode, file, true);
4792 ret = PTR_ERR(iter);
4794 /* Writes still need the seq_file to hold the private data */
4796 m = kzalloc(sizeof(*m), GFP_KERNEL);
4799 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4807 iter->trace_buffer = &tr->max_buffer;
4808 iter->cpu_file = tracing_get_cpu(inode);
4810 file->private_data = m;
4814 trace_array_put(tr);
4820 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4823 struct seq_file *m = filp->private_data;
4824 struct trace_iterator *iter = m->private;
4825 struct trace_array *tr = iter->tr;
4829 ret = tracing_update_buffers();
4833 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4837 mutex_lock(&trace_types_lock);
4839 if (tr->current_trace->use_max_tr) {
4846 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4850 if (tr->allocated_snapshot)
4854 /* Only allow per-cpu swap if the ring buffer supports it */
4855 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4856 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4861 if (!tr->allocated_snapshot) {
4862 ret = alloc_snapshot(tr);
4866 local_irq_disable();
4867 /* Now, we're going to swap */
4868 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4869 update_max_tr(tr, current, smp_processor_id());
4871 update_max_tr_single(tr, current, iter->cpu_file);
4875 if (tr->allocated_snapshot) {
4876 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4877 tracing_reset_online_cpus(&tr->max_buffer);
4879 tracing_reset(&tr->max_buffer, iter->cpu_file);
4889 mutex_unlock(&trace_types_lock);
4893 static int tracing_snapshot_release(struct inode *inode, struct file *file)
4895 struct seq_file *m = file->private_data;
4898 ret = tracing_release(inode, file);
4900 if (file->f_mode & FMODE_READ)
4903 /* If write only, the seq_file is just a stub */
4911 static int tracing_buffers_open(struct inode *inode, struct file *filp);
4912 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
4913 size_t count, loff_t *ppos);
4914 static int tracing_buffers_release(struct inode *inode, struct file *file);
4915 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4916 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
4918 static int snapshot_raw_open(struct inode *inode, struct file *filp)
4920 struct ftrace_buffer_info *info;
4923 ret = tracing_buffers_open(inode, filp);
4927 info = filp->private_data;
4929 if (info->iter.trace->use_max_tr) {
4930 tracing_buffers_release(inode, filp);
4934 info->iter.snapshot = true;
4935 info->iter.trace_buffer = &info->iter.tr->max_buffer;
4940 #endif /* CONFIG_TRACER_SNAPSHOT */
4943 static const struct file_operations tracing_max_lat_fops = {
4944 .open = tracing_open_generic,
4945 .read = tracing_max_lat_read,
4946 .write = tracing_max_lat_write,
4947 .llseek = generic_file_llseek,
4950 static const struct file_operations set_tracer_fops = {
4951 .open = tracing_open_generic,
4952 .read = tracing_set_trace_read,
4953 .write = tracing_set_trace_write,
4954 .llseek = generic_file_llseek,
4957 static const struct file_operations tracing_pipe_fops = {
4958 .open = tracing_open_pipe,
4959 .poll = tracing_poll_pipe,
4960 .read = tracing_read_pipe,
4961 .splice_read = tracing_splice_read_pipe,
4962 .release = tracing_release_pipe,
4963 .llseek = no_llseek,
4966 static const struct file_operations tracing_entries_fops = {
4967 .open = tracing_open_generic_tr,
4968 .read = tracing_entries_read,
4969 .write = tracing_entries_write,
4970 .llseek = generic_file_llseek,
4971 .release = tracing_release_generic_tr,
4974 static const struct file_operations tracing_total_entries_fops = {
4975 .open = tracing_open_generic_tr,
4976 .read = tracing_total_entries_read,
4977 .llseek = generic_file_llseek,
4978 .release = tracing_release_generic_tr,
4981 static const struct file_operations tracing_free_buffer_fops = {
4982 .open = tracing_open_generic_tr,
4983 .write = tracing_free_buffer_write,
4984 .release = tracing_free_buffer_release,
4987 static const struct file_operations tracing_mark_fops = {
4988 .open = tracing_open_generic_tr,
4989 .write = tracing_mark_write,
4990 .llseek = generic_file_llseek,
4991 .release = tracing_release_generic_tr,
4994 static const struct file_operations trace_clock_fops = {
4995 .open = tracing_clock_open,
4997 .llseek = seq_lseek,
4998 .release = tracing_single_release_tr,
4999 .write = tracing_clock_write,
5002 #ifdef CONFIG_TRACER_SNAPSHOT
5003 static const struct file_operations snapshot_fops = {
5004 .open = tracing_snapshot_open,
5006 .write = tracing_snapshot_write,
5007 .llseek = tracing_lseek,
5008 .release = tracing_snapshot_release,
5011 static const struct file_operations snapshot_raw_fops = {
5012 .open = snapshot_raw_open,
5013 .read = tracing_buffers_read,
5014 .release = tracing_buffers_release,
5015 .splice_read = tracing_buffers_splice_read,
5016 .llseek = no_llseek,
5019 #endif /* CONFIG_TRACER_SNAPSHOT */
5021 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5023 struct trace_array *tr = inode->i_private;
5024 struct ftrace_buffer_info *info;
5027 if (tracing_disabled)
5030 if (trace_array_get(tr) < 0)
5033 info = kzalloc(sizeof(*info), GFP_KERNEL);
5035 trace_array_put(tr);
5039 mutex_lock(&trace_types_lock);
5042 info->iter.cpu_file = tracing_get_cpu(inode);
5043 info->iter.trace = tr->current_trace;
5044 info->iter.trace_buffer = &tr->trace_buffer;
5046 /* Force reading ring buffer for first read */
5047 info->read = (unsigned int)-1;
5049 filp->private_data = info;
5051 mutex_unlock(&trace_types_lock);
5053 ret = nonseekable_open(inode, filp);
5055 trace_array_put(tr);
5061 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5063 struct ftrace_buffer_info *info = filp->private_data;
5064 struct trace_iterator *iter = &info->iter;
5066 return trace_poll(iter, filp, poll_table);
5070 tracing_buffers_read(struct file *filp, char __user *ubuf,
5071 size_t count, loff_t *ppos)
5073 struct ftrace_buffer_info *info = filp->private_data;
5074 struct trace_iterator *iter = &info->iter;
5081 mutex_lock(&trace_types_lock);
5083 #ifdef CONFIG_TRACER_MAX_TRACE
5084 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5091 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5097 /* Do we have previous read data to read? */
5098 if (info->read < PAGE_SIZE)
5102 trace_access_lock(iter->cpu_file);
5103 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5107 trace_access_unlock(iter->cpu_file);
5110 if (trace_empty(iter)) {
5111 if ((filp->f_flags & O_NONBLOCK)) {
5115 mutex_unlock(&trace_types_lock);
5116 iter->trace->wait_pipe(iter);
5117 mutex_lock(&trace_types_lock);
5118 if (signal_pending(current)) {
5130 size = PAGE_SIZE - info->read;
5134 ret = copy_to_user(ubuf, info->spare + info->read, size);
5145 mutex_unlock(&trace_types_lock);
5150 static int tracing_buffers_release(struct inode *inode, struct file *file)
5152 struct ftrace_buffer_info *info = file->private_data;
5153 struct trace_iterator *iter = &info->iter;
5155 mutex_lock(&trace_types_lock);
5157 __trace_array_put(iter->tr);
5160 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5163 mutex_unlock(&trace_types_lock);
5169 struct ring_buffer *buffer;
5174 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5175 struct pipe_buffer *buf)
5177 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5182 ring_buffer_free_read_page(ref->buffer, ref->page);
5187 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5188 struct pipe_buffer *buf)
5190 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5195 /* Pipe buffer operations for a buffer. */
5196 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5198 .map = generic_pipe_buf_map,
5199 .unmap = generic_pipe_buf_unmap,
5200 .confirm = generic_pipe_buf_confirm,
5201 .release = buffer_pipe_buf_release,
5202 .steal = generic_pipe_buf_steal,
5203 .get = buffer_pipe_buf_get,
5207 * Callback from splice_to_pipe(), if we need to release some pages
5208 * at the end of the spd in case we error'ed out in filling the pipe.
5210 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5212 struct buffer_ref *ref =
5213 (struct buffer_ref *)spd->partial[i].private;
5218 ring_buffer_free_read_page(ref->buffer, ref->page);
5220 spd->partial[i].private = 0;
5224 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5225 struct pipe_inode_info *pipe, size_t len,
5228 struct ftrace_buffer_info *info = file->private_data;
5229 struct trace_iterator *iter = &info->iter;
5230 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5231 struct page *pages_def[PIPE_DEF_BUFFERS];
5232 struct splice_pipe_desc spd = {
5234 .partial = partial_def,
5235 .nr_pages_max = PIPE_DEF_BUFFERS,
5237 .ops = &buffer_pipe_buf_ops,
5238 .spd_release = buffer_spd_release,
5240 struct buffer_ref *ref;
5241 int entries, size, i;
5244 mutex_lock(&trace_types_lock);
5246 #ifdef CONFIG_TRACER_MAX_TRACE
5247 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5253 if (splice_grow_spd(pipe, &spd)) {
5258 if (*ppos & (PAGE_SIZE - 1)) {
5263 if (len & (PAGE_SIZE - 1)) {
5264 if (len < PAGE_SIZE) {
5272 trace_access_lock(iter->cpu_file);
5273 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5275 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
5279 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5284 ref->buffer = iter->trace_buffer->buffer;
5285 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5291 r = ring_buffer_read_page(ref->buffer, &ref->page,
5292 len, iter->cpu_file, 1);
5294 ring_buffer_free_read_page(ref->buffer, ref->page);
5300 * zero out any left over data, this is going to
5303 size = ring_buffer_page_len(ref->page);
5304 if (size < PAGE_SIZE)
5305 memset(ref->page + size, 0, PAGE_SIZE - size);
5307 page = virt_to_page(ref->page);
5309 spd.pages[i] = page;
5310 spd.partial[i].len = PAGE_SIZE;
5311 spd.partial[i].offset = 0;
5312 spd.partial[i].private = (unsigned long)ref;
5316 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5319 trace_access_unlock(iter->cpu_file);
5322 /* did we read anything? */
5323 if (!spd.nr_pages) {
5324 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5328 mutex_unlock(&trace_types_lock);
5329 iter->trace->wait_pipe(iter);
5330 mutex_lock(&trace_types_lock);
5331 if (signal_pending(current)) {
5338 ret = splice_to_pipe(pipe, &spd);
5339 splice_shrink_spd(&spd);
5341 mutex_unlock(&trace_types_lock);
5346 static const struct file_operations tracing_buffers_fops = {
5347 .open = tracing_buffers_open,
5348 .read = tracing_buffers_read,
5349 .poll = tracing_buffers_poll,
5350 .release = tracing_buffers_release,
5351 .splice_read = tracing_buffers_splice_read,
5352 .llseek = no_llseek,
5356 tracing_stats_read(struct file *filp, char __user *ubuf,
5357 size_t count, loff_t *ppos)
5359 struct inode *inode = file_inode(filp);
5360 struct trace_array *tr = inode->i_private;
5361 struct trace_buffer *trace_buf = &tr->trace_buffer;
5362 int cpu = tracing_get_cpu(inode);
5363 struct trace_seq *s;
5365 unsigned long long t;
5366 unsigned long usec_rem;
5368 s = kmalloc(sizeof(*s), GFP_KERNEL);
5374 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5375 trace_seq_printf(s, "entries: %ld\n", cnt);
5377 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5378 trace_seq_printf(s, "overrun: %ld\n", cnt);
5380 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5381 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5383 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5384 trace_seq_printf(s, "bytes: %ld\n", cnt);
5386 if (trace_clocks[tr->clock_id].in_ns) {
5387 /* local or global for trace_clock */
5388 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5389 usec_rem = do_div(t, USEC_PER_SEC);
5390 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5393 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5394 usec_rem = do_div(t, USEC_PER_SEC);
5395 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5397 /* counter or tsc mode for trace_clock */
5398 trace_seq_printf(s, "oldest event ts: %llu\n",
5399 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5401 trace_seq_printf(s, "now ts: %llu\n",
5402 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5405 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5406 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5408 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5409 trace_seq_printf(s, "read events: %ld\n", cnt);
5411 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5418 static const struct file_operations tracing_stats_fops = {
5419 .open = tracing_open_generic_tr,
5420 .read = tracing_stats_read,
5421 .llseek = generic_file_llseek,
5422 .release = tracing_release_generic_tr,
5425 #ifdef CONFIG_DYNAMIC_FTRACE
5427 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5433 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5434 size_t cnt, loff_t *ppos)
5436 static char ftrace_dyn_info_buffer[1024];
5437 static DEFINE_MUTEX(dyn_info_mutex);
5438 unsigned long *p = filp->private_data;
5439 char *buf = ftrace_dyn_info_buffer;
5440 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5443 mutex_lock(&dyn_info_mutex);
5444 r = sprintf(buf, "%ld ", *p);
5446 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5449 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5451 mutex_unlock(&dyn_info_mutex);
5456 static const struct file_operations tracing_dyn_info_fops = {
5457 .open = tracing_open_generic,
5458 .read = tracing_read_dyn_info,
5459 .llseek = generic_file_llseek,
5461 #endif /* CONFIG_DYNAMIC_FTRACE */
5463 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5465 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5471 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5473 unsigned long *count = (long *)data;
5485 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5486 struct ftrace_probe_ops *ops, void *data)
5488 long count = (long)data;
5490 seq_printf(m, "%ps:", (void *)ip);
5492 seq_printf(m, "snapshot");
5495 seq_printf(m, ":unlimited\n");
5497 seq_printf(m, ":count=%ld\n", count);
5502 static struct ftrace_probe_ops snapshot_probe_ops = {
5503 .func = ftrace_snapshot,
5504 .print = ftrace_snapshot_print,
5507 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5508 .func = ftrace_count_snapshot,
5509 .print = ftrace_snapshot_print,
5513 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5514 char *glob, char *cmd, char *param, int enable)
5516 struct ftrace_probe_ops *ops;
5517 void *count = (void *)-1;
5521 /* hash funcs only work with set_ftrace_filter */
5525 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5527 if (glob[0] == '!') {
5528 unregister_ftrace_function_probe_func(glob+1, ops);
5535 number = strsep(¶m, ":");
5537 if (!strlen(number))
5541 * We use the callback data field (which is a pointer)
5544 ret = kstrtoul(number, 0, (unsigned long *)&count);
5549 ret = register_ftrace_function_probe(glob, ops, count);
5552 alloc_snapshot(&global_trace);
5554 return ret < 0 ? ret : 0;
5557 static struct ftrace_func_command ftrace_snapshot_cmd = {
5559 .func = ftrace_trace_snapshot_callback,
5562 static __init int register_snapshot_cmd(void)
5564 return register_ftrace_command(&ftrace_snapshot_cmd);
5567 static inline __init int register_snapshot_cmd(void) { return 0; }
5568 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5570 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5575 if (!debugfs_initialized())
5578 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5579 tr->dir = debugfs_create_dir("tracing", NULL);
5582 pr_warn_once("Could not create debugfs directory 'tracing'\n");
5587 struct dentry *tracing_init_dentry(void)
5589 return tracing_init_dentry_tr(&global_trace);
5592 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5594 struct dentry *d_tracer;
5597 return tr->percpu_dir;
5599 d_tracer = tracing_init_dentry_tr(tr);
5603 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5605 WARN_ONCE(!tr->percpu_dir,
5606 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5608 return tr->percpu_dir;
5611 static struct dentry *
5612 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
5613 void *data, long cpu, const struct file_operations *fops)
5615 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5617 if (ret) /* See tracing_get_cpu() */
5618 ret->d_inode->i_cdev = (void *)(cpu + 1);
5623 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5625 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5626 struct dentry *d_cpu;
5627 char cpu_dir[30]; /* 30 characters should be more than enough */
5632 snprintf(cpu_dir, 30, "cpu%ld", cpu);
5633 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5635 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5639 /* per cpu trace_pipe */
5640 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5641 tr, cpu, &tracing_pipe_fops);
5644 trace_create_cpu_file("trace", 0644, d_cpu,
5645 tr, cpu, &tracing_fops);
5647 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5648 tr, cpu, &tracing_buffers_fops);
5650 trace_create_cpu_file("stats", 0444, d_cpu,
5651 tr, cpu, &tracing_stats_fops);
5653 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5654 tr, cpu, &tracing_entries_fops);
5656 #ifdef CONFIG_TRACER_SNAPSHOT
5657 trace_create_cpu_file("snapshot", 0644, d_cpu,
5658 tr, cpu, &snapshot_fops);
5660 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5661 tr, cpu, &snapshot_raw_fops);
5665 #ifdef CONFIG_FTRACE_SELFTEST
5666 /* Let selftest have access to static functions in this file */
5667 #include "trace_selftest.c"
5670 struct trace_option_dentry {
5671 struct tracer_opt *opt;
5672 struct tracer_flags *flags;
5673 struct trace_array *tr;
5674 struct dentry *entry;
5678 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5681 struct trace_option_dentry *topt = filp->private_data;
5684 if (topt->flags->val & topt->opt->bit)
5689 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5693 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5696 struct trace_option_dentry *topt = filp->private_data;
5700 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5704 if (val != 0 && val != 1)
5707 if (!!(topt->flags->val & topt->opt->bit) != val) {
5708 mutex_lock(&trace_types_lock);
5709 ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
5711 mutex_unlock(&trace_types_lock);
5722 static const struct file_operations trace_options_fops = {
5723 .open = tracing_open_generic,
5724 .read = trace_options_read,
5725 .write = trace_options_write,
5726 .llseek = generic_file_llseek,
5730 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5733 long index = (long)filp->private_data;
5736 if (trace_flags & (1 << index))
5741 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5745 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5748 struct trace_array *tr = &global_trace;
5749 long index = (long)filp->private_data;
5753 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5757 if (val != 0 && val != 1)
5760 mutex_lock(&trace_types_lock);
5761 ret = set_tracer_flag(tr, 1 << index, val);
5762 mutex_unlock(&trace_types_lock);
5772 static const struct file_operations trace_options_core_fops = {
5773 .open = tracing_open_generic,
5774 .read = trace_options_core_read,
5775 .write = trace_options_core_write,
5776 .llseek = generic_file_llseek,
5779 struct dentry *trace_create_file(const char *name,
5781 struct dentry *parent,
5783 const struct file_operations *fops)
5787 ret = debugfs_create_file(name, mode, parent, data, fops);
5789 pr_warning("Could not create debugfs '%s' entry\n", name);
5795 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
5797 struct dentry *d_tracer;
5802 d_tracer = tracing_init_dentry_tr(tr);
5806 tr->options = debugfs_create_dir("options", d_tracer);
5808 pr_warning("Could not create debugfs directory 'options'\n");
5816 create_trace_option_file(struct trace_array *tr,
5817 struct trace_option_dentry *topt,
5818 struct tracer_flags *flags,
5819 struct tracer_opt *opt)
5821 struct dentry *t_options;
5823 t_options = trace_options_init_dentry(tr);
5827 topt->flags = flags;
5831 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
5832 &trace_options_fops);
5836 static struct trace_option_dentry *
5837 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
5839 struct trace_option_dentry *topts;
5840 struct tracer_flags *flags;
5841 struct tracer_opt *opts;
5847 flags = tracer->flags;
5849 if (!flags || !flags->opts)
5854 for (cnt = 0; opts[cnt].name; cnt++)
5857 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
5861 for (cnt = 0; opts[cnt].name; cnt++)
5862 create_trace_option_file(tr, &topts[cnt], flags,
5869 destroy_trace_option_files(struct trace_option_dentry *topts)
5876 for (cnt = 0; topts[cnt].opt; cnt++) {
5877 if (topts[cnt].entry)
5878 debugfs_remove(topts[cnt].entry);
5884 static struct dentry *
5885 create_trace_option_core_file(struct trace_array *tr,
5886 const char *option, long index)
5888 struct dentry *t_options;
5890 t_options = trace_options_init_dentry(tr);
5894 return trace_create_file(option, 0644, t_options, (void *)index,
5895 &trace_options_core_fops);
5898 static __init void create_trace_options_dir(struct trace_array *tr)
5900 struct dentry *t_options;
5903 t_options = trace_options_init_dentry(tr);
5907 for (i = 0; trace_options[i]; i++)
5908 create_trace_option_core_file(tr, trace_options[i], i);
5912 rb_simple_read(struct file *filp, char __user *ubuf,
5913 size_t cnt, loff_t *ppos)
5915 struct trace_array *tr = filp->private_data;
5919 r = tracer_tracing_is_on(tr);
5920 r = sprintf(buf, "%d\n", r);
5922 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5926 rb_simple_write(struct file *filp, const char __user *ubuf,
5927 size_t cnt, loff_t *ppos)
5929 struct trace_array *tr = filp->private_data;
5930 struct ring_buffer *buffer = tr->trace_buffer.buffer;
5934 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5939 mutex_lock(&trace_types_lock);
5941 tracer_tracing_on(tr);
5942 if (tr->current_trace->start)
5943 tr->current_trace->start(tr);
5945 tracer_tracing_off(tr);
5946 if (tr->current_trace->stop)
5947 tr->current_trace->stop(tr);
5949 mutex_unlock(&trace_types_lock);
5957 static const struct file_operations rb_simple_fops = {
5958 .open = tracing_open_generic_tr,
5959 .read = rb_simple_read,
5960 .write = rb_simple_write,
5961 .release = tracing_release_generic_tr,
5962 .llseek = default_llseek,
5965 struct dentry *trace_instance_dir;
5968 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
5971 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
5973 enum ring_buffer_flags rb_flags;
5975 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5979 buf->buffer = ring_buffer_alloc(size, rb_flags);
5983 buf->data = alloc_percpu(struct trace_array_cpu);
5985 ring_buffer_free(buf->buffer);
5989 /* Allocate the first page for all buffers */
5990 set_buffer_entries(&tr->trace_buffer,
5991 ring_buffer_size(tr->trace_buffer.buffer, 0));
5996 static int allocate_trace_buffers(struct trace_array *tr, int size)
6000 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6004 #ifdef CONFIG_TRACER_MAX_TRACE
6005 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6006 allocate_snapshot ? size : 1);
6008 ring_buffer_free(tr->trace_buffer.buffer);
6009 free_percpu(tr->trace_buffer.data);
6012 tr->allocated_snapshot = allocate_snapshot;
6015 * Only the top level trace array gets its snapshot allocated
6016 * from the kernel command line.
6018 allocate_snapshot = false;
6023 static int new_instance_create(const char *name)
6025 struct trace_array *tr;
6028 mutex_lock(&trace_types_lock);
6031 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6032 if (tr->name && strcmp(tr->name, name) == 0)
6037 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6041 tr->name = kstrdup(name, GFP_KERNEL);
6045 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6048 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6050 raw_spin_lock_init(&tr->start_lock);
6052 tr->current_trace = &nop_trace;
6054 INIT_LIST_HEAD(&tr->systems);
6055 INIT_LIST_HEAD(&tr->events);
6057 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6060 tr->dir = debugfs_create_dir(name, trace_instance_dir);
6064 ret = event_trace_add_tracer(tr->dir, tr);
6066 debugfs_remove_recursive(tr->dir);
6070 init_tracer_debugfs(tr, tr->dir);
6072 list_add(&tr->list, &ftrace_trace_arrays);
6074 mutex_unlock(&trace_types_lock);
6079 if (tr->trace_buffer.buffer)
6080 ring_buffer_free(tr->trace_buffer.buffer);
6081 free_cpumask_var(tr->tracing_cpumask);
6086 mutex_unlock(&trace_types_lock);
6092 static int instance_delete(const char *name)
6094 struct trace_array *tr;
6098 mutex_lock(&trace_types_lock);
6101 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6102 if (tr->name && strcmp(tr->name, name) == 0) {
6114 list_del(&tr->list);
6116 event_trace_del_tracer(tr);
6117 debugfs_remove_recursive(tr->dir);
6118 free_percpu(tr->trace_buffer.data);
6119 ring_buffer_free(tr->trace_buffer.buffer);
6127 mutex_unlock(&trace_types_lock);
6132 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
6134 struct dentry *parent;
6137 /* Paranoid: Make sure the parent is the "instances" directory */
6138 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6139 if (WARN_ON_ONCE(parent != trace_instance_dir))
6143 * The inode mutex is locked, but debugfs_create_dir() will also
6144 * take the mutex. As the instances directory can not be destroyed
6145 * or changed in any other way, it is safe to unlock it, and
6146 * let the dentry try. If two users try to make the same dir at
6147 * the same time, then the new_instance_create() will determine the
6150 mutex_unlock(&inode->i_mutex);
6152 ret = new_instance_create(dentry->d_iname);
6154 mutex_lock(&inode->i_mutex);
6159 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
6161 struct dentry *parent;
6164 /* Paranoid: Make sure the parent is the "instances" directory */
6165 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6166 if (WARN_ON_ONCE(parent != trace_instance_dir))
6169 /* The caller did a dget() on dentry */
6170 mutex_unlock(&dentry->d_inode->i_mutex);
6173 * The inode mutex is locked, but debugfs_create_dir() will also
6174 * take the mutex. As the instances directory can not be destroyed
6175 * or changed in any other way, it is safe to unlock it, and
6176 * let the dentry try. If two users try to make the same dir at
6177 * the same time, then the instance_delete() will determine the
6180 mutex_unlock(&inode->i_mutex);
6182 ret = instance_delete(dentry->d_iname);
6184 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6185 mutex_lock(&dentry->d_inode->i_mutex);
6190 static const struct inode_operations instance_dir_inode_operations = {
6191 .lookup = simple_lookup,
6192 .mkdir = instance_mkdir,
6193 .rmdir = instance_rmdir,
6196 static __init void create_trace_instances(struct dentry *d_tracer)
6198 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6199 if (WARN_ON(!trace_instance_dir))
6202 /* Hijack the dir inode operations, to allow mkdir */
6203 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6207 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
6211 trace_create_file("tracing_cpumask", 0644, d_tracer,
6212 tr, &tracing_cpumask_fops);
6214 trace_create_file("trace_options", 0644, d_tracer,
6215 tr, &tracing_iter_fops);
6217 trace_create_file("trace", 0644, d_tracer,
6220 trace_create_file("trace_pipe", 0444, d_tracer,
6221 tr, &tracing_pipe_fops);
6223 trace_create_file("buffer_size_kb", 0644, d_tracer,
6224 tr, &tracing_entries_fops);
6226 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6227 tr, &tracing_total_entries_fops);
6229 trace_create_file("free_buffer", 0200, d_tracer,
6230 tr, &tracing_free_buffer_fops);
6232 trace_create_file("trace_marker", 0220, d_tracer,
6233 tr, &tracing_mark_fops);
6235 trace_create_file("trace_clock", 0644, d_tracer, tr,
6238 trace_create_file("tracing_on", 0644, d_tracer,
6239 tr, &rb_simple_fops);
6241 #ifdef CONFIG_TRACER_SNAPSHOT
6242 trace_create_file("snapshot", 0644, d_tracer,
6243 tr, &snapshot_fops);
6246 for_each_tracing_cpu(cpu)
6247 tracing_init_debugfs_percpu(tr, cpu);
6251 static __init int tracer_init_debugfs(void)
6253 struct dentry *d_tracer;
6255 trace_access_lock_init();
6257 d_tracer = tracing_init_dentry();
6261 init_tracer_debugfs(&global_trace, d_tracer);
6263 trace_create_file("available_tracers", 0444, d_tracer,
6264 &global_trace, &show_traces_fops);
6266 trace_create_file("current_tracer", 0644, d_tracer,
6267 &global_trace, &set_tracer_fops);
6269 #ifdef CONFIG_TRACER_MAX_TRACE
6270 trace_create_file("tracing_max_latency", 0644, d_tracer,
6271 &tracing_max_latency, &tracing_max_lat_fops);
6274 trace_create_file("tracing_thresh", 0644, d_tracer,
6275 &tracing_thresh, &tracing_max_lat_fops);
6277 trace_create_file("README", 0444, d_tracer,
6278 NULL, &tracing_readme_fops);
6280 trace_create_file("saved_cmdlines", 0444, d_tracer,
6281 NULL, &tracing_saved_cmdlines_fops);
6283 #ifdef CONFIG_DYNAMIC_FTRACE
6284 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6285 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6288 create_trace_instances(d_tracer);
6290 create_trace_options_dir(&global_trace);
6295 static int trace_panic_handler(struct notifier_block *this,
6296 unsigned long event, void *unused)
6298 if (ftrace_dump_on_oops)
6299 ftrace_dump(ftrace_dump_on_oops);
6303 static struct notifier_block trace_panic_notifier = {
6304 .notifier_call = trace_panic_handler,
6306 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6309 static int trace_die_handler(struct notifier_block *self,
6315 if (ftrace_dump_on_oops)
6316 ftrace_dump(ftrace_dump_on_oops);
6324 static struct notifier_block trace_die_notifier = {
6325 .notifier_call = trace_die_handler,
6330 * printk is set to max of 1024, we really don't need it that big.
6331 * Nothing should be printing 1000 characters anyway.
6333 #define TRACE_MAX_PRINT 1000
6336 * Define here KERN_TRACE so that we have one place to modify
6337 * it if we decide to change what log level the ftrace dump
6340 #define KERN_TRACE KERN_EMERG
6343 trace_printk_seq(struct trace_seq *s)
6345 /* Probably should print a warning here. */
6346 if (s->len >= TRACE_MAX_PRINT)
6347 s->len = TRACE_MAX_PRINT;
6349 /* should be zero ended, but we are paranoid. */
6350 s->buffer[s->len] = 0;
6352 printk(KERN_TRACE "%s", s->buffer);
6357 void trace_init_global_iter(struct trace_iterator *iter)
6359 iter->tr = &global_trace;
6360 iter->trace = iter->tr->current_trace;
6361 iter->cpu_file = RING_BUFFER_ALL_CPUS;
6362 iter->trace_buffer = &global_trace.trace_buffer;
6364 if (iter->trace && iter->trace->open)
6365 iter->trace->open(iter);
6367 /* Annotate start of buffers if we had overruns */
6368 if (ring_buffer_overruns(iter->trace_buffer->buffer))
6369 iter->iter_flags |= TRACE_FILE_ANNOTATE;
6371 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6372 if (trace_clocks[iter->tr->clock_id].in_ns)
6373 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6376 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6378 /* use static because iter can be a bit big for the stack */
6379 static struct trace_iterator iter;
6380 static atomic_t dump_running;
6381 unsigned int old_userobj;
6382 unsigned long flags;
6385 /* Only allow one dump user at a time. */
6386 if (atomic_inc_return(&dump_running) != 1) {
6387 atomic_dec(&dump_running);
6392 * Always turn off tracing when we dump.
6393 * We don't need to show trace output of what happens
6394 * between multiple crashes.
6396 * If the user does a sysrq-z, then they can re-enable
6397 * tracing with echo 1 > tracing_on.
6401 local_irq_save(flags);
6403 /* Simulate the iterator */
6404 trace_init_global_iter(&iter);
6406 for_each_tracing_cpu(cpu) {
6407 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6410 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6412 /* don't look at user memory in panic mode */
6413 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6415 switch (oops_dump_mode) {
6417 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6420 iter.cpu_file = raw_smp_processor_id();
6425 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6426 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6429 printk(KERN_TRACE "Dumping ftrace buffer:\n");
6431 /* Did function tracer already get disabled? */
6432 if (ftrace_is_dead()) {
6433 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6434 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6438 * We need to stop all tracing on all CPUS to read the
6439 * the next buffer. This is a bit expensive, but is
6440 * not done often. We fill all what we can read,
6441 * and then release the locks again.
6444 while (!trace_empty(&iter)) {
6447 printk(KERN_TRACE "---------------------------------\n");
6451 /* reset all but tr, trace, and overruns */
6452 memset(&iter.seq, 0,
6453 sizeof(struct trace_iterator) -
6454 offsetof(struct trace_iterator, seq));
6455 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6458 if (trace_find_next_entry_inc(&iter) != NULL) {
6461 ret = print_trace_line(&iter);
6462 if (ret != TRACE_TYPE_NO_CONSUME)
6463 trace_consume(&iter);
6465 touch_nmi_watchdog();
6467 trace_printk_seq(&iter.seq);
6471 printk(KERN_TRACE " (ftrace buffer empty)\n");
6473 printk(KERN_TRACE "---------------------------------\n");
6476 trace_flags |= old_userobj;
6478 for_each_tracing_cpu(cpu) {
6479 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6481 atomic_dec(&dump_running);
6482 local_irq_restore(flags);
6484 EXPORT_SYMBOL_GPL(ftrace_dump);
6486 __init static int tracer_alloc_buffers(void)
6492 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6495 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
6496 goto out_free_buffer_mask;
6498 /* Only allocate trace_printk buffers if a trace_printk exists */
6499 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6500 /* Must be called before global_trace.buffer is allocated */
6501 trace_printk_init_buffers();
6503 /* To save memory, keep the ring buffer size to its minimum */
6504 if (ring_buffer_expanded)
6505 ring_buf_size = trace_buf_size;
6509 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6510 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
6512 raw_spin_lock_init(&global_trace.start_lock);
6514 /* Used for event triggers */
6515 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
6517 goto out_free_cpumask;
6519 /* TODO: make the number of buffers hot pluggable with CPUS */
6520 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6521 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6523 goto out_free_temp_buffer;
6526 if (global_trace.buffer_disabled)
6529 trace_init_cmdlines();
6532 * register_tracer() might reference current_trace, so it
6533 * needs to be set before we register anything. This is
6534 * just a bootstrap of current_trace anyway.
6536 global_trace.current_trace = &nop_trace;
6538 register_tracer(&nop_trace);
6540 /* All seems OK, enable tracing */
6541 tracing_disabled = 0;
6543 atomic_notifier_chain_register(&panic_notifier_list,
6544 &trace_panic_notifier);
6546 register_die_notifier(&trace_die_notifier);
6548 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6550 INIT_LIST_HEAD(&global_trace.systems);
6551 INIT_LIST_HEAD(&global_trace.events);
6552 list_add(&global_trace.list, &ftrace_trace_arrays);
6554 while (trace_boot_options) {
6557 option = strsep(&trace_boot_options, ",");
6558 trace_set_options(&global_trace, option);
6561 register_snapshot_cmd();
6565 out_free_temp_buffer:
6566 ring_buffer_free(temp_buffer);
6568 free_percpu(global_trace.trace_buffer.data);
6569 #ifdef CONFIG_TRACER_MAX_TRACE
6570 free_percpu(global_trace.max_buffer.data);
6572 free_cpumask_var(global_trace.tracing_cpumask);
6573 out_free_buffer_mask:
6574 free_cpumask_var(tracing_buffer_mask);
6579 __init static int clear_boot_tracer(void)
6582 * The default tracer at boot buffer is an init section.
6583 * This function is called in lateinit. If we did not
6584 * find the boot tracer, then clear it out, to prevent
6585 * later registration from accessing the buffer that is
6586 * about to be freed.
6588 if (!default_bootup_tracer)
6591 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6592 default_bootup_tracer);
6593 default_bootup_tracer = NULL;
6598 early_initcall(tracer_alloc_buffers);
6599 fs_initcall(tracer_init_debugfs);
6600 late_initcall(clear_boot_tracer);