1 // SPDX-License-Identifier: GPL-2.0
3 * OS Noise Tracer: computes the OS Noise suffered by a running thread.
4 * Timerlat Tracer: measures the wakeup latency of a timer triggered IRQ and thread.
6 * Based on "hwlat_detector" tracer by:
7 * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
8 * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
9 * With feedback from Clark Williams <williams@redhat.com>
11 * And also based on the rtsl tracer presented on:
12 * DE OLIVEIRA, Daniel Bristot, et al. Demystifying the real-time linux
13 * scheduling latency. In: 32nd Euromicro Conference on Real-Time Systems
14 * (ECRTS 2020). Schloss Dagstuhl-Leibniz-Zentrum fur Informatik, 2020.
16 * Copyright (C) 2021 Daniel Bristot de Oliveira, Red Hat, Inc. <bristot@redhat.com>
19 #include <linux/kthread.h>
20 #include <linux/tracefs.h>
21 #include <linux/uaccess.h>
22 #include <linux/cpumask.h>
23 #include <linux/delay.h>
24 #include <linux/sched/clock.h>
25 #include <uapi/linux/sched/types.h>
26 #include <linux/sched.h>
29 #ifdef CONFIG_X86_LOCAL_APIC
30 #include <asm/trace/irq_vectors.h>
31 #undef TRACE_INCLUDE_PATH
32 #undef TRACE_INCLUDE_FILE
33 #endif /* CONFIG_X86_LOCAL_APIC */
35 #include <trace/events/irq.h>
36 #include <trace/events/sched.h>
38 #define CREATE_TRACE_POINTS
39 #include <trace/events/osnoise.h>
44 #define BANNER "osnoise: "
45 #define DEFAULT_SAMPLE_PERIOD 1000000 /* 1s */
46 #define DEFAULT_SAMPLE_RUNTIME 1000000 /* 1s */
48 #define DEFAULT_TIMERLAT_PERIOD 1000 /* 1ms */
49 #define DEFAULT_TIMERLAT_PRIO 95 /* FIFO 95 */
52 * trace_array of the enabled osnoise/timerlat instances.
54 struct osnoise_instance {
55 struct list_head list;
56 struct trace_array *tr;
59 static struct list_head osnoise_instances;
61 static bool osnoise_has_registered_instances(void)
63 return !!list_first_or_null_rcu(&osnoise_instances,
64 struct osnoise_instance,
69 * osnoise_instance_registered - check if a tr is already registered
71 static int osnoise_instance_registered(struct trace_array *tr)
73 struct osnoise_instance *inst;
77 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
87 * osnoise_register_instance - register a new trace instance
89 * Register a trace_array *tr in the list of instances running
90 * osnoise/timerlat tracers.
92 static int osnoise_register_instance(struct trace_array *tr)
94 struct osnoise_instance *inst;
97 * register/unregister serialization is provided by trace's
100 lockdep_assert_held(&trace_types_lock);
102 inst = kmalloc(sizeof(*inst), GFP_KERNEL);
106 INIT_LIST_HEAD_RCU(&inst->list);
108 list_add_tail_rcu(&inst->list, &osnoise_instances);
114 * osnoise_unregister_instance - unregister a registered trace instance
116 * Remove the trace_array *tr from the list of instances running
117 * osnoise/timerlat tracers.
119 static void osnoise_unregister_instance(struct trace_array *tr)
121 struct osnoise_instance *inst;
125 * register/unregister serialization is provided by trace's
128 list_for_each_entry_rcu(inst, &osnoise_instances, list,
129 lockdep_is_held(&trace_types_lock)) {
130 if (inst->tr == tr) {
131 list_del_rcu(&inst->list);
160 #define IRQ_CONTEXT 0
161 #define THREAD_CONTEXT 1
163 * sofirq runtime info.
172 * thread runtime info.
181 * Runtime information: this structure saves the runtime information used by
182 * one sampling thread.
184 struct osnoise_variables {
185 struct task_struct *kthread;
190 struct osn_softirq softirq;
191 struct osn_thread thread;
196 * Per-cpu runtime information.
198 DEFINE_PER_CPU(struct osnoise_variables, per_cpu_osnoise_var);
201 * this_cpu_osn_var - Return the per-cpu osnoise_variables on its relative CPU
203 static inline struct osnoise_variables *this_cpu_osn_var(void)
205 return this_cpu_ptr(&per_cpu_osnoise_var);
208 #ifdef CONFIG_TIMERLAT_TRACER
210 * Runtime information for the timer mode.
212 struct timerlat_variables {
213 struct task_struct *kthread;
214 struct hrtimer timer;
221 DEFINE_PER_CPU(struct timerlat_variables, per_cpu_timerlat_var);
224 * this_cpu_tmr_var - Return the per-cpu timerlat_variables on its relative CPU
226 static inline struct timerlat_variables *this_cpu_tmr_var(void)
228 return this_cpu_ptr(&per_cpu_timerlat_var);
232 * tlat_var_reset - Reset the values of the given timerlat_variables
234 static inline void tlat_var_reset(void)
236 struct timerlat_variables *tlat_var;
239 * So far, all the values are initialized as 0, so
240 * zeroing the structure is perfect.
242 for_each_cpu(cpu, cpu_online_mask) {
243 tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
244 memset(tlat_var, 0, sizeof(*tlat_var));
247 #else /* CONFIG_TIMERLAT_TRACER */
248 #define tlat_var_reset() do {} while (0)
249 #endif /* CONFIG_TIMERLAT_TRACER */
252 * osn_var_reset - Reset the values of the given osnoise_variables
254 static inline void osn_var_reset(void)
256 struct osnoise_variables *osn_var;
260 * So far, all the values are initialized as 0, so
261 * zeroing the structure is perfect.
263 for_each_cpu(cpu, cpu_online_mask) {
264 osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
265 memset(osn_var, 0, sizeof(*osn_var));
270 * osn_var_reset_all - Reset the value of all per-cpu osnoise_variables
272 static inline void osn_var_reset_all(void)
279 * Tells NMIs to call back to the osnoise tracer to record timestamps.
281 bool trace_osnoise_callback_enabled;
284 * osnoise sample structure definition. Used to store the statistics of a
287 struct osnoise_sample {
288 u64 runtime; /* runtime */
289 u64 noise; /* noise */
290 u64 max_sample; /* max single noise sample */
291 int hw_count; /* # HW (incl. hypervisor) interference */
292 int nmi_count; /* # NMIs during this sample */
293 int irq_count; /* # IRQs during this sample */
294 int softirq_count; /* # softirqs during this sample */
295 int thread_count; /* # threads during this sample */
298 #ifdef CONFIG_TIMERLAT_TRACER
300 * timerlat sample structure definition. Used to store the statistics of
303 struct timerlat_sample {
304 u64 timer_latency; /* timer_latency */
305 unsigned int seqnum; /* unique sequence */
306 int context; /* timer context */
311 * Protect the interface.
313 struct mutex interface_lock;
318 static struct osnoise_data {
319 u64 sample_period; /* total sampling period */
320 u64 sample_runtime; /* active sampling portion of period */
321 u64 stop_tracing; /* stop trace in the internal operation (loop/irq) */
322 u64 stop_tracing_total; /* stop trace in the final operation (report/thread) */
323 #ifdef CONFIG_TIMERLAT_TRACER
324 u64 timerlat_period; /* timerlat period */
325 u64 print_stack; /* print IRQ stack if total > */
326 int timerlat_tracer; /* timerlat tracer */
328 bool tainted; /* infor users and developers about a problem */
330 .sample_period = DEFAULT_SAMPLE_PERIOD,
331 .sample_runtime = DEFAULT_SAMPLE_RUNTIME,
333 .stop_tracing_total = 0,
334 #ifdef CONFIG_TIMERLAT_TRACER
336 .timerlat_period = DEFAULT_TIMERLAT_PERIOD,
337 .timerlat_tracer = 0,
341 #ifdef CONFIG_TIMERLAT_TRACER
342 static inline bool timerlat_enabled(void)
344 return osnoise_data.timerlat_tracer;
347 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
349 struct timerlat_variables *tlat_var = this_cpu_tmr_var();
351 * If the timerlat is enabled, but the irq handler did
352 * not run yet enabling timerlat_tracer, do not trace.
354 if (!tlat_var->tracing_thread) {
355 osn_var->softirq.arrival_time = 0;
356 osn_var->softirq.delta_start = 0;
362 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
364 struct timerlat_variables *tlat_var = this_cpu_tmr_var();
366 * If the timerlat is enabled, but the irq handler did
367 * not run yet enabling timerlat_tracer, do not trace.
369 if (!tlat_var->tracing_thread) {
370 osn_var->thread.delta_start = 0;
371 osn_var->thread.arrival_time = 0;
376 #else /* CONFIG_TIMERLAT_TRACER */
377 static inline bool timerlat_enabled(void)
382 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
386 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
392 #ifdef CONFIG_PREEMPT_RT
394 * Print the osnoise header info.
396 static void print_osnoise_headers(struct seq_file *s)
398 if (osnoise_data.tainted)
399 seq_puts(s, "# osnoise is tainted!\n");
401 seq_puts(s, "# _-------=> irqs-off\n");
402 seq_puts(s, "# / _------=> need-resched\n");
403 seq_puts(s, "# | / _-----=> need-resched-lazy\n");
404 seq_puts(s, "# || / _----=> hardirq/softirq\n");
405 seq_puts(s, "# ||| / _---=> preempt-depth\n");
406 seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
407 seq_puts(s, "# ||||| / _-=> migrate-disable\n");
409 seq_puts(s, "# |||||| / ");
410 seq_puts(s, " MAX\n");
412 seq_puts(s, "# ||||| / ");
413 seq_puts(s, " SINGLE Interference counters:\n");
415 seq_puts(s, "# ||||||| RUNTIME ");
416 seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
418 seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP IN US ");
419 seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
421 seq_puts(s, "# | | | ||||||| | | ");
422 seq_puts(s, " | | | | | | | |\n");
424 #else /* CONFIG_PREEMPT_RT */
425 static void print_osnoise_headers(struct seq_file *s)
427 if (osnoise_data.tainted)
428 seq_puts(s, "# osnoise is tainted!\n");
430 seq_puts(s, "# _-----=> irqs-off\n");
431 seq_puts(s, "# / _----=> need-resched\n");
432 seq_puts(s, "# | / _---=> hardirq/softirq\n");
433 seq_puts(s, "# || / _--=> preempt-depth\n");
434 seq_puts(s, "# ||| / _-=> migrate-disable ");
435 seq_puts(s, " MAX\n");
436 seq_puts(s, "# |||| / delay ");
437 seq_puts(s, " SINGLE Interference counters:\n");
439 seq_puts(s, "# ||||| RUNTIME ");
440 seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
442 seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP IN US ");
443 seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
445 seq_puts(s, "# | | | ||||| | | ");
446 seq_puts(s, " | | | | | | | |\n");
448 #endif /* CONFIG_PREEMPT_RT */
451 * osnoise_taint - report an osnoise error.
453 #define osnoise_taint(msg) ({ \
454 struct osnoise_instance *inst; \
455 struct trace_buffer *buffer; \
458 list_for_each_entry_rcu(inst, &osnoise_instances, list) { \
459 buffer = inst->tr->array_buffer.buffer; \
460 trace_array_printk_buf(buffer, _THIS_IP_, msg); \
463 osnoise_data.tainted = true; \
467 * Record an osnoise_sample into the tracer buffer.
470 __trace_osnoise_sample(struct osnoise_sample *sample, struct trace_buffer *buffer)
472 struct trace_event_call *call = &event_osnoise;
473 struct ring_buffer_event *event;
474 struct osnoise_entry *entry;
476 event = trace_buffer_lock_reserve(buffer, TRACE_OSNOISE, sizeof(*entry),
480 entry = ring_buffer_event_data(event);
481 entry->runtime = sample->runtime;
482 entry->noise = sample->noise;
483 entry->max_sample = sample->max_sample;
484 entry->hw_count = sample->hw_count;
485 entry->nmi_count = sample->nmi_count;
486 entry->irq_count = sample->irq_count;
487 entry->softirq_count = sample->softirq_count;
488 entry->thread_count = sample->thread_count;
490 if (!call_filter_check_discard(call, entry, buffer, event))
491 trace_buffer_unlock_commit_nostack(buffer, event);
495 * Record an osnoise_sample on all osnoise instances.
497 static void trace_osnoise_sample(struct osnoise_sample *sample)
499 struct osnoise_instance *inst;
500 struct trace_buffer *buffer;
503 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
504 buffer = inst->tr->array_buffer.buffer;
505 __trace_osnoise_sample(sample, buffer);
510 #ifdef CONFIG_TIMERLAT_TRACER
512 * Print the timerlat header info.
514 #ifdef CONFIG_PREEMPT_RT
515 static void print_timerlat_headers(struct seq_file *s)
517 seq_puts(s, "# _-------=> irqs-off\n");
518 seq_puts(s, "# / _------=> need-resched\n");
519 seq_puts(s, "# | / _-----=> need-resched-lazy\n");
520 seq_puts(s, "# || / _----=> hardirq/softirq\n");
521 seq_puts(s, "# ||| / _---=> preempt-depth\n");
522 seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
523 seq_puts(s, "# ||||| / _-=> migrate-disable\n");
524 seq_puts(s, "# |||||| /\n");
525 seq_puts(s, "# ||||||| ACTIVATION\n");
526 seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP ID ");
527 seq_puts(s, " CONTEXT LATENCY\n");
528 seq_puts(s, "# | | | ||||||| | | ");
529 seq_puts(s, " | |\n");
531 #else /* CONFIG_PREEMPT_RT */
532 static void print_timerlat_headers(struct seq_file *s)
534 seq_puts(s, "# _-----=> irqs-off\n");
535 seq_puts(s, "# / _----=> need-resched\n");
536 seq_puts(s, "# | / _---=> hardirq/softirq\n");
537 seq_puts(s, "# || / _--=> preempt-depth\n");
538 seq_puts(s, "# ||| / _-=> migrate-disable\n");
539 seq_puts(s, "# |||| / delay\n");
540 seq_puts(s, "# ||||| ACTIVATION\n");
541 seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP ID ");
542 seq_puts(s, " CONTEXT LATENCY\n");
543 seq_puts(s, "# | | | ||||| | | ");
544 seq_puts(s, " | |\n");
546 #endif /* CONFIG_PREEMPT_RT */
549 __trace_timerlat_sample(struct timerlat_sample *sample, struct trace_buffer *buffer)
551 struct trace_event_call *call = &event_osnoise;
552 struct ring_buffer_event *event;
553 struct timerlat_entry *entry;
555 event = trace_buffer_lock_reserve(buffer, TRACE_TIMERLAT, sizeof(*entry),
559 entry = ring_buffer_event_data(event);
560 entry->seqnum = sample->seqnum;
561 entry->context = sample->context;
562 entry->timer_latency = sample->timer_latency;
564 if (!call_filter_check_discard(call, entry, buffer, event))
565 trace_buffer_unlock_commit_nostack(buffer, event);
569 * Record an timerlat_sample into the tracer buffer.
571 static void trace_timerlat_sample(struct timerlat_sample *sample)
573 struct osnoise_instance *inst;
574 struct trace_buffer *buffer;
577 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
578 buffer = inst->tr->array_buffer.buffer;
579 __trace_timerlat_sample(sample, buffer);
584 #ifdef CONFIG_STACKTRACE
586 #define MAX_CALLS 256
589 * Stack trace will take place only at IRQ level, so, no need
590 * to control nesting here.
595 unsigned long calls[MAX_CALLS];
598 static DEFINE_PER_CPU(struct trace_stack, trace_stack);
601 * timerlat_save_stack - save a stack trace without printing
603 * Save the current stack trace without printing. The
604 * stack will be printed later, after the end of the measurement.
606 static void timerlat_save_stack(int skip)
608 unsigned int size, nr_entries;
609 struct trace_stack *fstack;
611 fstack = this_cpu_ptr(&trace_stack);
613 size = ARRAY_SIZE(fstack->calls);
615 nr_entries = stack_trace_save(fstack->calls, size, skip);
617 fstack->stack_size = nr_entries * sizeof(unsigned long);
618 fstack->nr_entries = nr_entries;
625 __timerlat_dump_stack(struct trace_buffer *buffer, struct trace_stack *fstack, unsigned int size)
627 struct trace_event_call *call = &event_osnoise;
628 struct ring_buffer_event *event;
629 struct stack_entry *entry;
631 event = trace_buffer_lock_reserve(buffer, TRACE_STACK, sizeof(*entry) + size,
636 entry = ring_buffer_event_data(event);
638 memcpy(&entry->caller, fstack->calls, size);
639 entry->size = fstack->nr_entries;
641 if (!call_filter_check_discard(call, entry, buffer, event))
642 trace_buffer_unlock_commit_nostack(buffer, event);
646 * timerlat_dump_stack - dump a stack trace previously saved
648 static void timerlat_dump_stack(u64 latency)
650 struct osnoise_instance *inst;
651 struct trace_buffer *buffer;
652 struct trace_stack *fstack;
656 * trace only if latency > print_stack config, if enabled.
658 if (!osnoise_data.print_stack || osnoise_data.print_stack > latency)
661 preempt_disable_notrace();
662 fstack = this_cpu_ptr(&trace_stack);
663 size = fstack->stack_size;
666 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
667 buffer = inst->tr->array_buffer.buffer;
668 __timerlat_dump_stack(buffer, fstack, size);
672 preempt_enable_notrace();
674 #else /* CONFIG_STACKTRACE */
675 #define timerlat_dump_stack(u64 latency) do {} while (0)
676 #define timerlat_save_stack(a) do {} while (0)
677 #endif /* CONFIG_STACKTRACE */
678 #endif /* CONFIG_TIMERLAT_TRACER */
681 * Macros to encapsulate the time capturing infrastructure.
683 #define time_get() trace_clock_local()
684 #define time_to_us(x) div_u64(x, 1000)
685 #define time_sub(a, b) ((a) - (b))
688 * cond_move_irq_delta_start - Forward the delta_start of a running IRQ
690 * If an IRQ is preempted by an NMI, its delta_start is pushed forward
691 * to discount the NMI interference.
693 * See get_int_safe_duration().
696 cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)
698 if (osn_var->irq.delta_start)
699 osn_var->irq.delta_start += duration;
702 #ifndef CONFIG_PREEMPT_RT
704 * cond_move_softirq_delta_start - Forward the delta_start of a running softirq.
706 * If a softirq is preempted by an IRQ or NMI, its delta_start is pushed
707 * forward to discount the interference.
709 * See get_int_safe_duration().
712 cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)
714 if (osn_var->softirq.delta_start)
715 osn_var->softirq.delta_start += duration;
717 #else /* CONFIG_PREEMPT_RT */
718 #define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)
722 * cond_move_thread_delta_start - Forward the delta_start of a running thread
724 * If a noisy thread is preempted by an softirq, IRQ or NMI, its delta_start
725 * is pushed forward to discount the interference.
727 * See get_int_safe_duration().
730 cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)
732 if (osn_var->thread.delta_start)
733 osn_var->thread.delta_start += duration;
737 * get_int_safe_duration - Get the duration of a window
739 * The irq, softirq and thread varaibles need to have its duration without
740 * the interference from higher priority interrupts. Instead of keeping a
741 * variable to discount the interrupt interference from these variables, the
742 * starting time of these variables are pushed forward with the interrupt's
743 * duration. In this way, a single variable is used to:
745 * - Know if a given window is being measured.
746 * - Account its duration.
747 * - Discount the interference.
749 * To avoid getting inconsistent values, e.g.,:
753 * delta_start -= int duration;
755 * duration = now - delta_start;
757 * result: negative duration if the variable duration before the
758 * interrupt was smaller than the interrupt execution.
760 * A counter of interrupts is used. If the counter increased, try
761 * to capture an interference safe duration.
764 get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start)
766 u64 int_counter, now;
770 int_counter = local_read(&osn_var->int_counter);
771 /* synchronize with interrupts */
775 duration = (now - *delta_start);
777 /* synchronize with interrupts */
779 } while (int_counter != local_read(&osn_var->int_counter));
782 * This is an evidence of race conditions that cause
783 * a value to be "discounted" too much.
786 osnoise_taint("Negative duration!\n");
795 * set_int_safe_time - Save the current time on *time, aware of interference
797 * Get the time, taking into consideration a possible interference from
798 * higher priority interrupts.
800 * See get_int_safe_duration() for an explanation.
803 set_int_safe_time(struct osnoise_variables *osn_var, u64 *time)
808 int_counter = local_read(&osn_var->int_counter);
809 /* synchronize with interrupts */
814 /* synchronize with interrupts */
816 } while (int_counter != local_read(&osn_var->int_counter));
821 #ifdef CONFIG_TIMERLAT_TRACER
823 * copy_int_safe_time - Copy *src into *desc aware of interference
826 copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src)
831 int_counter = local_read(&osn_var->int_counter);
832 /* synchronize with interrupts */
837 /* synchronize with interrupts */
839 } while (int_counter != local_read(&osn_var->int_counter));
843 #endif /* CONFIG_TIMERLAT_TRACER */
846 * trace_osnoise_callback - NMI entry/exit callback
848 * This function is called at the entry and exit NMI code. The bool enter
849 * distinguishes between either case. This function is used to note a NMI
850 * occurrence, compute the noise caused by the NMI, and to remove the noise
851 * it is potentially causing on other interference variables.
853 void trace_osnoise_callback(bool enter)
855 struct osnoise_variables *osn_var = this_cpu_osn_var();
858 if (!osn_var->sampling)
862 * Currently trace_clock_local() calls sched_clock() and the
863 * generic version is not NMI safe.
865 if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK)) {
867 osn_var->nmi.delta_start = time_get();
868 local_inc(&osn_var->int_counter);
870 duration = time_get() - osn_var->nmi.delta_start;
872 trace_nmi_noise(osn_var->nmi.delta_start, duration);
874 cond_move_irq_delta_start(osn_var, duration);
875 cond_move_softirq_delta_start(osn_var, duration);
876 cond_move_thread_delta_start(osn_var, duration);
881 osn_var->nmi.count++;
885 * osnoise_trace_irq_entry - Note the starting of an IRQ
887 * Save the starting time of an IRQ. As IRQs are non-preemptive to other IRQs,
888 * it is safe to use a single variable (ons_var->irq) to save the statistics.
889 * The arrival_time is used to report... the arrival time. The delta_start
890 * is used to compute the duration at the IRQ exit handler. See
891 * cond_move_irq_delta_start().
893 void osnoise_trace_irq_entry(int id)
895 struct osnoise_variables *osn_var = this_cpu_osn_var();
897 if (!osn_var->sampling)
900 * This value will be used in the report, but not to compute
901 * the execution time, so it is safe to get it unsafe.
903 osn_var->irq.arrival_time = time_get();
904 set_int_safe_time(osn_var, &osn_var->irq.delta_start);
905 osn_var->irq.count++;
907 local_inc(&osn_var->int_counter);
911 * osnoise_irq_exit - Note the end of an IRQ, sava data and trace
913 * Computes the duration of the IRQ noise, and trace it. Also discounts the
914 * interference from other sources of noise could be currently being accounted.
916 void osnoise_trace_irq_exit(int id, const char *desc)
918 struct osnoise_variables *osn_var = this_cpu_osn_var();
921 if (!osn_var->sampling)
924 duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);
925 trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);
926 osn_var->irq.arrival_time = 0;
927 cond_move_softirq_delta_start(osn_var, duration);
928 cond_move_thread_delta_start(osn_var, duration);
932 * trace_irqentry_callback - Callback to the irq:irq_entry traceevent
934 * Used to note the starting of an IRQ occurece.
936 static void trace_irqentry_callback(void *data, int irq,
937 struct irqaction *action)
939 osnoise_trace_irq_entry(irq);
943 * trace_irqexit_callback - Callback to the irq:irq_exit traceevent
945 * Used to note the end of an IRQ occurece.
947 static void trace_irqexit_callback(void *data, int irq,
948 struct irqaction *action, int ret)
950 osnoise_trace_irq_exit(irq, action->name);
954 * arch specific register function.
956 int __weak osnoise_arch_register(void)
962 * arch specific unregister function.
964 void __weak osnoise_arch_unregister(void)
970 * hook_irq_events - Hook IRQ handling events
972 * This function hooks the IRQ related callbacks to the respective trace
975 static int hook_irq_events(void)
979 ret = register_trace_irq_handler_entry(trace_irqentry_callback, NULL);
983 ret = register_trace_irq_handler_exit(trace_irqexit_callback, NULL);
985 goto out_unregister_entry;
987 ret = osnoise_arch_register();
994 unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
995 out_unregister_entry:
996 unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1002 * unhook_irq_events - Unhook IRQ handling events
1004 * This function unhooks the IRQ related callbacks to the respective trace
1007 static void unhook_irq_events(void)
1009 osnoise_arch_unregister();
1010 unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1011 unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1014 #ifndef CONFIG_PREEMPT_RT
1016 * trace_softirq_entry_callback - Note the starting of a softirq
1018 * Save the starting time of a softirq. As softirqs are non-preemptive to
1019 * other softirqs, it is safe to use a single variable (ons_var->softirq)
1020 * to save the statistics. The arrival_time is used to report... the
1021 * arrival time. The delta_start is used to compute the duration at the
1022 * softirq exit handler. See cond_move_softirq_delta_start().
1024 static void trace_softirq_entry_callback(void *data, unsigned int vec_nr)
1026 struct osnoise_variables *osn_var = this_cpu_osn_var();
1028 if (!osn_var->sampling)
1031 * This value will be used in the report, but not to compute
1032 * the execution time, so it is safe to get it unsafe.
1034 osn_var->softirq.arrival_time = time_get();
1035 set_int_safe_time(osn_var, &osn_var->softirq.delta_start);
1036 osn_var->softirq.count++;
1038 local_inc(&osn_var->int_counter);
1042 * trace_softirq_exit_callback - Note the end of an softirq
1044 * Computes the duration of the softirq noise, and trace it. Also discounts the
1045 * interference from other sources of noise could be currently being accounted.
1047 static void trace_softirq_exit_callback(void *data, unsigned int vec_nr)
1049 struct osnoise_variables *osn_var = this_cpu_osn_var();
1052 if (!osn_var->sampling)
1055 if (unlikely(timerlat_enabled()))
1056 if (!timerlat_softirq_exit(osn_var))
1059 duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);
1060 trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);
1061 cond_move_thread_delta_start(osn_var, duration);
1062 osn_var->softirq.arrival_time = 0;
1066 * hook_softirq_events - Hook softirq handling events
1068 * This function hooks the softirq related callbacks to the respective trace
1071 static int hook_softirq_events(void)
1075 ret = register_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1079 ret = register_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1081 goto out_unreg_entry;
1086 unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1092 * unhook_softirq_events - Unhook softirq handling events
1094 * This function hooks the softirq related callbacks to the respective trace
1097 static void unhook_softirq_events(void)
1099 unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1100 unregister_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1102 #else /* CONFIG_PREEMPT_RT */
1104 * softirq are threads on the PREEMPT_RT mode.
1106 static int hook_softirq_events(void)
1110 static void unhook_softirq_events(void)
1116 * thread_entry - Record the starting of a thread noise window
1118 * It saves the context switch time for a noisy thread, and increments
1119 * the interference counters.
1122 thread_entry(struct osnoise_variables *osn_var, struct task_struct *t)
1124 if (!osn_var->sampling)
1127 * The arrival time will be used in the report, but not to compute
1128 * the execution time, so it is safe to get it unsafe.
1130 osn_var->thread.arrival_time = time_get();
1132 set_int_safe_time(osn_var, &osn_var->thread.delta_start);
1134 osn_var->thread.count++;
1135 local_inc(&osn_var->int_counter);
1139 * thread_exit - Report the end of a thread noise window
1141 * It computes the total noise from a thread, tracing if needed.
1144 thread_exit(struct osnoise_variables *osn_var, struct task_struct *t)
1148 if (!osn_var->sampling)
1151 if (unlikely(timerlat_enabled()))
1152 if (!timerlat_thread_exit(osn_var))
1155 duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);
1157 trace_thread_noise(t, osn_var->thread.arrival_time, duration);
1159 osn_var->thread.arrival_time = 0;
1163 * trace_sched_switch - sched:sched_switch trace event handler
1165 * This function is hooked to the sched:sched_switch trace event, and it is
1166 * used to record the beginning and to report the end of a thread noise window.
1169 trace_sched_switch_callback(void *data, bool preempt,
1170 struct task_struct *p,
1171 struct task_struct *n,
1172 unsigned int prev_state)
1174 struct osnoise_variables *osn_var = this_cpu_osn_var();
1176 if (p->pid != osn_var->pid)
1177 thread_exit(osn_var, p);
1179 if (n->pid != osn_var->pid)
1180 thread_entry(osn_var, n);
1184 * hook_thread_events - Hook the insturmentation for thread noise
1186 * Hook the osnoise tracer callbacks to handle the noise from other
1187 * threads on the necessary kernel events.
1189 static int hook_thread_events(void)
1193 ret = register_trace_sched_switch(trace_sched_switch_callback, NULL);
1201 * unhook_thread_events - *nhook the insturmentation for thread noise
1203 * Unook the osnoise tracer callbacks to handle the noise from other
1204 * threads on the necessary kernel events.
1206 static void unhook_thread_events(void)
1208 unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1212 * save_osn_sample_stats - Save the osnoise_sample statistics
1214 * Save the osnoise_sample statistics before the sampling phase. These
1215 * values will be used later to compute the diff betwneen the statistics
1216 * before and after the osnoise sampling.
1219 save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1221 s->nmi_count = osn_var->nmi.count;
1222 s->irq_count = osn_var->irq.count;
1223 s->softirq_count = osn_var->softirq.count;
1224 s->thread_count = osn_var->thread.count;
1228 * diff_osn_sample_stats - Compute the osnoise_sample statistics
1230 * After a sample period, compute the difference on the osnoise_sample
1231 * statistics. The struct osnoise_sample *s contains the statistics saved via
1232 * save_osn_sample_stats() before the osnoise sampling.
1235 diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1237 s->nmi_count = osn_var->nmi.count - s->nmi_count;
1238 s->irq_count = osn_var->irq.count - s->irq_count;
1239 s->softirq_count = osn_var->softirq.count - s->softirq_count;
1240 s->thread_count = osn_var->thread.count - s->thread_count;
1244 * osnoise_stop_tracing - Stop tracing and the tracer.
1246 static __always_inline void osnoise_stop_tracing(void)
1248 struct osnoise_instance *inst;
1249 struct trace_array *tr;
1252 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1254 trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
1255 "stop tracing hit on cpu %d\n", smp_processor_id());
1257 tracer_tracing_off(tr);
1263 * notify_new_max_latency - Notify a new max latency via fsnotify interface.
1265 static void notify_new_max_latency(u64 latency)
1267 struct osnoise_instance *inst;
1268 struct trace_array *tr;
1271 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1273 if (tracer_tracing_is_on(tr) && tr->max_latency < latency) {
1274 tr->max_latency = latency;
1275 latency_fsnotify(tr);
1282 * run_osnoise - Sample the time and look for osnoise
1284 * Used to capture the time, looking for potential osnoise latency repeatedly.
1285 * Different from hwlat_detector, it is called with preemption and interrupts
1286 * enabled. This allows irqs, softirqs and threads to run, interfering on the
1287 * osnoise sampling thread, as they would do with a regular thread.
1289 static int run_osnoise(void)
1291 struct osnoise_variables *osn_var = this_cpu_osn_var();
1292 u64 start, sample, last_sample;
1293 u64 last_int_count, int_count;
1294 s64 noise = 0, max_noise = 0;
1295 s64 total, last_total = 0;
1296 struct osnoise_sample s;
1297 unsigned int threshold;
1298 u64 runtime, stop_in;
1304 * Considers the current thread as the workload.
1306 osn_var->pid = current->pid;
1309 * Save the current stats for the diff
1311 save_osn_sample_stats(osn_var, &s);
1314 * if threshold is 0, use the default value of 5 us.
1316 threshold = tracing_thresh ? : 5000;
1319 * Make sure NMIs see sampling first
1321 osn_var->sampling = true;
1325 * Transform the *_us config to nanoseconds to avoid the
1326 * division on the main loop.
1328 runtime = osnoise_data.sample_runtime * NSEC_PER_USEC;
1329 stop_in = osnoise_data.stop_tracing * NSEC_PER_USEC;
1339 last_int_count = set_int_safe_time(osn_var, &last_sample);
1345 int_count = set_int_safe_time(osn_var, &sample);
1347 noise = time_sub(sample, last_sample);
1350 * This shouldn't happen.
1353 osnoise_taint("negative noise!");
1360 total = time_sub(sample, start);
1363 * Check for possible overflows.
1365 if (total < last_total) {
1366 osnoise_taint("total overflow!");
1372 if (noise >= threshold) {
1373 int interference = int_count - last_int_count;
1375 if (noise > max_noise)
1383 trace_sample_threshold(last_sample, noise, interference);
1385 if (osnoise_data.stop_tracing)
1386 if (noise > stop_in)
1387 osnoise_stop_tracing();
1391 * In some cases, notably when running on a nohz_full CPU with
1392 * a stopped tick PREEMPT_RCU has no way to account for QSs.
1393 * This will eventually cause unwarranted noise as PREEMPT_RCU
1394 * will force preemption as the means of ending the current
1395 * grace period. We avoid this problem by calling
1396 * rcu_momentary_dyntick_idle(), which performs a zero duration
1397 * EQS allowing PREEMPT_RCU to end the current grace period.
1398 * This call shouldn't be wrapped inside an RCU critical
1401 * Note that in non PREEMPT_RCU kernels QSs are handled through
1404 if (IS_ENABLED(CONFIG_PREEMPT_RCU)) {
1405 local_irq_disable();
1406 rcu_momentary_dyntick_idle();
1411 * For the non-preemptive kernel config: let threads runs, if
1416 last_sample = sample;
1417 last_int_count = int_count;
1419 } while (total < runtime && !kthread_should_stop());
1422 * Finish the above in the view for interrupts.
1426 osn_var->sampling = false;
1429 * Make sure sampling data is no longer updated.
1436 s.noise = time_to_us(sum_noise);
1437 s.runtime = time_to_us(total);
1438 s.max_sample = time_to_us(max_noise);
1439 s.hw_count = hw_count;
1441 /* Save interference stats info */
1442 diff_osn_sample_stats(osn_var, &s);
1444 trace_osnoise_sample(&s);
1446 notify_new_max_latency(max_noise);
1448 if (osnoise_data.stop_tracing_total)
1449 if (s.noise > osnoise_data.stop_tracing_total)
1450 osnoise_stop_tracing();
1457 static struct cpumask osnoise_cpumask;
1458 static struct cpumask save_cpumask;
1461 * osnoise_sleep - sleep until the next period
1463 static void osnoise_sleep(void)
1468 mutex_lock(&interface_lock);
1469 interval = osnoise_data.sample_period - osnoise_data.sample_runtime;
1470 mutex_unlock(&interface_lock);
1473 * differently from hwlat_detector, the osnoise tracer can run
1474 * without a pause because preemption is on.
1477 /* Let synchronize_rcu_tasks() make progress */
1478 cond_resched_tasks_rcu_qs();
1482 wake_time = ktime_add_us(ktime_get(), interval);
1483 __set_current_state(TASK_INTERRUPTIBLE);
1485 while (schedule_hrtimeout_range(&wake_time, 0, HRTIMER_MODE_ABS)) {
1486 if (kthread_should_stop())
1492 * osnoise_main - The osnoise detection kernel thread
1494 * Calls run_osnoise() function to measure the osnoise for the configured runtime,
1497 static int osnoise_main(void *data)
1500 while (!kthread_should_stop()) {
1508 #ifdef CONFIG_TIMERLAT_TRACER
1510 * timerlat_irq - hrtimer handler for timerlat.
1512 static enum hrtimer_restart timerlat_irq(struct hrtimer *timer)
1514 struct osnoise_variables *osn_var = this_cpu_osn_var();
1515 struct timerlat_variables *tlat;
1516 struct timerlat_sample s;
1521 * I am not sure if the timer was armed for this CPU. So, get
1522 * the timerlat struct from the timer itself, not from this
1525 tlat = container_of(timer, struct timerlat_variables, timer);
1527 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1530 * Enable the osnoise: events for thread an softirq.
1532 tlat->tracing_thread = true;
1534 osn_var->thread.arrival_time = time_get();
1537 * A hardirq is running: the timer IRQ. It is for sure preempting
1538 * a thread, and potentially preempting a softirq.
1540 * At this point, it is not interesting to know the duration of the
1541 * preempted thread (and maybe softirq), but how much time they will
1542 * delay the beginning of the execution of the timer thread.
1544 * To get the correct (net) delay added by the softirq, its delta_start
1545 * is set as the IRQ one. In this way, at the return of the IRQ, the delta
1546 * start of the sofitrq will be zeroed, accounting then only the time
1549 * The thread follows the same principle. However, if a softirq is
1550 * running, the thread needs to receive the softirq delta_start. The
1551 * reason being is that the softirq will be the last to be unfolded,
1552 * resseting the thread delay to zero.
1554 * The PREEMPT_RT is a special case, though. As softirqs run as threads
1555 * on RT, moving the thread is enough.
1557 if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) {
1558 copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1559 &osn_var->softirq.delta_start);
1561 copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,
1562 &osn_var->irq.delta_start);
1564 copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1565 &osn_var->irq.delta_start);
1569 * Compute the current time with the expected time.
1571 diff = now - tlat->abs_period;
1574 s.seqnum = tlat->count;
1575 s.timer_latency = diff;
1576 s.context = IRQ_CONTEXT;
1578 trace_timerlat_sample(&s);
1580 if (osnoise_data.stop_tracing) {
1581 if (time_to_us(diff) >= osnoise_data.stop_tracing) {
1584 * At this point, if stop_tracing is set and <= print_stack,
1585 * print_stack is set and would be printed in the thread handler.
1587 * Thus, print the stack trace as it is helpful to define the
1588 * root cause of an IRQ latency.
1590 if (osnoise_data.stop_tracing <= osnoise_data.print_stack) {
1591 timerlat_save_stack(0);
1592 timerlat_dump_stack(time_to_us(diff));
1595 osnoise_stop_tracing();
1596 notify_new_max_latency(diff);
1598 wake_up_process(tlat->kthread);
1600 return HRTIMER_NORESTART;
1604 wake_up_process(tlat->kthread);
1606 if (osnoise_data.print_stack)
1607 timerlat_save_stack(0);
1609 return HRTIMER_NORESTART;
1613 * wait_next_period - Wait for the next period for timerlat
1615 static int wait_next_period(struct timerlat_variables *tlat)
1617 ktime_t next_abs_period, now;
1618 u64 rel_period = osnoise_data.timerlat_period * 1000;
1620 now = hrtimer_cb_get_time(&tlat->timer);
1621 next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1624 * Save the next abs_period.
1626 tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1629 * If the new abs_period is in the past, skip the activation.
1631 while (ktime_compare(now, next_abs_period) > 0) {
1632 next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1633 tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1636 set_current_state(TASK_INTERRUPTIBLE);
1638 hrtimer_start(&tlat->timer, next_abs_period, HRTIMER_MODE_ABS_PINNED_HARD);
1644 * timerlat_main- Timerlat main
1646 static int timerlat_main(void *data)
1648 struct osnoise_variables *osn_var = this_cpu_osn_var();
1649 struct timerlat_variables *tlat = this_cpu_tmr_var();
1650 struct timerlat_sample s;
1651 struct sched_param sp;
1655 * Make the thread RT, that is how cyclictest is usually used.
1657 sp.sched_priority = DEFAULT_TIMERLAT_PRIO;
1658 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1661 tlat->tracing_thread = false;
1663 hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1664 tlat->timer.function = timerlat_irq;
1665 tlat->kthread = current;
1666 osn_var->pid = current->pid;
1668 * Anotate the arrival time.
1670 tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
1672 wait_next_period(tlat);
1674 osn_var->sampling = 1;
1676 while (!kthread_should_stop()) {
1677 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1678 diff = now - tlat->abs_period;
1680 s.seqnum = tlat->count;
1681 s.timer_latency = diff;
1682 s.context = THREAD_CONTEXT;
1684 trace_timerlat_sample(&s);
1686 notify_new_max_latency(diff);
1688 timerlat_dump_stack(time_to_us(diff));
1690 tlat->tracing_thread = false;
1691 if (osnoise_data.stop_tracing_total)
1692 if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
1693 osnoise_stop_tracing();
1695 wait_next_period(tlat);
1698 hrtimer_cancel(&tlat->timer);
1701 #else /* CONFIG_TIMERLAT_TRACER */
1702 static int timerlat_main(void *data)
1706 #endif /* CONFIG_TIMERLAT_TRACER */
1709 * stop_kthread - stop a workload thread
1711 static void stop_kthread(unsigned int cpu)
1713 struct task_struct *kthread;
1715 kthread = per_cpu(per_cpu_osnoise_var, cpu).kthread;
1717 kthread_stop(kthread);
1718 per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
1722 * stop_per_cpu_kthread - Stop per-cpu threads
1724 * Stop the osnoise sampling htread. Use this on unload and at system
1727 static void stop_per_cpu_kthreads(void)
1733 for_each_online_cpu(cpu)
1740 * start_kthread - Start a workload tread
1742 static int start_kthread(unsigned int cpu)
1744 struct task_struct *kthread;
1745 void *main = osnoise_main;
1748 if (timerlat_enabled()) {
1749 snprintf(comm, 24, "timerlat/%d", cpu);
1750 main = timerlat_main;
1752 snprintf(comm, 24, "osnoise/%d", cpu);
1755 kthread = kthread_run_on_cpu(main, NULL, cpu, comm);
1757 if (IS_ERR(kthread)) {
1758 pr_err(BANNER "could not start sampling thread\n");
1759 stop_per_cpu_kthreads();
1763 per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread;
1769 * start_per_cpu_kthread - Kick off per-cpu osnoise sampling kthreads
1771 * This starts the kernel thread that will look for osnoise on many
1774 static int start_per_cpu_kthreads(void)
1776 struct cpumask *current_mask = &save_cpumask;
1782 * Run only on online CPUs in which osnoise is allowed to run.
1784 cpumask_and(current_mask, cpu_online_mask, &osnoise_cpumask);
1786 for_each_possible_cpu(cpu)
1787 per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
1789 for_each_cpu(cpu, current_mask) {
1790 retval = start_kthread(cpu);
1793 stop_per_cpu_kthreads();
1803 #ifdef CONFIG_HOTPLUG_CPU
1804 static void osnoise_hotplug_workfn(struct work_struct *dummy)
1806 unsigned int cpu = smp_processor_id();
1808 mutex_lock(&trace_types_lock);
1810 if (!osnoise_has_registered_instances())
1811 goto out_unlock_trace;
1813 mutex_lock(&interface_lock);
1816 if (!cpumask_test_cpu(cpu, &osnoise_cpumask))
1823 mutex_unlock(&interface_lock);
1825 mutex_unlock(&trace_types_lock);
1828 static DECLARE_WORK(osnoise_hotplug_work, osnoise_hotplug_workfn);
1831 * osnoise_cpu_init - CPU hotplug online callback function
1833 static int osnoise_cpu_init(unsigned int cpu)
1835 schedule_work_on(cpu, &osnoise_hotplug_work);
1840 * osnoise_cpu_die - CPU hotplug offline callback function
1842 static int osnoise_cpu_die(unsigned int cpu)
1848 static void osnoise_init_hotplug_support(void)
1852 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/osnoise:online",
1853 osnoise_cpu_init, osnoise_cpu_die);
1855 pr_warn(BANNER "Error to init cpu hotplug support\n");
1859 #else /* CONFIG_HOTPLUG_CPU */
1860 static void osnoise_init_hotplug_support(void)
1864 #endif /* CONFIG_HOTPLUG_CPU */
1867 * osnoise_cpus_read - Read function for reading the "cpus" file
1868 * @filp: The active open file structure
1869 * @ubuf: The userspace provided buffer to read value into
1870 * @cnt: The maximum number of bytes to read
1871 * @ppos: The current "file" position
1873 * Prints the "cpus" output into the user-provided buffer.
1876 osnoise_cpus_read(struct file *filp, char __user *ubuf, size_t count,
1882 mutex_lock(&interface_lock);
1884 len = snprintf(NULL, 0, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask)) + 1;
1885 mask_str = kmalloc(len, GFP_KERNEL);
1891 len = snprintf(mask_str, len, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask));
1897 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
1902 mutex_unlock(&interface_lock);
1908 * osnoise_cpus_write - Write function for "cpus" entry
1909 * @filp: The active open file structure
1910 * @ubuf: The user buffer that contains the value to write
1911 * @cnt: The maximum number of bytes to write to "file"
1912 * @ppos: The current position in @file
1914 * This function provides a write implementation for the "cpus"
1915 * interface to the osnoise trace. By default, it lists all CPUs,
1916 * in this way, allowing osnoise threads to run on any online CPU
1917 * of the system. It serves to restrict the execution of osnoise to the
1918 * set of CPUs writing via this interface. Why not use "tracing_cpumask"?
1919 * Because the user might be interested in tracing what is running on
1920 * other CPUs. For instance, one might run osnoise in one HT CPU
1921 * while observing what is running on the sibling HT CPU.
1924 osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count,
1927 cpumask_var_t osnoise_cpumask_new;
1934 if (copy_from_user(buf, ubuf, count))
1937 if (!zalloc_cpumask_var(&osnoise_cpumask_new, GFP_KERNEL))
1940 err = cpulist_parse(buf, osnoise_cpumask_new);
1945 * trace_types_lock is taken to avoid concurrency on start/stop.
1947 mutex_lock(&trace_types_lock);
1948 running = osnoise_has_registered_instances();
1950 stop_per_cpu_kthreads();
1952 mutex_lock(&interface_lock);
1954 * osnoise_cpumask is read by CPU hotplug operations.
1958 cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new);
1961 mutex_unlock(&interface_lock);
1964 start_per_cpu_kthreads();
1965 mutex_unlock(&trace_types_lock);
1967 free_cpumask_var(osnoise_cpumask_new);
1971 free_cpumask_var(osnoise_cpumask_new);
1977 * osnoise/runtime_us: cannot be greater than the period.
1979 static struct trace_min_max_param osnoise_runtime = {
1980 .lock = &interface_lock,
1981 .val = &osnoise_data.sample_runtime,
1982 .max = &osnoise_data.sample_period,
1987 * osnoise/period_us: cannot be smaller than the runtime.
1989 static struct trace_min_max_param osnoise_period = {
1990 .lock = &interface_lock,
1991 .val = &osnoise_data.sample_period,
1993 .min = &osnoise_data.sample_runtime,
1997 * osnoise/stop_tracing_us: no limit.
1999 static struct trace_min_max_param osnoise_stop_tracing_in = {
2000 .lock = &interface_lock,
2001 .val = &osnoise_data.stop_tracing,
2007 * osnoise/stop_tracing_total_us: no limit.
2009 static struct trace_min_max_param osnoise_stop_tracing_total = {
2010 .lock = &interface_lock,
2011 .val = &osnoise_data.stop_tracing_total,
2016 #ifdef CONFIG_TIMERLAT_TRACER
2018 * osnoise/print_stack: print the stacktrace of the IRQ handler if the total
2019 * latency is higher than val.
2021 static struct trace_min_max_param osnoise_print_stack = {
2022 .lock = &interface_lock,
2023 .val = &osnoise_data.print_stack,
2029 * osnoise/timerlat_period: min 100 us, max 1 s
2031 u64 timerlat_min_period = 100;
2032 u64 timerlat_max_period = 1000000;
2033 static struct trace_min_max_param timerlat_period = {
2034 .lock = &interface_lock,
2035 .val = &osnoise_data.timerlat_period,
2036 .max = &timerlat_max_period,
2037 .min = &timerlat_min_period,
2041 static const struct file_operations cpus_fops = {
2042 .open = tracing_open_generic,
2043 .read = osnoise_cpus_read,
2044 .write = osnoise_cpus_write,
2045 .llseek = generic_file_llseek,
2048 #ifdef CONFIG_TIMERLAT_TRACER
2049 #ifdef CONFIG_STACKTRACE
2050 static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2054 tmp = tracefs_create_file("print_stack", TRACE_MODE_WRITE, top_dir,
2055 &osnoise_print_stack, &trace_min_max_fops);
2061 #else /* CONFIG_STACKTRACE */
2062 static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2066 #endif /* CONFIG_STACKTRACE */
2069 * init_timerlat_tracefs - A function to initialize the timerlat interface files
2071 static int init_timerlat_tracefs(struct dentry *top_dir)
2075 tmp = tracefs_create_file("timerlat_period_us", TRACE_MODE_WRITE, top_dir,
2076 &timerlat_period, &trace_min_max_fops);
2080 return init_timerlat_stack_tracefs(top_dir);
2082 #else /* CONFIG_TIMERLAT_TRACER */
2083 static int init_timerlat_tracefs(struct dentry *top_dir)
2087 #endif /* CONFIG_TIMERLAT_TRACER */
2090 * init_tracefs - A function to initialize the tracefs interface files
2092 * This function creates entries in tracefs for "osnoise" and "timerlat".
2093 * It creates these directories in the tracing directory, and within that
2094 * directory the use can change and view the configs.
2096 static int init_tracefs(void)
2098 struct dentry *top_dir;
2102 ret = tracing_init_dentry();
2106 top_dir = tracefs_create_dir("osnoise", NULL);
2110 tmp = tracefs_create_file("period_us", TRACE_MODE_WRITE, top_dir,
2111 &osnoise_period, &trace_min_max_fops);
2115 tmp = tracefs_create_file("runtime_us", TRACE_MODE_WRITE, top_dir,
2116 &osnoise_runtime, &trace_min_max_fops);
2120 tmp = tracefs_create_file("stop_tracing_us", TRACE_MODE_WRITE, top_dir,
2121 &osnoise_stop_tracing_in, &trace_min_max_fops);
2125 tmp = tracefs_create_file("stop_tracing_total_us", TRACE_MODE_WRITE, top_dir,
2126 &osnoise_stop_tracing_total, &trace_min_max_fops);
2130 tmp = trace_create_file("cpus", TRACE_MODE_WRITE, top_dir, NULL, &cpus_fops);
2134 ret = init_timerlat_tracefs(top_dir);
2141 tracefs_remove(top_dir);
2145 static int osnoise_hook_events(void)
2150 * Trace is already hooked, we are re-enabling from
2153 if (trace_osnoise_callback_enabled)
2156 retval = hook_irq_events();
2160 retval = hook_softirq_events();
2162 goto out_unhook_irq;
2164 retval = hook_thread_events();
2171 unhook_softirq_events();
2173 unhook_irq_events();
2177 static void osnoise_unhook_events(void)
2179 unhook_thread_events();
2180 unhook_softirq_events();
2181 unhook_irq_events();
2185 * osnoise_workload_start - start the workload and hook to events
2187 static int osnoise_workload_start(void)
2192 * Instances need to be registered after calling workload
2193 * start. Hence, if there is already an instance, the
2194 * workload was already registered. Otherwise, this
2195 * code is on the way to register the first instance,
2196 * and the workload will start.
2198 if (osnoise_has_registered_instances())
2201 osn_var_reset_all();
2203 retval = osnoise_hook_events();
2208 * Make sure that ftrace_nmi_enter/exit() see reset values
2209 * before enabling trace_osnoise_callback_enabled.
2212 trace_osnoise_callback_enabled = true;
2214 retval = start_per_cpu_kthreads();
2216 trace_osnoise_callback_enabled = false;
2218 * Make sure that ftrace_nmi_enter/exit() see
2219 * trace_osnoise_callback_enabled as false before continuing.
2223 osnoise_unhook_events();
2231 * osnoise_workload_stop - stop the workload and unhook the events
2233 static void osnoise_workload_stop(void)
2236 * Instances need to be unregistered before calling
2237 * stop. Hence, if there is a registered instance, more
2238 * than one instance is running, and the workload will not
2239 * yet stop. Otherwise, this code is on the way to disable
2240 * the last instance, and the workload can stop.
2242 if (osnoise_has_registered_instances())
2246 * If callbacks were already disabled in a previous stop
2247 * call, there is no need to disable then again.
2249 * For instance, this happens when tracing is stopped via:
2250 * echo 0 > tracing_on
2251 * echo nop > current_tracer.
2253 if (!trace_osnoise_callback_enabled)
2256 trace_osnoise_callback_enabled = false;
2258 * Make sure that ftrace_nmi_enter/exit() see
2259 * trace_osnoise_callback_enabled as false before continuing.
2263 stop_per_cpu_kthreads();
2265 osnoise_unhook_events();
2268 static void osnoise_tracer_start(struct trace_array *tr)
2273 * If the instance is already registered, there is no need to
2274 * register it again.
2276 if (osnoise_instance_registered(tr))
2279 retval = osnoise_workload_start();
2281 pr_err(BANNER "Error starting osnoise tracer\n");
2283 osnoise_register_instance(tr);
2286 static void osnoise_tracer_stop(struct trace_array *tr)
2288 osnoise_unregister_instance(tr);
2289 osnoise_workload_stop();
2292 static int osnoise_tracer_init(struct trace_array *tr)
2295 * Only allow osnoise tracer if timerlat tracer is not running
2298 if (timerlat_enabled())
2301 tr->max_latency = 0;
2303 osnoise_tracer_start(tr);
2307 static void osnoise_tracer_reset(struct trace_array *tr)
2309 osnoise_tracer_stop(tr);
2312 static struct tracer osnoise_tracer __read_mostly = {
2314 .init = osnoise_tracer_init,
2315 .reset = osnoise_tracer_reset,
2316 .start = osnoise_tracer_start,
2317 .stop = osnoise_tracer_stop,
2318 .print_header = print_osnoise_headers,
2319 .allow_instances = true,
2322 #ifdef CONFIG_TIMERLAT_TRACER
2323 static void timerlat_tracer_start(struct trace_array *tr)
2328 * If the instance is already registered, there is no need to
2329 * register it again.
2331 if (osnoise_instance_registered(tr))
2334 retval = osnoise_workload_start();
2336 pr_err(BANNER "Error starting timerlat tracer\n");
2338 osnoise_register_instance(tr);
2343 static void timerlat_tracer_stop(struct trace_array *tr)
2347 osnoise_unregister_instance(tr);
2350 * Instruct the threads to stop only if this is the last instance.
2352 if (!osnoise_has_registered_instances()) {
2353 for_each_online_cpu(cpu)
2354 per_cpu(per_cpu_osnoise_var, cpu).sampling = 0;
2357 osnoise_workload_stop();
2360 static int timerlat_tracer_init(struct trace_array *tr)
2363 * Only allow timerlat tracer if osnoise tracer is not running already.
2365 if (osnoise_has_registered_instances() && !osnoise_data.timerlat_tracer)
2369 * If this is the first instance, set timerlat_tracer to block
2370 * osnoise tracer start.
2372 if (!osnoise_has_registered_instances())
2373 osnoise_data.timerlat_tracer = 1;
2375 tr->max_latency = 0;
2376 timerlat_tracer_start(tr);
2381 static void timerlat_tracer_reset(struct trace_array *tr)
2383 timerlat_tracer_stop(tr);
2386 * If this is the last instance, reset timerlat_tracer allowing
2387 * osnoise to be started.
2389 if (!osnoise_has_registered_instances())
2390 osnoise_data.timerlat_tracer = 0;
2393 static struct tracer timerlat_tracer __read_mostly = {
2395 .init = timerlat_tracer_init,
2396 .reset = timerlat_tracer_reset,
2397 .start = timerlat_tracer_start,
2398 .stop = timerlat_tracer_stop,
2399 .print_header = print_timerlat_headers,
2400 .allow_instances = true,
2403 __init static int init_timerlat_tracer(void)
2405 return register_tracer(&timerlat_tracer);
2407 #else /* CONFIG_TIMERLAT_TRACER */
2408 __init static int init_timerlat_tracer(void)
2412 #endif /* CONFIG_TIMERLAT_TRACER */
2414 __init static int init_osnoise_tracer(void)
2418 mutex_init(&interface_lock);
2420 cpumask_copy(&osnoise_cpumask, cpu_all_mask);
2422 ret = register_tracer(&osnoise_tracer);
2424 pr_err(BANNER "Error registering osnoise!\n");
2428 ret = init_timerlat_tracer();
2430 pr_err(BANNER "Error registering timerlat!\n");
2434 osnoise_init_hotplug_support();
2436 INIT_LIST_HEAD_RCU(&osnoise_instances);
2442 late_initcall(init_osnoise_tracer);