1 // SPDX-License-Identifier: GPL-2.0
3 * OS Noise Tracer: computes the OS Noise suffered by a running thread.
4 * Timerlat Tracer: measures the wakeup latency of a timer triggered IRQ and thread.
6 * Based on "hwlat_detector" tracer by:
7 * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
8 * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
9 * With feedback from Clark Williams <williams@redhat.com>
11 * And also based on the rtsl tracer presented on:
12 * DE OLIVEIRA, Daniel Bristot, et al. Demystifying the real-time linux
13 * scheduling latency. In: 32nd Euromicro Conference on Real-Time Systems
14 * (ECRTS 2020). Schloss Dagstuhl-Leibniz-Zentrum fur Informatik, 2020.
16 * Copyright (C) 2021 Daniel Bristot de Oliveira, Red Hat, Inc. <bristot@redhat.com>
19 #include <linux/kthread.h>
20 #include <linux/tracefs.h>
21 #include <linux/uaccess.h>
22 #include <linux/cpumask.h>
23 #include <linux/delay.h>
24 #include <linux/sched/clock.h>
25 #include <uapi/linux/sched/types.h>
26 #include <linux/sched.h>
29 #ifdef CONFIG_X86_LOCAL_APIC
30 #include <asm/trace/irq_vectors.h>
31 #undef TRACE_INCLUDE_PATH
32 #undef TRACE_INCLUDE_FILE
33 #endif /* CONFIG_X86_LOCAL_APIC */
35 #include <trace/events/irq.h>
36 #include <trace/events/sched.h>
38 #define CREATE_TRACE_POINTS
39 #include <trace/events/osnoise.h>
44 #define BANNER "osnoise: "
45 #define DEFAULT_SAMPLE_PERIOD 1000000 /* 1s */
46 #define DEFAULT_SAMPLE_RUNTIME 1000000 /* 1s */
48 #define DEFAULT_TIMERLAT_PERIOD 1000 /* 1ms */
49 #define DEFAULT_TIMERLAT_PRIO 95 /* FIFO 95 */
52 * osnoise/options entries.
54 enum osnoise_options_index {
63 static const char * const osnoise_options_str[OSN_MAX] = {
67 "OSNOISE_PREEMPT_DISABLE",
68 "OSNOISE_IRQ_DISABLE" };
70 #define OSN_DEFAULT_OPTIONS 0x2
71 static unsigned long osnoise_options = OSN_DEFAULT_OPTIONS;
74 * trace_array of the enabled osnoise/timerlat instances.
76 struct osnoise_instance {
77 struct list_head list;
78 struct trace_array *tr;
81 static struct list_head osnoise_instances;
83 static bool osnoise_has_registered_instances(void)
85 return !!list_first_or_null_rcu(&osnoise_instances,
86 struct osnoise_instance,
91 * osnoise_instance_registered - check if a tr is already registered
93 static int osnoise_instance_registered(struct trace_array *tr)
95 struct osnoise_instance *inst;
99 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
109 * osnoise_register_instance - register a new trace instance
111 * Register a trace_array *tr in the list of instances running
112 * osnoise/timerlat tracers.
114 static int osnoise_register_instance(struct trace_array *tr)
116 struct osnoise_instance *inst;
119 * register/unregister serialization is provided by trace's
122 lockdep_assert_held(&trace_types_lock);
124 inst = kmalloc(sizeof(*inst), GFP_KERNEL);
128 INIT_LIST_HEAD_RCU(&inst->list);
130 list_add_tail_rcu(&inst->list, &osnoise_instances);
136 * osnoise_unregister_instance - unregister a registered trace instance
138 * Remove the trace_array *tr from the list of instances running
139 * osnoise/timerlat tracers.
141 static void osnoise_unregister_instance(struct trace_array *tr)
143 struct osnoise_instance *inst;
147 * register/unregister serialization is provided by trace's
150 list_for_each_entry_rcu(inst, &osnoise_instances, list,
151 lockdep_is_held(&trace_types_lock)) {
152 if (inst->tr == tr) {
153 list_del_rcu(&inst->list);
162 kvfree_rcu_mightsleep(inst);
182 #define IRQ_CONTEXT 0
183 #define THREAD_CONTEXT 1
185 * sofirq runtime info.
194 * thread runtime info.
203 * Runtime information: this structure saves the runtime information used by
204 * one sampling thread.
206 struct osnoise_variables {
207 struct task_struct *kthread;
212 struct osn_softirq softirq;
213 struct osn_thread thread;
218 * Per-cpu runtime information.
220 static DEFINE_PER_CPU(struct osnoise_variables, per_cpu_osnoise_var);
223 * this_cpu_osn_var - Return the per-cpu osnoise_variables on its relative CPU
225 static inline struct osnoise_variables *this_cpu_osn_var(void)
227 return this_cpu_ptr(&per_cpu_osnoise_var);
230 #ifdef CONFIG_TIMERLAT_TRACER
232 * Runtime information for the timer mode.
234 struct timerlat_variables {
235 struct task_struct *kthread;
236 struct hrtimer timer;
243 static DEFINE_PER_CPU(struct timerlat_variables, per_cpu_timerlat_var);
246 * this_cpu_tmr_var - Return the per-cpu timerlat_variables on its relative CPU
248 static inline struct timerlat_variables *this_cpu_tmr_var(void)
250 return this_cpu_ptr(&per_cpu_timerlat_var);
254 * tlat_var_reset - Reset the values of the given timerlat_variables
256 static inline void tlat_var_reset(void)
258 struct timerlat_variables *tlat_var;
261 * So far, all the values are initialized as 0, so
262 * zeroing the structure is perfect.
264 for_each_cpu(cpu, cpu_online_mask) {
265 tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
266 memset(tlat_var, 0, sizeof(*tlat_var));
269 #else /* CONFIG_TIMERLAT_TRACER */
270 #define tlat_var_reset() do {} while (0)
271 #endif /* CONFIG_TIMERLAT_TRACER */
274 * osn_var_reset - Reset the values of the given osnoise_variables
276 static inline void osn_var_reset(void)
278 struct osnoise_variables *osn_var;
282 * So far, all the values are initialized as 0, so
283 * zeroing the structure is perfect.
285 for_each_cpu(cpu, cpu_online_mask) {
286 osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
287 memset(osn_var, 0, sizeof(*osn_var));
292 * osn_var_reset_all - Reset the value of all per-cpu osnoise_variables
294 static inline void osn_var_reset_all(void)
301 * Tells NMIs to call back to the osnoise tracer to record timestamps.
303 bool trace_osnoise_callback_enabled;
306 * osnoise sample structure definition. Used to store the statistics of a
309 struct osnoise_sample {
310 u64 runtime; /* runtime */
311 u64 noise; /* noise */
312 u64 max_sample; /* max single noise sample */
313 int hw_count; /* # HW (incl. hypervisor) interference */
314 int nmi_count; /* # NMIs during this sample */
315 int irq_count; /* # IRQs during this sample */
316 int softirq_count; /* # softirqs during this sample */
317 int thread_count; /* # threads during this sample */
320 #ifdef CONFIG_TIMERLAT_TRACER
322 * timerlat sample structure definition. Used to store the statistics of
325 struct timerlat_sample {
326 u64 timer_latency; /* timer_latency */
327 unsigned int seqnum; /* unique sequence */
328 int context; /* timer context */
333 * Protect the interface.
335 static struct mutex interface_lock;
340 static struct osnoise_data {
341 u64 sample_period; /* total sampling period */
342 u64 sample_runtime; /* active sampling portion of period */
343 u64 stop_tracing; /* stop trace in the internal operation (loop/irq) */
344 u64 stop_tracing_total; /* stop trace in the final operation (report/thread) */
345 #ifdef CONFIG_TIMERLAT_TRACER
346 u64 timerlat_period; /* timerlat period */
347 u64 print_stack; /* print IRQ stack if total > */
348 int timerlat_tracer; /* timerlat tracer */
350 bool tainted; /* infor users and developers about a problem */
352 .sample_period = DEFAULT_SAMPLE_PERIOD,
353 .sample_runtime = DEFAULT_SAMPLE_RUNTIME,
355 .stop_tracing_total = 0,
356 #ifdef CONFIG_TIMERLAT_TRACER
358 .timerlat_period = DEFAULT_TIMERLAT_PERIOD,
359 .timerlat_tracer = 0,
363 #ifdef CONFIG_TIMERLAT_TRACER
364 static inline bool timerlat_enabled(void)
366 return osnoise_data.timerlat_tracer;
369 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
371 struct timerlat_variables *tlat_var = this_cpu_tmr_var();
373 * If the timerlat is enabled, but the irq handler did
374 * not run yet enabling timerlat_tracer, do not trace.
376 if (!tlat_var->tracing_thread) {
377 osn_var->softirq.arrival_time = 0;
378 osn_var->softirq.delta_start = 0;
384 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
386 struct timerlat_variables *tlat_var = this_cpu_tmr_var();
388 * If the timerlat is enabled, but the irq handler did
389 * not run yet enabling timerlat_tracer, do not trace.
391 if (!tlat_var->tracing_thread) {
392 osn_var->thread.delta_start = 0;
393 osn_var->thread.arrival_time = 0;
398 #else /* CONFIG_TIMERLAT_TRACER */
399 static inline bool timerlat_enabled(void)
404 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
408 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
414 #ifdef CONFIG_PREEMPT_RT
416 * Print the osnoise header info.
418 static void print_osnoise_headers(struct seq_file *s)
420 if (osnoise_data.tainted)
421 seq_puts(s, "# osnoise is tainted!\n");
423 seq_puts(s, "# _-------=> irqs-off\n");
424 seq_puts(s, "# / _------=> need-resched\n");
425 seq_puts(s, "# | / _-----=> need-resched-lazy\n");
426 seq_puts(s, "# || / _----=> hardirq/softirq\n");
427 seq_puts(s, "# ||| / _---=> preempt-depth\n");
428 seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
429 seq_puts(s, "# ||||| / _-=> migrate-disable\n");
431 seq_puts(s, "# |||||| / ");
432 seq_puts(s, " MAX\n");
434 seq_puts(s, "# ||||| / ");
435 seq_puts(s, " SINGLE Interference counters:\n");
437 seq_puts(s, "# ||||||| RUNTIME ");
438 seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
440 seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP IN US ");
441 seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
443 seq_puts(s, "# | | | ||||||| | | ");
444 seq_puts(s, " | | | | | | | |\n");
446 #else /* CONFIG_PREEMPT_RT */
447 static void print_osnoise_headers(struct seq_file *s)
449 if (osnoise_data.tainted)
450 seq_puts(s, "# osnoise is tainted!\n");
452 seq_puts(s, "# _-----=> irqs-off\n");
453 seq_puts(s, "# / _----=> need-resched\n");
454 seq_puts(s, "# | / _---=> hardirq/softirq\n");
455 seq_puts(s, "# || / _--=> preempt-depth\n");
456 seq_puts(s, "# ||| / _-=> migrate-disable ");
457 seq_puts(s, " MAX\n");
458 seq_puts(s, "# |||| / delay ");
459 seq_puts(s, " SINGLE Interference counters:\n");
461 seq_puts(s, "# ||||| RUNTIME ");
462 seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
464 seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP IN US ");
465 seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
467 seq_puts(s, "# | | | ||||| | | ");
468 seq_puts(s, " | | | | | | | |\n");
470 #endif /* CONFIG_PREEMPT_RT */
473 * osnoise_taint - report an osnoise error.
475 #define osnoise_taint(msg) ({ \
476 struct osnoise_instance *inst; \
477 struct trace_buffer *buffer; \
480 list_for_each_entry_rcu(inst, &osnoise_instances, list) { \
481 buffer = inst->tr->array_buffer.buffer; \
482 trace_array_printk_buf(buffer, _THIS_IP_, msg); \
485 osnoise_data.tainted = true; \
489 * Record an osnoise_sample into the tracer buffer.
492 __trace_osnoise_sample(struct osnoise_sample *sample, struct trace_buffer *buffer)
494 struct trace_event_call *call = &event_osnoise;
495 struct ring_buffer_event *event;
496 struct osnoise_entry *entry;
498 event = trace_buffer_lock_reserve(buffer, TRACE_OSNOISE, sizeof(*entry),
502 entry = ring_buffer_event_data(event);
503 entry->runtime = sample->runtime;
504 entry->noise = sample->noise;
505 entry->max_sample = sample->max_sample;
506 entry->hw_count = sample->hw_count;
507 entry->nmi_count = sample->nmi_count;
508 entry->irq_count = sample->irq_count;
509 entry->softirq_count = sample->softirq_count;
510 entry->thread_count = sample->thread_count;
512 if (!call_filter_check_discard(call, entry, buffer, event))
513 trace_buffer_unlock_commit_nostack(buffer, event);
517 * Record an osnoise_sample on all osnoise instances.
519 static void trace_osnoise_sample(struct osnoise_sample *sample)
521 struct osnoise_instance *inst;
522 struct trace_buffer *buffer;
525 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
526 buffer = inst->tr->array_buffer.buffer;
527 __trace_osnoise_sample(sample, buffer);
532 #ifdef CONFIG_TIMERLAT_TRACER
534 * Print the timerlat header info.
536 #ifdef CONFIG_PREEMPT_RT
537 static void print_timerlat_headers(struct seq_file *s)
539 seq_puts(s, "# _-------=> irqs-off\n");
540 seq_puts(s, "# / _------=> need-resched\n");
541 seq_puts(s, "# | / _-----=> need-resched-lazy\n");
542 seq_puts(s, "# || / _----=> hardirq/softirq\n");
543 seq_puts(s, "# ||| / _---=> preempt-depth\n");
544 seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
545 seq_puts(s, "# ||||| / _-=> migrate-disable\n");
546 seq_puts(s, "# |||||| /\n");
547 seq_puts(s, "# ||||||| ACTIVATION\n");
548 seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP ID ");
549 seq_puts(s, " CONTEXT LATENCY\n");
550 seq_puts(s, "# | | | ||||||| | | ");
551 seq_puts(s, " | |\n");
553 #else /* CONFIG_PREEMPT_RT */
554 static void print_timerlat_headers(struct seq_file *s)
556 seq_puts(s, "# _-----=> irqs-off\n");
557 seq_puts(s, "# / _----=> need-resched\n");
558 seq_puts(s, "# | / _---=> hardirq/softirq\n");
559 seq_puts(s, "# || / _--=> preempt-depth\n");
560 seq_puts(s, "# ||| / _-=> migrate-disable\n");
561 seq_puts(s, "# |||| / delay\n");
562 seq_puts(s, "# ||||| ACTIVATION\n");
563 seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP ID ");
564 seq_puts(s, " CONTEXT LATENCY\n");
565 seq_puts(s, "# | | | ||||| | | ");
566 seq_puts(s, " | |\n");
568 #endif /* CONFIG_PREEMPT_RT */
571 __trace_timerlat_sample(struct timerlat_sample *sample, struct trace_buffer *buffer)
573 struct trace_event_call *call = &event_osnoise;
574 struct ring_buffer_event *event;
575 struct timerlat_entry *entry;
577 event = trace_buffer_lock_reserve(buffer, TRACE_TIMERLAT, sizeof(*entry),
581 entry = ring_buffer_event_data(event);
582 entry->seqnum = sample->seqnum;
583 entry->context = sample->context;
584 entry->timer_latency = sample->timer_latency;
586 if (!call_filter_check_discard(call, entry, buffer, event))
587 trace_buffer_unlock_commit_nostack(buffer, event);
591 * Record an timerlat_sample into the tracer buffer.
593 static void trace_timerlat_sample(struct timerlat_sample *sample)
595 struct osnoise_instance *inst;
596 struct trace_buffer *buffer;
599 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
600 buffer = inst->tr->array_buffer.buffer;
601 __trace_timerlat_sample(sample, buffer);
606 #ifdef CONFIG_STACKTRACE
608 #define MAX_CALLS 256
611 * Stack trace will take place only at IRQ level, so, no need
612 * to control nesting here.
617 unsigned long calls[MAX_CALLS];
620 static DEFINE_PER_CPU(struct trace_stack, trace_stack);
623 * timerlat_save_stack - save a stack trace without printing
625 * Save the current stack trace without printing. The
626 * stack will be printed later, after the end of the measurement.
628 static void timerlat_save_stack(int skip)
630 unsigned int size, nr_entries;
631 struct trace_stack *fstack;
633 fstack = this_cpu_ptr(&trace_stack);
635 size = ARRAY_SIZE(fstack->calls);
637 nr_entries = stack_trace_save(fstack->calls, size, skip);
639 fstack->stack_size = nr_entries * sizeof(unsigned long);
640 fstack->nr_entries = nr_entries;
647 __timerlat_dump_stack(struct trace_buffer *buffer, struct trace_stack *fstack, unsigned int size)
649 struct trace_event_call *call = &event_osnoise;
650 struct ring_buffer_event *event;
651 struct stack_entry *entry;
653 event = trace_buffer_lock_reserve(buffer, TRACE_STACK, sizeof(*entry) + size,
658 entry = ring_buffer_event_data(event);
660 memcpy(&entry->caller, fstack->calls, size);
661 entry->size = fstack->nr_entries;
663 if (!call_filter_check_discard(call, entry, buffer, event))
664 trace_buffer_unlock_commit_nostack(buffer, event);
668 * timerlat_dump_stack - dump a stack trace previously saved
670 static void timerlat_dump_stack(u64 latency)
672 struct osnoise_instance *inst;
673 struct trace_buffer *buffer;
674 struct trace_stack *fstack;
678 * trace only if latency > print_stack config, if enabled.
680 if (!osnoise_data.print_stack || osnoise_data.print_stack > latency)
683 preempt_disable_notrace();
684 fstack = this_cpu_ptr(&trace_stack);
685 size = fstack->stack_size;
688 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
689 buffer = inst->tr->array_buffer.buffer;
690 __timerlat_dump_stack(buffer, fstack, size);
694 preempt_enable_notrace();
696 #else /* CONFIG_STACKTRACE */
697 #define timerlat_dump_stack(u64 latency) do {} while (0)
698 #define timerlat_save_stack(a) do {} while (0)
699 #endif /* CONFIG_STACKTRACE */
700 #endif /* CONFIG_TIMERLAT_TRACER */
703 * Macros to encapsulate the time capturing infrastructure.
705 #define time_get() trace_clock_local()
706 #define time_to_us(x) div_u64(x, 1000)
707 #define time_sub(a, b) ((a) - (b))
710 * cond_move_irq_delta_start - Forward the delta_start of a running IRQ
712 * If an IRQ is preempted by an NMI, its delta_start is pushed forward
713 * to discount the NMI interference.
715 * See get_int_safe_duration().
718 cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)
720 if (osn_var->irq.delta_start)
721 osn_var->irq.delta_start += duration;
724 #ifndef CONFIG_PREEMPT_RT
726 * cond_move_softirq_delta_start - Forward the delta_start of a running softirq.
728 * If a softirq is preempted by an IRQ or NMI, its delta_start is pushed
729 * forward to discount the interference.
731 * See get_int_safe_duration().
734 cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)
736 if (osn_var->softirq.delta_start)
737 osn_var->softirq.delta_start += duration;
739 #else /* CONFIG_PREEMPT_RT */
740 #define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)
744 * cond_move_thread_delta_start - Forward the delta_start of a running thread
746 * If a noisy thread is preempted by an softirq, IRQ or NMI, its delta_start
747 * is pushed forward to discount the interference.
749 * See get_int_safe_duration().
752 cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)
754 if (osn_var->thread.delta_start)
755 osn_var->thread.delta_start += duration;
759 * get_int_safe_duration - Get the duration of a window
761 * The irq, softirq and thread varaibles need to have its duration without
762 * the interference from higher priority interrupts. Instead of keeping a
763 * variable to discount the interrupt interference from these variables, the
764 * starting time of these variables are pushed forward with the interrupt's
765 * duration. In this way, a single variable is used to:
767 * - Know if a given window is being measured.
768 * - Account its duration.
769 * - Discount the interference.
771 * To avoid getting inconsistent values, e.g.,:
775 * delta_start -= int duration;
777 * duration = now - delta_start;
779 * result: negative duration if the variable duration before the
780 * interrupt was smaller than the interrupt execution.
782 * A counter of interrupts is used. If the counter increased, try
783 * to capture an interference safe duration.
786 get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start)
788 u64 int_counter, now;
792 int_counter = local_read(&osn_var->int_counter);
793 /* synchronize with interrupts */
797 duration = (now - *delta_start);
799 /* synchronize with interrupts */
801 } while (int_counter != local_read(&osn_var->int_counter));
804 * This is an evidence of race conditions that cause
805 * a value to be "discounted" too much.
808 osnoise_taint("Negative duration!\n");
817 * set_int_safe_time - Save the current time on *time, aware of interference
819 * Get the time, taking into consideration a possible interference from
820 * higher priority interrupts.
822 * See get_int_safe_duration() for an explanation.
825 set_int_safe_time(struct osnoise_variables *osn_var, u64 *time)
830 int_counter = local_read(&osn_var->int_counter);
831 /* synchronize with interrupts */
836 /* synchronize with interrupts */
838 } while (int_counter != local_read(&osn_var->int_counter));
843 #ifdef CONFIG_TIMERLAT_TRACER
845 * copy_int_safe_time - Copy *src into *desc aware of interference
848 copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src)
853 int_counter = local_read(&osn_var->int_counter);
854 /* synchronize with interrupts */
859 /* synchronize with interrupts */
861 } while (int_counter != local_read(&osn_var->int_counter));
865 #endif /* CONFIG_TIMERLAT_TRACER */
868 * trace_osnoise_callback - NMI entry/exit callback
870 * This function is called at the entry and exit NMI code. The bool enter
871 * distinguishes between either case. This function is used to note a NMI
872 * occurrence, compute the noise caused by the NMI, and to remove the noise
873 * it is potentially causing on other interference variables.
875 void trace_osnoise_callback(bool enter)
877 struct osnoise_variables *osn_var = this_cpu_osn_var();
880 if (!osn_var->sampling)
884 * Currently trace_clock_local() calls sched_clock() and the
885 * generic version is not NMI safe.
887 if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK)) {
889 osn_var->nmi.delta_start = time_get();
890 local_inc(&osn_var->int_counter);
892 duration = time_get() - osn_var->nmi.delta_start;
894 trace_nmi_noise(osn_var->nmi.delta_start, duration);
896 cond_move_irq_delta_start(osn_var, duration);
897 cond_move_softirq_delta_start(osn_var, duration);
898 cond_move_thread_delta_start(osn_var, duration);
903 osn_var->nmi.count++;
907 * osnoise_trace_irq_entry - Note the starting of an IRQ
909 * Save the starting time of an IRQ. As IRQs are non-preemptive to other IRQs,
910 * it is safe to use a single variable (ons_var->irq) to save the statistics.
911 * The arrival_time is used to report... the arrival time. The delta_start
912 * is used to compute the duration at the IRQ exit handler. See
913 * cond_move_irq_delta_start().
915 void osnoise_trace_irq_entry(int id)
917 struct osnoise_variables *osn_var = this_cpu_osn_var();
919 if (!osn_var->sampling)
922 * This value will be used in the report, but not to compute
923 * the execution time, so it is safe to get it unsafe.
925 osn_var->irq.arrival_time = time_get();
926 set_int_safe_time(osn_var, &osn_var->irq.delta_start);
927 osn_var->irq.count++;
929 local_inc(&osn_var->int_counter);
933 * osnoise_irq_exit - Note the end of an IRQ, sava data and trace
935 * Computes the duration of the IRQ noise, and trace it. Also discounts the
936 * interference from other sources of noise could be currently being accounted.
938 void osnoise_trace_irq_exit(int id, const char *desc)
940 struct osnoise_variables *osn_var = this_cpu_osn_var();
943 if (!osn_var->sampling)
946 duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);
947 trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);
948 osn_var->irq.arrival_time = 0;
949 cond_move_softirq_delta_start(osn_var, duration);
950 cond_move_thread_delta_start(osn_var, duration);
954 * trace_irqentry_callback - Callback to the irq:irq_entry traceevent
956 * Used to note the starting of an IRQ occurece.
958 static void trace_irqentry_callback(void *data, int irq,
959 struct irqaction *action)
961 osnoise_trace_irq_entry(irq);
965 * trace_irqexit_callback - Callback to the irq:irq_exit traceevent
967 * Used to note the end of an IRQ occurece.
969 static void trace_irqexit_callback(void *data, int irq,
970 struct irqaction *action, int ret)
972 osnoise_trace_irq_exit(irq, action->name);
976 * arch specific register function.
978 int __weak osnoise_arch_register(void)
984 * arch specific unregister function.
986 void __weak osnoise_arch_unregister(void)
992 * hook_irq_events - Hook IRQ handling events
994 * This function hooks the IRQ related callbacks to the respective trace
997 static int hook_irq_events(void)
1001 ret = register_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1005 ret = register_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1007 goto out_unregister_entry;
1009 ret = osnoise_arch_register();
1016 unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1017 out_unregister_entry:
1018 unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1024 * unhook_irq_events - Unhook IRQ handling events
1026 * This function unhooks the IRQ related callbacks to the respective trace
1029 static void unhook_irq_events(void)
1031 osnoise_arch_unregister();
1032 unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1033 unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1036 #ifndef CONFIG_PREEMPT_RT
1038 * trace_softirq_entry_callback - Note the starting of a softirq
1040 * Save the starting time of a softirq. As softirqs are non-preemptive to
1041 * other softirqs, it is safe to use a single variable (ons_var->softirq)
1042 * to save the statistics. The arrival_time is used to report... the
1043 * arrival time. The delta_start is used to compute the duration at the
1044 * softirq exit handler. See cond_move_softirq_delta_start().
1046 static void trace_softirq_entry_callback(void *data, unsigned int vec_nr)
1048 struct osnoise_variables *osn_var = this_cpu_osn_var();
1050 if (!osn_var->sampling)
1053 * This value will be used in the report, but not to compute
1054 * the execution time, so it is safe to get it unsafe.
1056 osn_var->softirq.arrival_time = time_get();
1057 set_int_safe_time(osn_var, &osn_var->softirq.delta_start);
1058 osn_var->softirq.count++;
1060 local_inc(&osn_var->int_counter);
1064 * trace_softirq_exit_callback - Note the end of an softirq
1066 * Computes the duration of the softirq noise, and trace it. Also discounts the
1067 * interference from other sources of noise could be currently being accounted.
1069 static void trace_softirq_exit_callback(void *data, unsigned int vec_nr)
1071 struct osnoise_variables *osn_var = this_cpu_osn_var();
1074 if (!osn_var->sampling)
1077 if (unlikely(timerlat_enabled()))
1078 if (!timerlat_softirq_exit(osn_var))
1081 duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);
1082 trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);
1083 cond_move_thread_delta_start(osn_var, duration);
1084 osn_var->softirq.arrival_time = 0;
1088 * hook_softirq_events - Hook softirq handling events
1090 * This function hooks the softirq related callbacks to the respective trace
1093 static int hook_softirq_events(void)
1097 ret = register_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1101 ret = register_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1103 goto out_unreg_entry;
1108 unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1114 * unhook_softirq_events - Unhook softirq handling events
1116 * This function hooks the softirq related callbacks to the respective trace
1119 static void unhook_softirq_events(void)
1121 unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1122 unregister_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1124 #else /* CONFIG_PREEMPT_RT */
1126 * softirq are threads on the PREEMPT_RT mode.
1128 static int hook_softirq_events(void)
1132 static void unhook_softirq_events(void)
1138 * thread_entry - Record the starting of a thread noise window
1140 * It saves the context switch time for a noisy thread, and increments
1141 * the interference counters.
1144 thread_entry(struct osnoise_variables *osn_var, struct task_struct *t)
1146 if (!osn_var->sampling)
1149 * The arrival time will be used in the report, but not to compute
1150 * the execution time, so it is safe to get it unsafe.
1152 osn_var->thread.arrival_time = time_get();
1154 set_int_safe_time(osn_var, &osn_var->thread.delta_start);
1156 osn_var->thread.count++;
1157 local_inc(&osn_var->int_counter);
1161 * thread_exit - Report the end of a thread noise window
1163 * It computes the total noise from a thread, tracing if needed.
1166 thread_exit(struct osnoise_variables *osn_var, struct task_struct *t)
1170 if (!osn_var->sampling)
1173 if (unlikely(timerlat_enabled()))
1174 if (!timerlat_thread_exit(osn_var))
1177 duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);
1179 trace_thread_noise(t, osn_var->thread.arrival_time, duration);
1181 osn_var->thread.arrival_time = 0;
1185 * trace_sched_switch - sched:sched_switch trace event handler
1187 * This function is hooked to the sched:sched_switch trace event, and it is
1188 * used to record the beginning and to report the end of a thread noise window.
1191 trace_sched_switch_callback(void *data, bool preempt,
1192 struct task_struct *p,
1193 struct task_struct *n,
1194 unsigned int prev_state)
1196 struct osnoise_variables *osn_var = this_cpu_osn_var();
1197 int workload = test_bit(OSN_WORKLOAD, &osnoise_options);
1199 if ((p->pid != osn_var->pid) || !workload)
1200 thread_exit(osn_var, p);
1202 if ((n->pid != osn_var->pid) || !workload)
1203 thread_entry(osn_var, n);
1207 * hook_thread_events - Hook the insturmentation for thread noise
1209 * Hook the osnoise tracer callbacks to handle the noise from other
1210 * threads on the necessary kernel events.
1212 static int hook_thread_events(void)
1216 ret = register_trace_sched_switch(trace_sched_switch_callback, NULL);
1224 * unhook_thread_events - *nhook the insturmentation for thread noise
1226 * Unook the osnoise tracer callbacks to handle the noise from other
1227 * threads on the necessary kernel events.
1229 static void unhook_thread_events(void)
1231 unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1235 * save_osn_sample_stats - Save the osnoise_sample statistics
1237 * Save the osnoise_sample statistics before the sampling phase. These
1238 * values will be used later to compute the diff betwneen the statistics
1239 * before and after the osnoise sampling.
1242 save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1244 s->nmi_count = osn_var->nmi.count;
1245 s->irq_count = osn_var->irq.count;
1246 s->softirq_count = osn_var->softirq.count;
1247 s->thread_count = osn_var->thread.count;
1251 * diff_osn_sample_stats - Compute the osnoise_sample statistics
1253 * After a sample period, compute the difference on the osnoise_sample
1254 * statistics. The struct osnoise_sample *s contains the statistics saved via
1255 * save_osn_sample_stats() before the osnoise sampling.
1258 diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1260 s->nmi_count = osn_var->nmi.count - s->nmi_count;
1261 s->irq_count = osn_var->irq.count - s->irq_count;
1262 s->softirq_count = osn_var->softirq.count - s->softirq_count;
1263 s->thread_count = osn_var->thread.count - s->thread_count;
1267 * osnoise_stop_tracing - Stop tracing and the tracer.
1269 static __always_inline void osnoise_stop_tracing(void)
1271 struct osnoise_instance *inst;
1272 struct trace_array *tr;
1275 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1277 trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
1278 "stop tracing hit on cpu %d\n", smp_processor_id());
1280 if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options))
1281 panic("tracer hit stop condition on CPU %d\n", smp_processor_id());
1283 tracer_tracing_off(tr);
1289 * notify_new_max_latency - Notify a new max latency via fsnotify interface.
1291 static void notify_new_max_latency(u64 latency)
1293 struct osnoise_instance *inst;
1294 struct trace_array *tr;
1297 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1299 if (tracer_tracing_is_on(tr) && tr->max_latency < latency) {
1300 tr->max_latency = latency;
1301 latency_fsnotify(tr);
1308 * run_osnoise - Sample the time and look for osnoise
1310 * Used to capture the time, looking for potential osnoise latency repeatedly.
1311 * Different from hwlat_detector, it is called with preemption and interrupts
1312 * enabled. This allows irqs, softirqs and threads to run, interfering on the
1313 * osnoise sampling thread, as they would do with a regular thread.
1315 static int run_osnoise(void)
1317 bool disable_irq = test_bit(OSN_IRQ_DISABLE, &osnoise_options);
1318 struct osnoise_variables *osn_var = this_cpu_osn_var();
1319 u64 start, sample, last_sample;
1320 u64 last_int_count, int_count;
1321 s64 noise = 0, max_noise = 0;
1322 s64 total, last_total = 0;
1323 struct osnoise_sample s;
1324 bool disable_preemption;
1325 unsigned int threshold;
1326 u64 runtime, stop_in;
1332 * Disabling preemption is only required if IRQs are enabled,
1333 * and the options is set on.
1335 disable_preemption = !disable_irq && test_bit(OSN_PREEMPT_DISABLE, &osnoise_options);
1338 * Considers the current thread as the workload.
1340 osn_var->pid = current->pid;
1343 * Save the current stats for the diff
1345 save_osn_sample_stats(osn_var, &s);
1348 * if threshold is 0, use the default value of 5 us.
1350 threshold = tracing_thresh ? : 5000;
1353 * Apply PREEMPT and IRQ disabled options.
1356 local_irq_disable();
1358 if (disable_preemption)
1362 * Make sure NMIs see sampling first
1364 osn_var->sampling = true;
1368 * Transform the *_us config to nanoseconds to avoid the
1369 * division on the main loop.
1371 runtime = osnoise_data.sample_runtime * NSEC_PER_USEC;
1372 stop_in = osnoise_data.stop_tracing * NSEC_PER_USEC;
1382 last_int_count = set_int_safe_time(osn_var, &last_sample);
1388 int_count = set_int_safe_time(osn_var, &sample);
1390 noise = time_sub(sample, last_sample);
1393 * This shouldn't happen.
1396 osnoise_taint("negative noise!");
1403 total = time_sub(sample, start);
1406 * Check for possible overflows.
1408 if (total < last_total) {
1409 osnoise_taint("total overflow!");
1415 if (noise >= threshold) {
1416 int interference = int_count - last_int_count;
1418 if (noise > max_noise)
1426 trace_sample_threshold(last_sample, noise, interference);
1428 if (osnoise_data.stop_tracing)
1429 if (noise > stop_in)
1430 osnoise_stop_tracing();
1434 * In some cases, notably when running on a nohz_full CPU with
1435 * a stopped tick PREEMPT_RCU has no way to account for QSs.
1436 * This will eventually cause unwarranted noise as PREEMPT_RCU
1437 * will force preemption as the means of ending the current
1438 * grace period. We avoid this problem by calling
1439 * rcu_momentary_dyntick_idle(), which performs a zero duration
1440 * EQS allowing PREEMPT_RCU to end the current grace period.
1441 * This call shouldn't be wrapped inside an RCU critical
1444 * Note that in non PREEMPT_RCU kernels QSs are handled through
1447 if (IS_ENABLED(CONFIG_PREEMPT_RCU)) {
1449 local_irq_disable();
1451 rcu_momentary_dyntick_idle();
1458 * For the non-preemptive kernel config: let threads runs, if
1459 * they so wish, unless set not do to so.
1461 if (!disable_irq && !disable_preemption)
1464 last_sample = sample;
1465 last_int_count = int_count;
1467 } while (total < runtime && !kthread_should_stop());
1470 * Finish the above in the view for interrupts.
1474 osn_var->sampling = false;
1477 * Make sure sampling data is no longer updated.
1482 * Return to the preemptive state.
1484 if (disable_preemption)
1493 s.noise = time_to_us(sum_noise);
1494 s.runtime = time_to_us(total);
1495 s.max_sample = time_to_us(max_noise);
1496 s.hw_count = hw_count;
1498 /* Save interference stats info */
1499 diff_osn_sample_stats(osn_var, &s);
1501 trace_osnoise_sample(&s);
1503 notify_new_max_latency(max_noise);
1505 if (osnoise_data.stop_tracing_total)
1506 if (s.noise > osnoise_data.stop_tracing_total)
1507 osnoise_stop_tracing();
1514 static struct cpumask osnoise_cpumask;
1515 static struct cpumask save_cpumask;
1518 * osnoise_sleep - sleep until the next period
1520 static void osnoise_sleep(void)
1525 mutex_lock(&interface_lock);
1526 interval = osnoise_data.sample_period - osnoise_data.sample_runtime;
1527 mutex_unlock(&interface_lock);
1530 * differently from hwlat_detector, the osnoise tracer can run
1531 * without a pause because preemption is on.
1534 /* Let synchronize_rcu_tasks() make progress */
1535 cond_resched_tasks_rcu_qs();
1539 wake_time = ktime_add_us(ktime_get(), interval);
1540 __set_current_state(TASK_INTERRUPTIBLE);
1542 while (schedule_hrtimeout(&wake_time, HRTIMER_MODE_ABS)) {
1543 if (kthread_should_stop())
1549 * osnoise_main - The osnoise detection kernel thread
1551 * Calls run_osnoise() function to measure the osnoise for the configured runtime,
1554 static int osnoise_main(void *data)
1557 while (!kthread_should_stop()) {
1565 #ifdef CONFIG_TIMERLAT_TRACER
1567 * timerlat_irq - hrtimer handler for timerlat.
1569 static enum hrtimer_restart timerlat_irq(struct hrtimer *timer)
1571 struct osnoise_variables *osn_var = this_cpu_osn_var();
1572 struct timerlat_variables *tlat;
1573 struct timerlat_sample s;
1578 * I am not sure if the timer was armed for this CPU. So, get
1579 * the timerlat struct from the timer itself, not from this
1582 tlat = container_of(timer, struct timerlat_variables, timer);
1584 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1587 * Enable the osnoise: events for thread an softirq.
1589 tlat->tracing_thread = true;
1591 osn_var->thread.arrival_time = time_get();
1594 * A hardirq is running: the timer IRQ. It is for sure preempting
1595 * a thread, and potentially preempting a softirq.
1597 * At this point, it is not interesting to know the duration of the
1598 * preempted thread (and maybe softirq), but how much time they will
1599 * delay the beginning of the execution of the timer thread.
1601 * To get the correct (net) delay added by the softirq, its delta_start
1602 * is set as the IRQ one. In this way, at the return of the IRQ, the delta
1603 * start of the sofitrq will be zeroed, accounting then only the time
1606 * The thread follows the same principle. However, if a softirq is
1607 * running, the thread needs to receive the softirq delta_start. The
1608 * reason being is that the softirq will be the last to be unfolded,
1609 * resseting the thread delay to zero.
1611 * The PREEMPT_RT is a special case, though. As softirqs run as threads
1612 * on RT, moving the thread is enough.
1614 if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) {
1615 copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1616 &osn_var->softirq.delta_start);
1618 copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,
1619 &osn_var->irq.delta_start);
1621 copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1622 &osn_var->irq.delta_start);
1626 * Compute the current time with the expected time.
1628 diff = now - tlat->abs_period;
1631 s.seqnum = tlat->count;
1632 s.timer_latency = diff;
1633 s.context = IRQ_CONTEXT;
1635 trace_timerlat_sample(&s);
1637 if (osnoise_data.stop_tracing) {
1638 if (time_to_us(diff) >= osnoise_data.stop_tracing) {
1641 * At this point, if stop_tracing is set and <= print_stack,
1642 * print_stack is set and would be printed in the thread handler.
1644 * Thus, print the stack trace as it is helpful to define the
1645 * root cause of an IRQ latency.
1647 if (osnoise_data.stop_tracing <= osnoise_data.print_stack) {
1648 timerlat_save_stack(0);
1649 timerlat_dump_stack(time_to_us(diff));
1652 osnoise_stop_tracing();
1653 notify_new_max_latency(diff);
1655 return HRTIMER_NORESTART;
1659 wake_up_process(tlat->kthread);
1661 if (osnoise_data.print_stack)
1662 timerlat_save_stack(0);
1664 return HRTIMER_NORESTART;
1668 * wait_next_period - Wait for the next period for timerlat
1670 static int wait_next_period(struct timerlat_variables *tlat)
1672 ktime_t next_abs_period, now;
1673 u64 rel_period = osnoise_data.timerlat_period * 1000;
1675 now = hrtimer_cb_get_time(&tlat->timer);
1676 next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1679 * Save the next abs_period.
1681 tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1684 * If the new abs_period is in the past, skip the activation.
1686 while (ktime_compare(now, next_abs_period) > 0) {
1687 next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1688 tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1691 set_current_state(TASK_INTERRUPTIBLE);
1693 hrtimer_start(&tlat->timer, next_abs_period, HRTIMER_MODE_ABS_PINNED_HARD);
1699 * timerlat_main- Timerlat main
1701 static int timerlat_main(void *data)
1703 struct osnoise_variables *osn_var = this_cpu_osn_var();
1704 struct timerlat_variables *tlat = this_cpu_tmr_var();
1705 struct timerlat_sample s;
1706 struct sched_param sp;
1710 * Make the thread RT, that is how cyclictest is usually used.
1712 sp.sched_priority = DEFAULT_TIMERLAT_PRIO;
1713 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1716 tlat->tracing_thread = false;
1718 hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1719 tlat->timer.function = timerlat_irq;
1720 tlat->kthread = current;
1721 osn_var->pid = current->pid;
1723 * Anotate the arrival time.
1725 tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
1727 wait_next_period(tlat);
1729 osn_var->sampling = 1;
1731 while (!kthread_should_stop()) {
1732 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1733 diff = now - tlat->abs_period;
1735 s.seqnum = tlat->count;
1736 s.timer_latency = diff;
1737 s.context = THREAD_CONTEXT;
1739 trace_timerlat_sample(&s);
1741 notify_new_max_latency(diff);
1743 timerlat_dump_stack(time_to_us(diff));
1745 tlat->tracing_thread = false;
1746 if (osnoise_data.stop_tracing_total)
1747 if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
1748 osnoise_stop_tracing();
1750 wait_next_period(tlat);
1753 hrtimer_cancel(&tlat->timer);
1756 #else /* CONFIG_TIMERLAT_TRACER */
1757 static int timerlat_main(void *data)
1761 #endif /* CONFIG_TIMERLAT_TRACER */
1764 * stop_kthread - stop a workload thread
1766 static void stop_kthread(unsigned int cpu)
1768 struct task_struct *kthread;
1770 kthread = per_cpu(per_cpu_osnoise_var, cpu).kthread;
1772 kthread_stop(kthread);
1773 per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
1775 if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
1776 per_cpu(per_cpu_osnoise_var, cpu).sampling = false;
1784 * stop_per_cpu_kthread - Stop per-cpu threads
1786 * Stop the osnoise sampling htread. Use this on unload and at system
1789 static void stop_per_cpu_kthreads(void)
1795 for_each_online_cpu(cpu)
1802 * start_kthread - Start a workload tread
1804 static int start_kthread(unsigned int cpu)
1806 struct task_struct *kthread;
1807 void *main = osnoise_main;
1810 if (timerlat_enabled()) {
1811 snprintf(comm, 24, "timerlat/%d", cpu);
1812 main = timerlat_main;
1814 /* if no workload, just return */
1815 if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
1816 per_cpu(per_cpu_osnoise_var, cpu).sampling = true;
1821 snprintf(comm, 24, "osnoise/%d", cpu);
1824 kthread = kthread_run_on_cpu(main, NULL, cpu, comm);
1826 if (IS_ERR(kthread)) {
1827 pr_err(BANNER "could not start sampling thread\n");
1828 stop_per_cpu_kthreads();
1832 per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread;
1838 * start_per_cpu_kthread - Kick off per-cpu osnoise sampling kthreads
1840 * This starts the kernel thread that will look for osnoise on many
1843 static int start_per_cpu_kthreads(void)
1845 struct cpumask *current_mask = &save_cpumask;
1851 * Run only on online CPUs in which osnoise is allowed to run.
1853 cpumask_and(current_mask, cpu_online_mask, &osnoise_cpumask);
1855 for_each_possible_cpu(cpu)
1856 per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
1858 for_each_cpu(cpu, current_mask) {
1859 retval = start_kthread(cpu);
1862 stop_per_cpu_kthreads();
1872 #ifdef CONFIG_HOTPLUG_CPU
1873 static void osnoise_hotplug_workfn(struct work_struct *dummy)
1875 unsigned int cpu = smp_processor_id();
1877 mutex_lock(&trace_types_lock);
1879 if (!osnoise_has_registered_instances())
1880 goto out_unlock_trace;
1882 mutex_lock(&interface_lock);
1885 if (!cpumask_test_cpu(cpu, &osnoise_cpumask))
1892 mutex_unlock(&interface_lock);
1894 mutex_unlock(&trace_types_lock);
1897 static DECLARE_WORK(osnoise_hotplug_work, osnoise_hotplug_workfn);
1900 * osnoise_cpu_init - CPU hotplug online callback function
1902 static int osnoise_cpu_init(unsigned int cpu)
1904 schedule_work_on(cpu, &osnoise_hotplug_work);
1909 * osnoise_cpu_die - CPU hotplug offline callback function
1911 static int osnoise_cpu_die(unsigned int cpu)
1917 static void osnoise_init_hotplug_support(void)
1921 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/osnoise:online",
1922 osnoise_cpu_init, osnoise_cpu_die);
1924 pr_warn(BANNER "Error to init cpu hotplug support\n");
1928 #else /* CONFIG_HOTPLUG_CPU */
1929 static void osnoise_init_hotplug_support(void)
1933 #endif /* CONFIG_HOTPLUG_CPU */
1936 * seq file functions for the osnoise/options file.
1938 static void *s_options_start(struct seq_file *s, loff_t *pos)
1942 mutex_lock(&interface_lock);
1944 if (option >= OSN_MAX)
1950 static void *s_options_next(struct seq_file *s, void *v, loff_t *pos)
1952 int option = ++(*pos);
1954 if (option >= OSN_MAX)
1960 static int s_options_show(struct seq_file *s, void *v)
1965 if (option == OSN_DEFAULTS) {
1966 if (osnoise_options == OSN_DEFAULT_OPTIONS)
1967 seq_printf(s, "%s", osnoise_options_str[option]);
1969 seq_printf(s, "NO_%s", osnoise_options_str[option]);
1973 if (test_bit(option, &osnoise_options))
1974 seq_printf(s, "%s", osnoise_options_str[option]);
1976 seq_printf(s, "NO_%s", osnoise_options_str[option]);
1979 if (option != OSN_MAX)
1985 static void s_options_stop(struct seq_file *s, void *v)
1988 mutex_unlock(&interface_lock);
1991 static const struct seq_operations osnoise_options_seq_ops = {
1992 .start = s_options_start,
1993 .next = s_options_next,
1994 .show = s_options_show,
1995 .stop = s_options_stop
1998 static int osnoise_options_open(struct inode *inode, struct file *file)
2000 return seq_open(file, &osnoise_options_seq_ops);
2004 * osnoise_options_write - Write function for "options" entry
2005 * @filp: The active open file structure
2006 * @ubuf: The user buffer that contains the value to write
2007 * @cnt: The maximum number of bytes to write to "file"
2008 * @ppos: The current position in @file
2010 * Writing the option name sets the option, writing the "NO_"
2011 * prefix in front of the option name disables it.
2013 * Writing "DEFAULTS" resets the option values to the default ones.
2015 static ssize_t osnoise_options_write(struct file *filp, const char __user *ubuf,
2016 size_t cnt, loff_t *ppos)
2018 int running, option, enable, retval;
2019 char buf[256], *option_str;
2024 if (copy_from_user(buf, ubuf, cnt))
2029 if (strncmp(buf, "NO_", 3)) {
2030 option_str = strstrip(buf);
2033 option_str = strstrip(&buf[3]);
2037 option = match_string(osnoise_options_str, OSN_MAX, option_str);
2042 * trace_types_lock is taken to avoid concurrency on start/stop.
2044 mutex_lock(&trace_types_lock);
2045 running = osnoise_has_registered_instances();
2047 stop_per_cpu_kthreads();
2049 mutex_lock(&interface_lock);
2051 * avoid CPU hotplug operations that might read options.
2058 if (option == OSN_DEFAULTS)
2059 osnoise_options = OSN_DEFAULT_OPTIONS;
2061 set_bit(option, &osnoise_options);
2063 if (option == OSN_DEFAULTS)
2066 clear_bit(option, &osnoise_options);
2070 mutex_unlock(&interface_lock);
2073 start_per_cpu_kthreads();
2074 mutex_unlock(&trace_types_lock);
2080 * osnoise_cpus_read - Read function for reading the "cpus" file
2081 * @filp: The active open file structure
2082 * @ubuf: The userspace provided buffer to read value into
2083 * @cnt: The maximum number of bytes to read
2084 * @ppos: The current "file" position
2086 * Prints the "cpus" output into the user-provided buffer.
2089 osnoise_cpus_read(struct file *filp, char __user *ubuf, size_t count,
2095 mutex_lock(&interface_lock);
2097 len = snprintf(NULL, 0, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask)) + 1;
2098 mask_str = kmalloc(len, GFP_KERNEL);
2104 len = snprintf(mask_str, len, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask));
2110 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
2115 mutex_unlock(&interface_lock);
2121 * osnoise_cpus_write - Write function for "cpus" entry
2122 * @filp: The active open file structure
2123 * @ubuf: The user buffer that contains the value to write
2124 * @cnt: The maximum number of bytes to write to "file"
2125 * @ppos: The current position in @file
2127 * This function provides a write implementation for the "cpus"
2128 * interface to the osnoise trace. By default, it lists all CPUs,
2129 * in this way, allowing osnoise threads to run on any online CPU
2130 * of the system. It serves to restrict the execution of osnoise to the
2131 * set of CPUs writing via this interface. Why not use "tracing_cpumask"?
2132 * Because the user might be interested in tracing what is running on
2133 * other CPUs. For instance, one might run osnoise in one HT CPU
2134 * while observing what is running on the sibling HT CPU.
2137 osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count,
2140 cpumask_var_t osnoise_cpumask_new;
2147 if (copy_from_user(buf, ubuf, count))
2150 if (!zalloc_cpumask_var(&osnoise_cpumask_new, GFP_KERNEL))
2153 err = cpulist_parse(buf, osnoise_cpumask_new);
2158 * trace_types_lock is taken to avoid concurrency on start/stop.
2160 mutex_lock(&trace_types_lock);
2161 running = osnoise_has_registered_instances();
2163 stop_per_cpu_kthreads();
2165 mutex_lock(&interface_lock);
2167 * osnoise_cpumask is read by CPU hotplug operations.
2171 cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new);
2174 mutex_unlock(&interface_lock);
2177 start_per_cpu_kthreads();
2178 mutex_unlock(&trace_types_lock);
2180 free_cpumask_var(osnoise_cpumask_new);
2184 free_cpumask_var(osnoise_cpumask_new);
2190 * osnoise/runtime_us: cannot be greater than the period.
2192 static struct trace_min_max_param osnoise_runtime = {
2193 .lock = &interface_lock,
2194 .val = &osnoise_data.sample_runtime,
2195 .max = &osnoise_data.sample_period,
2200 * osnoise/period_us: cannot be smaller than the runtime.
2202 static struct trace_min_max_param osnoise_period = {
2203 .lock = &interface_lock,
2204 .val = &osnoise_data.sample_period,
2206 .min = &osnoise_data.sample_runtime,
2210 * osnoise/stop_tracing_us: no limit.
2212 static struct trace_min_max_param osnoise_stop_tracing_in = {
2213 .lock = &interface_lock,
2214 .val = &osnoise_data.stop_tracing,
2220 * osnoise/stop_tracing_total_us: no limit.
2222 static struct trace_min_max_param osnoise_stop_tracing_total = {
2223 .lock = &interface_lock,
2224 .val = &osnoise_data.stop_tracing_total,
2229 #ifdef CONFIG_TIMERLAT_TRACER
2231 * osnoise/print_stack: print the stacktrace of the IRQ handler if the total
2232 * latency is higher than val.
2234 static struct trace_min_max_param osnoise_print_stack = {
2235 .lock = &interface_lock,
2236 .val = &osnoise_data.print_stack,
2242 * osnoise/timerlat_period: min 100 us, max 1 s
2244 static u64 timerlat_min_period = 100;
2245 static u64 timerlat_max_period = 1000000;
2246 static struct trace_min_max_param timerlat_period = {
2247 .lock = &interface_lock,
2248 .val = &osnoise_data.timerlat_period,
2249 .max = &timerlat_max_period,
2250 .min = &timerlat_min_period,
2254 static const struct file_operations cpus_fops = {
2255 .open = tracing_open_generic,
2256 .read = osnoise_cpus_read,
2257 .write = osnoise_cpus_write,
2258 .llseek = generic_file_llseek,
2261 static const struct file_operations osnoise_options_fops = {
2262 .open = osnoise_options_open,
2264 .llseek = seq_lseek,
2265 .release = seq_release,
2266 .write = osnoise_options_write
2269 #ifdef CONFIG_TIMERLAT_TRACER
2270 #ifdef CONFIG_STACKTRACE
2271 static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2275 tmp = tracefs_create_file("print_stack", TRACE_MODE_WRITE, top_dir,
2276 &osnoise_print_stack, &trace_min_max_fops);
2282 #else /* CONFIG_STACKTRACE */
2283 static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2287 #endif /* CONFIG_STACKTRACE */
2290 * init_timerlat_tracefs - A function to initialize the timerlat interface files
2292 static int init_timerlat_tracefs(struct dentry *top_dir)
2296 tmp = tracefs_create_file("timerlat_period_us", TRACE_MODE_WRITE, top_dir,
2297 &timerlat_period, &trace_min_max_fops);
2301 return init_timerlat_stack_tracefs(top_dir);
2303 #else /* CONFIG_TIMERLAT_TRACER */
2304 static int init_timerlat_tracefs(struct dentry *top_dir)
2308 #endif /* CONFIG_TIMERLAT_TRACER */
2311 * init_tracefs - A function to initialize the tracefs interface files
2313 * This function creates entries in tracefs for "osnoise" and "timerlat".
2314 * It creates these directories in the tracing directory, and within that
2315 * directory the use can change and view the configs.
2317 static int init_tracefs(void)
2319 struct dentry *top_dir;
2323 ret = tracing_init_dentry();
2327 top_dir = tracefs_create_dir("osnoise", NULL);
2331 tmp = tracefs_create_file("period_us", TRACE_MODE_WRITE, top_dir,
2332 &osnoise_period, &trace_min_max_fops);
2336 tmp = tracefs_create_file("runtime_us", TRACE_MODE_WRITE, top_dir,
2337 &osnoise_runtime, &trace_min_max_fops);
2341 tmp = tracefs_create_file("stop_tracing_us", TRACE_MODE_WRITE, top_dir,
2342 &osnoise_stop_tracing_in, &trace_min_max_fops);
2346 tmp = tracefs_create_file("stop_tracing_total_us", TRACE_MODE_WRITE, top_dir,
2347 &osnoise_stop_tracing_total, &trace_min_max_fops);
2351 tmp = trace_create_file("cpus", TRACE_MODE_WRITE, top_dir, NULL, &cpus_fops);
2355 tmp = trace_create_file("options", TRACE_MODE_WRITE, top_dir, NULL,
2356 &osnoise_options_fops);
2360 ret = init_timerlat_tracefs(top_dir);
2367 tracefs_remove(top_dir);
2371 static int osnoise_hook_events(void)
2376 * Trace is already hooked, we are re-enabling from
2379 if (trace_osnoise_callback_enabled)
2382 retval = hook_irq_events();
2386 retval = hook_softirq_events();
2388 goto out_unhook_irq;
2390 retval = hook_thread_events();
2397 unhook_softirq_events();
2399 unhook_irq_events();
2403 static void osnoise_unhook_events(void)
2405 unhook_thread_events();
2406 unhook_softirq_events();
2407 unhook_irq_events();
2411 * osnoise_workload_start - start the workload and hook to events
2413 static int osnoise_workload_start(void)
2418 * Instances need to be registered after calling workload
2419 * start. Hence, if there is already an instance, the
2420 * workload was already registered. Otherwise, this
2421 * code is on the way to register the first instance,
2422 * and the workload will start.
2424 if (osnoise_has_registered_instances())
2427 osn_var_reset_all();
2429 retval = osnoise_hook_events();
2434 * Make sure that ftrace_nmi_enter/exit() see reset values
2435 * before enabling trace_osnoise_callback_enabled.
2438 trace_osnoise_callback_enabled = true;
2440 retval = start_per_cpu_kthreads();
2442 trace_osnoise_callback_enabled = false;
2444 * Make sure that ftrace_nmi_enter/exit() see
2445 * trace_osnoise_callback_enabled as false before continuing.
2449 osnoise_unhook_events();
2457 * osnoise_workload_stop - stop the workload and unhook the events
2459 static void osnoise_workload_stop(void)
2462 * Instances need to be unregistered before calling
2463 * stop. Hence, if there is a registered instance, more
2464 * than one instance is running, and the workload will not
2465 * yet stop. Otherwise, this code is on the way to disable
2466 * the last instance, and the workload can stop.
2468 if (osnoise_has_registered_instances())
2472 * If callbacks were already disabled in a previous stop
2473 * call, there is no need to disable then again.
2475 * For instance, this happens when tracing is stopped via:
2476 * echo 0 > tracing_on
2477 * echo nop > current_tracer.
2479 if (!trace_osnoise_callback_enabled)
2482 trace_osnoise_callback_enabled = false;
2484 * Make sure that ftrace_nmi_enter/exit() see
2485 * trace_osnoise_callback_enabled as false before continuing.
2489 stop_per_cpu_kthreads();
2491 osnoise_unhook_events();
2494 static void osnoise_tracer_start(struct trace_array *tr)
2499 * If the instance is already registered, there is no need to
2500 * register it again.
2502 if (osnoise_instance_registered(tr))
2505 retval = osnoise_workload_start();
2507 pr_err(BANNER "Error starting osnoise tracer\n");
2509 osnoise_register_instance(tr);
2512 static void osnoise_tracer_stop(struct trace_array *tr)
2514 osnoise_unregister_instance(tr);
2515 osnoise_workload_stop();
2518 static int osnoise_tracer_init(struct trace_array *tr)
2521 * Only allow osnoise tracer if timerlat tracer is not running
2524 if (timerlat_enabled())
2527 tr->max_latency = 0;
2529 osnoise_tracer_start(tr);
2533 static void osnoise_tracer_reset(struct trace_array *tr)
2535 osnoise_tracer_stop(tr);
2538 static struct tracer osnoise_tracer __read_mostly = {
2540 .init = osnoise_tracer_init,
2541 .reset = osnoise_tracer_reset,
2542 .start = osnoise_tracer_start,
2543 .stop = osnoise_tracer_stop,
2544 .print_header = print_osnoise_headers,
2545 .allow_instances = true,
2548 #ifdef CONFIG_TIMERLAT_TRACER
2549 static void timerlat_tracer_start(struct trace_array *tr)
2554 * If the instance is already registered, there is no need to
2555 * register it again.
2557 if (osnoise_instance_registered(tr))
2560 retval = osnoise_workload_start();
2562 pr_err(BANNER "Error starting timerlat tracer\n");
2564 osnoise_register_instance(tr);
2569 static void timerlat_tracer_stop(struct trace_array *tr)
2573 osnoise_unregister_instance(tr);
2576 * Instruct the threads to stop only if this is the last instance.
2578 if (!osnoise_has_registered_instances()) {
2579 for_each_online_cpu(cpu)
2580 per_cpu(per_cpu_osnoise_var, cpu).sampling = 0;
2583 osnoise_workload_stop();
2586 static int timerlat_tracer_init(struct trace_array *tr)
2589 * Only allow timerlat tracer if osnoise tracer is not running already.
2591 if (osnoise_has_registered_instances() && !osnoise_data.timerlat_tracer)
2595 * If this is the first instance, set timerlat_tracer to block
2596 * osnoise tracer start.
2598 if (!osnoise_has_registered_instances())
2599 osnoise_data.timerlat_tracer = 1;
2601 tr->max_latency = 0;
2602 timerlat_tracer_start(tr);
2607 static void timerlat_tracer_reset(struct trace_array *tr)
2609 timerlat_tracer_stop(tr);
2612 * If this is the last instance, reset timerlat_tracer allowing
2613 * osnoise to be started.
2615 if (!osnoise_has_registered_instances())
2616 osnoise_data.timerlat_tracer = 0;
2619 static struct tracer timerlat_tracer __read_mostly = {
2621 .init = timerlat_tracer_init,
2622 .reset = timerlat_tracer_reset,
2623 .start = timerlat_tracer_start,
2624 .stop = timerlat_tracer_stop,
2625 .print_header = print_timerlat_headers,
2626 .allow_instances = true,
2629 __init static int init_timerlat_tracer(void)
2631 return register_tracer(&timerlat_tracer);
2633 #else /* CONFIG_TIMERLAT_TRACER */
2634 __init static int init_timerlat_tracer(void)
2638 #endif /* CONFIG_TIMERLAT_TRACER */
2640 __init static int init_osnoise_tracer(void)
2644 mutex_init(&interface_lock);
2646 cpumask_copy(&osnoise_cpumask, cpu_all_mask);
2648 ret = register_tracer(&osnoise_tracer);
2650 pr_err(BANNER "Error registering osnoise!\n");
2654 ret = init_timerlat_tracer();
2656 pr_err(BANNER "Error registering timerlat!\n");
2660 osnoise_init_hotplug_support();
2662 INIT_LIST_HEAD_RCU(&osnoise_instances);
2668 late_initcall(init_osnoise_tracer);