1 // SPDX-License-Identifier: GPL-2.0
3 * OS Noise Tracer: computes the OS Noise suffered by a running thread.
4 * Timerlat Tracer: measures the wakeup latency of a timer triggered IRQ and thread.
6 * Based on "hwlat_detector" tracer by:
7 * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
8 * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
9 * With feedback from Clark Williams <williams@redhat.com>
11 * And also based on the rtsl tracer presented on:
12 * DE OLIVEIRA, Daniel Bristot, et al. Demystifying the real-time linux
13 * scheduling latency. In: 32nd Euromicro Conference on Real-Time Systems
14 * (ECRTS 2020). Schloss Dagstuhl-Leibniz-Zentrum fur Informatik, 2020.
16 * Copyright (C) 2021 Daniel Bristot de Oliveira, Red Hat, Inc. <bristot@redhat.com>
19 #include <linux/kthread.h>
20 #include <linux/tracefs.h>
21 #include <linux/uaccess.h>
22 #include <linux/cpumask.h>
23 #include <linux/delay.h>
24 #include <linux/sched/clock.h>
25 #include <uapi/linux/sched/types.h>
26 #include <linux/sched.h>
29 #ifdef CONFIG_X86_LOCAL_APIC
30 #include <asm/trace/irq_vectors.h>
31 #undef TRACE_INCLUDE_PATH
32 #undef TRACE_INCLUDE_FILE
33 #endif /* CONFIG_X86_LOCAL_APIC */
35 #include <trace/events/irq.h>
36 #include <trace/events/sched.h>
38 #define CREATE_TRACE_POINTS
39 #include <trace/events/osnoise.h>
44 #define BANNER "osnoise: "
45 #define DEFAULT_SAMPLE_PERIOD 1000000 /* 1s */
46 #define DEFAULT_SAMPLE_RUNTIME 1000000 /* 1s */
48 #define DEFAULT_TIMERLAT_PERIOD 1000 /* 1ms */
49 #define DEFAULT_TIMERLAT_PRIO 95 /* FIFO 95 */
52 * osnoise/options entries.
54 enum osnoise_options_index {
63 static const char * const osnoise_options_str[OSN_MAX] = {
67 "OSNOISE_PREEMPT_DISABLE",
68 "OSNOISE_IRQ_DISABLE" };
70 #define OSN_DEFAULT_OPTIONS 0x2
71 static unsigned long osnoise_options = OSN_DEFAULT_OPTIONS;
74 * trace_array of the enabled osnoise/timerlat instances.
76 struct osnoise_instance {
77 struct list_head list;
78 struct trace_array *tr;
81 static struct list_head osnoise_instances;
83 static bool osnoise_has_registered_instances(void)
85 return !!list_first_or_null_rcu(&osnoise_instances,
86 struct osnoise_instance,
91 * osnoise_instance_registered - check if a tr is already registered
93 static int osnoise_instance_registered(struct trace_array *tr)
95 struct osnoise_instance *inst;
99 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
109 * osnoise_register_instance - register a new trace instance
111 * Register a trace_array *tr in the list of instances running
112 * osnoise/timerlat tracers.
114 static int osnoise_register_instance(struct trace_array *tr)
116 struct osnoise_instance *inst;
119 * register/unregister serialization is provided by trace's
122 lockdep_assert_held(&trace_types_lock);
124 inst = kmalloc(sizeof(*inst), GFP_KERNEL);
128 INIT_LIST_HEAD_RCU(&inst->list);
130 list_add_tail_rcu(&inst->list, &osnoise_instances);
136 * osnoise_unregister_instance - unregister a registered trace instance
138 * Remove the trace_array *tr from the list of instances running
139 * osnoise/timerlat tracers.
141 static void osnoise_unregister_instance(struct trace_array *tr)
143 struct osnoise_instance *inst;
147 * register/unregister serialization is provided by trace's
150 list_for_each_entry_rcu(inst, &osnoise_instances, list,
151 lockdep_is_held(&trace_types_lock)) {
152 if (inst->tr == tr) {
153 list_del_rcu(&inst->list);
162 kvfree_rcu_mightsleep(inst);
182 #define IRQ_CONTEXT 0
183 #define THREAD_CONTEXT 1
184 #define THREAD_URET 2
186 * sofirq runtime info.
195 * thread runtime info.
204 * Runtime information: this structure saves the runtime information used by
205 * one sampling thread.
207 struct osnoise_variables {
208 struct task_struct *kthread;
213 struct osn_softirq softirq;
214 struct osn_thread thread;
219 * Per-cpu runtime information.
221 static DEFINE_PER_CPU(struct osnoise_variables, per_cpu_osnoise_var);
224 * this_cpu_osn_var - Return the per-cpu osnoise_variables on its relative CPU
226 static inline struct osnoise_variables *this_cpu_osn_var(void)
228 return this_cpu_ptr(&per_cpu_osnoise_var);
231 #ifdef CONFIG_TIMERLAT_TRACER
233 * Runtime information for the timer mode.
235 struct timerlat_variables {
236 struct task_struct *kthread;
237 struct hrtimer timer;
242 bool uthread_migrate;
245 static DEFINE_PER_CPU(struct timerlat_variables, per_cpu_timerlat_var);
248 * this_cpu_tmr_var - Return the per-cpu timerlat_variables on its relative CPU
250 static inline struct timerlat_variables *this_cpu_tmr_var(void)
252 return this_cpu_ptr(&per_cpu_timerlat_var);
256 * tlat_var_reset - Reset the values of the given timerlat_variables
258 static inline void tlat_var_reset(void)
260 struct timerlat_variables *tlat_var;
263 * So far, all the values are initialized as 0, so
264 * zeroing the structure is perfect.
266 for_each_cpu(cpu, cpu_online_mask) {
267 tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
268 memset(tlat_var, 0, sizeof(*tlat_var));
271 #else /* CONFIG_TIMERLAT_TRACER */
272 #define tlat_var_reset() do {} while (0)
273 #endif /* CONFIG_TIMERLAT_TRACER */
276 * osn_var_reset - Reset the values of the given osnoise_variables
278 static inline void osn_var_reset(void)
280 struct osnoise_variables *osn_var;
284 * So far, all the values are initialized as 0, so
285 * zeroing the structure is perfect.
287 for_each_cpu(cpu, cpu_online_mask) {
288 osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
289 memset(osn_var, 0, sizeof(*osn_var));
294 * osn_var_reset_all - Reset the value of all per-cpu osnoise_variables
296 static inline void osn_var_reset_all(void)
303 * Tells NMIs to call back to the osnoise tracer to record timestamps.
305 bool trace_osnoise_callback_enabled;
308 * osnoise sample structure definition. Used to store the statistics of a
311 struct osnoise_sample {
312 u64 runtime; /* runtime */
313 u64 noise; /* noise */
314 u64 max_sample; /* max single noise sample */
315 int hw_count; /* # HW (incl. hypervisor) interference */
316 int nmi_count; /* # NMIs during this sample */
317 int irq_count; /* # IRQs during this sample */
318 int softirq_count; /* # softirqs during this sample */
319 int thread_count; /* # threads during this sample */
322 #ifdef CONFIG_TIMERLAT_TRACER
324 * timerlat sample structure definition. Used to store the statistics of
327 struct timerlat_sample {
328 u64 timer_latency; /* timer_latency */
329 unsigned int seqnum; /* unique sequence */
330 int context; /* timer context */
335 * Protect the interface.
337 static struct mutex interface_lock;
342 static struct osnoise_data {
343 u64 sample_period; /* total sampling period */
344 u64 sample_runtime; /* active sampling portion of period */
345 u64 stop_tracing; /* stop trace in the internal operation (loop/irq) */
346 u64 stop_tracing_total; /* stop trace in the final operation (report/thread) */
347 #ifdef CONFIG_TIMERLAT_TRACER
348 u64 timerlat_period; /* timerlat period */
349 u64 print_stack; /* print IRQ stack if total > */
350 int timerlat_tracer; /* timerlat tracer */
352 bool tainted; /* infor users and developers about a problem */
354 .sample_period = DEFAULT_SAMPLE_PERIOD,
355 .sample_runtime = DEFAULT_SAMPLE_RUNTIME,
357 .stop_tracing_total = 0,
358 #ifdef CONFIG_TIMERLAT_TRACER
360 .timerlat_period = DEFAULT_TIMERLAT_PERIOD,
361 .timerlat_tracer = 0,
365 #ifdef CONFIG_TIMERLAT_TRACER
366 static inline bool timerlat_enabled(void)
368 return osnoise_data.timerlat_tracer;
371 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
373 struct timerlat_variables *tlat_var = this_cpu_tmr_var();
375 * If the timerlat is enabled, but the irq handler did
376 * not run yet enabling timerlat_tracer, do not trace.
378 if (!tlat_var->tracing_thread) {
379 osn_var->softirq.arrival_time = 0;
380 osn_var->softirq.delta_start = 0;
386 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
388 struct timerlat_variables *tlat_var = this_cpu_tmr_var();
390 * If the timerlat is enabled, but the irq handler did
391 * not run yet enabling timerlat_tracer, do not trace.
393 if (!tlat_var->tracing_thread) {
394 osn_var->thread.delta_start = 0;
395 osn_var->thread.arrival_time = 0;
400 #else /* CONFIG_TIMERLAT_TRACER */
401 static inline bool timerlat_enabled(void)
406 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
410 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
416 #ifdef CONFIG_PREEMPT_RT
418 * Print the osnoise header info.
420 static void print_osnoise_headers(struct seq_file *s)
422 if (osnoise_data.tainted)
423 seq_puts(s, "# osnoise is tainted!\n");
425 seq_puts(s, "# _-------=> irqs-off\n");
426 seq_puts(s, "# / _------=> need-resched\n");
427 seq_puts(s, "# | / _-----=> need-resched-lazy\n");
428 seq_puts(s, "# || / _----=> hardirq/softirq\n");
429 seq_puts(s, "# ||| / _---=> preempt-depth\n");
430 seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
431 seq_puts(s, "# ||||| / _-=> migrate-disable\n");
433 seq_puts(s, "# |||||| / ");
434 seq_puts(s, " MAX\n");
436 seq_puts(s, "# ||||| / ");
437 seq_puts(s, " SINGLE Interference counters:\n");
439 seq_puts(s, "# ||||||| RUNTIME ");
440 seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
442 seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP IN US ");
443 seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
445 seq_puts(s, "# | | | ||||||| | | ");
446 seq_puts(s, " | | | | | | | |\n");
448 #else /* CONFIG_PREEMPT_RT */
449 static void print_osnoise_headers(struct seq_file *s)
451 if (osnoise_data.tainted)
452 seq_puts(s, "# osnoise is tainted!\n");
454 seq_puts(s, "# _-----=> irqs-off\n");
455 seq_puts(s, "# / _----=> need-resched\n");
456 seq_puts(s, "# | / _---=> hardirq/softirq\n");
457 seq_puts(s, "# || / _--=> preempt-depth\n");
458 seq_puts(s, "# ||| / _-=> migrate-disable ");
459 seq_puts(s, " MAX\n");
460 seq_puts(s, "# |||| / delay ");
461 seq_puts(s, " SINGLE Interference counters:\n");
463 seq_puts(s, "# ||||| RUNTIME ");
464 seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
466 seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP IN US ");
467 seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
469 seq_puts(s, "# | | | ||||| | | ");
470 seq_puts(s, " | | | | | | | |\n");
472 #endif /* CONFIG_PREEMPT_RT */
475 * osnoise_taint - report an osnoise error.
477 #define osnoise_taint(msg) ({ \
478 struct osnoise_instance *inst; \
479 struct trace_buffer *buffer; \
482 list_for_each_entry_rcu(inst, &osnoise_instances, list) { \
483 buffer = inst->tr->array_buffer.buffer; \
484 trace_array_printk_buf(buffer, _THIS_IP_, msg); \
487 osnoise_data.tainted = true; \
491 * Record an osnoise_sample into the tracer buffer.
494 __trace_osnoise_sample(struct osnoise_sample *sample, struct trace_buffer *buffer)
496 struct trace_event_call *call = &event_osnoise;
497 struct ring_buffer_event *event;
498 struct osnoise_entry *entry;
500 event = trace_buffer_lock_reserve(buffer, TRACE_OSNOISE, sizeof(*entry),
504 entry = ring_buffer_event_data(event);
505 entry->runtime = sample->runtime;
506 entry->noise = sample->noise;
507 entry->max_sample = sample->max_sample;
508 entry->hw_count = sample->hw_count;
509 entry->nmi_count = sample->nmi_count;
510 entry->irq_count = sample->irq_count;
511 entry->softirq_count = sample->softirq_count;
512 entry->thread_count = sample->thread_count;
514 if (!call_filter_check_discard(call, entry, buffer, event))
515 trace_buffer_unlock_commit_nostack(buffer, event);
519 * Record an osnoise_sample on all osnoise instances.
521 static void trace_osnoise_sample(struct osnoise_sample *sample)
523 struct osnoise_instance *inst;
524 struct trace_buffer *buffer;
527 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
528 buffer = inst->tr->array_buffer.buffer;
529 __trace_osnoise_sample(sample, buffer);
534 #ifdef CONFIG_TIMERLAT_TRACER
536 * Print the timerlat header info.
538 #ifdef CONFIG_PREEMPT_RT
539 static void print_timerlat_headers(struct seq_file *s)
541 seq_puts(s, "# _-------=> irqs-off\n");
542 seq_puts(s, "# / _------=> need-resched\n");
543 seq_puts(s, "# | / _-----=> need-resched-lazy\n");
544 seq_puts(s, "# || / _----=> hardirq/softirq\n");
545 seq_puts(s, "# ||| / _---=> preempt-depth\n");
546 seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
547 seq_puts(s, "# ||||| / _-=> migrate-disable\n");
548 seq_puts(s, "# |||||| /\n");
549 seq_puts(s, "# ||||||| ACTIVATION\n");
550 seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP ID ");
551 seq_puts(s, " CONTEXT LATENCY\n");
552 seq_puts(s, "# | | | ||||||| | | ");
553 seq_puts(s, " | |\n");
555 #else /* CONFIG_PREEMPT_RT */
556 static void print_timerlat_headers(struct seq_file *s)
558 seq_puts(s, "# _-----=> irqs-off\n");
559 seq_puts(s, "# / _----=> need-resched\n");
560 seq_puts(s, "# | / _---=> hardirq/softirq\n");
561 seq_puts(s, "# || / _--=> preempt-depth\n");
562 seq_puts(s, "# ||| / _-=> migrate-disable\n");
563 seq_puts(s, "# |||| / delay\n");
564 seq_puts(s, "# ||||| ACTIVATION\n");
565 seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP ID ");
566 seq_puts(s, " CONTEXT LATENCY\n");
567 seq_puts(s, "# | | | ||||| | | ");
568 seq_puts(s, " | |\n");
570 #endif /* CONFIG_PREEMPT_RT */
573 __trace_timerlat_sample(struct timerlat_sample *sample, struct trace_buffer *buffer)
575 struct trace_event_call *call = &event_osnoise;
576 struct ring_buffer_event *event;
577 struct timerlat_entry *entry;
579 event = trace_buffer_lock_reserve(buffer, TRACE_TIMERLAT, sizeof(*entry),
583 entry = ring_buffer_event_data(event);
584 entry->seqnum = sample->seqnum;
585 entry->context = sample->context;
586 entry->timer_latency = sample->timer_latency;
588 if (!call_filter_check_discard(call, entry, buffer, event))
589 trace_buffer_unlock_commit_nostack(buffer, event);
593 * Record an timerlat_sample into the tracer buffer.
595 static void trace_timerlat_sample(struct timerlat_sample *sample)
597 struct osnoise_instance *inst;
598 struct trace_buffer *buffer;
601 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
602 buffer = inst->tr->array_buffer.buffer;
603 __trace_timerlat_sample(sample, buffer);
608 #ifdef CONFIG_STACKTRACE
610 #define MAX_CALLS 256
613 * Stack trace will take place only at IRQ level, so, no need
614 * to control nesting here.
619 unsigned long calls[MAX_CALLS];
622 static DEFINE_PER_CPU(struct trace_stack, trace_stack);
625 * timerlat_save_stack - save a stack trace without printing
627 * Save the current stack trace without printing. The
628 * stack will be printed later, after the end of the measurement.
630 static void timerlat_save_stack(int skip)
632 unsigned int size, nr_entries;
633 struct trace_stack *fstack;
635 fstack = this_cpu_ptr(&trace_stack);
637 size = ARRAY_SIZE(fstack->calls);
639 nr_entries = stack_trace_save(fstack->calls, size, skip);
641 fstack->stack_size = nr_entries * sizeof(unsigned long);
642 fstack->nr_entries = nr_entries;
649 __timerlat_dump_stack(struct trace_buffer *buffer, struct trace_stack *fstack, unsigned int size)
651 struct trace_event_call *call = &event_osnoise;
652 struct ring_buffer_event *event;
653 struct stack_entry *entry;
655 event = trace_buffer_lock_reserve(buffer, TRACE_STACK, sizeof(*entry) + size,
660 entry = ring_buffer_event_data(event);
662 memcpy(&entry->caller, fstack->calls, size);
663 entry->size = fstack->nr_entries;
665 if (!call_filter_check_discard(call, entry, buffer, event))
666 trace_buffer_unlock_commit_nostack(buffer, event);
670 * timerlat_dump_stack - dump a stack trace previously saved
672 static void timerlat_dump_stack(u64 latency)
674 struct osnoise_instance *inst;
675 struct trace_buffer *buffer;
676 struct trace_stack *fstack;
680 * trace only if latency > print_stack config, if enabled.
682 if (!osnoise_data.print_stack || osnoise_data.print_stack > latency)
685 preempt_disable_notrace();
686 fstack = this_cpu_ptr(&trace_stack);
687 size = fstack->stack_size;
690 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
691 buffer = inst->tr->array_buffer.buffer;
692 __timerlat_dump_stack(buffer, fstack, size);
696 preempt_enable_notrace();
698 #else /* CONFIG_STACKTRACE */
699 #define timerlat_dump_stack(u64 latency) do {} while (0)
700 #define timerlat_save_stack(a) do {} while (0)
701 #endif /* CONFIG_STACKTRACE */
702 #endif /* CONFIG_TIMERLAT_TRACER */
705 * Macros to encapsulate the time capturing infrastructure.
707 #define time_get() trace_clock_local()
708 #define time_to_us(x) div_u64(x, 1000)
709 #define time_sub(a, b) ((a) - (b))
712 * cond_move_irq_delta_start - Forward the delta_start of a running IRQ
714 * If an IRQ is preempted by an NMI, its delta_start is pushed forward
715 * to discount the NMI interference.
717 * See get_int_safe_duration().
720 cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)
722 if (osn_var->irq.delta_start)
723 osn_var->irq.delta_start += duration;
726 #ifndef CONFIG_PREEMPT_RT
728 * cond_move_softirq_delta_start - Forward the delta_start of a running softirq.
730 * If a softirq is preempted by an IRQ or NMI, its delta_start is pushed
731 * forward to discount the interference.
733 * See get_int_safe_duration().
736 cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)
738 if (osn_var->softirq.delta_start)
739 osn_var->softirq.delta_start += duration;
741 #else /* CONFIG_PREEMPT_RT */
742 #define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)
746 * cond_move_thread_delta_start - Forward the delta_start of a running thread
748 * If a noisy thread is preempted by an softirq, IRQ or NMI, its delta_start
749 * is pushed forward to discount the interference.
751 * See get_int_safe_duration().
754 cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)
756 if (osn_var->thread.delta_start)
757 osn_var->thread.delta_start += duration;
761 * get_int_safe_duration - Get the duration of a window
763 * The irq, softirq and thread varaibles need to have its duration without
764 * the interference from higher priority interrupts. Instead of keeping a
765 * variable to discount the interrupt interference from these variables, the
766 * starting time of these variables are pushed forward with the interrupt's
767 * duration. In this way, a single variable is used to:
769 * - Know if a given window is being measured.
770 * - Account its duration.
771 * - Discount the interference.
773 * To avoid getting inconsistent values, e.g.,:
777 * delta_start -= int duration;
779 * duration = now - delta_start;
781 * result: negative duration if the variable duration before the
782 * interrupt was smaller than the interrupt execution.
784 * A counter of interrupts is used. If the counter increased, try
785 * to capture an interference safe duration.
788 get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start)
790 u64 int_counter, now;
794 int_counter = local_read(&osn_var->int_counter);
795 /* synchronize with interrupts */
799 duration = (now - *delta_start);
801 /* synchronize with interrupts */
803 } while (int_counter != local_read(&osn_var->int_counter));
806 * This is an evidence of race conditions that cause
807 * a value to be "discounted" too much.
810 osnoise_taint("Negative duration!\n");
819 * set_int_safe_time - Save the current time on *time, aware of interference
821 * Get the time, taking into consideration a possible interference from
822 * higher priority interrupts.
824 * See get_int_safe_duration() for an explanation.
827 set_int_safe_time(struct osnoise_variables *osn_var, u64 *time)
832 int_counter = local_read(&osn_var->int_counter);
833 /* synchronize with interrupts */
838 /* synchronize with interrupts */
840 } while (int_counter != local_read(&osn_var->int_counter));
845 #ifdef CONFIG_TIMERLAT_TRACER
847 * copy_int_safe_time - Copy *src into *desc aware of interference
850 copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src)
855 int_counter = local_read(&osn_var->int_counter);
856 /* synchronize with interrupts */
861 /* synchronize with interrupts */
863 } while (int_counter != local_read(&osn_var->int_counter));
867 #endif /* CONFIG_TIMERLAT_TRACER */
870 * trace_osnoise_callback - NMI entry/exit callback
872 * This function is called at the entry and exit NMI code. The bool enter
873 * distinguishes between either case. This function is used to note a NMI
874 * occurrence, compute the noise caused by the NMI, and to remove the noise
875 * it is potentially causing on other interference variables.
877 void trace_osnoise_callback(bool enter)
879 struct osnoise_variables *osn_var = this_cpu_osn_var();
882 if (!osn_var->sampling)
886 * Currently trace_clock_local() calls sched_clock() and the
887 * generic version is not NMI safe.
889 if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK)) {
891 osn_var->nmi.delta_start = time_get();
892 local_inc(&osn_var->int_counter);
894 duration = time_get() - osn_var->nmi.delta_start;
896 trace_nmi_noise(osn_var->nmi.delta_start, duration);
898 cond_move_irq_delta_start(osn_var, duration);
899 cond_move_softirq_delta_start(osn_var, duration);
900 cond_move_thread_delta_start(osn_var, duration);
905 osn_var->nmi.count++;
909 * osnoise_trace_irq_entry - Note the starting of an IRQ
911 * Save the starting time of an IRQ. As IRQs are non-preemptive to other IRQs,
912 * it is safe to use a single variable (ons_var->irq) to save the statistics.
913 * The arrival_time is used to report... the arrival time. The delta_start
914 * is used to compute the duration at the IRQ exit handler. See
915 * cond_move_irq_delta_start().
917 void osnoise_trace_irq_entry(int id)
919 struct osnoise_variables *osn_var = this_cpu_osn_var();
921 if (!osn_var->sampling)
924 * This value will be used in the report, but not to compute
925 * the execution time, so it is safe to get it unsafe.
927 osn_var->irq.arrival_time = time_get();
928 set_int_safe_time(osn_var, &osn_var->irq.delta_start);
929 osn_var->irq.count++;
931 local_inc(&osn_var->int_counter);
935 * osnoise_irq_exit - Note the end of an IRQ, sava data and trace
937 * Computes the duration of the IRQ noise, and trace it. Also discounts the
938 * interference from other sources of noise could be currently being accounted.
940 void osnoise_trace_irq_exit(int id, const char *desc)
942 struct osnoise_variables *osn_var = this_cpu_osn_var();
945 if (!osn_var->sampling)
948 duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);
949 trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);
950 osn_var->irq.arrival_time = 0;
951 cond_move_softirq_delta_start(osn_var, duration);
952 cond_move_thread_delta_start(osn_var, duration);
956 * trace_irqentry_callback - Callback to the irq:irq_entry traceevent
958 * Used to note the starting of an IRQ occurece.
960 static void trace_irqentry_callback(void *data, int irq,
961 struct irqaction *action)
963 osnoise_trace_irq_entry(irq);
967 * trace_irqexit_callback - Callback to the irq:irq_exit traceevent
969 * Used to note the end of an IRQ occurece.
971 static void trace_irqexit_callback(void *data, int irq,
972 struct irqaction *action, int ret)
974 osnoise_trace_irq_exit(irq, action->name);
978 * arch specific register function.
980 int __weak osnoise_arch_register(void)
986 * arch specific unregister function.
988 void __weak osnoise_arch_unregister(void)
994 * hook_irq_events - Hook IRQ handling events
996 * This function hooks the IRQ related callbacks to the respective trace
999 static int hook_irq_events(void)
1003 ret = register_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1007 ret = register_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1009 goto out_unregister_entry;
1011 ret = osnoise_arch_register();
1018 unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1019 out_unregister_entry:
1020 unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1026 * unhook_irq_events - Unhook IRQ handling events
1028 * This function unhooks the IRQ related callbacks to the respective trace
1031 static void unhook_irq_events(void)
1033 osnoise_arch_unregister();
1034 unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1035 unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1038 #ifndef CONFIG_PREEMPT_RT
1040 * trace_softirq_entry_callback - Note the starting of a softirq
1042 * Save the starting time of a softirq. As softirqs are non-preemptive to
1043 * other softirqs, it is safe to use a single variable (ons_var->softirq)
1044 * to save the statistics. The arrival_time is used to report... the
1045 * arrival time. The delta_start is used to compute the duration at the
1046 * softirq exit handler. See cond_move_softirq_delta_start().
1048 static void trace_softirq_entry_callback(void *data, unsigned int vec_nr)
1050 struct osnoise_variables *osn_var = this_cpu_osn_var();
1052 if (!osn_var->sampling)
1055 * This value will be used in the report, but not to compute
1056 * the execution time, so it is safe to get it unsafe.
1058 osn_var->softirq.arrival_time = time_get();
1059 set_int_safe_time(osn_var, &osn_var->softirq.delta_start);
1060 osn_var->softirq.count++;
1062 local_inc(&osn_var->int_counter);
1066 * trace_softirq_exit_callback - Note the end of an softirq
1068 * Computes the duration of the softirq noise, and trace it. Also discounts the
1069 * interference from other sources of noise could be currently being accounted.
1071 static void trace_softirq_exit_callback(void *data, unsigned int vec_nr)
1073 struct osnoise_variables *osn_var = this_cpu_osn_var();
1076 if (!osn_var->sampling)
1079 if (unlikely(timerlat_enabled()))
1080 if (!timerlat_softirq_exit(osn_var))
1083 duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);
1084 trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);
1085 cond_move_thread_delta_start(osn_var, duration);
1086 osn_var->softirq.arrival_time = 0;
1090 * hook_softirq_events - Hook softirq handling events
1092 * This function hooks the softirq related callbacks to the respective trace
1095 static int hook_softirq_events(void)
1099 ret = register_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1103 ret = register_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1105 goto out_unreg_entry;
1110 unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1116 * unhook_softirq_events - Unhook softirq handling events
1118 * This function hooks the softirq related callbacks to the respective trace
1121 static void unhook_softirq_events(void)
1123 unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1124 unregister_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1126 #else /* CONFIG_PREEMPT_RT */
1128 * softirq are threads on the PREEMPT_RT mode.
1130 static int hook_softirq_events(void)
1134 static void unhook_softirq_events(void)
1140 * thread_entry - Record the starting of a thread noise window
1142 * It saves the context switch time for a noisy thread, and increments
1143 * the interference counters.
1146 thread_entry(struct osnoise_variables *osn_var, struct task_struct *t)
1148 if (!osn_var->sampling)
1151 * The arrival time will be used in the report, but not to compute
1152 * the execution time, so it is safe to get it unsafe.
1154 osn_var->thread.arrival_time = time_get();
1156 set_int_safe_time(osn_var, &osn_var->thread.delta_start);
1158 osn_var->thread.count++;
1159 local_inc(&osn_var->int_counter);
1163 * thread_exit - Report the end of a thread noise window
1165 * It computes the total noise from a thread, tracing if needed.
1168 thread_exit(struct osnoise_variables *osn_var, struct task_struct *t)
1172 if (!osn_var->sampling)
1175 if (unlikely(timerlat_enabled()))
1176 if (!timerlat_thread_exit(osn_var))
1179 duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);
1181 trace_thread_noise(t, osn_var->thread.arrival_time, duration);
1183 osn_var->thread.arrival_time = 0;
1186 #ifdef CONFIG_TIMERLAT_TRACER
1188 * osnoise_stop_exception - Stop tracing and the tracer.
1190 static __always_inline void osnoise_stop_exception(char *msg, int cpu)
1192 struct osnoise_instance *inst;
1193 struct trace_array *tr;
1196 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1198 trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
1199 "stop tracing hit on cpu %d due to exception: %s\n",
1203 if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options))
1204 panic("tracer hit on cpu %d due to exception: %s\n",
1208 tracer_tracing_off(tr);
1214 * trace_sched_migrate_callback - sched:sched_migrate_task trace event handler
1216 * his function is hooked to the sched:sched_migrate_task trace event, and monitors
1217 * timerlat user-space thread migration.
1219 static void trace_sched_migrate_callback(void *data, struct task_struct *p, int dest_cpu)
1221 struct osnoise_variables *osn_var;
1222 long cpu = task_cpu(p);
1224 osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
1225 if (osn_var->pid == p->pid && dest_cpu != cpu) {
1226 per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1;
1227 osnoise_taint("timerlat user-thread migrated\n");
1228 osnoise_stop_exception("timerlat user-thread migrated", cpu);
1232 static int register_migration_monitor(void)
1237 * Timerlat thread migration check is only required when running timerlat in user-space.
1238 * Thus, enable callback only if timerlat is set with no workload.
1240 if (timerlat_enabled() && !test_bit(OSN_WORKLOAD, &osnoise_options))
1241 ret = register_trace_sched_migrate_task(trace_sched_migrate_callback, NULL);
1246 static void unregister_migration_monitor(void)
1248 if (timerlat_enabled() && !test_bit(OSN_WORKLOAD, &osnoise_options))
1249 unregister_trace_sched_migrate_task(trace_sched_migrate_callback, NULL);
1252 static int register_migration_monitor(void)
1256 static void unregister_migration_monitor(void) {}
1259 * trace_sched_switch - sched:sched_switch trace event handler
1261 * This function is hooked to the sched:sched_switch trace event, and it is
1262 * used to record the beginning and to report the end of a thread noise window.
1265 trace_sched_switch_callback(void *data, bool preempt,
1266 struct task_struct *p,
1267 struct task_struct *n,
1268 unsigned int prev_state)
1270 struct osnoise_variables *osn_var = this_cpu_osn_var();
1271 int workload = test_bit(OSN_WORKLOAD, &osnoise_options);
1273 if ((p->pid != osn_var->pid) || !workload)
1274 thread_exit(osn_var, p);
1276 if ((n->pid != osn_var->pid) || !workload)
1277 thread_entry(osn_var, n);
1281 * hook_thread_events - Hook the instrumentation for thread noise
1283 * Hook the osnoise tracer callbacks to handle the noise from other
1284 * threads on the necessary kernel events.
1286 static int hook_thread_events(void)
1290 ret = register_trace_sched_switch(trace_sched_switch_callback, NULL);
1294 ret = register_migration_monitor();
1301 unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1306 * unhook_thread_events - unhook the instrumentation for thread noise
1308 * Unook the osnoise tracer callbacks to handle the noise from other
1309 * threads on the necessary kernel events.
1311 static void unhook_thread_events(void)
1313 unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1314 unregister_migration_monitor();
1318 * save_osn_sample_stats - Save the osnoise_sample statistics
1320 * Save the osnoise_sample statistics before the sampling phase. These
1321 * values will be used later to compute the diff betwneen the statistics
1322 * before and after the osnoise sampling.
1325 save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1327 s->nmi_count = osn_var->nmi.count;
1328 s->irq_count = osn_var->irq.count;
1329 s->softirq_count = osn_var->softirq.count;
1330 s->thread_count = osn_var->thread.count;
1334 * diff_osn_sample_stats - Compute the osnoise_sample statistics
1336 * After a sample period, compute the difference on the osnoise_sample
1337 * statistics. The struct osnoise_sample *s contains the statistics saved via
1338 * save_osn_sample_stats() before the osnoise sampling.
1341 diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1343 s->nmi_count = osn_var->nmi.count - s->nmi_count;
1344 s->irq_count = osn_var->irq.count - s->irq_count;
1345 s->softirq_count = osn_var->softirq.count - s->softirq_count;
1346 s->thread_count = osn_var->thread.count - s->thread_count;
1350 * osnoise_stop_tracing - Stop tracing and the tracer.
1352 static __always_inline void osnoise_stop_tracing(void)
1354 struct osnoise_instance *inst;
1355 struct trace_array *tr;
1358 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1360 trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
1361 "stop tracing hit on cpu %d\n", smp_processor_id());
1363 if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options))
1364 panic("tracer hit stop condition on CPU %d\n", smp_processor_id());
1366 tracer_tracing_off(tr);
1372 * osnoise_has_tracing_on - Check if there is at least one instance on
1374 static __always_inline int osnoise_has_tracing_on(void)
1376 struct osnoise_instance *inst;
1377 int trace_is_on = 0;
1380 list_for_each_entry_rcu(inst, &osnoise_instances, list)
1381 trace_is_on += tracer_tracing_is_on(inst->tr);
1388 * notify_new_max_latency - Notify a new max latency via fsnotify interface.
1390 static void notify_new_max_latency(u64 latency)
1392 struct osnoise_instance *inst;
1393 struct trace_array *tr;
1396 list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1398 if (tracer_tracing_is_on(tr) && tr->max_latency < latency) {
1399 tr->max_latency = latency;
1400 latency_fsnotify(tr);
1407 * run_osnoise - Sample the time and look for osnoise
1409 * Used to capture the time, looking for potential osnoise latency repeatedly.
1410 * Different from hwlat_detector, it is called with preemption and interrupts
1411 * enabled. This allows irqs, softirqs and threads to run, interfering on the
1412 * osnoise sampling thread, as they would do with a regular thread.
1414 static int run_osnoise(void)
1416 bool disable_irq = test_bit(OSN_IRQ_DISABLE, &osnoise_options);
1417 struct osnoise_variables *osn_var = this_cpu_osn_var();
1418 u64 start, sample, last_sample;
1419 u64 last_int_count, int_count;
1420 s64 noise = 0, max_noise = 0;
1421 s64 total, last_total = 0;
1422 struct osnoise_sample s;
1423 bool disable_preemption;
1424 unsigned int threshold;
1425 u64 runtime, stop_in;
1431 * Disabling preemption is only required if IRQs are enabled,
1432 * and the options is set on.
1434 disable_preemption = !disable_irq && test_bit(OSN_PREEMPT_DISABLE, &osnoise_options);
1437 * Considers the current thread as the workload.
1439 osn_var->pid = current->pid;
1442 * Save the current stats for the diff
1444 save_osn_sample_stats(osn_var, &s);
1447 * if threshold is 0, use the default value of 5 us.
1449 threshold = tracing_thresh ? : 5000;
1452 * Apply PREEMPT and IRQ disabled options.
1455 local_irq_disable();
1457 if (disable_preemption)
1461 * Make sure NMIs see sampling first
1463 osn_var->sampling = true;
1467 * Transform the *_us config to nanoseconds to avoid the
1468 * division on the main loop.
1470 runtime = osnoise_data.sample_runtime * NSEC_PER_USEC;
1471 stop_in = osnoise_data.stop_tracing * NSEC_PER_USEC;
1481 last_int_count = set_int_safe_time(osn_var, &last_sample);
1487 int_count = set_int_safe_time(osn_var, &sample);
1489 noise = time_sub(sample, last_sample);
1492 * This shouldn't happen.
1495 osnoise_taint("negative noise!");
1502 total = time_sub(sample, start);
1505 * Check for possible overflows.
1507 if (total < last_total) {
1508 osnoise_taint("total overflow!");
1514 if (noise >= threshold) {
1515 int interference = int_count - last_int_count;
1517 if (noise > max_noise)
1525 trace_sample_threshold(last_sample, noise, interference);
1527 if (osnoise_data.stop_tracing)
1528 if (noise > stop_in)
1529 osnoise_stop_tracing();
1533 * In some cases, notably when running on a nohz_full CPU with
1534 * a stopped tick PREEMPT_RCU has no way to account for QSs.
1535 * This will eventually cause unwarranted noise as PREEMPT_RCU
1536 * will force preemption as the means of ending the current
1537 * grace period. We avoid this problem by calling
1538 * rcu_momentary_dyntick_idle(), which performs a zero duration
1539 * EQS allowing PREEMPT_RCU to end the current grace period.
1540 * This call shouldn't be wrapped inside an RCU critical
1543 * Note that in non PREEMPT_RCU kernels QSs are handled through
1546 if (IS_ENABLED(CONFIG_PREEMPT_RCU)) {
1548 local_irq_disable();
1550 rcu_momentary_dyntick_idle();
1557 * For the non-preemptive kernel config: let threads runs, if
1558 * they so wish, unless set not do to so.
1560 if (!disable_irq && !disable_preemption)
1563 last_sample = sample;
1564 last_int_count = int_count;
1566 } while (total < runtime && !kthread_should_stop());
1569 * Finish the above in the view for interrupts.
1573 osn_var->sampling = false;
1576 * Make sure sampling data is no longer updated.
1581 * Return to the preemptive state.
1583 if (disable_preemption)
1592 s.noise = time_to_us(sum_noise);
1593 s.runtime = time_to_us(total);
1594 s.max_sample = time_to_us(max_noise);
1595 s.hw_count = hw_count;
1597 /* Save interference stats info */
1598 diff_osn_sample_stats(osn_var, &s);
1600 trace_osnoise_sample(&s);
1602 notify_new_max_latency(max_noise);
1604 if (osnoise_data.stop_tracing_total)
1605 if (s.noise > osnoise_data.stop_tracing_total)
1606 osnoise_stop_tracing();
1613 static struct cpumask osnoise_cpumask;
1614 static struct cpumask save_cpumask;
1617 * osnoise_sleep - sleep until the next period
1619 static void osnoise_sleep(bool skip_period)
1624 mutex_lock(&interface_lock);
1626 interval = osnoise_data.sample_period;
1628 interval = osnoise_data.sample_period - osnoise_data.sample_runtime;
1629 mutex_unlock(&interface_lock);
1632 * differently from hwlat_detector, the osnoise tracer can run
1633 * without a pause because preemption is on.
1636 /* Let synchronize_rcu_tasks() make progress */
1637 cond_resched_tasks_rcu_qs();
1641 wake_time = ktime_add_us(ktime_get(), interval);
1642 __set_current_state(TASK_INTERRUPTIBLE);
1644 while (schedule_hrtimeout(&wake_time, HRTIMER_MODE_ABS)) {
1645 if (kthread_should_stop())
1651 * osnoise_migration_pending - checks if the task needs to migrate
1653 * osnoise/timerlat threads are per-cpu. If there is a pending request to
1654 * migrate the thread away from the current CPU, something bad has happened.
1655 * Play the good citizen and leave.
1657 * Returns 0 if it is safe to continue, 1 otherwise.
1659 static inline int osnoise_migration_pending(void)
1661 if (!current->migration_pending)
1665 * If migration is pending, there is a task waiting for the
1666 * tracer to enable migration. The tracer does not allow migration,
1667 * thus: taint and leave to unblock the blocked thread.
1669 osnoise_taint("migration requested to osnoise threads, leaving.");
1672 * Unset this thread from the threads managed by the interface.
1673 * The tracers are responsible for cleaning their env before
1676 mutex_lock(&interface_lock);
1677 this_cpu_osn_var()->kthread = NULL;
1678 mutex_unlock(&interface_lock);
1684 * osnoise_main - The osnoise detection kernel thread
1686 * Calls run_osnoise() function to measure the osnoise for the configured runtime,
1689 static int osnoise_main(void *data)
1691 unsigned long flags;
1694 * This thread was created pinned to the CPU using PF_NO_SETAFFINITY.
1695 * The problem is that cgroup does not allow PF_NO_SETAFFINITY thread.
1697 * To work around this limitation, disable migration and remove the
1701 raw_spin_lock_irqsave(¤t->pi_lock, flags);
1702 current->flags &= ~(PF_NO_SETAFFINITY);
1703 raw_spin_unlock_irqrestore(¤t->pi_lock, flags);
1705 while (!kthread_should_stop()) {
1706 if (osnoise_migration_pending())
1709 /* skip a period if tracing is off on all instances */
1710 if (!osnoise_has_tracing_on()) {
1711 osnoise_sleep(true);
1716 osnoise_sleep(false);
1723 #ifdef CONFIG_TIMERLAT_TRACER
1725 * timerlat_irq - hrtimer handler for timerlat.
1727 static enum hrtimer_restart timerlat_irq(struct hrtimer *timer)
1729 struct osnoise_variables *osn_var = this_cpu_osn_var();
1730 struct timerlat_variables *tlat;
1731 struct timerlat_sample s;
1736 * I am not sure if the timer was armed for this CPU. So, get
1737 * the timerlat struct from the timer itself, not from this
1740 tlat = container_of(timer, struct timerlat_variables, timer);
1742 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1745 * Enable the osnoise: events for thread an softirq.
1747 tlat->tracing_thread = true;
1749 osn_var->thread.arrival_time = time_get();
1752 * A hardirq is running: the timer IRQ. It is for sure preempting
1753 * a thread, and potentially preempting a softirq.
1755 * At this point, it is not interesting to know the duration of the
1756 * preempted thread (and maybe softirq), but how much time they will
1757 * delay the beginning of the execution of the timer thread.
1759 * To get the correct (net) delay added by the softirq, its delta_start
1760 * is set as the IRQ one. In this way, at the return of the IRQ, the delta
1761 * start of the sofitrq will be zeroed, accounting then only the time
1764 * The thread follows the same principle. However, if a softirq is
1765 * running, the thread needs to receive the softirq delta_start. The
1766 * reason being is that the softirq will be the last to be unfolded,
1767 * resseting the thread delay to zero.
1769 * The PREEMPT_RT is a special case, though. As softirqs run as threads
1770 * on RT, moving the thread is enough.
1772 if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) {
1773 copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1774 &osn_var->softirq.delta_start);
1776 copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,
1777 &osn_var->irq.delta_start);
1779 copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1780 &osn_var->irq.delta_start);
1784 * Compute the current time with the expected time.
1786 diff = now - tlat->abs_period;
1789 s.seqnum = tlat->count;
1790 s.timer_latency = diff;
1791 s.context = IRQ_CONTEXT;
1793 trace_timerlat_sample(&s);
1795 if (osnoise_data.stop_tracing) {
1796 if (time_to_us(diff) >= osnoise_data.stop_tracing) {
1799 * At this point, if stop_tracing is set and <= print_stack,
1800 * print_stack is set and would be printed in the thread handler.
1802 * Thus, print the stack trace as it is helpful to define the
1803 * root cause of an IRQ latency.
1805 if (osnoise_data.stop_tracing <= osnoise_data.print_stack) {
1806 timerlat_save_stack(0);
1807 timerlat_dump_stack(time_to_us(diff));
1810 osnoise_stop_tracing();
1811 notify_new_max_latency(diff);
1813 wake_up_process(tlat->kthread);
1815 return HRTIMER_NORESTART;
1819 wake_up_process(tlat->kthread);
1821 if (osnoise_data.print_stack)
1822 timerlat_save_stack(0);
1824 return HRTIMER_NORESTART;
1828 * wait_next_period - Wait for the next period for timerlat
1830 static int wait_next_period(struct timerlat_variables *tlat)
1832 ktime_t next_abs_period, now;
1833 u64 rel_period = osnoise_data.timerlat_period * 1000;
1835 now = hrtimer_cb_get_time(&tlat->timer);
1836 next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1839 * Save the next abs_period.
1841 tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1844 * If the new abs_period is in the past, skip the activation.
1846 while (ktime_compare(now, next_abs_period) > 0) {
1847 next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1848 tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1851 set_current_state(TASK_INTERRUPTIBLE);
1853 hrtimer_start(&tlat->timer, next_abs_period, HRTIMER_MODE_ABS_PINNED_HARD);
1859 * timerlat_main- Timerlat main
1861 static int timerlat_main(void *data)
1863 struct osnoise_variables *osn_var = this_cpu_osn_var();
1864 struct timerlat_variables *tlat = this_cpu_tmr_var();
1865 struct timerlat_sample s;
1866 struct sched_param sp;
1867 unsigned long flags;
1871 * Make the thread RT, that is how cyclictest is usually used.
1873 sp.sched_priority = DEFAULT_TIMERLAT_PRIO;
1874 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1877 * This thread was created pinned to the CPU using PF_NO_SETAFFINITY.
1878 * The problem is that cgroup does not allow PF_NO_SETAFFINITY thread.
1880 * To work around this limitation, disable migration and remove the
1884 raw_spin_lock_irqsave(¤t->pi_lock, flags);
1885 current->flags &= ~(PF_NO_SETAFFINITY);
1886 raw_spin_unlock_irqrestore(¤t->pi_lock, flags);
1889 tlat->tracing_thread = false;
1891 hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1892 tlat->timer.function = timerlat_irq;
1893 tlat->kthread = current;
1894 osn_var->pid = current->pid;
1896 * Anotate the arrival time.
1898 tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
1900 wait_next_period(tlat);
1902 osn_var->sampling = 1;
1904 while (!kthread_should_stop()) {
1906 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1907 diff = now - tlat->abs_period;
1909 s.seqnum = tlat->count;
1910 s.timer_latency = diff;
1911 s.context = THREAD_CONTEXT;
1913 trace_timerlat_sample(&s);
1915 notify_new_max_latency(diff);
1917 timerlat_dump_stack(time_to_us(diff));
1919 tlat->tracing_thread = false;
1920 if (osnoise_data.stop_tracing_total)
1921 if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
1922 osnoise_stop_tracing();
1924 if (osnoise_migration_pending())
1927 wait_next_period(tlat);
1930 hrtimer_cancel(&tlat->timer);
1934 #else /* CONFIG_TIMERLAT_TRACER */
1935 static int timerlat_main(void *data)
1939 #endif /* CONFIG_TIMERLAT_TRACER */
1942 * stop_kthread - stop a workload thread
1944 static void stop_kthread(unsigned int cpu)
1946 struct task_struct *kthread;
1948 kthread = per_cpu(per_cpu_osnoise_var, cpu).kthread;
1950 if (test_bit(OSN_WORKLOAD, &osnoise_options)) {
1951 kthread_stop(kthread);
1954 * This is a user thread waiting on the timerlat_fd. We need
1955 * to close all users, and the best way to guarantee this is
1956 * by killing the thread. NOTE: this is a purpose specific file.
1958 kill_pid(kthread->thread_pid, SIGKILL, 1);
1959 put_task_struct(kthread);
1961 per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
1963 /* if no workload, just return */
1964 if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
1966 * This is set in the osnoise tracer case.
1968 per_cpu(per_cpu_osnoise_var, cpu).sampling = false;
1976 * stop_per_cpu_kthread - Stop per-cpu threads
1978 * Stop the osnoise sampling htread. Use this on unload and at system
1981 static void stop_per_cpu_kthreads(void)
1987 for_each_online_cpu(cpu)
1994 * start_kthread - Start a workload tread
1996 static int start_kthread(unsigned int cpu)
1998 struct task_struct *kthread;
1999 void *main = osnoise_main;
2002 if (timerlat_enabled()) {
2003 snprintf(comm, 24, "timerlat/%d", cpu);
2004 main = timerlat_main;
2006 /* if no workload, just return */
2007 if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
2008 per_cpu(per_cpu_osnoise_var, cpu).sampling = true;
2012 snprintf(comm, 24, "osnoise/%d", cpu);
2015 kthread = kthread_run_on_cpu(main, NULL, cpu, comm);
2017 if (IS_ERR(kthread)) {
2018 pr_err(BANNER "could not start sampling thread\n");
2019 stop_per_cpu_kthreads();
2023 per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread;
2029 * start_per_cpu_kthread - Kick off per-cpu osnoise sampling kthreads
2031 * This starts the kernel thread that will look for osnoise on many
2034 static int start_per_cpu_kthreads(void)
2036 struct cpumask *current_mask = &save_cpumask;
2040 if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
2041 if (timerlat_enabled())
2047 * Run only on online CPUs in which osnoise is allowed to run.
2049 cpumask_and(current_mask, cpu_online_mask, &osnoise_cpumask);
2051 for_each_possible_cpu(cpu)
2052 per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
2054 for_each_cpu(cpu, current_mask) {
2055 retval = start_kthread(cpu);
2058 stop_per_cpu_kthreads();
2068 #ifdef CONFIG_HOTPLUG_CPU
2069 static void osnoise_hotplug_workfn(struct work_struct *dummy)
2071 unsigned int cpu = smp_processor_id();
2073 mutex_lock(&trace_types_lock);
2075 if (!osnoise_has_registered_instances())
2076 goto out_unlock_trace;
2078 mutex_lock(&interface_lock);
2081 if (!cpumask_test_cpu(cpu, &osnoise_cpumask))
2088 mutex_unlock(&interface_lock);
2090 mutex_unlock(&trace_types_lock);
2093 static DECLARE_WORK(osnoise_hotplug_work, osnoise_hotplug_workfn);
2096 * osnoise_cpu_init - CPU hotplug online callback function
2098 static int osnoise_cpu_init(unsigned int cpu)
2100 schedule_work_on(cpu, &osnoise_hotplug_work);
2105 * osnoise_cpu_die - CPU hotplug offline callback function
2107 static int osnoise_cpu_die(unsigned int cpu)
2113 static void osnoise_init_hotplug_support(void)
2117 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/osnoise:online",
2118 osnoise_cpu_init, osnoise_cpu_die);
2120 pr_warn(BANNER "Error to init cpu hotplug support\n");
2124 #else /* CONFIG_HOTPLUG_CPU */
2125 static void osnoise_init_hotplug_support(void)
2129 #endif /* CONFIG_HOTPLUG_CPU */
2132 * seq file functions for the osnoise/options file.
2134 static void *s_options_start(struct seq_file *s, loff_t *pos)
2138 mutex_lock(&interface_lock);
2140 if (option >= OSN_MAX)
2146 static void *s_options_next(struct seq_file *s, void *v, loff_t *pos)
2148 int option = ++(*pos);
2150 if (option >= OSN_MAX)
2156 static int s_options_show(struct seq_file *s, void *v)
2161 if (option == OSN_DEFAULTS) {
2162 if (osnoise_options == OSN_DEFAULT_OPTIONS)
2163 seq_printf(s, "%s", osnoise_options_str[option]);
2165 seq_printf(s, "NO_%s", osnoise_options_str[option]);
2169 if (test_bit(option, &osnoise_options))
2170 seq_printf(s, "%s", osnoise_options_str[option]);
2172 seq_printf(s, "NO_%s", osnoise_options_str[option]);
2175 if (option != OSN_MAX)
2181 static void s_options_stop(struct seq_file *s, void *v)
2184 mutex_unlock(&interface_lock);
2187 static const struct seq_operations osnoise_options_seq_ops = {
2188 .start = s_options_start,
2189 .next = s_options_next,
2190 .show = s_options_show,
2191 .stop = s_options_stop
2194 static int osnoise_options_open(struct inode *inode, struct file *file)
2196 return seq_open(file, &osnoise_options_seq_ops);
2200 * osnoise_options_write - Write function for "options" entry
2201 * @filp: The active open file structure
2202 * @ubuf: The user buffer that contains the value to write
2203 * @cnt: The maximum number of bytes to write to "file"
2204 * @ppos: The current position in @file
2206 * Writing the option name sets the option, writing the "NO_"
2207 * prefix in front of the option name disables it.
2209 * Writing "DEFAULTS" resets the option values to the default ones.
2211 static ssize_t osnoise_options_write(struct file *filp, const char __user *ubuf,
2212 size_t cnt, loff_t *ppos)
2214 int running, option, enable, retval;
2215 char buf[256], *option_str;
2220 if (copy_from_user(buf, ubuf, cnt))
2225 if (strncmp(buf, "NO_", 3)) {
2226 option_str = strstrip(buf);
2229 option_str = strstrip(&buf[3]);
2233 option = match_string(osnoise_options_str, OSN_MAX, option_str);
2238 * trace_types_lock is taken to avoid concurrency on start/stop.
2240 mutex_lock(&trace_types_lock);
2241 running = osnoise_has_registered_instances();
2243 stop_per_cpu_kthreads();
2245 mutex_lock(&interface_lock);
2247 * avoid CPU hotplug operations that might read options.
2254 if (option == OSN_DEFAULTS)
2255 osnoise_options = OSN_DEFAULT_OPTIONS;
2257 set_bit(option, &osnoise_options);
2259 if (option == OSN_DEFAULTS)
2262 clear_bit(option, &osnoise_options);
2266 mutex_unlock(&interface_lock);
2269 start_per_cpu_kthreads();
2270 mutex_unlock(&trace_types_lock);
2276 * osnoise_cpus_read - Read function for reading the "cpus" file
2277 * @filp: The active open file structure
2278 * @ubuf: The userspace provided buffer to read value into
2279 * @cnt: The maximum number of bytes to read
2280 * @ppos: The current "file" position
2282 * Prints the "cpus" output into the user-provided buffer.
2285 osnoise_cpus_read(struct file *filp, char __user *ubuf, size_t count,
2291 mutex_lock(&interface_lock);
2293 len = snprintf(NULL, 0, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask)) + 1;
2294 mask_str = kmalloc(len, GFP_KERNEL);
2300 len = snprintf(mask_str, len, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask));
2306 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
2311 mutex_unlock(&interface_lock);
2317 * osnoise_cpus_write - Write function for "cpus" entry
2318 * @filp: The active open file structure
2319 * @ubuf: The user buffer that contains the value to write
2320 * @cnt: The maximum number of bytes to write to "file"
2321 * @ppos: The current position in @file
2323 * This function provides a write implementation for the "cpus"
2324 * interface to the osnoise trace. By default, it lists all CPUs,
2325 * in this way, allowing osnoise threads to run on any online CPU
2326 * of the system. It serves to restrict the execution of osnoise to the
2327 * set of CPUs writing via this interface. Why not use "tracing_cpumask"?
2328 * Because the user might be interested in tracing what is running on
2329 * other CPUs. For instance, one might run osnoise in one HT CPU
2330 * while observing what is running on the sibling HT CPU.
2333 osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count,
2336 cpumask_var_t osnoise_cpumask_new;
2343 if (copy_from_user(buf, ubuf, count))
2346 if (!zalloc_cpumask_var(&osnoise_cpumask_new, GFP_KERNEL))
2349 err = cpulist_parse(buf, osnoise_cpumask_new);
2354 * trace_types_lock is taken to avoid concurrency on start/stop.
2356 mutex_lock(&trace_types_lock);
2357 running = osnoise_has_registered_instances();
2359 stop_per_cpu_kthreads();
2361 mutex_lock(&interface_lock);
2363 * osnoise_cpumask is read by CPU hotplug operations.
2367 cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new);
2370 mutex_unlock(&interface_lock);
2373 start_per_cpu_kthreads();
2374 mutex_unlock(&trace_types_lock);
2376 free_cpumask_var(osnoise_cpumask_new);
2380 free_cpumask_var(osnoise_cpumask_new);
2385 #ifdef CONFIG_TIMERLAT_TRACER
2386 static int timerlat_fd_open(struct inode *inode, struct file *file)
2388 struct osnoise_variables *osn_var;
2389 struct timerlat_variables *tlat;
2390 long cpu = (long) inode->i_cdev;
2392 mutex_lock(&interface_lock);
2395 * This file is accessible only if timerlat is enabled, and
2396 * NO_OSNOISE_WORKLOAD is set.
2398 if (!timerlat_enabled() || test_bit(OSN_WORKLOAD, &osnoise_options)) {
2399 mutex_unlock(&interface_lock);
2405 osn_var = this_cpu_osn_var();
2408 * The osn_var->pid holds the single access to this file.
2411 mutex_unlock(&interface_lock);
2417 * timerlat tracer is a per-cpu tracer. Check if the user-space too
2418 * is pinned to a single CPU. The tracer laters monitor if the task
2419 * migrates and then disables tracer if it does. However, it is
2420 * worth doing this basic acceptance test to avoid obviusly wrong
2423 if (current->nr_cpus_allowed > 1 || cpu != smp_processor_id()) {
2424 mutex_unlock(&interface_lock);
2430 * From now on, it is good to go.
2432 file->private_data = inode->i_cdev;
2434 get_task_struct(current);
2436 osn_var->kthread = current;
2437 osn_var->pid = current->pid;
2442 mutex_unlock(&interface_lock);
2444 tlat = this_cpu_tmr_var();
2452 * timerlat_fd_read - Read function for "timerlat_fd" file
2453 * @file: The active open file structure
2454 * @ubuf: The userspace provided buffer to read value into
2455 * @cnt: The maximum number of bytes to read
2456 * @ppos: The current "file" position
2458 * Prints 1 on timerlat, the number of interferences on osnoise, -1 on error.
2461 timerlat_fd_read(struct file *file, char __user *ubuf, size_t count,
2464 long cpu = (long) file->private_data;
2465 struct osnoise_variables *osn_var;
2466 struct timerlat_variables *tlat;
2467 struct timerlat_sample s;
2473 tlat = this_cpu_tmr_var();
2476 * While in user-space, the thread is migratable. There is nothing
2477 * we can do about it.
2478 * So, if the thread is running on another CPU, stop the machinery.
2480 if (cpu == smp_processor_id()) {
2481 if (tlat->uthread_migrate) {
2486 per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1;
2487 osnoise_taint("timerlat user thread migrate\n");
2488 osnoise_stop_tracing();
2493 osn_var = this_cpu_osn_var();
2496 * The timerlat in user-space runs in a different order:
2497 * the read() starts from the execution of the previous occurrence,
2498 * sleeping for the next occurrence.
2500 * So, skip if we are entering on read() before the first wakeup
2501 * from timerlat IRQ:
2503 if (likely(osn_var->sampling)) {
2504 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
2505 diff = now - tlat->abs_period;
2508 * it was not a timer firing, but some other signal?
2513 s.seqnum = tlat->count;
2514 s.timer_latency = diff;
2515 s.context = THREAD_URET;
2517 trace_timerlat_sample(&s);
2519 notify_new_max_latency(diff);
2521 tlat->tracing_thread = false;
2522 if (osnoise_data.stop_tracing_total)
2523 if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
2524 osnoise_stop_tracing();
2526 tlat->tracing_thread = false;
2527 tlat->kthread = current;
2529 hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
2530 tlat->timer.function = timerlat_irq;
2532 /* Annotate now to drift new period */
2533 tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
2535 osn_var->sampling = 1;
2538 /* wait for the next period */
2539 wait_next_period(tlat);
2541 /* This is the wakeup from this cycle */
2542 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
2543 diff = now - tlat->abs_period;
2546 * it was not a timer firing, but some other signal?
2551 s.seqnum = tlat->count;
2552 s.timer_latency = diff;
2553 s.context = THREAD_CONTEXT;
2555 trace_timerlat_sample(&s);
2557 if (osnoise_data.stop_tracing_total) {
2558 if (time_to_us(diff) >= osnoise_data.stop_tracing_total) {
2559 timerlat_dump_stack(time_to_us(diff));
2560 notify_new_max_latency(diff);
2561 osnoise_stop_tracing();
2570 static int timerlat_fd_release(struct inode *inode, struct file *file)
2572 struct osnoise_variables *osn_var;
2573 struct timerlat_variables *tlat_var;
2574 long cpu = (long) file->private_data;
2577 mutex_lock(&interface_lock);
2579 osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
2580 tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
2582 hrtimer_cancel(&tlat_var->timer);
2583 memset(tlat_var, 0, sizeof(*tlat_var));
2585 osn_var->sampling = 0;
2589 * We are leaving, not being stopped... see stop_kthread();
2591 if (osn_var->kthread) {
2592 put_task_struct(osn_var->kthread);
2593 osn_var->kthread = NULL;
2596 mutex_unlock(&interface_lock);
2603 * osnoise/runtime_us: cannot be greater than the period.
2605 static struct trace_min_max_param osnoise_runtime = {
2606 .lock = &interface_lock,
2607 .val = &osnoise_data.sample_runtime,
2608 .max = &osnoise_data.sample_period,
2613 * osnoise/period_us: cannot be smaller than the runtime.
2615 static struct trace_min_max_param osnoise_period = {
2616 .lock = &interface_lock,
2617 .val = &osnoise_data.sample_period,
2619 .min = &osnoise_data.sample_runtime,
2623 * osnoise/stop_tracing_us: no limit.
2625 static struct trace_min_max_param osnoise_stop_tracing_in = {
2626 .lock = &interface_lock,
2627 .val = &osnoise_data.stop_tracing,
2633 * osnoise/stop_tracing_total_us: no limit.
2635 static struct trace_min_max_param osnoise_stop_tracing_total = {
2636 .lock = &interface_lock,
2637 .val = &osnoise_data.stop_tracing_total,
2642 #ifdef CONFIG_TIMERLAT_TRACER
2644 * osnoise/print_stack: print the stacktrace of the IRQ handler if the total
2645 * latency is higher than val.
2647 static struct trace_min_max_param osnoise_print_stack = {
2648 .lock = &interface_lock,
2649 .val = &osnoise_data.print_stack,
2655 * osnoise/timerlat_period: min 100 us, max 1 s
2657 static u64 timerlat_min_period = 100;
2658 static u64 timerlat_max_period = 1000000;
2659 static struct trace_min_max_param timerlat_period = {
2660 .lock = &interface_lock,
2661 .val = &osnoise_data.timerlat_period,
2662 .max = &timerlat_max_period,
2663 .min = &timerlat_min_period,
2666 static const struct file_operations timerlat_fd_fops = {
2667 .open = timerlat_fd_open,
2668 .read = timerlat_fd_read,
2669 .release = timerlat_fd_release,
2670 .llseek = generic_file_llseek,
2674 static const struct file_operations cpus_fops = {
2675 .open = tracing_open_generic,
2676 .read = osnoise_cpus_read,
2677 .write = osnoise_cpus_write,
2678 .llseek = generic_file_llseek,
2681 static const struct file_operations osnoise_options_fops = {
2682 .open = osnoise_options_open,
2684 .llseek = seq_lseek,
2685 .release = seq_release,
2686 .write = osnoise_options_write
2689 #ifdef CONFIG_TIMERLAT_TRACER
2690 #ifdef CONFIG_STACKTRACE
2691 static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2695 tmp = tracefs_create_file("print_stack", TRACE_MODE_WRITE, top_dir,
2696 &osnoise_print_stack, &trace_min_max_fops);
2702 #else /* CONFIG_STACKTRACE */
2703 static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2707 #endif /* CONFIG_STACKTRACE */
2709 static int osnoise_create_cpu_timerlat_fd(struct dentry *top_dir)
2711 struct dentry *timerlat_fd;
2712 struct dentry *per_cpu;
2713 struct dentry *cpu_dir;
2714 char cpu_str[30]; /* see trace.c: tracing_init_tracefs_percpu() */
2718 * Why not using tracing instance per_cpu/ dir?
2720 * Because osnoise/timerlat have a single workload, having
2721 * multiple files like these are wast of memory.
2723 per_cpu = tracefs_create_dir("per_cpu", top_dir);
2727 for_each_possible_cpu(cpu) {
2728 snprintf(cpu_str, 30, "cpu%ld", cpu);
2729 cpu_dir = tracefs_create_dir(cpu_str, per_cpu);
2733 timerlat_fd = trace_create_file("timerlat_fd", TRACE_MODE_READ,
2734 cpu_dir, NULL, &timerlat_fd_fops);
2738 /* Record the CPU */
2739 d_inode(timerlat_fd)->i_cdev = (void *)(cpu);
2745 tracefs_remove(per_cpu);
2750 * init_timerlat_tracefs - A function to initialize the timerlat interface files
2752 static int init_timerlat_tracefs(struct dentry *top_dir)
2757 tmp = tracefs_create_file("timerlat_period_us", TRACE_MODE_WRITE, top_dir,
2758 &timerlat_period, &trace_min_max_fops);
2762 retval = osnoise_create_cpu_timerlat_fd(top_dir);
2766 return init_timerlat_stack_tracefs(top_dir);
2768 #else /* CONFIG_TIMERLAT_TRACER */
2769 static int init_timerlat_tracefs(struct dentry *top_dir)
2773 #endif /* CONFIG_TIMERLAT_TRACER */
2776 * init_tracefs - A function to initialize the tracefs interface files
2778 * This function creates entries in tracefs for "osnoise" and "timerlat".
2779 * It creates these directories in the tracing directory, and within that
2780 * directory the use can change and view the configs.
2782 static int init_tracefs(void)
2784 struct dentry *top_dir;
2788 ret = tracing_init_dentry();
2792 top_dir = tracefs_create_dir("osnoise", NULL);
2796 tmp = tracefs_create_file("period_us", TRACE_MODE_WRITE, top_dir,
2797 &osnoise_period, &trace_min_max_fops);
2801 tmp = tracefs_create_file("runtime_us", TRACE_MODE_WRITE, top_dir,
2802 &osnoise_runtime, &trace_min_max_fops);
2806 tmp = tracefs_create_file("stop_tracing_us", TRACE_MODE_WRITE, top_dir,
2807 &osnoise_stop_tracing_in, &trace_min_max_fops);
2811 tmp = tracefs_create_file("stop_tracing_total_us", TRACE_MODE_WRITE, top_dir,
2812 &osnoise_stop_tracing_total, &trace_min_max_fops);
2816 tmp = trace_create_file("cpus", TRACE_MODE_WRITE, top_dir, NULL, &cpus_fops);
2820 tmp = trace_create_file("options", TRACE_MODE_WRITE, top_dir, NULL,
2821 &osnoise_options_fops);
2825 ret = init_timerlat_tracefs(top_dir);
2832 tracefs_remove(top_dir);
2836 static int osnoise_hook_events(void)
2841 * Trace is already hooked, we are re-enabling from
2844 if (trace_osnoise_callback_enabled)
2847 retval = hook_irq_events();
2851 retval = hook_softirq_events();
2853 goto out_unhook_irq;
2855 retval = hook_thread_events();
2862 unhook_softirq_events();
2864 unhook_irq_events();
2868 static void osnoise_unhook_events(void)
2870 unhook_thread_events();
2871 unhook_softirq_events();
2872 unhook_irq_events();
2876 * osnoise_workload_start - start the workload and hook to events
2878 static int osnoise_workload_start(void)
2883 * Instances need to be registered after calling workload
2884 * start. Hence, if there is already an instance, the
2885 * workload was already registered. Otherwise, this
2886 * code is on the way to register the first instance,
2887 * and the workload will start.
2889 if (osnoise_has_registered_instances())
2892 osn_var_reset_all();
2894 retval = osnoise_hook_events();
2899 * Make sure that ftrace_nmi_enter/exit() see reset values
2900 * before enabling trace_osnoise_callback_enabled.
2903 trace_osnoise_callback_enabled = true;
2905 retval = start_per_cpu_kthreads();
2907 trace_osnoise_callback_enabled = false;
2909 * Make sure that ftrace_nmi_enter/exit() see
2910 * trace_osnoise_callback_enabled as false before continuing.
2914 osnoise_unhook_events();
2922 * osnoise_workload_stop - stop the workload and unhook the events
2924 static void osnoise_workload_stop(void)
2927 * Instances need to be unregistered before calling
2928 * stop. Hence, if there is a registered instance, more
2929 * than one instance is running, and the workload will not
2930 * yet stop. Otherwise, this code is on the way to disable
2931 * the last instance, and the workload can stop.
2933 if (osnoise_has_registered_instances())
2937 * If callbacks were already disabled in a previous stop
2938 * call, there is no need to disable then again.
2940 * For instance, this happens when tracing is stopped via:
2941 * echo 0 > tracing_on
2942 * echo nop > current_tracer.
2944 if (!trace_osnoise_callback_enabled)
2947 trace_osnoise_callback_enabled = false;
2949 * Make sure that ftrace_nmi_enter/exit() see
2950 * trace_osnoise_callback_enabled as false before continuing.
2954 stop_per_cpu_kthreads();
2956 osnoise_unhook_events();
2959 static void osnoise_tracer_start(struct trace_array *tr)
2964 * If the instance is already registered, there is no need to
2965 * register it again.
2967 if (osnoise_instance_registered(tr))
2970 retval = osnoise_workload_start();
2972 pr_err(BANNER "Error starting osnoise tracer\n");
2974 osnoise_register_instance(tr);
2977 static void osnoise_tracer_stop(struct trace_array *tr)
2979 osnoise_unregister_instance(tr);
2980 osnoise_workload_stop();
2983 static int osnoise_tracer_init(struct trace_array *tr)
2986 * Only allow osnoise tracer if timerlat tracer is not running
2989 if (timerlat_enabled())
2992 tr->max_latency = 0;
2994 osnoise_tracer_start(tr);
2998 static void osnoise_tracer_reset(struct trace_array *tr)
3000 osnoise_tracer_stop(tr);
3003 static struct tracer osnoise_tracer __read_mostly = {
3005 .init = osnoise_tracer_init,
3006 .reset = osnoise_tracer_reset,
3007 .start = osnoise_tracer_start,
3008 .stop = osnoise_tracer_stop,
3009 .print_header = print_osnoise_headers,
3010 .allow_instances = true,
3013 #ifdef CONFIG_TIMERLAT_TRACER
3014 static void timerlat_tracer_start(struct trace_array *tr)
3019 * If the instance is already registered, there is no need to
3020 * register it again.
3022 if (osnoise_instance_registered(tr))
3025 retval = osnoise_workload_start();
3027 pr_err(BANNER "Error starting timerlat tracer\n");
3029 osnoise_register_instance(tr);
3034 static void timerlat_tracer_stop(struct trace_array *tr)
3038 osnoise_unregister_instance(tr);
3041 * Instruct the threads to stop only if this is the last instance.
3043 if (!osnoise_has_registered_instances()) {
3044 for_each_online_cpu(cpu)
3045 per_cpu(per_cpu_osnoise_var, cpu).sampling = 0;
3048 osnoise_workload_stop();
3051 static int timerlat_tracer_init(struct trace_array *tr)
3054 * Only allow timerlat tracer if osnoise tracer is not running already.
3056 if (osnoise_has_registered_instances() && !osnoise_data.timerlat_tracer)
3060 * If this is the first instance, set timerlat_tracer to block
3061 * osnoise tracer start.
3063 if (!osnoise_has_registered_instances())
3064 osnoise_data.timerlat_tracer = 1;
3066 tr->max_latency = 0;
3067 timerlat_tracer_start(tr);
3072 static void timerlat_tracer_reset(struct trace_array *tr)
3074 timerlat_tracer_stop(tr);
3077 * If this is the last instance, reset timerlat_tracer allowing
3078 * osnoise to be started.
3080 if (!osnoise_has_registered_instances())
3081 osnoise_data.timerlat_tracer = 0;
3084 static struct tracer timerlat_tracer __read_mostly = {
3086 .init = timerlat_tracer_init,
3087 .reset = timerlat_tracer_reset,
3088 .start = timerlat_tracer_start,
3089 .stop = timerlat_tracer_stop,
3090 .print_header = print_timerlat_headers,
3091 .allow_instances = true,
3094 __init static int init_timerlat_tracer(void)
3096 return register_tracer(&timerlat_tracer);
3098 #else /* CONFIG_TIMERLAT_TRACER */
3099 __init static int init_timerlat_tracer(void)
3103 #endif /* CONFIG_TIMERLAT_TRACER */
3105 __init static int init_osnoise_tracer(void)
3109 mutex_init(&interface_lock);
3111 cpumask_copy(&osnoise_cpumask, cpu_all_mask);
3113 ret = register_tracer(&osnoise_tracer);
3115 pr_err(BANNER "Error registering osnoise!\n");
3119 ret = init_timerlat_tracer();
3121 pr_err(BANNER "Error registering timerlat!\n");
3125 osnoise_init_hotplug_support();
3127 INIT_LIST_HEAD_RCU(&osnoise_instances);
3133 late_initcall(init_osnoise_tracer);