kernel/trace/trace_preemptirq.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * preemptoff and irqoff tracepoints
   4  *
   5  * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org>
   6  */
   7
   8 #include <linux/kallsyms.h>
   9 #include <linux/uaccess.h>
  10 #include <linux/module.h>
  11 #include <linux/ftrace.h>
  12 #include <linux/kprobes.h>
  13 #include "trace.h"
  14
  15 #define CREATE_TRACE_POINTS
  16 #include <trace/events/preemptirq.h>
  17
  18 /*
  19  * Use regular trace points on architectures that implement noinstr
  20  * tooling: these calls will only happen with RCU enabled, which can
  21  * use a regular tracepoint.
  22  *
  23  * On older architectures, use the rcuidle tracing methods (which
  24  * aren't NMI-safe - so exclude NMI contexts):
  25  */
  26 #ifdef CONFIG_ARCH_WANTS_NO_INSTR
  27 #define trace(point)    trace_##point
  28 #else
  29 #define trace(point)    if (!in_nmi()) trace_##point##_rcuidle
  30 #endif
  31
  32 #ifdef CONFIG_TRACE_IRQFLAGS
  33 /* Per-cpu variable to prevent redundant calls when IRQs already off */
  34 static DEFINE_PER_CPU(int, tracing_irq_cpu);
  35
  36 /*
  37  * Like trace_hardirqs_on() but without the lockdep invocation. This is
  38  * used in the low level entry code where the ordering vs. RCU is important
  39  * and lockdep uses a staged approach which splits the lockdep hardirq
  40  * tracking into a RCU on and a RCU off section.
  41  */
  42 void trace_hardirqs_on_prepare(void)
  43 {
  44         if (this_cpu_read(tracing_irq_cpu)) {
  45                 trace(irq_enable)(CALLER_ADDR0, CALLER_ADDR1);
  46                 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
  47                 this_cpu_write(tracing_irq_cpu, 0);
  48         }
  49 }
  50 EXPORT_SYMBOL(trace_hardirqs_on_prepare);
  51 NOKPROBE_SYMBOL(trace_hardirqs_on_prepare);
  52
  53 void trace_hardirqs_on(void)
  54 {
  55         if (this_cpu_read(tracing_irq_cpu)) {
  56                 trace(irq_enable)(CALLER_ADDR0, CALLER_ADDR1);
  57                 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
  58                 this_cpu_write(tracing_irq_cpu, 0);
  59         }
  60
  61         lockdep_hardirqs_on_prepare();
  62         lockdep_hardirqs_on(CALLER_ADDR0);
  63 }
  64 EXPORT_SYMBOL(trace_hardirqs_on);
  65 NOKPROBE_SYMBOL(trace_hardirqs_on);
  66
  67 /*
  68  * Like trace_hardirqs_off() but without the lockdep invocation. This is
  69  * used in the low level entry code where the ordering vs. RCU is important
  70  * and lockdep uses a staged approach which splits the lockdep hardirq
  71  * tracking into a RCU on and a RCU off section.
  72  */
  73 void trace_hardirqs_off_finish(void)
  74 {
  75         if (!this_cpu_read(tracing_irq_cpu)) {
  76                 this_cpu_write(tracing_irq_cpu, 1);
  77                 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
  78                 trace(irq_disable)(CALLER_ADDR0, CALLER_ADDR1);
  79         }
  80
  81 }
  82 EXPORT_SYMBOL(trace_hardirqs_off_finish);
  83 NOKPROBE_SYMBOL(trace_hardirqs_off_finish);
  84
  85 void trace_hardirqs_off(void)
  86 {
  87         lockdep_hardirqs_off(CALLER_ADDR0);
  88
  89         if (!this_cpu_read(tracing_irq_cpu)) {
  90                 this_cpu_write(tracing_irq_cpu, 1);
  91                 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
  92                 trace(irq_disable)(CALLER_ADDR0, CALLER_ADDR1);
  93         }
  94 }
  95 EXPORT_SYMBOL(trace_hardirqs_off);
  96 NOKPROBE_SYMBOL(trace_hardirqs_off);
  97 #endif /* CONFIG_TRACE_IRQFLAGS */
  98
  99 #ifdef CONFIG_TRACE_PREEMPT_TOGGLE
 100
 101 void trace_preempt_on(unsigned long a0, unsigned long a1)
 102 {
 103         trace(preempt_enable)(a0, a1);
 104         tracer_preempt_on(a0, a1);
 105 }
 106
 107 void trace_preempt_off(unsigned long a0, unsigned long a1)
 108 {
 109         trace(preempt_disable)(a0, a1);
 110         tracer_preempt_off(a0, a1);
 111 }
 112 #endif