kernel/trace/trace_preemptirq.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * preemptoff and irqoff tracepoints
   4  *
   5  * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org>
   6  */
   7
   8 #include <linux/kallsyms.h>
   9 #include <linux/uaccess.h>
  10 #include <linux/module.h>
  11 #include <linux/ftrace.h>
  12 #include <linux/kprobes.h>
  13 #include "trace.h"
  14
  15 #define CREATE_TRACE_POINTS
  16 #include <trace/events/preemptirq.h>
  17
  18 #ifdef CONFIG_TRACE_IRQFLAGS
  19 /* Per-cpu variable to prevent redundant calls when IRQs already off */
  20 static DEFINE_PER_CPU(int, tracing_irq_cpu);
  21
  22 /*
  23  * Like trace_hardirqs_on() but without the lockdep invocation. This is
  24  * used in the low level entry code where the ordering vs. RCU is important
  25  * and lockdep uses a staged approach which splits the lockdep hardirq
  26  * tracking into a RCU on and a RCU off section.
  27  */
  28 void trace_hardirqs_on_prepare(void)
  29 {
  30         if (this_cpu_read(tracing_irq_cpu)) {
  31                 if (!in_nmi())
  32                         trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1);
  33                 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
  34                 this_cpu_write(tracing_irq_cpu, 0);
  35         }
  36 }
  37 EXPORT_SYMBOL(trace_hardirqs_on_prepare);
  38 NOKPROBE_SYMBOL(trace_hardirqs_on_prepare);
  39
  40 void trace_hardirqs_on(void)
  41 {
  42         if (this_cpu_read(tracing_irq_cpu)) {
  43                 if (!in_nmi())
  44                         trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
  45                 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
  46                 this_cpu_write(tracing_irq_cpu, 0);
  47         }
  48
  49         lockdep_hardirqs_on_prepare();
  50         lockdep_hardirqs_on(CALLER_ADDR0);
  51 }
  52 EXPORT_SYMBOL(trace_hardirqs_on);
  53 NOKPROBE_SYMBOL(trace_hardirqs_on);
  54
  55 /*
  56  * Like trace_hardirqs_off() but without the lockdep invocation. This is
  57  * used in the low level entry code where the ordering vs. RCU is important
  58  * and lockdep uses a staged approach which splits the lockdep hardirq
  59  * tracking into a RCU on and a RCU off section.
  60  */
  61 void trace_hardirqs_off_finish(void)
  62 {
  63         if (!this_cpu_read(tracing_irq_cpu)) {
  64                 this_cpu_write(tracing_irq_cpu, 1);
  65                 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
  66                 if (!in_nmi())
  67                         trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1);
  68         }
  69
  70 }
  71 EXPORT_SYMBOL(trace_hardirqs_off_finish);
  72 NOKPROBE_SYMBOL(trace_hardirqs_off_finish);
  73
  74 void trace_hardirqs_off(void)
  75 {
  76         lockdep_hardirqs_off(CALLER_ADDR0);
  77
  78         if (!this_cpu_read(tracing_irq_cpu)) {
  79                 this_cpu_write(tracing_irq_cpu, 1);
  80                 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
  81                 if (!in_nmi())
  82                         trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
  83         }
  84 }
  85 EXPORT_SYMBOL(trace_hardirqs_off);
  86 NOKPROBE_SYMBOL(trace_hardirqs_off);
  87
  88 __visible void trace_hardirqs_on_caller(unsigned long caller_addr)
  89 {
  90         if (this_cpu_read(tracing_irq_cpu)) {
  91                 if (!in_nmi())
  92                         trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr);
  93                 tracer_hardirqs_on(CALLER_ADDR0, caller_addr);
  94                 this_cpu_write(tracing_irq_cpu, 0);
  95         }
  96
  97         lockdep_hardirqs_on_prepare();
  98         lockdep_hardirqs_on(caller_addr);
  99 }
 100 EXPORT_SYMBOL(trace_hardirqs_on_caller);
 101 NOKPROBE_SYMBOL(trace_hardirqs_on_caller);
 102
 103 __visible void trace_hardirqs_off_caller(unsigned long caller_addr)
 104 {
 105         lockdep_hardirqs_off(caller_addr);
 106
 107         if (!this_cpu_read(tracing_irq_cpu)) {
 108                 this_cpu_write(tracing_irq_cpu, 1);
 109                 tracer_hardirqs_off(CALLER_ADDR0, caller_addr);
 110                 if (!in_nmi())
 111                         trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr);
 112         }
 113 }
 114 EXPORT_SYMBOL(trace_hardirqs_off_caller);
 115 NOKPROBE_SYMBOL(trace_hardirqs_off_caller);
 116 #endif /* CONFIG_TRACE_IRQFLAGS */
 117
 118 #ifdef CONFIG_TRACE_PREEMPT_TOGGLE
 119
 120 void trace_preempt_on(unsigned long a0, unsigned long a1)
 121 {
 122         if (!in_nmi())
 123                 trace_preempt_enable_rcuidle(a0, a1);
 124         tracer_preempt_on(a0, a1);
 125 }
 126
 127 void trace_preempt_off(unsigned long a0, unsigned long a1)
 128 {
 129         if (!in_nmi())
 130                 trace_preempt_disable_rcuidle(a0, a1);
 131         tracer_preempt_off(a0, a1);
 132 }
 133 #endif