include/linux/context_tracking_state.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _LINUX_CONTEXT_TRACKING_STATE_H
   3 #define _LINUX_CONTEXT_TRACKING_STATE_H
   4
   5 #include <linux/percpu.h>
   6 #include <linux/static_key.h>
   7 #include <linux/context_tracking_irq.h>
   8
   9 /* Offset to allow distinguishing irq vs. task-based idle entry/exit. */
  10 #define DYNTICK_IRQ_NONIDLE     ((LONG_MAX / 2) + 1)
  11
  12 enum ctx_state {
  13         CONTEXT_DISABLED        = -1,   /* returned by ct_state() if unknown */
  14         CONTEXT_KERNEL          = 0,
  15         CONTEXT_IDLE            = 1,
  16         CONTEXT_USER            = 2,
  17         CONTEXT_GUEST           = 3,
  18         CONTEXT_MAX             = 4,
  19 };
  20
  21 /* Even value for idle, else odd. */
  22 #define RCU_DYNTICKS_IDX CONTEXT_MAX
  23
  24 #define CT_STATE_MASK (CONTEXT_MAX - 1)
  25 #define CT_DYNTICKS_MASK (~CT_STATE_MASK)
  26
  27 struct context_tracking {
  28 #ifdef CONFIG_CONTEXT_TRACKING_USER
  29         /*
  30          * When active is false, probes are unset in order
  31          * to minimize overhead: TIF flags are cleared
  32          * and calls to user_enter/exit are ignored. This
  33          * may be further optimized using static keys.
  34          */
  35         bool active;
  36         int recursion;
  37 #endif
  38 #ifdef CONFIG_CONTEXT_TRACKING
  39         atomic_t state;
  40 #endif
  41 #ifdef CONFIG_CONTEXT_TRACKING_IDLE
  42         long dynticks_nesting;          /* Track process nesting level. */
  43         long dynticks_nmi_nesting;      /* Track irq/NMI nesting level. */
  44 #endif
  45 };
  46
  47 #ifdef CONFIG_CONTEXT_TRACKING
  48 DECLARE_PER_CPU(struct context_tracking, context_tracking);
  49 #endif
  50
  51 #ifdef CONFIG_CONTEXT_TRACKING_USER
  52 static __always_inline int __ct_state(void)
  53 {
  54         return raw_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK;
  55 }
  56 #endif
  57
  58 #ifdef CONFIG_CONTEXT_TRACKING_IDLE
  59 static __always_inline int ct_dynticks(void)
  60 {
  61         return atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_DYNTICKS_MASK;
  62 }
  63
  64 static __always_inline int ct_dynticks_cpu(int cpu)
  65 {
  66         struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
  67
  68         return atomic_read(&ct->state) & CT_DYNTICKS_MASK;
  69 }
  70
  71 static __always_inline int ct_dynticks_cpu_acquire(int cpu)
  72 {
  73         struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
  74
  75         return atomic_read_acquire(&ct->state) & CT_DYNTICKS_MASK;
  76 }
  77
  78 static __always_inline long ct_dynticks_nesting(void)
  79 {
  80         return __this_cpu_read(context_tracking.dynticks_nesting);
  81 }
  82
  83 static __always_inline long ct_dynticks_nesting_cpu(int cpu)
  84 {
  85         struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
  86
  87         return ct->dynticks_nesting;
  88 }
  89
  90 static __always_inline long ct_dynticks_nmi_nesting(void)
  91 {
  92         return __this_cpu_read(context_tracking.dynticks_nmi_nesting);
  93 }
  94
  95 static __always_inline long ct_dynticks_nmi_nesting_cpu(int cpu)
  96 {
  97         struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
  98
  99         return ct->dynticks_nmi_nesting;
 100 }
 101 #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */
 102
 103 #ifdef CONFIG_CONTEXT_TRACKING_USER
 104 extern struct static_key_false context_tracking_key;
 105
 106 static __always_inline bool context_tracking_enabled(void)
 107 {
 108         return static_branch_unlikely(&context_tracking_key);
 109 }
 110
 111 static __always_inline bool context_tracking_enabled_cpu(int cpu)
 112 {
 113         return context_tracking_enabled() && per_cpu(context_tracking.active, cpu);
 114 }
 115
 116 static inline bool context_tracking_enabled_this_cpu(void)
 117 {
 118         return context_tracking_enabled() && __this_cpu_read(context_tracking.active);
 119 }
 120
 121 /**
 122  * ct_state() - return the current context tracking state if known
 123  *
 124  * Returns the current cpu's context tracking state if context tracking
 125  * is enabled.  If context tracking is disabled, returns
 126  * CONTEXT_DISABLED.  This should be used primarily for debugging.
 127  */
 128 static __always_inline int ct_state(void)
 129 {
 130         int ret;
 131
 132         if (!context_tracking_enabled())
 133                 return CONTEXT_DISABLED;
 134
 135         preempt_disable();
 136         ret = __ct_state();
 137         preempt_enable();
 138
 139         return ret;
 140 }
 141
 142 #else
 143 static __always_inline bool context_tracking_enabled(void) { return false; }
 144 static __always_inline bool context_tracking_enabled_cpu(int cpu) { return false; }
 145 static __always_inline bool context_tracking_enabled_this_cpu(void) { return false; }
 146 #endif /* CONFIG_CONTEXT_TRACKING_USER */
 147
 148 #endif