1 #ifndef _LINUX_JUMP_LABEL_H
2 #define _LINUX_JUMP_LABEL_H
7 * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
8 * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com>
10 * Jump labels provide an interface to generate dynamic branches using
11 * self-modifying code. Assuming toolchain and architecture support the result
12 * of a "if (static_branch(&key))" statement is a unconditional branch (which
13 * defaults to false - and the true block is placed out of line).
15 * However at runtime we can change the 'static' branch target using
16 * jump_label_{inc,dec}(). These function as a 'reference' count on the key
17 * object and for as long as there are references all branches referring to
18 * that particular key will point to the (out of line) true block.
20 * Since this relies on modifying code the jump_label_{inc,dec}() functions
21 * must be considered absolute slow paths (machine wide synchronization etc.).
22 * OTOH, since the affected branches are unconditional their runtime overhead
23 * will be absolutely minimal, esp. in the default (off) case where the total
24 * effect is a single NOP of appropriate size. The on case will patch in a jump
25 * to the out-of-line block.
27 * When the control is directly exposed to userspace it is prudent to delay the
28 * decrement to avoid high frequency code modifications which can (and do)
29 * cause significant performance degradation. Struct jump_label_key_deferred and
30 * jump_label_dec_deferred() provide for this.
32 * Lacking toolchain and or architecture support, it falls back to a simple
36 #include <linux/types.h>
37 #include <linux/compiler.h>
38 #include <linux/workqueue.h>
40 #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
42 struct jump_label_key {
44 struct jump_entry *entries;
46 struct jump_label_mod *next;
50 struct jump_label_key_deferred {
51 struct jump_label_key key;
52 unsigned long timeout;
53 struct delayed_work work;
56 # include <asm/jump_label.h>
57 # define HAVE_JUMP_LABEL
58 #endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
60 enum jump_label_type {
61 JUMP_LABEL_DISABLE = 0,
67 #ifdef HAVE_JUMP_LABEL
70 #define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL, NULL}
72 #define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL}
75 static __always_inline bool static_branch(struct jump_label_key *key)
77 return arch_static_branch(key);
80 extern struct jump_entry __start___jump_table[];
81 extern struct jump_entry __stop___jump_table[];
83 extern void jump_label_init(void);
84 extern void jump_label_lock(void);
85 extern void jump_label_unlock(void);
86 extern void arch_jump_label_transform(struct jump_entry *entry,
87 enum jump_label_type type);
88 extern void arch_jump_label_transform_static(struct jump_entry *entry,
89 enum jump_label_type type);
90 extern int jump_label_text_reserved(void *start, void *end);
91 extern void jump_label_inc(struct jump_label_key *key);
92 extern void jump_label_dec(struct jump_label_key *key);
93 extern void jump_label_dec_deferred(struct jump_label_key_deferred *key);
94 extern bool jump_label_enabled(struct jump_label_key *key);
95 extern void jump_label_apply_nops(struct module *mod);
96 extern void jump_label_rate_limit(struct jump_label_key_deferred *key,
99 #else /* !HAVE_JUMP_LABEL */
101 #include <linux/atomic.h>
103 #define JUMP_LABEL_INIT {ATOMIC_INIT(0)}
105 struct jump_label_key {
109 static __always_inline void jump_label_init(void)
113 struct jump_label_key_deferred {
114 struct jump_label_key key;
117 static __always_inline bool static_branch(struct jump_label_key *key)
119 if (unlikely(atomic_read(&key->enabled)))
124 static inline void jump_label_inc(struct jump_label_key *key)
126 atomic_inc(&key->enabled);
129 static inline void jump_label_dec(struct jump_label_key *key)
131 atomic_dec(&key->enabled);
134 static inline void jump_label_dec_deferred(struct jump_label_key_deferred *key)
136 jump_label_dec(&key->key);
139 static inline int jump_label_text_reserved(void *start, void *end)
144 static inline void jump_label_lock(void) {}
145 static inline void jump_label_unlock(void) {}
147 static inline bool jump_label_enabled(struct jump_label_key *key)
149 return !!atomic_read(&key->enabled);
152 static inline int jump_label_apply_nops(struct module *mod)
157 static inline void jump_label_rate_limit(struct jump_label_key_deferred *key,
161 #endif /* HAVE_JUMP_LABEL */
163 #define jump_label_key_enabled ((struct jump_label_key){ .enabled = ATOMIC_INIT(1), })
164 #define jump_label_key_disabled ((struct jump_label_key){ .enabled = ATOMIC_INIT(0), })
166 #endif /* _LINUX_JUMP_LABEL_H */