1 // SPDX-License-Identifier: GPL-2.0
3 #define pr_fmt(fmt) "rethook: " fmt
6 #include <linux/kallsyms.h>
7 #include <linux/kprobes.h>
8 #include <linux/preempt.h>
9 #include <linux/rethook.h>
10 #include <linux/slab.h>
11 #include <linux/sort.h>
13 /* Return hook list (shadow stack by list) */
16 * This function is called from delayed_put_task_struct() when a task is
17 * dead and cleaned up to recycle any kretprobe instances associated with
18 * this task. These left over instances represent probed functions that
19 * have been called but will never return.
21 void rethook_flush_task(struct task_struct *tk)
23 struct rethook_node *rhn;
24 struct llist_node *node;
26 node = __llist_del_all(&tk->rethooks);
28 rhn = container_of(node, struct rethook_node, llist);
36 static void rethook_free_rcu(struct rcu_head *head)
38 struct rethook *rh = container_of(head, struct rethook, rcu);
39 struct rethook_node *rhn;
40 struct freelist_node *node;
45 rhn = container_of(node, struct rethook_node, freelist);
51 /* The rh->ref is the number of pooled node + 1 */
52 if (refcount_sub_and_test(count, &rh->ref))
57 * rethook_stop() - Stop using a rethook.
58 * @rh: the struct rethook to stop.
60 * Stop using a rethook to prepare for freeing it. If you want to wait for
61 * all running rethook handler before calling rethook_free(), you need to
62 * call this first and wait RCU, and call rethook_free().
64 void rethook_stop(struct rethook *rh)
66 rcu_assign_pointer(rh->handler, NULL);
70 * rethook_free() - Free struct rethook.
71 * @rh: the struct rethook to be freed.
73 * Free the rethook. Before calling this function, user must ensure the
74 * @rh::data is cleaned if needed (or, the handler can access it after
75 * calling this function.) This function will set the @rh to be freed
76 * after all rethook_node are freed (not soon). And the caller must
77 * not touch @rh after calling this.
79 void rethook_free(struct rethook *rh)
83 call_rcu(&rh->rcu, rethook_free_rcu);
86 static inline rethook_handler_t rethook_get_handler(struct rethook *rh)
88 return (rethook_handler_t)rcu_dereference_check(rh->handler,
89 rcu_read_lock_any_held());
93 * rethook_alloc() - Allocate struct rethook.
94 * @data: a data to pass the @handler when hooking the return.
95 * @handler: the return hook callback function.
97 * Allocate and initialize a new rethook with @data and @handler.
98 * Return NULL if memory allocation fails or @handler is NULL.
99 * Note that @handler == NULL means this rethook is going to be freed.
101 struct rethook *rethook_alloc(void *data, rethook_handler_t handler)
103 struct rethook *rh = kzalloc(sizeof(struct rethook), GFP_KERNEL);
105 if (!rh || !handler) {
111 rcu_assign_pointer(rh->handler, handler);
112 rh->pool.head = NULL;
113 refcount_set(&rh->ref, 1);
119 * rethook_add_node() - Add a new node to the rethook.
120 * @rh: the struct rethook.
121 * @node: the struct rethook_node to be added.
123 * Add @node to @rh. User must allocate @node (as a part of user's
124 * data structure.) The @node fields are initialized in this function.
126 void rethook_add_node(struct rethook *rh, struct rethook_node *node)
129 freelist_add(&node->freelist, &rh->pool);
130 refcount_inc(&rh->ref);
133 static void free_rethook_node_rcu(struct rcu_head *head)
135 struct rethook_node *node = container_of(head, struct rethook_node, rcu);
137 if (refcount_dec_and_test(&node->rethook->ref))
138 kfree(node->rethook);
143 * rethook_recycle() - return the node to rethook.
144 * @node: The struct rethook_node to be returned.
146 * Return back the @node to @node::rethook. If the @node::rethook is already
147 * marked as freed, this will free the @node.
149 void rethook_recycle(struct rethook_node *node)
151 rethook_handler_t handler;
153 handler = rethook_get_handler(node->rethook);
155 freelist_add(&node->freelist, &node->rethook->pool);
157 call_rcu(&node->rcu, free_rethook_node_rcu);
159 NOKPROBE_SYMBOL(rethook_recycle);
162 * rethook_try_get() - get an unused rethook node.
163 * @rh: The struct rethook which pools the nodes.
165 * Get an unused rethook node from @rh. If the node pool is empty, this
166 * will return NULL. Caller must disable preemption.
168 struct rethook_node *rethook_try_get(struct rethook *rh)
170 rethook_handler_t handler = rethook_get_handler(rh);
171 struct freelist_node *fn;
173 /* Check whether @rh is going to be freed. */
174 if (unlikely(!handler))
178 * This expects the caller will set up a rethook on a function entry.
179 * When the function returns, the rethook will eventually be reclaimed
180 * or released in the rethook_recycle() with call_rcu().
181 * This means the caller must be run in the RCU-availabe context.
183 if (unlikely(!rcu_is_watching()))
186 fn = freelist_try_get(&rh->pool);
190 return container_of(fn, struct rethook_node, freelist);
192 NOKPROBE_SYMBOL(rethook_try_get);
195 * rethook_hook() - Hook the current function return.
196 * @node: The struct rethook node to hook the function return.
197 * @regs: The struct pt_regs for the function entry.
198 * @mcount: True if this is called from mcount(ftrace) context.
200 * Hook the current running function return. This must be called when the
201 * function entry (or at least @regs must be the registers of the function
202 * entry.) @mcount is used for identifying the context. If this is called
203 * from ftrace (mcount) callback, @mcount must be set true. If this is called
204 * from the real function entry (e.g. kprobes) @mcount must be set false.
205 * This is because the way to hook the function return depends on the context.
207 void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount)
209 arch_rethook_prepare(node, regs, mcount);
210 __llist_add(&node->llist, ¤t->rethooks);
212 NOKPROBE_SYMBOL(rethook_hook);
214 /* This assumes the 'tsk' is the current task or is not running. */
215 static unsigned long __rethook_find_ret_addr(struct task_struct *tsk,
216 struct llist_node **cur)
218 struct rethook_node *rh = NULL;
219 struct llist_node *node = *cur;
222 node = tsk->rethooks.first;
227 rh = container_of(node, struct rethook_node, llist);
228 if (rh->ret_addr != (unsigned long)arch_rethook_trampoline) {
236 NOKPROBE_SYMBOL(__rethook_find_ret_addr);
239 * rethook_find_ret_addr -- Find correct return address modified by rethook
241 * @frame: A frame pointer
242 * @cur: a storage of the loop cursor llist_node pointer for next call
244 * Find the correct return address modified by a rethook on @tsk in unsigned
246 * The @tsk must be 'current' or a task which is not running. @frame is a hint
247 * to get the currect return address - which is compared with the
248 * rethook::frame field. The @cur is a loop cursor for searching the
249 * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
250 * first call, but '@cur' itself must NOT NULL.
252 * Returns found address value or zero if not found.
254 unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame,
255 struct llist_node **cur)
257 struct rethook_node *rhn = NULL;
260 if (WARN_ON_ONCE(!cur))
263 if (WARN_ON_ONCE(tsk != current && task_is_running(tsk)))
267 ret = __rethook_find_ret_addr(tsk, cur);
270 rhn = container_of(*cur, struct rethook_node, llist);
271 } while (rhn->frame != frame);
275 NOKPROBE_SYMBOL(rethook_find_ret_addr);
277 void __weak arch_rethook_fixup_return(struct pt_regs *regs,
278 unsigned long correct_ret_addr)
281 * Do nothing by default. If the architecture which uses a
282 * frame pointer to record real return address on the stack,
283 * it should fill this function to fixup the return address
284 * so that stacktrace works from the rethook handler.
288 /* This function will be called from each arch-defined trampoline. */
289 unsigned long rethook_trampoline_handler(struct pt_regs *regs,
292 struct llist_node *first, *node = NULL;
293 unsigned long correct_ret_addr;
294 rethook_handler_t handler;
295 struct rethook_node *rhn;
297 correct_ret_addr = __rethook_find_ret_addr(current, &node);
298 if (!correct_ret_addr) {
299 pr_err("rethook: Return address not found! Maybe there is a bug in the kernel\n");
303 instruction_pointer_set(regs, correct_ret_addr);
306 * These loops must be protected from rethook_free_rcu() because those
307 * are accessing 'rhn->rethook'.
309 preempt_disable_notrace();
312 * Run the handler on the shadow stack. Do not unlink the list here because
313 * stackdump inside the handlers needs to decode it.
315 first = current->rethooks.first;
317 rhn = container_of(first, struct rethook_node, llist);
318 if (WARN_ON_ONCE(rhn->frame != frame))
320 handler = rethook_get_handler(rhn->rethook);
322 handler(rhn, rhn->rethook->data,
323 correct_ret_addr, regs);
330 /* Fixup registers for returning to correct address. */
331 arch_rethook_fixup_return(regs, correct_ret_addr);
333 /* Unlink used shadow stack */
334 first = current->rethooks.first;
335 current->rethooks.first = node->next;
339 rhn = container_of(first, struct rethook_node, llist);
341 rethook_recycle(rhn);
343 preempt_enable_notrace();
345 return correct_ret_addr;
347 NOKPROBE_SYMBOL(rethook_trampoline_handler);