2 * core.c - Kernel Live Patching Core
4 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
5 * Copyright (C) 2014 SUSE
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 2
10 * of the License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, see <http://www.gnu.org/licenses/>.
21 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
23 #include <linux/module.h>
24 #include <linux/kernel.h>
25 #include <linux/mutex.h>
26 #include <linux/slab.h>
27 #include <linux/ftrace.h>
28 #include <linux/list.h>
29 #include <linux/kallsyms.h>
30 #include <linux/livepatch.h>
33 * struct klp_ops - structure for tracking registered ftrace ops structs
35 * A single ftrace_ops is shared between all enabled replacement functions
36 * (klp_func structs) which have the same old_addr. This allows the switch
37 * between function versions to happen instantaneously by updating the klp_ops
38 * struct's func_stack list. The winner is the klp_func at the top of the
39 * func_stack (front of the list).
41 * @node: node for the global klp_ops list
42 * @func_stack: list head for the stack of klp_func's (active func is on top)
43 * @fops: registered ftrace ops struct
46 struct list_head node;
47 struct list_head func_stack;
48 struct ftrace_ops fops;
52 * The klp_mutex protects the global lists and state transitions of any
53 * structure reachable from them. References to any structure must be obtained
54 * under mutex protection (except in klp_ftrace_handler(), which uses RCU to
55 * ensure it gets consistent data).
57 static DEFINE_MUTEX(klp_mutex);
59 static LIST_HEAD(klp_patches);
60 static LIST_HEAD(klp_ops);
62 static struct kobject *klp_root_kobj;
64 static struct klp_ops *klp_find_ops(unsigned long old_addr)
67 struct klp_func *func;
69 list_for_each_entry(ops, &klp_ops, node) {
70 func = list_first_entry(&ops->func_stack, struct klp_func,
72 if (func->old_addr == old_addr)
79 static bool klp_is_module(struct klp_object *obj)
84 static bool klp_is_object_loaded(struct klp_object *obj)
86 return !obj->name || obj->mod;
89 /* sets obj->mod if object is not vmlinux and module is found */
90 static void klp_find_object_module(struct klp_object *obj)
94 if (!klp_is_module(obj))
97 mutex_lock(&module_mutex);
99 * We do not want to block removal of patched modules and therefore
100 * we do not take a reference here. The patches are removed by
101 * a going module handler instead.
103 mod = find_module(obj->name);
105 * Do not mess work of the module coming and going notifiers.
106 * Note that the patch might still be needed before the going handler
107 * is called. Module functions can be called even in the GOING state
108 * until mod->exit() finishes. This is especially important for
109 * patches that modify semantic of the functions.
111 if (mod && mod->klp_alive)
114 mutex_unlock(&module_mutex);
117 /* klp_mutex must be held by caller */
118 static bool klp_is_patch_registered(struct klp_patch *patch)
120 struct klp_patch *mypatch;
122 list_for_each_entry(mypatch, &klp_patches, list)
123 if (mypatch == patch)
129 static bool klp_initialized(void)
131 return !!klp_root_kobj;
134 struct klp_find_arg {
139 * If count == 0, the symbol was not found. If count == 1, a unique
140 * match was found and addr is set. If count > 1, there is
141 * unresolvable ambiguity among "count" number of symbols with the same
142 * name in the same object.
147 static int klp_find_callback(void *data, const char *name,
148 struct module *mod, unsigned long addr)
150 struct klp_find_arg *args = data;
152 if ((mod && !args->objname) || (!mod && args->objname))
155 if (strcmp(args->name, name))
158 if (args->objname && strcmp(args->objname, mod->name))
162 * args->addr might be overwritten if another match is found
163 * but klp_find_object_symbol() handles this and only returns the
164 * addr if count == 1.
172 static int klp_find_object_symbol(const char *objname, const char *name,
175 struct klp_find_arg args = {
182 kallsyms_on_each_symbol(klp_find_callback, &args);
185 pr_err("symbol '%s' not found in symbol table\n", name);
186 else if (args.count > 1)
187 pr_err("unresolvable ambiguity (%lu matches) on symbol '%s' in object '%s'\n",
188 args.count, name, objname);
198 struct klp_verify_args {
200 const unsigned long addr;
203 static int klp_verify_callback(void *data, const char *name,
204 struct module *mod, unsigned long addr)
206 struct klp_verify_args *args = data;
209 !strcmp(args->name, name) &&
216 static int klp_verify_vmlinux_symbol(const char *name, unsigned long addr)
218 struct klp_verify_args args = {
223 if (kallsyms_on_each_symbol(klp_verify_callback, &args))
226 pr_err("symbol '%s' not found at specified address 0x%016lx, kernel mismatch?\n",
231 static int klp_find_verify_func_addr(struct klp_object *obj,
232 struct klp_func *func)
236 #if defined(CONFIG_RANDOMIZE_BASE)
237 /* KASLR is enabled, disregard old_addr from user */
241 if (!func->old_addr || klp_is_module(obj))
242 ret = klp_find_object_symbol(obj->name, func->old_name,
245 ret = klp_verify_vmlinux_symbol(func->old_name,
252 * external symbols are located outside the parent object (where the parent
253 * object is either vmlinux or the kmod being patched).
255 static int klp_find_external_symbol(struct module *pmod, const char *name,
258 const struct kernel_symbol *sym;
260 /* first, check if it's an exported symbol */
262 sym = find_symbol(name, NULL, NULL, true, true);
270 /* otherwise check if it's in another .o within the patch module */
271 return klp_find_object_symbol(pmod->name, name, addr);
274 static int klp_write_object_relocations(struct module *pmod,
275 struct klp_object *obj)
278 struct klp_reloc *reloc;
280 if (WARN_ON(!klp_is_object_loaded(obj)))
283 if (WARN_ON(!obj->relocs))
286 for (reloc = obj->relocs; reloc->name; reloc++) {
287 if (!klp_is_module(obj)) {
288 ret = klp_verify_vmlinux_symbol(reloc->name,
293 /* module, reloc->val needs to be discovered */
295 ret = klp_find_external_symbol(pmod,
299 ret = klp_find_object_symbol(obj->mod->name,
305 ret = klp_write_module_reloc(pmod, reloc->type, reloc->loc,
306 reloc->val + reloc->addend);
308 pr_err("relocation failed for symbol '%s' at 0x%016lx (%d)\n",
309 reloc->name, reloc->val, ret);
317 static void notrace klp_ftrace_handler(unsigned long ip,
318 unsigned long parent_ip,
319 struct ftrace_ops *fops,
320 struct pt_regs *regs)
323 struct klp_func *func;
325 ops = container_of(fops, struct klp_ops, fops);
328 func = list_first_or_null_rcu(&ops->func_stack, struct klp_func,
330 if (WARN_ON_ONCE(!func))
333 klp_arch_set_pc(regs, (unsigned long)func->new_func);
338 static void klp_disable_func(struct klp_func *func)
342 WARN_ON(func->state != KLP_ENABLED);
343 WARN_ON(!func->old_addr);
345 ops = klp_find_ops(func->old_addr);
349 if (list_is_singular(&ops->func_stack)) {
350 WARN_ON(unregister_ftrace_function(&ops->fops));
351 WARN_ON(ftrace_set_filter_ip(&ops->fops, func->old_addr, 1, 0));
353 list_del_rcu(&func->stack_node);
354 list_del(&ops->node);
357 list_del_rcu(&func->stack_node);
360 func->state = KLP_DISABLED;
363 static int klp_enable_func(struct klp_func *func)
368 if (WARN_ON(!func->old_addr))
371 if (WARN_ON(func->state != KLP_DISABLED))
374 ops = klp_find_ops(func->old_addr);
376 ops = kzalloc(sizeof(*ops), GFP_KERNEL);
380 ops->fops.func = klp_ftrace_handler;
381 ops->fops.flags = FTRACE_OPS_FL_SAVE_REGS |
382 FTRACE_OPS_FL_DYNAMIC |
383 FTRACE_OPS_FL_IPMODIFY;
385 list_add(&ops->node, &klp_ops);
387 INIT_LIST_HEAD(&ops->func_stack);
388 list_add_rcu(&func->stack_node, &ops->func_stack);
390 ret = ftrace_set_filter_ip(&ops->fops, func->old_addr, 0, 0);
392 pr_err("failed to set ftrace filter for function '%s' (%d)\n",
393 func->old_name, ret);
397 ret = register_ftrace_function(&ops->fops);
399 pr_err("failed to register ftrace handler for function '%s' (%d)\n",
400 func->old_name, ret);
401 ftrace_set_filter_ip(&ops->fops, func->old_addr, 1, 0);
407 list_add_rcu(&func->stack_node, &ops->func_stack);
410 func->state = KLP_ENABLED;
415 list_del_rcu(&func->stack_node);
416 list_del(&ops->node);
421 static void klp_disable_object(struct klp_object *obj)
423 struct klp_func *func;
425 klp_for_each_func(obj, func)
426 if (func->state == KLP_ENABLED)
427 klp_disable_func(func);
429 obj->state = KLP_DISABLED;
432 static int klp_enable_object(struct klp_object *obj)
434 struct klp_func *func;
437 if (WARN_ON(obj->state != KLP_DISABLED))
440 if (WARN_ON(!klp_is_object_loaded(obj)))
443 klp_for_each_func(obj, func) {
444 ret = klp_enable_func(func);
446 klp_disable_object(obj);
450 obj->state = KLP_ENABLED;
455 static int __klp_disable_patch(struct klp_patch *patch)
457 struct klp_object *obj;
459 /* enforce stacking: only the last enabled patch can be disabled */
460 if (!list_is_last(&patch->list, &klp_patches) &&
461 list_next_entry(patch, list)->state == KLP_ENABLED)
464 pr_notice("disabling patch '%s'\n", patch->mod->name);
466 klp_for_each_object(patch, obj) {
467 if (obj->state == KLP_ENABLED)
468 klp_disable_object(obj);
471 patch->state = KLP_DISABLED;
477 * klp_disable_patch() - disables a registered patch
478 * @patch: The registered, enabled patch to be disabled
480 * Unregisters the patched functions from ftrace.
482 * Return: 0 on success, otherwise error
484 int klp_disable_patch(struct klp_patch *patch)
488 mutex_lock(&klp_mutex);
490 if (!klp_is_patch_registered(patch)) {
495 if (patch->state == KLP_DISABLED) {
500 ret = __klp_disable_patch(patch);
503 mutex_unlock(&klp_mutex);
506 EXPORT_SYMBOL_GPL(klp_disable_patch);
508 static int __klp_enable_patch(struct klp_patch *patch)
510 struct klp_object *obj;
513 if (WARN_ON(patch->state != KLP_DISABLED))
516 /* enforce stacking: only the first disabled patch can be enabled */
517 if (patch->list.prev != &klp_patches &&
518 list_prev_entry(patch, list)->state == KLP_DISABLED)
521 pr_notice_once("tainting kernel with TAINT_LIVEPATCH\n");
522 add_taint(TAINT_LIVEPATCH, LOCKDEP_STILL_OK);
524 pr_notice("enabling patch '%s'\n", patch->mod->name);
526 klp_for_each_object(patch, obj) {
527 if (!klp_is_object_loaded(obj))
530 ret = klp_enable_object(obj);
535 patch->state = KLP_ENABLED;
540 WARN_ON(__klp_disable_patch(patch));
545 * klp_enable_patch() - enables a registered patch
546 * @patch: The registered, disabled patch to be enabled
548 * Performs the needed symbol lookups and code relocations,
549 * then registers the patched functions with ftrace.
551 * Return: 0 on success, otherwise error
553 int klp_enable_patch(struct klp_patch *patch)
557 mutex_lock(&klp_mutex);
559 if (!klp_is_patch_registered(patch)) {
564 ret = __klp_enable_patch(patch);
567 mutex_unlock(&klp_mutex);
570 EXPORT_SYMBOL_GPL(klp_enable_patch);
575 * /sys/kernel/livepatch
576 * /sys/kernel/livepatch/<patch>
577 * /sys/kernel/livepatch/<patch>/enabled
578 * /sys/kernel/livepatch/<patch>/<object>
579 * /sys/kernel/livepatch/<patch>/<object>/<func>
582 static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
583 const char *buf, size_t count)
585 struct klp_patch *patch;
589 ret = kstrtoul(buf, 10, &val);
593 if (val != KLP_DISABLED && val != KLP_ENABLED)
596 patch = container_of(kobj, struct klp_patch, kobj);
598 mutex_lock(&klp_mutex);
600 if (val == patch->state) {
601 /* already in requested state */
606 if (val == KLP_ENABLED) {
607 ret = __klp_enable_patch(patch);
611 ret = __klp_disable_patch(patch);
616 mutex_unlock(&klp_mutex);
621 mutex_unlock(&klp_mutex);
625 static ssize_t enabled_show(struct kobject *kobj,
626 struct kobj_attribute *attr, char *buf)
628 struct klp_patch *patch;
630 patch = container_of(kobj, struct klp_patch, kobj);
631 return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->state);
634 static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled);
635 static struct attribute *klp_patch_attrs[] = {
636 &enabled_kobj_attr.attr,
640 static void klp_kobj_release_patch(struct kobject *kobj)
643 * Once we have a consistency model we'll need to module_put() the
644 * patch module here. See klp_register_patch() for more details.
648 static struct kobj_type klp_ktype_patch = {
649 .release = klp_kobj_release_patch,
650 .sysfs_ops = &kobj_sysfs_ops,
651 .default_attrs = klp_patch_attrs,
654 static void klp_kobj_release_object(struct kobject *kobj)
658 static struct kobj_type klp_ktype_object = {
659 .release = klp_kobj_release_object,
660 .sysfs_ops = &kobj_sysfs_ops,
663 static void klp_kobj_release_func(struct kobject *kobj)
667 static struct kobj_type klp_ktype_func = {
668 .release = klp_kobj_release_func,
669 .sysfs_ops = &kobj_sysfs_ops,
673 * Free all functions' kobjects in the array up to some limit. When limit is
674 * NULL, all kobjects are freed.
676 static void klp_free_funcs_limited(struct klp_object *obj,
677 struct klp_func *limit)
679 struct klp_func *func;
681 for (func = obj->funcs; func->old_name && func != limit; func++)
682 kobject_put(&func->kobj);
685 /* Clean up when a patched object is unloaded */
686 static void klp_free_object_loaded(struct klp_object *obj)
688 struct klp_func *func;
692 klp_for_each_func(obj, func)
697 * Free all objects' kobjects in the array up to some limit. When limit is
698 * NULL, all kobjects are freed.
700 static void klp_free_objects_limited(struct klp_patch *patch,
701 struct klp_object *limit)
703 struct klp_object *obj;
705 for (obj = patch->objs; obj->funcs && obj != limit; obj++) {
706 klp_free_funcs_limited(obj, NULL);
707 kobject_put(&obj->kobj);
711 static void klp_free_patch(struct klp_patch *patch)
713 klp_free_objects_limited(patch, NULL);
714 if (!list_empty(&patch->list))
715 list_del(&patch->list);
716 kobject_put(&patch->kobj);
719 static int klp_init_func(struct klp_object *obj, struct klp_func *func)
721 INIT_LIST_HEAD(&func->stack_node);
722 func->state = KLP_DISABLED;
724 return kobject_init_and_add(&func->kobj, &klp_ktype_func,
725 &obj->kobj, "%s", func->old_name);
728 /* parts of the initialization that is done only when the object is loaded */
729 static int klp_init_object_loaded(struct klp_patch *patch,
730 struct klp_object *obj)
732 struct klp_func *func;
736 ret = klp_write_object_relocations(patch->mod, obj);
741 klp_for_each_func(obj, func) {
742 ret = klp_find_verify_func_addr(obj, func);
750 static int klp_init_object(struct klp_patch *patch, struct klp_object *obj)
752 struct klp_func *func;
759 obj->state = KLP_DISABLED;
762 klp_find_object_module(obj);
764 name = klp_is_module(obj) ? obj->name : "vmlinux";
765 ret = kobject_init_and_add(&obj->kobj, &klp_ktype_object,
766 &patch->kobj, "%s", name);
770 klp_for_each_func(obj, func) {
771 ret = klp_init_func(obj, func);
776 if (klp_is_object_loaded(obj)) {
777 ret = klp_init_object_loaded(patch, obj);
785 klp_free_funcs_limited(obj, func);
786 kobject_put(&obj->kobj);
790 static int klp_init_patch(struct klp_patch *patch)
792 struct klp_object *obj;
798 mutex_lock(&klp_mutex);
800 patch->state = KLP_DISABLED;
802 ret = kobject_init_and_add(&patch->kobj, &klp_ktype_patch,
803 klp_root_kobj, "%s", patch->mod->name);
807 klp_for_each_object(patch, obj) {
808 ret = klp_init_object(patch, obj);
813 list_add_tail(&patch->list, &klp_patches);
815 mutex_unlock(&klp_mutex);
820 klp_free_objects_limited(patch, obj);
821 kobject_put(&patch->kobj);
823 mutex_unlock(&klp_mutex);
828 * klp_unregister_patch() - unregisters a patch
829 * @patch: Disabled patch to be unregistered
831 * Frees the data structures and removes the sysfs interface.
833 * Return: 0 on success, otherwise error
835 int klp_unregister_patch(struct klp_patch *patch)
839 mutex_lock(&klp_mutex);
841 if (!klp_is_patch_registered(patch)) {
846 if (patch->state == KLP_ENABLED) {
851 klp_free_patch(patch);
854 mutex_unlock(&klp_mutex);
857 EXPORT_SYMBOL_GPL(klp_unregister_patch);
860 * klp_register_patch() - registers a patch
861 * @patch: Patch to be registered
863 * Initializes the data structure associated with the patch and
864 * creates the sysfs interface.
866 * Return: 0 on success, otherwise error
868 int klp_register_patch(struct klp_patch *patch)
872 if (!klp_initialized())
875 if (!patch || !patch->mod)
879 * A reference is taken on the patch module to prevent it from being
880 * unloaded. Right now, we don't allow patch modules to unload since
881 * there is currently no method to determine if a thread is still
882 * running in the patched code contained in the patch module once
883 * the ftrace registration is successful.
885 if (!try_module_get(patch->mod))
888 ret = klp_init_patch(patch);
890 module_put(patch->mod);
894 EXPORT_SYMBOL_GPL(klp_register_patch);
896 static int klp_module_notify_coming(struct klp_patch *patch,
897 struct klp_object *obj)
899 struct module *pmod = patch->mod;
900 struct module *mod = obj->mod;
903 ret = klp_init_object_loaded(patch, obj);
905 pr_warn("failed to initialize patch '%s' for module '%s' (%d)\n",
906 pmod->name, mod->name, ret);
910 if (patch->state == KLP_DISABLED)
913 pr_notice("applying patch '%s' to loading module '%s'\n",
914 pmod->name, mod->name);
916 ret = klp_enable_object(obj);
918 pr_warn("failed to apply patch '%s' to module '%s' (%d)\n",
919 pmod->name, mod->name, ret);
923 static void klp_module_notify_going(struct klp_patch *patch,
924 struct klp_object *obj)
926 struct module *pmod = patch->mod;
927 struct module *mod = obj->mod;
929 if (patch->state == KLP_DISABLED)
932 pr_notice("reverting patch '%s' on unloading module '%s'\n",
933 pmod->name, mod->name);
935 klp_disable_object(obj);
938 klp_free_object_loaded(obj);
941 static int klp_module_notify(struct notifier_block *nb, unsigned long action,
945 struct module *mod = data;
946 struct klp_patch *patch;
947 struct klp_object *obj;
949 if (action != MODULE_STATE_COMING && action != MODULE_STATE_GOING)
952 mutex_lock(&klp_mutex);
955 * Each module has to know that the notifier has been called.
956 * We never know what module will get patched by a new patch.
958 if (action == MODULE_STATE_COMING)
959 mod->klp_alive = true;
960 else /* MODULE_STATE_GOING */
961 mod->klp_alive = false;
963 list_for_each_entry(patch, &klp_patches, list) {
964 klp_for_each_object(patch, obj) {
965 if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
968 if (action == MODULE_STATE_COMING) {
970 ret = klp_module_notify_coming(patch, obj);
973 pr_warn("patch '%s' is in an inconsistent state!\n",
976 } else /* MODULE_STATE_GOING */
977 klp_module_notify_going(patch, obj);
983 mutex_unlock(&klp_mutex);
988 static struct notifier_block klp_module_nb = {
989 .notifier_call = klp_module_notify,
990 .priority = INT_MIN+1, /* called late but before ftrace notifier */
993 static int __init klp_init(void)
997 ret = klp_check_compiler_support();
999 pr_info("Your compiler is too old; turning off.\n");
1003 ret = register_module_notifier(&klp_module_nb);
1007 klp_root_kobj = kobject_create_and_add("livepatch", kernel_kobj);
1008 if (!klp_root_kobj) {
1016 unregister_module_notifier(&klp_module_nb);
1020 module_init(klp_init);