kernel/kprobes.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  *  Kernel Probes (KProbes)
   4  *
   5  * Copyright (C) IBM Corporation, 2002, 2004
   6  *
   7  * 2002-Oct     Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
   8  *              Probes initial implementation (includes suggestions from
   9  *              Rusty Russell).
  10  * 2004-Aug     Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
  11  *              hlists and exceptions notifier as suggested by Andi Kleen.
  12  * 2004-July    Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
  13  *              interface to access function arguments.
  14  * 2004-Sep     Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
  15  *              exceptions notifier to be first on the priority list.
  16  * 2005-May     Hien Nguyen <hien@us.ibm.com>, Jim Keniston
  17  *              <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
  18  *              <prasanna@in.ibm.com> added function-return probes.
  19  */
  20
  21 #define pr_fmt(fmt) "kprobes: " fmt
  22
  23 #include <linux/kprobes.h>
  24 #include <linux/hash.h>
  25 #include <linux/init.h>
  26 #include <linux/slab.h>
  27 #include <linux/stddef.h>
  28 #include <linux/export.h>
  29 #include <linux/moduleloader.h>
  30 #include <linux/kallsyms.h>
  31 #include <linux/freezer.h>
  32 #include <linux/seq_file.h>
  33 #include <linux/debugfs.h>
  34 #include <linux/sysctl.h>
  35 #include <linux/kdebug.h>
  36 #include <linux/memory.h>
  37 #include <linux/ftrace.h>
  38 #include <linux/cpu.h>
  39 #include <linux/jump_label.h>
  40 #include <linux/static_call.h>
  41 #include <linux/perf_event.h>
  42
  43 #include <asm/sections.h>
  44 #include <asm/cacheflush.h>
  45 #include <asm/errno.h>
  46 #include <linux/uaccess.h>
  47
  48 #define KPROBE_HASH_BITS 6
  49 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
  50
  51 #if !defined(CONFIG_OPTPROBES) || !defined(CONFIG_SYSCTL)
  52 #define kprobe_sysctls_init() do { } while (0)
  53 #endif
  54
  55 static int kprobes_initialized;
  56 /* kprobe_table can be accessed by
  57  * - Normal hlist traversal and RCU add/del under 'kprobe_mutex' is held.
  58  * Or
  59  * - RCU hlist traversal under disabling preempt (breakpoint handlers)
  60  */
  61 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
  62
  63 /* NOTE: change this value only with 'kprobe_mutex' held */
  64 static bool kprobes_all_disarmed;
  65
  66 /* This protects 'kprobe_table' and 'optimizing_list' */
  67 static DEFINE_MUTEX(kprobe_mutex);
  68 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance);
  69
  70 kprobe_opcode_t * __weak kprobe_lookup_name(const char *name,
  71                                         unsigned int __unused)
  72 {
  73         return ((kprobe_opcode_t *)(kallsyms_lookup_name(name)));
  74 }
  75
  76 /*
  77  * Blacklist -- list of 'struct kprobe_blacklist_entry' to store info where
  78  * kprobes can not probe.
  79  */
  80 static LIST_HEAD(kprobe_blacklist);
  81
  82 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
  83 /*
  84  * 'kprobe::ainsn.insn' points to the copy of the instruction to be
  85  * single-stepped. x86_64, POWER4 and above have no-exec support and
  86  * stepping on the instruction on a vmalloced/kmalloced/data page
  87  * is a recipe for disaster
  88  */
  89 struct kprobe_insn_page {
  90         struct list_head list;
  91         kprobe_opcode_t *insns;         /* Page of instruction slots */
  92         struct kprobe_insn_cache *cache;
  93         int nused;
  94         int ngarbage;
  95         char slot_used[];
  96 };
  97
  98 #define KPROBE_INSN_PAGE_SIZE(slots)                    \
  99         (offsetof(struct kprobe_insn_page, slot_used) + \
 100          (sizeof(char) * (slots)))
 101
 102 static int slots_per_page(struct kprobe_insn_cache *c)
 103 {
 104         return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
 105 }
 106
 107 enum kprobe_slot_state {
 108         SLOT_CLEAN = 0,
 109         SLOT_DIRTY = 1,
 110         SLOT_USED = 2,
 111 };
 112
 113 void __weak *alloc_insn_page(void)
 114 {
 115         /*
 116          * Use module_alloc() so this page is within +/- 2GB of where the
 117          * kernel image and loaded module images reside. This is required
 118          * for most of the architectures.
 119          * (e.g. x86-64 needs this to handle the %rip-relative fixups.)
 120          */
 121         return module_alloc(PAGE_SIZE);
 122 }
 123
 124 static void free_insn_page(void *page)
 125 {
 126         module_memfree(page);
 127 }
 128
 129 struct kprobe_insn_cache kprobe_insn_slots = {
 130         .mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex),
 131         .alloc = alloc_insn_page,
 132         .free = free_insn_page,
 133         .sym = KPROBE_INSN_PAGE_SYM,
 134         .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
 135         .insn_size = MAX_INSN_SIZE,
 136         .nr_garbage = 0,
 137 };
 138 static int collect_garbage_slots(struct kprobe_insn_cache *c);
 139
 140 /**
 141  * __get_insn_slot() - Find a slot on an executable page for an instruction.
 142  * We allocate an executable page if there's no room on existing ones.
 143  */
 144 kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c)
 145 {
 146         struct kprobe_insn_page *kip;
 147         kprobe_opcode_t *slot = NULL;
 148
 149         /* Since the slot array is not protected by rcu, we need a mutex */
 150         mutex_lock(&c->mutex);
 151  retry:
 152         rcu_read_lock();
 153         list_for_each_entry_rcu(kip, &c->pages, list) {
 154                 if (kip->nused < slots_per_page(c)) {
 155                         int i;
 156
 157                         for (i = 0; i < slots_per_page(c); i++) {
 158                                 if (kip->slot_used[i] == SLOT_CLEAN) {
 159                                         kip->slot_used[i] = SLOT_USED;
 160                                         kip->nused++;
 161                                         slot = kip->insns + (i * c->insn_size);
 162                                         rcu_read_unlock();
 163                                         goto out;
 164                                 }
 165                         }
 166                         /* kip->nused is broken. Fix it. */
 167                         kip->nused = slots_per_page(c);
 168                         WARN_ON(1);
 169                 }
 170         }
 171         rcu_read_unlock();
 172
 173         /* If there are any garbage slots, collect it and try again. */
 174         if (c->nr_garbage && collect_garbage_slots(c) == 0)
 175                 goto retry;
 176
 177         /* All out of space.  Need to allocate a new page. */
 178         kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
 179         if (!kip)
 180                 goto out;
 181
 182         kip->insns = c->alloc();
 183         if (!kip->insns) {
 184                 kfree(kip);
 185                 goto out;
 186         }
 187         INIT_LIST_HEAD(&kip->list);
 188         memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
 189         kip->slot_used[0] = SLOT_USED;
 190         kip->nused = 1;
 191         kip->ngarbage = 0;
 192         kip->cache = c;
 193         list_add_rcu(&kip->list, &c->pages);
 194         slot = kip->insns;
 195
 196         /* Record the perf ksymbol register event after adding the page */
 197         perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, (unsigned long)kip->insns,
 198                            PAGE_SIZE, false, c->sym);
 199 out:
 200         mutex_unlock(&c->mutex);
 201         return slot;
 202 }
 203
 204 /* Return true if all garbages are collected, otherwise false. */
 205 static bool collect_one_slot(struct kprobe_insn_page *kip, int idx)
 206 {
 207         kip->slot_used[idx] = SLOT_CLEAN;
 208         kip->nused--;
 209         if (kip->nused == 0) {
 210                 /*
 211                  * Page is no longer in use.  Free it unless
 212                  * it's the last one.  We keep the last one
 213                  * so as not to have to set it up again the
 214                  * next time somebody inserts a probe.
 215                  */
 216                 if (!list_is_singular(&kip->list)) {
 217                         /*
 218                          * Record perf ksymbol unregister event before removing
 219                          * the page.
 220                          */
 221                         perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
 222                                            (unsigned long)kip->insns, PAGE_SIZE, true,
 223                                            kip->cache->sym);
 224                         list_del_rcu(&kip->list);
 225                         synchronize_rcu();
 226                         kip->cache->free(kip->insns);
 227                         kfree(kip);
 228                 }
 229                 return true;
 230         }
 231         return false;
 232 }
 233
 234 static int collect_garbage_slots(struct kprobe_insn_cache *c)
 235 {
 236         struct kprobe_insn_page *kip, *next;
 237
 238         /* Ensure no-one is interrupted on the garbages */
 239         synchronize_rcu();
 240
 241         list_for_each_entry_safe(kip, next, &c->pages, list) {
 242                 int i;
 243
 244                 if (kip->ngarbage == 0)
 245                         continue;
 246                 kip->ngarbage = 0;      /* we will collect all garbages */
 247                 for (i = 0; i < slots_per_page(c); i++) {
 248                         if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i))
 249                                 break;
 250                 }
 251         }
 252         c->nr_garbage = 0;
 253         return 0;
 254 }
 255
 256 void __free_insn_slot(struct kprobe_insn_cache *c,
 257                       kprobe_opcode_t *slot, int dirty)
 258 {
 259         struct kprobe_insn_page *kip;
 260         long idx;
 261
 262         mutex_lock(&c->mutex);
 263         rcu_read_lock();
 264         list_for_each_entry_rcu(kip, &c->pages, list) {
 265                 idx = ((long)slot - (long)kip->insns) /
 266                         (c->insn_size * sizeof(kprobe_opcode_t));
 267                 if (idx >= 0 && idx < slots_per_page(c))
 268                         goto out;
 269         }
 270         /* Could not find this slot. */
 271         WARN_ON(1);
 272         kip = NULL;
 273 out:
 274         rcu_read_unlock();
 275         /* Mark and sweep: this may sleep */
 276         if (kip) {
 277                 /* Check double free */
 278                 WARN_ON(kip->slot_used[idx] != SLOT_USED);
 279                 if (dirty) {
 280                         kip->slot_used[idx] = SLOT_DIRTY;
 281                         kip->ngarbage++;
 282                         if (++c->nr_garbage > slots_per_page(c))
 283                                 collect_garbage_slots(c);
 284                 } else {
 285                         collect_one_slot(kip, idx);
 286                 }
 287         }
 288         mutex_unlock(&c->mutex);
 289 }
 290
 291 /*
 292  * Check given address is on the page of kprobe instruction slots.
 293  * This will be used for checking whether the address on a stack
 294  * is on a text area or not.
 295  */
 296 bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr)
 297 {
 298         struct kprobe_insn_page *kip;
 299         bool ret = false;
 300
 301         rcu_read_lock();
 302         list_for_each_entry_rcu(kip, &c->pages, list) {
 303                 if (addr >= (unsigned long)kip->insns &&
 304                     addr < (unsigned long)kip->insns + PAGE_SIZE) {
 305                         ret = true;
 306                         break;
 307                 }
 308         }
 309         rcu_read_unlock();
 310
 311         return ret;
 312 }
 313
 314 int kprobe_cache_get_kallsym(struct kprobe_insn_cache *c, unsigned int *symnum,
 315                              unsigned long *value, char *type, char *sym)
 316 {
 317         struct kprobe_insn_page *kip;
 318         int ret = -ERANGE;
 319
 320         rcu_read_lock();
 321         list_for_each_entry_rcu(kip, &c->pages, list) {
 322                 if ((*symnum)--)
 323                         continue;
 324                 strscpy(sym, c->sym, KSYM_NAME_LEN);
 325                 *type = 't';
 326                 *value = (unsigned long)kip->insns;
 327                 ret = 0;
 328                 break;
 329         }
 330         rcu_read_unlock();
 331
 332         return ret;
 333 }
 334
 335 #ifdef CONFIG_OPTPROBES
 336 void __weak *alloc_optinsn_page(void)
 337 {
 338         return alloc_insn_page();
 339 }
 340
 341 void __weak free_optinsn_page(void *page)
 342 {
 343         free_insn_page(page);
 344 }
 345
 346 /* For optimized_kprobe buffer */
 347 struct kprobe_insn_cache kprobe_optinsn_slots = {
 348         .mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex),
 349         .alloc = alloc_optinsn_page,
 350         .free = free_optinsn_page,
 351         .sym = KPROBE_OPTINSN_PAGE_SYM,
 352         .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
 353         /* .insn_size is initialized later */
 354         .nr_garbage = 0,
 355 };
 356 #endif
 357 #endif
 358
 359 /* We have preemption disabled.. so it is safe to use __ versions */
 360 static inline void set_kprobe_instance(struct kprobe *kp)
 361 {
 362         __this_cpu_write(kprobe_instance, kp);
 363 }
 364
 365 static inline void reset_kprobe_instance(void)
 366 {
 367         __this_cpu_write(kprobe_instance, NULL);
 368 }
 369
 370 /*
 371  * This routine is called either:
 372  *      - under the 'kprobe_mutex' - during kprobe_[un]register().
 373  *                              OR
 374  *      - with preemption disabled - from architecture specific code.
 375  */
 376 struct kprobe *get_kprobe(void *addr)
 377 {
 378         struct hlist_head *head;
 379         struct kprobe *p;
 380
 381         head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
 382         hlist_for_each_entry_rcu(p, head, hlist,
 383                                  lockdep_is_held(&kprobe_mutex)) {
 384                 if (p->addr == addr)
 385                         return p;
 386         }
 387
 388         return NULL;
 389 }
 390 NOKPROBE_SYMBOL(get_kprobe);
 391
 392 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
 393
 394 /* Return true if 'p' is an aggregator */
 395 static inline bool kprobe_aggrprobe(struct kprobe *p)
 396 {
 397         return p->pre_handler == aggr_pre_handler;
 398 }
 399
 400 /* Return true if 'p' is unused */
 401 static inline bool kprobe_unused(struct kprobe *p)
 402 {
 403         return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
 404                list_empty(&p->list);
 405 }
 406
 407 /* Keep all fields in the kprobe consistent. */
 408 static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)
 409 {
 410         memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
 411         memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
 412 }
 413
 414 #ifdef CONFIG_OPTPROBES
 415 /* NOTE: This is protected by 'kprobe_mutex'. */
 416 static bool kprobes_allow_optimization;
 417
 418 /*
 419  * Call all 'kprobe::pre_handler' on the list, but ignores its return value.
 420  * This must be called from arch-dep optimized caller.
 421  */
 422 void opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
 423 {
 424         struct kprobe *kp;
 425
 426         list_for_each_entry_rcu(kp, &p->list, list) {
 427                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
 428                         set_kprobe_instance(kp);
 429                         kp->pre_handler(kp, regs);
 430                 }
 431                 reset_kprobe_instance();
 432         }
 433 }
 434 NOKPROBE_SYMBOL(opt_pre_handler);
 435
 436 /* Free optimized instructions and optimized_kprobe */
 437 static void free_aggr_kprobe(struct kprobe *p)
 438 {
 439         struct optimized_kprobe *op;
 440
 441         op = container_of(p, struct optimized_kprobe, kp);
 442         arch_remove_optimized_kprobe(op);
 443         arch_remove_kprobe(p);
 444         kfree(op);
 445 }
 446
 447 /* Return true if the kprobe is ready for optimization. */
 448 static inline int kprobe_optready(struct kprobe *p)
 449 {
 450         struct optimized_kprobe *op;
 451
 452         if (kprobe_aggrprobe(p)) {
 453                 op = container_of(p, struct optimized_kprobe, kp);
 454                 return arch_prepared_optinsn(&op->optinsn);
 455         }
 456
 457         return 0;
 458 }
 459
 460 /* Return true if the kprobe is disarmed. Note: p must be on hash list */
 461 bool kprobe_disarmed(struct kprobe *p)
 462 {
 463         struct optimized_kprobe *op;
 464
 465         /* If kprobe is not aggr/opt probe, just return kprobe is disabled */
 466         if (!kprobe_aggrprobe(p))
 467                 return kprobe_disabled(p);
 468
 469         op = container_of(p, struct optimized_kprobe, kp);
 470
 471         return kprobe_disabled(p) && list_empty(&op->list);
 472 }
 473
 474 /* Return true if the probe is queued on (un)optimizing lists */
 475 static bool kprobe_queued(struct kprobe *p)
 476 {
 477         struct optimized_kprobe *op;
 478
 479         if (kprobe_aggrprobe(p)) {
 480                 op = container_of(p, struct optimized_kprobe, kp);
 481                 if (!list_empty(&op->list))
 482                         return true;
 483         }
 484         return false;
 485 }
 486
 487 /*
 488  * Return an optimized kprobe whose optimizing code replaces
 489  * instructions including 'addr' (exclude breakpoint).
 490  */
 491 static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr)
 492 {
 493         int i;
 494         struct kprobe *p = NULL;
 495         struct optimized_kprobe *op;
 496
 497         /* Don't check i == 0, since that is a breakpoint case. */
 498         for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++)
 499                 p = get_kprobe(addr - i);
 500
 501         if (p && kprobe_optready(p)) {
 502                 op = container_of(p, struct optimized_kprobe, kp);
 503                 if (arch_within_optimized_kprobe(op, addr))
 504                         return p;
 505         }
 506
 507         return NULL;
 508 }
 509
 510 /* Optimization staging list, protected by 'kprobe_mutex' */
 511 static LIST_HEAD(optimizing_list);
 512 static LIST_HEAD(unoptimizing_list);
 513 static LIST_HEAD(freeing_list);
 514
 515 static void kprobe_optimizer(struct work_struct *work);
 516 static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
 517 #define OPTIMIZE_DELAY 5
 518
 519 /*
 520  * Optimize (replace a breakpoint with a jump) kprobes listed on
 521  * 'optimizing_list'.
 522  */
 523 static void do_optimize_kprobes(void)
 524 {
 525         lockdep_assert_held(&text_mutex);
 526         /*
 527          * The optimization/unoptimization refers 'online_cpus' via
 528          * stop_machine() and cpu-hotplug modifies the 'online_cpus'.
 529          * And same time, 'text_mutex' will be held in cpu-hotplug and here.
 530          * This combination can cause a deadlock (cpu-hotplug tries to lock
 531          * 'text_mutex' but stop_machine() can not be done because
 532          * the 'online_cpus' has been changed)
 533          * To avoid this deadlock, caller must have locked cpu-hotplug
 534          * for preventing cpu-hotplug outside of 'text_mutex' locking.
 535          */
 536         lockdep_assert_cpus_held();
 537
 538         /* Optimization never be done when disarmed */
 539         if (kprobes_all_disarmed || !kprobes_allow_optimization ||
 540             list_empty(&optimizing_list))
 541                 return;
 542
 543         arch_optimize_kprobes(&optimizing_list);
 544 }
 545
 546 /*
 547  * Unoptimize (replace a jump with a breakpoint and remove the breakpoint
 548  * if need) kprobes listed on 'unoptimizing_list'.
 549  */
 550 static void do_unoptimize_kprobes(void)
 551 {
 552         struct optimized_kprobe *op, *tmp;
 553
 554         lockdep_assert_held(&text_mutex);
 555         /* See comment in do_optimize_kprobes() */
 556         lockdep_assert_cpus_held();
 557
 558         if (!list_empty(&unoptimizing_list))
 559                 arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list);
 560
 561         /* Loop on 'freeing_list' for disarming and removing from kprobe hash list */
 562         list_for_each_entry_safe(op, tmp, &freeing_list, list) {
 563                 /* Switching from detour code to origin */
 564                 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 565                 /* Disarm probes if marked disabled and not gone */
 566                 if (kprobe_disabled(&op->kp) && !kprobe_gone(&op->kp))
 567                         arch_disarm_kprobe(&op->kp);
 568                 if (kprobe_unused(&op->kp)) {
 569                         /*
 570                          * Remove unused probes from hash list. After waiting
 571                          * for synchronization, these probes are reclaimed.
 572                          * (reclaiming is done by do_free_cleaned_kprobes().)
 573                          */
 574                         hlist_del_rcu(&op->kp.hlist);
 575                 } else
 576                         list_del_init(&op->list);
 577         }
 578 }
 579
 580 /* Reclaim all kprobes on the 'freeing_list' */
 581 static void do_free_cleaned_kprobes(void)
 582 {
 583         struct optimized_kprobe *op, *tmp;
 584
 585         list_for_each_entry_safe(op, tmp, &freeing_list, list) {
 586                 list_del_init(&op->list);
 587                 if (WARN_ON_ONCE(!kprobe_unused(&op->kp))) {
 588                         /*
 589                          * This must not happen, but if there is a kprobe
 590                          * still in use, keep it on kprobes hash list.
 591                          */
 592                         continue;
 593                 }
 594                 free_aggr_kprobe(&op->kp);
 595         }
 596 }
 597
 598 /* Start optimizer after OPTIMIZE_DELAY passed */
 599 static void kick_kprobe_optimizer(void)
 600 {
 601         schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
 602 }
 603
 604 /* Kprobe jump optimizer */
 605 static void kprobe_optimizer(struct work_struct *work)
 606 {
 607         mutex_lock(&kprobe_mutex);
 608         cpus_read_lock();
 609         mutex_lock(&text_mutex);
 610
 611         /*
 612          * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
 613          * kprobes before waiting for quiesence period.
 614          */
 615         do_unoptimize_kprobes();
 616
 617         /*
 618          * Step 2: Wait for quiesence period to ensure all potentially
 619          * preempted tasks to have normally scheduled. Because optprobe
 620          * may modify multiple instructions, there is a chance that Nth
 621          * instruction is preempted. In that case, such tasks can return
 622          * to 2nd-Nth byte of jump instruction. This wait is for avoiding it.
 623          * Note that on non-preemptive kernel, this is transparently converted
 624          * to synchronoze_sched() to wait for all interrupts to have completed.
 625          */
 626         synchronize_rcu_tasks();
 627
 628         /* Step 3: Optimize kprobes after quiesence period */
 629         do_optimize_kprobes();
 630
 631         /* Step 4: Free cleaned kprobes after quiesence period */
 632         do_free_cleaned_kprobes();
 633
 634         mutex_unlock(&text_mutex);
 635         cpus_read_unlock();
 636
 637         /* Step 5: Kick optimizer again if needed */
 638         if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list))
 639                 kick_kprobe_optimizer();
 640
 641         mutex_unlock(&kprobe_mutex);
 642 }
 643
 644 /* Wait for completing optimization and unoptimization */
 645 void wait_for_kprobe_optimizer(void)
 646 {
 647         mutex_lock(&kprobe_mutex);
 648
 649         while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) {
 650                 mutex_unlock(&kprobe_mutex);
 651
 652                 /* This will also make 'optimizing_work' execute immmediately */
 653                 flush_delayed_work(&optimizing_work);
 654                 /* 'optimizing_work' might not have been queued yet, relax */
 655                 cpu_relax();
 656
 657                 mutex_lock(&kprobe_mutex);
 658         }
 659
 660         mutex_unlock(&kprobe_mutex);
 661 }
 662
 663 bool optprobe_queued_unopt(struct optimized_kprobe *op)
 664 {
 665         struct optimized_kprobe *_op;
 666
 667         list_for_each_entry(_op, &unoptimizing_list, list) {
 668                 if (op == _op)
 669                         return true;
 670         }
 671
 672         return false;
 673 }
 674
 675 /* Optimize kprobe if p is ready to be optimized */
 676 static void optimize_kprobe(struct kprobe *p)
 677 {
 678         struct optimized_kprobe *op;
 679
 680         /* Check if the kprobe is disabled or not ready for optimization. */
 681         if (!kprobe_optready(p) || !kprobes_allow_optimization ||
 682             (kprobe_disabled(p) || kprobes_all_disarmed))
 683                 return;
 684
 685         /* kprobes with 'post_handler' can not be optimized */
 686         if (p->post_handler)
 687                 return;
 688
 689         op = container_of(p, struct optimized_kprobe, kp);
 690
 691         /* Check there is no other kprobes at the optimized instructions */
 692         if (arch_check_optimized_kprobe(op) < 0)
 693                 return;
 694
 695         /* Check if it is already optimized. */
 696         if (op->kp.flags & KPROBE_FLAG_OPTIMIZED) {
 697                 if (optprobe_queued_unopt(op)) {
 698                         /* This is under unoptimizing. Just dequeue the probe */
 699                         list_del_init(&op->list);
 700                 }
 701                 return;
 702         }
 703         op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
 704
 705         /*
 706          * On the 'unoptimizing_list' and 'optimizing_list',
 707          * 'op' must have OPTIMIZED flag
 708          */
 709         if (WARN_ON_ONCE(!list_empty(&op->list)))
 710                 return;
 711
 712         list_add(&op->list, &optimizing_list);
 713         kick_kprobe_optimizer();
 714 }
 715
 716 /* Short cut to direct unoptimizing */
 717 static void force_unoptimize_kprobe(struct optimized_kprobe *op)
 718 {
 719         lockdep_assert_cpus_held();
 720         arch_unoptimize_kprobe(op);
 721         op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 722 }
 723
 724 /* Unoptimize a kprobe if p is optimized */
 725 static void unoptimize_kprobe(struct kprobe *p, bool force)
 726 {
 727         struct optimized_kprobe *op;
 728
 729         if (!kprobe_aggrprobe(p) || kprobe_disarmed(p))
 730                 return; /* This is not an optprobe nor optimized */
 731
 732         op = container_of(p, struct optimized_kprobe, kp);
 733         if (!kprobe_optimized(p))
 734                 return;
 735
 736         if (!list_empty(&op->list)) {
 737                 if (optprobe_queued_unopt(op)) {
 738                         /* Queued in unoptimizing queue */
 739                         if (force) {
 740                                 /*
 741                                  * Forcibly unoptimize the kprobe here, and queue it
 742                                  * in the freeing list for release afterwards.
 743                                  */
 744                                 force_unoptimize_kprobe(op);
 745                                 list_move(&op->list, &freeing_list);
 746                         }
 747                 } else {
 748                         /* Dequeue from the optimizing queue */
 749                         list_del_init(&op->list);
 750                         op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 751                 }
 752                 return;
 753         }
 754
 755         /* Optimized kprobe case */
 756         if (force) {
 757                 /* Forcibly update the code: this is a special case */
 758                 force_unoptimize_kprobe(op);
 759         } else {
 760                 list_add(&op->list, &unoptimizing_list);
 761                 kick_kprobe_optimizer();
 762         }
 763 }
 764
 765 /* Cancel unoptimizing for reusing */
 766 static int reuse_unused_kprobe(struct kprobe *ap)
 767 {
 768         struct optimized_kprobe *op;
 769
 770         /*
 771          * Unused kprobe MUST be on the way of delayed unoptimizing (means
 772          * there is still a relative jump) and disabled.
 773          */
 774         op = container_of(ap, struct optimized_kprobe, kp);
 775         WARN_ON_ONCE(list_empty(&op->list));
 776         /* Enable the probe again */
 777         ap->flags &= ~KPROBE_FLAG_DISABLED;
 778         /* Optimize it again. (remove from 'op->list') */
 779         if (!kprobe_optready(ap))
 780                 return -EINVAL;
 781
 782         optimize_kprobe(ap);
 783         return 0;
 784 }
 785
 786 /* Remove optimized instructions */
 787 static void kill_optimized_kprobe(struct kprobe *p)
 788 {
 789         struct optimized_kprobe *op;
 790
 791         op = container_of(p, struct optimized_kprobe, kp);
 792         if (!list_empty(&op->list))
 793                 /* Dequeue from the (un)optimization queue */
 794                 list_del_init(&op->list);
 795         op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 796
 797         if (kprobe_unused(p)) {
 798                 /*
 799                  * Unused kprobe is on unoptimizing or freeing list. We move it
 800                  * to freeing_list and let the kprobe_optimizer() remove it from
 801                  * the kprobe hash list and free it.
 802                  */
 803                 if (optprobe_queued_unopt(op))
 804                         list_move(&op->list, &freeing_list);
 805         }
 806
 807         /* Don't touch the code, because it is already freed. */
 808         arch_remove_optimized_kprobe(op);
 809 }
 810
 811 static inline
 812 void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p)
 813 {
 814         if (!kprobe_ftrace(p))
 815                 arch_prepare_optimized_kprobe(op, p);
 816 }
 817
 818 /* Try to prepare optimized instructions */
 819 static void prepare_optimized_kprobe(struct kprobe *p)
 820 {
 821         struct optimized_kprobe *op;
 822
 823         op = container_of(p, struct optimized_kprobe, kp);
 824         __prepare_optimized_kprobe(op, p);
 825 }
 826
 827 /* Allocate new optimized_kprobe and try to prepare optimized instructions. */
 828 static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
 829 {
 830         struct optimized_kprobe *op;
 831
 832         op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
 833         if (!op)
 834                 return NULL;
 835
 836         INIT_LIST_HEAD(&op->list);
 837         op->kp.addr = p->addr;
 838         __prepare_optimized_kprobe(op, p);
 839
 840         return &op->kp;
 841 }
 842
 843 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
 844
 845 /*
 846  * Prepare an optimized_kprobe and optimize it.
 847  * NOTE: 'p' must be a normal registered kprobe.
 848  */
 849 static void try_to_optimize_kprobe(struct kprobe *p)
 850 {
 851         struct kprobe *ap;
 852         struct optimized_kprobe *op;
 853
 854         /* Impossible to optimize ftrace-based kprobe. */
 855         if (kprobe_ftrace(p))
 856                 return;
 857
 858         /* For preparing optimization, jump_label_text_reserved() is called. */
 859         cpus_read_lock();
 860         jump_label_lock();
 861         mutex_lock(&text_mutex);
 862
 863         ap = alloc_aggr_kprobe(p);
 864         if (!ap)
 865                 goto out;
 866
 867         op = container_of(ap, struct optimized_kprobe, kp);
 868         if (!arch_prepared_optinsn(&op->optinsn)) {
 869                 /* If failed to setup optimizing, fallback to kprobe. */
 870                 arch_remove_optimized_kprobe(op);
 871                 kfree(op);
 872                 goto out;
 873         }
 874
 875         init_aggr_kprobe(ap, p);
 876         optimize_kprobe(ap);    /* This just kicks optimizer thread. */
 877
 878 out:
 879         mutex_unlock(&text_mutex);
 880         jump_label_unlock();
 881         cpus_read_unlock();
 882 }
 883
 884 static void optimize_all_kprobes(void)
 885 {
 886         struct hlist_head *head;
 887         struct kprobe *p;
 888         unsigned int i;
 889
 890         mutex_lock(&kprobe_mutex);
 891         /* If optimization is already allowed, just return. */
 892         if (kprobes_allow_optimization)
 893                 goto out;
 894
 895         cpus_read_lock();
 896         kprobes_allow_optimization = true;
 897         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
 898                 head = &kprobe_table[i];
 899                 hlist_for_each_entry(p, head, hlist)
 900                         if (!kprobe_disabled(p))
 901                                 optimize_kprobe(p);
 902         }
 903         cpus_read_unlock();
 904         pr_info("kprobe jump-optimization is enabled. All kprobes are optimized if possible.\n");
 905 out:
 906         mutex_unlock(&kprobe_mutex);
 907 }
 908
 909 #ifdef CONFIG_SYSCTL
 910 static void unoptimize_all_kprobes(void)
 911 {
 912         struct hlist_head *head;
 913         struct kprobe *p;
 914         unsigned int i;
 915
 916         mutex_lock(&kprobe_mutex);
 917         /* If optimization is already prohibited, just return. */
 918         if (!kprobes_allow_optimization) {
 919                 mutex_unlock(&kprobe_mutex);
 920                 return;
 921         }
 922
 923         cpus_read_lock();
 924         kprobes_allow_optimization = false;
 925         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
 926                 head = &kprobe_table[i];
 927                 hlist_for_each_entry(p, head, hlist) {
 928                         if (!kprobe_disabled(p))
 929                                 unoptimize_kprobe(p, false);
 930                 }
 931         }
 932         cpus_read_unlock();
 933         mutex_unlock(&kprobe_mutex);
 934
 935         /* Wait for unoptimizing completion. */
 936         wait_for_kprobe_optimizer();
 937         pr_info("kprobe jump-optimization is disabled. All kprobes are based on software breakpoint.\n");
 938 }
 939
 940 static DEFINE_MUTEX(kprobe_sysctl_mutex);
 941 static int sysctl_kprobes_optimization;
 942 static int proc_kprobes_optimization_handler(struct ctl_table *table,
 943                                              int write, void *buffer,
 944                                              size_t *length, loff_t *ppos)
 945 {
 946         int ret;
 947
 948         mutex_lock(&kprobe_sysctl_mutex);
 949         sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
 950         ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
 951
 952         if (sysctl_kprobes_optimization)
 953                 optimize_all_kprobes();
 954         else
 955                 unoptimize_all_kprobes();
 956         mutex_unlock(&kprobe_sysctl_mutex);
 957
 958         return ret;
 959 }
 960
 961 static struct ctl_table kprobe_sysctls[] = {
 962         {
 963                 .procname       = "kprobes-optimization",
 964                 .data           = &sysctl_kprobes_optimization,
 965                 .maxlen         = sizeof(int),
 966                 .mode           = 0644,
 967                 .proc_handler   = proc_kprobes_optimization_handler,
 968                 .extra1         = SYSCTL_ZERO,
 969                 .extra2         = SYSCTL_ONE,
 970         },
 971         {}
 972 };
 973
 974 static void __init kprobe_sysctls_init(void)
 975 {
 976         register_sysctl_init("debug", kprobe_sysctls);
 977 }
 978 #endif /* CONFIG_SYSCTL */
 979
 980 /* Put a breakpoint for a probe. */
 981 static void __arm_kprobe(struct kprobe *p)
 982 {
 983         struct kprobe *_p;
 984
 985         lockdep_assert_held(&text_mutex);
 986
 987         /* Find the overlapping optimized kprobes. */
 988         _p = get_optimized_kprobe(p->addr);
 989         if (unlikely(_p))
 990                 /* Fallback to unoptimized kprobe */
 991                 unoptimize_kprobe(_p, true);
 992
 993         arch_arm_kprobe(p);
 994         optimize_kprobe(p);     /* Try to optimize (add kprobe to a list) */
 995 }
 996
 997 /* Remove the breakpoint of a probe. */
 998 static void __disarm_kprobe(struct kprobe *p, bool reopt)
 999 {
1000         struct kprobe *_p;
1001
1002         lockdep_assert_held(&text_mutex);
1003
1004         /* Try to unoptimize */
1005         unoptimize_kprobe(p, kprobes_all_disarmed);
1006
1007         if (!kprobe_queued(p)) {
1008                 arch_disarm_kprobe(p);
1009                 /* If another kprobe was blocked, re-optimize it. */
1010                 _p = get_optimized_kprobe(p->addr);
1011                 if (unlikely(_p) && reopt)
1012                         optimize_kprobe(_p);
1013         }
1014         /*
1015          * TODO: Since unoptimization and real disarming will be done by
1016          * the worker thread, we can not check whether another probe are
1017          * unoptimized because of this probe here. It should be re-optimized
1018          * by the worker thread.
1019          */
1020 }
1021
1022 #else /* !CONFIG_OPTPROBES */
1023
1024 #define optimize_kprobe(p)                      do {} while (0)
1025 #define unoptimize_kprobe(p, f)                 do {} while (0)
1026 #define kill_optimized_kprobe(p)                do {} while (0)
1027 #define prepare_optimized_kprobe(p)             do {} while (0)
1028 #define try_to_optimize_kprobe(p)               do {} while (0)
1029 #define __arm_kprobe(p)                         arch_arm_kprobe(p)
1030 #define __disarm_kprobe(p, o)                   arch_disarm_kprobe(p)
1031 #define kprobe_disarmed(p)                      kprobe_disabled(p)
1032 #define wait_for_kprobe_optimizer()             do {} while (0)
1033
1034 static int reuse_unused_kprobe(struct kprobe *ap)
1035 {
1036         /*
1037          * If the optimized kprobe is NOT supported, the aggr kprobe is
1038          * released at the same time that the last aggregated kprobe is
1039          * unregistered.
1040          * Thus there should be no chance to reuse unused kprobe.
1041          */
1042         WARN_ON_ONCE(1);
1043         return -EINVAL;
1044 }
1045
1046 static void free_aggr_kprobe(struct kprobe *p)
1047 {
1048         arch_remove_kprobe(p);
1049         kfree(p);
1050 }
1051
1052 static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
1053 {
1054         return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
1055 }
1056 #endif /* CONFIG_OPTPROBES */
1057
1058 #ifdef CONFIG_KPROBES_ON_FTRACE
1059 static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
1060         .func = kprobe_ftrace_handler,
1061         .flags = FTRACE_OPS_FL_SAVE_REGS,
1062 };
1063
1064 static struct ftrace_ops kprobe_ipmodify_ops __read_mostly = {
1065         .func = kprobe_ftrace_handler,
1066         .flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY,
1067 };
1068
1069 static int kprobe_ipmodify_enabled;
1070 static int kprobe_ftrace_enabled;
1071
1072 static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
1073                                int *cnt)
1074 {
1075         int ret;
1076
1077         lockdep_assert_held(&kprobe_mutex);
1078
1079         ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0);
1080         if (WARN_ONCE(ret < 0, "Failed to arm kprobe-ftrace at %pS (error %d)\n", p->addr, ret))
1081                 return ret;
1082
1083         if (*cnt == 0) {
1084                 ret = register_ftrace_function(ops);
1085                 if (WARN(ret < 0, "Failed to register kprobe-ftrace (error %d)\n", ret))
1086                         goto err_ftrace;
1087         }
1088
1089         (*cnt)++;
1090         return ret;
1091
1092 err_ftrace:
1093         /*
1094          * At this point, sinec ops is not registered, we should be sefe from
1095          * registering empty filter.
1096          */
1097         ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1098         return ret;
1099 }
1100
1101 static int arm_kprobe_ftrace(struct kprobe *p)
1102 {
1103         bool ipmodify = (p->post_handler != NULL);
1104
1105         return __arm_kprobe_ftrace(p,
1106                 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1107                 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1108 }
1109
1110 static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
1111                                   int *cnt)
1112 {
1113         int ret;
1114
1115         lockdep_assert_held(&kprobe_mutex);
1116
1117         if (*cnt == 1) {
1118                 ret = unregister_ftrace_function(ops);
1119                 if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (error %d)\n", ret))
1120                         return ret;
1121         }
1122
1123         (*cnt)--;
1124
1125         ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1126         WARN_ONCE(ret < 0, "Failed to disarm kprobe-ftrace at %pS (error %d)\n",
1127                   p->addr, ret);
1128         return ret;
1129 }
1130
1131 static int disarm_kprobe_ftrace(struct kprobe *p)
1132 {
1133         bool ipmodify = (p->post_handler != NULL);
1134
1135         return __disarm_kprobe_ftrace(p,
1136                 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1137                 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1138 }
1139 #else   /* !CONFIG_KPROBES_ON_FTRACE */
1140 static inline int arm_kprobe_ftrace(struct kprobe *p)
1141 {
1142         return -ENODEV;
1143 }
1144
1145 static inline int disarm_kprobe_ftrace(struct kprobe *p)
1146 {
1147         return -ENODEV;
1148 }
1149 #endif
1150
1151 static int prepare_kprobe(struct kprobe *p)
1152 {
1153         /* Must ensure p->addr is really on ftrace */
1154         if (kprobe_ftrace(p))
1155                 return arch_prepare_kprobe_ftrace(p);
1156
1157         return arch_prepare_kprobe(p);
1158 }
1159
1160 static int arm_kprobe(struct kprobe *kp)
1161 {
1162         if (unlikely(kprobe_ftrace(kp)))
1163                 return arm_kprobe_ftrace(kp);
1164
1165         cpus_read_lock();
1166         mutex_lock(&text_mutex);
1167         __arm_kprobe(kp);
1168         mutex_unlock(&text_mutex);
1169         cpus_read_unlock();
1170
1171         return 0;
1172 }
1173
1174 static int disarm_kprobe(struct kprobe *kp, bool reopt)
1175 {
1176         if (unlikely(kprobe_ftrace(kp)))
1177                 return disarm_kprobe_ftrace(kp);
1178
1179         cpus_read_lock();
1180         mutex_lock(&text_mutex);
1181         __disarm_kprobe(kp, reopt);
1182         mutex_unlock(&text_mutex);
1183         cpus_read_unlock();
1184
1185         return 0;
1186 }
1187
1188 /*
1189  * Aggregate handlers for multiple kprobes support - these handlers
1190  * take care of invoking the individual kprobe handlers on p->list
1191  */
1192 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
1193 {
1194         struct kprobe *kp;
1195
1196         list_for_each_entry_rcu(kp, &p->list, list) {
1197                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
1198                         set_kprobe_instance(kp);
1199                         if (kp->pre_handler(kp, regs))
1200                                 return 1;
1201                 }
1202                 reset_kprobe_instance();
1203         }
1204         return 0;
1205 }
1206 NOKPROBE_SYMBOL(aggr_pre_handler);
1207
1208 static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
1209                               unsigned long flags)
1210 {
1211         struct kprobe *kp;
1212
1213         list_for_each_entry_rcu(kp, &p->list, list) {
1214                 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
1215                         set_kprobe_instance(kp);
1216                         kp->post_handler(kp, regs, flags);
1217                         reset_kprobe_instance();
1218                 }
1219         }
1220 }
1221 NOKPROBE_SYMBOL(aggr_post_handler);
1222
1223 /* Walks the list and increments 'nmissed' if 'p' has child probes. */
1224 void kprobes_inc_nmissed_count(struct kprobe *p)
1225 {
1226         struct kprobe *kp;
1227
1228         if (!kprobe_aggrprobe(p)) {
1229                 p->nmissed++;
1230         } else {
1231                 list_for_each_entry_rcu(kp, &p->list, list)
1232                         kp->nmissed++;
1233         }
1234 }
1235 NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
1236
1237 static struct kprobe kprobe_busy = {
1238         .addr = (void *) get_kprobe,
1239 };
1240
1241 void kprobe_busy_begin(void)
1242 {
1243         struct kprobe_ctlblk *kcb;
1244
1245         preempt_disable();
1246         __this_cpu_write(current_kprobe, &kprobe_busy);
1247         kcb = get_kprobe_ctlblk();
1248         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
1249 }
1250
1251 void kprobe_busy_end(void)
1252 {
1253         __this_cpu_write(current_kprobe, NULL);
1254         preempt_enable();
1255 }
1256
1257 /* Add the new probe to 'ap->list'. */
1258 static int add_new_kprobe(struct kprobe *ap, struct kprobe *p)
1259 {
1260         if (p->post_handler)
1261                 unoptimize_kprobe(ap, true);    /* Fall back to normal kprobe */
1262
1263         list_add_rcu(&p->list, &ap->list);
1264         if (p->post_handler && !ap->post_handler)
1265                 ap->post_handler = aggr_post_handler;
1266
1267         return 0;
1268 }
1269
1270 /*
1271  * Fill in the required fields of the aggregator kprobe. Replace the
1272  * earlier kprobe in the hlist with the aggregator kprobe.
1273  */
1274 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
1275 {
1276         /* Copy the insn slot of 'p' to 'ap'. */
1277         copy_kprobe(p, ap);
1278         flush_insn_slot(ap);
1279         ap->addr = p->addr;
1280         ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
1281         ap->pre_handler = aggr_pre_handler;
1282         /* We don't care the kprobe which has gone. */
1283         if (p->post_handler && !kprobe_gone(p))
1284                 ap->post_handler = aggr_post_handler;
1285
1286         INIT_LIST_HEAD(&ap->list);
1287         INIT_HLIST_NODE(&ap->hlist);
1288
1289         list_add_rcu(&p->list, &ap->list);
1290         hlist_replace_rcu(&p->hlist, &ap->hlist);
1291 }
1292
1293 /*
1294  * This registers the second or subsequent kprobe at the same address.
1295  */
1296 static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p)
1297 {
1298         int ret = 0;
1299         struct kprobe *ap = orig_p;
1300
1301         cpus_read_lock();
1302
1303         /* For preparing optimization, jump_label_text_reserved() is called */
1304         jump_label_lock();
1305         mutex_lock(&text_mutex);
1306
1307         if (!kprobe_aggrprobe(orig_p)) {
1308                 /* If 'orig_p' is not an 'aggr_kprobe', create new one. */
1309                 ap = alloc_aggr_kprobe(orig_p);
1310                 if (!ap) {
1311                         ret = -ENOMEM;
1312                         goto out;
1313                 }
1314                 init_aggr_kprobe(ap, orig_p);
1315         } else if (kprobe_unused(ap)) {
1316                 /* This probe is going to die. Rescue it */
1317                 ret = reuse_unused_kprobe(ap);
1318                 if (ret)
1319                         goto out;
1320         }
1321
1322         if (kprobe_gone(ap)) {
1323                 /*
1324                  * Attempting to insert new probe at the same location that
1325                  * had a probe in the module vaddr area which already
1326                  * freed. So, the instruction slot has already been
1327                  * released. We need a new slot for the new probe.
1328                  */
1329                 ret = arch_prepare_kprobe(ap);
1330                 if (ret)
1331                         /*
1332                          * Even if fail to allocate new slot, don't need to
1333                          * free the 'ap'. It will be used next time, or
1334                          * freed by unregister_kprobe().
1335                          */
1336                         goto out;
1337
1338                 /* Prepare optimized instructions if possible. */
1339                 prepare_optimized_kprobe(ap);
1340
1341                 /*
1342                  * Clear gone flag to prevent allocating new slot again, and
1343                  * set disabled flag because it is not armed yet.
1344                  */
1345                 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1346                             | KPROBE_FLAG_DISABLED;
1347         }
1348
1349         /* Copy the insn slot of 'p' to 'ap'. */
1350         copy_kprobe(ap, p);
1351         ret = add_new_kprobe(ap, p);
1352
1353 out:
1354         mutex_unlock(&text_mutex);
1355         jump_label_unlock();
1356         cpus_read_unlock();
1357
1358         if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) {
1359                 ap->flags &= ~KPROBE_FLAG_DISABLED;
1360                 if (!kprobes_all_disarmed) {
1361                         /* Arm the breakpoint again. */
1362                         ret = arm_kprobe(ap);
1363                         if (ret) {
1364                                 ap->flags |= KPROBE_FLAG_DISABLED;
1365                                 list_del_rcu(&p->list);
1366                                 synchronize_rcu();
1367                         }
1368                 }
1369         }
1370         return ret;
1371 }
1372
1373 bool __weak arch_within_kprobe_blacklist(unsigned long addr)
1374 {
1375         /* The '__kprobes' functions and entry code must not be probed. */
1376         return addr >= (unsigned long)__kprobes_text_start &&
1377                addr < (unsigned long)__kprobes_text_end;
1378 }
1379
1380 static bool __within_kprobe_blacklist(unsigned long addr)
1381 {
1382         struct kprobe_blacklist_entry *ent;
1383
1384         if (arch_within_kprobe_blacklist(addr))
1385                 return true;
1386         /*
1387          * If 'kprobe_blacklist' is defined, check the address and
1388          * reject any probe registration in the prohibited area.
1389          */
1390         list_for_each_entry(ent, &kprobe_blacklist, list) {
1391                 if (addr >= ent->start_addr && addr < ent->end_addr)
1392                         return true;
1393         }
1394         return false;
1395 }
1396
1397 bool within_kprobe_blacklist(unsigned long addr)
1398 {
1399         char symname[KSYM_NAME_LEN], *p;
1400
1401         if (__within_kprobe_blacklist(addr))
1402                 return true;
1403
1404         /* Check if the address is on a suffixed-symbol */
1405         if (!lookup_symbol_name(addr, symname)) {
1406                 p = strchr(symname, '.');
1407                 if (!p)
1408                         return false;
1409                 *p = '\0';
1410                 addr = (unsigned long)kprobe_lookup_name(symname, 0);
1411                 if (addr)
1412                         return __within_kprobe_blacklist(addr);
1413         }
1414         return false;
1415 }
1416
1417 /*
1418  * arch_adjust_kprobe_addr - adjust the address
1419  * @addr: symbol base address
1420  * @offset: offset within the symbol
1421  * @on_func_entry: was this @addr+@offset on the function entry
1422  *
1423  * Typically returns @addr + @offset, except for special cases where the
1424  * function might be prefixed by a CFI landing pad, in that case any offset
1425  * inside the landing pad is mapped to the first 'real' instruction of the
1426  * symbol.
1427  *
1428  * Specifically, for things like IBT/BTI, skip the resp. ENDBR/BTI.C
1429  * instruction at +0.
1430  */
1431 kprobe_opcode_t *__weak arch_adjust_kprobe_addr(unsigned long addr,
1432                                                 unsigned long offset,
1433                                                 bool *on_func_entry)
1434 {
1435         *on_func_entry = !offset;
1436         return (kprobe_opcode_t *)(addr + offset);
1437 }
1438
1439 /*
1440  * If 'symbol_name' is specified, look it up and add the 'offset'
1441  * to it. This way, we can specify a relative address to a symbol.
1442  * This returns encoded errors if it fails to look up symbol or invalid
1443  * combination of parameters.
1444  */
1445 static kprobe_opcode_t *
1446 _kprobe_addr(kprobe_opcode_t *addr, const char *symbol_name,
1447              unsigned long offset, bool *on_func_entry)
1448 {
1449         if ((symbol_name && addr) || (!symbol_name && !addr))
1450                 goto invalid;
1451
1452         if (symbol_name) {
1453                 /*
1454                  * Input: @sym + @offset
1455                  * Output: @addr + @offset
1456                  *
1457                  * NOTE: kprobe_lookup_name() does *NOT* fold the offset
1458                  *       argument into it's output!
1459                  */
1460                 addr = kprobe_lookup_name(symbol_name, offset);
1461                 if (!addr)
1462                         return ERR_PTR(-ENOENT);
1463         }
1464
1465         /*
1466          * So here we have @addr + @offset, displace it into a new
1467          * @addr' + @offset' where @addr' is the symbol start address.
1468          */
1469         addr = (void *)addr + offset;
1470         if (!kallsyms_lookup_size_offset((unsigned long)addr, NULL, &offset))
1471                 return ERR_PTR(-ENOENT);
1472         addr = (void *)addr - offset;
1473
1474         /*
1475          * Then ask the architecture to re-combine them, taking care of
1476          * magical function entry details while telling us if this was indeed
1477          * at the start of the function.
1478          */
1479         addr = arch_adjust_kprobe_addr((unsigned long)addr, offset, on_func_entry);
1480         if (addr)
1481                 return addr;
1482
1483 invalid:
1484         return ERR_PTR(-EINVAL);
1485 }
1486
1487 static kprobe_opcode_t *kprobe_addr(struct kprobe *p)
1488 {
1489         bool on_func_entry;
1490         return _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry);
1491 }
1492
1493 /*
1494  * Check the 'p' is valid and return the aggregator kprobe
1495  * at the same address.
1496  */
1497 static struct kprobe *__get_valid_kprobe(struct kprobe *p)
1498 {
1499         struct kprobe *ap, *list_p;
1500
1501         lockdep_assert_held(&kprobe_mutex);
1502
1503         ap = get_kprobe(p->addr);
1504         if (unlikely(!ap))
1505                 return NULL;
1506
1507         if (p != ap) {
1508                 list_for_each_entry(list_p, &ap->list, list)
1509                         if (list_p == p)
1510                         /* kprobe p is a valid probe */
1511                                 goto valid;
1512                 return NULL;
1513         }
1514 valid:
1515         return ap;
1516 }
1517
1518 /*
1519  * Warn and return error if the kprobe is being re-registered since
1520  * there must be a software bug.
1521  */
1522 static inline int warn_kprobe_rereg(struct kprobe *p)
1523 {
1524         int ret = 0;
1525
1526         mutex_lock(&kprobe_mutex);
1527         if (WARN_ON_ONCE(__get_valid_kprobe(p)))
1528                 ret = -EINVAL;
1529         mutex_unlock(&kprobe_mutex);
1530
1531         return ret;
1532 }
1533
1534 static int check_ftrace_location(struct kprobe *p)
1535 {
1536         unsigned long addr = (unsigned long)p->addr;
1537
1538         if (ftrace_location(addr) == addr) {
1539 #ifdef CONFIG_KPROBES_ON_FTRACE
1540                 p->flags |= KPROBE_FLAG_FTRACE;
1541 #else   /* !CONFIG_KPROBES_ON_FTRACE */
1542                 return -EINVAL;
1543 #endif
1544         }
1545         return 0;
1546 }
1547
1548 static bool is_cfi_preamble_symbol(unsigned long addr)
1549 {
1550         char symbuf[KSYM_NAME_LEN];
1551
1552         if (lookup_symbol_name(addr, symbuf))
1553                 return false;
1554
1555         return str_has_prefix("__cfi_", symbuf) ||
1556                 str_has_prefix("__pfx_", symbuf);
1557 }
1558
1559 static int check_kprobe_address_safe(struct kprobe *p,
1560                                      struct module **probed_mod)
1561 {
1562         int ret;
1563
1564         ret = check_ftrace_location(p);
1565         if (ret)
1566                 return ret;
1567         jump_label_lock();
1568         preempt_disable();
1569
1570         /* Ensure it is not in reserved area nor out of text */
1571         if (!(core_kernel_text((unsigned long) p->addr) ||
1572             is_module_text_address((unsigned long) p->addr)) ||
1573             in_gate_area_no_mm((unsigned long) p->addr) ||
1574             within_kprobe_blacklist((unsigned long) p->addr) ||
1575             jump_label_text_reserved(p->addr, p->addr) ||
1576             static_call_text_reserved(p->addr, p->addr) ||
1577             find_bug((unsigned long)p->addr) ||
1578             is_cfi_preamble_symbol((unsigned long)p->addr)) {
1579                 ret = -EINVAL;
1580                 goto out;
1581         }
1582
1583         /* Check if 'p' is probing a module. */
1584         *probed_mod = __module_text_address((unsigned long) p->addr);
1585         if (*probed_mod) {
1586                 /*
1587                  * We must hold a refcount of the probed module while updating
1588                  * its code to prohibit unexpected unloading.
1589                  */
1590                 if (unlikely(!try_module_get(*probed_mod))) {
1591                         ret = -ENOENT;
1592                         goto out;
1593                 }
1594
1595                 /*
1596                  * If the module freed '.init.text', we couldn't insert
1597                  * kprobes in there.
1598                  */
1599                 if (within_module_init((unsigned long)p->addr, *probed_mod) &&
1600                     (*probed_mod)->state != MODULE_STATE_COMING) {
1601                         module_put(*probed_mod);
1602                         *probed_mod = NULL;
1603                         ret = -ENOENT;
1604                 }
1605         }
1606 out:
1607         preempt_enable();
1608         jump_label_unlock();
1609
1610         return ret;
1611 }
1612
1613 int register_kprobe(struct kprobe *p)
1614 {
1615         int ret;
1616         struct kprobe *old_p;
1617         struct module *probed_mod;
1618         kprobe_opcode_t *addr;
1619         bool on_func_entry;
1620
1621         /* Adjust probe address from symbol */
1622         addr = _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry);
1623         if (IS_ERR(addr))
1624                 return PTR_ERR(addr);
1625         p->addr = addr;
1626
1627         ret = warn_kprobe_rereg(p);
1628         if (ret)
1629                 return ret;
1630
1631         /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1632         p->flags &= KPROBE_FLAG_DISABLED;
1633         p->nmissed = 0;
1634         INIT_LIST_HEAD(&p->list);
1635
1636         ret = check_kprobe_address_safe(p, &probed_mod);
1637         if (ret)
1638                 return ret;
1639
1640         mutex_lock(&kprobe_mutex);
1641
1642         if (on_func_entry)
1643                 p->flags |= KPROBE_FLAG_ON_FUNC_ENTRY;
1644
1645         old_p = get_kprobe(p->addr);
1646         if (old_p) {
1647                 /* Since this may unoptimize 'old_p', locking 'text_mutex'. */
1648                 ret = register_aggr_kprobe(old_p, p);
1649                 goto out;
1650         }
1651
1652         cpus_read_lock();
1653         /* Prevent text modification */
1654         mutex_lock(&text_mutex);
1655         ret = prepare_kprobe(p);
1656         mutex_unlock(&text_mutex);
1657         cpus_read_unlock();
1658         if (ret)
1659                 goto out;
1660
1661         INIT_HLIST_NODE(&p->hlist);
1662         hlist_add_head_rcu(&p->hlist,
1663                        &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1664
1665         if (!kprobes_all_disarmed && !kprobe_disabled(p)) {
1666                 ret = arm_kprobe(p);
1667                 if (ret) {
1668                         hlist_del_rcu(&p->hlist);
1669                         synchronize_rcu();
1670                         goto out;
1671                 }
1672         }
1673
1674         /* Try to optimize kprobe */
1675         try_to_optimize_kprobe(p);
1676 out:
1677         mutex_unlock(&kprobe_mutex);
1678
1679         if (probed_mod)
1680                 module_put(probed_mod);
1681
1682         return ret;
1683 }
1684 EXPORT_SYMBOL_GPL(register_kprobe);
1685
1686 /* Check if all probes on the 'ap' are disabled. */
1687 static bool aggr_kprobe_disabled(struct kprobe *ap)
1688 {
1689         struct kprobe *kp;
1690
1691         lockdep_assert_held(&kprobe_mutex);
1692
1693         list_for_each_entry(kp, &ap->list, list)
1694                 if (!kprobe_disabled(kp))
1695                         /*
1696                          * Since there is an active probe on the list,
1697                          * we can't disable this 'ap'.
1698                          */
1699                         return false;
1700
1701         return true;
1702 }
1703
1704 static struct kprobe *__disable_kprobe(struct kprobe *p)
1705 {
1706         struct kprobe *orig_p;
1707         int ret;
1708
1709         lockdep_assert_held(&kprobe_mutex);
1710
1711         /* Get an original kprobe for return */
1712         orig_p = __get_valid_kprobe(p);
1713         if (unlikely(orig_p == NULL))
1714                 return ERR_PTR(-EINVAL);
1715
1716         if (!kprobe_disabled(p)) {
1717                 /* Disable probe if it is a child probe */
1718                 if (p != orig_p)
1719                         p->flags |= KPROBE_FLAG_DISABLED;
1720
1721                 /* Try to disarm and disable this/parent probe */
1722                 if (p == orig_p || aggr_kprobe_disabled(orig_p)) {
1723                         /*
1724                          * Don't be lazy here.  Even if 'kprobes_all_disarmed'
1725                          * is false, 'orig_p' might not have been armed yet.
1726                          * Note arm_all_kprobes() __tries__ to arm all kprobes
1727                          * on the best effort basis.
1728                          */
1729                         if (!kprobes_all_disarmed && !kprobe_disabled(orig_p)) {
1730                                 ret = disarm_kprobe(orig_p, true);
1731                                 if (ret) {
1732                                         p->flags &= ~KPROBE_FLAG_DISABLED;
1733                                         return ERR_PTR(ret);
1734                                 }
1735                         }
1736                         orig_p->flags |= KPROBE_FLAG_DISABLED;
1737                 }
1738         }
1739
1740         return orig_p;
1741 }
1742
1743 /*
1744  * Unregister a kprobe without a scheduler synchronization.
1745  */
1746 static int __unregister_kprobe_top(struct kprobe *p)
1747 {
1748         struct kprobe *ap, *list_p;
1749
1750         /* Disable kprobe. This will disarm it if needed. */
1751         ap = __disable_kprobe(p);
1752         if (IS_ERR(ap))
1753                 return PTR_ERR(ap);
1754
1755         if (ap == p)
1756                 /*
1757                  * This probe is an independent(and non-optimized) kprobe
1758                  * (not an aggrprobe). Remove from the hash list.
1759                  */
1760                 goto disarmed;
1761
1762         /* Following process expects this probe is an aggrprobe */
1763         WARN_ON(!kprobe_aggrprobe(ap));
1764
1765         if (list_is_singular(&ap->list) && kprobe_disarmed(ap))
1766                 /*
1767                  * !disarmed could be happen if the probe is under delayed
1768                  * unoptimizing.
1769                  */
1770                 goto disarmed;
1771         else {
1772                 /* If disabling probe has special handlers, update aggrprobe */
1773                 if (p->post_handler && !kprobe_gone(p)) {
1774                         list_for_each_entry(list_p, &ap->list, list) {
1775                                 if ((list_p != p) && (list_p->post_handler))
1776                                         goto noclean;
1777                         }
1778                         /*
1779                          * For the kprobe-on-ftrace case, we keep the
1780                          * post_handler setting to identify this aggrprobe
1781                          * armed with kprobe_ipmodify_ops.
1782                          */
1783                         if (!kprobe_ftrace(ap))
1784                                 ap->post_handler = NULL;
1785                 }
1786 noclean:
1787                 /*
1788                  * Remove from the aggrprobe: this path will do nothing in
1789                  * __unregister_kprobe_bottom().
1790                  */
1791                 list_del_rcu(&p->list);
1792                 if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
1793                         /*
1794                          * Try to optimize this probe again, because post
1795                          * handler may have been changed.
1796                          */
1797                         optimize_kprobe(ap);
1798         }
1799         return 0;
1800
1801 disarmed:
1802         hlist_del_rcu(&ap->hlist);
1803         return 0;
1804 }
1805
1806 static void __unregister_kprobe_bottom(struct kprobe *p)
1807 {
1808         struct kprobe *ap;
1809
1810         if (list_empty(&p->list))
1811                 /* This is an independent kprobe */
1812                 arch_remove_kprobe(p);
1813         else if (list_is_singular(&p->list)) {
1814                 /* This is the last child of an aggrprobe */
1815                 ap = list_entry(p->list.next, struct kprobe, list);
1816                 list_del(&p->list);
1817                 free_aggr_kprobe(ap);
1818         }
1819         /* Otherwise, do nothing. */
1820 }
1821
1822 int register_kprobes(struct kprobe **kps, int num)
1823 {
1824         int i, ret = 0;
1825
1826         if (num <= 0)
1827                 return -EINVAL;
1828         for (i = 0; i < num; i++) {
1829                 ret = register_kprobe(kps[i]);
1830                 if (ret < 0) {
1831                         if (i > 0)
1832                                 unregister_kprobes(kps, i);
1833                         break;
1834                 }
1835         }
1836         return ret;
1837 }
1838 EXPORT_SYMBOL_GPL(register_kprobes);
1839
1840 void unregister_kprobe(struct kprobe *p)
1841 {
1842         unregister_kprobes(&p, 1);
1843 }
1844 EXPORT_SYMBOL_GPL(unregister_kprobe);
1845
1846 void unregister_kprobes(struct kprobe **kps, int num)
1847 {
1848         int i;
1849
1850         if (num <= 0)
1851                 return;
1852         mutex_lock(&kprobe_mutex);
1853         for (i = 0; i < num; i++)
1854                 if (__unregister_kprobe_top(kps[i]) < 0)
1855                         kps[i]->addr = NULL;
1856         mutex_unlock(&kprobe_mutex);
1857
1858         synchronize_rcu();
1859         for (i = 0; i < num; i++)
1860                 if (kps[i]->addr)
1861                         __unregister_kprobe_bottom(kps[i]);
1862 }
1863 EXPORT_SYMBOL_GPL(unregister_kprobes);
1864
1865 int __weak kprobe_exceptions_notify(struct notifier_block *self,
1866                                         unsigned long val, void *data)
1867 {
1868         return NOTIFY_DONE;
1869 }
1870 NOKPROBE_SYMBOL(kprobe_exceptions_notify);
1871
1872 static struct notifier_block kprobe_exceptions_nb = {
1873         .notifier_call = kprobe_exceptions_notify,
1874         .priority = 0x7fffffff /* we need to be notified first */
1875 };
1876
1877 #ifdef CONFIG_KRETPROBES
1878
1879 #if !defined(CONFIG_KRETPROBE_ON_RETHOOK)
1880 static void free_rp_inst_rcu(struct rcu_head *head)
1881 {
1882         struct kretprobe_instance *ri = container_of(head, struct kretprobe_instance, rcu);
1883
1884         if (refcount_dec_and_test(&ri->rph->ref))
1885                 kfree(ri->rph);
1886         kfree(ri);
1887 }
1888 NOKPROBE_SYMBOL(free_rp_inst_rcu);
1889
1890 static void recycle_rp_inst(struct kretprobe_instance *ri)
1891 {
1892         struct kretprobe *rp = get_kretprobe(ri);
1893
1894         if (likely(rp))
1895                 freelist_add(&ri->freelist, &rp->freelist);
1896         else
1897                 call_rcu(&ri->rcu, free_rp_inst_rcu);
1898 }
1899 NOKPROBE_SYMBOL(recycle_rp_inst);
1900
1901 /*
1902  * This function is called from delayed_put_task_struct() when a task is
1903  * dead and cleaned up to recycle any kretprobe instances associated with
1904  * this task. These left over instances represent probed functions that
1905  * have been called but will never return.
1906  */
1907 void kprobe_flush_task(struct task_struct *tk)
1908 {
1909         struct kretprobe_instance *ri;
1910         struct llist_node *node;
1911
1912         /* Early boot, not yet initialized. */
1913         if (unlikely(!kprobes_initialized))
1914                 return;
1915
1916         kprobe_busy_begin();
1917
1918         node = __llist_del_all(&tk->kretprobe_instances);
1919         while (node) {
1920                 ri = container_of(node, struct kretprobe_instance, llist);
1921                 node = node->next;
1922
1923                 recycle_rp_inst(ri);
1924         }
1925
1926         kprobe_busy_end();
1927 }
1928 NOKPROBE_SYMBOL(kprobe_flush_task);
1929
1930 static inline void free_rp_inst(struct kretprobe *rp)
1931 {
1932         struct kretprobe_instance *ri;
1933         struct freelist_node *node;
1934         int count = 0;
1935
1936         node = rp->freelist.head;
1937         while (node) {
1938                 ri = container_of(node, struct kretprobe_instance, freelist);
1939                 node = node->next;
1940
1941                 kfree(ri);
1942                 count++;
1943         }
1944
1945         if (refcount_sub_and_test(count, &rp->rph->ref)) {
1946                 kfree(rp->rph);
1947                 rp->rph = NULL;
1948         }
1949 }
1950
1951 /* This assumes the 'tsk' is the current task or the is not running. */
1952 static kprobe_opcode_t *__kretprobe_find_ret_addr(struct task_struct *tsk,
1953                                                   struct llist_node **cur)
1954 {
1955         struct kretprobe_instance *ri = NULL;
1956         struct llist_node *node = *cur;
1957
1958         if (!node)
1959                 node = tsk->kretprobe_instances.first;
1960         else
1961                 node = node->next;
1962
1963         while (node) {
1964                 ri = container_of(node, struct kretprobe_instance, llist);
1965                 if (ri->ret_addr != kretprobe_trampoline_addr()) {
1966                         *cur = node;
1967                         return ri->ret_addr;
1968                 }
1969                 node = node->next;
1970         }
1971         return NULL;
1972 }
1973 NOKPROBE_SYMBOL(__kretprobe_find_ret_addr);
1974
1975 /**
1976  * kretprobe_find_ret_addr -- Find correct return address modified by kretprobe
1977  * @tsk: Target task
1978  * @fp: A frame pointer
1979  * @cur: a storage of the loop cursor llist_node pointer for next call
1980  *
1981  * Find the correct return address modified by a kretprobe on @tsk in unsigned
1982  * long type. If it finds the return address, this returns that address value,
1983  * or this returns 0.
1984  * The @tsk must be 'current' or a task which is not running. @fp is a hint
1985  * to get the currect return address - which is compared with the
1986  * kretprobe_instance::fp field. The @cur is a loop cursor for searching the
1987  * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
1988  * first call, but '@cur' itself must NOT NULL.
1989  */
1990 unsigned long kretprobe_find_ret_addr(struct task_struct *tsk, void *fp,
1991                                       struct llist_node **cur)
1992 {
1993         struct kretprobe_instance *ri = NULL;
1994         kprobe_opcode_t *ret;
1995
1996         if (WARN_ON_ONCE(!cur))
1997                 return 0;
1998
1999         do {
2000                 ret = __kretprobe_find_ret_addr(tsk, cur);
2001                 if (!ret)
2002                         break;
2003                 ri = container_of(*cur, struct kretprobe_instance, llist);
2004         } while (ri->fp != fp);
2005
2006         return (unsigned long)ret;
2007 }
2008 NOKPROBE_SYMBOL(kretprobe_find_ret_addr);
2009
2010 void __weak arch_kretprobe_fixup_return(struct pt_regs *regs,
2011                                         kprobe_opcode_t *correct_ret_addr)
2012 {
2013         /*
2014          * Do nothing by default. Please fill this to update the fake return
2015          * address on the stack with the correct one on each arch if possible.
2016          */
2017 }
2018
2019 unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
2020                                              void *frame_pointer)
2021 {
2022         struct kretprobe_instance *ri = NULL;
2023         struct llist_node *first, *node = NULL;
2024         kprobe_opcode_t *correct_ret_addr;
2025         struct kretprobe *rp;
2026
2027         /* Find correct address and all nodes for this frame. */
2028         correct_ret_addr = __kretprobe_find_ret_addr(current, &node);
2029         if (!correct_ret_addr) {
2030                 pr_err("kretprobe: Return address not found, not execute handler. Maybe there is a bug in the kernel.\n");
2031                 BUG_ON(1);
2032         }
2033
2034         /*
2035          * Set the return address as the instruction pointer, because if the
2036          * user handler calls stack_trace_save_regs() with this 'regs',
2037          * the stack trace will start from the instruction pointer.
2038          */
2039         instruction_pointer_set(regs, (unsigned long)correct_ret_addr);
2040
2041         /* Run the user handler of the nodes. */
2042         first = current->kretprobe_instances.first;
2043         while (first) {
2044                 ri = container_of(first, struct kretprobe_instance, llist);
2045
2046                 if (WARN_ON_ONCE(ri->fp != frame_pointer))
2047                         break;
2048
2049                 rp = get_kretprobe(ri);
2050                 if (rp && rp->handler) {
2051                         struct kprobe *prev = kprobe_running();
2052
2053                         __this_cpu_write(current_kprobe, &rp->kp);
2054                         ri->ret_addr = correct_ret_addr;
2055                         rp->handler(ri, regs);
2056                         __this_cpu_write(current_kprobe, prev);
2057                 }
2058                 if (first == node)
2059                         break;
2060
2061                 first = first->next;
2062         }
2063
2064         arch_kretprobe_fixup_return(regs, correct_ret_addr);
2065
2066         /* Unlink all nodes for this frame. */
2067         first = current->kretprobe_instances.first;
2068         current->kretprobe_instances.first = node->next;
2069         node->next = NULL;
2070
2071         /* Recycle free instances. */
2072         while (first) {
2073                 ri = container_of(first, struct kretprobe_instance, llist);
2074                 first = first->next;
2075
2076                 recycle_rp_inst(ri);
2077         }
2078
2079         return (unsigned long)correct_ret_addr;
2080 }
2081 NOKPROBE_SYMBOL(__kretprobe_trampoline_handler)
2082
2083 /*
2084  * This kprobe pre_handler is registered with every kretprobe. When probe
2085  * hits it will set up the return probe.
2086  */
2087 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
2088 {
2089         struct kretprobe *rp = container_of(p, struct kretprobe, kp);
2090         struct kretprobe_instance *ri;
2091         struct freelist_node *fn;
2092
2093         fn = freelist_try_get(&rp->freelist);
2094         if (!fn) {
2095                 rp->nmissed++;
2096                 return 0;
2097         }
2098
2099         ri = container_of(fn, struct kretprobe_instance, freelist);
2100
2101         if (rp->entry_handler && rp->entry_handler(ri, regs)) {
2102                 freelist_add(&ri->freelist, &rp->freelist);
2103                 return 0;
2104         }
2105
2106         arch_prepare_kretprobe(ri, regs);
2107
2108         __llist_add(&ri->llist, &current->kretprobe_instances);
2109
2110         return 0;
2111 }
2112 NOKPROBE_SYMBOL(pre_handler_kretprobe);
2113 #else /* CONFIG_KRETPROBE_ON_RETHOOK */
2114 /*
2115  * This kprobe pre_handler is registered with every kretprobe. When probe
2116  * hits it will set up the return probe.
2117  */
2118 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
2119 {
2120         struct kretprobe *rp = container_of(p, struct kretprobe, kp);
2121         struct kretprobe_instance *ri;
2122         struct rethook_node *rhn;
2123
2124         rhn = rethook_try_get(rp->rh);
2125         if (!rhn) {
2126                 rp->nmissed++;
2127                 return 0;
2128         }
2129
2130         ri = container_of(rhn, struct kretprobe_instance, node);
2131
2132         if (rp->entry_handler && rp->entry_handler(ri, regs))
2133                 rethook_recycle(rhn);
2134         else
2135                 rethook_hook(rhn, regs, kprobe_ftrace(p));
2136
2137         return 0;
2138 }
2139 NOKPROBE_SYMBOL(pre_handler_kretprobe);
2140
2141 static void kretprobe_rethook_handler(struct rethook_node *rh, void *data,
2142                                       unsigned long ret_addr,
2143                                       struct pt_regs *regs)
2144 {
2145         struct kretprobe *rp = (struct kretprobe *)data;
2146         struct kretprobe_instance *ri;
2147         struct kprobe_ctlblk *kcb;
2148
2149         /* The data must NOT be null. This means rethook data structure is broken. */
2150         if (WARN_ON_ONCE(!data) || !rp->handler)
2151                 return;
2152
2153         __this_cpu_write(current_kprobe, &rp->kp);
2154         kcb = get_kprobe_ctlblk();
2155         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
2156
2157         ri = container_of(rh, struct kretprobe_instance, node);
2158         rp->handler(ri, regs);
2159
2160         __this_cpu_write(current_kprobe, NULL);
2161 }
2162 NOKPROBE_SYMBOL(kretprobe_rethook_handler);
2163
2164 #endif /* !CONFIG_KRETPROBE_ON_RETHOOK */
2165
2166 /**
2167  * kprobe_on_func_entry() -- check whether given address is function entry
2168  * @addr: Target address
2169  * @sym:  Target symbol name
2170  * @offset: The offset from the symbol or the address
2171  *
2172  * This checks whether the given @addr+@offset or @sym+@offset is on the
2173  * function entry address or not.
2174  * This returns 0 if it is the function entry, or -EINVAL if it is not.
2175  * And also it returns -ENOENT if it fails the symbol or address lookup.
2176  * Caller must pass @addr or @sym (either one must be NULL), or this
2177  * returns -EINVAL.
2178  */
2179 int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
2180 {
2181         bool on_func_entry;
2182         kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset, &on_func_entry);
2183
2184         if (IS_ERR(kp_addr))
2185                 return PTR_ERR(kp_addr);
2186
2187         if (!on_func_entry)
2188                 return -EINVAL;
2189
2190         return 0;
2191 }
2192
2193 int register_kretprobe(struct kretprobe *rp)
2194 {
2195         int ret;
2196         struct kretprobe_instance *inst;
2197         int i;
2198         void *addr;
2199
2200         ret = kprobe_on_func_entry(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset);
2201         if (ret)
2202                 return ret;
2203
2204         /* If only 'rp->kp.addr' is specified, check reregistering kprobes */
2205         if (rp->kp.addr && warn_kprobe_rereg(&rp->kp))
2206                 return -EINVAL;
2207
2208         if (kretprobe_blacklist_size) {
2209                 addr = kprobe_addr(&rp->kp);
2210                 if (IS_ERR(addr))
2211                         return PTR_ERR(addr);
2212
2213                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2214                         if (kretprobe_blacklist[i].addr == addr)
2215                                 return -EINVAL;
2216                 }
2217         }
2218
2219         if (rp->data_size > KRETPROBE_MAX_DATA_SIZE)
2220                 return -E2BIG;
2221
2222         rp->kp.pre_handler = pre_handler_kretprobe;
2223         rp->kp.post_handler = NULL;
2224
2225         /* Pre-allocate memory for max kretprobe instances */
2226         if (rp->maxactive <= 0)
2227                 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
2228
2229 #ifdef CONFIG_KRETPROBE_ON_RETHOOK
2230         rp->rh = rethook_alloc((void *)rp, kretprobe_rethook_handler);
2231         if (!rp->rh)
2232                 return -ENOMEM;
2233
2234         for (i = 0; i < rp->maxactive; i++) {
2235                 inst = kzalloc(sizeof(struct kretprobe_instance) +
2236                                rp->data_size, GFP_KERNEL);
2237                 if (inst == NULL) {
2238                         rethook_free(rp->rh);
2239                         rp->rh = NULL;
2240                         return -ENOMEM;
2241                 }
2242                 rethook_add_node(rp->rh, &inst->node);
2243         }
2244         rp->nmissed = 0;
2245         /* Establish function entry probe point */
2246         ret = register_kprobe(&rp->kp);
2247         if (ret != 0) {
2248                 rethook_free(rp->rh);
2249                 rp->rh = NULL;
2250         }
2251 #else   /* !CONFIG_KRETPROBE_ON_RETHOOK */
2252         rp->freelist.head = NULL;
2253         rp->rph = kzalloc(sizeof(struct kretprobe_holder), GFP_KERNEL);
2254         if (!rp->rph)
2255                 return -ENOMEM;
2256
2257         rp->rph->rp = rp;
2258         for (i = 0; i < rp->maxactive; i++) {
2259                 inst = kzalloc(sizeof(struct kretprobe_instance) +
2260                                rp->data_size, GFP_KERNEL);
2261                 if (inst == NULL) {
2262                         refcount_set(&rp->rph->ref, i);
2263                         free_rp_inst(rp);
2264                         return -ENOMEM;
2265                 }
2266                 inst->rph = rp->rph;
2267                 freelist_add(&inst->freelist, &rp->freelist);
2268         }
2269         refcount_set(&rp->rph->ref, i);
2270
2271         rp->nmissed = 0;
2272         /* Establish function entry probe point */
2273         ret = register_kprobe(&rp->kp);
2274         if (ret != 0)
2275                 free_rp_inst(rp);
2276 #endif
2277         return ret;
2278 }
2279 EXPORT_SYMBOL_GPL(register_kretprobe);
2280
2281 int register_kretprobes(struct kretprobe **rps, int num)
2282 {
2283         int ret = 0, i;
2284
2285         if (num <= 0)
2286                 return -EINVAL;
2287         for (i = 0; i < num; i++) {
2288                 ret = register_kretprobe(rps[i]);
2289                 if (ret < 0) {
2290                         if (i > 0)
2291                                 unregister_kretprobes(rps, i);
2292                         break;
2293                 }
2294         }
2295         return ret;
2296 }
2297 EXPORT_SYMBOL_GPL(register_kretprobes);
2298
2299 void unregister_kretprobe(struct kretprobe *rp)
2300 {
2301         unregister_kretprobes(&rp, 1);
2302 }
2303 EXPORT_SYMBOL_GPL(unregister_kretprobe);
2304
2305 void unregister_kretprobes(struct kretprobe **rps, int num)
2306 {
2307         int i;
2308
2309         if (num <= 0)
2310                 return;
2311         mutex_lock(&kprobe_mutex);
2312         for (i = 0; i < num; i++) {
2313                 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
2314                         rps[i]->kp.addr = NULL;
2315 #ifdef CONFIG_KRETPROBE_ON_RETHOOK
2316                 rethook_free(rps[i]->rh);
2317 #else
2318                 rps[i]->rph->rp = NULL;
2319 #endif
2320         }
2321         mutex_unlock(&kprobe_mutex);
2322
2323         synchronize_rcu();
2324         for (i = 0; i < num; i++) {
2325                 if (rps[i]->kp.addr) {
2326                         __unregister_kprobe_bottom(&rps[i]->kp);
2327 #ifndef CONFIG_KRETPROBE_ON_RETHOOK
2328                         free_rp_inst(rps[i]);
2329 #endif
2330                 }
2331         }
2332 }
2333 EXPORT_SYMBOL_GPL(unregister_kretprobes);
2334
2335 #else /* CONFIG_KRETPROBES */
2336 int register_kretprobe(struct kretprobe *rp)
2337 {
2338         return -EOPNOTSUPP;
2339 }
2340 EXPORT_SYMBOL_GPL(register_kretprobe);
2341
2342 int register_kretprobes(struct kretprobe **rps, int num)
2343 {
2344         return -EOPNOTSUPP;
2345 }
2346 EXPORT_SYMBOL_GPL(register_kretprobes);
2347
2348 void unregister_kretprobe(struct kretprobe *rp)
2349 {
2350 }
2351 EXPORT_SYMBOL_GPL(unregister_kretprobe);
2352
2353 void unregister_kretprobes(struct kretprobe **rps, int num)
2354 {
2355 }
2356 EXPORT_SYMBOL_GPL(unregister_kretprobes);
2357
2358 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
2359 {
2360         return 0;
2361 }
2362 NOKPROBE_SYMBOL(pre_handler_kretprobe);
2363
2364 #endif /* CONFIG_KRETPROBES */
2365
2366 /* Set the kprobe gone and remove its instruction buffer. */
2367 static void kill_kprobe(struct kprobe *p)
2368 {
2369         struct kprobe *kp;
2370
2371         lockdep_assert_held(&kprobe_mutex);
2372
2373         /*
2374          * The module is going away. We should disarm the kprobe which
2375          * is using ftrace, because ftrace framework is still available at
2376          * 'MODULE_STATE_GOING' notification.
2377          */
2378         if (kprobe_ftrace(p) && !kprobe_disabled(p) && !kprobes_all_disarmed)
2379                 disarm_kprobe_ftrace(p);
2380
2381         p->flags |= KPROBE_FLAG_GONE;
2382         if (kprobe_aggrprobe(p)) {
2383                 /*
2384                  * If this is an aggr_kprobe, we have to list all the
2385                  * chained probes and mark them GONE.
2386                  */
2387                 list_for_each_entry(kp, &p->list, list)
2388                         kp->flags |= KPROBE_FLAG_GONE;
2389                 p->post_handler = NULL;
2390                 kill_optimized_kprobe(p);
2391         }
2392         /*
2393          * Here, we can remove insn_slot safely, because no thread calls
2394          * the original probed function (which will be freed soon) any more.
2395          */
2396         arch_remove_kprobe(p);
2397 }
2398
2399 /* Disable one kprobe */
2400 int disable_kprobe(struct kprobe *kp)
2401 {
2402         int ret = 0;
2403         struct kprobe *p;
2404
2405         mutex_lock(&kprobe_mutex);
2406
2407         /* Disable this kprobe */
2408         p = __disable_kprobe(kp);
2409         if (IS_ERR(p))
2410                 ret = PTR_ERR(p);
2411
2412         mutex_unlock(&kprobe_mutex);
2413         return ret;
2414 }
2415 EXPORT_SYMBOL_GPL(disable_kprobe);
2416
2417 /* Enable one kprobe */
2418 int enable_kprobe(struct kprobe *kp)
2419 {
2420         int ret = 0;
2421         struct kprobe *p;
2422
2423         mutex_lock(&kprobe_mutex);
2424
2425         /* Check whether specified probe is valid. */
2426         p = __get_valid_kprobe(kp);
2427         if (unlikely(p == NULL)) {
2428                 ret = -EINVAL;
2429                 goto out;
2430         }
2431
2432         if (kprobe_gone(kp)) {
2433                 /* This kprobe has gone, we couldn't enable it. */
2434                 ret = -EINVAL;
2435                 goto out;
2436         }
2437
2438         if (p != kp)
2439                 kp->flags &= ~KPROBE_FLAG_DISABLED;
2440
2441         if (!kprobes_all_disarmed && kprobe_disabled(p)) {
2442                 p->flags &= ~KPROBE_FLAG_DISABLED;
2443                 ret = arm_kprobe(p);
2444                 if (ret) {
2445                         p->flags |= KPROBE_FLAG_DISABLED;
2446                         if (p != kp)
2447                                 kp->flags |= KPROBE_FLAG_DISABLED;
2448                 }
2449         }
2450 out:
2451         mutex_unlock(&kprobe_mutex);
2452         return ret;
2453 }
2454 EXPORT_SYMBOL_GPL(enable_kprobe);
2455
2456 /* Caller must NOT call this in usual path. This is only for critical case */
2457 void dump_kprobe(struct kprobe *kp)
2458 {
2459         pr_err("Dump kprobe:\n.symbol_name = %s, .offset = %x, .addr = %pS\n",
2460                kp->symbol_name, kp->offset, kp->addr);
2461 }
2462 NOKPROBE_SYMBOL(dump_kprobe);
2463
2464 int kprobe_add_ksym_blacklist(unsigned long entry)
2465 {
2466         struct kprobe_blacklist_entry *ent;
2467         unsigned long offset = 0, size = 0;
2468
2469         if (!kernel_text_address(entry) ||
2470             !kallsyms_lookup_size_offset(entry, &size, &offset))
2471                 return -EINVAL;
2472
2473         ent = kmalloc(sizeof(*ent), GFP_KERNEL);
2474         if (!ent)
2475                 return -ENOMEM;
2476         ent->start_addr = entry;
2477         ent->end_addr = entry + size;
2478         INIT_LIST_HEAD(&ent->list);
2479         list_add_tail(&ent->list, &kprobe_blacklist);
2480
2481         return (int)size;
2482 }
2483
2484 /* Add all symbols in given area into kprobe blacklist */
2485 int kprobe_add_area_blacklist(unsigned long start, unsigned long end)
2486 {
2487         unsigned long entry;
2488         int ret = 0;
2489
2490         for (entry = start; entry < end; entry += ret) {
2491                 ret = kprobe_add_ksym_blacklist(entry);
2492                 if (ret < 0)
2493                         return ret;
2494                 if (ret == 0)   /* In case of alias symbol */
2495                         ret = 1;
2496         }
2497         return 0;
2498 }
2499
2500 /* Remove all symbols in given area from kprobe blacklist */
2501 static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end)
2502 {
2503         struct kprobe_blacklist_entry *ent, *n;
2504
2505         list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) {
2506                 if (ent->start_addr < start || ent->start_addr >= end)
2507                         continue;
2508                 list_del(&ent->list);
2509                 kfree(ent);
2510         }
2511 }
2512
2513 static void kprobe_remove_ksym_blacklist(unsigned long entry)
2514 {
2515         kprobe_remove_area_blacklist(entry, entry + 1);
2516 }
2517
2518 int __weak arch_kprobe_get_kallsym(unsigned int *symnum, unsigned long *value,
2519                                    char *type, char *sym)
2520 {
2521         return -ERANGE;
2522 }
2523
2524 int kprobe_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
2525                        char *sym)
2526 {
2527 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
2528         if (!kprobe_cache_get_kallsym(&kprobe_insn_slots, &symnum, value, type, sym))
2529                 return 0;
2530 #ifdef CONFIG_OPTPROBES
2531         if (!kprobe_cache_get_kallsym(&kprobe_optinsn_slots, &symnum, value, type, sym))
2532                 return 0;
2533 #endif
2534 #endif
2535         if (!arch_kprobe_get_kallsym(&symnum, value, type, sym))
2536                 return 0;
2537         return -ERANGE;
2538 }
2539
2540 int __init __weak arch_populate_kprobe_blacklist(void)
2541 {
2542         return 0;
2543 }
2544
2545 /*
2546  * Lookup and populate the kprobe_blacklist.
2547  *
2548  * Unlike the kretprobe blacklist, we'll need to determine
2549  * the range of addresses that belong to the said functions,
2550  * since a kprobe need not necessarily be at the beginning
2551  * of a function.
2552  */
2553 static int __init populate_kprobe_blacklist(unsigned long *start,
2554                                              unsigned long *end)
2555 {
2556         unsigned long entry;
2557         unsigned long *iter;
2558         int ret;
2559
2560         for (iter = start; iter < end; iter++) {
2561                 entry = (unsigned long)dereference_symbol_descriptor((void *)*iter);
2562                 ret = kprobe_add_ksym_blacklist(entry);
2563                 if (ret == -EINVAL)
2564                         continue;
2565                 if (ret < 0)
2566                         return ret;
2567         }
2568
2569         /* Symbols in '__kprobes_text' are blacklisted */
2570         ret = kprobe_add_area_blacklist((unsigned long)__kprobes_text_start,
2571                                         (unsigned long)__kprobes_text_end);
2572         if (ret)
2573                 return ret;
2574
2575         /* Symbols in 'noinstr' section are blacklisted */
2576         ret = kprobe_add_area_blacklist((unsigned long)__noinstr_text_start,
2577                                         (unsigned long)__noinstr_text_end);
2578
2579         return ret ? : arch_populate_kprobe_blacklist();
2580 }
2581
2582 static void add_module_kprobe_blacklist(struct module *mod)
2583 {
2584         unsigned long start, end;
2585         int i;
2586
2587         if (mod->kprobe_blacklist) {
2588                 for (i = 0; i < mod->num_kprobe_blacklist; i++)
2589                         kprobe_add_ksym_blacklist(mod->kprobe_blacklist[i]);
2590         }
2591
2592         start = (unsigned long)mod->kprobes_text_start;
2593         if (start) {
2594                 end = start + mod->kprobes_text_size;
2595                 kprobe_add_area_blacklist(start, end);
2596         }
2597
2598         start = (unsigned long)mod->noinstr_text_start;
2599         if (start) {
2600                 end = start + mod->noinstr_text_size;
2601                 kprobe_add_area_blacklist(start, end);
2602         }
2603 }
2604
2605 static void remove_module_kprobe_blacklist(struct module *mod)
2606 {
2607         unsigned long start, end;
2608         int i;
2609
2610         if (mod->kprobe_blacklist) {
2611                 for (i = 0; i < mod->num_kprobe_blacklist; i++)
2612                         kprobe_remove_ksym_blacklist(mod->kprobe_blacklist[i]);
2613         }
2614
2615         start = (unsigned long)mod->kprobes_text_start;
2616         if (start) {
2617                 end = start + mod->kprobes_text_size;
2618                 kprobe_remove_area_blacklist(start, end);
2619         }
2620
2621         start = (unsigned long)mod->noinstr_text_start;
2622         if (start) {
2623                 end = start + mod->noinstr_text_size;
2624                 kprobe_remove_area_blacklist(start, end);
2625         }
2626 }
2627
2628 /* Module notifier call back, checking kprobes on the module */
2629 static int kprobes_module_callback(struct notifier_block *nb,
2630                                    unsigned long val, void *data)
2631 {
2632         struct module *mod = data;
2633         struct hlist_head *head;
2634         struct kprobe *p;
2635         unsigned int i;
2636         int checkcore = (val == MODULE_STATE_GOING);
2637
2638         if (val == MODULE_STATE_COMING) {
2639                 mutex_lock(&kprobe_mutex);
2640                 add_module_kprobe_blacklist(mod);
2641                 mutex_unlock(&kprobe_mutex);
2642         }
2643         if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
2644                 return NOTIFY_DONE;
2645
2646         /*
2647          * When 'MODULE_STATE_GOING' was notified, both of module '.text' and
2648          * '.init.text' sections would be freed. When 'MODULE_STATE_LIVE' was
2649          * notified, only '.init.text' section would be freed. We need to
2650          * disable kprobes which have been inserted in the sections.
2651          */
2652         mutex_lock(&kprobe_mutex);
2653         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2654                 head = &kprobe_table[i];
2655                 hlist_for_each_entry(p, head, hlist)
2656                         if (within_module_init((unsigned long)p->addr, mod) ||
2657                             (checkcore &&
2658                              within_module_core((unsigned long)p->addr, mod))) {
2659                                 /*
2660                                  * The vaddr this probe is installed will soon
2661                                  * be vfreed buy not synced to disk. Hence,
2662                                  * disarming the breakpoint isn't needed.
2663                                  *
2664                                  * Note, this will also move any optimized probes
2665                                  * that are pending to be removed from their
2666                                  * corresponding lists to the 'freeing_list' and
2667                                  * will not be touched by the delayed
2668                                  * kprobe_optimizer() work handler.
2669                                  */
2670                                 kill_kprobe(p);
2671                         }
2672         }
2673         if (val == MODULE_STATE_GOING)
2674                 remove_module_kprobe_blacklist(mod);
2675         mutex_unlock(&kprobe_mutex);
2676         return NOTIFY_DONE;
2677 }
2678
2679 static struct notifier_block kprobe_module_nb = {
2680         .notifier_call = kprobes_module_callback,
2681         .priority = 0
2682 };
2683
2684 void kprobe_free_init_mem(void)
2685 {
2686         void *start = (void *)(&__init_begin);
2687         void *end = (void *)(&__init_end);
2688         struct hlist_head *head;
2689         struct kprobe *p;
2690         int i;
2691
2692         mutex_lock(&kprobe_mutex);
2693
2694         /* Kill all kprobes on initmem because the target code has been freed. */
2695         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2696                 head = &kprobe_table[i];
2697                 hlist_for_each_entry(p, head, hlist) {
2698                         if (start <= (void *)p->addr && (void *)p->addr < end)
2699                                 kill_kprobe(p);
2700                 }
2701         }
2702
2703         mutex_unlock(&kprobe_mutex);
2704 }
2705
2706 static int __init init_kprobes(void)
2707 {
2708         int i, err;
2709
2710         /* FIXME allocate the probe table, currently defined statically */
2711         /* initialize all list heads */
2712         for (i = 0; i < KPROBE_TABLE_SIZE; i++)
2713                 INIT_HLIST_HEAD(&kprobe_table[i]);
2714
2715         err = populate_kprobe_blacklist(__start_kprobe_blacklist,
2716                                         __stop_kprobe_blacklist);
2717         if (err)
2718                 pr_err("Failed to populate blacklist (error %d), kprobes not restricted, be careful using them!\n", err);
2719
2720         if (kretprobe_blacklist_size) {
2721                 /* lookup the function address from its name */
2722                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2723                         kretprobe_blacklist[i].addr =
2724                                 kprobe_lookup_name(kretprobe_blacklist[i].name, 0);
2725                         if (!kretprobe_blacklist[i].addr)
2726                                 pr_err("Failed to lookup symbol '%s' for kretprobe blacklist. Maybe the target function is removed or renamed.\n",
2727                                        kretprobe_blacklist[i].name);
2728                 }
2729         }
2730
2731         /* By default, kprobes are armed */
2732         kprobes_all_disarmed = false;
2733
2734 #if defined(CONFIG_OPTPROBES) && defined(__ARCH_WANT_KPROBES_INSN_SLOT)
2735         /* Init 'kprobe_optinsn_slots' for allocation */
2736         kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2737 #endif
2738
2739         err = arch_init_kprobes();
2740         if (!err)
2741                 err = register_die_notifier(&kprobe_exceptions_nb);
2742         if (!err)
2743                 err = register_module_notifier(&kprobe_module_nb);
2744
2745         kprobes_initialized = (err == 0);
2746         kprobe_sysctls_init();
2747         return err;
2748 }
2749 early_initcall(init_kprobes);
2750
2751 #if defined(CONFIG_OPTPROBES)
2752 static int __init init_optprobes(void)
2753 {
2754         /*
2755          * Enable kprobe optimization - this kicks the optimizer which
2756          * depends on synchronize_rcu_tasks() and ksoftirqd, that is
2757          * not spawned in early initcall. So delay the optimization.
2758          */
2759         optimize_all_kprobes();
2760
2761         return 0;
2762 }
2763 subsys_initcall(init_optprobes);
2764 #endif
2765
2766 #ifdef CONFIG_DEBUG_FS
2767 static void report_probe(struct seq_file *pi, struct kprobe *p,
2768                 const char *sym, int offset, char *modname, struct kprobe *pp)
2769 {
2770         char *kprobe_type;
2771         void *addr = p->addr;
2772
2773         if (p->pre_handler == pre_handler_kretprobe)
2774                 kprobe_type = "r";
2775         else
2776                 kprobe_type = "k";
2777
2778         if (!kallsyms_show_value(pi->file->f_cred))
2779                 addr = NULL;
2780
2781         if (sym)
2782                 seq_printf(pi, "%px  %s  %s+0x%x  %s ",
2783                         addr, kprobe_type, sym, offset,
2784                         (modname ? modname : " "));
2785         else    /* try to use %pS */
2786                 seq_printf(pi, "%px  %s  %pS ",
2787                         addr, kprobe_type, p->addr);
2788
2789         if (!pp)
2790                 pp = p;
2791         seq_printf(pi, "%s%s%s%s\n",
2792                 (kprobe_gone(p) ? "[GONE]" : ""),
2793                 ((kprobe_disabled(p) && !kprobe_gone(p)) ?  "[DISABLED]" : ""),
2794                 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""),
2795                 (kprobe_ftrace(pp) ? "[FTRACE]" : ""));
2796 }
2797
2798 static void *kprobe_seq_start(struct seq_file *f, loff_t *pos)
2799 {
2800         return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2801 }
2802
2803 static void *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
2804 {
2805         (*pos)++;
2806         if (*pos >= KPROBE_TABLE_SIZE)
2807                 return NULL;
2808         return pos;
2809 }
2810
2811 static void kprobe_seq_stop(struct seq_file *f, void *v)
2812 {
2813         /* Nothing to do */
2814 }
2815
2816 static int show_kprobe_addr(struct seq_file *pi, void *v)
2817 {
2818         struct hlist_head *head;
2819         struct kprobe *p, *kp;
2820         const char *sym = NULL;
2821         unsigned int i = *(loff_t *) v;
2822         unsigned long offset = 0;
2823         char *modname, namebuf[KSYM_NAME_LEN];
2824
2825         head = &kprobe_table[i];
2826         preempt_disable();
2827         hlist_for_each_entry_rcu(p, head, hlist) {
2828                 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
2829                                         &offset, &modname, namebuf);
2830                 if (kprobe_aggrprobe(p)) {
2831                         list_for_each_entry_rcu(kp, &p->list, list)
2832                                 report_probe(pi, kp, sym, offset, modname, p);
2833                 } else
2834                         report_probe(pi, p, sym, offset, modname, NULL);
2835         }
2836         preempt_enable();
2837         return 0;
2838 }
2839
2840 static const struct seq_operations kprobes_sops = {
2841         .start = kprobe_seq_start,
2842         .next  = kprobe_seq_next,
2843         .stop  = kprobe_seq_stop,
2844         .show  = show_kprobe_addr
2845 };
2846
2847 DEFINE_SEQ_ATTRIBUTE(kprobes);
2848
2849 /* kprobes/blacklist -- shows which functions can not be probed */
2850 static void *kprobe_blacklist_seq_start(struct seq_file *m, loff_t *pos)
2851 {
2852         mutex_lock(&kprobe_mutex);
2853         return seq_list_start(&kprobe_blacklist, *pos);
2854 }
2855
2856 static void *kprobe_blacklist_seq_next(struct seq_file *m, void *v, loff_t *pos)
2857 {
2858         return seq_list_next(v, &kprobe_blacklist, pos);
2859 }
2860
2861 static int kprobe_blacklist_seq_show(struct seq_file *m, void *v)
2862 {
2863         struct kprobe_blacklist_entry *ent =
2864                 list_entry(v, struct kprobe_blacklist_entry, list);
2865
2866         /*
2867          * If '/proc/kallsyms' is not showing kernel address, we won't
2868          * show them here either.
2869          */
2870         if (!kallsyms_show_value(m->file->f_cred))
2871                 seq_printf(m, "0x%px-0x%px\t%ps\n", NULL, NULL,
2872                            (void *)ent->start_addr);
2873         else
2874                 seq_printf(m, "0x%px-0x%px\t%ps\n", (void *)ent->start_addr,
2875                            (void *)ent->end_addr, (void *)ent->start_addr);
2876         return 0;
2877 }
2878
2879 static void kprobe_blacklist_seq_stop(struct seq_file *f, void *v)
2880 {
2881         mutex_unlock(&kprobe_mutex);
2882 }
2883
2884 static const struct seq_operations kprobe_blacklist_sops = {
2885         .start = kprobe_blacklist_seq_start,
2886         .next  = kprobe_blacklist_seq_next,
2887         .stop  = kprobe_blacklist_seq_stop,
2888         .show  = kprobe_blacklist_seq_show,
2889 };
2890 DEFINE_SEQ_ATTRIBUTE(kprobe_blacklist);
2891
2892 static int arm_all_kprobes(void)
2893 {
2894         struct hlist_head *head;
2895         struct kprobe *p;
2896         unsigned int i, total = 0, errors = 0;
2897         int err, ret = 0;
2898
2899         mutex_lock(&kprobe_mutex);
2900
2901         /* If kprobes are armed, just return */
2902         if (!kprobes_all_disarmed)
2903                 goto already_enabled;
2904
2905         /*
2906          * optimize_kprobe() called by arm_kprobe() checks
2907          * kprobes_all_disarmed, so set kprobes_all_disarmed before
2908          * arm_kprobe.
2909          */
2910         kprobes_all_disarmed = false;
2911         /* Arming kprobes doesn't optimize kprobe itself */
2912         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2913                 head = &kprobe_table[i];
2914                 /* Arm all kprobes on a best-effort basis */
2915                 hlist_for_each_entry(p, head, hlist) {
2916                         if (!kprobe_disabled(p)) {
2917                                 err = arm_kprobe(p);
2918                                 if (err)  {
2919                                         errors++;
2920                                         ret = err;
2921                                 }
2922                                 total++;
2923                         }
2924                 }
2925         }
2926
2927         if (errors)
2928                 pr_warn("Kprobes globally enabled, but failed to enable %d out of %d probes. Please check which kprobes are kept disabled via debugfs.\n",
2929                         errors, total);
2930         else
2931                 pr_info("Kprobes globally enabled\n");
2932
2933 already_enabled:
2934         mutex_unlock(&kprobe_mutex);
2935         return ret;
2936 }
2937
2938 static int disarm_all_kprobes(void)
2939 {
2940         struct hlist_head *head;
2941         struct kprobe *p;
2942         unsigned int i, total = 0, errors = 0;
2943         int err, ret = 0;
2944
2945         mutex_lock(&kprobe_mutex);
2946
2947         /* If kprobes are already disarmed, just return */
2948         if (kprobes_all_disarmed) {
2949                 mutex_unlock(&kprobe_mutex);
2950                 return 0;
2951         }
2952
2953         kprobes_all_disarmed = true;
2954
2955         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2956                 head = &kprobe_table[i];
2957                 /* Disarm all kprobes on a best-effort basis */
2958                 hlist_for_each_entry(p, head, hlist) {
2959                         if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) {
2960                                 err = disarm_kprobe(p, false);
2961                                 if (err) {
2962                                         errors++;
2963                                         ret = err;
2964                                 }
2965                                 total++;
2966                         }
2967                 }
2968         }
2969
2970         if (errors)
2971                 pr_warn("Kprobes globally disabled, but failed to disable %d out of %d probes. Please check which kprobes are kept enabled via debugfs.\n",
2972                         errors, total);
2973         else
2974                 pr_info("Kprobes globally disabled\n");
2975
2976         mutex_unlock(&kprobe_mutex);
2977
2978         /* Wait for disarming all kprobes by optimizer */
2979         wait_for_kprobe_optimizer();
2980
2981         return ret;
2982 }
2983
2984 /*
2985  * XXX: The debugfs bool file interface doesn't allow for callbacks
2986  * when the bool state is switched. We can reuse that facility when
2987  * available
2988  */
2989 static ssize_t read_enabled_file_bool(struct file *file,
2990                char __user *user_buf, size_t count, loff_t *ppos)
2991 {
2992         char buf[3];
2993
2994         if (!kprobes_all_disarmed)
2995                 buf[0] = '1';
2996         else
2997                 buf[0] = '0';
2998         buf[1] = '\n';
2999         buf[2] = 0x00;
3000         return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
3001 }
3002
3003 static ssize_t write_enabled_file_bool(struct file *file,
3004                const char __user *user_buf, size_t count, loff_t *ppos)
3005 {
3006         bool enable;
3007         int ret;
3008
3009         ret = kstrtobool_from_user(user_buf, count, &enable);
3010         if (ret)
3011                 return ret;
3012
3013         ret = enable ? arm_all_kprobes() : disarm_all_kprobes();
3014         if (ret)
3015                 return ret;
3016
3017         return count;
3018 }
3019
3020 static const struct file_operations fops_kp = {
3021         .read =         read_enabled_file_bool,
3022         .write =        write_enabled_file_bool,
3023         .llseek =       default_llseek,
3024 };
3025
3026 static int __init debugfs_kprobe_init(void)
3027 {
3028         struct dentry *dir;
3029
3030         dir = debugfs_create_dir("kprobes", NULL);
3031
3032         debugfs_create_file("list", 0400, dir, NULL, &kprobes_fops);
3033
3034         debugfs_create_file("enabled", 0600, dir, NULL, &fops_kp);
3035
3036         debugfs_create_file("blacklist", 0400, dir, NULL,
3037                             &kprobe_blacklist_fops);
3038
3039         return 0;
3040 }
3041
3042 late_initcall(debugfs_kprobe_init);
3043 #endif /* CONFIG_DEBUG_FS */