kernel/kprobes.c

   1 /*
   2  *  Kernel Probes (KProbes)
   3  *  kernel/kprobes.c
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  *
  19  * Copyright (C) IBM Corporation, 2002, 2004
  20  *
  21  * 2002-Oct     Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
  22  *              Probes initial implementation (includes suggestions from
  23  *              Rusty Russell).
  24  * 2004-Aug     Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
  25  *              hlists and exceptions notifier as suggested by Andi Kleen.
  26  * 2004-July    Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
  27  *              interface to access function arguments.
  28  * 2004-Sep     Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
  29  *              exceptions notifier to be first on the priority list.
  30  * 2005-May     Hien Nguyen <hien@us.ibm.com>, Jim Keniston
  31  *              <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
  32  *              <prasanna@in.ibm.com> added function-return probes.
  33  */
  34 #include <linux/kprobes.h>
  35 #include <linux/hash.h>
  36 #include <linux/init.h>
  37 #include <linux/slab.h>
  38 #include <linux/stddef.h>
  39 #include <linux/module.h>
  40 #include <linux/moduleloader.h>
  41 #include <linux/kallsyms.h>
  42 #include <linux/freezer.h>
  43 #include <linux/seq_file.h>
  44 #include <linux/debugfs.h>
  45 #include <linux/sysctl.h>
  46 #include <linux/kdebug.h>
  47 #include <linux/memory.h>
  48 #include <linux/ftrace.h>
  49 #include <linux/cpu.h>
  50
  51 #include <asm-generic/sections.h>
  52 #include <asm/cacheflush.h>
  53 #include <asm/errno.h>
  54 #include <asm/uaccess.h>
  55
  56 #define KPROBE_HASH_BITS 6
  57 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
  58
  59
  60 /*
  61  * Some oddball architectures like 64bit powerpc have function descriptors
  62  * so this must be overridable.
  63  */
  64 #ifndef kprobe_lookup_name
  65 #define kprobe_lookup_name(name, addr) \
  66         addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
  67 #endif
  68
  69 static int kprobes_initialized;
  70 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
  71 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
  72
  73 /* NOTE: change this value only with kprobe_mutex held */
  74 static bool kprobes_all_disarmed;
  75
  76 static DEFINE_MUTEX(kprobe_mutex);      /* Protects kprobe_table */
  77 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
  78 static struct {
  79         spinlock_t lock ____cacheline_aligned_in_smp;
  80 } kretprobe_table_locks[KPROBE_TABLE_SIZE];
  81
  82 static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
  83 {
  84         return &(kretprobe_table_locks[hash].lock);
  85 }
  86
  87 /*
  88  * Normally, functions that we'd want to prohibit kprobes in, are marked
  89  * __kprobes. But, there are cases where such functions already belong to
  90  * a different section (__sched for preempt_schedule)
  91  *
  92  * For such cases, we now have a blacklist
  93  */
  94 static struct kprobe_blackpoint kprobe_blacklist[] = {
  95         {"preempt_schedule",},
  96         {"native_get_debugreg",},
  97         {"irq_entries_start",},
  98         {"common_interrupt",},
  99         {"mcount",},    /* mcount can be called from everywhere */
 100         {NULL}    /* Terminator */
 101 };
 102
 103 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
 104 /*
 105  * kprobe->ainsn.insn points to the copy of the instruction to be
 106  * single-stepped. x86_64, POWER4 and above have no-exec support and
 107  * stepping on the instruction on a vmalloced/kmalloced/data page
 108  * is a recipe for disaster
 109  */
 110 struct kprobe_insn_page {
 111         struct list_head list;
 112         kprobe_opcode_t *insns;         /* Page of instruction slots */
 113         int nused;
 114         int ngarbage;
 115         char slot_used[];
 116 };
 117
 118 #define KPROBE_INSN_PAGE_SIZE(slots)                    \
 119         (offsetof(struct kprobe_insn_page, slot_used) + \
 120          (sizeof(char) * (slots)))
 121
 122 struct kprobe_insn_cache {
 123         struct list_head pages; /* list of kprobe_insn_page */
 124         size_t insn_size;       /* size of instruction slot */
 125         int nr_garbage;
 126 };
 127
 128 static int slots_per_page(struct kprobe_insn_cache *c)
 129 {
 130         return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
 131 }
 132
 133 enum kprobe_slot_state {
 134         SLOT_CLEAN = 0,
 135         SLOT_DIRTY = 1,
 136         SLOT_USED = 2,
 137 };
 138
 139 static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_slots */
 140 static struct kprobe_insn_cache kprobe_insn_slots = {
 141         .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
 142         .insn_size = MAX_INSN_SIZE,
 143         .nr_garbage = 0,
 144 };
 145 static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c);
 146
 147 /**
 148  * __get_insn_slot() - Find a slot on an executable page for an instruction.
 149  * We allocate an executable page if there's no room on existing ones.
 150  */
 151 static kprobe_opcode_t __kprobes *__get_insn_slot(struct kprobe_insn_cache *c)
 152 {
 153         struct kprobe_insn_page *kip;
 154
 155  retry:
 156         list_for_each_entry(kip, &c->pages, list) {
 157                 if (kip->nused < slots_per_page(c)) {
 158                         int i;
 159                         for (i = 0; i < slots_per_page(c); i++) {
 160                                 if (kip->slot_used[i] == SLOT_CLEAN) {
 161                                         kip->slot_used[i] = SLOT_USED;
 162                                         kip->nused++;
 163                                         return kip->insns + (i * c->insn_size);
 164                                 }
 165                         }
 166                         /* kip->nused is broken. Fix it. */
 167                         kip->nused = slots_per_page(c);
 168                         WARN_ON(1);
 169                 }
 170         }
 171
 172         /* If there are any garbage slots, collect it and try again. */
 173         if (c->nr_garbage && collect_garbage_slots(c) == 0)
 174                 goto retry;
 175
 176         /* All out of space.  Need to allocate a new page. */
 177         kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
 178         if (!kip)
 179                 return NULL;
 180
 181         /*
 182          * Use module_alloc so this page is within +/- 2GB of where the
 183          * kernel image and loaded module images reside. This is required
 184          * so x86_64 can correctly handle the %rip-relative fixups.
 185          */
 186         kip->insns = module_alloc(PAGE_SIZE);
 187         if (!kip->insns) {
 188                 kfree(kip);
 189                 return NULL;
 190         }
 191         INIT_LIST_HEAD(&kip->list);
 192         memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
 193         kip->slot_used[0] = SLOT_USED;
 194         kip->nused = 1;
 195         kip->ngarbage = 0;
 196         list_add(&kip->list, &c->pages);
 197         return kip->insns;
 198 }
 199
 200
 201 kprobe_opcode_t __kprobes *get_insn_slot(void)
 202 {
 203         kprobe_opcode_t *ret = NULL;
 204
 205         mutex_lock(&kprobe_insn_mutex);
 206         ret = __get_insn_slot(&kprobe_insn_slots);
 207         mutex_unlock(&kprobe_insn_mutex);
 208
 209         return ret;
 210 }
 211
 212 /* Return 1 if all garbages are collected, otherwise 0. */
 213 static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
 214 {
 215         kip->slot_used[idx] = SLOT_CLEAN;
 216         kip->nused--;
 217         if (kip->nused == 0) {
 218                 /*
 219                  * Page is no longer in use.  Free it unless
 220                  * it's the last one.  We keep the last one
 221                  * so as not to have to set it up again the
 222                  * next time somebody inserts a probe.
 223                  */
 224                 if (!list_is_singular(&kip->list)) {
 225                         list_del(&kip->list);
 226                         module_free(NULL, kip->insns);
 227                         kfree(kip);
 228                 }
 229                 return 1;
 230         }
 231         return 0;
 232 }
 233
 234 static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c)
 235 {
 236         struct kprobe_insn_page *kip, *next;
 237
 238         /* Ensure no-one is interrupted on the garbages */
 239         synchronize_sched();
 240
 241         list_for_each_entry_safe(kip, next, &c->pages, list) {
 242                 int i;
 243                 if (kip->ngarbage == 0)
 244                         continue;
 245                 kip->ngarbage = 0;      /* we will collect all garbages */
 246                 for (i = 0; i < slots_per_page(c); i++) {
 247                         if (kip->slot_used[i] == SLOT_DIRTY &&
 248                             collect_one_slot(kip, i))
 249                                 break;
 250                 }
 251         }
 252         c->nr_garbage = 0;
 253         return 0;
 254 }
 255
 256 static void __kprobes __free_insn_slot(struct kprobe_insn_cache *c,
 257                                        kprobe_opcode_t *slot, int dirty)
 258 {
 259         struct kprobe_insn_page *kip;
 260
 261         list_for_each_entry(kip, &c->pages, list) {
 262                 long idx = ((long)slot - (long)kip->insns) /
 263                                 (c->insn_size * sizeof(kprobe_opcode_t));
 264                 if (idx >= 0 && idx < slots_per_page(c)) {
 265                         WARN_ON(kip->slot_used[idx] != SLOT_USED);
 266                         if (dirty) {
 267                                 kip->slot_used[idx] = SLOT_DIRTY;
 268                                 kip->ngarbage++;
 269                                 if (++c->nr_garbage > slots_per_page(c))
 270                                         collect_garbage_slots(c);
 271                         } else
 272                                 collect_one_slot(kip, idx);
 273                         return;
 274                 }
 275         }
 276         /* Could not free this slot. */
 277         WARN_ON(1);
 278 }
 279
 280 void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
 281 {
 282         mutex_lock(&kprobe_insn_mutex);
 283         __free_insn_slot(&kprobe_insn_slots, slot, dirty);
 284         mutex_unlock(&kprobe_insn_mutex);
 285 }
 286 #ifdef CONFIG_OPTPROBES
 287 /* For optimized_kprobe buffer */
 288 static DEFINE_MUTEX(kprobe_optinsn_mutex); /* Protects kprobe_optinsn_slots */
 289 static struct kprobe_insn_cache kprobe_optinsn_slots = {
 290         .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
 291         /* .insn_size is initialized later */
 292         .nr_garbage = 0,
 293 };
 294 /* Get a slot for optimized_kprobe buffer */
 295 kprobe_opcode_t __kprobes *get_optinsn_slot(void)
 296 {
 297         kprobe_opcode_t *ret = NULL;
 298
 299         mutex_lock(&kprobe_optinsn_mutex);
 300         ret = __get_insn_slot(&kprobe_optinsn_slots);
 301         mutex_unlock(&kprobe_optinsn_mutex);
 302
 303         return ret;
 304 }
 305
 306 void __kprobes free_optinsn_slot(kprobe_opcode_t * slot, int dirty)
 307 {
 308         mutex_lock(&kprobe_optinsn_mutex);
 309         __free_insn_slot(&kprobe_optinsn_slots, slot, dirty);
 310         mutex_unlock(&kprobe_optinsn_mutex);
 311 }
 312 #endif
 313 #endif
 314
 315 /* We have preemption disabled.. so it is safe to use __ versions */
 316 static inline void set_kprobe_instance(struct kprobe *kp)
 317 {
 318         __get_cpu_var(kprobe_instance) = kp;
 319 }
 320
 321 static inline void reset_kprobe_instance(void)
 322 {
 323         __get_cpu_var(kprobe_instance) = NULL;
 324 }
 325
 326 /*
 327  * This routine is called either:
 328  *      - under the kprobe_mutex - during kprobe_[un]register()
 329  *                              OR
 330  *      - with preemption disabled - from arch/xxx/kernel/kprobes.c
 331  */
 332 struct kprobe __kprobes *get_kprobe(void *addr)
 333 {
 334         struct hlist_head *head;
 335         struct hlist_node *node;
 336         struct kprobe *p;
 337
 338         head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
 339         hlist_for_each_entry_rcu(p, node, head, hlist) {
 340                 if (p->addr == addr)
 341                         return p;
 342         }
 343
 344         return NULL;
 345 }
 346
 347 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
 348
 349 /* Return true if the kprobe is an aggregator */
 350 static inline int kprobe_aggrprobe(struct kprobe *p)
 351 {
 352         return p->pre_handler == aggr_pre_handler;
 353 }
 354
 355 /*
 356  * Keep all fields in the kprobe consistent
 357  */
 358 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
 359 {
 360         memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
 361         memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
 362 }
 363
 364 #ifdef CONFIG_OPTPROBES
 365 /* NOTE: change this value only with kprobe_mutex held */
 366 static bool kprobes_allow_optimization;
 367
 368 /*
 369  * Call all pre_handler on the list, but ignores its return value.
 370  * This must be called from arch-dep optimized caller.
 371  */
 372 void __kprobes opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
 373 {
 374         struct kprobe *kp;
 375
 376         list_for_each_entry_rcu(kp, &p->list, list) {
 377                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
 378                         set_kprobe_instance(kp);
 379                         kp->pre_handler(kp, regs);
 380                 }
 381                 reset_kprobe_instance();
 382         }
 383 }
 384
 385 /* Return true(!0) if the kprobe is ready for optimization. */
 386 static inline int kprobe_optready(struct kprobe *p)
 387 {
 388         struct optimized_kprobe *op;
 389
 390         if (kprobe_aggrprobe(p)) {
 391                 op = container_of(p, struct optimized_kprobe, kp);
 392                 return arch_prepared_optinsn(&op->optinsn);
 393         }
 394
 395         return 0;
 396 }
 397
 398 /*
 399  * Return an optimized kprobe whose optimizing code replaces
 400  * instructions including addr (exclude breakpoint).
 401  */
 402 struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr)
 403 {
 404         int i;
 405         struct kprobe *p = NULL;
 406         struct optimized_kprobe *op;
 407
 408         /* Don't check i == 0, since that is a breakpoint case. */
 409         for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)
 410                 p = get_kprobe((void *)(addr - i));
 411
 412         if (p && kprobe_optready(p)) {
 413                 op = container_of(p, struct optimized_kprobe, kp);
 414                 if (arch_within_optimized_kprobe(op, addr))
 415                         return p;
 416         }
 417
 418         return NULL;
 419 }
 420
 421 /* Optimization staging list, protected by kprobe_mutex */
 422 static LIST_HEAD(optimizing_list);
 423
 424 static void kprobe_optimizer(struct work_struct *work);
 425 static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
 426 #define OPTIMIZE_DELAY 5
 427
 428 /* Kprobe jump optimizer */
 429 static __kprobes void kprobe_optimizer(struct work_struct *work)
 430 {
 431         struct optimized_kprobe *op, *tmp;
 432
 433         /* Lock modules while optimizing kprobes */
 434         mutex_lock(&module_mutex);
 435         mutex_lock(&kprobe_mutex);
 436         if (kprobes_all_disarmed || !kprobes_allow_optimization)
 437                 goto end;
 438
 439         /*
 440          * Wait for quiesence period to ensure all running interrupts
 441          * are done. Because optprobe may modify multiple instructions
 442          * there is a chance that Nth instruction is interrupted. In that
 443          * case, running interrupt can return to 2nd-Nth byte of jump
 444          * instruction. This wait is for avoiding it.
 445          */
 446         synchronize_sched();
 447
 448         /*
 449          * The optimization/unoptimization refers online_cpus via
 450          * stop_machine() and cpu-hotplug modifies online_cpus.
 451          * And same time, text_mutex will be held in cpu-hotplug and here.
 452          * This combination can cause a deadlock (cpu-hotplug try to lock
 453          * text_mutex but stop_machine can not be done because online_cpus
 454          * has been changed)
 455          * To avoid this deadlock, we need to call get_online_cpus()
 456          * for preventing cpu-hotplug outside of text_mutex locking.
 457          */
 458         get_online_cpus();
 459         mutex_lock(&text_mutex);
 460         list_for_each_entry_safe(op, tmp, &optimizing_list, list) {
 461                 WARN_ON(kprobe_disabled(&op->kp));
 462                 if (arch_optimize_kprobe(op) < 0)
 463                         op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 464                 list_del_init(&op->list);
 465         }
 466         mutex_unlock(&text_mutex);
 467         put_online_cpus();
 468 end:
 469         mutex_unlock(&kprobe_mutex);
 470         mutex_unlock(&module_mutex);
 471 }
 472
 473 /* Optimize kprobe if p is ready to be optimized */
 474 static __kprobes void optimize_kprobe(struct kprobe *p)
 475 {
 476         struct optimized_kprobe *op;
 477
 478         /* Check if the kprobe is disabled or not ready for optimization. */
 479         if (!kprobe_optready(p) || !kprobes_allow_optimization ||
 480             (kprobe_disabled(p) || kprobes_all_disarmed))
 481                 return;
 482
 483         /* Both of break_handler and post_handler are not supported. */
 484         if (p->break_handler || p->post_handler)
 485                 return;
 486
 487         op = container_of(p, struct optimized_kprobe, kp);
 488
 489         /* Check there is no other kprobes at the optimized instructions */
 490         if (arch_check_optimized_kprobe(op) < 0)
 491                 return;
 492
 493         /* Check if it is already optimized. */
 494         if (op->kp.flags & KPROBE_FLAG_OPTIMIZED)
 495                 return;
 496
 497         op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
 498         list_add(&op->list, &optimizing_list);
 499         if (!delayed_work_pending(&optimizing_work))
 500                 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
 501 }
 502
 503 /* Unoptimize a kprobe if p is optimized */
 504 static __kprobes void unoptimize_kprobe(struct kprobe *p)
 505 {
 506         struct optimized_kprobe *op;
 507
 508         if ((p->flags & KPROBE_FLAG_OPTIMIZED) && kprobe_aggrprobe(p)) {
 509                 op = container_of(p, struct optimized_kprobe, kp);
 510                 if (!list_empty(&op->list))
 511                         /* Dequeue from the optimization queue */
 512                         list_del_init(&op->list);
 513                 else
 514                         /* Replace jump with break */
 515                         arch_unoptimize_kprobe(op);
 516                 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 517         }
 518 }
 519
 520 /* Remove optimized instructions */
 521 static void __kprobes kill_optimized_kprobe(struct kprobe *p)
 522 {
 523         struct optimized_kprobe *op;
 524
 525         op = container_of(p, struct optimized_kprobe, kp);
 526         if (!list_empty(&op->list)) {
 527                 /* Dequeue from the optimization queue */
 528                 list_del_init(&op->list);
 529                 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 530         }
 531         /* Don't unoptimize, because the target code will be freed. */
 532         arch_remove_optimized_kprobe(op);
 533 }
 534
 535 /* Try to prepare optimized instructions */
 536 static __kprobes void prepare_optimized_kprobe(struct kprobe *p)
 537 {
 538         struct optimized_kprobe *op;
 539
 540         op = container_of(p, struct optimized_kprobe, kp);
 541         arch_prepare_optimized_kprobe(op);
 542 }
 543
 544 /* Free optimized instructions and optimized_kprobe */
 545 static __kprobes void free_aggr_kprobe(struct kprobe *p)
 546 {
 547         struct optimized_kprobe *op;
 548
 549         op = container_of(p, struct optimized_kprobe, kp);
 550         arch_remove_optimized_kprobe(op);
 551         kfree(op);
 552 }
 553
 554 /* Allocate new optimized_kprobe and try to prepare optimized instructions */
 555 static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
 556 {
 557         struct optimized_kprobe *op;
 558
 559         op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
 560         if (!op)
 561                 return NULL;
 562
 563         INIT_LIST_HEAD(&op->list);
 564         op->kp.addr = p->addr;
 565         arch_prepare_optimized_kprobe(op);
 566
 567         return &op->kp;
 568 }
 569
 570 static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
 571
 572 /*
 573  * Prepare an optimized_kprobe and optimize it
 574  * NOTE: p must be a normal registered kprobe
 575  */
 576 static __kprobes void try_to_optimize_kprobe(struct kprobe *p)
 577 {
 578         struct kprobe *ap;
 579         struct optimized_kprobe *op;
 580
 581         ap = alloc_aggr_kprobe(p);
 582         if (!ap)
 583                 return;
 584
 585         op = container_of(ap, struct optimized_kprobe, kp);
 586         if (!arch_prepared_optinsn(&op->optinsn)) {
 587                 /* If failed to setup optimizing, fallback to kprobe */
 588                 free_aggr_kprobe(ap);
 589                 return;
 590         }
 591
 592         init_aggr_kprobe(ap, p);
 593         optimize_kprobe(ap);
 594 }
 595
 596 #ifdef CONFIG_SYSCTL
 597 static void __kprobes optimize_all_kprobes(void)
 598 {
 599         struct hlist_head *head;
 600         struct hlist_node *node;
 601         struct kprobe *p;
 602         unsigned int i;
 603
 604         /* If optimization is already allowed, just return */
 605         if (kprobes_allow_optimization)
 606                 return;
 607
 608         kprobes_allow_optimization = true;
 609         mutex_lock(&text_mutex);
 610         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
 611                 head = &kprobe_table[i];
 612                 hlist_for_each_entry_rcu(p, node, head, hlist)
 613                         if (!kprobe_disabled(p))
 614                                 optimize_kprobe(p);
 615         }
 616         mutex_unlock(&text_mutex);
 617         printk(KERN_INFO "Kprobes globally optimized\n");
 618 }
 619
 620 static void __kprobes unoptimize_all_kprobes(void)
 621 {
 622         struct hlist_head *head;
 623         struct hlist_node *node;
 624         struct kprobe *p;
 625         unsigned int i;
 626
 627         /* If optimization is already prohibited, just return */
 628         if (!kprobes_allow_optimization)
 629                 return;
 630
 631         kprobes_allow_optimization = false;
 632         printk(KERN_INFO "Kprobes globally unoptimized\n");
 633         get_online_cpus();      /* For avoiding text_mutex deadlock */
 634         mutex_lock(&text_mutex);
 635         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
 636                 head = &kprobe_table[i];
 637                 hlist_for_each_entry_rcu(p, node, head, hlist) {
 638                         if (!kprobe_disabled(p))
 639                                 unoptimize_kprobe(p);
 640                 }
 641         }
 642
 643         mutex_unlock(&text_mutex);
 644         put_online_cpus();
 645         /* Allow all currently running kprobes to complete */
 646         synchronize_sched();
 647 }
 648
 649 int sysctl_kprobes_optimization;
 650 int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
 651                                       void __user *buffer, size_t *length,
 652                                       loff_t *ppos)
 653 {
 654         int ret;
 655
 656         mutex_lock(&kprobe_mutex);
 657         sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
 658         ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
 659
 660         if (sysctl_kprobes_optimization)
 661                 optimize_all_kprobes();
 662         else
 663                 unoptimize_all_kprobes();
 664         mutex_unlock(&kprobe_mutex);
 665
 666         return ret;
 667 }
 668 #endif /* CONFIG_SYSCTL */
 669
 670 static void __kprobes __arm_kprobe(struct kprobe *p)
 671 {
 672         struct kprobe *old_p;
 673
 674         /* Check collision with other optimized kprobes */
 675         old_p = get_optimized_kprobe((unsigned long)p->addr);
 676         if (unlikely(old_p))
 677                 unoptimize_kprobe(old_p); /* Fallback to unoptimized kprobe */
 678
 679         arch_arm_kprobe(p);
 680         optimize_kprobe(p);     /* Try to optimize (add kprobe to a list) */
 681 }
 682
 683 static void __kprobes __disarm_kprobe(struct kprobe *p)
 684 {
 685         struct kprobe *old_p;
 686
 687         unoptimize_kprobe(p);   /* Try to unoptimize */
 688         arch_disarm_kprobe(p);
 689
 690         /* If another kprobe was blocked, optimize it. */
 691         old_p = get_optimized_kprobe((unsigned long)p->addr);
 692         if (unlikely(old_p))
 693                 optimize_kprobe(old_p);
 694 }
 695
 696 #else /* !CONFIG_OPTPROBES */
 697
 698 #define optimize_kprobe(p)                      do {} while (0)
 699 #define unoptimize_kprobe(p)                    do {} while (0)
 700 #define kill_optimized_kprobe(p)                do {} while (0)
 701 #define prepare_optimized_kprobe(p)             do {} while (0)
 702 #define try_to_optimize_kprobe(p)               do {} while (0)
 703 #define __arm_kprobe(p)                         arch_arm_kprobe(p)
 704 #define __disarm_kprobe(p)                      arch_disarm_kprobe(p)
 705
 706 static __kprobes void free_aggr_kprobe(struct kprobe *p)
 707 {
 708         kfree(p);
 709 }
 710
 711 static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
 712 {
 713         return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
 714 }
 715 #endif /* CONFIG_OPTPROBES */
 716
 717 /* Arm a kprobe with text_mutex */
 718 static void __kprobes arm_kprobe(struct kprobe *kp)
 719 {
 720         /*
 721          * Here, since __arm_kprobe() doesn't use stop_machine(),
 722          * this doesn't cause deadlock on text_mutex. So, we don't
 723          * need get_online_cpus().
 724          */
 725         mutex_lock(&text_mutex);
 726         __arm_kprobe(kp);
 727         mutex_unlock(&text_mutex);
 728 }
 729
 730 /* Disarm a kprobe with text_mutex */
 731 static void __kprobes disarm_kprobe(struct kprobe *kp)
 732 {
 733         get_online_cpus();      /* For avoiding text_mutex deadlock */
 734         mutex_lock(&text_mutex);
 735         __disarm_kprobe(kp);
 736         mutex_unlock(&text_mutex);
 737         put_online_cpus();
 738 }
 739
 740 /*
 741  * Aggregate handlers for multiple kprobes support - these handlers
 742  * take care of invoking the individual kprobe handlers on p->list
 743  */
 744 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
 745 {
 746         struct kprobe *kp;
 747
 748         list_for_each_entry_rcu(kp, &p->list, list) {
 749                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
 750                         set_kprobe_instance(kp);
 751                         if (kp->pre_handler(kp, regs))
 752                                 return 1;
 753                 }
 754                 reset_kprobe_instance();
 755         }
 756         return 0;
 757 }
 758
 759 static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
 760                                         unsigned long flags)
 761 {
 762         struct kprobe *kp;
 763
 764         list_for_each_entry_rcu(kp, &p->list, list) {
 765                 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
 766                         set_kprobe_instance(kp);
 767                         kp->post_handler(kp, regs, flags);
 768                         reset_kprobe_instance();
 769                 }
 770         }
 771 }
 772
 773 static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
 774                                         int trapnr)
 775 {
 776         struct kprobe *cur = __get_cpu_var(kprobe_instance);
 777
 778         /*
 779          * if we faulted "during" the execution of a user specified
 780          * probe handler, invoke just that probe's fault handler
 781          */
 782         if (cur && cur->fault_handler) {
 783                 if (cur->fault_handler(cur, regs, trapnr))
 784                         return 1;
 785         }
 786         return 0;
 787 }
 788
 789 static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
 790 {
 791         struct kprobe *cur = __get_cpu_var(kprobe_instance);
 792         int ret = 0;
 793
 794         if (cur && cur->break_handler) {
 795                 if (cur->break_handler(cur, regs))
 796                         ret = 1;
 797         }
 798         reset_kprobe_instance();
 799         return ret;
 800 }
 801
 802 /* Walks the list and increments nmissed count for multiprobe case */
 803 void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
 804 {
 805         struct kprobe *kp;
 806         if (!kprobe_aggrprobe(p)) {
 807                 p->nmissed++;
 808         } else {
 809                 list_for_each_entry_rcu(kp, &p->list, list)
 810                         kp->nmissed++;
 811         }
 812         return;
 813 }
 814
 815 void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
 816                                 struct hlist_head *head)
 817 {
 818         struct kretprobe *rp = ri->rp;
 819
 820         /* remove rp inst off the rprobe_inst_table */
 821         hlist_del(&ri->hlist);
 822         INIT_HLIST_NODE(&ri->hlist);
 823         if (likely(rp)) {
 824                 spin_lock(&rp->lock);
 825                 hlist_add_head(&ri->hlist, &rp->free_instances);
 826                 spin_unlock(&rp->lock);
 827         } else
 828                 /* Unregistering */
 829                 hlist_add_head(&ri->hlist, head);
 830 }
 831
 832 void __kprobes kretprobe_hash_lock(struct task_struct *tsk,
 833                          struct hlist_head **head, unsigned long *flags)
 834 {
 835         unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
 836         spinlock_t *hlist_lock;
 837
 838         *head = &kretprobe_inst_table[hash];
 839         hlist_lock = kretprobe_table_lock_ptr(hash);
 840         spin_lock_irqsave(hlist_lock, *flags);
 841 }
 842
 843 static void __kprobes kretprobe_table_lock(unsigned long hash,
 844         unsigned long *flags)
 845 {
 846         spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
 847         spin_lock_irqsave(hlist_lock, *flags);
 848 }
 849
 850 void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,
 851         unsigned long *flags)
 852 {
 853         unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
 854         spinlock_t *hlist_lock;
 855
 856         hlist_lock = kretprobe_table_lock_ptr(hash);
 857         spin_unlock_irqrestore(hlist_lock, *flags);
 858 }
 859
 860 void __kprobes kretprobe_table_unlock(unsigned long hash, unsigned long *flags)
 861 {
 862         spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
 863         spin_unlock_irqrestore(hlist_lock, *flags);
 864 }
 865
 866 /*
 867  * This function is called from finish_task_switch when task tk becomes dead,
 868  * so that we can recycle any function-return probe instances associated
 869  * with this task. These left over instances represent probed functions
 870  * that have been called but will never return.
 871  */
 872 void __kprobes kprobe_flush_task(struct task_struct *tk)
 873 {
 874         struct kretprobe_instance *ri;
 875         struct hlist_head *head, empty_rp;
 876         struct hlist_node *node, *tmp;
 877         unsigned long hash, flags = 0;
 878
 879         if (unlikely(!kprobes_initialized))
 880                 /* Early boot.  kretprobe_table_locks not yet initialized. */
 881                 return;
 882
 883         hash = hash_ptr(tk, KPROBE_HASH_BITS);
 884         head = &kretprobe_inst_table[hash];
 885         kretprobe_table_lock(hash, &flags);
 886         hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
 887                 if (ri->task == tk)
 888                         recycle_rp_inst(ri, &empty_rp);
 889         }
 890         kretprobe_table_unlock(hash, &flags);
 891         INIT_HLIST_HEAD(&empty_rp);
 892         hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
 893                 hlist_del(&ri->hlist);
 894                 kfree(ri);
 895         }
 896 }
 897
 898 static inline void free_rp_inst(struct kretprobe *rp)
 899 {
 900         struct kretprobe_instance *ri;
 901         struct hlist_node *pos, *next;
 902
 903         hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) {
 904                 hlist_del(&ri->hlist);
 905                 kfree(ri);
 906         }
 907 }
 908
 909 static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
 910 {
 911         unsigned long flags, hash;
 912         struct kretprobe_instance *ri;
 913         struct hlist_node *pos, *next;
 914         struct hlist_head *head;
 915
 916         /* No race here */
 917         for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
 918                 kretprobe_table_lock(hash, &flags);
 919                 head = &kretprobe_inst_table[hash];
 920                 hlist_for_each_entry_safe(ri, pos, next, head, hlist) {
 921                         if (ri->rp == rp)
 922                                 ri->rp = NULL;
 923                 }
 924                 kretprobe_table_unlock(hash, &flags);
 925         }
 926         free_rp_inst(rp);
 927 }
 928
 929 /*
 930 * Add the new probe to ap->list. Fail if this is the
 931 * second jprobe at the address - two jprobes can't coexist
 932 */
 933 static int __kprobes add_new_kprobe(struct kprobe *ap, struct kprobe *p)
 934 {
 935         BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
 936
 937         if (p->break_handler || p->post_handler)
 938                 unoptimize_kprobe(ap);  /* Fall back to normal kprobe */
 939
 940         if (p->break_handler) {
 941                 if (ap->break_handler)
 942                         return -EEXIST;
 943                 list_add_tail_rcu(&p->list, &ap->list);
 944                 ap->break_handler = aggr_break_handler;
 945         } else
 946                 list_add_rcu(&p->list, &ap->list);
 947         if (p->post_handler && !ap->post_handler)
 948                 ap->post_handler = aggr_post_handler;
 949
 950         if (kprobe_disabled(ap) && !kprobe_disabled(p)) {
 951                 ap->flags &= ~KPROBE_FLAG_DISABLED;
 952                 if (!kprobes_all_disarmed)
 953                         /* Arm the breakpoint again. */
 954                         __arm_kprobe(ap);
 955         }
 956         return 0;
 957 }
 958
 959 /*
 960  * Fill in the required fields of the "manager kprobe". Replace the
 961  * earlier kprobe in the hlist with the manager kprobe
 962  */
 963 static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
 964 {
 965         /* Copy p's insn slot to ap */
 966         copy_kprobe(p, ap);
 967         flush_insn_slot(ap);
 968         ap->addr = p->addr;
 969         ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
 970         ap->pre_handler = aggr_pre_handler;
 971         ap->fault_handler = aggr_fault_handler;
 972         /* We don't care the kprobe which has gone. */
 973         if (p->post_handler && !kprobe_gone(p))
 974                 ap->post_handler = aggr_post_handler;
 975         if (p->break_handler && !kprobe_gone(p))
 976                 ap->break_handler = aggr_break_handler;
 977
 978         INIT_LIST_HEAD(&ap->list);
 979         INIT_HLIST_NODE(&ap->hlist);
 980
 981         list_add_rcu(&p->list, &ap->list);
 982         hlist_replace_rcu(&p->hlist, &ap->hlist);
 983 }
 984
 985 /*
 986  * This is the second or subsequent kprobe at the address - handle
 987  * the intricacies
 988  */
 989 static int __kprobes register_aggr_kprobe(struct kprobe *old_p,
 990                                           struct kprobe *p)
 991 {
 992         int ret = 0;
 993         struct kprobe *ap = old_p;
 994
 995         if (!kprobe_aggrprobe(old_p)) {
 996                 /* If old_p is not an aggr_kprobe, create new aggr_kprobe. */
 997                 ap = alloc_aggr_kprobe(old_p);
 998                 if (!ap)
 999                         return -ENOMEM;
1000                 init_aggr_kprobe(ap, old_p);
1001         }
1002
1003         if (kprobe_gone(ap)) {
1004                 /*
1005                  * Attempting to insert new probe at the same location that
1006                  * had a probe in the module vaddr area which already
1007                  * freed. So, the instruction slot has already been
1008                  * released. We need a new slot for the new probe.
1009                  */
1010                 ret = arch_prepare_kprobe(ap);
1011                 if (ret)
1012                         /*
1013                          * Even if fail to allocate new slot, don't need to
1014                          * free aggr_probe. It will be used next time, or
1015                          * freed by unregister_kprobe.
1016                          */
1017                         return ret;
1018
1019                 /* Prepare optimized instructions if possible. */
1020                 prepare_optimized_kprobe(ap);
1021
1022                 /*
1023                  * Clear gone flag to prevent allocating new slot again, and
1024                  * set disabled flag because it is not armed yet.
1025                  */
1026                 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1027                             | KPROBE_FLAG_DISABLED;
1028         }
1029
1030         /* Copy ap's insn slot to p */
1031         copy_kprobe(ap, p);
1032         return add_new_kprobe(ap, p);
1033 }
1034
1035 /* Try to disable aggr_kprobe, and return 1 if succeeded.*/
1036 static int __kprobes try_to_disable_aggr_kprobe(struct kprobe *p)
1037 {
1038         struct kprobe *kp;
1039
1040         list_for_each_entry_rcu(kp, &p->list, list) {
1041                 if (!kprobe_disabled(kp))
1042                         /*
1043                          * There is an active probe on the list.
1044                          * We can't disable aggr_kprobe.
1045                          */
1046                         return 0;
1047         }
1048         p->flags |= KPROBE_FLAG_DISABLED;
1049         return 1;
1050 }
1051
1052 static int __kprobes in_kprobes_functions(unsigned long addr)
1053 {
1054         struct kprobe_blackpoint *kb;
1055
1056         if (addr >= (unsigned long)__kprobes_text_start &&
1057             addr < (unsigned long)__kprobes_text_end)
1058                 return -EINVAL;
1059         /*
1060          * If there exists a kprobe_blacklist, verify and
1061          * fail any probe registration in the prohibited area
1062          */
1063         for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1064                 if (kb->start_addr) {
1065                         if (addr >= kb->start_addr &&
1066                             addr < (kb->start_addr + kb->range))
1067                                 return -EINVAL;
1068                 }
1069         }
1070         return 0;
1071 }
1072
1073 /*
1074  * If we have a symbol_name argument, look it up and add the offset field
1075  * to it. This way, we can specify a relative address to a symbol.
1076  */
1077 static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p)
1078 {
1079         kprobe_opcode_t *addr = p->addr;
1080         if (p->symbol_name) {
1081                 if (addr)
1082                         return NULL;
1083                 kprobe_lookup_name(p->symbol_name, addr);
1084         }
1085
1086         if (!addr)
1087                 return NULL;
1088         return (kprobe_opcode_t *)(((char *)addr) + p->offset);
1089 }
1090
1091 /* Check passed kprobe is valid and return kprobe in kprobe_table. */
1092 static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
1093 {
1094         struct kprobe *old_p, *list_p;
1095
1096         old_p = get_kprobe(p->addr);
1097         if (unlikely(!old_p))
1098                 return NULL;
1099
1100         if (p != old_p) {
1101                 list_for_each_entry_rcu(list_p, &old_p->list, list)
1102                         if (list_p == p)
1103                         /* kprobe p is a valid probe */
1104                                 goto valid;
1105                 return NULL;
1106         }
1107 valid:
1108         return old_p;
1109 }
1110
1111 /* Return error if the kprobe is being re-registered */
1112 static inline int check_kprobe_rereg(struct kprobe *p)
1113 {
1114         int ret = 0;
1115         struct kprobe *old_p;
1116
1117         mutex_lock(&kprobe_mutex);
1118         old_p = __get_valid_kprobe(p);
1119         if (old_p)
1120                 ret = -EINVAL;
1121         mutex_unlock(&kprobe_mutex);
1122         return ret;
1123 }
1124
1125 int __kprobes register_kprobe(struct kprobe *p)
1126 {
1127         int ret = 0;
1128         struct kprobe *old_p;
1129         struct module *probed_mod;
1130         kprobe_opcode_t *addr;
1131
1132         addr = kprobe_addr(p);
1133         if (!addr)
1134                 return -EINVAL;
1135         p->addr = addr;
1136
1137         ret = check_kprobe_rereg(p);
1138         if (ret)
1139                 return ret;
1140
1141         preempt_disable();
1142         if (!kernel_text_address((unsigned long) p->addr) ||
1143             in_kprobes_functions((unsigned long) p->addr) ||
1144             ftrace_text_reserved(p->addr, p->addr)) {
1145                 preempt_enable();
1146                 return -EINVAL;
1147         }
1148
1149         /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1150         p->flags &= KPROBE_FLAG_DISABLED;
1151
1152         /*
1153          * Check if are we probing a module.
1154          */
1155         probed_mod = __module_text_address((unsigned long) p->addr);
1156         if (probed_mod) {
1157                 /*
1158                  * We must hold a refcount of the probed module while updating
1159                  * its code to prohibit unexpected unloading.
1160                  */
1161                 if (unlikely(!try_module_get(probed_mod))) {
1162                         preempt_enable();
1163                         return -EINVAL;
1164                 }
1165                 /*
1166                  * If the module freed .init.text, we couldn't insert
1167                  * kprobes in there.
1168                  */
1169                 if (within_module_init((unsigned long)p->addr, probed_mod) &&
1170                     probed_mod->state != MODULE_STATE_COMING) {
1171                         module_put(probed_mod);
1172                         preempt_enable();
1173                         return -EINVAL;
1174                 }
1175         }
1176         preempt_enable();
1177
1178         p->nmissed = 0;
1179         INIT_LIST_HEAD(&p->list);
1180         mutex_lock(&kprobe_mutex);
1181
1182         get_online_cpus();      /* For avoiding text_mutex deadlock. */
1183         mutex_lock(&text_mutex);
1184
1185         old_p = get_kprobe(p->addr);
1186         if (old_p) {
1187                 /* Since this may unoptimize old_p, locking text_mutex. */
1188                 ret = register_aggr_kprobe(old_p, p);
1189                 goto out;
1190         }
1191
1192         ret = arch_prepare_kprobe(p);
1193         if (ret)
1194                 goto out;
1195
1196         INIT_HLIST_NODE(&p->hlist);
1197         hlist_add_head_rcu(&p->hlist,
1198                        &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1199
1200         if (!kprobes_all_disarmed && !kprobe_disabled(p))
1201                 __arm_kprobe(p);
1202
1203         /* Try to optimize kprobe */
1204         try_to_optimize_kprobe(p);
1205
1206 out:
1207         mutex_unlock(&text_mutex);
1208         put_online_cpus();
1209         mutex_unlock(&kprobe_mutex);
1210
1211         if (probed_mod)
1212                 module_put(probed_mod);
1213
1214         return ret;
1215 }
1216 EXPORT_SYMBOL_GPL(register_kprobe);
1217
1218 /*
1219  * Unregister a kprobe without a scheduler synchronization.
1220  */
1221 static int __kprobes __unregister_kprobe_top(struct kprobe *p)
1222 {
1223         struct kprobe *old_p, *list_p;
1224
1225         old_p = __get_valid_kprobe(p);
1226         if (old_p == NULL)
1227                 return -EINVAL;
1228
1229         if (old_p == p ||
1230             (kprobe_aggrprobe(old_p) &&
1231              list_is_singular(&old_p->list))) {
1232                 /*
1233                  * Only probe on the hash list. Disarm only if kprobes are
1234                  * enabled and not gone - otherwise, the breakpoint would
1235                  * already have been removed. We save on flushing icache.
1236                  */
1237                 if (!kprobes_all_disarmed && !kprobe_disabled(old_p))
1238                         disarm_kprobe(old_p);
1239                 hlist_del_rcu(&old_p->hlist);
1240         } else {
1241                 if (p->break_handler && !kprobe_gone(p))
1242                         old_p->break_handler = NULL;
1243                 if (p->post_handler && !kprobe_gone(p)) {
1244                         list_for_each_entry_rcu(list_p, &old_p->list, list) {
1245                                 if ((list_p != p) && (list_p->post_handler))
1246                                         goto noclean;
1247                         }
1248                         old_p->post_handler = NULL;
1249                 }
1250 noclean:
1251                 list_del_rcu(&p->list);
1252                 if (!kprobe_disabled(old_p)) {
1253                         try_to_disable_aggr_kprobe(old_p);
1254                         if (!kprobes_all_disarmed) {
1255                                 if (kprobe_disabled(old_p))
1256                                         disarm_kprobe(old_p);
1257                                 else
1258                                         /* Try to optimize this probe again */
1259                                         optimize_kprobe(old_p);
1260                         }
1261                 }
1262         }
1263         return 0;
1264 }
1265
1266 static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
1267 {
1268         struct kprobe *old_p;
1269
1270         if (list_empty(&p->list))
1271                 arch_remove_kprobe(p);
1272         else if (list_is_singular(&p->list)) {
1273                 /* "p" is the last child of an aggr_kprobe */
1274                 old_p = list_entry(p->list.next, struct kprobe, list);
1275                 list_del(&p->list);
1276                 arch_remove_kprobe(old_p);
1277                 free_aggr_kprobe(old_p);
1278         }
1279 }
1280
1281 int __kprobes register_kprobes(struct kprobe **kps, int num)
1282 {
1283         int i, ret = 0;
1284
1285         if (num <= 0)
1286                 return -EINVAL;
1287         for (i = 0; i < num; i++) {
1288                 ret = register_kprobe(kps[i]);
1289                 if (ret < 0) {
1290                         if (i > 0)
1291                                 unregister_kprobes(kps, i);
1292                         break;
1293                 }
1294         }
1295         return ret;
1296 }
1297 EXPORT_SYMBOL_GPL(register_kprobes);
1298
1299 void __kprobes unregister_kprobe(struct kprobe *p)
1300 {
1301         unregister_kprobes(&p, 1);
1302 }
1303 EXPORT_SYMBOL_GPL(unregister_kprobe);
1304
1305 void __kprobes unregister_kprobes(struct kprobe **kps, int num)
1306 {
1307         int i;
1308
1309         if (num <= 0)
1310                 return;
1311         mutex_lock(&kprobe_mutex);
1312         for (i = 0; i < num; i++)
1313                 if (__unregister_kprobe_top(kps[i]) < 0)
1314                         kps[i]->addr = NULL;
1315         mutex_unlock(&kprobe_mutex);
1316
1317         synchronize_sched();
1318         for (i = 0; i < num; i++)
1319                 if (kps[i]->addr)
1320                         __unregister_kprobe_bottom(kps[i]);
1321 }
1322 EXPORT_SYMBOL_GPL(unregister_kprobes);
1323
1324 static struct notifier_block kprobe_exceptions_nb = {
1325         .notifier_call = kprobe_exceptions_notify,
1326         .priority = 0x7fffffff /* we need to be notified first */
1327 };
1328
1329 unsigned long __weak arch_deref_entry_point(void *entry)
1330 {
1331         return (unsigned long)entry;
1332 }
1333
1334 int __kprobes register_jprobes(struct jprobe **jps, int num)
1335 {
1336         struct jprobe *jp;
1337         int ret = 0, i;
1338
1339         if (num <= 0)
1340                 return -EINVAL;
1341         for (i = 0; i < num; i++) {
1342                 unsigned long addr;
1343                 jp = jps[i];
1344                 addr = arch_deref_entry_point(jp->entry);
1345
1346                 if (!kernel_text_address(addr))
1347                         ret = -EINVAL;
1348                 else {
1349                         /* Todo: Verify probepoint is a function entry point */
1350                         jp->kp.pre_handler = setjmp_pre_handler;
1351                         jp->kp.break_handler = longjmp_break_handler;
1352                         ret = register_kprobe(&jp->kp);
1353                 }
1354                 if (ret < 0) {
1355                         if (i > 0)
1356                                 unregister_jprobes(jps, i);
1357                         break;
1358                 }
1359         }
1360         return ret;
1361 }
1362 EXPORT_SYMBOL_GPL(register_jprobes);
1363
1364 int __kprobes register_jprobe(struct jprobe *jp)
1365 {
1366         return register_jprobes(&jp, 1);
1367 }
1368 EXPORT_SYMBOL_GPL(register_jprobe);
1369
1370 void __kprobes unregister_jprobe(struct jprobe *jp)
1371 {
1372         unregister_jprobes(&jp, 1);
1373 }
1374 EXPORT_SYMBOL_GPL(unregister_jprobe);
1375
1376 void __kprobes unregister_jprobes(struct jprobe **jps, int num)
1377 {
1378         int i;
1379
1380         if (num <= 0)
1381                 return;
1382         mutex_lock(&kprobe_mutex);
1383         for (i = 0; i < num; i++)
1384                 if (__unregister_kprobe_top(&jps[i]->kp) < 0)
1385                         jps[i]->kp.addr = NULL;
1386         mutex_unlock(&kprobe_mutex);
1387
1388         synchronize_sched();
1389         for (i = 0; i < num; i++) {
1390                 if (jps[i]->kp.addr)
1391                         __unregister_kprobe_bottom(&jps[i]->kp);
1392         }
1393 }
1394 EXPORT_SYMBOL_GPL(unregister_jprobes);
1395
1396 #ifdef CONFIG_KRETPROBES
1397 /*
1398  * This kprobe pre_handler is registered with every kretprobe. When probe
1399  * hits it will set up the return probe.
1400  */
1401 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1402                                            struct pt_regs *regs)
1403 {
1404         struct kretprobe *rp = container_of(p, struct kretprobe, kp);
1405         unsigned long hash, flags = 0;
1406         struct kretprobe_instance *ri;
1407
1408         /*TODO: consider to only swap the RA after the last pre_handler fired */
1409         hash = hash_ptr(current, KPROBE_HASH_BITS);
1410         spin_lock_irqsave(&rp->lock, flags);
1411         if (!hlist_empty(&rp->free_instances)) {
1412                 ri = hlist_entry(rp->free_instances.first,
1413                                 struct kretprobe_instance, hlist);
1414                 hlist_del(&ri->hlist);
1415                 spin_unlock_irqrestore(&rp->lock, flags);
1416
1417                 ri->rp = rp;
1418                 ri->task = current;
1419
1420                 if (rp->entry_handler && rp->entry_handler(ri, regs))
1421                         return 0;
1422
1423                 arch_prepare_kretprobe(ri, regs);
1424
1425                 /* XXX(hch): why is there no hlist_move_head? */
1426                 INIT_HLIST_NODE(&ri->hlist);
1427                 kretprobe_table_lock(hash, &flags);
1428                 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
1429                 kretprobe_table_unlock(hash, &flags);
1430         } else {
1431                 rp->nmissed++;
1432                 spin_unlock_irqrestore(&rp->lock, flags);
1433         }
1434         return 0;
1435 }
1436
1437 int __kprobes register_kretprobe(struct kretprobe *rp)
1438 {
1439         int ret = 0;
1440         struct kretprobe_instance *inst;
1441         int i;
1442         void *addr;
1443
1444         if (kretprobe_blacklist_size) {
1445                 addr = kprobe_addr(&rp->kp);
1446                 if (!addr)
1447                         return -EINVAL;
1448
1449                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1450                         if (kretprobe_blacklist[i].addr == addr)
1451                                 return -EINVAL;
1452                 }
1453         }
1454
1455         rp->kp.pre_handler = pre_handler_kretprobe;
1456         rp->kp.post_handler = NULL;
1457         rp->kp.fault_handler = NULL;
1458         rp->kp.break_handler = NULL;
1459
1460         /* Pre-allocate memory for max kretprobe instances */
1461         if (rp->maxactive <= 0) {
1462 #ifdef CONFIG_PREEMPT
1463                 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
1464 #else
1465                 rp->maxactive = num_possible_cpus();
1466 #endif
1467         }
1468         spin_lock_init(&rp->lock);
1469         INIT_HLIST_HEAD(&rp->free_instances);
1470         for (i = 0; i < rp->maxactive; i++) {
1471                 inst = kmalloc(sizeof(struct kretprobe_instance) +
1472                                rp->data_size, GFP_KERNEL);
1473                 if (inst == NULL) {
1474                         free_rp_inst(rp);
1475                         return -ENOMEM;
1476                 }
1477                 INIT_HLIST_NODE(&inst->hlist);
1478                 hlist_add_head(&inst->hlist, &rp->free_instances);
1479         }
1480
1481         rp->nmissed = 0;
1482         /* Establish function entry probe point */
1483         ret = register_kprobe(&rp->kp);
1484         if (ret != 0)
1485                 free_rp_inst(rp);
1486         return ret;
1487 }
1488 EXPORT_SYMBOL_GPL(register_kretprobe);
1489
1490 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1491 {
1492         int ret = 0, i;
1493
1494         if (num <= 0)
1495                 return -EINVAL;
1496         for (i = 0; i < num; i++) {
1497                 ret = register_kretprobe(rps[i]);
1498                 if (ret < 0) {
1499                         if (i > 0)
1500                                 unregister_kretprobes(rps, i);
1501                         break;
1502                 }
1503         }
1504         return ret;
1505 }
1506 EXPORT_SYMBOL_GPL(register_kretprobes);
1507
1508 void __kprobes unregister_kretprobe(struct kretprobe *rp)
1509 {
1510         unregister_kretprobes(&rp, 1);
1511 }
1512 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1513
1514 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1515 {
1516         int i;
1517
1518         if (num <= 0)
1519                 return;
1520         mutex_lock(&kprobe_mutex);
1521         for (i = 0; i < num; i++)
1522                 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1523                         rps[i]->kp.addr = NULL;
1524         mutex_unlock(&kprobe_mutex);
1525
1526         synchronize_sched();
1527         for (i = 0; i < num; i++) {
1528                 if (rps[i]->kp.addr) {
1529                         __unregister_kprobe_bottom(&rps[i]->kp);
1530                         cleanup_rp_inst(rps[i]);
1531                 }
1532         }
1533 }
1534 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1535
1536 #else /* CONFIG_KRETPROBES */
1537 int __kprobes register_kretprobe(struct kretprobe *rp)
1538 {
1539         return -ENOSYS;
1540 }
1541 EXPORT_SYMBOL_GPL(register_kretprobe);
1542
1543 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1544 {
1545         return -ENOSYS;
1546 }
1547 EXPORT_SYMBOL_GPL(register_kretprobes);
1548
1549 void __kprobes unregister_kretprobe(struct kretprobe *rp)
1550 {
1551 }
1552 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1553
1554 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1555 {
1556 }
1557 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1558
1559 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1560                                            struct pt_regs *regs)
1561 {
1562         return 0;
1563 }
1564
1565 #endif /* CONFIG_KRETPROBES */
1566
1567 /* Set the kprobe gone and remove its instruction buffer. */
1568 static void __kprobes kill_kprobe(struct kprobe *p)
1569 {
1570         struct kprobe *kp;
1571
1572         p->flags |= KPROBE_FLAG_GONE;
1573         if (kprobe_aggrprobe(p)) {
1574                 /*
1575                  * If this is an aggr_kprobe, we have to list all the
1576                  * chained probes and mark them GONE.
1577                  */
1578                 list_for_each_entry_rcu(kp, &p->list, list)
1579                         kp->flags |= KPROBE_FLAG_GONE;
1580                 p->post_handler = NULL;
1581                 p->break_handler = NULL;
1582                 kill_optimized_kprobe(p);
1583         }
1584         /*
1585          * Here, we can remove insn_slot safely, because no thread calls
1586          * the original probed function (which will be freed soon) any more.
1587          */
1588         arch_remove_kprobe(p);
1589 }
1590
1591 void __kprobes dump_kprobe(struct kprobe *kp)
1592 {
1593         printk(KERN_WARNING "Dumping kprobe:\n");
1594         printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
1595                kp->symbol_name, kp->addr, kp->offset);
1596 }
1597
1598 /* Module notifier call back, checking kprobes on the module */
1599 static int __kprobes kprobes_module_callback(struct notifier_block *nb,
1600                                              unsigned long val, void *data)
1601 {
1602         struct module *mod = data;
1603         struct hlist_head *head;
1604         struct hlist_node *node;
1605         struct kprobe *p;
1606         unsigned int i;
1607         int checkcore = (val == MODULE_STATE_GOING);
1608
1609         if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
1610                 return NOTIFY_DONE;
1611
1612         /*
1613          * When MODULE_STATE_GOING was notified, both of module .text and
1614          * .init.text sections would be freed. When MODULE_STATE_LIVE was
1615          * notified, only .init.text section would be freed. We need to
1616          * disable kprobes which have been inserted in the sections.
1617          */
1618         mutex_lock(&kprobe_mutex);
1619         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1620                 head = &kprobe_table[i];
1621                 hlist_for_each_entry_rcu(p, node, head, hlist)
1622                         if (within_module_init((unsigned long)p->addr, mod) ||
1623                             (checkcore &&
1624                              within_module_core((unsigned long)p->addr, mod))) {
1625                                 /*
1626                                  * The vaddr this probe is installed will soon
1627                                  * be vfreed buy not synced to disk. Hence,
1628                                  * disarming the breakpoint isn't needed.
1629                                  */
1630                                 kill_kprobe(p);
1631                         }
1632         }
1633         mutex_unlock(&kprobe_mutex);
1634         return NOTIFY_DONE;
1635 }
1636
1637 static struct notifier_block kprobe_module_nb = {
1638         .notifier_call = kprobes_module_callback,
1639         .priority = 0
1640 };
1641
1642 static int __init init_kprobes(void)
1643 {
1644         int i, err = 0;
1645         unsigned long offset = 0, size = 0;
1646         char *modname, namebuf[128];
1647         const char *symbol_name;
1648         void *addr;
1649         struct kprobe_blackpoint *kb;
1650
1651         /* FIXME allocate the probe table, currently defined statically */
1652         /* initialize all list heads */
1653         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1654                 INIT_HLIST_HEAD(&kprobe_table[i]);
1655                 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1656                 spin_lock_init(&(kretprobe_table_locks[i].lock));
1657         }
1658
1659         /*
1660          * Lookup and populate the kprobe_blacklist.
1661          *
1662          * Unlike the kretprobe blacklist, we'll need to determine
1663          * the range of addresses that belong to the said functions,
1664          * since a kprobe need not necessarily be at the beginning
1665          * of a function.
1666          */
1667         for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1668                 kprobe_lookup_name(kb->name, addr);
1669                 if (!addr)
1670                         continue;
1671
1672                 kb->start_addr = (unsigned long)addr;
1673                 symbol_name = kallsyms_lookup(kb->start_addr,
1674                                 &size, &offset, &modname, namebuf);
1675                 if (!symbol_name)
1676                         kb->range = 0;
1677                 else
1678                         kb->range = size;
1679         }
1680
1681         if (kretprobe_blacklist_size) {
1682                 /* lookup the function address from its name */
1683                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1684                         kprobe_lookup_name(kretprobe_blacklist[i].name,
1685                                            kretprobe_blacklist[i].addr);
1686                         if (!kretprobe_blacklist[i].addr)
1687                                 printk("kretprobe: lookup failed: %s\n",
1688                                        kretprobe_blacklist[i].name);
1689                 }
1690         }
1691
1692 #if defined(CONFIG_OPTPROBES)
1693 #if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
1694         /* Init kprobe_optinsn_slots */
1695         kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
1696 #endif
1697         /* By default, kprobes can be optimized */
1698         kprobes_allow_optimization = true;
1699 #endif
1700
1701         /* By default, kprobes are armed */
1702         kprobes_all_disarmed = false;
1703
1704         err = arch_init_kprobes();
1705         if (!err)
1706                 err = register_die_notifier(&kprobe_exceptions_nb);
1707         if (!err)
1708                 err = register_module_notifier(&kprobe_module_nb);
1709
1710         kprobes_initialized = (err == 0);
1711
1712         if (!err)
1713                 init_test_probes();
1714         return err;
1715 }
1716
1717 #ifdef CONFIG_DEBUG_FS
1718 static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p,
1719                 const char *sym, int offset, char *modname, struct kprobe *pp)
1720 {
1721         char *kprobe_type;
1722
1723         if (p->pre_handler == pre_handler_kretprobe)
1724                 kprobe_type = "r";
1725         else if (p->pre_handler == setjmp_pre_handler)
1726                 kprobe_type = "j";
1727         else
1728                 kprobe_type = "k";
1729
1730         if (sym)
1731                 seq_printf(pi, "%p  %s  %s+0x%x  %s ",
1732                         p->addr, kprobe_type, sym, offset,
1733                         (modname ? modname : " "));
1734         else
1735                 seq_printf(pi, "%p  %s  %p ",
1736                         p->addr, kprobe_type, p->addr);
1737
1738         if (!pp)
1739                 pp = p;
1740         seq_printf(pi, "%s%s%s\n",
1741                 (kprobe_gone(p) ? "[GONE]" : ""),
1742                 ((kprobe_disabled(p) && !kprobe_gone(p)) ?  "[DISABLED]" : ""),
1743                 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""));
1744 }
1745
1746 static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos)
1747 {
1748         return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
1749 }
1750
1751 static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
1752 {
1753         (*pos)++;
1754         if (*pos >= KPROBE_TABLE_SIZE)
1755                 return NULL;
1756         return pos;
1757 }
1758
1759 static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v)
1760 {
1761         /* Nothing to do */
1762 }
1763
1764 static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v)
1765 {
1766         struct hlist_head *head;
1767         struct hlist_node *node;
1768         struct kprobe *p, *kp;
1769         const char *sym = NULL;
1770         unsigned int i = *(loff_t *) v;
1771         unsigned long offset = 0;
1772         char *modname, namebuf[128];
1773
1774         head = &kprobe_table[i];
1775         preempt_disable();
1776         hlist_for_each_entry_rcu(p, node, head, hlist) {
1777                 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
1778                                         &offset, &modname, namebuf);
1779                 if (kprobe_aggrprobe(p)) {
1780                         list_for_each_entry_rcu(kp, &p->list, list)
1781                                 report_probe(pi, kp, sym, offset, modname, p);
1782                 } else
1783                         report_probe(pi, p, sym, offset, modname, NULL);
1784         }
1785         preempt_enable();
1786         return 0;
1787 }
1788
1789 static const struct seq_operations kprobes_seq_ops = {
1790         .start = kprobe_seq_start,
1791         .next  = kprobe_seq_next,
1792         .stop  = kprobe_seq_stop,
1793         .show  = show_kprobe_addr
1794 };
1795
1796 static int __kprobes kprobes_open(struct inode *inode, struct file *filp)
1797 {
1798         return seq_open(filp, &kprobes_seq_ops);
1799 }
1800
1801 static const struct file_operations debugfs_kprobes_operations = {
1802         .open           = kprobes_open,
1803         .read           = seq_read,
1804         .llseek         = seq_lseek,
1805         .release        = seq_release,
1806 };
1807
1808 /* Disable one kprobe */
1809 int __kprobes disable_kprobe(struct kprobe *kp)
1810 {
1811         int ret = 0;
1812         struct kprobe *p;
1813
1814         mutex_lock(&kprobe_mutex);
1815
1816         /* Check whether specified probe is valid. */
1817         p = __get_valid_kprobe(kp);
1818         if (unlikely(p == NULL)) {
1819                 ret = -EINVAL;
1820                 goto out;
1821         }
1822
1823         /* If the probe is already disabled (or gone), just return */
1824         if (kprobe_disabled(kp))
1825                 goto out;
1826
1827         kp->flags |= KPROBE_FLAG_DISABLED;
1828         if (p != kp)
1829                 /* When kp != p, p is always enabled. */
1830                 try_to_disable_aggr_kprobe(p);
1831
1832         if (!kprobes_all_disarmed && kprobe_disabled(p))
1833                 disarm_kprobe(p);
1834 out:
1835         mutex_unlock(&kprobe_mutex);
1836         return ret;
1837 }
1838 EXPORT_SYMBOL_GPL(disable_kprobe);
1839
1840 /* Enable one kprobe */
1841 int __kprobes enable_kprobe(struct kprobe *kp)
1842 {
1843         int ret = 0;
1844         struct kprobe *p;
1845
1846         mutex_lock(&kprobe_mutex);
1847
1848         /* Check whether specified probe is valid. */
1849         p = __get_valid_kprobe(kp);
1850         if (unlikely(p == NULL)) {
1851                 ret = -EINVAL;
1852                 goto out;
1853         }
1854
1855         if (kprobe_gone(kp)) {
1856                 /* This kprobe has gone, we couldn't enable it. */
1857                 ret = -EINVAL;
1858                 goto out;
1859         }
1860
1861         if (p != kp)
1862                 kp->flags &= ~KPROBE_FLAG_DISABLED;
1863
1864         if (!kprobes_all_disarmed && kprobe_disabled(p)) {
1865                 p->flags &= ~KPROBE_FLAG_DISABLED;
1866                 arm_kprobe(p);
1867         }
1868 out:
1869         mutex_unlock(&kprobe_mutex);
1870         return ret;
1871 }
1872 EXPORT_SYMBOL_GPL(enable_kprobe);
1873
1874 static void __kprobes arm_all_kprobes(void)
1875 {
1876         struct hlist_head *head;
1877         struct hlist_node *node;
1878         struct kprobe *p;
1879         unsigned int i;
1880
1881         mutex_lock(&kprobe_mutex);
1882
1883         /* If kprobes are armed, just return */
1884         if (!kprobes_all_disarmed)
1885                 goto already_enabled;
1886
1887         /* Arming kprobes doesn't optimize kprobe itself */
1888         mutex_lock(&text_mutex);
1889         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1890                 head = &kprobe_table[i];
1891                 hlist_for_each_entry_rcu(p, node, head, hlist)
1892                         if (!kprobe_disabled(p))
1893                                 __arm_kprobe(p);
1894         }
1895         mutex_unlock(&text_mutex);
1896
1897         kprobes_all_disarmed = false;
1898         printk(KERN_INFO "Kprobes globally enabled\n");
1899
1900 already_enabled:
1901         mutex_unlock(&kprobe_mutex);
1902         return;
1903 }
1904
1905 static void __kprobes disarm_all_kprobes(void)
1906 {
1907         struct hlist_head *head;
1908         struct hlist_node *node;
1909         struct kprobe *p;
1910         unsigned int i;
1911
1912         mutex_lock(&kprobe_mutex);
1913
1914         /* If kprobes are already disarmed, just return */
1915         if (kprobes_all_disarmed)
1916                 goto already_disabled;
1917
1918         kprobes_all_disarmed = true;
1919         printk(KERN_INFO "Kprobes globally disabled\n");
1920
1921         /*
1922          * Here we call get_online_cpus() for avoiding text_mutex deadlock,
1923          * because disarming may also unoptimize kprobes.
1924          */
1925         get_online_cpus();
1926         mutex_lock(&text_mutex);
1927         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1928                 head = &kprobe_table[i];
1929                 hlist_for_each_entry_rcu(p, node, head, hlist) {
1930                         if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p))
1931                                 __disarm_kprobe(p);
1932                 }
1933         }
1934
1935         mutex_unlock(&text_mutex);
1936         put_online_cpus();
1937         mutex_unlock(&kprobe_mutex);
1938         /* Allow all currently running kprobes to complete */
1939         synchronize_sched();
1940         return;
1941
1942 already_disabled:
1943         mutex_unlock(&kprobe_mutex);
1944         return;
1945 }
1946
1947 /*
1948  * XXX: The debugfs bool file interface doesn't allow for callbacks
1949  * when the bool state is switched. We can reuse that facility when
1950  * available
1951  */
1952 static ssize_t read_enabled_file_bool(struct file *file,
1953                char __user *user_buf, size_t count, loff_t *ppos)
1954 {
1955         char buf[3];
1956
1957         if (!kprobes_all_disarmed)
1958                 buf[0] = '1';
1959         else
1960                 buf[0] = '0';
1961         buf[1] = '\n';
1962         buf[2] = 0x00;
1963         return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1964 }
1965
1966 static ssize_t write_enabled_file_bool(struct file *file,
1967                const char __user *user_buf, size_t count, loff_t *ppos)
1968 {
1969         char buf[32];
1970         int buf_size;
1971
1972         buf_size = min(count, (sizeof(buf)-1));
1973         if (copy_from_user(buf, user_buf, buf_size))
1974                 return -EFAULT;
1975
1976         switch (buf[0]) {
1977         case 'y':
1978         case 'Y':
1979         case '1':
1980                 arm_all_kprobes();
1981                 break;
1982         case 'n':
1983         case 'N':
1984         case '0':
1985                 disarm_all_kprobes();
1986                 break;
1987         }
1988
1989         return count;
1990 }
1991
1992 static const struct file_operations fops_kp = {
1993         .read =         read_enabled_file_bool,
1994         .write =        write_enabled_file_bool,
1995 };
1996
1997 static int __kprobes debugfs_kprobe_init(void)
1998 {
1999         struct dentry *dir, *file;
2000         unsigned int value = 1;
2001
2002         dir = debugfs_create_dir("kprobes", NULL);
2003         if (!dir)
2004                 return -ENOMEM;
2005
2006         file = debugfs_create_file("list", 0444, dir, NULL,
2007                                 &debugfs_kprobes_operations);
2008         if (!file) {
2009                 debugfs_remove(dir);
2010                 return -ENOMEM;
2011         }
2012
2013         file = debugfs_create_file("enabled", 0600, dir,
2014                                         &value, &fops_kp);
2015         if (!file) {
2016                 debugfs_remove(dir);
2017                 return -ENOMEM;
2018         }
2019
2020         return 0;
2021 }
2022
2023 late_initcall(debugfs_kprobe_init);
2024 #endif /* CONFIG_DEBUG_FS */
2025
2026 module_init(init_kprobes);
2027
2028 /* defined in arch/.../kernel/kprobes.c */
2029 EXPORT_SYMBOL_GPL(jprobe_return);