kernel/kprobes.c

   1 /*
   2  *  Kernel Probes (KProbes)
   3  *  kernel/kprobes.c
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  *
  19  * Copyright (C) IBM Corporation, 2002, 2004
  20  *
  21  * 2002-Oct     Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
  22  *              Probes initial implementation (includes suggestions from
  23  *              Rusty Russell).
  24  * 2004-Aug     Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
  25  *              hlists and exceptions notifier as suggested by Andi Kleen.
  26  * 2004-July    Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
  27  *              interface to access function arguments.
  28  * 2004-Sep     Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
  29  *              exceptions notifier to be first on the priority list.
  30  * 2005-May     Hien Nguyen <hien@us.ibm.com>, Jim Keniston
  31  *              <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
  32  *              <prasanna@in.ibm.com> added function-return probes.
  33  */
  34 #include <linux/kprobes.h>
  35 #include <linux/hash.h>
  36 #include <linux/init.h>
  37 #include <linux/slab.h>
  38 #include <linux/stddef.h>
  39 #include <linux/module.h>
  40 #include <linux/moduleloader.h>
  41 #include <linux/kallsyms.h>
  42 #include <linux/freezer.h>
  43 #include <linux/seq_file.h>
  44 #include <linux/debugfs.h>
  45 #include <linux/sysctl.h>
  46 #include <linux/kdebug.h>
  47 #include <linux/memory.h>
  48 #include <linux/ftrace.h>
  49 #include <linux/cpu.h>
  50 #include <linux/jump_label.h>
  51
  52 #include <asm-generic/sections.h>
  53 #include <asm/cacheflush.h>
  54 #include <asm/errno.h>
  55 #include <asm/uaccess.h>
  56
  57 #define KPROBE_HASH_BITS 6
  58 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
  59
  60
  61 /*
  62  * Some oddball architectures like 64bit powerpc have function descriptors
  63  * so this must be overridable.
  64  */
  65 #ifndef kprobe_lookup_name
  66 #define kprobe_lookup_name(name, addr) \
  67         addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
  68 #endif
  69
  70 static int kprobes_initialized;
  71 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
  72 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
  73
  74 /* NOTE: change this value only with kprobe_mutex held */
  75 static bool kprobes_all_disarmed;
  76
  77 /* This protects kprobe_table and optimizing_list */
  78 static DEFINE_MUTEX(kprobe_mutex);
  79 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
  80 static struct {
  81         spinlock_t lock ____cacheline_aligned_in_smp;
  82 } kretprobe_table_locks[KPROBE_TABLE_SIZE];
  83
  84 static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
  85 {
  86         return &(kretprobe_table_locks[hash].lock);
  87 }
  88
  89 /*
  90  * Normally, functions that we'd want to prohibit kprobes in, are marked
  91  * __kprobes. But, there are cases where such functions already belong to
  92  * a different section (__sched for preempt_schedule)
  93  *
  94  * For such cases, we now have a blacklist
  95  */
  96 static struct kprobe_blackpoint kprobe_blacklist[] = {
  97         {"preempt_schedule",},
  98         {"native_get_debugreg",},
  99         {"irq_entries_start",},
 100         {"common_interrupt",},
 101         {"mcount",},    /* mcount can be called from everywhere */
 102         {NULL}    /* Terminator */
 103 };
 104
 105 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
 106 /*
 107  * kprobe->ainsn.insn points to the copy of the instruction to be
 108  * single-stepped. x86_64, POWER4 and above have no-exec support and
 109  * stepping on the instruction on a vmalloced/kmalloced/data page
 110  * is a recipe for disaster
 111  */
 112 struct kprobe_insn_page {
 113         struct list_head list;
 114         kprobe_opcode_t *insns;         /* Page of instruction slots */
 115         int nused;
 116         int ngarbage;
 117         char slot_used[];
 118 };
 119
 120 #define KPROBE_INSN_PAGE_SIZE(slots)                    \
 121         (offsetof(struct kprobe_insn_page, slot_used) + \
 122          (sizeof(char) * (slots)))
 123
 124 struct kprobe_insn_cache {
 125         struct list_head pages; /* list of kprobe_insn_page */
 126         size_t insn_size;       /* size of instruction slot */
 127         int nr_garbage;
 128 };
 129
 130 static int slots_per_page(struct kprobe_insn_cache *c)
 131 {
 132         return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
 133 }
 134
 135 enum kprobe_slot_state {
 136         SLOT_CLEAN = 0,
 137         SLOT_DIRTY = 1,
 138         SLOT_USED = 2,
 139 };
 140
 141 static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_slots */
 142 static struct kprobe_insn_cache kprobe_insn_slots = {
 143         .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
 144         .insn_size = MAX_INSN_SIZE,
 145         .nr_garbage = 0,
 146 };
 147 static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c);
 148
 149 /**
 150  * __get_insn_slot() - Find a slot on an executable page for an instruction.
 151  * We allocate an executable page if there's no room on existing ones.
 152  */
 153 static kprobe_opcode_t __kprobes *__get_insn_slot(struct kprobe_insn_cache *c)
 154 {
 155         struct kprobe_insn_page *kip;
 156
 157  retry:
 158         list_for_each_entry(kip, &c->pages, list) {
 159                 if (kip->nused < slots_per_page(c)) {
 160                         int i;
 161                         for (i = 0; i < slots_per_page(c); i++) {
 162                                 if (kip->slot_used[i] == SLOT_CLEAN) {
 163                                         kip->slot_used[i] = SLOT_USED;
 164                                         kip->nused++;
 165                                         return kip->insns + (i * c->insn_size);
 166                                 }
 167                         }
 168                         /* kip->nused is broken. Fix it. */
 169                         kip->nused = slots_per_page(c);
 170                         WARN_ON(1);
 171                 }
 172         }
 173
 174         /* If there are any garbage slots, collect it and try again. */
 175         if (c->nr_garbage && collect_garbage_slots(c) == 0)
 176                 goto retry;
 177
 178         /* All out of space.  Need to allocate a new page. */
 179         kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
 180         if (!kip)
 181                 return NULL;
 182
 183         /*
 184          * Use module_alloc so this page is within +/- 2GB of where the
 185          * kernel image and loaded module images reside. This is required
 186          * so x86_64 can correctly handle the %rip-relative fixups.
 187          */
 188         kip->insns = module_alloc(PAGE_SIZE);
 189         if (!kip->insns) {
 190                 kfree(kip);
 191                 return NULL;
 192         }
 193         INIT_LIST_HEAD(&kip->list);
 194         memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
 195         kip->slot_used[0] = SLOT_USED;
 196         kip->nused = 1;
 197         kip->ngarbage = 0;
 198         list_add(&kip->list, &c->pages);
 199         return kip->insns;
 200 }
 201
 202
 203 kprobe_opcode_t __kprobes *get_insn_slot(void)
 204 {
 205         kprobe_opcode_t *ret = NULL;
 206
 207         mutex_lock(&kprobe_insn_mutex);
 208         ret = __get_insn_slot(&kprobe_insn_slots);
 209         mutex_unlock(&kprobe_insn_mutex);
 210
 211         return ret;
 212 }
 213
 214 /* Return 1 if all garbages are collected, otherwise 0. */
 215 static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
 216 {
 217         kip->slot_used[idx] = SLOT_CLEAN;
 218         kip->nused--;
 219         if (kip->nused == 0) {
 220                 /*
 221                  * Page is no longer in use.  Free it unless
 222                  * it's the last one.  We keep the last one
 223                  * so as not to have to set it up again the
 224                  * next time somebody inserts a probe.
 225                  */
 226                 if (!list_is_singular(&kip->list)) {
 227                         list_del(&kip->list);
 228                         module_free(NULL, kip->insns);
 229                         kfree(kip);
 230                 }
 231                 return 1;
 232         }
 233         return 0;
 234 }
 235
 236 static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c)
 237 {
 238         struct kprobe_insn_page *kip, *next;
 239
 240         /* Ensure no-one is interrupted on the garbages */
 241         synchronize_sched();
 242
 243         list_for_each_entry_safe(kip, next, &c->pages, list) {
 244                 int i;
 245                 if (kip->ngarbage == 0)
 246                         continue;
 247                 kip->ngarbage = 0;      /* we will collect all garbages */
 248                 for (i = 0; i < slots_per_page(c); i++) {
 249                         if (kip->slot_used[i] == SLOT_DIRTY &&
 250                             collect_one_slot(kip, i))
 251                                 break;
 252                 }
 253         }
 254         c->nr_garbage = 0;
 255         return 0;
 256 }
 257
 258 static void __kprobes __free_insn_slot(struct kprobe_insn_cache *c,
 259                                        kprobe_opcode_t *slot, int dirty)
 260 {
 261         struct kprobe_insn_page *kip;
 262
 263         list_for_each_entry(kip, &c->pages, list) {
 264                 long idx = ((long)slot - (long)kip->insns) /
 265                                 (c->insn_size * sizeof(kprobe_opcode_t));
 266                 if (idx >= 0 && idx < slots_per_page(c)) {
 267                         WARN_ON(kip->slot_used[idx] != SLOT_USED);
 268                         if (dirty) {
 269                                 kip->slot_used[idx] = SLOT_DIRTY;
 270                                 kip->ngarbage++;
 271                                 if (++c->nr_garbage > slots_per_page(c))
 272                                         collect_garbage_slots(c);
 273                         } else
 274                                 collect_one_slot(kip, idx);
 275                         return;
 276                 }
 277         }
 278         /* Could not free this slot. */
 279         WARN_ON(1);
 280 }
 281
 282 void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
 283 {
 284         mutex_lock(&kprobe_insn_mutex);
 285         __free_insn_slot(&kprobe_insn_slots, slot, dirty);
 286         mutex_unlock(&kprobe_insn_mutex);
 287 }
 288 #ifdef CONFIG_OPTPROBES
 289 /* For optimized_kprobe buffer */
 290 static DEFINE_MUTEX(kprobe_optinsn_mutex); /* Protects kprobe_optinsn_slots */
 291 static struct kprobe_insn_cache kprobe_optinsn_slots = {
 292         .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
 293         /* .insn_size is initialized later */
 294         .nr_garbage = 0,
 295 };
 296 /* Get a slot for optimized_kprobe buffer */
 297 kprobe_opcode_t __kprobes *get_optinsn_slot(void)
 298 {
 299         kprobe_opcode_t *ret = NULL;
 300
 301         mutex_lock(&kprobe_optinsn_mutex);
 302         ret = __get_insn_slot(&kprobe_optinsn_slots);
 303         mutex_unlock(&kprobe_optinsn_mutex);
 304
 305         return ret;
 306 }
 307
 308 void __kprobes free_optinsn_slot(kprobe_opcode_t * slot, int dirty)
 309 {
 310         mutex_lock(&kprobe_optinsn_mutex);
 311         __free_insn_slot(&kprobe_optinsn_slots, slot, dirty);
 312         mutex_unlock(&kprobe_optinsn_mutex);
 313 }
 314 #endif
 315 #endif
 316
 317 /* We have preemption disabled.. so it is safe to use __ versions */
 318 static inline void set_kprobe_instance(struct kprobe *kp)
 319 {
 320         __get_cpu_var(kprobe_instance) = kp;
 321 }
 322
 323 static inline void reset_kprobe_instance(void)
 324 {
 325         __get_cpu_var(kprobe_instance) = NULL;
 326 }
 327
 328 /*
 329  * This routine is called either:
 330  *      - under the kprobe_mutex - during kprobe_[un]register()
 331  *                              OR
 332  *      - with preemption disabled - from arch/xxx/kernel/kprobes.c
 333  */
 334 struct kprobe __kprobes *get_kprobe(void *addr)
 335 {
 336         struct hlist_head *head;
 337         struct hlist_node *node;
 338         struct kprobe *p;
 339
 340         head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
 341         hlist_for_each_entry_rcu(p, node, head, hlist) {
 342                 if (p->addr == addr)
 343                         return p;
 344         }
 345
 346         return NULL;
 347 }
 348
 349 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
 350
 351 /* Return true if the kprobe is an aggregator */
 352 static inline int kprobe_aggrprobe(struct kprobe *p)
 353 {
 354         return p->pre_handler == aggr_pre_handler;
 355 }
 356
 357 /* Return true(!0) if the kprobe is unused */
 358 static inline int kprobe_unused(struct kprobe *p)
 359 {
 360         return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
 361                list_empty(&p->list);
 362 }
 363
 364 /*
 365  * Keep all fields in the kprobe consistent
 366  */
 367 static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)
 368 {
 369         memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
 370         memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
 371 }
 372
 373 #ifdef CONFIG_OPTPROBES
 374 /* NOTE: change this value only with kprobe_mutex held */
 375 static bool kprobes_allow_optimization;
 376
 377 /*
 378  * Call all pre_handler on the list, but ignores its return value.
 379  * This must be called from arch-dep optimized caller.
 380  */
 381 void __kprobes opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
 382 {
 383         struct kprobe *kp;
 384
 385         list_for_each_entry_rcu(kp, &p->list, list) {
 386                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
 387                         set_kprobe_instance(kp);
 388                         kp->pre_handler(kp, regs);
 389                 }
 390                 reset_kprobe_instance();
 391         }
 392 }
 393
 394 /* Free optimized instructions and optimized_kprobe */
 395 static __kprobes void free_aggr_kprobe(struct kprobe *p)
 396 {
 397         struct optimized_kprobe *op;
 398
 399         op = container_of(p, struct optimized_kprobe, kp);
 400         arch_remove_optimized_kprobe(op);
 401         arch_remove_kprobe(p);
 402         kfree(op);
 403 }
 404
 405 /* Return true(!0) if the kprobe is ready for optimization. */
 406 static inline int kprobe_optready(struct kprobe *p)
 407 {
 408         struct optimized_kprobe *op;
 409
 410         if (kprobe_aggrprobe(p)) {
 411                 op = container_of(p, struct optimized_kprobe, kp);
 412                 return arch_prepared_optinsn(&op->optinsn);
 413         }
 414
 415         return 0;
 416 }
 417
 418 /* Return true(!0) if the kprobe is disarmed. Note: p must be on hash list */
 419 static inline int kprobe_disarmed(struct kprobe *p)
 420 {
 421         struct optimized_kprobe *op;
 422
 423         /* If kprobe is not aggr/opt probe, just return kprobe is disabled */
 424         if (!kprobe_aggrprobe(p))
 425                 return kprobe_disabled(p);
 426
 427         op = container_of(p, struct optimized_kprobe, kp);
 428
 429         return kprobe_disabled(p) && list_empty(&op->list);
 430 }
 431
 432 /* Return true(!0) if the probe is queued on (un)optimizing lists */
 433 static int __kprobes kprobe_queued(struct kprobe *p)
 434 {
 435         struct optimized_kprobe *op;
 436
 437         if (kprobe_aggrprobe(p)) {
 438                 op = container_of(p, struct optimized_kprobe, kp);
 439                 if (!list_empty(&op->list))
 440                         return 1;
 441         }
 442         return 0;
 443 }
 444
 445 /*
 446  * Return an optimized kprobe whose optimizing code replaces
 447  * instructions including addr (exclude breakpoint).
 448  */
 449 static struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr)
 450 {
 451         int i;
 452         struct kprobe *p = NULL;
 453         struct optimized_kprobe *op;
 454
 455         /* Don't check i == 0, since that is a breakpoint case. */
 456         for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)
 457                 p = get_kprobe((void *)(addr - i));
 458
 459         if (p && kprobe_optready(p)) {
 460                 op = container_of(p, struct optimized_kprobe, kp);
 461                 if (arch_within_optimized_kprobe(op, addr))
 462                         return p;
 463         }
 464
 465         return NULL;
 466 }
 467
 468 /* Optimization staging list, protected by kprobe_mutex */
 469 static LIST_HEAD(optimizing_list);
 470 static LIST_HEAD(unoptimizing_list);
 471
 472 static void kprobe_optimizer(struct work_struct *work);
 473 static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
 474 static DECLARE_COMPLETION(optimizer_comp);
 475 #define OPTIMIZE_DELAY 5
 476
 477 /*
 478  * Optimize (replace a breakpoint with a jump) kprobes listed on
 479  * optimizing_list.
 480  */
 481 static __kprobes void do_optimize_kprobes(void)
 482 {
 483         struct optimized_kprobe *op, *tmp;
 484
 485         /* Optimization never be done when disarmed */
 486         if (kprobes_all_disarmed || !kprobes_allow_optimization ||
 487             list_empty(&optimizing_list))
 488                 return;
 489
 490         /*
 491          * The optimization/unoptimization refers online_cpus via
 492          * stop_machine() and cpu-hotplug modifies online_cpus.
 493          * And same time, text_mutex will be held in cpu-hotplug and here.
 494          * This combination can cause a deadlock (cpu-hotplug try to lock
 495          * text_mutex but stop_machine can not be done because online_cpus
 496          * has been changed)
 497          * To avoid this deadlock, we need to call get_online_cpus()
 498          * for preventing cpu-hotplug outside of text_mutex locking.
 499          */
 500         get_online_cpus();
 501         mutex_lock(&text_mutex);
 502         list_for_each_entry_safe(op, tmp, &optimizing_list, list) {
 503                 WARN_ON(kprobe_disabled(&op->kp));
 504                 if (arch_optimize_kprobe(op) < 0)
 505                         op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 506                 list_del_init(&op->list);
 507         }
 508         mutex_unlock(&text_mutex);
 509         put_online_cpus();
 510 }
 511
 512 /*
 513  * Unoptimize (replace a jump with a breakpoint and remove the breakpoint
 514  * if need) kprobes listed on unoptimizing_list.
 515  */
 516 static __kprobes void do_unoptimize_kprobes(struct list_head *free_list)
 517 {
 518         struct optimized_kprobe *op, *tmp;
 519
 520         /* Unoptimization must be done anytime */
 521         if (list_empty(&unoptimizing_list))
 522                 return;
 523
 524         /* Ditto to do_optimize_kprobes */
 525         get_online_cpus();
 526         mutex_lock(&text_mutex);
 527         list_for_each_entry_safe(op, tmp, &unoptimizing_list, list) {
 528                 /* Unoptimize kprobes */
 529                 arch_unoptimize_kprobe(op);
 530                 /* Disarm probes if marked disabled */
 531                 if (kprobe_disabled(&op->kp))
 532                         arch_disarm_kprobe(&op->kp);
 533                 if (kprobe_unused(&op->kp)) {
 534                         /*
 535                          * Remove unused probes from hash list. After waiting
 536                          * for synchronization, these probes are reclaimed.
 537                          * (reclaiming is done by do_free_cleaned_kprobes.)
 538                          */
 539                         hlist_del_rcu(&op->kp.hlist);
 540                         /* Move only unused probes on free_list */
 541                         list_move(&op->list, free_list);
 542                 } else
 543                         list_del_init(&op->list);
 544         }
 545         mutex_unlock(&text_mutex);
 546         put_online_cpus();
 547 }
 548
 549 /* Reclaim all kprobes on the free_list */
 550 static __kprobes void do_free_cleaned_kprobes(struct list_head *free_list)
 551 {
 552         struct optimized_kprobe *op, *tmp;
 553
 554         list_for_each_entry_safe(op, tmp, free_list, list) {
 555                 BUG_ON(!kprobe_unused(&op->kp));
 556                 list_del_init(&op->list);
 557                 free_aggr_kprobe(&op->kp);
 558         }
 559 }
 560
 561 /* Start optimizer after OPTIMIZE_DELAY passed */
 562 static __kprobes void kick_kprobe_optimizer(void)
 563 {
 564         if (!delayed_work_pending(&optimizing_work))
 565                 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
 566 }
 567
 568 /* Kprobe jump optimizer */
 569 static __kprobes void kprobe_optimizer(struct work_struct *work)
 570 {
 571         LIST_HEAD(free_list);
 572
 573         /* Lock modules while optimizing kprobes */
 574         mutex_lock(&module_mutex);
 575         mutex_lock(&kprobe_mutex);
 576
 577         /*
 578          * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
 579          * kprobes before waiting for quiesence period.
 580          */
 581         do_unoptimize_kprobes(&free_list);
 582
 583         /*
 584          * Step 2: Wait for quiesence period to ensure all running interrupts
 585          * are done. Because optprobe may modify multiple instructions
 586          * there is a chance that Nth instruction is interrupted. In that
 587          * case, running interrupt can return to 2nd-Nth byte of jump
 588          * instruction. This wait is for avoiding it.
 589          */
 590         synchronize_sched();
 591
 592         /* Step 3: Optimize kprobes after quiesence period */
 593         do_optimize_kprobes();
 594
 595         /* Step 4: Free cleaned kprobes after quiesence period */
 596         do_free_cleaned_kprobes(&free_list);
 597
 598         mutex_unlock(&kprobe_mutex);
 599         mutex_unlock(&module_mutex);
 600
 601         /* Wake up all waiters */
 602         complete_all(&optimizer_comp);
 603 }
 604
 605 /* Wait for completing optimization and unoptimization */
 606 static __kprobes void wait_for_kprobe_optimizer(void)
 607 {
 608         if (delayed_work_pending(&optimizing_work))
 609                 wait_for_completion(&optimizer_comp);
 610 }
 611
 612 /* Optimize kprobe if p is ready to be optimized */
 613 static __kprobes void optimize_kprobe(struct kprobe *p)
 614 {
 615         struct optimized_kprobe *op;
 616
 617         /* Check if the kprobe is disabled or not ready for optimization. */
 618         if (!kprobe_optready(p) || !kprobes_allow_optimization ||
 619             (kprobe_disabled(p) || kprobes_all_disarmed))
 620                 return;
 621
 622         /* Both of break_handler and post_handler are not supported. */
 623         if (p->break_handler || p->post_handler)
 624                 return;
 625
 626         op = container_of(p, struct optimized_kprobe, kp);
 627
 628         /* Check there is no other kprobes at the optimized instructions */
 629         if (arch_check_optimized_kprobe(op) < 0)
 630                 return;
 631
 632         /* Check if it is already optimized. */
 633         if (op->kp.flags & KPROBE_FLAG_OPTIMIZED)
 634                 return;
 635         op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
 636
 637         if (!list_empty(&op->list))
 638                 /* This is under unoptimizing. Just dequeue the probe */
 639                 list_del_init(&op->list);
 640         else {
 641                 list_add(&op->list, &optimizing_list);
 642                 kick_kprobe_optimizer();
 643         }
 644 }
 645
 646 /* Short cut to direct unoptimizing */
 647 static __kprobes void force_unoptimize_kprobe(struct optimized_kprobe *op)
 648 {
 649         get_online_cpus();
 650         arch_unoptimize_kprobe(op);
 651         put_online_cpus();
 652         if (kprobe_disabled(&op->kp))
 653                 arch_disarm_kprobe(&op->kp);
 654 }
 655
 656 /* Unoptimize a kprobe if p is optimized */
 657 static __kprobes void unoptimize_kprobe(struct kprobe *p, bool force)
 658 {
 659         struct optimized_kprobe *op;
 660
 661         if (!kprobe_aggrprobe(p) || kprobe_disarmed(p))
 662                 return; /* This is not an optprobe nor optimized */
 663
 664         op = container_of(p, struct optimized_kprobe, kp);
 665         if (!kprobe_optimized(p)) {
 666                 /* Unoptimized or unoptimizing case */
 667                 if (force && !list_empty(&op->list)) {
 668                         /*
 669                          * Only if this is unoptimizing kprobe and forced,
 670                          * forcibly unoptimize it. (No need to unoptimize
 671                          * unoptimized kprobe again :)
 672                          */
 673                         list_del_init(&op->list);
 674                         force_unoptimize_kprobe(op);
 675                 }
 676                 return;
 677         }
 678
 679         op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 680         if (!list_empty(&op->list)) {
 681                 /* Dequeue from the optimization queue */
 682                 list_del_init(&op->list);
 683                 return;
 684         }
 685         /* Optimized kprobe case */
 686         if (force)
 687                 /* Forcibly update the code: this is a special case */
 688                 force_unoptimize_kprobe(op);
 689         else {
 690                 list_add(&op->list, &unoptimizing_list);
 691                 kick_kprobe_optimizer();
 692         }
 693 }
 694
 695 /* Cancel unoptimizing for reusing */
 696 static void reuse_unused_kprobe(struct kprobe *ap)
 697 {
 698         struct optimized_kprobe *op;
 699
 700         BUG_ON(!kprobe_unused(ap));
 701         /*
 702          * Unused kprobe MUST be on the way of delayed unoptimizing (means
 703          * there is still a relative jump) and disabled.
 704          */
 705         op = container_of(ap, struct optimized_kprobe, kp);
 706         if (unlikely(list_empty(&op->list)))
 707                 printk(KERN_WARNING "Warning: found a stray unused "
 708                         "aggrprobe@%p\n", ap->addr);
 709         /* Enable the probe again */
 710         ap->flags &= ~KPROBE_FLAG_DISABLED;
 711         /* Optimize it again (remove from op->list) */
 712         BUG_ON(!kprobe_optready(ap));
 713         optimize_kprobe(ap);
 714 }
 715
 716 /* Remove optimized instructions */
 717 static void __kprobes kill_optimized_kprobe(struct kprobe *p)
 718 {
 719         struct optimized_kprobe *op;
 720
 721         op = container_of(p, struct optimized_kprobe, kp);
 722         if (!list_empty(&op->list))
 723                 /* Dequeue from the (un)optimization queue */
 724                 list_del_init(&op->list);
 725
 726         op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
 727         /* Don't touch the code, because it is already freed. */
 728         arch_remove_optimized_kprobe(op);
 729 }
 730
 731 /* Try to prepare optimized instructions */
 732 static __kprobes void prepare_optimized_kprobe(struct kprobe *p)
 733 {
 734         struct optimized_kprobe *op;
 735
 736         op = container_of(p, struct optimized_kprobe, kp);
 737         arch_prepare_optimized_kprobe(op);
 738 }
 739
 740 /* Allocate new optimized_kprobe and try to prepare optimized instructions */
 741 static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
 742 {
 743         struct optimized_kprobe *op;
 744
 745         op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
 746         if (!op)
 747                 return NULL;
 748
 749         INIT_LIST_HEAD(&op->list);
 750         op->kp.addr = p->addr;
 751         arch_prepare_optimized_kprobe(op);
 752
 753         return &op->kp;
 754 }
 755
 756 static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
 757
 758 /*
 759  * Prepare an optimized_kprobe and optimize it
 760  * NOTE: p must be a normal registered kprobe
 761  */
 762 static __kprobes void try_to_optimize_kprobe(struct kprobe *p)
 763 {
 764         struct kprobe *ap;
 765         struct optimized_kprobe *op;
 766
 767         ap = alloc_aggr_kprobe(p);
 768         if (!ap)
 769                 return;
 770
 771         op = container_of(ap, struct optimized_kprobe, kp);
 772         if (!arch_prepared_optinsn(&op->optinsn)) {
 773                 /* If failed to setup optimizing, fallback to kprobe */
 774                 arch_remove_optimized_kprobe(op);
 775                 kfree(op);
 776                 return;
 777         }
 778
 779         init_aggr_kprobe(ap, p);
 780         optimize_kprobe(ap);
 781 }
 782
 783 #ifdef CONFIG_SYSCTL
 784 /* This should be called with kprobe_mutex locked */
 785 static void __kprobes optimize_all_kprobes(void)
 786 {
 787         struct hlist_head *head;
 788         struct hlist_node *node;
 789         struct kprobe *p;
 790         unsigned int i;
 791
 792         /* If optimization is already allowed, just return */
 793         if (kprobes_allow_optimization)
 794                 return;
 795
 796         kprobes_allow_optimization = true;
 797         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
 798                 head = &kprobe_table[i];
 799                 hlist_for_each_entry_rcu(p, node, head, hlist)
 800                         if (!kprobe_disabled(p))
 801                                 optimize_kprobe(p);
 802         }
 803         printk(KERN_INFO "Kprobes globally optimized\n");
 804 }
 805
 806 /* This should be called with kprobe_mutex locked */
 807 static void __kprobes unoptimize_all_kprobes(void)
 808 {
 809         struct hlist_head *head;
 810         struct hlist_node *node;
 811         struct kprobe *p;
 812         unsigned int i;
 813
 814         /* If optimization is already prohibited, just return */
 815         if (!kprobes_allow_optimization)
 816                 return;
 817
 818         kprobes_allow_optimization = false;
 819         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
 820                 head = &kprobe_table[i];
 821                 hlist_for_each_entry_rcu(p, node, head, hlist) {
 822                         if (!kprobe_disabled(p))
 823                                 unoptimize_kprobe(p, false);
 824                 }
 825         }
 826         /* Wait for unoptimizing completion */
 827         wait_for_kprobe_optimizer();
 828         printk(KERN_INFO "Kprobes globally unoptimized\n");
 829 }
 830
 831 int sysctl_kprobes_optimization;
 832 int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
 833                                       void __user *buffer, size_t *length,
 834                                       loff_t *ppos)
 835 {
 836         int ret;
 837
 838         mutex_lock(&kprobe_mutex);
 839         sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
 840         ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
 841
 842         if (sysctl_kprobes_optimization)
 843                 optimize_all_kprobes();
 844         else
 845                 unoptimize_all_kprobes();
 846         mutex_unlock(&kprobe_mutex);
 847
 848         return ret;
 849 }
 850 #endif /* CONFIG_SYSCTL */
 851
 852 /* Put a breakpoint for a probe. Must be called with text_mutex locked */
 853 static void __kprobes __arm_kprobe(struct kprobe *p)
 854 {
 855         struct kprobe *_p;
 856
 857         /* Check collision with other optimized kprobes */
 858         _p = get_optimized_kprobe((unsigned long)p->addr);
 859         if (unlikely(_p))
 860                 /* Fallback to unoptimized kprobe */
 861                 unoptimize_kprobe(_p, true);
 862
 863         arch_arm_kprobe(p);
 864         optimize_kprobe(p);     /* Try to optimize (add kprobe to a list) */
 865 }
 866
 867 /* Remove the breakpoint of a probe. Must be called with text_mutex locked */
 868 static void __kprobes __disarm_kprobe(struct kprobe *p, bool reopt)
 869 {
 870         struct kprobe *_p;
 871
 872         unoptimize_kprobe(p, false);    /* Try to unoptimize */
 873
 874         if (!kprobe_queued(p)) {
 875                 arch_disarm_kprobe(p);
 876                 /* If another kprobe was blocked, optimize it. */
 877                 _p = get_optimized_kprobe((unsigned long)p->addr);
 878                 if (unlikely(_p) && reopt)
 879                         optimize_kprobe(_p);
 880         }
 881         /* TODO: reoptimize others after unoptimized this probe */
 882 }
 883
 884 #else /* !CONFIG_OPTPROBES */
 885
 886 #define optimize_kprobe(p)                      do {} while (0)
 887 #define unoptimize_kprobe(p, f)                 do {} while (0)
 888 #define kill_optimized_kprobe(p)                do {} while (0)
 889 #define prepare_optimized_kprobe(p)             do {} while (0)
 890 #define try_to_optimize_kprobe(p)               do {} while (0)
 891 #define __arm_kprobe(p)                         arch_arm_kprobe(p)
 892 #define __disarm_kprobe(p, o)                   arch_disarm_kprobe(p)
 893 #define kprobe_disarmed(p)                      kprobe_disabled(p)
 894 #define wait_for_kprobe_optimizer()             do {} while (0)
 895
 896 /* There should be no unused kprobes can be reused without optimization */
 897 static void reuse_unused_kprobe(struct kprobe *ap)
 898 {
 899         printk(KERN_ERR "Error: There should be no unused kprobe here.\n");
 900         BUG_ON(kprobe_unused(ap));
 901 }
 902
 903 static __kprobes void free_aggr_kprobe(struct kprobe *p)
 904 {
 905         arch_remove_kprobe(p);
 906         kfree(p);
 907 }
 908
 909 static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
 910 {
 911         return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
 912 }
 913 #endif /* CONFIG_OPTPROBES */
 914
 915 /* Arm a kprobe with text_mutex */
 916 static void __kprobes arm_kprobe(struct kprobe *kp)
 917 {
 918         /*
 919          * Here, since __arm_kprobe() doesn't use stop_machine(),
 920          * this doesn't cause deadlock on text_mutex. So, we don't
 921          * need get_online_cpus().
 922          */
 923         mutex_lock(&text_mutex);
 924         __arm_kprobe(kp);
 925         mutex_unlock(&text_mutex);
 926 }
 927
 928 /* Disarm a kprobe with text_mutex */
 929 static void __kprobes disarm_kprobe(struct kprobe *kp)
 930 {
 931         /* Ditto */
 932         mutex_lock(&text_mutex);
 933         __disarm_kprobe(kp, true);
 934         mutex_unlock(&text_mutex);
 935 }
 936
 937 /*
 938  * Aggregate handlers for multiple kprobes support - these handlers
 939  * take care of invoking the individual kprobe handlers on p->list
 940  */
 941 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
 942 {
 943         struct kprobe *kp;
 944
 945         list_for_each_entry_rcu(kp, &p->list, list) {
 946                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
 947                         set_kprobe_instance(kp);
 948                         if (kp->pre_handler(kp, regs))
 949                                 return 1;
 950                 }
 951                 reset_kprobe_instance();
 952         }
 953         return 0;
 954 }
 955
 956 static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
 957                                         unsigned long flags)
 958 {
 959         struct kprobe *kp;
 960
 961         list_for_each_entry_rcu(kp, &p->list, list) {
 962                 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
 963                         set_kprobe_instance(kp);
 964                         kp->post_handler(kp, regs, flags);
 965                         reset_kprobe_instance();
 966                 }
 967         }
 968 }
 969
 970 static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
 971                                         int trapnr)
 972 {
 973         struct kprobe *cur = __get_cpu_var(kprobe_instance);
 974
 975         /*
 976          * if we faulted "during" the execution of a user specified
 977          * probe handler, invoke just that probe's fault handler
 978          */
 979         if (cur && cur->fault_handler) {
 980                 if (cur->fault_handler(cur, regs, trapnr))
 981                         return 1;
 982         }
 983         return 0;
 984 }
 985
 986 static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
 987 {
 988         struct kprobe *cur = __get_cpu_var(kprobe_instance);
 989         int ret = 0;
 990
 991         if (cur && cur->break_handler) {
 992                 if (cur->break_handler(cur, regs))
 993                         ret = 1;
 994         }
 995         reset_kprobe_instance();
 996         return ret;
 997 }
 998
 999 /* Walks the list and increments nmissed count for multiprobe case */
1000 void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
1001 {
1002         struct kprobe *kp;
1003         if (!kprobe_aggrprobe(p)) {
1004                 p->nmissed++;
1005         } else {
1006                 list_for_each_entry_rcu(kp, &p->list, list)
1007                         kp->nmissed++;
1008         }
1009         return;
1010 }
1011
1012 void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
1013                                 struct hlist_head *head)
1014 {
1015         struct kretprobe *rp = ri->rp;
1016
1017         /* remove rp inst off the rprobe_inst_table */
1018         hlist_del(&ri->hlist);
1019         INIT_HLIST_NODE(&ri->hlist);
1020         if (likely(rp)) {
1021                 spin_lock(&rp->lock);
1022                 hlist_add_head(&ri->hlist, &rp->free_instances);
1023                 spin_unlock(&rp->lock);
1024         } else
1025                 /* Unregistering */
1026                 hlist_add_head(&ri->hlist, head);
1027 }
1028
1029 void __kprobes kretprobe_hash_lock(struct task_struct *tsk,
1030                          struct hlist_head **head, unsigned long *flags)
1031 __acquires(hlist_lock)
1032 {
1033         unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
1034         spinlock_t *hlist_lock;
1035
1036         *head = &kretprobe_inst_table[hash];
1037         hlist_lock = kretprobe_table_lock_ptr(hash);
1038         spin_lock_irqsave(hlist_lock, *flags);
1039 }
1040
1041 static void __kprobes kretprobe_table_lock(unsigned long hash,
1042         unsigned long *flags)
1043 __acquires(hlist_lock)
1044 {
1045         spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1046         spin_lock_irqsave(hlist_lock, *flags);
1047 }
1048
1049 void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,
1050         unsigned long *flags)
1051 __releases(hlist_lock)
1052 {
1053         unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
1054         spinlock_t *hlist_lock;
1055
1056         hlist_lock = kretprobe_table_lock_ptr(hash);
1057         spin_unlock_irqrestore(hlist_lock, *flags);
1058 }
1059
1060 static void __kprobes kretprobe_table_unlock(unsigned long hash,
1061        unsigned long *flags)
1062 __releases(hlist_lock)
1063 {
1064         spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1065         spin_unlock_irqrestore(hlist_lock, *flags);
1066 }
1067
1068 /*
1069  * This function is called from finish_task_switch when task tk becomes dead,
1070  * so that we can recycle any function-return probe instances associated
1071  * with this task. These left over instances represent probed functions
1072  * that have been called but will never return.
1073  */
1074 void __kprobes kprobe_flush_task(struct task_struct *tk)
1075 {
1076         struct kretprobe_instance *ri;
1077         struct hlist_head *head, empty_rp;
1078         struct hlist_node *node, *tmp;
1079         unsigned long hash, flags = 0;
1080
1081         if (unlikely(!kprobes_initialized))
1082                 /* Early boot.  kretprobe_table_locks not yet initialized. */
1083                 return;
1084
1085         hash = hash_ptr(tk, KPROBE_HASH_BITS);
1086         head = &kretprobe_inst_table[hash];
1087         kretprobe_table_lock(hash, &flags);
1088         hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
1089                 if (ri->task == tk)
1090                         recycle_rp_inst(ri, &empty_rp);
1091         }
1092         kretprobe_table_unlock(hash, &flags);
1093         INIT_HLIST_HEAD(&empty_rp);
1094         hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
1095                 hlist_del(&ri->hlist);
1096                 kfree(ri);
1097         }
1098 }
1099
1100 static inline void free_rp_inst(struct kretprobe *rp)
1101 {
1102         struct kretprobe_instance *ri;
1103         struct hlist_node *pos, *next;
1104
1105         hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) {
1106                 hlist_del(&ri->hlist);
1107                 kfree(ri);
1108         }
1109 }
1110
1111 static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
1112 {
1113         unsigned long flags, hash;
1114         struct kretprobe_instance *ri;
1115         struct hlist_node *pos, *next;
1116         struct hlist_head *head;
1117
1118         /* No race here */
1119         for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
1120                 kretprobe_table_lock(hash, &flags);
1121                 head = &kretprobe_inst_table[hash];
1122                 hlist_for_each_entry_safe(ri, pos, next, head, hlist) {
1123                         if (ri->rp == rp)
1124                                 ri->rp = NULL;
1125                 }
1126                 kretprobe_table_unlock(hash, &flags);
1127         }
1128         free_rp_inst(rp);
1129 }
1130
1131 /*
1132 * Add the new probe to ap->list. Fail if this is the
1133 * second jprobe at the address - two jprobes can't coexist
1134 */
1135 static int __kprobes add_new_kprobe(struct kprobe *ap, struct kprobe *p)
1136 {
1137         BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
1138
1139         if (p->break_handler || p->post_handler)
1140                 unoptimize_kprobe(ap, true);    /* Fall back to normal kprobe */
1141
1142         if (p->break_handler) {
1143                 if (ap->break_handler)
1144                         return -EEXIST;
1145                 list_add_tail_rcu(&p->list, &ap->list);
1146                 ap->break_handler = aggr_break_handler;
1147         } else
1148                 list_add_rcu(&p->list, &ap->list);
1149         if (p->post_handler && !ap->post_handler)
1150                 ap->post_handler = aggr_post_handler;
1151
1152         if (kprobe_disabled(ap) && !kprobe_disabled(p)) {
1153                 ap->flags &= ~KPROBE_FLAG_DISABLED;
1154                 if (!kprobes_all_disarmed)
1155                         /* Arm the breakpoint again. */
1156                         __arm_kprobe(ap);
1157         }
1158         return 0;
1159 }
1160
1161 /*
1162  * Fill in the required fields of the "manager kprobe". Replace the
1163  * earlier kprobe in the hlist with the manager kprobe
1164  */
1165 static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
1166 {
1167         /* Copy p's insn slot to ap */
1168         copy_kprobe(p, ap);
1169         flush_insn_slot(ap);
1170         ap->addr = p->addr;
1171         ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
1172         ap->pre_handler = aggr_pre_handler;
1173         ap->fault_handler = aggr_fault_handler;
1174         /* We don't care the kprobe which has gone. */
1175         if (p->post_handler && !kprobe_gone(p))
1176                 ap->post_handler = aggr_post_handler;
1177         if (p->break_handler && !kprobe_gone(p))
1178                 ap->break_handler = aggr_break_handler;
1179
1180         INIT_LIST_HEAD(&ap->list);
1181         INIT_HLIST_NODE(&ap->hlist);
1182
1183         list_add_rcu(&p->list, &ap->list);
1184         hlist_replace_rcu(&p->hlist, &ap->hlist);
1185 }
1186
1187 /*
1188  * This is the second or subsequent kprobe at the address - handle
1189  * the intricacies
1190  */
1191 static int __kprobes register_aggr_kprobe(struct kprobe *orig_p,
1192                                           struct kprobe *p)
1193 {
1194         int ret = 0;
1195         struct kprobe *ap = orig_p;
1196
1197         if (!kprobe_aggrprobe(orig_p)) {
1198                 /* If orig_p is not an aggr_kprobe, create new aggr_kprobe. */
1199                 ap = alloc_aggr_kprobe(orig_p);
1200                 if (!ap)
1201                         return -ENOMEM;
1202                 init_aggr_kprobe(ap, orig_p);
1203         } else if (kprobe_unused(ap))
1204                 /* This probe is going to die. Rescue it */
1205                 reuse_unused_kprobe(ap);
1206
1207         if (kprobe_gone(ap)) {
1208                 /*
1209                  * Attempting to insert new probe at the same location that
1210                  * had a probe in the module vaddr area which already
1211                  * freed. So, the instruction slot has already been
1212                  * released. We need a new slot for the new probe.
1213                  */
1214                 ret = arch_prepare_kprobe(ap);
1215                 if (ret)
1216                         /*
1217                          * Even if fail to allocate new slot, don't need to
1218                          * free aggr_probe. It will be used next time, or
1219                          * freed by unregister_kprobe.
1220                          */
1221                         return ret;
1222
1223                 /* Prepare optimized instructions if possible. */
1224                 prepare_optimized_kprobe(ap);
1225
1226                 /*
1227                  * Clear gone flag to prevent allocating new slot again, and
1228                  * set disabled flag because it is not armed yet.
1229                  */
1230                 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1231                             | KPROBE_FLAG_DISABLED;
1232         }
1233
1234         /* Copy ap's insn slot to p */
1235         copy_kprobe(ap, p);
1236         return add_new_kprobe(ap, p);
1237 }
1238
1239 static int __kprobes in_kprobes_functions(unsigned long addr)
1240 {
1241         struct kprobe_blackpoint *kb;
1242
1243         if (addr >= (unsigned long)__kprobes_text_start &&
1244             addr < (unsigned long)__kprobes_text_end)
1245                 return -EINVAL;
1246         /*
1247          * If there exists a kprobe_blacklist, verify and
1248          * fail any probe registration in the prohibited area
1249          */
1250         for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1251                 if (kb->start_addr) {
1252                         if (addr >= kb->start_addr &&
1253                             addr < (kb->start_addr + kb->range))
1254                                 return -EINVAL;
1255                 }
1256         }
1257         return 0;
1258 }
1259
1260 /*
1261  * If we have a symbol_name argument, look it up and add the offset field
1262  * to it. This way, we can specify a relative address to a symbol.
1263  */
1264 static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p)
1265 {
1266         kprobe_opcode_t *addr = p->addr;
1267         if (p->symbol_name) {
1268                 if (addr)
1269                         return NULL;
1270                 kprobe_lookup_name(p->symbol_name, addr);
1271         }
1272
1273         if (!addr)
1274                 return NULL;
1275         return (kprobe_opcode_t *)(((char *)addr) + p->offset);
1276 }
1277
1278 /* Check passed kprobe is valid and return kprobe in kprobe_table. */
1279 static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
1280 {
1281         struct kprobe *ap, *list_p;
1282
1283         ap = get_kprobe(p->addr);
1284         if (unlikely(!ap))
1285                 return NULL;
1286
1287         if (p != ap) {
1288                 list_for_each_entry_rcu(list_p, &ap->list, list)
1289                         if (list_p == p)
1290                         /* kprobe p is a valid probe */
1291                                 goto valid;
1292                 return NULL;
1293         }
1294 valid:
1295         return ap;
1296 }
1297
1298 /* Return error if the kprobe is being re-registered */
1299 static inline int check_kprobe_rereg(struct kprobe *p)
1300 {
1301         int ret = 0;
1302
1303         mutex_lock(&kprobe_mutex);
1304         if (__get_valid_kprobe(p))
1305                 ret = -EINVAL;
1306         mutex_unlock(&kprobe_mutex);
1307
1308         return ret;
1309 }
1310
1311 int __kprobes register_kprobe(struct kprobe *p)
1312 {
1313         int ret = 0;
1314         struct kprobe *old_p;
1315         struct module *probed_mod;
1316         kprobe_opcode_t *addr;
1317
1318         addr = kprobe_addr(p);
1319         if (!addr)
1320                 return -EINVAL;
1321         p->addr = addr;
1322
1323         ret = check_kprobe_rereg(p);
1324         if (ret)
1325                 return ret;
1326
1327         jump_label_lock();
1328         preempt_disable();
1329         if (!kernel_text_address((unsigned long) p->addr) ||
1330             in_kprobes_functions((unsigned long) p->addr) ||
1331             ftrace_text_reserved(p->addr, p->addr) ||
1332             jump_label_text_reserved(p->addr, p->addr))
1333                 goto fail_with_jump_label;
1334
1335         /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1336         p->flags &= KPROBE_FLAG_DISABLED;
1337
1338         /*
1339          * Check if are we probing a module.
1340          */
1341         probed_mod = __module_text_address((unsigned long) p->addr);
1342         if (probed_mod) {
1343                 /*
1344                  * We must hold a refcount of the probed module while updating
1345                  * its code to prohibit unexpected unloading.
1346                  */
1347                 if (unlikely(!try_module_get(probed_mod)))
1348                         goto fail_with_jump_label;
1349
1350                 /*
1351                  * If the module freed .init.text, we couldn't insert
1352                  * kprobes in there.
1353                  */
1354                 if (within_module_init((unsigned long)p->addr, probed_mod) &&
1355                     probed_mod->state != MODULE_STATE_COMING) {
1356                         module_put(probed_mod);
1357                         goto fail_with_jump_label;
1358                 }
1359         }
1360         preempt_enable();
1361         jump_label_unlock();
1362
1363         p->nmissed = 0;
1364         INIT_LIST_HEAD(&p->list);
1365         mutex_lock(&kprobe_mutex);
1366
1367         jump_label_lock(); /* needed to call jump_label_text_reserved() */
1368
1369         get_online_cpus();      /* For avoiding text_mutex deadlock. */
1370         mutex_lock(&text_mutex);
1371
1372         old_p = get_kprobe(p->addr);
1373         if (old_p) {
1374                 /* Since this may unoptimize old_p, locking text_mutex. */
1375                 ret = register_aggr_kprobe(old_p, p);
1376                 goto out;
1377         }
1378
1379         ret = arch_prepare_kprobe(p);
1380         if (ret)
1381                 goto out;
1382
1383         INIT_HLIST_NODE(&p->hlist);
1384         hlist_add_head_rcu(&p->hlist,
1385                        &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1386
1387         if (!kprobes_all_disarmed && !kprobe_disabled(p))
1388                 __arm_kprobe(p);
1389
1390         /* Try to optimize kprobe */
1391         try_to_optimize_kprobe(p);
1392
1393 out:
1394         mutex_unlock(&text_mutex);
1395         put_online_cpus();
1396         jump_label_unlock();
1397         mutex_unlock(&kprobe_mutex);
1398
1399         if (probed_mod)
1400                 module_put(probed_mod);
1401
1402         return ret;
1403
1404 fail_with_jump_label:
1405         preempt_enable();
1406         jump_label_unlock();
1407         return -EINVAL;
1408 }
1409 EXPORT_SYMBOL_GPL(register_kprobe);
1410
1411 /* Check if all probes on the aggrprobe are disabled */
1412 static int __kprobes aggr_kprobe_disabled(struct kprobe *ap)
1413 {
1414         struct kprobe *kp;
1415
1416         list_for_each_entry_rcu(kp, &ap->list, list)
1417                 if (!kprobe_disabled(kp))
1418                         /*
1419                          * There is an active probe on the list.
1420                          * We can't disable this ap.
1421                          */
1422                         return 0;
1423
1424         return 1;
1425 }
1426
1427 /* Disable one kprobe: Make sure called under kprobe_mutex is locked */
1428 static struct kprobe *__kprobes __disable_kprobe(struct kprobe *p)
1429 {
1430         struct kprobe *orig_p;
1431
1432         /* Get an original kprobe for return */
1433         orig_p = __get_valid_kprobe(p);
1434         if (unlikely(orig_p == NULL))
1435                 return NULL;
1436
1437         if (!kprobe_disabled(p)) {
1438                 /* Disable probe if it is a child probe */
1439                 if (p != orig_p)
1440                         p->flags |= KPROBE_FLAG_DISABLED;
1441
1442                 /* Try to disarm and disable this/parent probe */
1443                 if (p == orig_p || aggr_kprobe_disabled(orig_p)) {
1444                         disarm_kprobe(orig_p);
1445                         orig_p->flags |= KPROBE_FLAG_DISABLED;
1446                 }
1447         }
1448
1449         return orig_p;
1450 }
1451
1452 /*
1453  * Unregister a kprobe without a scheduler synchronization.
1454  */
1455 static int __kprobes __unregister_kprobe_top(struct kprobe *p)
1456 {
1457         struct kprobe *ap, *list_p;
1458
1459         /* Disable kprobe. This will disarm it if needed. */
1460         ap = __disable_kprobe(p);
1461         if (ap == NULL)
1462                 return -EINVAL;
1463
1464         if (ap == p)
1465                 /*
1466                  * This probe is an independent(and non-optimized) kprobe
1467                  * (not an aggrprobe). Remove from the hash list.
1468                  */
1469                 goto disarmed;
1470
1471         /* Following process expects this probe is an aggrprobe */
1472         WARN_ON(!kprobe_aggrprobe(ap));
1473
1474         if (list_is_singular(&ap->list) && kprobe_disarmed(ap))
1475                 /*
1476                  * !disarmed could be happen if the probe is under delayed
1477                  * unoptimizing.
1478                  */
1479                 goto disarmed;
1480         else {
1481                 /* If disabling probe has special handlers, update aggrprobe */
1482                 if (p->break_handler && !kprobe_gone(p))
1483                         ap->break_handler = NULL;
1484                 if (p->post_handler && !kprobe_gone(p)) {
1485                         list_for_each_entry_rcu(list_p, &ap->list, list) {
1486                                 if ((list_p != p) && (list_p->post_handler))
1487                                         goto noclean;
1488                         }
1489                         ap->post_handler = NULL;
1490                 }
1491 noclean:
1492                 /*
1493                  * Remove from the aggrprobe: this path will do nothing in
1494                  * __unregister_kprobe_bottom().
1495                  */
1496                 list_del_rcu(&p->list);
1497                 if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
1498                         /*
1499                          * Try to optimize this probe again, because post
1500                          * handler may have been changed.
1501                          */
1502                         optimize_kprobe(ap);
1503         }
1504         return 0;
1505
1506 disarmed:
1507         BUG_ON(!kprobe_disarmed(ap));
1508         hlist_del_rcu(&ap->hlist);
1509         return 0;
1510 }
1511
1512 static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
1513 {
1514         struct kprobe *ap;
1515
1516         if (list_empty(&p->list))
1517                 /* This is an independent kprobe */
1518                 arch_remove_kprobe(p);
1519         else if (list_is_singular(&p->list)) {
1520                 /* This is the last child of an aggrprobe */
1521                 ap = list_entry(p->list.next, struct kprobe, list);
1522                 list_del(&p->list);
1523                 free_aggr_kprobe(ap);
1524         }
1525         /* Otherwise, do nothing. */
1526 }
1527
1528 int __kprobes register_kprobes(struct kprobe **kps, int num)
1529 {
1530         int i, ret = 0;
1531
1532         if (num <= 0)
1533                 return -EINVAL;
1534         for (i = 0; i < num; i++) {
1535                 ret = register_kprobe(kps[i]);
1536                 if (ret < 0) {
1537                         if (i > 0)
1538                                 unregister_kprobes(kps, i);
1539                         break;
1540                 }
1541         }
1542         return ret;
1543 }
1544 EXPORT_SYMBOL_GPL(register_kprobes);
1545
1546 void __kprobes unregister_kprobe(struct kprobe *p)
1547 {
1548         unregister_kprobes(&p, 1);
1549 }
1550 EXPORT_SYMBOL_GPL(unregister_kprobe);
1551
1552 void __kprobes unregister_kprobes(struct kprobe **kps, int num)
1553 {
1554         int i;
1555
1556         if (num <= 0)
1557                 return;
1558         mutex_lock(&kprobe_mutex);
1559         for (i = 0; i < num; i++)
1560                 if (__unregister_kprobe_top(kps[i]) < 0)
1561                         kps[i]->addr = NULL;
1562         mutex_unlock(&kprobe_mutex);
1563
1564         synchronize_sched();
1565         for (i = 0; i < num; i++)
1566                 if (kps[i]->addr)
1567                         __unregister_kprobe_bottom(kps[i]);
1568 }
1569 EXPORT_SYMBOL_GPL(unregister_kprobes);
1570
1571 static struct notifier_block kprobe_exceptions_nb = {
1572         .notifier_call = kprobe_exceptions_notify,
1573         .priority = 0x7fffffff /* we need to be notified first */
1574 };
1575
1576 unsigned long __weak arch_deref_entry_point(void *entry)
1577 {
1578         return (unsigned long)entry;
1579 }
1580
1581 int __kprobes register_jprobes(struct jprobe **jps, int num)
1582 {
1583         struct jprobe *jp;
1584         int ret = 0, i;
1585
1586         if (num <= 0)
1587                 return -EINVAL;
1588         for (i = 0; i < num; i++) {
1589                 unsigned long addr, offset;
1590                 jp = jps[i];
1591                 addr = arch_deref_entry_point(jp->entry);
1592
1593                 /* Verify probepoint is a function entry point */
1594                 if (kallsyms_lookup_size_offset(addr, NULL, &offset) &&
1595                     offset == 0) {
1596                         jp->kp.pre_handler = setjmp_pre_handler;
1597                         jp->kp.break_handler = longjmp_break_handler;
1598                         ret = register_kprobe(&jp->kp);
1599                 } else
1600                         ret = -EINVAL;
1601
1602                 if (ret < 0) {
1603                         if (i > 0)
1604                                 unregister_jprobes(jps, i);
1605                         break;
1606                 }
1607         }
1608         return ret;
1609 }
1610 EXPORT_SYMBOL_GPL(register_jprobes);
1611
1612 int __kprobes register_jprobe(struct jprobe *jp)
1613 {
1614         return register_jprobes(&jp, 1);
1615 }
1616 EXPORT_SYMBOL_GPL(register_jprobe);
1617
1618 void __kprobes unregister_jprobe(struct jprobe *jp)
1619 {
1620         unregister_jprobes(&jp, 1);
1621 }
1622 EXPORT_SYMBOL_GPL(unregister_jprobe);
1623
1624 void __kprobes unregister_jprobes(struct jprobe **jps, int num)
1625 {
1626         int i;
1627
1628         if (num <= 0)
1629                 return;
1630         mutex_lock(&kprobe_mutex);
1631         for (i = 0; i < num; i++)
1632                 if (__unregister_kprobe_top(&jps[i]->kp) < 0)
1633                         jps[i]->kp.addr = NULL;
1634         mutex_unlock(&kprobe_mutex);
1635
1636         synchronize_sched();
1637         for (i = 0; i < num; i++) {
1638                 if (jps[i]->kp.addr)
1639                         __unregister_kprobe_bottom(&jps[i]->kp);
1640         }
1641 }
1642 EXPORT_SYMBOL_GPL(unregister_jprobes);
1643
1644 #ifdef CONFIG_KRETPROBES
1645 /*
1646  * This kprobe pre_handler is registered with every kretprobe. When probe
1647  * hits it will set up the return probe.
1648  */
1649 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1650                                            struct pt_regs *regs)
1651 {
1652         struct kretprobe *rp = container_of(p, struct kretprobe, kp);
1653         unsigned long hash, flags = 0;
1654         struct kretprobe_instance *ri;
1655
1656         /*TODO: consider to only swap the RA after the last pre_handler fired */
1657         hash = hash_ptr(current, KPROBE_HASH_BITS);
1658         spin_lock_irqsave(&rp->lock, flags);
1659         if (!hlist_empty(&rp->free_instances)) {
1660                 ri = hlist_entry(rp->free_instances.first,
1661                                 struct kretprobe_instance, hlist);
1662                 hlist_del(&ri->hlist);
1663                 spin_unlock_irqrestore(&rp->lock, flags);
1664
1665                 ri->rp = rp;
1666                 ri->task = current;
1667
1668                 if (rp->entry_handler && rp->entry_handler(ri, regs))
1669                         return 0;
1670
1671                 arch_prepare_kretprobe(ri, regs);
1672
1673                 /* XXX(hch): why is there no hlist_move_head? */
1674                 INIT_HLIST_NODE(&ri->hlist);
1675                 kretprobe_table_lock(hash, &flags);
1676                 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
1677                 kretprobe_table_unlock(hash, &flags);
1678         } else {
1679                 rp->nmissed++;
1680                 spin_unlock_irqrestore(&rp->lock, flags);
1681         }
1682         return 0;
1683 }
1684
1685 int __kprobes register_kretprobe(struct kretprobe *rp)
1686 {
1687         int ret = 0;
1688         struct kretprobe_instance *inst;
1689         int i;
1690         void *addr;
1691
1692         if (kretprobe_blacklist_size) {
1693                 addr = kprobe_addr(&rp->kp);
1694                 if (!addr)
1695                         return -EINVAL;
1696
1697                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1698                         if (kretprobe_blacklist[i].addr == addr)
1699                                 return -EINVAL;
1700                 }
1701         }
1702
1703         rp->kp.pre_handler = pre_handler_kretprobe;
1704         rp->kp.post_handler = NULL;
1705         rp->kp.fault_handler = NULL;
1706         rp->kp.break_handler = NULL;
1707
1708         /* Pre-allocate memory for max kretprobe instances */
1709         if (rp->maxactive <= 0) {
1710 #ifdef CONFIG_PREEMPT
1711                 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
1712 #else
1713                 rp->maxactive = num_possible_cpus();
1714 #endif
1715         }
1716         spin_lock_init(&rp->lock);
1717         INIT_HLIST_HEAD(&rp->free_instances);
1718         for (i = 0; i < rp->maxactive; i++) {
1719                 inst = kmalloc(sizeof(struct kretprobe_instance) +
1720                                rp->data_size, GFP_KERNEL);
1721                 if (inst == NULL) {
1722                         free_rp_inst(rp);
1723                         return -ENOMEM;
1724                 }
1725                 INIT_HLIST_NODE(&inst->hlist);
1726                 hlist_add_head(&inst->hlist, &rp->free_instances);
1727         }
1728
1729         rp->nmissed = 0;
1730         /* Establish function entry probe point */
1731         ret = register_kprobe(&rp->kp);
1732         if (ret != 0)
1733                 free_rp_inst(rp);
1734         return ret;
1735 }
1736 EXPORT_SYMBOL_GPL(register_kretprobe);
1737
1738 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1739 {
1740         int ret = 0, i;
1741
1742         if (num <= 0)
1743                 return -EINVAL;
1744         for (i = 0; i < num; i++) {
1745                 ret = register_kretprobe(rps[i]);
1746                 if (ret < 0) {
1747                         if (i > 0)
1748                                 unregister_kretprobes(rps, i);
1749                         break;
1750                 }
1751         }
1752         return ret;
1753 }
1754 EXPORT_SYMBOL_GPL(register_kretprobes);
1755
1756 void __kprobes unregister_kretprobe(struct kretprobe *rp)
1757 {
1758         unregister_kretprobes(&rp, 1);
1759 }
1760 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1761
1762 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1763 {
1764         int i;
1765
1766         if (num <= 0)
1767                 return;
1768         mutex_lock(&kprobe_mutex);
1769         for (i = 0; i < num; i++)
1770                 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1771                         rps[i]->kp.addr = NULL;
1772         mutex_unlock(&kprobe_mutex);
1773
1774         synchronize_sched();
1775         for (i = 0; i < num; i++) {
1776                 if (rps[i]->kp.addr) {
1777                         __unregister_kprobe_bottom(&rps[i]->kp);
1778                         cleanup_rp_inst(rps[i]);
1779                 }
1780         }
1781 }
1782 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1783
1784 #else /* CONFIG_KRETPROBES */
1785 int __kprobes register_kretprobe(struct kretprobe *rp)
1786 {
1787         return -ENOSYS;
1788 }
1789 EXPORT_SYMBOL_GPL(register_kretprobe);
1790
1791 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1792 {
1793         return -ENOSYS;
1794 }
1795 EXPORT_SYMBOL_GPL(register_kretprobes);
1796
1797 void __kprobes unregister_kretprobe(struct kretprobe *rp)
1798 {
1799 }
1800 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1801
1802 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1803 {
1804 }
1805 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1806
1807 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1808                                            struct pt_regs *regs)
1809 {
1810         return 0;
1811 }
1812
1813 #endif /* CONFIG_KRETPROBES */
1814
1815 /* Set the kprobe gone and remove its instruction buffer. */
1816 static void __kprobes kill_kprobe(struct kprobe *p)
1817 {
1818         struct kprobe *kp;
1819
1820         p->flags |= KPROBE_FLAG_GONE;
1821         if (kprobe_aggrprobe(p)) {
1822                 /*
1823                  * If this is an aggr_kprobe, we have to list all the
1824                  * chained probes and mark them GONE.
1825                  */
1826                 list_for_each_entry_rcu(kp, &p->list, list)
1827                         kp->flags |= KPROBE_FLAG_GONE;
1828                 p->post_handler = NULL;
1829                 p->break_handler = NULL;
1830                 kill_optimized_kprobe(p);
1831         }
1832         /*
1833          * Here, we can remove insn_slot safely, because no thread calls
1834          * the original probed function (which will be freed soon) any more.
1835          */
1836         arch_remove_kprobe(p);
1837 }
1838
1839 /* Disable one kprobe */
1840 int __kprobes disable_kprobe(struct kprobe *kp)
1841 {
1842         int ret = 0;
1843
1844         mutex_lock(&kprobe_mutex);
1845
1846         /* Disable this kprobe */
1847         if (__disable_kprobe(kp) == NULL)
1848                 ret = -EINVAL;
1849
1850         mutex_unlock(&kprobe_mutex);
1851         return ret;
1852 }
1853 EXPORT_SYMBOL_GPL(disable_kprobe);
1854
1855 /* Enable one kprobe */
1856 int __kprobes enable_kprobe(struct kprobe *kp)
1857 {
1858         int ret = 0;
1859         struct kprobe *p;
1860
1861         mutex_lock(&kprobe_mutex);
1862
1863         /* Check whether specified probe is valid. */
1864         p = __get_valid_kprobe(kp);
1865         if (unlikely(p == NULL)) {
1866                 ret = -EINVAL;
1867                 goto out;
1868         }
1869
1870         if (kprobe_gone(kp)) {
1871                 /* This kprobe has gone, we couldn't enable it. */
1872                 ret = -EINVAL;
1873                 goto out;
1874         }
1875
1876         if (p != kp)
1877                 kp->flags &= ~KPROBE_FLAG_DISABLED;
1878
1879         if (!kprobes_all_disarmed && kprobe_disabled(p)) {
1880                 p->flags &= ~KPROBE_FLAG_DISABLED;
1881                 arm_kprobe(p);
1882         }
1883 out:
1884         mutex_unlock(&kprobe_mutex);
1885         return ret;
1886 }
1887 EXPORT_SYMBOL_GPL(enable_kprobe);
1888
1889 void __kprobes dump_kprobe(struct kprobe *kp)
1890 {
1891         printk(KERN_WARNING "Dumping kprobe:\n");
1892         printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
1893                kp->symbol_name, kp->addr, kp->offset);
1894 }
1895
1896 /* Module notifier call back, checking kprobes on the module */
1897 static int __kprobes kprobes_module_callback(struct notifier_block *nb,
1898                                              unsigned long val, void *data)
1899 {
1900         struct module *mod = data;
1901         struct hlist_head *head;
1902         struct hlist_node *node;
1903         struct kprobe *p;
1904         unsigned int i;
1905         int checkcore = (val == MODULE_STATE_GOING);
1906
1907         if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
1908                 return NOTIFY_DONE;
1909
1910         /*
1911          * When MODULE_STATE_GOING was notified, both of module .text and
1912          * .init.text sections would be freed. When MODULE_STATE_LIVE was
1913          * notified, only .init.text section would be freed. We need to
1914          * disable kprobes which have been inserted in the sections.
1915          */
1916         mutex_lock(&kprobe_mutex);
1917         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1918                 head = &kprobe_table[i];
1919                 hlist_for_each_entry_rcu(p, node, head, hlist)
1920                         if (within_module_init((unsigned long)p->addr, mod) ||
1921                             (checkcore &&
1922                              within_module_core((unsigned long)p->addr, mod))) {
1923                                 /*
1924                                  * The vaddr this probe is installed will soon
1925                                  * be vfreed buy not synced to disk. Hence,
1926                                  * disarming the breakpoint isn't needed.
1927                                  */
1928                                 kill_kprobe(p);
1929                         }
1930         }
1931         mutex_unlock(&kprobe_mutex);
1932         return NOTIFY_DONE;
1933 }
1934
1935 static struct notifier_block kprobe_module_nb = {
1936         .notifier_call = kprobes_module_callback,
1937         .priority = 0
1938 };
1939
1940 static int __init init_kprobes(void)
1941 {
1942         int i, err = 0;
1943         unsigned long offset = 0, size = 0;
1944         char *modname, namebuf[128];
1945         const char *symbol_name;
1946         void *addr;
1947         struct kprobe_blackpoint *kb;
1948
1949         /* FIXME allocate the probe table, currently defined statically */
1950         /* initialize all list heads */
1951         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1952                 INIT_HLIST_HEAD(&kprobe_table[i]);
1953                 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1954                 spin_lock_init(&(kretprobe_table_locks[i].lock));
1955         }
1956
1957         /*
1958          * Lookup and populate the kprobe_blacklist.
1959          *
1960          * Unlike the kretprobe blacklist, we'll need to determine
1961          * the range of addresses that belong to the said functions,
1962          * since a kprobe need not necessarily be at the beginning
1963          * of a function.
1964          */
1965         for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1966                 kprobe_lookup_name(kb->name, addr);
1967                 if (!addr)
1968                         continue;
1969
1970                 kb->start_addr = (unsigned long)addr;
1971                 symbol_name = kallsyms_lookup(kb->start_addr,
1972                                 &size, &offset, &modname, namebuf);
1973                 if (!symbol_name)
1974                         kb->range = 0;
1975                 else
1976                         kb->range = size;
1977         }
1978
1979         if (kretprobe_blacklist_size) {
1980                 /* lookup the function address from its name */
1981                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1982                         kprobe_lookup_name(kretprobe_blacklist[i].name,
1983                                            kretprobe_blacklist[i].addr);
1984                         if (!kretprobe_blacklist[i].addr)
1985                                 printk("kretprobe: lookup failed: %s\n",
1986                                        kretprobe_blacklist[i].name);
1987                 }
1988         }
1989
1990 #if defined(CONFIG_OPTPROBES)
1991 #if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
1992         /* Init kprobe_optinsn_slots */
1993         kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
1994 #endif
1995         /* By default, kprobes can be optimized */
1996         kprobes_allow_optimization = true;
1997 #endif
1998
1999         /* By default, kprobes are armed */
2000         kprobes_all_disarmed = false;
2001
2002         err = arch_init_kprobes();
2003         if (!err)
2004                 err = register_die_notifier(&kprobe_exceptions_nb);
2005         if (!err)
2006                 err = register_module_notifier(&kprobe_module_nb);
2007
2008         kprobes_initialized = (err == 0);
2009
2010         if (!err)
2011                 init_test_probes();
2012         return err;
2013 }
2014
2015 #ifdef CONFIG_DEBUG_FS
2016 static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p,
2017                 const char *sym, int offset, char *modname, struct kprobe *pp)
2018 {
2019         char *kprobe_type;
2020
2021         if (p->pre_handler == pre_handler_kretprobe)
2022                 kprobe_type = "r";
2023         else if (p->pre_handler == setjmp_pre_handler)
2024                 kprobe_type = "j";
2025         else
2026                 kprobe_type = "k";
2027
2028         if (sym)
2029                 seq_printf(pi, "%p  %s  %s+0x%x  %s ",
2030                         p->addr, kprobe_type, sym, offset,
2031                         (modname ? modname : " "));
2032         else
2033                 seq_printf(pi, "%p  %s  %p ",
2034                         p->addr, kprobe_type, p->addr);
2035
2036         if (!pp)
2037                 pp = p;
2038         seq_printf(pi, "%s%s%s\n",
2039                 (kprobe_gone(p) ? "[GONE]" : ""),
2040                 ((kprobe_disabled(p) && !kprobe_gone(p)) ?  "[DISABLED]" : ""),
2041                 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""));
2042 }
2043
2044 static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos)
2045 {
2046         return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2047 }
2048
2049 static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
2050 {
2051         (*pos)++;
2052         if (*pos >= KPROBE_TABLE_SIZE)
2053                 return NULL;
2054         return pos;
2055 }
2056
2057 static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v)
2058 {
2059         /* Nothing to do */
2060 }
2061
2062 static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v)
2063 {
2064         struct hlist_head *head;
2065         struct hlist_node *node;
2066         struct kprobe *p, *kp;
2067         const char *sym = NULL;
2068         unsigned int i = *(loff_t *) v;
2069         unsigned long offset = 0;
2070         char *modname, namebuf[128];
2071
2072         head = &kprobe_table[i];
2073         preempt_disable();
2074         hlist_for_each_entry_rcu(p, node, head, hlist) {
2075                 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
2076                                         &offset, &modname, namebuf);
2077                 if (kprobe_aggrprobe(p)) {
2078                         list_for_each_entry_rcu(kp, &p->list, list)
2079                                 report_probe(pi, kp, sym, offset, modname, p);
2080                 } else
2081                         report_probe(pi, p, sym, offset, modname, NULL);
2082         }
2083         preempt_enable();
2084         return 0;
2085 }
2086
2087 static const struct seq_operations kprobes_seq_ops = {
2088         .start = kprobe_seq_start,
2089         .next  = kprobe_seq_next,
2090         .stop  = kprobe_seq_stop,
2091         .show  = show_kprobe_addr
2092 };
2093
2094 static int __kprobes kprobes_open(struct inode *inode, struct file *filp)
2095 {
2096         return seq_open(filp, &kprobes_seq_ops);
2097 }
2098
2099 static const struct file_operations debugfs_kprobes_operations = {
2100         .open           = kprobes_open,
2101         .read           = seq_read,
2102         .llseek         = seq_lseek,
2103         .release        = seq_release,
2104 };
2105
2106 static void __kprobes arm_all_kprobes(void)
2107 {
2108         struct hlist_head *head;
2109         struct hlist_node *node;
2110         struct kprobe *p;
2111         unsigned int i;
2112
2113         mutex_lock(&kprobe_mutex);
2114
2115         /* If kprobes are armed, just return */
2116         if (!kprobes_all_disarmed)
2117                 goto already_enabled;
2118
2119         /* Arming kprobes doesn't optimize kprobe itself */
2120         mutex_lock(&text_mutex);
2121         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2122                 head = &kprobe_table[i];
2123                 hlist_for_each_entry_rcu(p, node, head, hlist)
2124                         if (!kprobe_disabled(p))
2125                                 __arm_kprobe(p);
2126         }
2127         mutex_unlock(&text_mutex);
2128
2129         kprobes_all_disarmed = false;
2130         printk(KERN_INFO "Kprobes globally enabled\n");
2131
2132 already_enabled:
2133         mutex_unlock(&kprobe_mutex);
2134         return;
2135 }
2136
2137 static void __kprobes disarm_all_kprobes(void)
2138 {
2139         struct hlist_head *head;
2140         struct hlist_node *node;
2141         struct kprobe *p;
2142         unsigned int i;
2143
2144         mutex_lock(&kprobe_mutex);
2145
2146         /* If kprobes are already disarmed, just return */
2147         if (kprobes_all_disarmed) {
2148                 mutex_unlock(&kprobe_mutex);
2149                 return;
2150         }
2151
2152         kprobes_all_disarmed = true;
2153         printk(KERN_INFO "Kprobes globally disabled\n");
2154
2155         mutex_lock(&text_mutex);
2156         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2157                 head = &kprobe_table[i];
2158                 hlist_for_each_entry_rcu(p, node, head, hlist) {
2159                         if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p))
2160                                 __disarm_kprobe(p, false);
2161                 }
2162         }
2163         mutex_unlock(&text_mutex);
2164         mutex_unlock(&kprobe_mutex);
2165
2166         /* Wait for disarming all kprobes by optimizer */
2167         wait_for_kprobe_optimizer();
2168 }
2169
2170 /*
2171  * XXX: The debugfs bool file interface doesn't allow for callbacks
2172  * when the bool state is switched. We can reuse that facility when
2173  * available
2174  */
2175 static ssize_t read_enabled_file_bool(struct file *file,
2176                char __user *user_buf, size_t count, loff_t *ppos)
2177 {
2178         char buf[3];
2179
2180         if (!kprobes_all_disarmed)
2181                 buf[0] = '1';
2182         else
2183                 buf[0] = '0';
2184         buf[1] = '\n';
2185         buf[2] = 0x00;
2186         return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
2187 }
2188
2189 static ssize_t write_enabled_file_bool(struct file *file,
2190                const char __user *user_buf, size_t count, loff_t *ppos)
2191 {
2192         char buf[32];
2193         int buf_size;
2194
2195         buf_size = min(count, (sizeof(buf)-1));
2196         if (copy_from_user(buf, user_buf, buf_size))
2197                 return -EFAULT;
2198
2199         switch (buf[0]) {
2200         case 'y':
2201         case 'Y':
2202         case '1':
2203                 arm_all_kprobes();
2204                 break;
2205         case 'n':
2206         case 'N':
2207         case '0':
2208                 disarm_all_kprobes();
2209                 break;
2210         }
2211
2212         return count;
2213 }
2214
2215 static const struct file_operations fops_kp = {
2216         .read =         read_enabled_file_bool,
2217         .write =        write_enabled_file_bool,
2218         .llseek =       default_llseek,
2219 };
2220
2221 static int __kprobes debugfs_kprobe_init(void)
2222 {
2223         struct dentry *dir, *file;
2224         unsigned int value = 1;
2225
2226         dir = debugfs_create_dir("kprobes", NULL);
2227         if (!dir)
2228                 return -ENOMEM;
2229
2230         file = debugfs_create_file("list", 0444, dir, NULL,
2231                                 &debugfs_kprobes_operations);
2232         if (!file) {
2233                 debugfs_remove(dir);
2234                 return -ENOMEM;
2235         }
2236
2237         file = debugfs_create_file("enabled", 0600, dir,
2238                                         &value, &fops_kp);
2239         if (!file) {
2240                 debugfs_remove(dir);
2241                 return -ENOMEM;
2242         }
2243
2244         return 0;
2245 }
2246
2247 late_initcall(debugfs_kprobe_init);
2248 #endif /* CONFIG_DEBUG_FS */
2249
2250 module_init(init_kprobes);
2251
2252 /* defined in arch/.../kernel/kprobes.c */
2253 EXPORT_SYMBOL_GPL(jprobe_return);