lib/debugobjects.c

   1 /*
   2  * Generic infrastructure for lifetime debugging of objects.
   3  *
   4  * Started by Thomas Gleixner
   5  *
   6  * Copyright (C) 2008, Thomas Gleixner <tglx@linutronix.de>
   7  *
   8  * For licencing details see kernel-base/COPYING
   9  */
  10
  11 #define pr_fmt(fmt) "ODEBUG: " fmt
  12
  13 #include <linux/debugobjects.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/sched.h>
  16 #include <linux/sched/task_stack.h>
  17 #include <linux/seq_file.h>
  18 #include <linux/debugfs.h>
  19 #include <linux/slab.h>
  20 #include <linux/hash.h>
  21 #include <linux/kmemleak.h>
  22
  23 #define ODEBUG_HASH_BITS        14
  24 #define ODEBUG_HASH_SIZE        (1 << ODEBUG_HASH_BITS)
  25
  26 #define ODEBUG_POOL_SIZE        1024
  27 #define ODEBUG_POOL_MIN_LEVEL   256
  28 #define ODEBUG_POOL_PERCPU_SIZE 64
  29
  30 #define ODEBUG_CHUNK_SHIFT      PAGE_SHIFT
  31 #define ODEBUG_CHUNK_SIZE       (1 << ODEBUG_CHUNK_SHIFT)
  32 #define ODEBUG_CHUNK_MASK       (~(ODEBUG_CHUNK_SIZE - 1))
  33
  34 struct debug_bucket {
  35         struct hlist_head       list;
  36         raw_spinlock_t          lock;
  37 };
  38
  39 /*
  40  * Debug object percpu free list
  41  * Access is protected by disabling irq
  42  */
  43 struct debug_percpu_free {
  44         struct hlist_head       free_objs;
  45         int                     obj_free;
  46 };
  47
  48 static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
  49
  50 static struct debug_bucket      obj_hash[ODEBUG_HASH_SIZE];
  51
  52 static struct debug_obj         obj_static_pool[ODEBUG_POOL_SIZE] __initdata;
  53
  54 static DEFINE_RAW_SPINLOCK(pool_lock);
  55
  56 static HLIST_HEAD(obj_pool);
  57 static HLIST_HEAD(obj_to_free);
  58
  59 /*
  60  * Because of the presence of percpu free pools, obj_pool_free will
  61  * under-count those in the percpu free pools. Similarly, obj_pool_used
  62  * will over-count those in the percpu free pools. Adjustments will be
  63  * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used
  64  * can be off.
  65  */
  66 static int                      obj_pool_min_free = ODEBUG_POOL_SIZE;
  67 static int                      obj_pool_free = ODEBUG_POOL_SIZE;
  68 static int                      obj_pool_used;
  69 static int                      obj_pool_max_used;
  70 /* The number of objs on the global free list */
  71 static int                      obj_nr_tofree;
  72
  73 static int                      debug_objects_maxchain __read_mostly;
  74 static int __maybe_unused       debug_objects_maxchecked __read_mostly;
  75 static int                      debug_objects_fixups __read_mostly;
  76 static int                      debug_objects_warnings __read_mostly;
  77 static int                      debug_objects_enabled __read_mostly
  78                                 = CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT;
  79 static int                      debug_objects_pool_size __read_mostly
  80                                 = ODEBUG_POOL_SIZE;
  81 static int                      debug_objects_pool_min_level __read_mostly
  82                                 = ODEBUG_POOL_MIN_LEVEL;
  83 static struct debug_obj_descr   *descr_test  __read_mostly;
  84 static struct kmem_cache        *obj_cache __read_mostly;
  85
  86 /*
  87  * Track numbers of kmem_cache_alloc()/free() calls done.
  88  */
  89 static int                      debug_objects_allocated;
  90 static int                      debug_objects_freed;
  91
  92 static void free_obj_work(struct work_struct *work);
  93 static DECLARE_WORK(debug_obj_work, free_obj_work);
  94
  95 static int __init enable_object_debug(char *str)
  96 {
  97         debug_objects_enabled = 1;
  98         return 0;
  99 }
 100
 101 static int __init disable_object_debug(char *str)
 102 {
 103         debug_objects_enabled = 0;
 104         return 0;
 105 }
 106
 107 early_param("debug_objects", enable_object_debug);
 108 early_param("no_debug_objects", disable_object_debug);
 109
 110 static const char *obj_states[ODEBUG_STATE_MAX] = {
 111         [ODEBUG_STATE_NONE]             = "none",
 112         [ODEBUG_STATE_INIT]             = "initialized",
 113         [ODEBUG_STATE_INACTIVE]         = "inactive",
 114         [ODEBUG_STATE_ACTIVE]           = "active",
 115         [ODEBUG_STATE_DESTROYED]        = "destroyed",
 116         [ODEBUG_STATE_NOTAVAILABLE]     = "not available",
 117 };
 118
 119 static void fill_pool(void)
 120 {
 121         gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
 122         struct debug_obj *new, *obj;
 123         unsigned long flags;
 124
 125         if (likely(obj_pool_free >= debug_objects_pool_min_level))
 126                 return;
 127
 128         /*
 129          * Reuse objs from the global free list; they will be reinitialized
 130          * when allocating.
 131          */
 132         while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
 133                 raw_spin_lock_irqsave(&pool_lock, flags);
 134                 /*
 135                  * Recheck with the lock held as the worker thread might have
 136                  * won the race and freed the global free list already.
 137                  */
 138                 if (obj_nr_tofree) {
 139                         obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
 140                         hlist_del(&obj->node);
 141                         obj_nr_tofree--;
 142                         hlist_add_head(&obj->node, &obj_pool);
 143                         obj_pool_free++;
 144                 }
 145                 raw_spin_unlock_irqrestore(&pool_lock, flags);
 146         }
 147
 148         if (unlikely(!obj_cache))
 149                 return;
 150
 151         while (obj_pool_free < debug_objects_pool_min_level) {
 152
 153                 new = kmem_cache_zalloc(obj_cache, gfp);
 154                 if (!new)
 155                         return;
 156
 157                 raw_spin_lock_irqsave(&pool_lock, flags);
 158                 hlist_add_head(&new->node, &obj_pool);
 159                 debug_objects_allocated++;
 160                 obj_pool_free++;
 161                 raw_spin_unlock_irqrestore(&pool_lock, flags);
 162         }
 163 }
 164
 165 /*
 166  * Lookup an object in the hash bucket.
 167  */
 168 static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b)
 169 {
 170         struct debug_obj *obj;
 171         int cnt = 0;
 172
 173         hlist_for_each_entry(obj, &b->list, node) {
 174                 cnt++;
 175                 if (obj->object == addr)
 176                         return obj;
 177         }
 178         if (cnt > debug_objects_maxchain)
 179                 debug_objects_maxchain = cnt;
 180
 181         return NULL;
 182 }
 183
 184 /*
 185  * Allocate a new object from the hlist
 186  */
 187 static struct debug_obj *__alloc_object(struct hlist_head *list)
 188 {
 189         struct debug_obj *obj = NULL;
 190
 191         if (list->first) {
 192                 obj = hlist_entry(list->first, typeof(*obj), node);
 193                 hlist_del(&obj->node);
 194         }
 195
 196         return obj;
 197 }
 198
 199 /*
 200  * Allocate a new object. If the pool is empty, switch off the debugger.
 201  * Must be called with interrupts disabled.
 202  */
 203 static struct debug_obj *
 204 alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)
 205 {
 206         struct debug_percpu_free *percpu_pool;
 207         struct debug_obj *obj;
 208
 209         if (likely(obj_cache)) {
 210                 percpu_pool = this_cpu_ptr(&percpu_obj_pool);
 211                 obj = __alloc_object(&percpu_pool->free_objs);
 212                 if (obj) {
 213                         percpu_pool->obj_free--;
 214                         goto init_obj;
 215                 }
 216         }
 217
 218         raw_spin_lock(&pool_lock);
 219         obj = __alloc_object(&obj_pool);
 220         if (obj) {
 221                 obj_pool_used++;
 222                 if (obj_pool_used > obj_pool_max_used)
 223                         obj_pool_max_used = obj_pool_used;
 224
 225                 obj_pool_free--;
 226                 if (obj_pool_free < obj_pool_min_free)
 227                         obj_pool_min_free = obj_pool_free;
 228         }
 229         raw_spin_unlock(&pool_lock);
 230
 231 init_obj:
 232         if (obj) {
 233                 obj->object = addr;
 234                 obj->descr  = descr;
 235                 obj->state  = ODEBUG_STATE_NONE;
 236                 obj->astate = 0;
 237                 hlist_add_head(&obj->node, &b->list);
 238         }
 239         return obj;
 240 }
 241
 242 /*
 243  * workqueue function to free objects.
 244  *
 245  * To reduce contention on the global pool_lock, the actual freeing of
 246  * debug objects will be delayed if the pool_lock is busy.
 247  */
 248 static void free_obj_work(struct work_struct *work)
 249 {
 250         struct hlist_node *tmp;
 251         struct debug_obj *obj;
 252         unsigned long flags;
 253         HLIST_HEAD(tofree);
 254
 255         if (!raw_spin_trylock_irqsave(&pool_lock, flags))
 256                 return;
 257
 258         /*
 259          * The objs on the pool list might be allocated before the work is
 260          * run, so recheck if pool list it full or not, if not fill pool
 261          * list from the global free list
 262          */
 263         while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
 264                 obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
 265                 hlist_del(&obj->node);
 266                 hlist_add_head(&obj->node, &obj_pool);
 267                 obj_pool_free++;
 268                 obj_nr_tofree--;
 269         }
 270
 271         /*
 272          * Pool list is already full and there are still objs on the free
 273          * list. Move remaining free objs to a temporary list to free the
 274          * memory outside the pool_lock held region.
 275          */
 276         if (obj_nr_tofree) {
 277                 hlist_move_list(&obj_to_free, &tofree);
 278                 debug_objects_freed += obj_nr_tofree;
 279                 obj_nr_tofree = 0;
 280         }
 281         raw_spin_unlock_irqrestore(&pool_lock, flags);
 282
 283         hlist_for_each_entry_safe(obj, tmp, &tofree, node) {
 284                 hlist_del(&obj->node);
 285                 kmem_cache_free(obj_cache, obj);
 286         }
 287 }
 288
 289 static bool __free_object(struct debug_obj *obj)
 290 {
 291         unsigned long flags;
 292         bool work;
 293         struct debug_percpu_free *percpu_pool;
 294
 295         local_irq_save(flags);
 296         /*
 297          * Try to free it into the percpu pool first.
 298          */
 299         percpu_pool = this_cpu_ptr(&percpu_obj_pool);
 300         if (obj_cache && percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) {
 301                 hlist_add_head(&obj->node, &percpu_pool->free_objs);
 302                 percpu_pool->obj_free++;
 303                 local_irq_restore(flags);
 304                 return false;
 305         }
 306
 307         raw_spin_lock(&pool_lock);
 308         work = (obj_pool_free > debug_objects_pool_size) && obj_cache;
 309         obj_pool_used--;
 310
 311         if (work) {
 312                 obj_nr_tofree++;
 313                 hlist_add_head(&obj->node, &obj_to_free);
 314         } else {
 315                 obj_pool_free++;
 316                 hlist_add_head(&obj->node, &obj_pool);
 317         }
 318         raw_spin_unlock(&pool_lock);
 319         local_irq_restore(flags);
 320         return work;
 321 }
 322
 323 /*
 324  * Put the object back into the pool and schedule work to free objects
 325  * if necessary.
 326  */
 327 static void free_object(struct debug_obj *obj)
 328 {
 329         if (__free_object(obj))
 330                 schedule_work(&debug_obj_work);
 331 }
 332
 333 /*
 334  * We run out of memory. That means we probably have tons of objects
 335  * allocated.
 336  */
 337 static void debug_objects_oom(void)
 338 {
 339         struct debug_bucket *db = obj_hash;
 340         struct hlist_node *tmp;
 341         HLIST_HEAD(freelist);
 342         struct debug_obj *obj;
 343         unsigned long flags;
 344         int i;
 345
 346         pr_warn("Out of memory. ODEBUG disabled\n");
 347
 348         for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
 349                 raw_spin_lock_irqsave(&db->lock, flags);
 350                 hlist_move_list(&db->list, &freelist);
 351                 raw_spin_unlock_irqrestore(&db->lock, flags);
 352
 353                 /* Now free them */
 354                 hlist_for_each_entry_safe(obj, tmp, &freelist, node) {
 355                         hlist_del(&obj->node);
 356                         free_object(obj);
 357                 }
 358         }
 359 }
 360
 361 /*
 362  * We use the pfn of the address for the hash. That way we can check
 363  * for freed objects simply by checking the affected bucket.
 364  */
 365 static struct debug_bucket *get_bucket(unsigned long addr)
 366 {
 367         unsigned long hash;
 368
 369         hash = hash_long((addr >> ODEBUG_CHUNK_SHIFT), ODEBUG_HASH_BITS);
 370         return &obj_hash[hash];
 371 }
 372
 373 static void debug_print_object(struct debug_obj *obj, char *msg)
 374 {
 375         struct debug_obj_descr *descr = obj->descr;
 376         static int limit;
 377
 378         if (limit < 5 && descr != descr_test) {
 379                 void *hint = descr->debug_hint ?
 380                         descr->debug_hint(obj->object) : NULL;
 381                 limit++;
 382                 WARN(1, KERN_ERR "ODEBUG: %s %s (active state %u) "
 383                                  "object type: %s hint: %pS\n",
 384                         msg, obj_states[obj->state], obj->astate,
 385                         descr->name, hint);
 386         }
 387         debug_objects_warnings++;
 388 }
 389
 390 /*
 391  * Try to repair the damage, so we have a better chance to get useful
 392  * debug output.
 393  */
 394 static bool
 395 debug_object_fixup(bool (*fixup)(void *addr, enum debug_obj_state state),
 396                    void * addr, enum debug_obj_state state)
 397 {
 398         if (fixup && fixup(addr, state)) {
 399                 debug_objects_fixups++;
 400                 return true;
 401         }
 402         return false;
 403 }
 404
 405 static void debug_object_is_on_stack(void *addr, int onstack)
 406 {
 407         int is_on_stack;
 408         static int limit;
 409
 410         if (limit > 4)
 411                 return;
 412
 413         is_on_stack = object_is_on_stack(addr);
 414         if (is_on_stack == onstack)
 415                 return;
 416
 417         limit++;
 418         if (is_on_stack)
 419                 pr_warn("object %p is on stack %p, but NOT annotated.\n", addr,
 420                          task_stack_page(current));
 421         else
 422                 pr_warn("object %p is NOT on stack %p, but annotated.\n", addr,
 423                          task_stack_page(current));
 424
 425         WARN_ON(1);
 426 }
 427
 428 static void
 429 __debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack)
 430 {
 431         enum debug_obj_state state;
 432         struct debug_bucket *db;
 433         struct debug_obj *obj;
 434         unsigned long flags;
 435
 436         fill_pool();
 437
 438         db = get_bucket((unsigned long) addr);
 439
 440         raw_spin_lock_irqsave(&db->lock, flags);
 441
 442         obj = lookup_object(addr, db);
 443         if (!obj) {
 444                 obj = alloc_object(addr, db, descr);
 445                 if (!obj) {
 446                         debug_objects_enabled = 0;
 447                         raw_spin_unlock_irqrestore(&db->lock, flags);
 448                         debug_objects_oom();
 449                         return;
 450                 }
 451                 debug_object_is_on_stack(addr, onstack);
 452         }
 453
 454         switch (obj->state) {
 455         case ODEBUG_STATE_NONE:
 456         case ODEBUG_STATE_INIT:
 457         case ODEBUG_STATE_INACTIVE:
 458                 obj->state = ODEBUG_STATE_INIT;
 459                 break;
 460
 461         case ODEBUG_STATE_ACTIVE:
 462                 debug_print_object(obj, "init");
 463                 state = obj->state;
 464                 raw_spin_unlock_irqrestore(&db->lock, flags);
 465                 debug_object_fixup(descr->fixup_init, addr, state);
 466                 return;
 467
 468         case ODEBUG_STATE_DESTROYED:
 469                 debug_print_object(obj, "init");
 470                 break;
 471         default:
 472                 break;
 473         }
 474
 475         raw_spin_unlock_irqrestore(&db->lock, flags);
 476 }
 477
 478 /**
 479  * debug_object_init - debug checks when an object is initialized
 480  * @addr:       address of the object
 481  * @descr:      pointer to an object specific debug description structure
 482  */
 483 void debug_object_init(void *addr, struct debug_obj_descr *descr)
 484 {
 485         if (!debug_objects_enabled)
 486                 return;
 487
 488         __debug_object_init(addr, descr, 0);
 489 }
 490 EXPORT_SYMBOL_GPL(debug_object_init);
 491
 492 /**
 493  * debug_object_init_on_stack - debug checks when an object on stack is
 494  *                              initialized
 495  * @addr:       address of the object
 496  * @descr:      pointer to an object specific debug description structure
 497  */
 498 void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr)
 499 {
 500         if (!debug_objects_enabled)
 501                 return;
 502
 503         __debug_object_init(addr, descr, 1);
 504 }
 505 EXPORT_SYMBOL_GPL(debug_object_init_on_stack);
 506
 507 /**
 508  * debug_object_activate - debug checks when an object is activated
 509  * @addr:       address of the object
 510  * @descr:      pointer to an object specific debug description structure
 511  * Returns 0 for success, -EINVAL for check failed.
 512  */
 513 int debug_object_activate(void *addr, struct debug_obj_descr *descr)
 514 {
 515         enum debug_obj_state state;
 516         struct debug_bucket *db;
 517         struct debug_obj *obj;
 518         unsigned long flags;
 519         int ret;
 520         struct debug_obj o = { .object = addr,
 521                                .state = ODEBUG_STATE_NOTAVAILABLE,
 522                                .descr = descr };
 523
 524         if (!debug_objects_enabled)
 525                 return 0;
 526
 527         db = get_bucket((unsigned long) addr);
 528
 529         raw_spin_lock_irqsave(&db->lock, flags);
 530
 531         obj = lookup_object(addr, db);
 532         if (obj) {
 533                 switch (obj->state) {
 534                 case ODEBUG_STATE_INIT:
 535                 case ODEBUG_STATE_INACTIVE:
 536                         obj->state = ODEBUG_STATE_ACTIVE;
 537                         ret = 0;
 538                         break;
 539
 540                 case ODEBUG_STATE_ACTIVE:
 541                         debug_print_object(obj, "activate");
 542                         state = obj->state;
 543                         raw_spin_unlock_irqrestore(&db->lock, flags);
 544                         ret = debug_object_fixup(descr->fixup_activate, addr, state);
 545                         return ret ? 0 : -EINVAL;
 546
 547                 case ODEBUG_STATE_DESTROYED:
 548                         debug_print_object(obj, "activate");
 549                         ret = -EINVAL;
 550                         break;
 551                 default:
 552                         ret = 0;
 553                         break;
 554                 }
 555                 raw_spin_unlock_irqrestore(&db->lock, flags);
 556                 return ret;
 557         }
 558
 559         raw_spin_unlock_irqrestore(&db->lock, flags);
 560         /*
 561          * We are here when a static object is activated. We
 562          * let the type specific code confirm whether this is
 563          * true or not. if true, we just make sure that the
 564          * static object is tracked in the object tracker. If
 565          * not, this must be a bug, so we try to fix it up.
 566          */
 567         if (descr->is_static_object && descr->is_static_object(addr)) {
 568                 /* track this static object */
 569                 debug_object_init(addr, descr);
 570                 debug_object_activate(addr, descr);
 571         } else {
 572                 debug_print_object(&o, "activate");
 573                 ret = debug_object_fixup(descr->fixup_activate, addr,
 574                                         ODEBUG_STATE_NOTAVAILABLE);
 575                 return ret ? 0 : -EINVAL;
 576         }
 577         return 0;
 578 }
 579 EXPORT_SYMBOL_GPL(debug_object_activate);
 580
 581 /**
 582  * debug_object_deactivate - debug checks when an object is deactivated
 583  * @addr:       address of the object
 584  * @descr:      pointer to an object specific debug description structure
 585  */
 586 void debug_object_deactivate(void *addr, struct debug_obj_descr *descr)
 587 {
 588         struct debug_bucket *db;
 589         struct debug_obj *obj;
 590         unsigned long flags;
 591
 592         if (!debug_objects_enabled)
 593                 return;
 594
 595         db = get_bucket((unsigned long) addr);
 596
 597         raw_spin_lock_irqsave(&db->lock, flags);
 598
 599         obj = lookup_object(addr, db);
 600         if (obj) {
 601                 switch (obj->state) {
 602                 case ODEBUG_STATE_INIT:
 603                 case ODEBUG_STATE_INACTIVE:
 604                 case ODEBUG_STATE_ACTIVE:
 605                         if (!obj->astate)
 606                                 obj->state = ODEBUG_STATE_INACTIVE;
 607                         else
 608                                 debug_print_object(obj, "deactivate");
 609                         break;
 610
 611                 case ODEBUG_STATE_DESTROYED:
 612                         debug_print_object(obj, "deactivate");
 613                         break;
 614                 default:
 615                         break;
 616                 }
 617         } else {
 618                 struct debug_obj o = { .object = addr,
 619                                        .state = ODEBUG_STATE_NOTAVAILABLE,
 620                                        .descr = descr };
 621
 622                 debug_print_object(&o, "deactivate");
 623         }
 624
 625         raw_spin_unlock_irqrestore(&db->lock, flags);
 626 }
 627 EXPORT_SYMBOL_GPL(debug_object_deactivate);
 628
 629 /**
 630  * debug_object_destroy - debug checks when an object is destroyed
 631  * @addr:       address of the object
 632  * @descr:      pointer to an object specific debug description structure
 633  */
 634 void debug_object_destroy(void *addr, struct debug_obj_descr *descr)
 635 {
 636         enum debug_obj_state state;
 637         struct debug_bucket *db;
 638         struct debug_obj *obj;
 639         unsigned long flags;
 640
 641         if (!debug_objects_enabled)
 642                 return;
 643
 644         db = get_bucket((unsigned long) addr);
 645
 646         raw_spin_lock_irqsave(&db->lock, flags);
 647
 648         obj = lookup_object(addr, db);
 649         if (!obj)
 650                 goto out_unlock;
 651
 652         switch (obj->state) {
 653         case ODEBUG_STATE_NONE:
 654         case ODEBUG_STATE_INIT:
 655         case ODEBUG_STATE_INACTIVE:
 656                 obj->state = ODEBUG_STATE_DESTROYED;
 657                 break;
 658         case ODEBUG_STATE_ACTIVE:
 659                 debug_print_object(obj, "destroy");
 660                 state = obj->state;
 661                 raw_spin_unlock_irqrestore(&db->lock, flags);
 662                 debug_object_fixup(descr->fixup_destroy, addr, state);
 663                 return;
 664
 665         case ODEBUG_STATE_DESTROYED:
 666                 debug_print_object(obj, "destroy");
 667                 break;
 668         default:
 669                 break;
 670         }
 671 out_unlock:
 672         raw_spin_unlock_irqrestore(&db->lock, flags);
 673 }
 674 EXPORT_SYMBOL_GPL(debug_object_destroy);
 675
 676 /**
 677  * debug_object_free - debug checks when an object is freed
 678  * @addr:       address of the object
 679  * @descr:      pointer to an object specific debug description structure
 680  */
 681 void debug_object_free(void *addr, struct debug_obj_descr *descr)
 682 {
 683         enum debug_obj_state state;
 684         struct debug_bucket *db;
 685         struct debug_obj *obj;
 686         unsigned long flags;
 687
 688         if (!debug_objects_enabled)
 689                 return;
 690
 691         db = get_bucket((unsigned long) addr);
 692
 693         raw_spin_lock_irqsave(&db->lock, flags);
 694
 695         obj = lookup_object(addr, db);
 696         if (!obj)
 697                 goto out_unlock;
 698
 699         switch (obj->state) {
 700         case ODEBUG_STATE_ACTIVE:
 701                 debug_print_object(obj, "free");
 702                 state = obj->state;
 703                 raw_spin_unlock_irqrestore(&db->lock, flags);
 704                 debug_object_fixup(descr->fixup_free, addr, state);
 705                 return;
 706         default:
 707                 hlist_del(&obj->node);
 708                 raw_spin_unlock_irqrestore(&db->lock, flags);
 709                 free_object(obj);
 710                 return;
 711         }
 712 out_unlock:
 713         raw_spin_unlock_irqrestore(&db->lock, flags);
 714 }
 715 EXPORT_SYMBOL_GPL(debug_object_free);
 716
 717 /**
 718  * debug_object_assert_init - debug checks when object should be init-ed
 719  * @addr:       address of the object
 720  * @descr:      pointer to an object specific debug description structure
 721  */
 722 void debug_object_assert_init(void *addr, struct debug_obj_descr *descr)
 723 {
 724         struct debug_bucket *db;
 725         struct debug_obj *obj;
 726         unsigned long flags;
 727
 728         if (!debug_objects_enabled)
 729                 return;
 730
 731         db = get_bucket((unsigned long) addr);
 732
 733         raw_spin_lock_irqsave(&db->lock, flags);
 734
 735         obj = lookup_object(addr, db);
 736         if (!obj) {
 737                 struct debug_obj o = { .object = addr,
 738                                        .state = ODEBUG_STATE_NOTAVAILABLE,
 739                                        .descr = descr };
 740
 741                 raw_spin_unlock_irqrestore(&db->lock, flags);
 742                 /*
 743                  * Maybe the object is static, and we let the type specific
 744                  * code confirm. Track this static object if true, else invoke
 745                  * fixup.
 746                  */
 747                 if (descr->is_static_object && descr->is_static_object(addr)) {
 748                         /* Track this static object */
 749                         debug_object_init(addr, descr);
 750                 } else {
 751                         debug_print_object(&o, "assert_init");
 752                         debug_object_fixup(descr->fixup_assert_init, addr,
 753                                            ODEBUG_STATE_NOTAVAILABLE);
 754                 }
 755                 return;
 756         }
 757
 758         raw_spin_unlock_irqrestore(&db->lock, flags);
 759 }
 760 EXPORT_SYMBOL_GPL(debug_object_assert_init);
 761
 762 /**
 763  * debug_object_active_state - debug checks object usage state machine
 764  * @addr:       address of the object
 765  * @descr:      pointer to an object specific debug description structure
 766  * @expect:     expected state
 767  * @next:       state to move to if expected state is found
 768  */
 769 void
 770 debug_object_active_state(void *addr, struct debug_obj_descr *descr,
 771                           unsigned int expect, unsigned int next)
 772 {
 773         struct debug_bucket *db;
 774         struct debug_obj *obj;
 775         unsigned long flags;
 776
 777         if (!debug_objects_enabled)
 778                 return;
 779
 780         db = get_bucket((unsigned long) addr);
 781
 782         raw_spin_lock_irqsave(&db->lock, flags);
 783
 784         obj = lookup_object(addr, db);
 785         if (obj) {
 786                 switch (obj->state) {
 787                 case ODEBUG_STATE_ACTIVE:
 788                         if (obj->astate == expect)
 789                                 obj->astate = next;
 790                         else
 791                                 debug_print_object(obj, "active_state");
 792                         break;
 793
 794                 default:
 795                         debug_print_object(obj, "active_state");
 796                         break;
 797                 }
 798         } else {
 799                 struct debug_obj o = { .object = addr,
 800                                        .state = ODEBUG_STATE_NOTAVAILABLE,
 801                                        .descr = descr };
 802
 803                 debug_print_object(&o, "active_state");
 804         }
 805
 806         raw_spin_unlock_irqrestore(&db->lock, flags);
 807 }
 808 EXPORT_SYMBOL_GPL(debug_object_active_state);
 809
 810 #ifdef CONFIG_DEBUG_OBJECTS_FREE
 811 static void __debug_check_no_obj_freed(const void *address, unsigned long size)
 812 {
 813         unsigned long flags, oaddr, saddr, eaddr, paddr, chunks;
 814         struct debug_obj_descr *descr;
 815         enum debug_obj_state state;
 816         struct debug_bucket *db;
 817         struct hlist_node *tmp;
 818         struct debug_obj *obj;
 819         int cnt, objs_checked = 0;
 820         bool work = false;
 821
 822         saddr = (unsigned long) address;
 823         eaddr = saddr + size;
 824         paddr = saddr & ODEBUG_CHUNK_MASK;
 825         chunks = ((eaddr - paddr) + (ODEBUG_CHUNK_SIZE - 1));
 826         chunks >>= ODEBUG_CHUNK_SHIFT;
 827
 828         for (;chunks > 0; chunks--, paddr += ODEBUG_CHUNK_SIZE) {
 829                 db = get_bucket(paddr);
 830
 831 repeat:
 832                 cnt = 0;
 833                 raw_spin_lock_irqsave(&db->lock, flags);
 834                 hlist_for_each_entry_safe(obj, tmp, &db->list, node) {
 835                         cnt++;
 836                         oaddr = (unsigned long) obj->object;
 837                         if (oaddr < saddr || oaddr >= eaddr)
 838                                 continue;
 839
 840                         switch (obj->state) {
 841                         case ODEBUG_STATE_ACTIVE:
 842                                 debug_print_object(obj, "free");
 843                                 descr = obj->descr;
 844                                 state = obj->state;
 845                                 raw_spin_unlock_irqrestore(&db->lock, flags);
 846                                 debug_object_fixup(descr->fixup_free,
 847                                                    (void *) oaddr, state);
 848                                 goto repeat;
 849                         default:
 850                                 hlist_del(&obj->node);
 851                                 work |= __free_object(obj);
 852                                 break;
 853                         }
 854                 }
 855                 raw_spin_unlock_irqrestore(&db->lock, flags);
 856
 857                 if (cnt > debug_objects_maxchain)
 858                         debug_objects_maxchain = cnt;
 859
 860                 objs_checked += cnt;
 861         }
 862
 863         if (objs_checked > debug_objects_maxchecked)
 864                 debug_objects_maxchecked = objs_checked;
 865
 866         /* Schedule work to actually kmem_cache_free() objects */
 867         if (work)
 868                 schedule_work(&debug_obj_work);
 869 }
 870
 871 void debug_check_no_obj_freed(const void *address, unsigned long size)
 872 {
 873         if (debug_objects_enabled)
 874                 __debug_check_no_obj_freed(address, size);
 875 }
 876 #endif
 877
 878 #ifdef CONFIG_DEBUG_FS
 879
 880 static int debug_stats_show(struct seq_file *m, void *v)
 881 {
 882         int cpu, obj_percpu_free = 0;
 883
 884         for_each_possible_cpu(cpu)
 885                 obj_percpu_free += per_cpu(percpu_obj_pool.obj_free, cpu);
 886
 887         seq_printf(m, "max_chain     :%d\n", debug_objects_maxchain);
 888         seq_printf(m, "max_checked   :%d\n", debug_objects_maxchecked);
 889         seq_printf(m, "warnings      :%d\n", debug_objects_warnings);
 890         seq_printf(m, "fixups        :%d\n", debug_objects_fixups);
 891         seq_printf(m, "pool_free     :%d\n", obj_pool_free + obj_percpu_free);
 892         seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
 893         seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
 894         seq_printf(m, "pool_used     :%d\n", obj_pool_used - obj_percpu_free);
 895         seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
 896         seq_printf(m, "on_free_list  :%d\n", obj_nr_tofree);
 897         seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
 898         seq_printf(m, "objs_freed    :%d\n", debug_objects_freed);
 899         return 0;
 900 }
 901
 902 static int debug_stats_open(struct inode *inode, struct file *filp)
 903 {
 904         return single_open(filp, debug_stats_show, NULL);
 905 }
 906
 907 static const struct file_operations debug_stats_fops = {
 908         .open           = debug_stats_open,
 909         .read           = seq_read,
 910         .llseek         = seq_lseek,
 911         .release        = single_release,
 912 };
 913
 914 static int __init debug_objects_init_debugfs(void)
 915 {
 916         struct dentry *dbgdir;
 917
 918         if (!debug_objects_enabled)
 919                 return 0;
 920
 921         dbgdir = debugfs_create_dir("debug_objects", NULL);
 922
 923         debugfs_create_file("stats", 0444, dbgdir, NULL, &debug_stats_fops);
 924
 925         return 0;
 926 }
 927 __initcall(debug_objects_init_debugfs);
 928
 929 #else
 930 static inline void debug_objects_init_debugfs(void) { }
 931 #endif
 932
 933 #ifdef CONFIG_DEBUG_OBJECTS_SELFTEST
 934
 935 /* Random data structure for the self test */
 936 struct self_test {
 937         unsigned long   dummy1[6];
 938         int             static_init;
 939         unsigned long   dummy2[3];
 940 };
 941
 942 static __initdata struct debug_obj_descr descr_type_test;
 943
 944 static bool __init is_static_object(void *addr)
 945 {
 946         struct self_test *obj = addr;
 947
 948         return obj->static_init;
 949 }
 950
 951 /*
 952  * fixup_init is called when:
 953  * - an active object is initialized
 954  */
 955 static bool __init fixup_init(void *addr, enum debug_obj_state state)
 956 {
 957         struct self_test *obj = addr;
 958
 959         switch (state) {
 960         case ODEBUG_STATE_ACTIVE:
 961                 debug_object_deactivate(obj, &descr_type_test);
 962                 debug_object_init(obj, &descr_type_test);
 963                 return true;
 964         default:
 965                 return false;
 966         }
 967 }
 968
 969 /*
 970  * fixup_activate is called when:
 971  * - an active object is activated
 972  * - an unknown non-static object is activated
 973  */
 974 static bool __init fixup_activate(void *addr, enum debug_obj_state state)
 975 {
 976         struct self_test *obj = addr;
 977
 978         switch (state) {
 979         case ODEBUG_STATE_NOTAVAILABLE:
 980                 return true;
 981         case ODEBUG_STATE_ACTIVE:
 982                 debug_object_deactivate(obj, &descr_type_test);
 983                 debug_object_activate(obj, &descr_type_test);
 984                 return true;
 985
 986         default:
 987                 return false;
 988         }
 989 }
 990
 991 /*
 992  * fixup_destroy is called when:
 993  * - an active object is destroyed
 994  */
 995 static bool __init fixup_destroy(void *addr, enum debug_obj_state state)
 996 {
 997         struct self_test *obj = addr;
 998
 999         switch (state) {
1000         case ODEBUG_STATE_ACTIVE:
1001                 debug_object_deactivate(obj, &descr_type_test);
1002                 debug_object_destroy(obj, &descr_type_test);
1003                 return true;
1004         default:
1005                 return false;
1006         }
1007 }
1008
1009 /*
1010  * fixup_free is called when:
1011  * - an active object is freed
1012  */
1013 static bool __init fixup_free(void *addr, enum debug_obj_state state)
1014 {
1015         struct self_test *obj = addr;
1016
1017         switch (state) {
1018         case ODEBUG_STATE_ACTIVE:
1019                 debug_object_deactivate(obj, &descr_type_test);
1020                 debug_object_free(obj, &descr_type_test);
1021                 return true;
1022         default:
1023                 return false;
1024         }
1025 }
1026
1027 static int __init
1028 check_results(void *addr, enum debug_obj_state state, int fixups, int warnings)
1029 {
1030         struct debug_bucket *db;
1031         struct debug_obj *obj;
1032         unsigned long flags;
1033         int res = -EINVAL;
1034
1035         db = get_bucket((unsigned long) addr);
1036
1037         raw_spin_lock_irqsave(&db->lock, flags);
1038
1039         obj = lookup_object(addr, db);
1040         if (!obj && state != ODEBUG_STATE_NONE) {
1041                 WARN(1, KERN_ERR "ODEBUG: selftest object not found\n");
1042                 goto out;
1043         }
1044         if (obj && obj->state != state) {
1045                 WARN(1, KERN_ERR "ODEBUG: selftest wrong state: %d != %d\n",
1046                        obj->state, state);
1047                 goto out;
1048         }
1049         if (fixups != debug_objects_fixups) {
1050                 WARN(1, KERN_ERR "ODEBUG: selftest fixups failed %d != %d\n",
1051                        fixups, debug_objects_fixups);
1052                 goto out;
1053         }
1054         if (warnings != debug_objects_warnings) {
1055                 WARN(1, KERN_ERR "ODEBUG: selftest warnings failed %d != %d\n",
1056                        warnings, debug_objects_warnings);
1057                 goto out;
1058         }
1059         res = 0;
1060 out:
1061         raw_spin_unlock_irqrestore(&db->lock, flags);
1062         if (res)
1063                 debug_objects_enabled = 0;
1064         return res;
1065 }
1066
1067 static __initdata struct debug_obj_descr descr_type_test = {
1068         .name                   = "selftest",
1069         .is_static_object       = is_static_object,
1070         .fixup_init             = fixup_init,
1071         .fixup_activate         = fixup_activate,
1072         .fixup_destroy          = fixup_destroy,
1073         .fixup_free             = fixup_free,
1074 };
1075
1076 static __initdata struct self_test obj = { .static_init = 0 };
1077
1078 static void __init debug_objects_selftest(void)
1079 {
1080         int fixups, oldfixups, warnings, oldwarnings;
1081         unsigned long flags;
1082
1083         local_irq_save(flags);
1084
1085         fixups = oldfixups = debug_objects_fixups;
1086         warnings = oldwarnings = debug_objects_warnings;
1087         descr_test = &descr_type_test;
1088
1089         debug_object_init(&obj, &descr_type_test);
1090         if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
1091                 goto out;
1092         debug_object_activate(&obj, &descr_type_test);
1093         if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1094                 goto out;
1095         debug_object_activate(&obj, &descr_type_test);
1096         if (check_results(&obj, ODEBUG_STATE_ACTIVE, ++fixups, ++warnings))
1097                 goto out;
1098         debug_object_deactivate(&obj, &descr_type_test);
1099         if (check_results(&obj, ODEBUG_STATE_INACTIVE, fixups, warnings))
1100                 goto out;
1101         debug_object_destroy(&obj, &descr_type_test);
1102         if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, warnings))
1103                 goto out;
1104         debug_object_init(&obj, &descr_type_test);
1105         if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1106                 goto out;
1107         debug_object_activate(&obj, &descr_type_test);
1108         if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1109                 goto out;
1110         debug_object_deactivate(&obj, &descr_type_test);
1111         if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1112                 goto out;
1113         debug_object_free(&obj, &descr_type_test);
1114         if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
1115                 goto out;
1116
1117         obj.static_init = 1;
1118         debug_object_activate(&obj, &descr_type_test);
1119         if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1120                 goto out;
1121         debug_object_init(&obj, &descr_type_test);
1122         if (check_results(&obj, ODEBUG_STATE_INIT, ++fixups, ++warnings))
1123                 goto out;
1124         debug_object_free(&obj, &descr_type_test);
1125         if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
1126                 goto out;
1127
1128 #ifdef CONFIG_DEBUG_OBJECTS_FREE
1129         debug_object_init(&obj, &descr_type_test);
1130         if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
1131                 goto out;
1132         debug_object_activate(&obj, &descr_type_test);
1133         if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1134                 goto out;
1135         __debug_check_no_obj_freed(&obj, sizeof(obj));
1136         if (check_results(&obj, ODEBUG_STATE_NONE, ++fixups, ++warnings))
1137                 goto out;
1138 #endif
1139         pr_info("selftest passed\n");
1140
1141 out:
1142         debug_objects_fixups = oldfixups;
1143         debug_objects_warnings = oldwarnings;
1144         descr_test = NULL;
1145
1146         local_irq_restore(flags);
1147 }
1148 #else
1149 static inline void debug_objects_selftest(void) { }
1150 #endif
1151
1152 /*
1153  * Called during early boot to initialize the hash buckets and link
1154  * the static object pool objects into the poll list. After this call
1155  * the object tracker is fully operational.
1156  */
1157 void __init debug_objects_early_init(void)
1158 {
1159         int i;
1160
1161         for (i = 0; i < ODEBUG_HASH_SIZE; i++)
1162                 raw_spin_lock_init(&obj_hash[i].lock);
1163
1164         for (i = 0; i < ODEBUG_POOL_SIZE; i++)
1165                 hlist_add_head(&obj_static_pool[i].node, &obj_pool);
1166 }
1167
1168 /*
1169  * Convert the statically allocated objects to dynamic ones:
1170  */
1171 static int __init debug_objects_replace_static_objects(void)
1172 {
1173         struct debug_bucket *db = obj_hash;
1174         struct hlist_node *tmp;
1175         struct debug_obj *obj, *new;
1176         HLIST_HEAD(objects);
1177         int i, cnt = 0;
1178
1179         for (i = 0; i < ODEBUG_POOL_SIZE; i++) {
1180                 obj = kmem_cache_zalloc(obj_cache, GFP_KERNEL);
1181                 if (!obj)
1182                         goto free;
1183                 hlist_add_head(&obj->node, &objects);
1184         }
1185
1186         /*
1187          * debug_objects_mem_init() is now called early that only one CPU is up
1188          * and interrupts have been disabled, so it is safe to replace the
1189          * active object references.
1190          */
1191
1192         /* Remove the statically allocated objects from the pool */
1193         hlist_for_each_entry_safe(obj, tmp, &obj_pool, node)
1194                 hlist_del(&obj->node);
1195         /* Move the allocated objects to the pool */
1196         hlist_move_list(&objects, &obj_pool);
1197
1198         /* Replace the active object references */
1199         for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
1200                 hlist_move_list(&db->list, &objects);
1201
1202                 hlist_for_each_entry(obj, &objects, node) {
1203                         new = hlist_entry(obj_pool.first, typeof(*obj), node);
1204                         hlist_del(&new->node);
1205                         /* copy object data */
1206                         *new = *obj;
1207                         hlist_add_head(&new->node, &db->list);
1208                         cnt++;
1209                 }
1210         }
1211
1212         pr_debug("%d of %d active objects replaced\n",
1213                  cnt, obj_pool_used);
1214         return 0;
1215 free:
1216         hlist_for_each_entry_safe(obj, tmp, &objects, node) {
1217                 hlist_del(&obj->node);
1218                 kmem_cache_free(obj_cache, obj);
1219         }
1220         return -ENOMEM;
1221 }
1222
1223 /*
1224  * Called after the kmem_caches are functional to setup a dedicated
1225  * cache pool, which has the SLAB_DEBUG_OBJECTS flag set. This flag
1226  * prevents that the debug code is called on kmem_cache_free() for the
1227  * debug tracker objects to avoid recursive calls.
1228  */
1229 void __init debug_objects_mem_init(void)
1230 {
1231         int cpu;
1232
1233         if (!debug_objects_enabled)
1234                 return;
1235
1236         /*
1237          * Initialize the percpu object pools
1238          *
1239          * Initialization is not strictly necessary, but was done for
1240          * completeness.
1241          */
1242         for_each_possible_cpu(cpu)
1243                 INIT_HLIST_HEAD(&per_cpu(percpu_obj_pool.free_objs, cpu));
1244
1245         obj_cache = kmem_cache_create("debug_objects_cache",
1246                                       sizeof (struct debug_obj), 0,
1247                                       SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE,
1248                                       NULL);
1249
1250         if (!obj_cache || debug_objects_replace_static_objects()) {
1251                 debug_objects_enabled = 0;
1252                 kmem_cache_destroy(obj_cache);
1253                 pr_warn("out of memory.\n");
1254         } else
1255                 debug_objects_selftest();
1256 }