kernel/livepatch/core.c

   1 /*
   2  * core.c - Kernel Live Patching Core
   3  *
   4  * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
   5  * Copyright (C) 2014 SUSE
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation; either version 2
  10  * of the License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  22
  23 #include <linux/module.h>
  24 #include <linux/kernel.h>
  25 #include <linux/mutex.h>
  26 #include <linux/slab.h>
  27 #include <linux/list.h>
  28 #include <linux/kallsyms.h>
  29 #include <linux/livepatch.h>
  30 #include <linux/elf.h>
  31 #include <linux/moduleloader.h>
  32 #include <linux/completion.h>
  33 #include <asm/cacheflush.h>
  34 #include "core.h"
  35 #include "patch.h"
  36 #include "transition.h"
  37
  38 /*
  39  * klp_mutex is a coarse lock which serializes access to klp data.  All
  40  * accesses to klp-related variables and structures must have mutex protection,
  41  * except within the following functions which carefully avoid the need for it:
  42  *
  43  * - klp_ftrace_handler()
  44  * - klp_update_patch_state()
  45  */
  46 DEFINE_MUTEX(klp_mutex);
  47
  48 static LIST_HEAD(klp_patches);
  49
  50 static struct kobject *klp_root_kobj;
  51
  52 static bool klp_is_module(struct klp_object *obj)
  53 {
  54         return obj->name;
  55 }
  56
  57 /* sets obj->mod if object is not vmlinux and module is found */
  58 static void klp_find_object_module(struct klp_object *obj)
  59 {
  60         struct module *mod;
  61
  62         if (!klp_is_module(obj))
  63                 return;
  64
  65         mutex_lock(&module_mutex);
  66         /*
  67          * We do not want to block removal of patched modules and therefore
  68          * we do not take a reference here. The patches are removed by
  69          * klp_module_going() instead.
  70          */
  71         mod = find_module(obj->name);
  72         /*
  73          * Do not mess work of klp_module_coming() and klp_module_going().
  74          * Note that the patch might still be needed before klp_module_going()
  75          * is called. Module functions can be called even in the GOING state
  76          * until mod->exit() finishes. This is especially important for
  77          * patches that modify semantic of the functions.
  78          */
  79         if (mod && mod->klp_alive)
  80                 obj->mod = mod;
  81
  82         mutex_unlock(&module_mutex);
  83 }
  84
  85 static bool klp_is_patch_registered(struct klp_patch *patch)
  86 {
  87         struct klp_patch *mypatch;
  88
  89         list_for_each_entry(mypatch, &klp_patches, list)
  90                 if (mypatch == patch)
  91                         return true;
  92
  93         return false;
  94 }
  95
  96 static bool klp_initialized(void)
  97 {
  98         return !!klp_root_kobj;
  99 }
 100
 101 struct klp_find_arg {
 102         const char *objname;
 103         const char *name;
 104         unsigned long addr;
 105         unsigned long count;
 106         unsigned long pos;
 107 };
 108
 109 static int klp_find_callback(void *data, const char *name,
 110                              struct module *mod, unsigned long addr)
 111 {
 112         struct klp_find_arg *args = data;
 113
 114         if ((mod && !args->objname) || (!mod && args->objname))
 115                 return 0;
 116
 117         if (strcmp(args->name, name))
 118                 return 0;
 119
 120         if (args->objname && strcmp(args->objname, mod->name))
 121                 return 0;
 122
 123         args->addr = addr;
 124         args->count++;
 125
 126         /*
 127          * Finish the search when the symbol is found for the desired position
 128          * or the position is not defined for a non-unique symbol.
 129          */
 130         if ((args->pos && (args->count == args->pos)) ||
 131             (!args->pos && (args->count > 1)))
 132                 return 1;
 133
 134         return 0;
 135 }
 136
 137 static int klp_find_object_symbol(const char *objname, const char *name,
 138                                   unsigned long sympos, unsigned long *addr)
 139 {
 140         struct klp_find_arg args = {
 141                 .objname = objname,
 142                 .name = name,
 143                 .addr = 0,
 144                 .count = 0,
 145                 .pos = sympos,
 146         };
 147
 148         mutex_lock(&module_mutex);
 149         if (objname)
 150                 module_kallsyms_on_each_symbol(klp_find_callback, &args);
 151         else
 152                 kallsyms_on_each_symbol(klp_find_callback, &args);
 153         mutex_unlock(&module_mutex);
 154
 155         /*
 156          * Ensure an address was found. If sympos is 0, ensure symbol is unique;
 157          * otherwise ensure the symbol position count matches sympos.
 158          */
 159         if (args.addr == 0)
 160                 pr_err("symbol '%s' not found in symbol table\n", name);
 161         else if (args.count > 1 && sympos == 0) {
 162                 pr_err("unresolvable ambiguity for symbol '%s' in object '%s'\n",
 163                        name, objname);
 164         } else if (sympos != args.count && sympos > 0) {
 165                 pr_err("symbol position %lu for symbol '%s' in object '%s' not found\n",
 166                        sympos, name, objname ? objname : "vmlinux");
 167         } else {
 168                 *addr = args.addr;
 169                 return 0;
 170         }
 171
 172         *addr = 0;
 173         return -EINVAL;
 174 }
 175
 176 static int klp_resolve_symbols(Elf_Shdr *relasec, struct module *pmod)
 177 {
 178         int i, cnt, vmlinux, ret;
 179         char objname[MODULE_NAME_LEN];
 180         char symname[KSYM_NAME_LEN];
 181         char *strtab = pmod->core_kallsyms.strtab;
 182         Elf_Rela *relas;
 183         Elf_Sym *sym;
 184         unsigned long sympos, addr;
 185
 186         /*
 187          * Since the field widths for objname and symname in the sscanf()
 188          * call are hard-coded and correspond to MODULE_NAME_LEN and
 189          * KSYM_NAME_LEN respectively, we must make sure that MODULE_NAME_LEN
 190          * and KSYM_NAME_LEN have the values we expect them to have.
 191          *
 192          * Because the value of MODULE_NAME_LEN can differ among architectures,
 193          * we use the smallest/strictest upper bound possible (56, based on
 194          * the current definition of MODULE_NAME_LEN) to prevent overflows.
 195          */
 196         BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 128);
 197
 198         relas = (Elf_Rela *) relasec->sh_addr;
 199         /* For each rela in this klp relocation section */
 200         for (i = 0; i < relasec->sh_size / sizeof(Elf_Rela); i++) {
 201                 sym = pmod->core_kallsyms.symtab + ELF_R_SYM(relas[i].r_info);
 202                 if (sym->st_shndx != SHN_LIVEPATCH) {
 203                         pr_err("symbol %s is not marked as a livepatch symbol\n",
 204                                strtab + sym->st_name);
 205                         return -EINVAL;
 206                 }
 207
 208                 /* Format: .klp.sym.objname.symname,sympos */
 209                 cnt = sscanf(strtab + sym->st_name,
 210                              ".klp.sym.%55[^.].%127[^,],%lu",
 211                              objname, symname, &sympos);
 212                 if (cnt != 3) {
 213                         pr_err("symbol %s has an incorrectly formatted name\n",
 214                                strtab + sym->st_name);
 215                         return -EINVAL;
 216                 }
 217
 218                 /* klp_find_object_symbol() treats a NULL objname as vmlinux */
 219                 vmlinux = !strcmp(objname, "vmlinux");
 220                 ret = klp_find_object_symbol(vmlinux ? NULL : objname,
 221                                              symname, sympos, &addr);
 222                 if (ret)
 223                         return ret;
 224
 225                 sym->st_value = addr;
 226         }
 227
 228         return 0;
 229 }
 230
 231 static int klp_write_object_relocations(struct module *pmod,
 232                                         struct klp_object *obj)
 233 {
 234         int i, cnt, ret = 0;
 235         const char *objname, *secname;
 236         char sec_objname[MODULE_NAME_LEN];
 237         Elf_Shdr *sec;
 238
 239         if (WARN_ON(!klp_is_object_loaded(obj)))
 240                 return -EINVAL;
 241
 242         objname = klp_is_module(obj) ? obj->name : "vmlinux";
 243
 244         /* For each klp relocation section */
 245         for (i = 1; i < pmod->klp_info->hdr.e_shnum; i++) {
 246                 sec = pmod->klp_info->sechdrs + i;
 247                 secname = pmod->klp_info->secstrings + sec->sh_name;
 248                 if (!(sec->sh_flags & SHF_RELA_LIVEPATCH))
 249                         continue;
 250
 251                 /*
 252                  * Format: .klp.rela.sec_objname.section_name
 253                  * See comment in klp_resolve_symbols() for an explanation
 254                  * of the selected field width value.
 255                  */
 256                 cnt = sscanf(secname, ".klp.rela.%55[^.]", sec_objname);
 257                 if (cnt != 1) {
 258                         pr_err("section %s has an incorrectly formatted name\n",
 259                                secname);
 260                         ret = -EINVAL;
 261                         break;
 262                 }
 263
 264                 if (strcmp(objname, sec_objname))
 265                         continue;
 266
 267                 ret = klp_resolve_symbols(sec, pmod);
 268                 if (ret)
 269                         break;
 270
 271                 ret = apply_relocate_add(pmod->klp_info->sechdrs,
 272                                          pmod->core_kallsyms.strtab,
 273                                          pmod->klp_info->symndx, i, pmod);
 274                 if (ret)
 275                         break;
 276         }
 277
 278         return ret;
 279 }
 280
 281 static int __klp_disable_patch(struct klp_patch *patch)
 282 {
 283         struct klp_object *obj;
 284
 285         if (WARN_ON(!patch->enabled))
 286                 return -EINVAL;
 287
 288         if (klp_transition_patch)
 289                 return -EBUSY;
 290
 291         /* enforce stacking: only the last enabled patch can be disabled */
 292         if (!list_is_last(&patch->list, &klp_patches) &&
 293             list_next_entry(patch, list)->enabled)
 294                 return -EBUSY;
 295
 296         klp_init_transition(patch, KLP_UNPATCHED);
 297
 298         klp_for_each_object(patch, obj)
 299                 if (obj->patched)
 300                         klp_pre_unpatch_callback(obj);
 301
 302         /*
 303          * Enforce the order of the func->transition writes in
 304          * klp_init_transition() and the TIF_PATCH_PENDING writes in
 305          * klp_start_transition().  In the rare case where klp_ftrace_handler()
 306          * is called shortly after klp_update_patch_state() switches the task,
 307          * this ensures the handler sees that func->transition is set.
 308          */
 309         smp_wmb();
 310
 311         klp_start_transition();
 312         klp_try_complete_transition();
 313         patch->enabled = false;
 314
 315         return 0;
 316 }
 317
 318 /**
 319  * klp_disable_patch() - disables a registered patch
 320  * @patch:      The registered, enabled patch to be disabled
 321  *
 322  * Unregisters the patched functions from ftrace.
 323  *
 324  * Return: 0 on success, otherwise error
 325  */
 326 int klp_disable_patch(struct klp_patch *patch)
 327 {
 328         int ret;
 329
 330         mutex_lock(&klp_mutex);
 331
 332         if (!klp_is_patch_registered(patch)) {
 333                 ret = -EINVAL;
 334                 goto err;
 335         }
 336
 337         if (!patch->enabled) {
 338                 ret = -EINVAL;
 339                 goto err;
 340         }
 341
 342         ret = __klp_disable_patch(patch);
 343
 344 err:
 345         mutex_unlock(&klp_mutex);
 346         return ret;
 347 }
 348 EXPORT_SYMBOL_GPL(klp_disable_patch);
 349
 350 static int __klp_enable_patch(struct klp_patch *patch)
 351 {
 352         struct klp_object *obj;
 353         int ret;
 354
 355         if (klp_transition_patch)
 356                 return -EBUSY;
 357
 358         if (WARN_ON(patch->enabled))
 359                 return -EINVAL;
 360
 361         /* enforce stacking: only the first disabled patch can be enabled */
 362         if (patch->list.prev != &klp_patches &&
 363             !list_prev_entry(patch, list)->enabled)
 364                 return -EBUSY;
 365
 366         /*
 367          * A reference is taken on the patch module to prevent it from being
 368          * unloaded.
 369          *
 370          * Note: For immediate (no consistency model) patches we don't allow
 371          * patch modules to unload since there is no safe/sane method to
 372          * determine if a thread is still running in the patched code contained
 373          * in the patch module once the ftrace registration is successful.
 374          */
 375         if (!try_module_get(patch->mod))
 376                 return -ENODEV;
 377
 378         pr_notice("enabling patch '%s'\n", patch->mod->name);
 379
 380         klp_init_transition(patch, KLP_PATCHED);
 381
 382         /*
 383          * Enforce the order of the func->transition writes in
 384          * klp_init_transition() and the ops->func_stack writes in
 385          * klp_patch_object(), so that klp_ftrace_handler() will see the
 386          * func->transition updates before the handler is registered and the
 387          * new funcs become visible to the handler.
 388          */
 389         smp_wmb();
 390
 391         klp_for_each_object(patch, obj) {
 392                 if (!klp_is_object_loaded(obj))
 393                         continue;
 394
 395                 ret = klp_pre_patch_callback(obj);
 396                 if (ret) {
 397                         pr_warn("pre-patch callback failed for object '%s'\n",
 398                                 klp_is_module(obj) ? obj->name : "vmlinux");
 399                         goto err;
 400                 }
 401
 402                 ret = klp_patch_object(obj);
 403                 if (ret) {
 404                         pr_warn("failed to patch object '%s'\n",
 405                                 klp_is_module(obj) ? obj->name : "vmlinux");
 406                         goto err;
 407                 }
 408         }
 409
 410         klp_start_transition();
 411         klp_try_complete_transition();
 412         patch->enabled = true;
 413
 414         return 0;
 415 err:
 416         pr_warn("failed to enable patch '%s'\n", patch->mod->name);
 417
 418         klp_cancel_transition();
 419         return ret;
 420 }
 421
 422 /**
 423  * klp_enable_patch() - enables a registered patch
 424  * @patch:      The registered, disabled patch to be enabled
 425  *
 426  * Performs the needed symbol lookups and code relocations,
 427  * then registers the patched functions with ftrace.
 428  *
 429  * Return: 0 on success, otherwise error
 430  */
 431 int klp_enable_patch(struct klp_patch *patch)
 432 {
 433         int ret;
 434
 435         mutex_lock(&klp_mutex);
 436
 437         if (!klp_is_patch_registered(patch)) {
 438                 ret = -EINVAL;
 439                 goto err;
 440         }
 441
 442         ret = __klp_enable_patch(patch);
 443
 444 err:
 445         mutex_unlock(&klp_mutex);
 446         return ret;
 447 }
 448 EXPORT_SYMBOL_GPL(klp_enable_patch);
 449
 450 /*
 451  * Sysfs Interface
 452  *
 453  * /sys/kernel/livepatch
 454  * /sys/kernel/livepatch/<patch>
 455  * /sys/kernel/livepatch/<patch>/enabled
 456  * /sys/kernel/livepatch/<patch>/transition
 457  * /sys/kernel/livepatch/<patch>/<object>
 458  * /sys/kernel/livepatch/<patch>/<object>/<function,sympos>
 459  */
 460
 461 static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
 462                              const char *buf, size_t count)
 463 {
 464         struct klp_patch *patch;
 465         int ret;
 466         bool enabled;
 467
 468         ret = kstrtobool(buf, &enabled);
 469         if (ret)
 470                 return ret;
 471
 472         patch = container_of(kobj, struct klp_patch, kobj);
 473
 474         mutex_lock(&klp_mutex);
 475
 476         if (!klp_is_patch_registered(patch)) {
 477                 /*
 478                  * Module with the patch could either disappear meanwhile or is
 479                  * not properly initialized yet.
 480                  */
 481                 ret = -EINVAL;
 482                 goto err;
 483         }
 484
 485         if (patch->enabled == enabled) {
 486                 /* already in requested state */
 487                 ret = -EINVAL;
 488                 goto err;
 489         }
 490
 491         if (patch == klp_transition_patch) {
 492                 klp_reverse_transition();
 493         } else if (enabled) {
 494                 ret = __klp_enable_patch(patch);
 495                 if (ret)
 496                         goto err;
 497         } else {
 498                 ret = __klp_disable_patch(patch);
 499                 if (ret)
 500                         goto err;
 501         }
 502
 503         mutex_unlock(&klp_mutex);
 504
 505         return count;
 506
 507 err:
 508         mutex_unlock(&klp_mutex);
 509         return ret;
 510 }
 511
 512 static ssize_t enabled_show(struct kobject *kobj,
 513                             struct kobj_attribute *attr, char *buf)
 514 {
 515         struct klp_patch *patch;
 516
 517         patch = container_of(kobj, struct klp_patch, kobj);
 518         return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled);
 519 }
 520
 521 static ssize_t transition_show(struct kobject *kobj,
 522                                struct kobj_attribute *attr, char *buf)
 523 {
 524         struct klp_patch *patch;
 525
 526         patch = container_of(kobj, struct klp_patch, kobj);
 527         return snprintf(buf, PAGE_SIZE-1, "%d\n",
 528                         patch == klp_transition_patch);
 529 }
 530
 531 static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled);
 532 static struct kobj_attribute transition_kobj_attr = __ATTR_RO(transition);
 533 static struct attribute *klp_patch_attrs[] = {
 534         &enabled_kobj_attr.attr,
 535         &transition_kobj_attr.attr,
 536         NULL
 537 };
 538
 539 static void klp_kobj_release_patch(struct kobject *kobj)
 540 {
 541         struct klp_patch *patch;
 542
 543         patch = container_of(kobj, struct klp_patch, kobj);
 544         complete(&patch->finish);
 545 }
 546
 547 static struct kobj_type klp_ktype_patch = {
 548         .release = klp_kobj_release_patch,
 549         .sysfs_ops = &kobj_sysfs_ops,
 550         .default_attrs = klp_patch_attrs,
 551 };
 552
 553 static void klp_kobj_release_object(struct kobject *kobj)
 554 {
 555 }
 556
 557 static struct kobj_type klp_ktype_object = {
 558         .release = klp_kobj_release_object,
 559         .sysfs_ops = &kobj_sysfs_ops,
 560 };
 561
 562 static void klp_kobj_release_func(struct kobject *kobj)
 563 {
 564 }
 565
 566 static struct kobj_type klp_ktype_func = {
 567         .release = klp_kobj_release_func,
 568         .sysfs_ops = &kobj_sysfs_ops,
 569 };
 570
 571 /*
 572  * Free all functions' kobjects in the array up to some limit. When limit is
 573  * NULL, all kobjects are freed.
 574  */
 575 static void klp_free_funcs_limited(struct klp_object *obj,
 576                                    struct klp_func *limit)
 577 {
 578         struct klp_func *func;
 579
 580         for (func = obj->funcs; func->old_name && func != limit; func++)
 581                 kobject_put(&func->kobj);
 582 }
 583
 584 /* Clean up when a patched object is unloaded */
 585 static void klp_free_object_loaded(struct klp_object *obj)
 586 {
 587         struct klp_func *func;
 588
 589         obj->mod = NULL;
 590
 591         klp_for_each_func(obj, func)
 592                 func->old_addr = 0;
 593 }
 594
 595 /*
 596  * Free all objects' kobjects in the array up to some limit. When limit is
 597  * NULL, all kobjects are freed.
 598  */
 599 static void klp_free_objects_limited(struct klp_patch *patch,
 600                                      struct klp_object *limit)
 601 {
 602         struct klp_object *obj;
 603
 604         for (obj = patch->objs; obj->funcs && obj != limit; obj++) {
 605                 klp_free_funcs_limited(obj, NULL);
 606                 kobject_put(&obj->kobj);
 607         }
 608 }
 609
 610 static void klp_free_patch(struct klp_patch *patch)
 611 {
 612         klp_free_objects_limited(patch, NULL);
 613         if (!list_empty(&patch->list))
 614                 list_del(&patch->list);
 615 }
 616
 617 static int klp_init_func(struct klp_object *obj, struct klp_func *func)
 618 {
 619         if (!func->old_name || !func->new_func)
 620                 return -EINVAL;
 621
 622         INIT_LIST_HEAD(&func->stack_node);
 623         func->patched = false;
 624         func->transition = false;
 625
 626         /* The format for the sysfs directory is <function,sympos> where sympos
 627          * is the nth occurrence of this symbol in kallsyms for the patched
 628          * object. If the user selects 0 for old_sympos, then 1 will be used
 629          * since a unique symbol will be the first occurrence.
 630          */
 631         return kobject_init_and_add(&func->kobj, &klp_ktype_func,
 632                                     &obj->kobj, "%s,%lu", func->old_name,
 633                                     func->old_sympos ? func->old_sympos : 1);
 634 }
 635
 636 /* Arches may override this to finish any remaining arch-specific tasks */
 637 void __weak arch_klp_init_object_loaded(struct klp_patch *patch,
 638                                         struct klp_object *obj)
 639 {
 640 }
 641
 642 /* parts of the initialization that is done only when the object is loaded */
 643 static int klp_init_object_loaded(struct klp_patch *patch,
 644                                   struct klp_object *obj)
 645 {
 646         struct klp_func *func;
 647         int ret;
 648
 649         module_disable_ro(patch->mod);
 650         ret = klp_write_object_relocations(patch->mod, obj);
 651         if (ret) {
 652                 module_enable_ro(patch->mod, true);
 653                 return ret;
 654         }
 655
 656         arch_klp_init_object_loaded(patch, obj);
 657         module_enable_ro(patch->mod, true);
 658
 659         klp_for_each_func(obj, func) {
 660                 ret = klp_find_object_symbol(obj->name, func->old_name,
 661                                              func->old_sympos,
 662                                              &func->old_addr);
 663                 if (ret)
 664                         return ret;
 665
 666                 ret = kallsyms_lookup_size_offset(func->old_addr,
 667                                                   &func->old_size, NULL);
 668                 if (!ret) {
 669                         pr_err("kallsyms size lookup failed for '%s'\n",
 670                                func->old_name);
 671                         return -ENOENT;
 672                 }
 673
 674                 ret = kallsyms_lookup_size_offset((unsigned long)func->new_func,
 675                                                   &func->new_size, NULL);
 676                 if (!ret) {
 677                         pr_err("kallsyms size lookup failed for '%s' replacement\n",
 678                                func->old_name);
 679                         return -ENOENT;
 680                 }
 681         }
 682
 683         return 0;
 684 }
 685
 686 static int klp_init_object(struct klp_patch *patch, struct klp_object *obj)
 687 {
 688         struct klp_func *func;
 689         int ret;
 690         const char *name;
 691
 692         if (!obj->funcs)
 693                 return -EINVAL;
 694
 695         obj->patched = false;
 696         obj->mod = NULL;
 697
 698         klp_find_object_module(obj);
 699
 700         name = klp_is_module(obj) ? obj->name : "vmlinux";
 701         ret = kobject_init_and_add(&obj->kobj, &klp_ktype_object,
 702                                    &patch->kobj, "%s", name);
 703         if (ret)
 704                 return ret;
 705
 706         klp_for_each_func(obj, func) {
 707                 ret = klp_init_func(obj, func);
 708                 if (ret)
 709                         goto free;
 710         }
 711
 712         if (klp_is_object_loaded(obj)) {
 713                 ret = klp_init_object_loaded(patch, obj);
 714                 if (ret)
 715                         goto free;
 716         }
 717
 718         return 0;
 719
 720 free:
 721         klp_free_funcs_limited(obj, func);
 722         kobject_put(&obj->kobj);
 723         return ret;
 724 }
 725
 726 static int klp_init_patch(struct klp_patch *patch)
 727 {
 728         struct klp_object *obj;
 729         int ret;
 730
 731         if (!patch->objs)
 732                 return -EINVAL;
 733
 734         mutex_lock(&klp_mutex);
 735
 736         patch->enabled = false;
 737         init_completion(&patch->finish);
 738
 739         ret = kobject_init_and_add(&patch->kobj, &klp_ktype_patch,
 740                                    klp_root_kobj, "%s", patch->mod->name);
 741         if (ret) {
 742                 mutex_unlock(&klp_mutex);
 743                 return ret;
 744         }
 745
 746         klp_for_each_object(patch, obj) {
 747                 ret = klp_init_object(patch, obj);
 748                 if (ret)
 749                         goto free;
 750         }
 751
 752         list_add_tail(&patch->list, &klp_patches);
 753
 754         mutex_unlock(&klp_mutex);
 755
 756         return 0;
 757
 758 free:
 759         klp_free_objects_limited(patch, obj);
 760
 761         mutex_unlock(&klp_mutex);
 762
 763         kobject_put(&patch->kobj);
 764         wait_for_completion(&patch->finish);
 765
 766         return ret;
 767 }
 768
 769 /**
 770  * klp_unregister_patch() - unregisters a patch
 771  * @patch:      Disabled patch to be unregistered
 772  *
 773  * Frees the data structures and removes the sysfs interface.
 774  *
 775  * Return: 0 on success, otherwise error
 776  */
 777 int klp_unregister_patch(struct klp_patch *patch)
 778 {
 779         int ret;
 780
 781         mutex_lock(&klp_mutex);
 782
 783         if (!klp_is_patch_registered(patch)) {
 784                 ret = -EINVAL;
 785                 goto err;
 786         }
 787
 788         if (patch->enabled) {
 789                 ret = -EBUSY;
 790                 goto err;
 791         }
 792
 793         klp_free_patch(patch);
 794
 795         mutex_unlock(&klp_mutex);
 796
 797         kobject_put(&patch->kobj);
 798         wait_for_completion(&patch->finish);
 799
 800         return 0;
 801 err:
 802         mutex_unlock(&klp_mutex);
 803         return ret;
 804 }
 805 EXPORT_SYMBOL_GPL(klp_unregister_patch);
 806
 807 /**
 808  * klp_register_patch() - registers a patch
 809  * @patch:      Patch to be registered
 810  *
 811  * Initializes the data structure associated with the patch and
 812  * creates the sysfs interface.
 813  *
 814  * There is no need to take the reference on the patch module here. It is done
 815  * later when the patch is enabled.
 816  *
 817  * Return: 0 on success, otherwise error
 818  */
 819 int klp_register_patch(struct klp_patch *patch)
 820 {
 821         if (!patch || !patch->mod)
 822                 return -EINVAL;
 823
 824         if (!is_livepatch_module(patch->mod)) {
 825                 pr_err("module %s is not marked as a livepatch module\n",
 826                        patch->mod->name);
 827                 return -EINVAL;
 828         }
 829
 830         if (!klp_initialized())
 831                 return -ENODEV;
 832
 833         /*
 834          * Architectures without reliable stack traces have to set
 835          * patch->immediate because there's currently no way to patch kthreads
 836          * with the consistency model.
 837          */
 838         if (!klp_have_reliable_stack() && !patch->immediate) {
 839                 pr_err("This architecture doesn't have support for the livepatch consistency model.\n");
 840                 return -ENOSYS;
 841         }
 842
 843         return klp_init_patch(patch);
 844 }
 845 EXPORT_SYMBOL_GPL(klp_register_patch);
 846
 847 /*
 848  * Remove parts of patches that touch a given kernel module. The list of
 849  * patches processed might be limited. When limit is NULL, all patches
 850  * will be handled.
 851  */
 852 static void klp_cleanup_module_patches_limited(struct module *mod,
 853                                                struct klp_patch *limit)
 854 {
 855         struct klp_patch *patch;
 856         struct klp_object *obj;
 857
 858         list_for_each_entry(patch, &klp_patches, list) {
 859                 if (patch == limit)
 860                         break;
 861
 862                 klp_for_each_object(patch, obj) {
 863                         if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
 864                                 continue;
 865
 866                         /*
 867                          * Only unpatch the module if the patch is enabled or
 868                          * is in transition.
 869                          */
 870                         if (patch->enabled || patch == klp_transition_patch) {
 871
 872                                 if (patch != klp_transition_patch)
 873                                         klp_pre_unpatch_callback(obj);
 874
 875                                 pr_notice("reverting patch '%s' on unloading module '%s'\n",
 876                                           patch->mod->name, obj->mod->name);
 877                                 klp_unpatch_object(obj);
 878
 879                                 klp_post_unpatch_callback(obj);
 880                         }
 881
 882                         klp_free_object_loaded(obj);
 883                         break;
 884                 }
 885         }
 886 }
 887
 888 int klp_module_coming(struct module *mod)
 889 {
 890         int ret;
 891         struct klp_patch *patch;
 892         struct klp_object *obj;
 893
 894         if (WARN_ON(mod->state != MODULE_STATE_COMING))
 895                 return -EINVAL;
 896
 897         mutex_lock(&klp_mutex);
 898         /*
 899          * Each module has to know that klp_module_coming()
 900          * has been called. We never know what module will
 901          * get patched by a new patch.
 902          */
 903         mod->klp_alive = true;
 904
 905         list_for_each_entry(patch, &klp_patches, list) {
 906                 klp_for_each_object(patch, obj) {
 907                         if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
 908                                 continue;
 909
 910                         obj->mod = mod;
 911
 912                         ret = klp_init_object_loaded(patch, obj);
 913                         if (ret) {
 914                                 pr_warn("failed to initialize patch '%s' for module '%s' (%d)\n",
 915                                         patch->mod->name, obj->mod->name, ret);
 916                                 goto err;
 917                         }
 918
 919                         /*
 920                          * Only patch the module if the patch is enabled or is
 921                          * in transition.
 922                          */
 923                         if (!patch->enabled && patch != klp_transition_patch)
 924                                 break;
 925
 926                         pr_notice("applying patch '%s' to loading module '%s'\n",
 927                                   patch->mod->name, obj->mod->name);
 928
 929                         ret = klp_pre_patch_callback(obj);
 930                         if (ret) {
 931                                 pr_warn("pre-patch callback failed for object '%s'\n",
 932                                         obj->name);
 933                                 goto err;
 934                         }
 935
 936                         ret = klp_patch_object(obj);
 937                         if (ret) {
 938                                 pr_warn("failed to apply patch '%s' to module '%s' (%d)\n",
 939                                         patch->mod->name, obj->mod->name, ret);
 940
 941                                 klp_post_unpatch_callback(obj);
 942                                 goto err;
 943                         }
 944
 945                         if (patch != klp_transition_patch)
 946                                 klp_post_patch_callback(obj);
 947
 948                         break;
 949                 }
 950         }
 951
 952         mutex_unlock(&klp_mutex);
 953
 954         return 0;
 955
 956 err:
 957         /*
 958          * If a patch is unsuccessfully applied, return
 959          * error to the module loader.
 960          */
 961         pr_warn("patch '%s' failed for module '%s', refusing to load module '%s'\n",
 962                 patch->mod->name, obj->mod->name, obj->mod->name);
 963         mod->klp_alive = false;
 964         klp_cleanup_module_patches_limited(mod, patch);
 965         mutex_unlock(&klp_mutex);
 966
 967         return ret;
 968 }
 969
 970 void klp_module_going(struct module *mod)
 971 {
 972         if (WARN_ON(mod->state != MODULE_STATE_GOING &&
 973                     mod->state != MODULE_STATE_COMING))
 974                 return;
 975
 976         mutex_lock(&klp_mutex);
 977         /*
 978          * Each module has to know that klp_module_going()
 979          * has been called. We never know what module will
 980          * get patched by a new patch.
 981          */
 982         mod->klp_alive = false;
 983
 984         klp_cleanup_module_patches_limited(mod, NULL);
 985
 986         mutex_unlock(&klp_mutex);
 987 }
 988
 989 static int __init klp_init(void)
 990 {
 991         int ret;
 992
 993         ret = klp_check_compiler_support();
 994         if (ret) {
 995                 pr_info("Your compiler is too old; turning off.\n");
 996                 return -EINVAL;
 997         }
 998
 999         klp_root_kobj = kobject_create_and_add("livepatch", kernel_kobj);
1000         if (!klp_root_kobj)
1001                 return -ENOMEM;
1002
1003         return 0;
1004 }
1005
1006 module_init(klp_init);