arch/x86/platform/efi/efi.c

   1 /*
   2  * Common EFI (Extensible Firmware Interface) support functions
   3  * Based on Extensible Firmware Interface Specification version 1.0
   4  *
   5  * Copyright (C) 1999 VA Linux Systems
   6  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
   7  * Copyright (C) 1999-2002 Hewlett-Packard Co.
   8  *      David Mosberger-Tang <davidm@hpl.hp.com>
   9  *      Stephane Eranian <eranian@hpl.hp.com>
  10  * Copyright (C) 2005-2008 Intel Co.
  11  *      Fenghua Yu <fenghua.yu@intel.com>
  12  *      Bibo Mao <bibo.mao@intel.com>
  13  *      Chandramouli Narayanan <mouli@linux.intel.com>
  14  *      Huang Ying <ying.huang@intel.com>
  15  * Copyright (C) 2013 SuSE Labs
  16  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
  17  *
  18  * Copied from efi_32.c to eliminate the duplicated code between EFI
  19  * 32/64 support code. --ying 2007-10-26
  20  *
  21  * All EFI Runtime Services are not implemented yet as EFI only
  22  * supports physical mode addressing on SoftSDV. This is to be fixed
  23  * in a future version.  --drummond 1999-07-20
  24  *
  25  * Implemented EFI runtime services and virtual mode calls.  --davidm
  26  *
  27  * Goutham Rao: <goutham.rao@intel.com>
  28  *      Skip non-WB memory and ignore empty memory ranges.
  29  */
  30
  31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  32
  33 #include <linux/kernel.h>
  34 #include <linux/init.h>
  35 #include <linux/efi.h>
  36 #include <linux/efi-bgrt.h>
  37 #include <linux/export.h>
  38 #include <linux/bootmem.h>
  39 #include <linux/slab.h>
  40 #include <linux/memblock.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/time.h>
  44 #include <linux/io.h>
  45 #include <linux/reboot.h>
  46 #include <linux/bcd.h>
  47
  48 #include <asm/setup.h>
  49 #include <asm/efi.h>
  50 #include <asm/time.h>
  51 #include <asm/cacheflush.h>
  52 #include <asm/tlbflush.h>
  53 #include <asm/x86_init.h>
  54 #include <asm/rtc.h>
  55 #include <asm/uv/uv.h>
  56
  57 #define EFI_DEBUG
  58
  59 struct efi_memory_map memmap;
  60
  61 static struct efi efi_phys __initdata;
  62 static efi_system_table_t efi_systab __initdata;
  63
  64 static efi_config_table_type_t arch_tables[] __initdata = {
  65 #ifdef CONFIG_X86_UV
  66         {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
  67 #endif
  68         {NULL_GUID, NULL, NULL},
  69 };
  70
  71 u64 efi_setup;          /* efi setup_data physical address */
  72
  73 static int add_efi_memmap __initdata;
  74 static int __init setup_add_efi_memmap(char *arg)
  75 {
  76         add_efi_memmap = 1;
  77         return 0;
  78 }
  79 early_param("add_efi_memmap", setup_add_efi_memmap);
  80
  81 static efi_status_t __init phys_efi_set_virtual_address_map(
  82         unsigned long memory_map_size,
  83         unsigned long descriptor_size,
  84         u32 descriptor_version,
  85         efi_memory_desc_t *virtual_map)
  86 {
  87         efi_status_t status;
  88         unsigned long flags;
  89         pgd_t *save_pgd;
  90
  91         save_pgd = efi_call_phys_prolog();
  92
  93         /* Disable interrupts around EFI calls: */
  94         local_irq_save(flags);
  95         status = efi_call_phys(efi_phys.set_virtual_address_map,
  96                                memory_map_size, descriptor_size,
  97                                descriptor_version, virtual_map);
  98         local_irq_restore(flags);
  99
 100         efi_call_phys_epilog(save_pgd);
 101
 102         return status;
 103 }
 104
 105 void efi_get_time(struct timespec *now)
 106 {
 107         efi_status_t status;
 108         efi_time_t eft;
 109         efi_time_cap_t cap;
 110
 111         status = efi.get_time(&eft, &cap);
 112         if (status != EFI_SUCCESS)
 113                 pr_err("Oops: efitime: can't read time!\n");
 114
 115         now->tv_sec = mktime(eft.year, eft.month, eft.day, eft.hour,
 116                              eft.minute, eft.second);
 117         now->tv_nsec = 0;
 118 }
 119
 120 void __init efi_find_mirror(void)
 121 {
 122         void *p;
 123         u64 mirror_size = 0, total_size = 0;
 124
 125         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 126                 efi_memory_desc_t *md = p;
 127                 unsigned long long start = md->phys_addr;
 128                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 129
 130                 total_size += size;
 131                 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
 132                         memblock_mark_mirror(start, size);
 133                         mirror_size += size;
 134                 }
 135         }
 136         if (mirror_size)
 137                 pr_info("Memory: %lldM/%lldM mirrored memory\n",
 138                         mirror_size>>20, total_size>>20);
 139 }
 140
 141 /*
 142  * Tell the kernel about the EFI memory map.  This might include
 143  * more than the max 128 entries that can fit in the e820 legacy
 144  * (zeropage) memory map.
 145  */
 146
 147 static void __init do_add_efi_memmap(void)
 148 {
 149         void *p;
 150
 151         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 152                 efi_memory_desc_t *md = p;
 153                 unsigned long long start = md->phys_addr;
 154                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 155                 int e820_type;
 156
 157                 switch (md->type) {
 158                 case EFI_LOADER_CODE:
 159                 case EFI_LOADER_DATA:
 160                 case EFI_BOOT_SERVICES_CODE:
 161                 case EFI_BOOT_SERVICES_DATA:
 162                 case EFI_CONVENTIONAL_MEMORY:
 163                         if (md->attribute & EFI_MEMORY_WB)
 164                                 e820_type = E820_RAM;
 165                         else
 166                                 e820_type = E820_RESERVED;
 167                         break;
 168                 case EFI_ACPI_RECLAIM_MEMORY:
 169                         e820_type = E820_ACPI;
 170                         break;
 171                 case EFI_ACPI_MEMORY_NVS:
 172                         e820_type = E820_NVS;
 173                         break;
 174                 case EFI_UNUSABLE_MEMORY:
 175                         e820_type = E820_UNUSABLE;
 176                         break;
 177                 default:
 178                         /*
 179                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
 180                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
 181                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
 182                          */
 183                         e820_type = E820_RESERVED;
 184                         break;
 185                 }
 186                 e820_add_region(start, size, e820_type);
 187         }
 188         sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
 189 }
 190
 191 int __init efi_memblock_x86_reserve_range(void)
 192 {
 193         struct efi_info *e = &boot_params.efi_info;
 194         unsigned long pmap;
 195
 196         if (efi_enabled(EFI_PARAVIRT))
 197                 return 0;
 198
 199 #ifdef CONFIG_X86_32
 200         /* Can't handle data above 4GB at this time */
 201         if (e->efi_memmap_hi) {
 202                 pr_err("Memory map is above 4GB, disabling EFI.\n");
 203                 return -EINVAL;
 204         }
 205         pmap =  e->efi_memmap;
 206 #else
 207         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
 208 #endif
 209         memmap.phys_map         = (void *)pmap;
 210         memmap.nr_map           = e->efi_memmap_size /
 211                                   e->efi_memdesc_size;
 212         memmap.desc_size        = e->efi_memdesc_size;
 213         memmap.desc_version     = e->efi_memdesc_version;
 214
 215         memblock_reserve(pmap, memmap.nr_map * memmap.desc_size);
 216
 217         efi.memmap = &memmap;
 218
 219         return 0;
 220 }
 221
 222 static void __init print_efi_memmap(void)
 223 {
 224 #ifdef EFI_DEBUG
 225         efi_memory_desc_t *md;
 226         void *p;
 227         int i;
 228
 229         for (p = memmap.map, i = 0;
 230              p < memmap.map_end;
 231              p += memmap.desc_size, i++) {
 232                 char buf[64];
 233
 234                 md = p;
 235                 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx) (%lluMB)\n",
 236                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 237                         md->phys_addr,
 238                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT),
 239                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
 240         }
 241 #endif  /*  EFI_DEBUG  */
 242 }
 243
 244 void __init efi_unmap_memmap(void)
 245 {
 246         clear_bit(EFI_MEMMAP, &efi.flags);
 247         if (memmap.map) {
 248                 early_memunmap(memmap.map, memmap.nr_map * memmap.desc_size);
 249                 memmap.map = NULL;
 250         }
 251 }
 252
 253 static int __init efi_systab_init(void *phys)
 254 {
 255         if (efi_enabled(EFI_64BIT)) {
 256                 efi_system_table_64_t *systab64;
 257                 struct efi_setup_data *data = NULL;
 258                 u64 tmp = 0;
 259
 260                 if (efi_setup) {
 261                         data = early_memremap(efi_setup, sizeof(*data));
 262                         if (!data)
 263                                 return -ENOMEM;
 264                 }
 265                 systab64 = early_memremap((unsigned long)phys,
 266                                          sizeof(*systab64));
 267                 if (systab64 == NULL) {
 268                         pr_err("Couldn't map the system table!\n");
 269                         if (data)
 270                                 early_memunmap(data, sizeof(*data));
 271                         return -ENOMEM;
 272                 }
 273
 274                 efi_systab.hdr = systab64->hdr;
 275                 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
 276                                               systab64->fw_vendor;
 277                 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
 278                 efi_systab.fw_revision = systab64->fw_revision;
 279                 efi_systab.con_in_handle = systab64->con_in_handle;
 280                 tmp |= systab64->con_in_handle;
 281                 efi_systab.con_in = systab64->con_in;
 282                 tmp |= systab64->con_in;
 283                 efi_systab.con_out_handle = systab64->con_out_handle;
 284                 tmp |= systab64->con_out_handle;
 285                 efi_systab.con_out = systab64->con_out;
 286                 tmp |= systab64->con_out;
 287                 efi_systab.stderr_handle = systab64->stderr_handle;
 288                 tmp |= systab64->stderr_handle;
 289                 efi_systab.stderr = systab64->stderr;
 290                 tmp |= systab64->stderr;
 291                 efi_systab.runtime = data ?
 292                                      (void *)(unsigned long)data->runtime :
 293                                      (void *)(unsigned long)systab64->runtime;
 294                 tmp |= data ? data->runtime : systab64->runtime;
 295                 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
 296                 tmp |= systab64->boottime;
 297                 efi_systab.nr_tables = systab64->nr_tables;
 298                 efi_systab.tables = data ? (unsigned long)data->tables :
 299                                            systab64->tables;
 300                 tmp |= data ? data->tables : systab64->tables;
 301
 302                 early_memunmap(systab64, sizeof(*systab64));
 303                 if (data)
 304                         early_memunmap(data, sizeof(*data));
 305 #ifdef CONFIG_X86_32
 306                 if (tmp >> 32) {
 307                         pr_err("EFI data located above 4GB, disabling EFI.\n");
 308                         return -EINVAL;
 309                 }
 310 #endif
 311         } else {
 312                 efi_system_table_32_t *systab32;
 313
 314                 systab32 = early_memremap((unsigned long)phys,
 315                                          sizeof(*systab32));
 316                 if (systab32 == NULL) {
 317                         pr_err("Couldn't map the system table!\n");
 318                         return -ENOMEM;
 319                 }
 320
 321                 efi_systab.hdr = systab32->hdr;
 322                 efi_systab.fw_vendor = systab32->fw_vendor;
 323                 efi_systab.fw_revision = systab32->fw_revision;
 324                 efi_systab.con_in_handle = systab32->con_in_handle;
 325                 efi_systab.con_in = systab32->con_in;
 326                 efi_systab.con_out_handle = systab32->con_out_handle;
 327                 efi_systab.con_out = systab32->con_out;
 328                 efi_systab.stderr_handle = systab32->stderr_handle;
 329                 efi_systab.stderr = systab32->stderr;
 330                 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
 331                 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
 332                 efi_systab.nr_tables = systab32->nr_tables;
 333                 efi_systab.tables = systab32->tables;
 334
 335                 early_memunmap(systab32, sizeof(*systab32));
 336         }
 337
 338         efi.systab = &efi_systab;
 339
 340         /*
 341          * Verify the EFI Table
 342          */
 343         if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 344                 pr_err("System table signature incorrect!\n");
 345                 return -EINVAL;
 346         }
 347         if ((efi.systab->hdr.revision >> 16) == 0)
 348                 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
 349                        efi.systab->hdr.revision >> 16,
 350                        efi.systab->hdr.revision & 0xffff);
 351
 352         set_bit(EFI_SYSTEM_TABLES, &efi.flags);
 353
 354         return 0;
 355 }
 356
 357 static int __init efi_runtime_init32(void)
 358 {
 359         efi_runtime_services_32_t *runtime;
 360
 361         runtime = early_memremap((unsigned long)efi.systab->runtime,
 362                         sizeof(efi_runtime_services_32_t));
 363         if (!runtime) {
 364                 pr_err("Could not map the runtime service table!\n");
 365                 return -ENOMEM;
 366         }
 367
 368         /*
 369          * We will only need *early* access to the SetVirtualAddressMap
 370          * EFI runtime service. All other runtime services will be called
 371          * via the virtual mapping.
 372          */
 373         efi_phys.set_virtual_address_map =
 374                         (efi_set_virtual_address_map_t *)
 375                         (unsigned long)runtime->set_virtual_address_map;
 376         early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
 377
 378         return 0;
 379 }
 380
 381 static int __init efi_runtime_init64(void)
 382 {
 383         efi_runtime_services_64_t *runtime;
 384
 385         runtime = early_memremap((unsigned long)efi.systab->runtime,
 386                         sizeof(efi_runtime_services_64_t));
 387         if (!runtime) {
 388                 pr_err("Could not map the runtime service table!\n");
 389                 return -ENOMEM;
 390         }
 391
 392         /*
 393          * We will only need *early* access to the SetVirtualAddressMap
 394          * EFI runtime service. All other runtime services will be called
 395          * via the virtual mapping.
 396          */
 397         efi_phys.set_virtual_address_map =
 398                         (efi_set_virtual_address_map_t *)
 399                         (unsigned long)runtime->set_virtual_address_map;
 400         early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
 401
 402         return 0;
 403 }
 404
 405 static int __init efi_runtime_init(void)
 406 {
 407         int rv;
 408
 409         /*
 410          * Check out the runtime services table. We need to map
 411          * the runtime services table so that we can grab the physical
 412          * address of several of the EFI runtime functions, needed to
 413          * set the firmware into virtual mode.
 414          *
 415          * When EFI_PARAVIRT is in force then we could not map runtime
 416          * service memory region because we do not have direct access to it.
 417          * However, runtime services are available through proxy functions
 418          * (e.g. in case of Xen dom0 EFI implementation they call special
 419          * hypercall which executes relevant EFI functions) and that is why
 420          * they are always enabled.
 421          */
 422
 423         if (!efi_enabled(EFI_PARAVIRT)) {
 424                 if (efi_enabled(EFI_64BIT))
 425                         rv = efi_runtime_init64();
 426                 else
 427                         rv = efi_runtime_init32();
 428
 429                 if (rv)
 430                         return rv;
 431         }
 432
 433         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 434
 435         return 0;
 436 }
 437
 438 static int __init efi_memmap_init(void)
 439 {
 440         if (efi_enabled(EFI_PARAVIRT))
 441                 return 0;
 442
 443         /* Map the EFI memory map */
 444         memmap.map = early_memremap((unsigned long)memmap.phys_map,
 445                                    memmap.nr_map * memmap.desc_size);
 446         if (memmap.map == NULL) {
 447                 pr_err("Could not map the memory map!\n");
 448                 return -ENOMEM;
 449         }
 450         memmap.map_end = memmap.map + (memmap.nr_map * memmap.desc_size);
 451
 452         if (add_efi_memmap)
 453                 do_add_efi_memmap();
 454
 455         set_bit(EFI_MEMMAP, &efi.flags);
 456
 457         return 0;
 458 }
 459
 460 void __init efi_init(void)
 461 {
 462         efi_char16_t *c16;
 463         char vendor[100] = "unknown";
 464         int i = 0;
 465         void *tmp;
 466
 467 #ifdef CONFIG_X86_32
 468         if (boot_params.efi_info.efi_systab_hi ||
 469             boot_params.efi_info.efi_memmap_hi) {
 470                 pr_info("Table located above 4GB, disabling EFI.\n");
 471                 return;
 472         }
 473         efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
 474 #else
 475         efi_phys.systab = (efi_system_table_t *)
 476                           (boot_params.efi_info.efi_systab |
 477                           ((__u64)boot_params.efi_info.efi_systab_hi<<32));
 478 #endif
 479
 480         if (efi_systab_init(efi_phys.systab))
 481                 return;
 482
 483         efi.config_table = (unsigned long)efi.systab->tables;
 484         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
 485         efi.runtime      = (unsigned long)efi.systab->runtime;
 486
 487         /*
 488          * Show what we know for posterity
 489          */
 490         c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
 491         if (c16) {
 492                 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
 493                         vendor[i] = *c16++;
 494                 vendor[i] = '\0';
 495         } else
 496                 pr_err("Could not map the firmware vendor!\n");
 497         early_memunmap(tmp, 2);
 498
 499         pr_info("EFI v%u.%.02u by %s\n",
 500                 efi.systab->hdr.revision >> 16,
 501                 efi.systab->hdr.revision & 0xffff, vendor);
 502
 503         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
 504                 return;
 505
 506         if (efi_config_init(arch_tables))
 507                 return;
 508
 509         /*
 510          * Note: We currently don't support runtime services on an EFI
 511          * that doesn't match the kernel 32/64-bit mode.
 512          */
 513
 514         if (!efi_runtime_supported())
 515                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
 516         else {
 517                 if (efi_runtime_disabled() || efi_runtime_init())
 518                         return;
 519         }
 520         if (efi_memmap_init())
 521                 return;
 522
 523         if (efi_enabled(EFI_DBG))
 524                 print_efi_memmap();
 525
 526         efi_esrt_init();
 527 }
 528
 529 void __init efi_late_init(void)
 530 {
 531         efi_bgrt_init();
 532 }
 533
 534 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
 535 {
 536         u64 addr, npages;
 537
 538         addr = md->virt_addr;
 539         npages = md->num_pages;
 540
 541         memrange_efi_to_native(&addr, &npages);
 542
 543         if (executable)
 544                 set_memory_x(addr, npages);
 545         else
 546                 set_memory_nx(addr, npages);
 547 }
 548
 549 void __init runtime_code_page_mkexec(void)
 550 {
 551         efi_memory_desc_t *md;
 552         void *p;
 553
 554         /* Make EFI runtime service code area executable */
 555         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 556                 md = p;
 557
 558                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
 559                         continue;
 560
 561                 efi_set_executable(md, true);
 562         }
 563 }
 564
 565 void __init efi_memory_uc(u64 addr, unsigned long size)
 566 {
 567         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
 568         u64 npages;
 569
 570         npages = round_up(size, page_shift) / page_shift;
 571         memrange_efi_to_native(&addr, &npages);
 572         set_memory_uc(addr, npages);
 573 }
 574
 575 void __init old_map_region(efi_memory_desc_t *md)
 576 {
 577         u64 start_pfn, end_pfn, end;
 578         unsigned long size;
 579         void *va;
 580
 581         start_pfn = PFN_DOWN(md->phys_addr);
 582         size      = md->num_pages << PAGE_SHIFT;
 583         end       = md->phys_addr + size;
 584         end_pfn   = PFN_UP(end);
 585
 586         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
 587                 va = __va(md->phys_addr);
 588
 589                 if (!(md->attribute & EFI_MEMORY_WB))
 590                         efi_memory_uc((u64)(unsigned long)va, size);
 591         } else
 592                 va = efi_ioremap(md->phys_addr, size,
 593                                  md->type, md->attribute);
 594
 595         md->virt_addr = (u64) (unsigned long) va;
 596         if (!va)
 597                 pr_err("ioremap of 0x%llX failed!\n",
 598                        (unsigned long long)md->phys_addr);
 599 }
 600
 601 /* Merge contiguous regions of the same type and attribute */
 602 static void __init efi_merge_regions(void)
 603 {
 604         void *p;
 605         efi_memory_desc_t *md, *prev_md = NULL;
 606
 607         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 608                 u64 prev_size;
 609                 md = p;
 610
 611                 if (!prev_md) {
 612                         prev_md = md;
 613                         continue;
 614                 }
 615
 616                 if (prev_md->type != md->type ||
 617                     prev_md->attribute != md->attribute) {
 618                         prev_md = md;
 619                         continue;
 620                 }
 621
 622                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
 623
 624                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
 625                         prev_md->num_pages += md->num_pages;
 626                         md->type = EFI_RESERVED_TYPE;
 627                         md->attribute = 0;
 628                         continue;
 629                 }
 630                 prev_md = md;
 631         }
 632 }
 633
 634 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
 635 {
 636         unsigned long size;
 637         u64 end, systab;
 638
 639         size = md->num_pages << EFI_PAGE_SHIFT;
 640         end = md->phys_addr + size;
 641         systab = (u64)(unsigned long)efi_phys.systab;
 642         if (md->phys_addr <= systab && systab < end) {
 643                 systab += md->virt_addr - md->phys_addr;
 644                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
 645         }
 646 }
 647
 648 static void __init save_runtime_map(void)
 649 {
 650 #ifdef CONFIG_KEXEC
 651         efi_memory_desc_t *md;
 652         void *tmp, *p, *q = NULL;
 653         int count = 0;
 654
 655         if (efi_enabled(EFI_OLD_MEMMAP))
 656                 return;
 657
 658         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 659                 md = p;
 660
 661                 if (!(md->attribute & EFI_MEMORY_RUNTIME) ||
 662                     (md->type == EFI_BOOT_SERVICES_CODE) ||
 663                     (md->type == EFI_BOOT_SERVICES_DATA))
 664                         continue;
 665                 tmp = krealloc(q, (count + 1) * memmap.desc_size, GFP_KERNEL);
 666                 if (!tmp)
 667                         goto out;
 668                 q = tmp;
 669
 670                 memcpy(q + count * memmap.desc_size, md, memmap.desc_size);
 671                 count++;
 672         }
 673
 674         efi_runtime_map_setup(q, count, memmap.desc_size);
 675         return;
 676
 677 out:
 678         kfree(q);
 679         pr_err("Error saving runtime map, efi runtime on kexec non-functional!!\n");
 680 #endif
 681 }
 682
 683 static void *realloc_pages(void *old_memmap, int old_shift)
 684 {
 685         void *ret;
 686
 687         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
 688         if (!ret)
 689                 goto out;
 690
 691         /*
 692          * A first-time allocation doesn't have anything to copy.
 693          */
 694         if (!old_memmap)
 695                 return ret;
 696
 697         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
 698
 699 out:
 700         free_pages((unsigned long)old_memmap, old_shift);
 701         return ret;
 702 }
 703
 704 /*
 705  * Map the efi memory ranges of the runtime services and update new_mmap with
 706  * virtual addresses.
 707  */
 708 static void * __init efi_map_regions(int *count, int *pg_shift)
 709 {
 710         void *p, *new_memmap = NULL;
 711         unsigned long left = 0;
 712         efi_memory_desc_t *md;
 713
 714         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 715                 md = p;
 716                 if (!(md->attribute & EFI_MEMORY_RUNTIME)) {
 717 #ifdef CONFIG_X86_64
 718                         if (md->type != EFI_BOOT_SERVICES_CODE &&
 719                             md->type != EFI_BOOT_SERVICES_DATA)
 720 #endif
 721                                 continue;
 722                 }
 723
 724                 efi_map_region(md);
 725                 get_systab_virt_addr(md);
 726
 727                 if (left < memmap.desc_size) {
 728                         new_memmap = realloc_pages(new_memmap, *pg_shift);
 729                         if (!new_memmap)
 730                                 return NULL;
 731
 732                         left += PAGE_SIZE << *pg_shift;
 733                         (*pg_shift)++;
 734                 }
 735
 736                 memcpy(new_memmap + (*count * memmap.desc_size), md,
 737                        memmap.desc_size);
 738
 739                 left -= memmap.desc_size;
 740                 (*count)++;
 741         }
 742
 743         return new_memmap;
 744 }
 745
 746 static void __init kexec_enter_virtual_mode(void)
 747 {
 748 #ifdef CONFIG_KEXEC
 749         efi_memory_desc_t *md;
 750         void *p;
 751
 752         efi.systab = NULL;
 753
 754         /*
 755          * We don't do virtual mode, since we don't do runtime services, on
 756          * non-native EFI
 757          */
 758         if (!efi_is_native()) {
 759                 efi_unmap_memmap();
 760                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 761                 return;
 762         }
 763
 764         /*
 765         * Map efi regions which were passed via setup_data. The virt_addr is a
 766         * fixed addr which was used in first kernel of a kexec boot.
 767         */
 768         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 769                 md = p;
 770                 efi_map_region_fixed(md); /* FIXME: add error handling */
 771                 get_systab_virt_addr(md);
 772         }
 773
 774         save_runtime_map();
 775
 776         BUG_ON(!efi.systab);
 777
 778         efi_sync_low_kernel_mappings();
 779
 780         /*
 781          * Now that EFI is in virtual mode, update the function
 782          * pointers in the runtime service table to the new virtual addresses.
 783          *
 784          * Call EFI services through wrapper functions.
 785          */
 786         efi.runtime_version = efi_systab.hdr.revision;
 787
 788         efi_native_runtime_setup();
 789
 790         efi.set_virtual_address_map = NULL;
 791
 792         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
 793                 runtime_code_page_mkexec();
 794
 795         /* clean DUMMY object */
 796         efi_delete_dummy_variable();
 797 #endif
 798 }
 799
 800 /*
 801  * This function will switch the EFI runtime services to virtual mode.
 802  * Essentially, we look through the EFI memmap and map every region that
 803  * has the runtime attribute bit set in its memory descriptor into the
 804  * ->trampoline_pgd page table using a top-down VA allocation scheme.
 805  *
 806  * The old method which used to update that memory descriptor with the
 807  * virtual address obtained from ioremap() is still supported when the
 808  * kernel is booted with efi=old_map on its command line. Same old
 809  * method enabled the runtime services to be called without having to
 810  * thunk back into physical mode for every invocation.
 811  *
 812  * The new method does a pagetable switch in a preemption-safe manner
 813  * so that we're in a different address space when calling a runtime
 814  * function. For function arguments passing we do copy the PGDs of the
 815  * kernel page table into ->trampoline_pgd prior to each call.
 816  *
 817  * Specially for kexec boot, efi runtime maps in previous kernel should
 818  * be passed in via setup_data. In that case runtime ranges will be mapped
 819  * to the same virtual addresses as the first kernel, see
 820  * kexec_enter_virtual_mode().
 821  */
 822 static void __init __efi_enter_virtual_mode(void)
 823 {
 824         int count = 0, pg_shift = 0;
 825         void *new_memmap = NULL;
 826         efi_status_t status;
 827
 828         efi.systab = NULL;
 829
 830         efi_merge_regions();
 831         new_memmap = efi_map_regions(&count, &pg_shift);
 832         if (!new_memmap) {
 833                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
 834                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 835                 return;
 836         }
 837
 838         save_runtime_map();
 839
 840         BUG_ON(!efi.systab);
 841
 842         if (efi_setup_page_tables(__pa(new_memmap), 1 << pg_shift)) {
 843                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 844                 return;
 845         }
 846
 847         efi_sync_low_kernel_mappings();
 848         efi_dump_pagetable();
 849
 850         if (efi_is_native()) {
 851                 status = phys_efi_set_virtual_address_map(
 852                                 memmap.desc_size * count,
 853                                 memmap.desc_size,
 854                                 memmap.desc_version,
 855                                 (efi_memory_desc_t *)__pa(new_memmap));
 856         } else {
 857                 status = efi_thunk_set_virtual_address_map(
 858                                 efi_phys.set_virtual_address_map,
 859                                 memmap.desc_size * count,
 860                                 memmap.desc_size,
 861                                 memmap.desc_version,
 862                                 (efi_memory_desc_t *)__pa(new_memmap));
 863         }
 864
 865         if (status != EFI_SUCCESS) {
 866                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
 867                          status);
 868                 panic("EFI call to SetVirtualAddressMap() failed!");
 869         }
 870
 871         /*
 872          * Now that EFI is in virtual mode, update the function
 873          * pointers in the runtime service table to the new virtual addresses.
 874          *
 875          * Call EFI services through wrapper functions.
 876          */
 877         efi.runtime_version = efi_systab.hdr.revision;
 878
 879         if (efi_is_native())
 880                 efi_native_runtime_setup();
 881         else
 882                 efi_thunk_runtime_setup();
 883
 884         efi.set_virtual_address_map = NULL;
 885
 886         efi_runtime_mkexec();
 887
 888         /*
 889          * We mapped the descriptor array into the EFI pagetable above but we're
 890          * not unmapping it here. Here's why:
 891          *
 892          * We're copying select PGDs from the kernel page table to the EFI page
 893          * table and when we do so and make changes to those PGDs like unmapping
 894          * stuff from them, those changes appear in the kernel page table and we
 895          * go boom.
 896          *
 897          * From setup_real_mode():
 898          *
 899          * ...
 900          * trampoline_pgd[0] = init_level4_pgt[pgd_index(__PAGE_OFFSET)].pgd;
 901          *
 902          * In this particular case, our allocation is in PGD 0 of the EFI page
 903          * table but we've copied that PGD from PGD[272] of the EFI page table:
 904          *
 905          *      pgd_index(__PAGE_OFFSET = 0xffff880000000000) = 272
 906          *
 907          * where the direct memory mapping in kernel space is.
 908          *
 909          * new_memmap's VA comes from that direct mapping and thus clearing it,
 910          * it would get cleared in the kernel page table too.
 911          *
 912          * efi_cleanup_page_tables(__pa(new_memmap), 1 << pg_shift);
 913          */
 914         free_pages((unsigned long)new_memmap, pg_shift);
 915
 916         /* clean DUMMY object */
 917         efi_delete_dummy_variable();
 918 }
 919
 920 void __init efi_enter_virtual_mode(void)
 921 {
 922         if (efi_enabled(EFI_PARAVIRT))
 923                 return;
 924
 925         if (efi_setup)
 926                 kexec_enter_virtual_mode();
 927         else
 928                 __efi_enter_virtual_mode();
 929 }
 930
 931 /*
 932  * Convenience functions to obtain memory types and attributes
 933  */
 934 u32 efi_mem_type(unsigned long phys_addr)
 935 {
 936         efi_memory_desc_t *md;
 937         void *p;
 938
 939         if (!efi_enabled(EFI_MEMMAP))
 940                 return 0;
 941
 942         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 943                 md = p;
 944                 if ((md->phys_addr <= phys_addr) &&
 945                     (phys_addr < (md->phys_addr +
 946                                   (md->num_pages << EFI_PAGE_SHIFT))))
 947                         return md->type;
 948         }
 949         return 0;
 950 }
 951
 952 u64 efi_mem_attributes(unsigned long phys_addr)
 953 {
 954         efi_memory_desc_t *md;
 955         void *p;
 956
 957         if (!efi_enabled(EFI_MEMMAP))
 958                 return 0;
 959
 960         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 961                 md = p;
 962                 if ((md->phys_addr <= phys_addr) &&
 963                     (phys_addr < (md->phys_addr +
 964                                   (md->num_pages << EFI_PAGE_SHIFT))))
 965                         return md->attribute;
 966         }
 967         return 0;
 968 }
 969
 970 static int __init arch_parse_efi_cmdline(char *str)
 971 {
 972         if (parse_option_str(str, "old_map"))
 973                 set_bit(EFI_OLD_MEMMAP, &efi.flags);
 974         if (parse_option_str(str, "debug"))
 975                 set_bit(EFI_DBG, &efi.flags);
 976
 977         return 0;
 978 }
 979 early_param("efi", arch_parse_efi_cmdline);