1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Common EFI (Extensible Firmware Interface) support functions
4 * Based on Extensible Firmware Interface Specification version 1.0
5 *
6 * Copyright (C) 1999 VA Linux Systems
7 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
8 * Copyright (C) 1999-2002 Hewlett-Packard Co.
9 * David Mosberger-Tang <davidm@hpl.hp.com>
10 * Stephane Eranian <eranian@hpl.hp.com>
11 * Copyright (C) 2005-2008 Intel Co.
12 * Fenghua Yu <fenghua.yu@intel.com>
13 * Bibo Mao <bibo.mao@intel.com>
14 * Chandramouli Narayanan <mouli@linux.intel.com>
15 * Huang Ying <ying.huang@intel.com>
16 * Copyright (C) 2013 SuSE Labs
17 * Borislav Petkov <bp@suse.de> - runtime services VA mapping
18 *
19 * Copied from efi_32.c to eliminate the duplicated code between EFI
20 * 32/64 support code. --ying 2007-10-26
21 *
22 * All EFI Runtime Services are not implemented yet as EFI only
23 * supports physical mode addressing on SoftSDV. This is to be fixed
24 * in a future version. --drummond 1999-07-20
25 *
26 * Implemented EFI runtime services and virtual mode calls. --davidm
27 *
28 * Goutham Rao: <goutham.rao@intel.com>
29 * Skip non-WB memory and ignore empty memory ranges.
30 */
31
32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33
34 #include <linux/kernel.h>
35 #include <linux/init.h>
36 #include <linux/efi.h>
37 #include <linux/efi-bgrt.h>
38 #include <linux/export.h>
39 #include <linux/memblock.h>
40 #include <linux/slab.h>
41 #include <linux/spinlock.h>
42 #include <linux/uaccess.h>
43 #include <linux/time.h>
44 #include <linux/io.h>
45 #include <linux/reboot.h>
46 #include <linux/bcd.h>
47
48 #include <asm/setup.h>
49 #include <asm/efi.h>
50 #include <asm/e820/api.h>
51 #include <asm/time.h>
52 #include <asm/set_memory.h>
53 #include <asm/tlbflush.h>
54 #include <asm/x86_init.h>
55 #include <asm/uv/uv.h>
56
57 static struct efi efi_phys __initdata;
58 static efi_system_table_t efi_systab __initdata;
59
60 static efi_config_table_type_t arch_tables[] __initdata = {
61 #ifdef CONFIG_X86_UV
62 {UV_SYSTEM_TABLE_GUID, "UVsystab", &uv_systab_phys},
63 #endif
64 {NULL_GUID, NULL, NULL},
65 };
66
67 static const unsigned long * const efi_tables[] = {
68 &efi.mps,
69 &efi.acpi,
70 &efi.acpi20,
71 &efi.smbios,
72 &efi.smbios3,
73 &efi.boot_info,
74 &efi.hcdp,
75 &efi.uga,
76 #ifdef CONFIG_X86_UV
77 &uv_systab_phys,
78 #endif
79 &efi.fw_vendor,
80 &efi.runtime,
81 &efi.config_table,
82 &efi.esrt,
83 &efi.properties_table,
84 &efi.mem_attr_table,
85 #ifdef CONFIG_EFI_RCI2_TABLE
86 &rci2_table_phys,
87 #endif
88 &efi.tpm_log,
89 &efi.tpm_final_log,
90 };
91
92 u64 efi_setup; /* efi setup_data physical address */
93
94 static int add_efi_memmap __initdata;
setup_add_efi_memmap(char * arg)95 static int __init setup_add_efi_memmap(char *arg)
96 {
97 add_efi_memmap = 1;
98 return 0;
99 }
100 early_param("add_efi_memmap", setup_add_efi_memmap);
101
phys_efi_set_virtual_address_map(unsigned long memory_map_size,unsigned long descriptor_size,u32 descriptor_version,efi_memory_desc_t * virtual_map)102 static efi_status_t __init phys_efi_set_virtual_address_map(
103 unsigned long memory_map_size,
104 unsigned long descriptor_size,
105 u32 descriptor_version,
106 efi_memory_desc_t *virtual_map)
107 {
108 efi_status_t status;
109 unsigned long flags;
110 pgd_t *save_pgd;
111
112 save_pgd = efi_call_phys_prolog();
113 if (!save_pgd)
114 return EFI_ABORTED;
115
116 /* Disable interrupts around EFI calls: */
117 local_irq_save(flags);
118 status = efi_call_phys(efi_phys.set_virtual_address_map,
119 memory_map_size, descriptor_size,
120 descriptor_version, virtual_map);
121 local_irq_restore(flags);
122
123 efi_call_phys_epilog(save_pgd);
124
125 return status;
126 }
127
efi_find_mirror(void)128 void __init efi_find_mirror(void)
129 {
130 efi_memory_desc_t *md;
131 u64 mirror_size = 0, total_size = 0;
132
133 for_each_efi_memory_desc(md) {
134 unsigned long long start = md->phys_addr;
135 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
136
137 total_size += size;
138 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
139 memblock_mark_mirror(start, size);
140 mirror_size += size;
141 }
142 }
143 if (mirror_size)
144 pr_info("Memory: %lldM/%lldM mirrored memory\n",
145 mirror_size>>20, total_size>>20);
146 }
147
148 /*
149 * Tell the kernel about the EFI memory map. This might include
150 * more than the max 128 entries that can fit in the e820 legacy
151 * (zeropage) memory map.
152 */
153
do_add_efi_memmap(void)154 static void __init do_add_efi_memmap(void)
155 {
156 efi_memory_desc_t *md;
157
158 for_each_efi_memory_desc(md) {
159 unsigned long long start = md->phys_addr;
160 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
161 int e820_type;
162
163 switch (md->type) {
164 case EFI_LOADER_CODE:
165 case EFI_LOADER_DATA:
166 case EFI_BOOT_SERVICES_CODE:
167 case EFI_BOOT_SERVICES_DATA:
168 case EFI_CONVENTIONAL_MEMORY:
169 if (md->attribute & EFI_MEMORY_WB)
170 e820_type = E820_TYPE_RAM;
171 else
172 e820_type = E820_TYPE_RESERVED;
173 break;
174 case EFI_ACPI_RECLAIM_MEMORY:
175 e820_type = E820_TYPE_ACPI;
176 break;
177 case EFI_ACPI_MEMORY_NVS:
178 e820_type = E820_TYPE_NVS;
179 break;
180 case EFI_UNUSABLE_MEMORY:
181 e820_type = E820_TYPE_UNUSABLE;
182 break;
183 case EFI_PERSISTENT_MEMORY:
184 e820_type = E820_TYPE_PMEM;
185 break;
186 default:
187 /*
188 * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
189 * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
190 * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
191 */
192 e820_type = E820_TYPE_RESERVED;
193 break;
194 }
195 e820__range_add(start, size, e820_type);
196 }
197 e820__update_table(e820_table);
198 }
199
efi_memblock_x86_reserve_range(void)200 int __init efi_memblock_x86_reserve_range(void)
201 {
202 struct efi_info *e = &boot_params.efi_info;
203 struct efi_memory_map_data data;
204 phys_addr_t pmap;
205 int rv;
206
207 if (efi_enabled(EFI_PARAVIRT))
208 return 0;
209
210 #ifdef CONFIG_X86_32
211 /* Can't handle data above 4GB at this time */
212 if (e->efi_memmap_hi) {
213 pr_err("Memory map is above 4GB, disabling EFI.\n");
214 return -EINVAL;
215 }
216 pmap = e->efi_memmap;
217 #else
218 pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
219 #endif
220 data.phys_map = pmap;
221 data.size = e->efi_memmap_size;
222 data.desc_size = e->efi_memdesc_size;
223 data.desc_version = e->efi_memdesc_version;
224
225 rv = efi_memmap_init_early(&data);
226 if (rv)
227 return rv;
228
229 if (add_efi_memmap)
230 do_add_efi_memmap();
231
232 WARN(efi.memmap.desc_version != 1,
233 "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
234 efi.memmap.desc_version);
235
236 memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size);
237
238 return 0;
239 }
240
241 #define OVERFLOW_ADDR_SHIFT (64 - EFI_PAGE_SHIFT)
242 #define OVERFLOW_ADDR_MASK (U64_MAX << OVERFLOW_ADDR_SHIFT)
243 #define U64_HIGH_BIT (~(U64_MAX >> 1))
244
efi_memmap_entry_valid(const efi_memory_desc_t * md,int i)245 static bool __init efi_memmap_entry_valid(const efi_memory_desc_t *md, int i)
246 {
247 u64 end = (md->num_pages << EFI_PAGE_SHIFT) + md->phys_addr - 1;
248 u64 end_hi = 0;
249 char buf[64];
250
251 if (md->num_pages == 0) {
252 end = 0;
253 } else if (md->num_pages > EFI_PAGES_MAX ||
254 EFI_PAGES_MAX - md->num_pages <
255 (md->phys_addr >> EFI_PAGE_SHIFT)) {
256 end_hi = (md->num_pages & OVERFLOW_ADDR_MASK)
257 >> OVERFLOW_ADDR_SHIFT;
258
259 if ((md->phys_addr & U64_HIGH_BIT) && !(end & U64_HIGH_BIT))
260 end_hi += 1;
261 } else {
262 return true;
263 }
264
265 pr_warn_once(FW_BUG "Invalid EFI memory map entries:\n");
266
267 if (end_hi) {
268 pr_warn("mem%02u: %s range=[0x%016llx-0x%llx%016llx] (invalid)\n",
269 i, efi_md_typeattr_format(buf, sizeof(buf), md),
270 md->phys_addr, end_hi, end);
271 } else {
272 pr_warn("mem%02u: %s range=[0x%016llx-0x%016llx] (invalid)\n",
273 i, efi_md_typeattr_format(buf, sizeof(buf), md),
274 md->phys_addr, end);
275 }
276 return false;
277 }
278
efi_clean_memmap(void)279 static void __init efi_clean_memmap(void)
280 {
281 efi_memory_desc_t *out = efi.memmap.map;
282 const efi_memory_desc_t *in = out;
283 const efi_memory_desc_t *end = efi.memmap.map_end;
284 int i, n_removal;
285
286 for (i = n_removal = 0; in < end; i++) {
287 if (efi_memmap_entry_valid(in, i)) {
288 if (out != in)
289 memcpy(out, in, efi.memmap.desc_size);
290 out = (void *)out + efi.memmap.desc_size;
291 } else {
292 n_removal++;
293 }
294 in = (void *)in + efi.memmap.desc_size;
295 }
296
297 if (n_removal > 0) {
298 u64 size = efi.memmap.nr_map - n_removal;
299
300 pr_warn("Removing %d invalid memory map entries.\n", n_removal);
301 efi_memmap_install(efi.memmap.phys_map, size);
302 }
303 }
304
efi_print_memmap(void)305 void __init efi_print_memmap(void)
306 {
307 efi_memory_desc_t *md;
308 int i = 0;
309
310 for_each_efi_memory_desc(md) {
311 char buf[64];
312
313 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n",
314 i++, efi_md_typeattr_format(buf, sizeof(buf), md),
315 md->phys_addr,
316 md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1,
317 (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
318 }
319 }
320
efi_systab_init(void * phys)321 static int __init efi_systab_init(void *phys)
322 {
323 if (efi_enabled(EFI_64BIT)) {
324 efi_system_table_64_t *systab64;
325 struct efi_setup_data *data = NULL;
326 u64 tmp = 0;
327
328 if (efi_setup) {
329 data = early_memremap(efi_setup, sizeof(*data));
330 if (!data)
331 return -ENOMEM;
332 }
333 systab64 = early_memremap((unsigned long)phys,
334 sizeof(*systab64));
335 if (systab64 == NULL) {
336 pr_err("Couldn't map the system table!\n");
337 if (data)
338 early_memunmap(data, sizeof(*data));
339 return -ENOMEM;
340 }
341
342 efi_systab.hdr = systab64->hdr;
343 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
344 systab64->fw_vendor;
345 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
346 efi_systab.fw_revision = systab64->fw_revision;
347 efi_systab.con_in_handle = systab64->con_in_handle;
348 tmp |= systab64->con_in_handle;
349 efi_systab.con_in = systab64->con_in;
350 tmp |= systab64->con_in;
351 efi_systab.con_out_handle = systab64->con_out_handle;
352 tmp |= systab64->con_out_handle;
353 efi_systab.con_out = systab64->con_out;
354 tmp |= systab64->con_out;
355 efi_systab.stderr_handle = systab64->stderr_handle;
356 tmp |= systab64->stderr_handle;
357 efi_systab.stderr = systab64->stderr;
358 tmp |= systab64->stderr;
359 efi_systab.runtime = data ?
360 (void *)(unsigned long)data->runtime :
361 (void *)(unsigned long)systab64->runtime;
362 tmp |= data ? data->runtime : systab64->runtime;
363 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
364 tmp |= systab64->boottime;
365 efi_systab.nr_tables = systab64->nr_tables;
366 efi_systab.tables = data ? (unsigned long)data->tables :
367 systab64->tables;
368 tmp |= data ? data->tables : systab64->tables;
369
370 early_memunmap(systab64, sizeof(*systab64));
371 if (data)
372 early_memunmap(data, sizeof(*data));
373 #ifdef CONFIG_X86_32
374 if (tmp >> 32) {
375 pr_err("EFI data located above 4GB, disabling EFI.\n");
376 return -EINVAL;
377 }
378 #endif
379 } else {
380 efi_system_table_32_t *systab32;
381
382 systab32 = early_memremap((unsigned long)phys,
383 sizeof(*systab32));
384 if (systab32 == NULL) {
385 pr_err("Couldn't map the system table!\n");
386 return -ENOMEM;
387 }
388
389 efi_systab.hdr = systab32->hdr;
390 efi_systab.fw_vendor = systab32->fw_vendor;
391 efi_systab.fw_revision = systab32->fw_revision;
392 efi_systab.con_in_handle = systab32->con_in_handle;
393 efi_systab.con_in = systab32->con_in;
394 efi_systab.con_out_handle = systab32->con_out_handle;
395 efi_systab.con_out = systab32->con_out;
396 efi_systab.stderr_handle = systab32->stderr_handle;
397 efi_systab.stderr = systab32->stderr;
398 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
399 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
400 efi_systab.nr_tables = systab32->nr_tables;
401 efi_systab.tables = systab32->tables;
402
403 early_memunmap(systab32, sizeof(*systab32));
404 }
405
406 efi.systab = &efi_systab;
407
408 /*
409 * Verify the EFI Table
410 */
411 if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
412 pr_err("System table signature incorrect!\n");
413 return -EINVAL;
414 }
415 if ((efi.systab->hdr.revision >> 16) == 0)
416 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
417 efi.systab->hdr.revision >> 16,
418 efi.systab->hdr.revision & 0xffff);
419
420 return 0;
421 }
422
efi_runtime_init32(void)423 static int __init efi_runtime_init32(void)
424 {
425 efi_runtime_services_32_t *runtime;
426
427 runtime = early_memremap((unsigned long)efi.systab->runtime,
428 sizeof(efi_runtime_services_32_t));
429 if (!runtime) {
430 pr_err("Could not map the runtime service table!\n");
431 return -ENOMEM;
432 }
433
434 /*
435 * We will only need *early* access to the SetVirtualAddressMap
436 * EFI runtime service. All other runtime services will be called
437 * via the virtual mapping.
438 */
439 efi_phys.set_virtual_address_map =
440 (efi_set_virtual_address_map_t *)
441 (unsigned long)runtime->set_virtual_address_map;
442 early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
443
444 return 0;
445 }
446
efi_runtime_init64(void)447 static int __init efi_runtime_init64(void)
448 {
449 efi_runtime_services_64_t *runtime;
450
451 runtime = early_memremap((unsigned long)efi.systab->runtime,
452 sizeof(efi_runtime_services_64_t));
453 if (!runtime) {
454 pr_err("Could not map the runtime service table!\n");
455 return -ENOMEM;
456 }
457
458 /*
459 * We will only need *early* access to the SetVirtualAddressMap
460 * EFI runtime service. All other runtime services will be called
461 * via the virtual mapping.
462 */
463 efi_phys.set_virtual_address_map =
464 (efi_set_virtual_address_map_t *)
465 (unsigned long)runtime->set_virtual_address_map;
466 early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
467
468 return 0;
469 }
470
efi_runtime_init(void)471 static int __init efi_runtime_init(void)
472 {
473 int rv;
474
475 /*
476 * Check out the runtime services table. We need to map
477 * the runtime services table so that we can grab the physical
478 * address of several of the EFI runtime functions, needed to
479 * set the firmware into virtual mode.
480 *
481 * When EFI_PARAVIRT is in force then we could not map runtime
482 * service memory region because we do not have direct access to it.
483 * However, runtime services are available through proxy functions
484 * (e.g. in case of Xen dom0 EFI implementation they call special
485 * hypercall which executes relevant EFI functions) and that is why
486 * they are always enabled.
487 */
488
489 if (!efi_enabled(EFI_PARAVIRT)) {
490 if (efi_enabled(EFI_64BIT))
491 rv = efi_runtime_init64();
492 else
493 rv = efi_runtime_init32();
494
495 if (rv)
496 return rv;
497 }
498
499 set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
500
501 return 0;
502 }
503
efi_init(void)504 void __init efi_init(void)
505 {
506 efi_char16_t *c16;
507 char vendor[100] = "unknown";
508 int i = 0;
509
510 #ifdef CONFIG_X86_32
511 if (boot_params.efi_info.efi_systab_hi ||
512 boot_params.efi_info.efi_memmap_hi) {
513 pr_info("Table located above 4GB, disabling EFI.\n");
514 return;
515 }
516 efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
517 #else
518 efi_phys.systab = (efi_system_table_t *)
519 (boot_params.efi_info.efi_systab |
520 ((__u64)boot_params.efi_info.efi_systab_hi<<32));
521 #endif
522
523 if (efi_systab_init(efi_phys.systab))
524 return;
525
526 efi.config_table = (unsigned long)efi.systab->tables;
527 efi.fw_vendor = (unsigned long)efi.systab->fw_vendor;
528 efi.runtime = (unsigned long)efi.systab->runtime;
529
530 /*
531 * Show what we know for posterity
532 */
533 c16 = early_memremap_ro(efi.systab->fw_vendor,
534 sizeof(vendor) * sizeof(efi_char16_t));
535 if (c16) {
536 for (i = 0; i < sizeof(vendor) - 1 && c16[i]; ++i)
537 vendor[i] = c16[i];
538 vendor[i] = '\0';
539 early_memunmap(c16, sizeof(vendor) * sizeof(efi_char16_t));
540 } else {
541 pr_err("Could not map the firmware vendor!\n");
542 }
543
544 pr_info("EFI v%u.%.02u by %s\n",
545 efi.systab->hdr.revision >> 16,
546 efi.systab->hdr.revision & 0xffff, vendor);
547
548 if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
549 return;
550
551 if (efi_config_init(arch_tables))
552 return;
553
554 /*
555 * Note: We currently don't support runtime services on an EFI
556 * that doesn't match the kernel 32/64-bit mode.
557 */
558
559 if (!efi_runtime_supported())
560 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
561 else {
562 if (efi_runtime_disabled() || efi_runtime_init()) {
563 efi_memmap_unmap();
564 return;
565 }
566 }
567
568 efi_clean_memmap();
569
570 if (efi_enabled(EFI_DBG))
571 efi_print_memmap();
572 }
573
efi_set_executable(efi_memory_desc_t * md,bool executable)574 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
575 {
576 u64 addr, npages;
577
578 addr = md->virt_addr;
579 npages = md->num_pages;
580
581 memrange_efi_to_native(&addr, &npages);
582
583 if (executable)
584 set_memory_x(addr, npages);
585 else
586 set_memory_nx(addr, npages);
587 }
588
runtime_code_page_mkexec(void)589 void __init runtime_code_page_mkexec(void)
590 {
591 efi_memory_desc_t *md;
592
593 /* Make EFI runtime service code area executable */
594 for_each_efi_memory_desc(md) {
595 if (md->type != EFI_RUNTIME_SERVICES_CODE)
596 continue;
597
598 efi_set_executable(md, true);
599 }
600 }
601
efi_memory_uc(u64 addr,unsigned long size)602 void __init efi_memory_uc(u64 addr, unsigned long size)
603 {
604 unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
605 u64 npages;
606
607 npages = round_up(size, page_shift) / page_shift;
608 memrange_efi_to_native(&addr, &npages);
609 set_memory_uc(addr, npages);
610 }
611
old_map_region(efi_memory_desc_t * md)612 void __init old_map_region(efi_memory_desc_t *md)
613 {
614 u64 start_pfn, end_pfn, end;
615 unsigned long size;
616 void *va;
617
618 start_pfn = PFN_DOWN(md->phys_addr);
619 size = md->num_pages << PAGE_SHIFT;
620 end = md->phys_addr + size;
621 end_pfn = PFN_UP(end);
622
623 if (pfn_range_is_mapped(start_pfn, end_pfn)) {
624 va = __va(md->phys_addr);
625
626 if (!(md->attribute & EFI_MEMORY_WB))
627 efi_memory_uc((u64)(unsigned long)va, size);
628 } else
629 va = efi_ioremap(md->phys_addr, size,
630 md->type, md->attribute);
631
632 md->virt_addr = (u64) (unsigned long) va;
633 if (!va)
634 pr_err("ioremap of 0x%llX failed!\n",
635 (unsigned long long)md->phys_addr);
636 }
637
638 /* Merge contiguous regions of the same type and attribute */
efi_merge_regions(void)639 static void __init efi_merge_regions(void)
640 {
641 efi_memory_desc_t *md, *prev_md = NULL;
642
643 for_each_efi_memory_desc(md) {
644 u64 prev_size;
645
646 if (!prev_md) {
647 prev_md = md;
648 continue;
649 }
650
651 if (prev_md->type != md->type ||
652 prev_md->attribute != md->attribute) {
653 prev_md = md;
654 continue;
655 }
656
657 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
658
659 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
660 prev_md->num_pages += md->num_pages;
661 md->type = EFI_RESERVED_TYPE;
662 md->attribute = 0;
663 continue;
664 }
665 prev_md = md;
666 }
667 }
668
get_systab_virt_addr(efi_memory_desc_t * md)669 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
670 {
671 unsigned long size;
672 u64 end, systab;
673
674 size = md->num_pages << EFI_PAGE_SHIFT;
675 end = md->phys_addr + size;
676 systab = (u64)(unsigned long)efi_phys.systab;
677 if (md->phys_addr <= systab && systab < end) {
678 systab += md->virt_addr - md->phys_addr;
679 efi.systab = (efi_system_table_t *)(unsigned long)systab;
680 }
681 }
682
realloc_pages(void * old_memmap,int old_shift)683 static void *realloc_pages(void *old_memmap, int old_shift)
684 {
685 void *ret;
686
687 ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
688 if (!ret)
689 goto out;
690
691 /*
692 * A first-time allocation doesn't have anything to copy.
693 */
694 if (!old_memmap)
695 return ret;
696
697 memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
698
699 out:
700 free_pages((unsigned long)old_memmap, old_shift);
701 return ret;
702 }
703
704 /*
705 * Iterate the EFI memory map in reverse order because the regions
706 * will be mapped top-down. The end result is the same as if we had
707 * mapped things forward, but doesn't require us to change the
708 * existing implementation of efi_map_region().
709 */
efi_map_next_entry_reverse(void * entry)710 static inline void *efi_map_next_entry_reverse(void *entry)
711 {
712 /* Initial call */
713 if (!entry)
714 return efi.memmap.map_end - efi.memmap.desc_size;
715
716 entry -= efi.memmap.desc_size;
717 if (entry < efi.memmap.map)
718 return NULL;
719
720 return entry;
721 }
722
723 /*
724 * efi_map_next_entry - Return the next EFI memory map descriptor
725 * @entry: Previous EFI memory map descriptor
726 *
727 * This is a helper function to iterate over the EFI memory map, which
728 * we do in different orders depending on the current configuration.
729 *
730 * To begin traversing the memory map @entry must be %NULL.
731 *
732 * Returns %NULL when we reach the end of the memory map.
733 */
efi_map_next_entry(void * entry)734 static void *efi_map_next_entry(void *entry)
735 {
736 if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) {
737 /*
738 * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE
739 * config table feature requires us to map all entries
740 * in the same order as they appear in the EFI memory
741 * map. That is to say, entry N must have a lower
742 * virtual address than entry N+1. This is because the
743 * firmware toolchain leaves relative references in
744 * the code/data sections, which are split and become
745 * separate EFI memory regions. Mapping things
746 * out-of-order leads to the firmware accessing
747 * unmapped addresses.
748 *
749 * Since we need to map things this way whether or not
750 * the kernel actually makes use of
751 * EFI_PROPERTIES_TABLE, let's just switch to this
752 * scheme by default for 64-bit.
753 */
754 return efi_map_next_entry_reverse(entry);
755 }
756
757 /* Initial call */
758 if (!entry)
759 return efi.memmap.map;
760
761 entry += efi.memmap.desc_size;
762 if (entry >= efi.memmap.map_end)
763 return NULL;
764
765 return entry;
766 }
767
should_map_region(efi_memory_desc_t * md)768 static bool should_map_region(efi_memory_desc_t *md)
769 {
770 /*
771 * Runtime regions always require runtime mappings (obviously).
772 */
773 if (md->attribute & EFI_MEMORY_RUNTIME)
774 return true;
775
776 /*
777 * 32-bit EFI doesn't suffer from the bug that requires us to
778 * reserve boot services regions, and mixed mode support
779 * doesn't exist for 32-bit kernels.
780 */
781 if (IS_ENABLED(CONFIG_X86_32))
782 return false;
783
784 /*
785 * Map all of RAM so that we can access arguments in the 1:1
786 * mapping when making EFI runtime calls.
787 */
788 if (IS_ENABLED(CONFIG_EFI_MIXED) && !efi_is_native()) {
789 if (md->type == EFI_CONVENTIONAL_MEMORY ||
790 md->type == EFI_LOADER_DATA ||
791 md->type == EFI_LOADER_CODE)
792 return true;
793 }
794
795 /*
796 * Map boot services regions as a workaround for buggy
797 * firmware that accesses them even when they shouldn't.
798 *
799 * See efi_{reserve,free}_boot_services().
800 */
801 if (md->type == EFI_BOOT_SERVICES_CODE ||
802 md->type == EFI_BOOT_SERVICES_DATA)
803 return true;
804
805 return false;
806 }
807
808 /*
809 * Map the efi memory ranges of the runtime services and update new_mmap with
810 * virtual addresses.
811 */
efi_map_regions(int * count,int * pg_shift)812 static void * __init efi_map_regions(int *count, int *pg_shift)
813 {
814 void *p, *new_memmap = NULL;
815 unsigned long left = 0;
816 unsigned long desc_size;
817 efi_memory_desc_t *md;
818
819 desc_size = efi.memmap.desc_size;
820
821 p = NULL;
822 while ((p = efi_map_next_entry(p))) {
823 md = p;
824
825 if (!should_map_region(md))
826 continue;
827
828 efi_map_region(md);
829 get_systab_virt_addr(md);
830
831 if (left < desc_size) {
832 new_memmap = realloc_pages(new_memmap, *pg_shift);
833 if (!new_memmap)
834 return NULL;
835
836 left += PAGE_SIZE << *pg_shift;
837 (*pg_shift)++;
838 }
839
840 memcpy(new_memmap + (*count * desc_size), md, desc_size);
841
842 left -= desc_size;
843 (*count)++;
844 }
845
846 return new_memmap;
847 }
848
kexec_enter_virtual_mode(void)849 static void __init kexec_enter_virtual_mode(void)
850 {
851 #ifdef CONFIG_KEXEC_CORE
852 efi_memory_desc_t *md;
853 unsigned int num_pages;
854
855 efi.systab = NULL;
856
857 /*
858 * We don't do virtual mode, since we don't do runtime services, on
859 * non-native EFI. With efi=old_map, we don't do runtime services in
860 * kexec kernel because in the initial boot something else might
861 * have been mapped at these virtual addresses.
862 */
863 if (!efi_is_native() || efi_enabled(EFI_OLD_MEMMAP)) {
864 efi_memmap_unmap();
865 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
866 return;
867 }
868
869 if (efi_alloc_page_tables()) {
870 pr_err("Failed to allocate EFI page tables\n");
871 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
872 return;
873 }
874
875 /*
876 * Map efi regions which were passed via setup_data. The virt_addr is a
877 * fixed addr which was used in first kernel of a kexec boot.
878 */
879 for_each_efi_memory_desc(md) {
880 efi_map_region_fixed(md); /* FIXME: add error handling */
881 get_systab_virt_addr(md);
882 }
883
884 /*
885 * Unregister the early EFI memmap from efi_init() and install
886 * the new EFI memory map.
887 */
888 efi_memmap_unmap();
889
890 if (efi_memmap_init_late(efi.memmap.phys_map,
891 efi.memmap.desc_size * efi.memmap.nr_map)) {
892 pr_err("Failed to remap late EFI memory map\n");
893 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
894 return;
895 }
896
897 BUG_ON(!efi.systab);
898
899 num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE);
900 num_pages >>= PAGE_SHIFT;
901
902 if (efi_setup_page_tables(efi.memmap.phys_map, num_pages)) {
903 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
904 return;
905 }
906
907 efi_sync_low_kernel_mappings();
908
909 /*
910 * Now that EFI is in virtual mode, update the function
911 * pointers in the runtime service table to the new virtual addresses.
912 *
913 * Call EFI services through wrapper functions.
914 */
915 efi.runtime_version = efi_systab.hdr.revision;
916
917 efi_native_runtime_setup();
918
919 efi.set_virtual_address_map = NULL;
920
921 if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
922 runtime_code_page_mkexec();
923 #endif
924 }
925
926 /*
927 * This function will switch the EFI runtime services to virtual mode.
928 * Essentially, we look through the EFI memmap and map every region that
929 * has the runtime attribute bit set in its memory descriptor into the
930 * efi_pgd page table.
931 *
932 * The old method which used to update that memory descriptor with the
933 * virtual address obtained from ioremap() is still supported when the
934 * kernel is booted with efi=old_map on its command line. Same old
935 * method enabled the runtime services to be called without having to
936 * thunk back into physical mode for every invocation.
937 *
938 * The new method does a pagetable switch in a preemption-safe manner
939 * so that we're in a different address space when calling a runtime
940 * function. For function arguments passing we do copy the PUDs of the
941 * kernel page table into efi_pgd prior to each call.
942 *
943 * Specially for kexec boot, efi runtime maps in previous kernel should
944 * be passed in via setup_data. In that case runtime ranges will be mapped
945 * to the same virtual addresses as the first kernel, see
946 * kexec_enter_virtual_mode().
947 */
__efi_enter_virtual_mode(void)948 static void __init __efi_enter_virtual_mode(void)
949 {
950 int count = 0, pg_shift = 0;
951 void *new_memmap = NULL;
952 efi_status_t status;
953 unsigned long pa;
954
955 efi.systab = NULL;
956
957 if (efi_alloc_page_tables()) {
958 pr_err("Failed to allocate EFI page tables\n");
959 goto err;
960 }
961
962 efi_merge_regions();
963 new_memmap = efi_map_regions(&count, &pg_shift);
964 if (!new_memmap) {
965 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
966 goto err;
967 }
968
969 pa = __pa(new_memmap);
970
971 /*
972 * Unregister the early EFI memmap from efi_init() and install
973 * the new EFI memory map that we are about to pass to the
974 * firmware via SetVirtualAddressMap().
975 */
976 efi_memmap_unmap();
977
978 if (efi_memmap_init_late(pa, efi.memmap.desc_size * count)) {
979 pr_err("Failed to remap late EFI memory map\n");
980 goto err;
981 }
982
983 if (efi_enabled(EFI_DBG)) {
984 pr_info("EFI runtime memory map:\n");
985 efi_print_memmap();
986 }
987
988 if (WARN_ON(!efi.systab))
989 goto err;
990
991 if (efi_setup_page_tables(pa, 1 << pg_shift))
992 goto err;
993
994 efi_sync_low_kernel_mappings();
995
996 if (efi_is_native()) {
997 status = phys_efi_set_virtual_address_map(
998 efi.memmap.desc_size * count,
999 efi.memmap.desc_size,
1000 efi.memmap.desc_version,
1001 (efi_memory_desc_t *)pa);
1002 } else {
1003 status = efi_thunk_set_virtual_address_map(
1004 efi_phys.set_virtual_address_map,
1005 efi.memmap.desc_size * count,
1006 efi.memmap.desc_size,
1007 efi.memmap.desc_version,
1008 (efi_memory_desc_t *)pa);
1009 }
1010
1011 if (status != EFI_SUCCESS) {
1012 pr_err("Unable to switch EFI into virtual mode (status=%lx)!\n",
1013 status);
1014 goto err;
1015 }
1016
1017 efi_free_boot_services();
1018
1019 /*
1020 * Now that EFI is in virtual mode, update the function
1021 * pointers in the runtime service table to the new virtual addresses.
1022 *
1023 * Call EFI services through wrapper functions.
1024 */
1025 efi.runtime_version = efi_systab.hdr.revision;
1026
1027 if (efi_is_native())
1028 efi_native_runtime_setup();
1029 else
1030 efi_thunk_runtime_setup();
1031
1032 efi.set_virtual_address_map = NULL;
1033
1034 /*
1035 * Apply more restrictive page table mapping attributes now that
1036 * SVAM() has been called and the firmware has performed all
1037 * necessary relocation fixups for the new virtual addresses.
1038 */
1039 efi_runtime_update_mappings();
1040
1041 /* clean DUMMY object */
1042 efi_delete_dummy_variable();
1043 return;
1044
1045 err:
1046 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
1047 }
1048
efi_enter_virtual_mode(void)1049 void __init efi_enter_virtual_mode(void)
1050 {
1051 if (efi_enabled(EFI_PARAVIRT))
1052 return;
1053
1054 if (efi_setup)
1055 kexec_enter_virtual_mode();
1056 else
1057 __efi_enter_virtual_mode();
1058
1059 efi_dump_pagetable();
1060 }
1061
arch_parse_efi_cmdline(char * str)1062 static int __init arch_parse_efi_cmdline(char *str)
1063 {
1064 if (!str) {
1065 pr_warn("need at least one option\n");
1066 return -EINVAL;
1067 }
1068
1069 if (parse_option_str(str, "old_map"))
1070 set_bit(EFI_OLD_MEMMAP, &efi.flags);
1071
1072 return 0;
1073 }
1074 early_param("efi", arch_parse_efi_cmdline);
1075
efi_is_table_address(unsigned long phys_addr)1076 bool efi_is_table_address(unsigned long phys_addr)
1077 {
1078 unsigned int i;
1079
1080 if (phys_addr == EFI_INVALID_TABLE_ADDR)
1081 return false;
1082
1083 for (i = 0; i < ARRAY_SIZE(efi_tables); i++)
1084 if (*(efi_tables[i]) == phys_addr)
1085 return true;
1086
1087 return false;
1088 }
1089