1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * x86_64 specific EFI support functions
4 * Based on Extensible Firmware Interface Specification version 1.0
5 *
6 * Copyright (C) 2005-2008 Intel Co.
7 * Fenghua Yu <fenghua.yu@intel.com>
8 * Bibo Mao <bibo.mao@intel.com>
9 * Chandramouli Narayanan <mouli@linux.intel.com>
10 * Huang Ying <ying.huang@intel.com>
11 *
12 * Code to convert EFI to E820 map has been implemented in elilo bootloader
13 * based on a EFI patch by Edgar Hucek. Based on the E820 map, the page table
14 * is setup appropriately for EFI runtime code.
15 * - mouli 06/14/2007.
16 *
17 */
18
19 #define pr_fmt(fmt) "efi: " fmt
20
21 #include <linux/kernel.h>
22 #include <linux/init.h>
23 #include <linux/mm.h>
24 #include <linux/types.h>
25 #include <linux/spinlock.h>
26 #include <linux/bootmem.h>
27 #include <linux/ioport.h>
28 #include <linux/init.h>
29 #include <linux/mc146818rtc.h>
30 #include <linux/efi.h>
31 #include <linux/uaccess.h>
32 #include <linux/io.h>
33 #include <linux/reboot.h>
34 #include <linux/slab.h>
35 #include <linux/ucs2_string.h>
36
37 #include <asm/setup.h>
38 #include <asm/page.h>
39 #include <asm/e820/api.h>
40 #include <asm/pgtable.h>
41 #include <asm/tlbflush.h>
42 #include <asm/proto.h>
43 #include <asm/efi.h>
44 #include <asm/cacheflush.h>
45 #include <asm/fixmap.h>
46 #include <asm/realmode.h>
47 #include <asm/time.h>
48 #include <asm/pgalloc.h>
49
50 /*
51 * We allocate runtime services regions top-down, starting from -4G, i.e.
52 * 0xffff_ffff_0000_0000 and limit EFI VA mapping space to 64G.
53 */
54 static u64 efi_va = EFI_VA_START;
55
56 struct efi_scratch efi_scratch;
57
early_code_mapping_set_exec(int executable)58 static void __init early_code_mapping_set_exec(int executable)
59 {
60 efi_memory_desc_t *md;
61
62 if (!(__supported_pte_mask & _PAGE_NX))
63 return;
64
65 /* Make EFI service code area executable */
66 for_each_efi_memory_desc(md) {
67 if (md->type == EFI_RUNTIME_SERVICES_CODE ||
68 md->type == EFI_BOOT_SERVICES_CODE)
69 efi_set_executable(md, executable);
70 }
71 }
72
efi_call_phys_prolog(void)73 pgd_t * __init efi_call_phys_prolog(void)
74 {
75 unsigned long vaddr, addr_pgd, addr_p4d, addr_pud;
76 pgd_t *save_pgd, *pgd_k, *pgd_efi;
77 p4d_t *p4d, *p4d_k, *p4d_efi;
78 pud_t *pud;
79
80 int pgd;
81 int n_pgds, i, j;
82
83 if (!efi_enabled(EFI_OLD_MEMMAP)) {
84 save_pgd = (pgd_t *)__read_cr3();
85 write_cr3((unsigned long)efi_scratch.efi_pgt);
86 goto out;
87 }
88
89 early_code_mapping_set_exec(1);
90
91 n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT), PGDIR_SIZE);
92 save_pgd = kmalloc_array(n_pgds, sizeof(*save_pgd), GFP_KERNEL);
93
94 /*
95 * Build 1:1 identity mapping for efi=old_map usage. Note that
96 * PAGE_OFFSET is PGDIR_SIZE aligned when KASLR is disabled, while
97 * it is PUD_SIZE ALIGNED with KASLR enabled. So for a given physical
98 * address X, the pud_index(X) != pud_index(__va(X)), we can only copy
99 * PUD entry of __va(X) to fill in pud entry of X to build 1:1 mapping.
100 * This means here we can only reuse the PMD tables of the direct mapping.
101 */
102 for (pgd = 0; pgd < n_pgds; pgd++) {
103 addr_pgd = (unsigned long)(pgd * PGDIR_SIZE);
104 vaddr = (unsigned long)__va(pgd * PGDIR_SIZE);
105 pgd_efi = pgd_offset_k(addr_pgd);
106 save_pgd[pgd] = *pgd_efi;
107
108 p4d = p4d_alloc(&init_mm, pgd_efi, addr_pgd);
109 if (!p4d) {
110 pr_err("Failed to allocate p4d table!\n");
111 goto out;
112 }
113
114 for (i = 0; i < PTRS_PER_P4D; i++) {
115 addr_p4d = addr_pgd + i * P4D_SIZE;
116 p4d_efi = p4d + p4d_index(addr_p4d);
117
118 pud = pud_alloc(&init_mm, p4d_efi, addr_p4d);
119 if (!pud) {
120 pr_err("Failed to allocate pud table!\n");
121 goto out;
122 }
123
124 for (j = 0; j < PTRS_PER_PUD; j++) {
125 addr_pud = addr_p4d + j * PUD_SIZE;
126
127 if (addr_pud > (max_pfn << PAGE_SHIFT))
128 break;
129
130 vaddr = (unsigned long)__va(addr_pud);
131
132 pgd_k = pgd_offset_k(vaddr);
133 p4d_k = p4d_offset(pgd_k, vaddr);
134 pud[j] = *pud_offset(p4d_k, vaddr);
135 }
136 }
137 pgd_offset_k(pgd * PGDIR_SIZE)->pgd &= ~_PAGE_NX;
138 }
139
140 out:
141 __flush_tlb_all();
142
143 return save_pgd;
144 }
145
efi_call_phys_epilog(pgd_t * save_pgd)146 void __init efi_call_phys_epilog(pgd_t *save_pgd)
147 {
148 /*
149 * After the lock is released, the original page table is restored.
150 */
151 int pgd_idx, i;
152 int nr_pgds;
153 pgd_t *pgd;
154 p4d_t *p4d;
155 pud_t *pud;
156
157 if (!efi_enabled(EFI_OLD_MEMMAP)) {
158 write_cr3((unsigned long)save_pgd);
159 __flush_tlb_all();
160 return;
161 }
162
163 nr_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT) , PGDIR_SIZE);
164
165 for (pgd_idx = 0; pgd_idx < nr_pgds; pgd_idx++) {
166 pgd = pgd_offset_k(pgd_idx * PGDIR_SIZE);
167 set_pgd(pgd_offset_k(pgd_idx * PGDIR_SIZE), save_pgd[pgd_idx]);
168
169 if (!pgd_present(*pgd))
170 continue;
171
172 for (i = 0; i < PTRS_PER_P4D; i++) {
173 p4d = p4d_offset(pgd,
174 pgd_idx * PGDIR_SIZE + i * P4D_SIZE);
175
176 if (!p4d_present(*p4d))
177 continue;
178
179 pud = (pud_t *)p4d_page_vaddr(*p4d);
180 pud_free(&init_mm, pud);
181 }
182
183 p4d = (p4d_t *)pgd_page_vaddr(*pgd);
184 p4d_free(&init_mm, p4d);
185 }
186
187 kfree(save_pgd);
188
189 __flush_tlb_all();
190 early_code_mapping_set_exec(0);
191 }
192
193 static pgd_t *efi_pgd;
194
195 /*
196 * We need our own copy of the higher levels of the page tables
197 * because we want to avoid inserting EFI region mappings (EFI_VA_END
198 * to EFI_VA_START) into the standard kernel page tables. Everything
199 * else can be shared, see efi_sync_low_kernel_mappings().
200 *
201 * We don't want the pgd on the pgd_list and cannot use pgd_alloc() for the
202 * allocation.
203 */
efi_alloc_page_tables(void)204 int __init efi_alloc_page_tables(void)
205 {
206 pgd_t *pgd;
207 p4d_t *p4d;
208 pud_t *pud;
209 gfp_t gfp_mask;
210
211 if (efi_enabled(EFI_OLD_MEMMAP))
212 return 0;
213
214 gfp_mask = GFP_KERNEL | __GFP_ZERO;
215 efi_pgd = (pgd_t *)__get_free_pages(gfp_mask, PGD_ALLOCATION_ORDER);
216 if (!efi_pgd)
217 return -ENOMEM;
218
219 pgd = efi_pgd + pgd_index(EFI_VA_END);
220 p4d = p4d_alloc(&init_mm, pgd, EFI_VA_END);
221 if (!p4d) {
222 free_page((unsigned long)efi_pgd);
223 return -ENOMEM;
224 }
225
226 pud = pud_alloc(&init_mm, p4d, EFI_VA_END);
227 if (!pud) {
228 if (CONFIG_PGTABLE_LEVELS > 4)
229 free_page((unsigned long) pgd_page_vaddr(*pgd));
230 free_pages((unsigned long)efi_pgd, PGD_ALLOCATION_ORDER);
231 return -ENOMEM;
232 }
233
234 return 0;
235 }
236
237 /*
238 * Add low kernel mappings for passing arguments to EFI functions.
239 */
efi_sync_low_kernel_mappings(void)240 void efi_sync_low_kernel_mappings(void)
241 {
242 unsigned num_entries;
243 pgd_t *pgd_k, *pgd_efi;
244 p4d_t *p4d_k, *p4d_efi;
245 pud_t *pud_k, *pud_efi;
246
247 if (efi_enabled(EFI_OLD_MEMMAP))
248 return;
249
250 /*
251 * We can share all PGD entries apart from the one entry that
252 * covers the EFI runtime mapping space.
253 *
254 * Make sure the EFI runtime region mappings are guaranteed to
255 * only span a single PGD entry and that the entry also maps
256 * other important kernel regions.
257 */
258 BUILD_BUG_ON(pgd_index(EFI_VA_END) != pgd_index(MODULES_END));
259 BUILD_BUG_ON((EFI_VA_START & PGDIR_MASK) !=
260 (EFI_VA_END & PGDIR_MASK));
261
262 pgd_efi = efi_pgd + pgd_index(PAGE_OFFSET);
263 pgd_k = pgd_offset_k(PAGE_OFFSET);
264
265 num_entries = pgd_index(EFI_VA_END) - pgd_index(PAGE_OFFSET);
266 memcpy(pgd_efi, pgd_k, sizeof(pgd_t) * num_entries);
267
268 /*
269 * As with PGDs, we share all P4D entries apart from the one entry
270 * that covers the EFI runtime mapping space.
271 */
272 BUILD_BUG_ON(p4d_index(EFI_VA_END) != p4d_index(MODULES_END));
273 BUILD_BUG_ON((EFI_VA_START & P4D_MASK) != (EFI_VA_END & P4D_MASK));
274
275 pgd_efi = efi_pgd + pgd_index(EFI_VA_END);
276 pgd_k = pgd_offset_k(EFI_VA_END);
277 p4d_efi = p4d_offset(pgd_efi, 0);
278 p4d_k = p4d_offset(pgd_k, 0);
279
280 num_entries = p4d_index(EFI_VA_END);
281 memcpy(p4d_efi, p4d_k, sizeof(p4d_t) * num_entries);
282
283 /*
284 * We share all the PUD entries apart from those that map the
285 * EFI regions. Copy around them.
286 */
287 BUILD_BUG_ON((EFI_VA_START & ~PUD_MASK) != 0);
288 BUILD_BUG_ON((EFI_VA_END & ~PUD_MASK) != 0);
289
290 p4d_efi = p4d_offset(pgd_efi, EFI_VA_END);
291 p4d_k = p4d_offset(pgd_k, EFI_VA_END);
292 pud_efi = pud_offset(p4d_efi, 0);
293 pud_k = pud_offset(p4d_k, 0);
294
295 num_entries = pud_index(EFI_VA_END);
296 memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries);
297
298 pud_efi = pud_offset(p4d_efi, EFI_VA_START);
299 pud_k = pud_offset(p4d_k, EFI_VA_START);
300
301 num_entries = PTRS_PER_PUD - pud_index(EFI_VA_START);
302 memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries);
303 }
304
305 /*
306 * Wrapper for slow_virt_to_phys() that handles NULL addresses.
307 */
308 static inline phys_addr_t
virt_to_phys_or_null_size(void * va,unsigned long size)309 virt_to_phys_or_null_size(void *va, unsigned long size)
310 {
311 bool bad_size;
312
313 if (!va)
314 return 0;
315
316 if (virt_addr_valid(va))
317 return virt_to_phys(va);
318
319 /*
320 * A fully aligned variable on the stack is guaranteed not to
321 * cross a page bounary. Try to catch strings on the stack by
322 * checking that 'size' is a power of two.
323 */
324 bad_size = size > PAGE_SIZE || !is_power_of_2(size);
325
326 WARN_ON(!IS_ALIGNED((unsigned long)va, size) || bad_size);
327
328 return slow_virt_to_phys(va);
329 }
330
331 #define virt_to_phys_or_null(addr) \
332 virt_to_phys_or_null_size((addr), sizeof(*(addr)))
333
efi_setup_page_tables(unsigned long pa_memmap,unsigned num_pages)334 int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages)
335 {
336 unsigned long pfn, text, pf;
337 struct page *page;
338 unsigned npages;
339 pgd_t *pgd;
340
341 if (efi_enabled(EFI_OLD_MEMMAP))
342 return 0;
343
344 /*
345 * Since the PGD is encrypted, set the encryption mask so that when
346 * this value is loaded into cr3 the PGD will be decrypted during
347 * the pagetable walk.
348 */
349 efi_scratch.efi_pgt = (pgd_t *)__sme_pa(efi_pgd);
350 pgd = efi_pgd;
351
352 /*
353 * It can happen that the physical address of new_memmap lands in memory
354 * which is not mapped in the EFI page table. Therefore we need to go
355 * and ident-map those pages containing the map before calling
356 * phys_efi_set_virtual_address_map().
357 */
358 pfn = pa_memmap >> PAGE_SHIFT;
359 pf = _PAGE_NX | _PAGE_RW | _PAGE_ENC;
360 if (kernel_map_pages_in_pgd(pgd, pfn, pa_memmap, num_pages, pf)) {
361 pr_err("Error ident-mapping new memmap (0x%lx)!\n", pa_memmap);
362 return 1;
363 }
364
365 efi_scratch.use_pgd = true;
366
367 /*
368 * Certain firmware versions are way too sentimential and still believe
369 * they are exclusive and unquestionable owners of the first physical page,
370 * even though they explicitly mark it as EFI_CONVENTIONAL_MEMORY
371 * (but then write-access it later during SetVirtualAddressMap()).
372 *
373 * Create a 1:1 mapping for this page, to avoid triple faults during early
374 * boot with such firmware. We are free to hand this page to the BIOS,
375 * as trim_bios_range() will reserve the first page and isolate it away
376 * from memory allocators anyway.
377 */
378 if (kernel_map_pages_in_pgd(pgd, 0x0, 0x0, 1, _PAGE_RW)) {
379 pr_err("Failed to create 1:1 mapping for the first page!\n");
380 return 1;
381 }
382
383 /*
384 * When making calls to the firmware everything needs to be 1:1
385 * mapped and addressable with 32-bit pointers. Map the kernel
386 * text and allocate a new stack because we can't rely on the
387 * stack pointer being < 4GB.
388 */
389 if (!IS_ENABLED(CONFIG_EFI_MIXED) || efi_is_native())
390 return 0;
391
392 page = alloc_page(GFP_KERNEL|__GFP_DMA32);
393 if (!page) {
394 pr_err("Unable to allocate EFI runtime stack < 4GB\n");
395 return 1;
396 }
397
398 efi_scratch.phys_stack = page_to_phys(page + 1); /* stack grows down */
399
400 npages = (_etext - _text) >> PAGE_SHIFT;
401 text = __pa(_text);
402 pfn = text >> PAGE_SHIFT;
403
404 pf = _PAGE_RW | _PAGE_ENC;
405 if (kernel_map_pages_in_pgd(pgd, pfn, text, npages, pf)) {
406 pr_err("Failed to map kernel text 1:1\n");
407 return 1;
408 }
409
410 return 0;
411 }
412
__map_region(efi_memory_desc_t * md,u64 va)413 static void __init __map_region(efi_memory_desc_t *md, u64 va)
414 {
415 unsigned long flags = _PAGE_RW;
416 unsigned long pfn;
417 pgd_t *pgd = efi_pgd;
418
419 if (!(md->attribute & EFI_MEMORY_WB))
420 flags |= _PAGE_PCD;
421
422 pfn = md->phys_addr >> PAGE_SHIFT;
423 if (kernel_map_pages_in_pgd(pgd, pfn, va, md->num_pages, flags))
424 pr_warn("Error mapping PA 0x%llx -> VA 0x%llx!\n",
425 md->phys_addr, va);
426 }
427
efi_map_region(efi_memory_desc_t * md)428 void __init efi_map_region(efi_memory_desc_t *md)
429 {
430 unsigned long size = md->num_pages << PAGE_SHIFT;
431 u64 pa = md->phys_addr;
432
433 if (efi_enabled(EFI_OLD_MEMMAP))
434 return old_map_region(md);
435
436 /*
437 * Make sure the 1:1 mappings are present as a catch-all for b0rked
438 * firmware which doesn't update all internal pointers after switching
439 * to virtual mode and would otherwise crap on us.
440 */
441 __map_region(md, md->phys_addr);
442
443 /*
444 * Enforce the 1:1 mapping as the default virtual address when
445 * booting in EFI mixed mode, because even though we may be
446 * running a 64-bit kernel, the firmware may only be 32-bit.
447 */
448 if (!efi_is_native () && IS_ENABLED(CONFIG_EFI_MIXED)) {
449 md->virt_addr = md->phys_addr;
450 return;
451 }
452
453 efi_va -= size;
454
455 /* Is PA 2M-aligned? */
456 if (!(pa & (PMD_SIZE - 1))) {
457 efi_va &= PMD_MASK;
458 } else {
459 u64 pa_offset = pa & (PMD_SIZE - 1);
460 u64 prev_va = efi_va;
461
462 /* get us the same offset within this 2M page */
463 efi_va = (efi_va & PMD_MASK) + pa_offset;
464
465 if (efi_va > prev_va)
466 efi_va -= PMD_SIZE;
467 }
468
469 if (efi_va < EFI_VA_END) {
470 pr_warn(FW_WARN "VA address range overflow!\n");
471 return;
472 }
473
474 /* Do the VA map */
475 __map_region(md, efi_va);
476 md->virt_addr = efi_va;
477 }
478
479 /*
480 * kexec kernel will use efi_map_region_fixed to map efi runtime memory ranges.
481 * md->virt_addr is the original virtual address which had been mapped in kexec
482 * 1st kernel.
483 */
efi_map_region_fixed(efi_memory_desc_t * md)484 void __init efi_map_region_fixed(efi_memory_desc_t *md)
485 {
486 __map_region(md, md->phys_addr);
487 __map_region(md, md->virt_addr);
488 }
489
efi_ioremap(unsigned long phys_addr,unsigned long size,u32 type,u64 attribute)490 void __iomem *__init efi_ioremap(unsigned long phys_addr, unsigned long size,
491 u32 type, u64 attribute)
492 {
493 unsigned long last_map_pfn;
494
495 if (type == EFI_MEMORY_MAPPED_IO)
496 return ioremap(phys_addr, size);
497
498 last_map_pfn = init_memory_mapping(phys_addr, phys_addr + size);
499 if ((last_map_pfn << PAGE_SHIFT) < phys_addr + size) {
500 unsigned long top = last_map_pfn << PAGE_SHIFT;
501 efi_ioremap(top, size - (top - phys_addr), type, attribute);
502 }
503
504 if (!(attribute & EFI_MEMORY_WB))
505 efi_memory_uc((u64)(unsigned long)__va(phys_addr), size);
506
507 return (void __iomem *)__va(phys_addr);
508 }
509
parse_efi_setup(u64 phys_addr,u32 data_len)510 void __init parse_efi_setup(u64 phys_addr, u32 data_len)
511 {
512 efi_setup = phys_addr + sizeof(struct setup_data);
513 }
514
efi_update_mappings(efi_memory_desc_t * md,unsigned long pf)515 static int __init efi_update_mappings(efi_memory_desc_t *md, unsigned long pf)
516 {
517 unsigned long pfn;
518 pgd_t *pgd = efi_pgd;
519 int err1, err2;
520
521 /* Update the 1:1 mapping */
522 pfn = md->phys_addr >> PAGE_SHIFT;
523 err1 = kernel_map_pages_in_pgd(pgd, pfn, md->phys_addr, md->num_pages, pf);
524 if (err1) {
525 pr_err("Error while updating 1:1 mapping PA 0x%llx -> VA 0x%llx!\n",
526 md->phys_addr, md->virt_addr);
527 }
528
529 err2 = kernel_map_pages_in_pgd(pgd, pfn, md->virt_addr, md->num_pages, pf);
530 if (err2) {
531 pr_err("Error while updating VA mapping PA 0x%llx -> VA 0x%llx!\n",
532 md->phys_addr, md->virt_addr);
533 }
534
535 return err1 || err2;
536 }
537
efi_update_mem_attr(struct mm_struct * mm,efi_memory_desc_t * md)538 static int __init efi_update_mem_attr(struct mm_struct *mm, efi_memory_desc_t *md)
539 {
540 unsigned long pf = 0;
541
542 if (md->attribute & EFI_MEMORY_XP)
543 pf |= _PAGE_NX;
544
545 if (!(md->attribute & EFI_MEMORY_RO))
546 pf |= _PAGE_RW;
547
548 return efi_update_mappings(md, pf);
549 }
550
efi_runtime_update_mappings(void)551 void __init efi_runtime_update_mappings(void)
552 {
553 efi_memory_desc_t *md;
554
555 if (efi_enabled(EFI_OLD_MEMMAP)) {
556 if (__supported_pte_mask & _PAGE_NX)
557 runtime_code_page_mkexec();
558 return;
559 }
560
561 /*
562 * Use the EFI Memory Attribute Table for mapping permissions if it
563 * exists, since it is intended to supersede EFI_PROPERTIES_TABLE.
564 */
565 if (efi_enabled(EFI_MEM_ATTR)) {
566 efi_memattr_apply_permissions(NULL, efi_update_mem_attr);
567 return;
568 }
569
570 /*
571 * EFI_MEMORY_ATTRIBUTES_TABLE is intended to replace
572 * EFI_PROPERTIES_TABLE. So, use EFI_PROPERTIES_TABLE to update
573 * permissions only if EFI_MEMORY_ATTRIBUTES_TABLE is not
574 * published by the firmware. Even if we find a buggy implementation of
575 * EFI_MEMORY_ATTRIBUTES_TABLE, don't fall back to
576 * EFI_PROPERTIES_TABLE, because of the same reason.
577 */
578
579 if (!efi_enabled(EFI_NX_PE_DATA))
580 return;
581
582 for_each_efi_memory_desc(md) {
583 unsigned long pf = 0;
584
585 if (!(md->attribute & EFI_MEMORY_RUNTIME))
586 continue;
587
588 if (!(md->attribute & EFI_MEMORY_WB))
589 pf |= _PAGE_PCD;
590
591 if ((md->attribute & EFI_MEMORY_XP) ||
592 (md->type == EFI_RUNTIME_SERVICES_DATA))
593 pf |= _PAGE_NX;
594
595 if (!(md->attribute & EFI_MEMORY_RO) &&
596 (md->type != EFI_RUNTIME_SERVICES_CODE))
597 pf |= _PAGE_RW;
598
599 efi_update_mappings(md, pf);
600 }
601 }
602
efi_dump_pagetable(void)603 void __init efi_dump_pagetable(void)
604 {
605 #ifdef CONFIG_EFI_PGT_DUMP
606 if (efi_enabled(EFI_OLD_MEMMAP))
607 ptdump_walk_pgd_level(NULL, swapper_pg_dir);
608 else
609 ptdump_walk_pgd_level(NULL, efi_pgd);
610 #endif
611 }
612
613 #ifdef CONFIG_EFI_MIXED
614 extern efi_status_t efi64_thunk(u32, ...);
615
616 #define runtime_service32(func) \
617 ({ \
618 u32 table = (u32)(unsigned long)efi.systab; \
619 u32 *rt, *___f; \
620 \
621 rt = (u32 *)(table + offsetof(efi_system_table_32_t, runtime)); \
622 ___f = (u32 *)(*rt + offsetof(efi_runtime_services_32_t, func)); \
623 *___f; \
624 })
625
626 /*
627 * Switch to the EFI page tables early so that we can access the 1:1
628 * runtime services mappings which are not mapped in any other page
629 * tables. This function must be called before runtime_service32().
630 *
631 * Also, disable interrupts because the IDT points to 64-bit handlers,
632 * which aren't going to function correctly when we switch to 32-bit.
633 */
634 #define efi_thunk(f, ...) \
635 ({ \
636 efi_status_t __s; \
637 unsigned long __flags; \
638 u32 __func; \
639 \
640 local_irq_save(__flags); \
641 arch_efi_call_virt_setup(); \
642 \
643 __func = runtime_service32(f); \
644 __s = efi64_thunk(__func, __VA_ARGS__); \
645 \
646 arch_efi_call_virt_teardown(); \
647 local_irq_restore(__flags); \
648 \
649 __s; \
650 })
651
efi_thunk_set_virtual_address_map(void * phys_set_virtual_address_map,unsigned long memory_map_size,unsigned long descriptor_size,u32 descriptor_version,efi_memory_desc_t * virtual_map)652 efi_status_t efi_thunk_set_virtual_address_map(
653 void *phys_set_virtual_address_map,
654 unsigned long memory_map_size,
655 unsigned long descriptor_size,
656 u32 descriptor_version,
657 efi_memory_desc_t *virtual_map)
658 {
659 efi_status_t status;
660 unsigned long flags;
661 u32 func;
662
663 efi_sync_low_kernel_mappings();
664 local_irq_save(flags);
665
666 efi_scratch.prev_cr3 = __read_cr3();
667 write_cr3((unsigned long)efi_scratch.efi_pgt);
668 __flush_tlb_all();
669
670 func = (u32)(unsigned long)phys_set_virtual_address_map;
671 status = efi64_thunk(func, memory_map_size, descriptor_size,
672 descriptor_version, virtual_map);
673
674 write_cr3(efi_scratch.prev_cr3);
675 __flush_tlb_all();
676 local_irq_restore(flags);
677
678 return status;
679 }
680
efi_thunk_get_time(efi_time_t * tm,efi_time_cap_t * tc)681 static efi_status_t efi_thunk_get_time(efi_time_t *tm, efi_time_cap_t *tc)
682 {
683 efi_status_t status;
684 u32 phys_tm, phys_tc;
685
686 spin_lock(&rtc_lock);
687
688 phys_tm = virt_to_phys_or_null(tm);
689 phys_tc = virt_to_phys_or_null(tc);
690
691 status = efi_thunk(get_time, phys_tm, phys_tc);
692
693 spin_unlock(&rtc_lock);
694
695 return status;
696 }
697
efi_thunk_set_time(efi_time_t * tm)698 static efi_status_t efi_thunk_set_time(efi_time_t *tm)
699 {
700 efi_status_t status;
701 u32 phys_tm;
702
703 spin_lock(&rtc_lock);
704
705 phys_tm = virt_to_phys_or_null(tm);
706
707 status = efi_thunk(set_time, phys_tm);
708
709 spin_unlock(&rtc_lock);
710
711 return status;
712 }
713
714 static efi_status_t
efi_thunk_get_wakeup_time(efi_bool_t * enabled,efi_bool_t * pending,efi_time_t * tm)715 efi_thunk_get_wakeup_time(efi_bool_t *enabled, efi_bool_t *pending,
716 efi_time_t *tm)
717 {
718 efi_status_t status;
719 u32 phys_enabled, phys_pending, phys_tm;
720
721 spin_lock(&rtc_lock);
722
723 phys_enabled = virt_to_phys_or_null(enabled);
724 phys_pending = virt_to_phys_or_null(pending);
725 phys_tm = virt_to_phys_or_null(tm);
726
727 status = efi_thunk(get_wakeup_time, phys_enabled,
728 phys_pending, phys_tm);
729
730 spin_unlock(&rtc_lock);
731
732 return status;
733 }
734
735 static efi_status_t
efi_thunk_set_wakeup_time(efi_bool_t enabled,efi_time_t * tm)736 efi_thunk_set_wakeup_time(efi_bool_t enabled, efi_time_t *tm)
737 {
738 efi_status_t status;
739 u32 phys_tm;
740
741 spin_lock(&rtc_lock);
742
743 phys_tm = virt_to_phys_or_null(tm);
744
745 status = efi_thunk(set_wakeup_time, enabled, phys_tm);
746
747 spin_unlock(&rtc_lock);
748
749 return status;
750 }
751
efi_name_size(efi_char16_t * name)752 static unsigned long efi_name_size(efi_char16_t *name)
753 {
754 return ucs2_strsize(name, EFI_VAR_NAME_LEN) + 1;
755 }
756
757 static efi_status_t
efi_thunk_get_variable(efi_char16_t * name,efi_guid_t * vendor,u32 * attr,unsigned long * data_size,void * data)758 efi_thunk_get_variable(efi_char16_t *name, efi_guid_t *vendor,
759 u32 *attr, unsigned long *data_size, void *data)
760 {
761 efi_status_t status;
762 u32 phys_name, phys_vendor, phys_attr;
763 u32 phys_data_size, phys_data;
764
765 phys_data_size = virt_to_phys_or_null(data_size);
766 phys_vendor = virt_to_phys_or_null(vendor);
767 phys_name = virt_to_phys_or_null_size(name, efi_name_size(name));
768 phys_attr = virt_to_phys_or_null(attr);
769 phys_data = virt_to_phys_or_null_size(data, *data_size);
770
771 status = efi_thunk(get_variable, phys_name, phys_vendor,
772 phys_attr, phys_data_size, phys_data);
773
774 return status;
775 }
776
777 static efi_status_t
efi_thunk_set_variable(efi_char16_t * name,efi_guid_t * vendor,u32 attr,unsigned long data_size,void * data)778 efi_thunk_set_variable(efi_char16_t *name, efi_guid_t *vendor,
779 u32 attr, unsigned long data_size, void *data)
780 {
781 u32 phys_name, phys_vendor, phys_data;
782 efi_status_t status;
783
784 phys_name = virt_to_phys_or_null_size(name, efi_name_size(name));
785 phys_vendor = virt_to_phys_or_null(vendor);
786 phys_data = virt_to_phys_or_null_size(data, data_size);
787
788 /* If data_size is > sizeof(u32) we've got problems */
789 status = efi_thunk(set_variable, phys_name, phys_vendor,
790 attr, data_size, phys_data);
791
792 return status;
793 }
794
795 static efi_status_t
efi_thunk_get_next_variable(unsigned long * name_size,efi_char16_t * name,efi_guid_t * vendor)796 efi_thunk_get_next_variable(unsigned long *name_size,
797 efi_char16_t *name,
798 efi_guid_t *vendor)
799 {
800 efi_status_t status;
801 u32 phys_name_size, phys_name, phys_vendor;
802
803 phys_name_size = virt_to_phys_or_null(name_size);
804 phys_vendor = virt_to_phys_or_null(vendor);
805 phys_name = virt_to_phys_or_null_size(name, *name_size);
806
807 status = efi_thunk(get_next_variable, phys_name_size,
808 phys_name, phys_vendor);
809
810 return status;
811 }
812
813 static efi_status_t
efi_thunk_get_next_high_mono_count(u32 * count)814 efi_thunk_get_next_high_mono_count(u32 *count)
815 {
816 efi_status_t status;
817 u32 phys_count;
818
819 phys_count = virt_to_phys_or_null(count);
820 status = efi_thunk(get_next_high_mono_count, phys_count);
821
822 return status;
823 }
824
825 static void
efi_thunk_reset_system(int reset_type,efi_status_t status,unsigned long data_size,efi_char16_t * data)826 efi_thunk_reset_system(int reset_type, efi_status_t status,
827 unsigned long data_size, efi_char16_t *data)
828 {
829 u32 phys_data;
830
831 phys_data = virt_to_phys_or_null_size(data, data_size);
832
833 efi_thunk(reset_system, reset_type, status, data_size, phys_data);
834 }
835
836 static efi_status_t
efi_thunk_update_capsule(efi_capsule_header_t ** capsules,unsigned long count,unsigned long sg_list)837 efi_thunk_update_capsule(efi_capsule_header_t **capsules,
838 unsigned long count, unsigned long sg_list)
839 {
840 /*
841 * To properly support this function we would need to repackage
842 * 'capsules' because the firmware doesn't understand 64-bit
843 * pointers.
844 */
845 return EFI_UNSUPPORTED;
846 }
847
848 static efi_status_t
efi_thunk_query_variable_info(u32 attr,u64 * storage_space,u64 * remaining_space,u64 * max_variable_size)849 efi_thunk_query_variable_info(u32 attr, u64 *storage_space,
850 u64 *remaining_space,
851 u64 *max_variable_size)
852 {
853 efi_status_t status;
854 u32 phys_storage, phys_remaining, phys_max;
855
856 if (efi.runtime_version < EFI_2_00_SYSTEM_TABLE_REVISION)
857 return EFI_UNSUPPORTED;
858
859 phys_storage = virt_to_phys_or_null(storage_space);
860 phys_remaining = virt_to_phys_or_null(remaining_space);
861 phys_max = virt_to_phys_or_null(max_variable_size);
862
863 status = efi_thunk(query_variable_info, attr, phys_storage,
864 phys_remaining, phys_max);
865
866 return status;
867 }
868
869 static efi_status_t
efi_thunk_query_capsule_caps(efi_capsule_header_t ** capsules,unsigned long count,u64 * max_size,int * reset_type)870 efi_thunk_query_capsule_caps(efi_capsule_header_t **capsules,
871 unsigned long count, u64 *max_size,
872 int *reset_type)
873 {
874 /*
875 * To properly support this function we would need to repackage
876 * 'capsules' because the firmware doesn't understand 64-bit
877 * pointers.
878 */
879 return EFI_UNSUPPORTED;
880 }
881
efi_thunk_runtime_setup(void)882 void efi_thunk_runtime_setup(void)
883 {
884 efi.get_time = efi_thunk_get_time;
885 efi.set_time = efi_thunk_set_time;
886 efi.get_wakeup_time = efi_thunk_get_wakeup_time;
887 efi.set_wakeup_time = efi_thunk_set_wakeup_time;
888 efi.get_variable = efi_thunk_get_variable;
889 efi.get_next_variable = efi_thunk_get_next_variable;
890 efi.set_variable = efi_thunk_set_variable;
891 efi.get_next_high_mono_count = efi_thunk_get_next_high_mono_count;
892 efi.reset_system = efi_thunk_reset_system;
893 efi.query_variable_info = efi_thunk_query_variable_info;
894 efi.update_capsule = efi_thunk_update_capsule;
895 efi.query_capsule_caps = efi_thunk_query_capsule_caps;
896 }
897 #endif /* CONFIG_EFI_MIXED */
898