• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* ----------------------------------------------------------------------- *
2  *
3  *   Copyright 2014 Intel Corporation; author: H. Peter Anvin
4  *
5  *   This program is free software; you can redistribute it and/or modify it
6  *   under the terms and conditions of the GNU General Public License,
7  *   version 2, as published by the Free Software Foundation.
8  *
9  *   This program is distributed in the hope it will be useful, but WITHOUT
10  *   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  *   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12  *   more details.
13  *
14  * ----------------------------------------------------------------------- */
15 
16 /*
17  * The IRET instruction, when returning to a 16-bit segment, only
18  * restores the bottom 16 bits of the user space stack pointer.  This
19  * causes some 16-bit software to break, but it also leaks kernel state
20  * to user space.
21  *
22  * This works around this by creating percpu "ministacks", each of which
23  * is mapped 2^16 times 64K apart.  When we detect that the return SS is
24  * on the LDT, we copy the IRET frame to the ministack and use the
25  * relevant alias to return to userspace.  The ministacks are mapped
26  * readonly, so if the IRET fault we promote #GP to #DF which is an IST
27  * vector and thus has its own stack; we then do the fixup in the #DF
28  * handler.
29  *
30  * This file sets up the ministacks and the related page tables.  The
31  * actual ministack invocation is in entry_64.S.
32  */
33 
34 #include <linux/init.h>
35 #include <linux/init_task.h>
36 #include <linux/kernel.h>
37 #include <linux/percpu.h>
38 #include <linux/gfp.h>
39 #include <linux/random.h>
40 #include <asm/pgtable.h>
41 #include <asm/pgalloc.h>
42 #include <asm/setup.h>
43 #include <asm/espfix.h>
44 
45 /*
46  * Note: we only need 6*8 = 48 bytes for the espfix stack, but round
47  * it up to a cache line to avoid unnecessary sharing.
48  */
49 #define ESPFIX_STACK_SIZE	(8*8UL)
50 #define ESPFIX_STACKS_PER_PAGE	(PAGE_SIZE/ESPFIX_STACK_SIZE)
51 
52 /* There is address space for how many espfix pages? */
53 #define ESPFIX_PAGE_SPACE	(1UL << (PGDIR_SHIFT-PAGE_SHIFT-16))
54 
55 #define ESPFIX_MAX_CPUS		(ESPFIX_STACKS_PER_PAGE * ESPFIX_PAGE_SPACE)
56 #if CONFIG_NR_CPUS > ESPFIX_MAX_CPUS
57 # error "Need more than one PGD for the ESPFIX hack"
58 #endif
59 
60 #define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO)
61 
62 /* This contains the *bottom* address of the espfix stack */
63 DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_stack);
64 DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_waddr);
65 
66 /* Initialization mutex - should this be a spinlock? */
67 static DEFINE_MUTEX(espfix_init_mutex);
68 
69 /* Page allocation bitmap - each page serves ESPFIX_STACKS_PER_PAGE CPUs */
70 #define ESPFIX_MAX_PAGES  DIV_ROUND_UP(CONFIG_NR_CPUS, ESPFIX_STACKS_PER_PAGE)
71 static void *espfix_pages[ESPFIX_MAX_PAGES];
72 
73 static __page_aligned_bss pud_t espfix_pud_page[PTRS_PER_PUD]
74 	__aligned(PAGE_SIZE);
75 
76 static unsigned int page_random, slot_random;
77 
78 /*
79  * This returns the bottom address of the espfix stack for a specific CPU.
80  * The math allows for a non-power-of-two ESPFIX_STACK_SIZE, in which case
81  * we have to account for some amount of padding at the end of each page.
82  */
espfix_base_addr(unsigned int cpu)83 static inline unsigned long espfix_base_addr(unsigned int cpu)
84 {
85 	unsigned long page, slot;
86 	unsigned long addr;
87 
88 	page = (cpu / ESPFIX_STACKS_PER_PAGE) ^ page_random;
89 	slot = (cpu + slot_random) % ESPFIX_STACKS_PER_PAGE;
90 	addr = (page << PAGE_SHIFT) + (slot * ESPFIX_STACK_SIZE);
91 	addr = (addr & 0xffffUL) | ((addr & ~0xffffUL) << 16);
92 	addr += ESPFIX_BASE_ADDR;
93 	return addr;
94 }
95 
96 #define PTE_STRIDE        (65536/PAGE_SIZE)
97 #define ESPFIX_PTE_CLONES (PTRS_PER_PTE/PTE_STRIDE)
98 #define ESPFIX_PMD_CLONES PTRS_PER_PMD
99 #define ESPFIX_PUD_CLONES (65536/(ESPFIX_PTE_CLONES*ESPFIX_PMD_CLONES))
100 
101 #define PGTABLE_PROT	  ((_KERNPG_TABLE & ~_PAGE_RW) | _PAGE_NX)
102 
init_espfix_random(void)103 static void init_espfix_random(void)
104 {
105 	unsigned long rand;
106 
107 	/*
108 	 * This is run before the entropy pools are initialized,
109 	 * but this is hopefully better than nothing.
110 	 */
111 	if (!arch_get_random_long(&rand)) {
112 		/* The constant is an arbitrary large prime */
113 		rdtscll(rand);
114 		rand *= 0xc345c6b72fd16123UL;
115 	}
116 
117 	slot_random = rand % ESPFIX_STACKS_PER_PAGE;
118 	page_random = (rand / ESPFIX_STACKS_PER_PAGE)
119 		& (ESPFIX_PAGE_SPACE - 1);
120 }
121 
init_espfix_bsp(void)122 void __init init_espfix_bsp(void)
123 {
124 	pgd_t *pgd_p;
125 	pteval_t ptemask;
126 
127 	ptemask = __supported_pte_mask;
128 
129 	/* Install the espfix pud into the kernel page directory */
130 	pgd_p = &init_level4_pgt[pgd_index(ESPFIX_BASE_ADDR)];
131 	pgd_populate(&init_mm, pgd_p, (pud_t *)espfix_pud_page);
132 
133 	/* Randomize the locations */
134 	init_espfix_random();
135 
136 	/* The rest is the same as for any other processor */
137 	init_espfix_ap();
138 }
139 
init_espfix_ap(void)140 void init_espfix_ap(void)
141 {
142 	unsigned int cpu, page;
143 	unsigned long addr;
144 	pud_t pud, *pud_p;
145 	pmd_t pmd, *pmd_p;
146 	pte_t pte, *pte_p;
147 	int n;
148 	void *stack_page;
149 	pteval_t ptemask;
150 
151 	/* We only have to do this once... */
152 	if (likely(this_cpu_read(espfix_stack)))
153 		return;		/* Already initialized */
154 
155 	cpu = smp_processor_id();
156 	addr = espfix_base_addr(cpu);
157 	page = cpu/ESPFIX_STACKS_PER_PAGE;
158 
159 	/* Did another CPU already set this up? */
160 	stack_page = ACCESS_ONCE(espfix_pages[page]);
161 	if (likely(stack_page))
162 		goto done;
163 
164 	mutex_lock(&espfix_init_mutex);
165 
166 	/* Did we race on the lock? */
167 	stack_page = ACCESS_ONCE(espfix_pages[page]);
168 	if (stack_page)
169 		goto unlock_done;
170 
171 	ptemask = __supported_pte_mask;
172 
173 	pud_p = &espfix_pud_page[pud_index(addr)];
174 	pud = *pud_p;
175 	if (!pud_present(pud)) {
176 		pmd_p = (pmd_t *)__get_free_page(PGALLOC_GFP);
177 		pud = __pud(__pa(pmd_p) | (PGTABLE_PROT & ptemask));
178 		paravirt_alloc_pmd(&init_mm, __pa(pmd_p) >> PAGE_SHIFT);
179 		for (n = 0; n < ESPFIX_PUD_CLONES; n++)
180 			set_pud(&pud_p[n], pud);
181 	}
182 
183 	pmd_p = pmd_offset(&pud, addr);
184 	pmd = *pmd_p;
185 	if (!pmd_present(pmd)) {
186 		pte_p = (pte_t *)__get_free_page(PGALLOC_GFP);
187 		pmd = __pmd(__pa(pte_p) | (PGTABLE_PROT & ptemask));
188 		paravirt_alloc_pte(&init_mm, __pa(pte_p) >> PAGE_SHIFT);
189 		for (n = 0; n < ESPFIX_PMD_CLONES; n++)
190 			set_pmd(&pmd_p[n], pmd);
191 	}
192 
193 	pte_p = pte_offset_kernel(&pmd, addr);
194 	stack_page = (void *)__get_free_page(GFP_KERNEL);
195 	pte = __pte(__pa(stack_page) | (__PAGE_KERNEL_RO & ptemask));
196 	for (n = 0; n < ESPFIX_PTE_CLONES; n++)
197 		set_pte(&pte_p[n*PTE_STRIDE], pte);
198 
199 	/* Job is done for this CPU and any CPU which shares this page */
200 	ACCESS_ONCE(espfix_pages[page]) = stack_page;
201 
202 unlock_done:
203 	mutex_unlock(&espfix_init_mutex);
204 done:
205 	this_cpu_write(espfix_stack, addr);
206 	this_cpu_write(espfix_waddr, (unsigned long)stack_page
207 		       + (addr & ~PAGE_MASK));
208 }
209