• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2017 Arm Ltd.
3 #define pr_fmt(fmt) "sdei: " fmt
4 
5 #include <linux/arm-smccc.h>
6 #include <linux/arm_sdei.h>
7 #include <linux/hardirq.h>
8 #include <linux/irqflags.h>
9 #include <linux/sched/task_stack.h>
10 #include <linux/scs.h>
11 #include <linux/uaccess.h>
12 
13 #include <asm/alternative.h>
14 #include <asm/exception.h>
15 #include <asm/kprobes.h>
16 #include <asm/mmu.h>
17 #include <asm/ptrace.h>
18 #include <asm/sections.h>
19 #include <asm/stacktrace.h>
20 #include <asm/sysreg.h>
21 #include <asm/vmap_stack.h>
22 
23 unsigned long sdei_exit_mode;
24 
25 /*
26  * VMAP'd stacks checking for stack overflow on exception using sp as a scratch
27  * register, meaning SDEI has to switch to its own stack. We need two stacks as
28  * a critical event may interrupt a normal event that has just taken a
29  * synchronous exception, and is using sp as scratch register. For a critical
30  * event interrupting a normal event, we can't reliably tell if we were on the
31  * sdei stack.
32  * For now, we allocate stacks when the driver is probed.
33  */
34 DECLARE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
35 DECLARE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
36 
37 #ifdef CONFIG_VMAP_STACK
38 DEFINE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
39 DEFINE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
40 #endif
41 
42 DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr);
43 DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr);
44 
45 #ifdef CONFIG_SHADOW_CALL_STACK
46 DEFINE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr);
47 DEFINE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr);
48 #endif
49 
50 DEFINE_PER_CPU(struct sdei_registered_event *, sdei_active_normal_event);
51 DEFINE_PER_CPU(struct sdei_registered_event *, sdei_active_critical_event);
52 
_free_sdei_stack(unsigned long * __percpu * ptr,int cpu)53 static void _free_sdei_stack(unsigned long * __percpu *ptr, int cpu)
54 {
55 	unsigned long *p;
56 
57 	p = per_cpu(*ptr, cpu);
58 	if (p) {
59 		per_cpu(*ptr, cpu) = NULL;
60 		vfree(p);
61 	}
62 }
63 
free_sdei_stacks(void)64 static void free_sdei_stacks(void)
65 {
66 	int cpu;
67 
68 	if (!IS_ENABLED(CONFIG_VMAP_STACK))
69 		return;
70 
71 	for_each_possible_cpu(cpu) {
72 		_free_sdei_stack(&sdei_stack_normal_ptr, cpu);
73 		_free_sdei_stack(&sdei_stack_critical_ptr, cpu);
74 	}
75 }
76 
_init_sdei_stack(unsigned long * __percpu * ptr,int cpu)77 static int _init_sdei_stack(unsigned long * __percpu *ptr, int cpu)
78 {
79 	unsigned long *p;
80 
81 	p = arch_alloc_vmap_stack(SDEI_STACK_SIZE, cpu_to_node(cpu));
82 	if (!p)
83 		return -ENOMEM;
84 	per_cpu(*ptr, cpu) = p;
85 
86 	return 0;
87 }
88 
init_sdei_stacks(void)89 static int init_sdei_stacks(void)
90 {
91 	int cpu;
92 	int err = 0;
93 
94 	if (!IS_ENABLED(CONFIG_VMAP_STACK))
95 		return 0;
96 
97 	for_each_possible_cpu(cpu) {
98 		err = _init_sdei_stack(&sdei_stack_normal_ptr, cpu);
99 		if (err)
100 			break;
101 		err = _init_sdei_stack(&sdei_stack_critical_ptr, cpu);
102 		if (err)
103 			break;
104 	}
105 
106 	if (err)
107 		free_sdei_stacks();
108 
109 	return err;
110 }
111 
_free_sdei_scs(unsigned long * __percpu * ptr,int cpu)112 static void _free_sdei_scs(unsigned long * __percpu *ptr, int cpu)
113 {
114 	void *s;
115 
116 	s = per_cpu(*ptr, cpu);
117 	if (s) {
118 		per_cpu(*ptr, cpu) = NULL;
119 		scs_free(s);
120 	}
121 }
122 
free_sdei_scs(void)123 static void free_sdei_scs(void)
124 {
125 	int cpu;
126 
127 	for_each_possible_cpu(cpu) {
128 		_free_sdei_scs(&sdei_shadow_call_stack_normal_ptr, cpu);
129 		_free_sdei_scs(&sdei_shadow_call_stack_critical_ptr, cpu);
130 	}
131 }
132 
_init_sdei_scs(unsigned long * __percpu * ptr,int cpu)133 static int _init_sdei_scs(unsigned long * __percpu *ptr, int cpu)
134 {
135 	void *s;
136 
137 	s = scs_alloc(cpu_to_node(cpu));
138 	if (!s)
139 		return -ENOMEM;
140 	per_cpu(*ptr, cpu) = s;
141 
142 	return 0;
143 }
144 
init_sdei_scs(void)145 static int init_sdei_scs(void)
146 {
147 	int cpu;
148 	int err = 0;
149 
150 	if (!IS_ENABLED(CONFIG_SHADOW_CALL_STACK))
151 		return 0;
152 
153 	for_each_possible_cpu(cpu) {
154 		err = _init_sdei_scs(&sdei_shadow_call_stack_normal_ptr, cpu);
155 		if (err)
156 			break;
157 		err = _init_sdei_scs(&sdei_shadow_call_stack_critical_ptr, cpu);
158 		if (err)
159 			break;
160 	}
161 
162 	if (err)
163 		free_sdei_scs();
164 
165 	return err;
166 }
167 
on_sdei_normal_stack(unsigned long sp,unsigned long size,struct stack_info * info)168 static bool on_sdei_normal_stack(unsigned long sp, unsigned long size,
169 				 struct stack_info *info)
170 {
171 	unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
172 	unsigned long high = low + SDEI_STACK_SIZE;
173 
174 	return on_stack(sp, size, low, high, STACK_TYPE_SDEI_NORMAL, info);
175 }
176 
on_sdei_critical_stack(unsigned long sp,unsigned long size,struct stack_info * info)177 static bool on_sdei_critical_stack(unsigned long sp, unsigned long size,
178 				   struct stack_info *info)
179 {
180 	unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr);
181 	unsigned long high = low + SDEI_STACK_SIZE;
182 
183 	return on_stack(sp, size, low, high, STACK_TYPE_SDEI_CRITICAL, info);
184 }
185 
_on_sdei_stack(unsigned long sp,unsigned long size,struct stack_info * info)186 bool _on_sdei_stack(unsigned long sp, unsigned long size, struct stack_info *info)
187 {
188 	if (!IS_ENABLED(CONFIG_VMAP_STACK))
189 		return false;
190 
191 	if (on_sdei_critical_stack(sp, size, info))
192 		return true;
193 
194 	if (on_sdei_normal_stack(sp, size, info))
195 		return true;
196 
197 	return false;
198 }
199 
sdei_arch_get_entry_point(int conduit)200 unsigned long sdei_arch_get_entry_point(int conduit)
201 {
202 	/*
203 	 * SDEI works between adjacent exception levels. If we booted at EL1 we
204 	 * assume a hypervisor is marshalling events. If we booted at EL2 and
205 	 * dropped to EL1 because we don't support VHE, then we can't support
206 	 * SDEI.
207 	 */
208 	if (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) {
209 		pr_err("Not supported on this hardware/boot configuration\n");
210 		goto out_err;
211 	}
212 
213 	if (init_sdei_stacks())
214 		goto out_err;
215 
216 	if (init_sdei_scs())
217 		goto out_err_free_stacks;
218 
219 	sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
220 
221 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
222 	if (arm64_kernel_unmapped_at_el0()) {
223 		unsigned long offset;
224 
225 		offset = (unsigned long)__sdei_asm_entry_trampoline -
226 			 (unsigned long)__entry_tramp_text_start;
227 		return TRAMP_VALIAS + offset;
228 	} else
229 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
230 		return (unsigned long)__sdei_asm_handler;
231 
232 out_err_free_stacks:
233 	free_sdei_stacks();
234 out_err:
235 	return 0;
236 }
237 
238 /*
239  * do_sdei_event() returns one of:
240  *  SDEI_EV_HANDLED -  success, return to the interrupted context.
241  *  SDEI_EV_FAILED  -  failure, return this error code to firmare.
242  *  virtual-address -  success, return to this address.
243  */
do_sdei_event(struct pt_regs * regs,struct sdei_registered_event * arg)244 unsigned long __kprobes do_sdei_event(struct pt_regs *regs,
245 				      struct sdei_registered_event *arg)
246 {
247 	u32 mode;
248 	int i, err = 0;
249 	int clobbered_registers = 4;
250 	u64 elr = read_sysreg(elr_el1);
251 	u32 kernel_mode = read_sysreg(CurrentEL) | 1;	/* +SPSel */
252 	unsigned long vbar = read_sysreg(vbar_el1);
253 
254 	if (arm64_kernel_unmapped_at_el0())
255 		clobbered_registers++;
256 
257 	/* Retrieve the missing registers values */
258 	for (i = 0; i < clobbered_registers; i++) {
259 		/* from within the handler, this call always succeeds */
260 		sdei_api_event_context(i, &regs->regs[i]);
261 	}
262 
263 	err = sdei_event_handler(regs, arg);
264 	if (err)
265 		return SDEI_EV_FAILED;
266 
267 	if (elr != read_sysreg(elr_el1)) {
268 		/*
269 		 * We took a synchronous exception from the SDEI handler.
270 		 * This could deadlock, and if you interrupt KVM it will
271 		 * hyp-panic instead.
272 		 */
273 		pr_warn("unsafe: exception during handler\n");
274 	}
275 
276 	mode = regs->pstate & (PSR_MODE32_BIT | PSR_MODE_MASK);
277 
278 	/*
279 	 * If we interrupted the kernel with interrupts masked, we always go
280 	 * back to wherever we came from.
281 	 */
282 	if (mode == kernel_mode && !interrupts_enabled(regs))
283 		return SDEI_EV_HANDLED;
284 
285 	/*
286 	 * Otherwise, we pretend this was an IRQ. This lets user space tasks
287 	 * receive signals before we return to them, and KVM to invoke it's
288 	 * world switch to do the same.
289 	 *
290 	 * See DDI0487B.a Table D1-7 'Vector offsets from vector table base
291 	 * address'.
292 	 */
293 	if (mode == kernel_mode)
294 		return vbar + 0x280;
295 	else if (mode & PSR_MODE32_BIT)
296 		return vbar + 0x680;
297 
298 	return vbar + 0x480;
299 }
300