1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * KVM nVHE hypervisor stack tracing support.
4 *
5 * The unwinder implementation depends on the nVHE mode:
6 *
7 * 1) Non-protected nVHE mode - the host can directly access the
8 * HYP stack pages and unwind the HYP stack in EL1. This saves having
9 * to allocate shared buffers for the host to read the unwinded
10 * stacktrace.
11 *
12 * 2) pKVM (protected nVHE) mode - the host cannot directly access
13 * the HYP memory. The stack is unwinded in EL2 and dumped to a shared
14 * buffer where the host can read and print the stacktrace.
15 *
16 * Copyright (C) 2022 Google LLC
17 */
18
19 #include <linux/kvm.h>
20 #include <linux/kvm_host.h>
21
22 #include <asm/kvm_mmu.h>
23 #include <asm/stacktrace/nvhe.h>
24 #include <asm/kvm_pkvm_module.h>
25
stackinfo_get_overflow(void)26 static struct stack_info stackinfo_get_overflow(void)
27 {
28 struct kvm_nvhe_stacktrace_info *stacktrace_info
29 = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
30 unsigned long low = (unsigned long)stacktrace_info->overflow_stack_base;
31 unsigned long high = low + OVERFLOW_STACK_SIZE;
32
33 return (struct stack_info) {
34 .low = low,
35 .high = high,
36 };
37 }
38
stackinfo_get_overflow_kern_va(void)39 static struct stack_info stackinfo_get_overflow_kern_va(void)
40 {
41 unsigned long low = (unsigned long)this_cpu_ptr_nvhe_sym(overflow_stack);
42 unsigned long high = low + OVERFLOW_STACK_SIZE;
43
44 return (struct stack_info) {
45 .low = low,
46 .high = high,
47 };
48 }
49
stackinfo_get_hyp(void)50 static struct stack_info stackinfo_get_hyp(void)
51 {
52 struct kvm_nvhe_stacktrace_info *stacktrace_info
53 = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
54 unsigned long low = (unsigned long)stacktrace_info->stack_base;
55 unsigned long high = low + NVHE_STACK_SIZE;
56
57 return (struct stack_info) {
58 .low = low,
59 .high = high,
60 };
61 }
62
stackinfo_get_hyp_kern_va(void)63 static struct stack_info stackinfo_get_hyp_kern_va(void)
64 {
65 unsigned long low = (unsigned long)*this_cpu_ptr(&kvm_arm_hyp_stack_base);
66 unsigned long high = low + NVHE_STACK_SIZE;
67
68 return (struct stack_info) {
69 .low = low,
70 .high = high,
71 };
72 }
73
74 /*
75 * kvm_nvhe_stack_kern_va - Convert KVM nVHE HYP stack addresses to a kernel VAs
76 *
77 * The nVHE hypervisor stack is mapped in the flexible 'private' VA range, to
78 * allow for guard pages below the stack. Consequently, the fixed offset address
79 * translation macros won't work here.
80 *
81 * The kernel VA is calculated as an offset from the kernel VA of the hypervisor
82 * stack base.
83 *
84 * Returns true on success and updates @addr to its corresponding kernel VA;
85 * otherwise returns false.
86 */
kvm_nvhe_stack_kern_va(unsigned long * addr,unsigned long size)87 static bool kvm_nvhe_stack_kern_va(unsigned long *addr, unsigned long size)
88 {
89 struct stack_info stack_hyp, stack_kern;
90
91 stack_hyp = stackinfo_get_hyp();
92 stack_kern = stackinfo_get_hyp_kern_va();
93 if (stackinfo_on_stack(&stack_hyp, *addr, size))
94 goto found;
95
96 stack_hyp = stackinfo_get_overflow();
97 stack_kern = stackinfo_get_overflow_kern_va();
98 if (stackinfo_on_stack(&stack_hyp, *addr, size))
99 goto found;
100
101 return false;
102
103 found:
104 *addr = *addr - stack_hyp.low + stack_kern.low;
105 return true;
106 }
107
108 /*
109 * Convert a KVN nVHE HYP frame record address to a kernel VA
110 */
kvm_nvhe_stack_kern_record_va(unsigned long * addr)111 static bool kvm_nvhe_stack_kern_record_va(unsigned long *addr)
112 {
113 return kvm_nvhe_stack_kern_va(addr, 16);
114 }
115
unwind_next(struct unwind_state * state)116 static int unwind_next(struct unwind_state *state)
117 {
118 /*
119 * The FP is in the hypervisor VA space. Convert it to the kernel VA
120 * space so it can be unwound by the regular unwind functions.
121 */
122 if (!kvm_nvhe_stack_kern_record_va(&state->fp))
123 return -EINVAL;
124
125 return unwind_next_frame_record(state);
126 }
127
unwind(struct unwind_state * state,stack_trace_consume_fn consume_entry,void * cookie)128 static void unwind(struct unwind_state *state,
129 stack_trace_consume_fn consume_entry, void *cookie)
130 {
131 while (1) {
132 int ret;
133
134 if (!consume_entry(cookie, state->pc))
135 break;
136 ret = unwind_next(state);
137 if (ret < 0)
138 break;
139 }
140 }
141
142 /*
143 * kvm_nvhe_dump_backtrace_entry - Symbolize and print an nVHE backtrace entry
144 *
145 * @arg : the hypervisor offset, used for address translation
146 * @where : the program counter corresponding to the stack frame
147 */
kvm_nvhe_dump_backtrace_entry(void * arg,unsigned long where)148 static bool kvm_nvhe_dump_backtrace_entry(void *arg, unsigned long where)
149 {
150 unsigned long va_mask = GENMASK_ULL(hyp_va_bits - 1, 0);
151 unsigned long hyp_offset = (unsigned long)arg;
152 unsigned long mod_addr = pkvm_el2_mod_kern_va(where & va_mask);
153 unsigned long where_kaslr;
154
155 if (mod_addr) {
156 where_kaslr = where = mod_addr;
157 } else {
158 /* Mask tags and convert to kern addr */
159 where = (where & va_mask) + hyp_offset;
160 where_kaslr = where + kaslr_offset();
161 }
162
163 kvm_err(" [<%016lx>] %pB\n", where, (void *)(where_kaslr));
164
165 return true;
166 }
167
kvm_nvhe_dump_backtrace_start(void)168 static void kvm_nvhe_dump_backtrace_start(void)
169 {
170 kvm_err("nVHE call trace:\n");
171 }
172
kvm_nvhe_dump_backtrace_end(void)173 static void kvm_nvhe_dump_backtrace_end(void)
174 {
175 kvm_err("---[ end nVHE call trace ]---\n");
176 }
177
178 /*
179 * hyp_dump_backtrace - Dump the non-protected nVHE backtrace.
180 *
181 * @hyp_offset: hypervisor offset, used for address translation.
182 *
183 * The host can directly access HYP stack pages in non-protected
184 * mode, so the unwinding is done directly from EL1. This removes
185 * the need for shared buffers between host and hypervisor for
186 * the stacktrace.
187 */
hyp_dump_backtrace(unsigned long hyp_offset)188 static void hyp_dump_backtrace(unsigned long hyp_offset)
189 {
190 struct kvm_nvhe_stacktrace_info *stacktrace_info;
191 struct stack_info stacks[] = {
192 stackinfo_get_overflow_kern_va(),
193 stackinfo_get_hyp_kern_va(),
194 };
195 struct unwind_state state = {
196 .stacks = stacks,
197 .nr_stacks = ARRAY_SIZE(stacks),
198 };
199
200 stacktrace_info = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
201
202 kvm_nvhe_unwind_init(&state, stacktrace_info->fp, stacktrace_info->pc);
203
204 kvm_nvhe_dump_backtrace_start();
205 unwind(&state, kvm_nvhe_dump_backtrace_entry, (void *)hyp_offset);
206 kvm_nvhe_dump_backtrace_end();
207 }
208
209 #ifdef CONFIG_PKVM_STACKTRACE
210 DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)],
211 pkvm_stacktrace);
212
213 /*
214 * pkvm_dump_backtrace - Dump the protected nVHE HYP backtrace.
215 *
216 * @hyp_offset: hypervisor offset, used for address translation.
217 *
218 * Dumping of the pKVM HYP backtrace is done by reading the
219 * stack addresses from the shared stacktrace buffer, since the
220 * host cannot directly access hypervisor memory in protected
221 * mode.
222 */
pkvm_dump_backtrace(unsigned long hyp_offset)223 static void pkvm_dump_backtrace(unsigned long hyp_offset)
224 {
225 unsigned long *stacktrace
226 = (unsigned long *) this_cpu_ptr_nvhe_sym(pkvm_stacktrace);
227 int i;
228
229 kvm_nvhe_dump_backtrace_start();
230 /* The saved stacktrace is terminated by a null entry */
231 for (i = 0;
232 i < ARRAY_SIZE(kvm_nvhe_sym(pkvm_stacktrace)) && stacktrace[i];
233 i++)
234 kvm_nvhe_dump_backtrace_entry((void *)hyp_offset, stacktrace[i]);
235 kvm_nvhe_dump_backtrace_end();
236 }
237 #else /* !CONFIG_PKVM_STACKTRACE */
pkvm_dump_backtrace(unsigned long hyp_offset)238 static void pkvm_dump_backtrace(unsigned long hyp_offset)
239 {
240 kvm_err("Cannot dump pKVM nVHE stacktrace: !CONFIG_PKVM_STACKTRACE\n");
241 }
242 #endif /* CONFIG_PKVM_STACKTRACE */
243
244 /*
245 * kvm_nvhe_dump_backtrace - Dump KVM nVHE hypervisor backtrace.
246 *
247 * @hyp_offset: hypervisor offset, used for address translation.
248 */
kvm_nvhe_dump_backtrace(unsigned long hyp_offset)249 void kvm_nvhe_dump_backtrace(unsigned long hyp_offset)
250 {
251 if (is_protected_kvm_enabled())
252 pkvm_dump_backtrace(hyp_offset);
253 else
254 hyp_dump_backtrace(hyp_offset);
255 }
256