1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2023 SiFive
4 * Author: Andy Chiu <andy.chiu@sifive.com>
5 */
6 #include <linux/export.h>
7 #include <linux/sched/signal.h>
8 #include <linux/types.h>
9 #include <linux/slab.h>
10 #include <linux/sched.h>
11 #include <linux/uaccess.h>
12 #include <linux/prctl.h>
13
14 #include <asm/thread_info.h>
15 #include <asm/processor.h>
16 #include <asm/insn.h>
17 #include <asm/vector.h>
18 #include <asm/csr.h>
19 #include <asm/elf.h>
20 #include <asm/ptrace.h>
21 #include <asm/bug.h>
22
23 static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE);
24 static struct kmem_cache *riscv_v_user_cachep;
25 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
26 static struct kmem_cache *riscv_v_kernel_cachep;
27 #endif
28
29 unsigned long riscv_v_vsize __read_mostly;
30 EXPORT_SYMBOL_GPL(riscv_v_vsize);
31
riscv_v_setup_vsize(void)32 int riscv_v_setup_vsize(void)
33 {
34 unsigned long this_vsize;
35
36 /* There are 32 vector registers with vlenb length. */
37 riscv_v_enable();
38 this_vsize = csr_read(CSR_VLENB) * 32;
39 riscv_v_disable();
40
41 if (!riscv_v_vsize) {
42 riscv_v_vsize = this_vsize;
43 return 0;
44 }
45
46 if (riscv_v_vsize != this_vsize) {
47 WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems");
48 return -EOPNOTSUPP;
49 }
50
51 return 0;
52 }
53
riscv_v_setup_ctx_cache(void)54 void __init riscv_v_setup_ctx_cache(void)
55 {
56 if (!has_vector())
57 return;
58
59 riscv_v_user_cachep = kmem_cache_create_usercopy("riscv_vector_ctx",
60 riscv_v_vsize, 16, SLAB_PANIC,
61 0, riscv_v_vsize, NULL);
62 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
63 riscv_v_kernel_cachep = kmem_cache_create("riscv_vector_kctx",
64 riscv_v_vsize, 16,
65 SLAB_PANIC, NULL);
66 #endif
67 }
68
insn_is_vector(u32 insn_buf)69 static bool insn_is_vector(u32 insn_buf)
70 {
71 u32 opcode = insn_buf & __INSN_OPCODE_MASK;
72 u32 width, csr;
73
74 /*
75 * All V-related instructions, including CSR operations are 4-Byte. So,
76 * do not handle if the instruction length is not 4-Byte.
77 */
78 if (unlikely(GET_INSN_LENGTH(insn_buf) != 4))
79 return false;
80
81 switch (opcode) {
82 case RVV_OPCODE_VECTOR:
83 return true;
84 case RVV_OPCODE_VL:
85 case RVV_OPCODE_VS:
86 width = RVV_EXRACT_VL_VS_WIDTH(insn_buf);
87 if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 ||
88 width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64)
89 return true;
90
91 break;
92 case RVG_OPCODE_SYSTEM:
93 csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf);
94 if ((csr >= CSR_VSTART && csr <= CSR_VCSR) ||
95 (csr >= CSR_VL && csr <= CSR_VLENB))
96 return true;
97 }
98
99 return false;
100 }
101
riscv_v_thread_zalloc(struct kmem_cache * cache,struct __riscv_v_ext_state * ctx)102 static int riscv_v_thread_zalloc(struct kmem_cache *cache,
103 struct __riscv_v_ext_state *ctx)
104 {
105 void *datap;
106
107 datap = kmem_cache_zalloc(cache, GFP_KERNEL);
108 if (!datap)
109 return -ENOMEM;
110
111 ctx->datap = datap;
112 memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap));
113 return 0;
114 }
115
riscv_v_thread_alloc(struct task_struct * tsk)116 void riscv_v_thread_alloc(struct task_struct *tsk)
117 {
118 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
119 riscv_v_thread_zalloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate);
120 #endif
121 }
122
riscv_v_thread_free(struct task_struct * tsk)123 void riscv_v_thread_free(struct task_struct *tsk)
124 {
125 if (tsk->thread.vstate.datap)
126 kmem_cache_free(riscv_v_user_cachep, tsk->thread.vstate.datap);
127 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
128 if (tsk->thread.kernel_vstate.datap)
129 kmem_cache_free(riscv_v_kernel_cachep, tsk->thread.kernel_vstate.datap);
130 #endif
131 }
132
133 #define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK)
134 #define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2)
135 #define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK)
136 #define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT))
riscv_v_ctrl_get_cur(struct task_struct * tsk)137 static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk)
138 {
139 return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl);
140 }
141
riscv_v_ctrl_get_next(struct task_struct * tsk)142 static inline int riscv_v_ctrl_get_next(struct task_struct *tsk)
143 {
144 return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl);
145 }
146
riscv_v_ctrl_test_inherit(struct task_struct * tsk)147 static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk)
148 {
149 return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl);
150 }
151
riscv_v_ctrl_set(struct task_struct * tsk,int cur,int nxt,bool inherit)152 static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt,
153 bool inherit)
154 {
155 unsigned long ctrl;
156
157 ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK;
158 ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt);
159 if (inherit)
160 ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT;
161 tsk->thread.vstate_ctrl &= ~PR_RISCV_V_VSTATE_CTRL_MASK;
162 tsk->thread.vstate_ctrl |= ctrl;
163 }
164
riscv_v_vstate_ctrl_user_allowed(void)165 bool riscv_v_vstate_ctrl_user_allowed(void)
166 {
167 return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON;
168 }
169 EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed);
170
riscv_v_first_use_handler(struct pt_regs * regs)171 bool riscv_v_first_use_handler(struct pt_regs *regs)
172 {
173 u32 __user *epc = (u32 __user *)regs->epc;
174 u32 insn = (u32)regs->badaddr;
175
176 /* Do not handle if V is not supported, or disabled */
177 if (!(ELF_HWCAP & COMPAT_HWCAP_ISA_V))
178 return false;
179
180 /* If V has been enabled then it is not the first-use trap */
181 if (riscv_v_vstate_query(regs))
182 return false;
183
184 /* Get the instruction */
185 if (!insn) {
186 if (__get_user(insn, epc))
187 return false;
188 }
189
190 /* Filter out non-V instructions */
191 if (!insn_is_vector(insn))
192 return false;
193
194 /* Sanity check. datap should be null by the time of the first-use trap */
195 WARN_ON(current->thread.vstate.datap);
196
197 /*
198 * Now we sure that this is a V instruction. And it executes in the
199 * context where VS has been off. So, try to allocate the user's V
200 * context and resume execution.
201 */
202 if (riscv_v_thread_zalloc(riscv_v_user_cachep, ¤t->thread.vstate)) {
203 force_sig(SIGBUS);
204 return true;
205 }
206 riscv_v_vstate_on(regs);
207 riscv_v_vstate_set_restore(current, regs);
208 return true;
209 }
210
riscv_v_vstate_ctrl_init(struct task_struct * tsk)211 void riscv_v_vstate_ctrl_init(struct task_struct *tsk)
212 {
213 bool inherit;
214 int cur, next;
215
216 if (!has_vector())
217 return;
218
219 next = riscv_v_ctrl_get_next(tsk);
220 if (!next) {
221 if (READ_ONCE(riscv_v_implicit_uacc))
222 cur = PR_RISCV_V_VSTATE_CTRL_ON;
223 else
224 cur = PR_RISCV_V_VSTATE_CTRL_OFF;
225 } else {
226 cur = next;
227 }
228 /* Clear next mask if inherit-bit is not set */
229 inherit = riscv_v_ctrl_test_inherit(tsk);
230 if (!inherit)
231 next = PR_RISCV_V_VSTATE_CTRL_DEFAULT;
232
233 riscv_v_ctrl_set(tsk, cur, next, inherit);
234 }
235
riscv_v_vstate_ctrl_get_current(void)236 long riscv_v_vstate_ctrl_get_current(void)
237 {
238 if (!has_vector())
239 return -EINVAL;
240
241 return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK;
242 }
243
riscv_v_vstate_ctrl_set_current(unsigned long arg)244 long riscv_v_vstate_ctrl_set_current(unsigned long arg)
245 {
246 bool inherit;
247 int cur, next;
248
249 if (!has_vector())
250 return -EINVAL;
251
252 if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK)
253 return -EINVAL;
254
255 cur = VSTATE_CTRL_GET_CUR(arg);
256 switch (cur) {
257 case PR_RISCV_V_VSTATE_CTRL_OFF:
258 /* Do not allow user to turn off V if current is not off */
259 if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF)
260 return -EPERM;
261
262 break;
263 case PR_RISCV_V_VSTATE_CTRL_ON:
264 break;
265 case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
266 cur = riscv_v_ctrl_get_cur(current);
267 break;
268 default:
269 return -EINVAL;
270 }
271
272 next = VSTATE_CTRL_GET_NEXT(arg);
273 inherit = VSTATE_CTRL_GET_INHERIT(arg);
274 switch (next) {
275 case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
276 case PR_RISCV_V_VSTATE_CTRL_OFF:
277 case PR_RISCV_V_VSTATE_CTRL_ON:
278 riscv_v_ctrl_set(current, cur, next, inherit);
279 return 0;
280 }
281
282 return -EINVAL;
283 }
284
285 #ifdef CONFIG_SYSCTL
286
287 static struct ctl_table riscv_v_default_vstate_table[] = {
288 {
289 .procname = "riscv_v_default_allow",
290 .data = &riscv_v_implicit_uacc,
291 .maxlen = sizeof(riscv_v_implicit_uacc),
292 .mode = 0644,
293 .proc_handler = proc_dobool,
294 },
295 { }
296 };
297
riscv_v_sysctl_init(void)298 static int __init riscv_v_sysctl_init(void)
299 {
300 if (has_vector())
301 if (!register_sysctl("abi", riscv_v_default_vstate_table))
302 return -EINVAL;
303 return 0;
304 }
305
306 #else /* ! CONFIG_SYSCTL */
riscv_v_sysctl_init(void)307 static int __init riscv_v_sysctl_init(void) { return 0; }
308 #endif /* ! CONFIG_SYSCTL */
309
riscv_v_init(void)310 static int riscv_v_init(void)
311 {
312 return riscv_v_sysctl_init();
313 }
314 core_initcall(riscv_v_init);
315