1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020 Western Digital Corporation or its affiliates.
4 */
5 #include <linux/kernel.h>
6 #include <linux/init.h>
7 #include <linux/mm.h>
8 #include <linux/module.h>
9 #include <linux/perf_event.h>
10 #include <linux/irq.h>
11 #include <linux/stringify.h>
12
13 #include <asm/processor.h>
14 #include <asm/ptrace.h>
15 #include <asm/csr.h>
16 #include <asm/entry-common.h>
17 #include <asm/hwprobe.h>
18 #include <asm/cpufeature.h>
19
20 #define INSN_MATCH_LB 0x3
21 #define INSN_MASK_LB 0x707f
22 #define INSN_MATCH_LH 0x1003
23 #define INSN_MASK_LH 0x707f
24 #define INSN_MATCH_LW 0x2003
25 #define INSN_MASK_LW 0x707f
26 #define INSN_MATCH_LD 0x3003
27 #define INSN_MASK_LD 0x707f
28 #define INSN_MATCH_LBU 0x4003
29 #define INSN_MASK_LBU 0x707f
30 #define INSN_MATCH_LHU 0x5003
31 #define INSN_MASK_LHU 0x707f
32 #define INSN_MATCH_LWU 0x6003
33 #define INSN_MASK_LWU 0x707f
34 #define INSN_MATCH_SB 0x23
35 #define INSN_MASK_SB 0x707f
36 #define INSN_MATCH_SH 0x1023
37 #define INSN_MASK_SH 0x707f
38 #define INSN_MATCH_SW 0x2023
39 #define INSN_MASK_SW 0x707f
40 #define INSN_MATCH_SD 0x3023
41 #define INSN_MASK_SD 0x707f
42
43 #define INSN_MATCH_FLW 0x2007
44 #define INSN_MASK_FLW 0x707f
45 #define INSN_MATCH_FLD 0x3007
46 #define INSN_MASK_FLD 0x707f
47 #define INSN_MATCH_FLQ 0x4007
48 #define INSN_MASK_FLQ 0x707f
49 #define INSN_MATCH_FSW 0x2027
50 #define INSN_MASK_FSW 0x707f
51 #define INSN_MATCH_FSD 0x3027
52 #define INSN_MASK_FSD 0x707f
53 #define INSN_MATCH_FSQ 0x4027
54 #define INSN_MASK_FSQ 0x707f
55
56 #define INSN_MATCH_C_LD 0x6000
57 #define INSN_MASK_C_LD 0xe003
58 #define INSN_MATCH_C_SD 0xe000
59 #define INSN_MASK_C_SD 0xe003
60 #define INSN_MATCH_C_LW 0x4000
61 #define INSN_MASK_C_LW 0xe003
62 #define INSN_MATCH_C_SW 0xc000
63 #define INSN_MASK_C_SW 0xe003
64 #define INSN_MATCH_C_LDSP 0x6002
65 #define INSN_MASK_C_LDSP 0xe003
66 #define INSN_MATCH_C_SDSP 0xe002
67 #define INSN_MASK_C_SDSP 0xe003
68 #define INSN_MATCH_C_LWSP 0x4002
69 #define INSN_MASK_C_LWSP 0xe003
70 #define INSN_MATCH_C_SWSP 0xc002
71 #define INSN_MASK_C_SWSP 0xe003
72
73 #define INSN_MATCH_C_FLD 0x2000
74 #define INSN_MASK_C_FLD 0xe003
75 #define INSN_MATCH_C_FLW 0x6000
76 #define INSN_MASK_C_FLW 0xe003
77 #define INSN_MATCH_C_FSD 0xa000
78 #define INSN_MASK_C_FSD 0xe003
79 #define INSN_MATCH_C_FSW 0xe000
80 #define INSN_MASK_C_FSW 0xe003
81 #define INSN_MATCH_C_FLDSP 0x2002
82 #define INSN_MASK_C_FLDSP 0xe003
83 #define INSN_MATCH_C_FSDSP 0xa002
84 #define INSN_MASK_C_FSDSP 0xe003
85 #define INSN_MATCH_C_FLWSP 0x6002
86 #define INSN_MASK_C_FLWSP 0xe003
87 #define INSN_MATCH_C_FSWSP 0xe002
88 #define INSN_MASK_C_FSWSP 0xe003
89
90 #define INSN_MATCH_C_LHU 0x8400
91 #define INSN_MASK_C_LHU 0xfc43
92 #define INSN_MATCH_C_LH 0x8440
93 #define INSN_MASK_C_LH 0xfc43
94 #define INSN_MATCH_C_SH 0x8c00
95 #define INSN_MASK_C_SH 0xfc43
96
97 #define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4)
98
99 #if defined(CONFIG_64BIT)
100 #define LOG_REGBYTES 3
101 #define XLEN 64
102 #else
103 #define LOG_REGBYTES 2
104 #define XLEN 32
105 #endif
106 #define REGBYTES (1 << LOG_REGBYTES)
107 #define XLEN_MINUS_16 ((XLEN) - 16)
108
109 #define SH_RD 7
110 #define SH_RS1 15
111 #define SH_RS2 20
112 #define SH_RS2C 2
113
114 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
115 #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \
116 (RV_X(x, 10, 3) << 3) | \
117 (RV_X(x, 5, 1) << 6))
118 #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \
119 (RV_X(x, 5, 2) << 6))
120 #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \
121 (RV_X(x, 12, 1) << 5) | \
122 (RV_X(x, 2, 2) << 6))
123 #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \
124 (RV_X(x, 12, 1) << 5) | \
125 (RV_X(x, 2, 3) << 6))
126 #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \
127 (RV_X(x, 7, 2) << 6))
128 #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \
129 (RV_X(x, 7, 3) << 6))
130 #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3))
131 #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3))
132 #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5)
133
134 #define SHIFT_RIGHT(x, y) \
135 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y)))
136
137 #define REG_MASK \
138 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES))
139
140 #define REG_OFFSET(insn, pos) \
141 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK)
142
143 #define REG_PTR(insn, pos, regs) \
144 (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))
145
146 #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs))
147 #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs))
148 #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs))
149 #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs))
150 #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs))
151 #define GET_SP(regs) (*REG_PTR(2, 0, regs))
152 #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val))
153 #define IMM_I(insn) ((s32)(insn) >> 20)
154 #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \
155 (s32)(((insn) >> 7) & 0x1f))
156 #define MASK_FUNCT3 0x7000
157
158 #define GET_PRECISION(insn) (((insn) >> 25) & 3)
159 #define GET_RM(insn) (((insn) >> 12) & 7)
160 #define PRECISION_S 0
161 #define PRECISION_D 1
162
163 #ifdef CONFIG_FPU
164
165 #define FP_GET_RD(insn) (insn >> 7 & 0x1F)
166
167 extern void put_f32_reg(unsigned long fp_reg, unsigned long value);
168
set_f32_rd(unsigned long insn,struct pt_regs * regs,unsigned long val)169 static int set_f32_rd(unsigned long insn, struct pt_regs *regs,
170 unsigned long val)
171 {
172 unsigned long fp_reg = FP_GET_RD(insn);
173
174 put_f32_reg(fp_reg, val);
175 regs->status |= SR_FS_DIRTY;
176
177 return 0;
178 }
179
180 extern void put_f64_reg(unsigned long fp_reg, unsigned long value);
181
set_f64_rd(unsigned long insn,struct pt_regs * regs,u64 val)182 static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val)
183 {
184 unsigned long fp_reg = FP_GET_RD(insn);
185 unsigned long value;
186
187 #if __riscv_xlen == 32
188 value = (unsigned long) &val;
189 #else
190 value = val;
191 #endif
192 put_f64_reg(fp_reg, value);
193 regs->status |= SR_FS_DIRTY;
194
195 return 0;
196 }
197
198 #if __riscv_xlen == 32
199 extern void get_f64_reg(unsigned long fp_reg, u64 *value);
200
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)201 static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset,
202 struct pt_regs *regs)
203 {
204 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
205 u64 val;
206
207 get_f64_reg(fp_reg, &val);
208 regs->status |= SR_FS_DIRTY;
209
210 return val;
211 }
212 #else
213
214 extern unsigned long get_f64_reg(unsigned long fp_reg);
215
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)216 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
217 struct pt_regs *regs)
218 {
219 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
220 unsigned long val;
221
222 val = get_f64_reg(fp_reg);
223 regs->status |= SR_FS_DIRTY;
224
225 return val;
226 }
227
228 #endif
229
230 extern unsigned long get_f32_reg(unsigned long fp_reg);
231
get_f32_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)232 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
233 struct pt_regs *regs)
234 {
235 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
236 unsigned long val;
237
238 val = get_f32_reg(fp_reg);
239 regs->status |= SR_FS_DIRTY;
240
241 return val;
242 }
243
244 #else /* CONFIG_FPU */
set_f32_rd(unsigned long insn,struct pt_regs * regs,unsigned long val)245 static void set_f32_rd(unsigned long insn, struct pt_regs *regs,
246 unsigned long val) {}
247
set_f64_rd(unsigned long insn,struct pt_regs * regs,u64 val)248 static void set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) {}
249
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)250 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
251 struct pt_regs *regs)
252 {
253 return 0;
254 }
255
get_f32_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)256 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
257 struct pt_regs *regs)
258 {
259 return 0;
260 }
261
262 #endif
263
264 #define GET_F64_RS2(insn, regs) (get_f64_rs(insn, 20, regs))
265 #define GET_F64_RS2C(insn, regs) (get_f64_rs(insn, 2, regs))
266 #define GET_F64_RS2S(insn, regs) (get_f64_rs(RVC_RS2S(insn), 0, regs))
267
268 #define GET_F32_RS2(insn, regs) (get_f32_rs(insn, 20, regs))
269 #define GET_F32_RS2C(insn, regs) (get_f32_rs(insn, 2, regs))
270 #define GET_F32_RS2S(insn, regs) (get_f32_rs(RVC_RS2S(insn), 0, regs))
271
272 #define __read_insn(regs, insn, insn_addr, type) \
273 ({ \
274 int __ret; \
275 \
276 if (user_mode(regs)) { \
277 __ret = __get_user(insn, (type __user *) insn_addr); \
278 } else { \
279 insn = *(type *)insn_addr; \
280 __ret = 0; \
281 } \
282 \
283 __ret; \
284 })
285
get_insn(struct pt_regs * regs,ulong epc,ulong * r_insn)286 static inline int get_insn(struct pt_regs *regs, ulong epc, ulong *r_insn)
287 {
288 ulong insn = 0;
289
290 if (epc & 0x2) {
291 ulong tmp = 0;
292
293 if (__read_insn(regs, insn, epc, u16))
294 return -EFAULT;
295 /* __get_user() uses regular "lw" which sign extend the loaded
296 * value make sure to clear higher order bits in case we "or" it
297 * below with the upper 16 bits half.
298 */
299 insn &= GENMASK(15, 0);
300 if ((insn & __INSN_LENGTH_MASK) != __INSN_LENGTH_32) {
301 *r_insn = insn;
302 return 0;
303 }
304 epc += sizeof(u16);
305 if (__read_insn(regs, tmp, epc, u16))
306 return -EFAULT;
307 *r_insn = (tmp << 16) | insn;
308
309 return 0;
310 } else {
311 if (__read_insn(regs, insn, epc, u32))
312 return -EFAULT;
313 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) {
314 *r_insn = insn;
315 return 0;
316 }
317 insn &= GENMASK(15, 0);
318 *r_insn = insn;
319
320 return 0;
321 }
322 }
323
324 union reg_data {
325 u8 data_bytes[8];
326 ulong data_ulong;
327 u64 data_u64;
328 };
329
330 static bool unaligned_ctl __read_mostly;
331
332 /* sysctl hooks */
333 int unaligned_enabled __read_mostly = 1; /* Enabled by default */
334
handle_misaligned_load(struct pt_regs * regs)335 int handle_misaligned_load(struct pt_regs *regs)
336 {
337 union reg_data val;
338 unsigned long epc = regs->epc;
339 unsigned long insn;
340 unsigned long addr = regs->badaddr;
341 int fp = 0, shift = 0, len = 0;
342
343 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
344
345 #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
346 *this_cpu_ptr(&misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED;
347 #endif
348
349 if (!unaligned_enabled)
350 return -1;
351
352 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
353 return -1;
354
355 if (get_insn(regs, epc, &insn))
356 return -1;
357
358 regs->epc = 0;
359
360 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
361 len = 4;
362 shift = 8 * (sizeof(unsigned long) - len);
363 #if defined(CONFIG_64BIT)
364 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
365 len = 8;
366 shift = 8 * (sizeof(unsigned long) - len);
367 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
368 len = 4;
369 #endif
370 } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) {
371 fp = 1;
372 len = 8;
373 } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) {
374 fp = 1;
375 len = 4;
376 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
377 len = 2;
378 shift = 8 * (sizeof(unsigned long) - len);
379 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
380 len = 2;
381 #if defined(CONFIG_64BIT)
382 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
383 len = 8;
384 shift = 8 * (sizeof(unsigned long) - len);
385 insn = RVC_RS2S(insn) << SH_RD;
386 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
387 ((insn >> SH_RD) & 0x1f)) {
388 len = 8;
389 shift = 8 * (sizeof(unsigned long) - len);
390 #endif
391 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
392 len = 4;
393 shift = 8 * (sizeof(unsigned long) - len);
394 insn = RVC_RS2S(insn) << SH_RD;
395 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
396 ((insn >> SH_RD) & 0x1f)) {
397 len = 4;
398 shift = 8 * (sizeof(unsigned long) - len);
399 } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) {
400 fp = 1;
401 len = 8;
402 insn = RVC_RS2S(insn) << SH_RD;
403 } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) {
404 fp = 1;
405 len = 8;
406 #if defined(CONFIG_32BIT)
407 } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) {
408 fp = 1;
409 len = 4;
410 insn = RVC_RS2S(insn) << SH_RD;
411 } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) {
412 fp = 1;
413 len = 4;
414 #endif
415 } else if ((insn & INSN_MASK_C_LHU) == INSN_MATCH_C_LHU) {
416 len = 2;
417 insn = RVC_RS2S(insn) << SH_RD;
418 } else if ((insn & INSN_MASK_C_LH) == INSN_MATCH_C_LH) {
419 len = 2;
420 shift = 8 * (sizeof(ulong) - len);
421 insn = RVC_RS2S(insn) << SH_RD;
422 } else {
423 regs->epc = epc;
424 return -1;
425 }
426
427 if (!IS_ENABLED(CONFIG_FPU) && fp)
428 return -EOPNOTSUPP;
429
430 val.data_u64 = 0;
431 if (user_mode(regs)) {
432 if (copy_from_user(&val, (u8 __user *)addr, len))
433 return -1;
434 } else {
435 memcpy(&val, (u8 *)addr, len);
436 }
437
438 if (!fp)
439 SET_RD(insn, regs, (long)(val.data_ulong << shift) >> shift);
440 else if (len == 8)
441 set_f64_rd(insn, regs, val.data_u64);
442 else
443 set_f32_rd(insn, regs, val.data_ulong);
444
445 regs->epc = epc + INSN_LEN(insn);
446
447 return 0;
448 }
449
handle_misaligned_store(struct pt_regs * regs)450 int handle_misaligned_store(struct pt_regs *regs)
451 {
452 union reg_data val;
453 unsigned long epc = regs->epc;
454 unsigned long insn;
455 unsigned long addr = regs->badaddr;
456 int len = 0, fp = 0;
457
458 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
459
460 if (!unaligned_enabled)
461 return -1;
462
463 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
464 return -1;
465
466 if (get_insn(regs, epc, &insn))
467 return -1;
468
469 regs->epc = 0;
470
471 val.data_ulong = GET_RS2(insn, regs);
472
473 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
474 len = 4;
475 #if defined(CONFIG_64BIT)
476 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
477 len = 8;
478 #endif
479 } else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) {
480 fp = 1;
481 len = 8;
482 val.data_u64 = GET_F64_RS2(insn, regs);
483 } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) {
484 fp = 1;
485 len = 4;
486 val.data_ulong = GET_F32_RS2(insn, regs);
487 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
488 len = 2;
489 #if defined(CONFIG_64BIT)
490 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
491 len = 8;
492 val.data_ulong = GET_RS2S(insn, regs);
493 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) {
494 len = 8;
495 val.data_ulong = GET_RS2C(insn, regs);
496 #endif
497 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
498 len = 4;
499 val.data_ulong = GET_RS2S(insn, regs);
500 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) {
501 len = 4;
502 val.data_ulong = GET_RS2C(insn, regs);
503 } else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) {
504 fp = 1;
505 len = 8;
506 val.data_u64 = GET_F64_RS2S(insn, regs);
507 } else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) {
508 fp = 1;
509 len = 8;
510 val.data_u64 = GET_F64_RS2C(insn, regs);
511 #if !defined(CONFIG_64BIT)
512 } else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) {
513 fp = 1;
514 len = 4;
515 val.data_ulong = GET_F32_RS2S(insn, regs);
516 } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) {
517 fp = 1;
518 len = 4;
519 val.data_ulong = GET_F32_RS2C(insn, regs);
520 #endif
521 } else if ((insn & INSN_MASK_C_SH) == INSN_MATCH_C_SH) {
522 len = 2;
523 val.data_ulong = GET_RS2S(insn, regs);
524 } else {
525 regs->epc = epc;
526 return -1;
527 }
528
529 if (!IS_ENABLED(CONFIG_FPU) && fp)
530 return -EOPNOTSUPP;
531
532 if (user_mode(regs)) {
533 if (copy_to_user((u8 __user *)addr, &val, len))
534 return -1;
535 } else {
536 memcpy((u8 *)addr, &val, len);
537 }
538
539 regs->epc = epc + INSN_LEN(insn);
540
541 return 0;
542 }
543
check_unaligned_access_emulated(struct work_struct * work __always_unused)544 void check_unaligned_access_emulated(struct work_struct *work __always_unused)
545 {
546 int cpu = smp_processor_id();
547 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu);
548 unsigned long tmp_var, tmp_val;
549
550 *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
551
552 __asm__ __volatile__ (
553 " "REG_L" %[tmp], 1(%[ptr])\n"
554 : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory");
555
556 /*
557 * If unaligned_ctl is already set, this means that we detected that all
558 * CPUS uses emulated misaligned access at boot time. If that changed
559 * when hotplugging the new cpu, this is something we don't handle.
560 */
561 if (unlikely(unaligned_ctl && (*mas_ptr != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED))) {
562 pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n");
563 while (true)
564 cpu_relax();
565 }
566 }
567
check_unaligned_access_emulated_all_cpus(void)568 bool check_unaligned_access_emulated_all_cpus(void)
569 {
570 int cpu;
571
572 /*
573 * We can only support PR_UNALIGN controls if all CPUs have misaligned
574 * accesses emulated since tasks requesting such control can run on any
575 * CPU.
576 */
577 schedule_on_each_cpu(check_unaligned_access_emulated);
578
579 for_each_online_cpu(cpu)
580 if (per_cpu(misaligned_access_speed, cpu)
581 != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED)
582 return false;
583
584 unaligned_ctl = true;
585 return true;
586 }
587
unaligned_ctl_available(void)588 bool unaligned_ctl_available(void)
589 {
590 return unaligned_ctl;
591 }
592