1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright 2022-2024 Rivos, Inc
4 */
5
6 #ifndef _ASM_CPUFEATURE_H
7 #define _ASM_CPUFEATURE_H
8
9 #include <linux/bitmap.h>
10 #include <linux/jump_label.h>
11 #include <linux/workqueue.h>
12 #include <asm/hwcap.h>
13 #include <asm/alternative-macros.h>
14 #include <asm/errno.h>
15
16 /*
17 * These are probed via a device_initcall(), via either the SBI or directly
18 * from the corresponding CSRs.
19 */
20 struct riscv_cpuinfo {
21 unsigned long mvendorid;
22 unsigned long marchid;
23 unsigned long mimpid;
24 };
25
26 struct riscv_isainfo {
27 DECLARE_BITMAP(isa, RISCV_ISA_EXT_MAX);
28 };
29
30 DECLARE_PER_CPU(struct riscv_cpuinfo, riscv_cpuinfo);
31
32 /* Per-cpu ISA extensions. */
33 extern struct riscv_isainfo hart_isa[NR_CPUS];
34
35 void riscv_user_isa_enable(void);
36
37 #define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size, _validate) { \
38 .name = #_name, \
39 .property = #_name, \
40 .id = _id, \
41 .subset_ext_ids = _subset_exts, \
42 .subset_ext_size = _subset_exts_size, \
43 .validate = _validate \
44 }
45
46 #define __RISCV_ISA_EXT_DATA(_name, _id) _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0, NULL)
47
48 #define __RISCV_ISA_EXT_DATA_VALIDATE(_name, _id, _validate) \
49 _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0, _validate)
50
51 /* Used to declare pure "lasso" extension (Zk for instance) */
52 #define __RISCV_ISA_EXT_BUNDLE(_name, _bundled_exts) \
53 _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, \
54 ARRAY_SIZE(_bundled_exts), NULL)
55
56 /* Used to declare extensions that are a superset of other extensions (Zvbb for instance) */
57 #define __RISCV_ISA_EXT_SUPERSET(_name, _id, _sub_exts) \
58 _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), NULL)
59 #define __RISCV_ISA_EXT_SUPERSET_VALIDATE(_name, _id, _sub_exts, _validate) \
60 _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), _validate)
61
62 #if defined(CONFIG_RISCV_MISALIGNED)
63 bool check_unaligned_access_emulated_all_cpus(void);
64 void check_unaligned_access_emulated(struct work_struct *work __always_unused);
65 void unaligned_emulation_finish(void);
66 bool unaligned_ctl_available(void);
67 DECLARE_PER_CPU(long, misaligned_access_speed);
68 #else
unaligned_ctl_available(void)69 static inline bool unaligned_ctl_available(void)
70 {
71 return false;
72 }
73 #endif
74
75 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)
76 DECLARE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key);
77
has_fast_unaligned_accesses(void)78 static __always_inline bool has_fast_unaligned_accesses(void)
79 {
80 return static_branch_likely(&fast_unaligned_access_speed_key);
81 }
82 #else
has_fast_unaligned_accesses(void)83 static __always_inline bool has_fast_unaligned_accesses(void)
84 {
85 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
86 return true;
87 else
88 return false;
89 }
90 #endif
91
92 unsigned long riscv_get_elf_hwcap(void);
93
94 struct riscv_isa_ext_data {
95 const unsigned int id;
96 const char *name;
97 const char *property;
98 const unsigned int *subset_ext_ids;
99 const unsigned int subset_ext_size;
100 int (*validate)(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap);
101 };
102
103 extern const struct riscv_isa_ext_data riscv_isa_ext[];
104 extern const size_t riscv_isa_ext_count;
105 extern bool riscv_isa_fallback;
106
107 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap);
108
109 #define STANDARD_EXT 0
110
111 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit);
112 #define riscv_isa_extension_available(isa_bitmap, ext) \
113 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_##ext)
114
__riscv_has_extension_likely(const unsigned long vendor,const unsigned long ext)115 static __always_inline bool __riscv_has_extension_likely(const unsigned long vendor,
116 const unsigned long ext)
117 {
118 asm goto(ALTERNATIVE("j %l[l_no]", "nop", %[vendor], %[ext], 1)
119 :
120 : [vendor] "i" (vendor), [ext] "i" (ext)
121 :
122 : l_no);
123
124 return true;
125 l_no:
126 return false;
127 }
128
__riscv_has_extension_unlikely(const unsigned long vendor,const unsigned long ext)129 static __always_inline bool __riscv_has_extension_unlikely(const unsigned long vendor,
130 const unsigned long ext)
131 {
132 asm goto(ALTERNATIVE("nop", "j %l[l_yes]", %[vendor], %[ext], 1)
133 :
134 : [vendor] "i" (vendor), [ext] "i" (ext)
135 :
136 : l_yes);
137
138 return false;
139 l_yes:
140 return true;
141 }
142
riscv_has_extension_unlikely(const unsigned long ext)143 static __always_inline bool riscv_has_extension_unlikely(const unsigned long ext)
144 {
145 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
146
147 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
148 return __riscv_has_extension_unlikely(STANDARD_EXT, ext);
149
150 return __riscv_isa_extension_available(NULL, ext);
151 }
152
riscv_has_extension_likely(const unsigned long ext)153 static __always_inline bool riscv_has_extension_likely(const unsigned long ext)
154 {
155 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
156
157 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
158 return __riscv_has_extension_likely(STANDARD_EXT, ext);
159
160 return __riscv_isa_extension_available(NULL, ext);
161 }
162
riscv_cpu_has_extension_likely(int cpu,const unsigned long ext)163 static __always_inline bool riscv_cpu_has_extension_likely(int cpu, const unsigned long ext)
164 {
165 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
166
167 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) &&
168 __riscv_has_extension_likely(STANDARD_EXT, ext))
169 return true;
170
171 return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);
172 }
173
riscv_cpu_has_extension_unlikely(int cpu,const unsigned long ext)174 static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsigned long ext)
175 {
176 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
177
178 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) &&
179 __riscv_has_extension_unlikely(STANDARD_EXT, ext))
180 return true;
181
182 return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);
183 }
184
185 #endif
186