1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* -*- linux-c -*- ------------------------------------------------------- *
3 *
4 * Copyright 2002 H. Peter Anvin - All Rights Reserved
5 *
6 * ----------------------------------------------------------------------- */
7
8 /*
9 * raid6/algos.c
10 *
11 * Algorithm list and algorithm selection for RAID-6
12 */
13
14 #include <linux/raid/pq.h>
15 #ifndef __KERNEL__
16 #include <sys/mman.h>
17 #include <stdio.h>
18 #else
19 #include <linux/module.h>
20 #include <linux/gfp.h>
21 #if !RAID6_USE_EMPTY_ZERO_PAGE
22 /* In .bss so it's zeroed */
23 const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(256)));
24 EXPORT_SYMBOL(raid6_empty_zero_page);
25 #endif
26 #endif
27
28 struct raid6_calls raid6_call;
29 EXPORT_SYMBOL_GPL(raid6_call);
30
31 const struct raid6_calls * const raid6_algos[] = {
32 #if defined(__i386__) && !defined(__arch_um__)
33 #ifdef CONFIG_AS_AVX512
34 &raid6_avx512x2,
35 &raid6_avx512x1,
36 #endif
37 &raid6_avx2x2,
38 &raid6_avx2x1,
39 &raid6_sse2x2,
40 &raid6_sse2x1,
41 &raid6_sse1x2,
42 &raid6_sse1x1,
43 &raid6_mmxx2,
44 &raid6_mmxx1,
45 #endif
46 #if defined(__x86_64__) && !defined(__arch_um__)
47 #ifdef CONFIG_AS_AVX512
48 &raid6_avx512x4,
49 &raid6_avx512x2,
50 &raid6_avx512x1,
51 #endif
52 &raid6_avx2x4,
53 &raid6_avx2x2,
54 &raid6_avx2x1,
55 &raid6_sse2x4,
56 &raid6_sse2x2,
57 &raid6_sse2x1,
58 #endif
59 #ifdef CONFIG_ALTIVEC
60 &raid6_vpermxor8,
61 &raid6_vpermxor4,
62 &raid6_vpermxor2,
63 &raid6_vpermxor1,
64 &raid6_altivec8,
65 &raid6_altivec4,
66 &raid6_altivec2,
67 &raid6_altivec1,
68 #endif
69 #if defined(CONFIG_S390)
70 &raid6_s390vx8,
71 #endif
72 #ifdef CONFIG_KERNEL_MODE_NEON
73 &raid6_neonx8,
74 &raid6_neonx4,
75 &raid6_neonx2,
76 &raid6_neonx1,
77 #endif
78 #ifdef CONFIG_LOONGARCH
79 #ifdef CONFIG_CPU_HAS_LASX
80 &raid6_lasx,
81 #endif
82 #ifdef CONFIG_CPU_HAS_LSX
83 &raid6_lsx,
84 #endif
85 #endif
86 #if defined(__ia64__)
87 &raid6_intx32,
88 &raid6_intx16,
89 #endif
90 &raid6_intx8,
91 &raid6_intx4,
92 &raid6_intx2,
93 &raid6_intx1,
94 NULL
95 };
96
97 void (*raid6_2data_recov)(int, size_t, int, int, void **);
98 EXPORT_SYMBOL_GPL(raid6_2data_recov);
99
100 void (*raid6_datap_recov)(int, size_t, int, void **);
101 EXPORT_SYMBOL_GPL(raid6_datap_recov);
102
103 const struct raid6_recov_calls *const raid6_recov_algos[] = {
104 #ifdef CONFIG_X86
105 #ifdef CONFIG_AS_AVX512
106 &raid6_recov_avx512,
107 #endif
108 &raid6_recov_avx2,
109 &raid6_recov_ssse3,
110 #endif
111 #ifdef CONFIG_S390
112 &raid6_recov_s390xc,
113 #endif
114 #if defined(CONFIG_KERNEL_MODE_NEON)
115 &raid6_recov_neon,
116 #endif
117 #ifdef CONFIG_LOONGARCH
118 #ifdef CONFIG_CPU_HAS_LASX
119 &raid6_recov_lasx,
120 #endif
121 #ifdef CONFIG_CPU_HAS_LSX
122 &raid6_recov_lsx,
123 #endif
124 #endif
125 &raid6_recov_intx1,
126 NULL
127 };
128
129 #ifdef __KERNEL__
130 #define RAID6_TIME_JIFFIES_LG2 4
131 #else
132 /* Need more time to be stable in userspace */
133 #define RAID6_TIME_JIFFIES_LG2 9
134 #define time_before(x, y) ((x) < (y))
135 #endif
136
137 #define RAID6_TEST_DISKS 8
138 #define RAID6_TEST_DISKS_ORDER 3
139
raid6_choose_recov(void)140 static inline const struct raid6_recov_calls *raid6_choose_recov(void)
141 {
142 const struct raid6_recov_calls *const *algo;
143 const struct raid6_recov_calls *best;
144
145 for (best = NULL, algo = raid6_recov_algos; *algo; algo++)
146 if (!best || (*algo)->priority > best->priority)
147 if (!(*algo)->valid || (*algo)->valid())
148 best = *algo;
149
150 if (best) {
151 raid6_2data_recov = best->data2;
152 raid6_datap_recov = best->datap;
153
154 pr_info("raid6: using %s recovery algorithm\n", best->name);
155 } else
156 pr_err("raid6: Yikes! No recovery algorithm found!\n");
157
158 return best;
159 }
160
raid6_choose_gen(void * (* const dptrs)[RAID6_TEST_DISKS],const int disks)161 static inline const struct raid6_calls *raid6_choose_gen(
162 void *(*const dptrs)[RAID6_TEST_DISKS], const int disks)
163 {
164 unsigned long perf, bestgenperf, bestxorperf, j0, j1;
165 int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */
166 const struct raid6_calls *const *algo;
167 const struct raid6_calls *best;
168
169 for (bestgenperf = 0, bestxorperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) {
170 if (!best || (*algo)->prefer >= best->prefer) {
171 if ((*algo)->valid && !(*algo)->valid())
172 continue;
173
174 if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
175 best = *algo;
176 break;
177 }
178
179 perf = 0;
180
181 preempt_disable();
182 j0 = jiffies;
183 while ((j1 = jiffies) == j0)
184 cpu_relax();
185 while (time_before(jiffies,
186 j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
187 (*algo)->gen_syndrome(disks, PAGE_SIZE, *dptrs);
188 perf++;
189 }
190 preempt_enable();
191
192 if (perf > bestgenperf) {
193 bestgenperf = perf;
194 best = *algo;
195 }
196 pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name,
197 (perf * HZ * (disks-2)) >>
198 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
199
200 if (!(*algo)->xor_syndrome)
201 continue;
202
203 perf = 0;
204
205 preempt_disable();
206 j0 = jiffies;
207 while ((j1 = jiffies) == j0)
208 cpu_relax();
209 while (time_before(jiffies,
210 j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
211 (*algo)->xor_syndrome(disks, start, stop,
212 PAGE_SIZE, *dptrs);
213 perf++;
214 }
215 preempt_enable();
216
217 if (best == *algo)
218 bestxorperf = perf;
219
220 pr_info("raid6: %-8s xor() %5ld MB/s\n", (*algo)->name,
221 (perf * HZ * (disks-2)) >>
222 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
223 }
224 }
225
226 if (best) {
227 if (IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
228 pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
229 best->name,
230 (bestgenperf * HZ * (disks-2)) >>
231 (20 - PAGE_SHIFT+RAID6_TIME_JIFFIES_LG2));
232 if (best->xor_syndrome)
233 pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n",
234 (bestxorperf * HZ * (disks-2)) >>
235 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
236 } else
237 pr_info("raid6: skip pq benchmark and using algorithm %s\n",
238 best->name);
239 raid6_call = *best;
240 } else
241 pr_err("raid6: Yikes! No algorithm found!\n");
242
243 return best;
244 }
245
246
247 /* Try to pick the best algorithm */
248 /* This code uses the gfmul table as convenient data set to abuse */
249
raid6_select_algo(void)250 int __init raid6_select_algo(void)
251 {
252 const int disks = RAID6_TEST_DISKS;
253
254 const struct raid6_calls *gen_best;
255 const struct raid6_recov_calls *rec_best;
256 char *disk_ptr, *p;
257 void *dptrs[RAID6_TEST_DISKS];
258 int i, cycle;
259
260 /* prepare the buffer and fill it circularly with gfmul table */
261 disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER);
262 if (!disk_ptr) {
263 pr_err("raid6: Yikes! No memory available.\n");
264 return -ENOMEM;
265 }
266
267 p = disk_ptr;
268 for (i = 0; i < disks; i++)
269 dptrs[i] = p + PAGE_SIZE * i;
270
271 cycle = ((disks - 2) * PAGE_SIZE) / 65536;
272 for (i = 0; i < cycle; i++) {
273 memcpy(p, raid6_gfmul, 65536);
274 p += 65536;
275 }
276
277 if ((disks - 2) * PAGE_SIZE % 65536)
278 memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536);
279
280 /* select raid gen_syndrome function */
281 gen_best = raid6_choose_gen(&dptrs, disks);
282
283 /* select raid recover functions */
284 rec_best = raid6_choose_recov();
285
286 free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER);
287
288 return gen_best && rec_best ? 0 : -EINVAL;
289 }
290
raid6_exit(void)291 static void raid6_exit(void)
292 {
293 do { } while (0);
294 }
295
296 subsys_initcall(raid6_select_algo);
297 module_exit(raid6_exit);
298 MODULE_LICENSE("GPL");
299 MODULE_DESCRIPTION("RAID6 Q-syndrome calculations");
300