1 /**************************************************************************
2 *
3 * Copyright 2008 Dennis Smit
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * on the rights to use, copy, modify, merge, publish, distribute, sub
10 * license, and/or sell copies of the Software, and to permit persons to whom
11 * the Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
20 * AUTHORS, COPYRIGHT HOLDERS, AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
21 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
22 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
23 * USE OR OTHER DEALINGS IN THE SOFTWARE.
24 *
25 **************************************************************************/
26
27 /**
28 * @file
29 * CPU feature detection.
30 *
31 * @author Dennis Smit
32 * @author Based on the work of Eric Anholt <anholt@FreeBSD.org>
33 */
34
35 #include "pipe/p_config.h"
36
37 #include "u_debug.h"
38 #include "u_cpu_detect.h"
39
40 #if defined(PIPE_ARCH_PPC)
41 #if defined(PIPE_OS_APPLE)
42 #include <sys/sysctl.h>
43 #else
44 #include <signal.h>
45 #include <setjmp.h>
46 #endif
47 #endif
48
49 #if defined(PIPE_OS_NETBSD) || defined(PIPE_OS_OPENBSD)
50 #include <sys/param.h>
51 #include <sys/sysctl.h>
52 #include <machine/cpu.h>
53 #endif
54
55 #if defined(PIPE_OS_FREEBSD) || defined(PIPE_OS_DRAGONFLY)
56 #include <sys/types.h>
57 #include <sys/sysctl.h>
58 #endif
59
60 #if defined(PIPE_OS_LINUX)
61 #include <signal.h>
62 #endif
63
64 #ifdef PIPE_OS_UNIX
65 #include <unistd.h>
66 #endif
67
68 #if defined(PIPE_OS_WINDOWS)
69 #include <windows.h>
70 #if defined(PIPE_CC_MSVC)
71 #include <intrin.h>
72 #endif
73 #endif
74
75
76 #ifdef DEBUG
77 DEBUG_GET_ONCE_BOOL_OPTION(dump_cpu, "GALLIUM_DUMP_CPU", FALSE)
78 #endif
79
80
81 struct util_cpu_caps util_cpu_caps;
82
83 #if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
84 static int has_cpuid(void);
85 #endif
86
87
88 #if defined(PIPE_ARCH_PPC) && !defined(PIPE_OS_APPLE)
89 static jmp_buf __lv_powerpc_jmpbuf;
90 static volatile sig_atomic_t __lv_powerpc_canjump = 0;
91
92 static void
sigill_handler(int sig)93 sigill_handler(int sig)
94 {
95 if (!__lv_powerpc_canjump) {
96 signal (sig, SIG_DFL);
97 raise (sig);
98 }
99
100 __lv_powerpc_canjump = 0;
101 longjmp(__lv_powerpc_jmpbuf, 1);
102 }
103 #endif
104
105 #if defined(PIPE_ARCH_PPC)
106 static void
check_os_altivec_support(void)107 check_os_altivec_support(void)
108 {
109 #if defined(PIPE_OS_APPLE)
110 int sels[2] = {CTL_HW, HW_VECTORUNIT};
111 int has_vu = 0;
112 int len = sizeof (has_vu);
113 int err;
114
115 err = sysctl(sels, 2, &has_vu, &len, NULL, 0);
116
117 if (err == 0) {
118 if (has_vu != 0) {
119 util_cpu_caps.has_altivec = 1;
120 }
121 }
122 #else /* !PIPE_OS_APPLE */
123 /* not on Apple/Darwin, do it the brute-force way */
124 /* this is borrowed from the libmpeg2 library */
125 signal(SIGILL, sigill_handler);
126 if (setjmp(__lv_powerpc_jmpbuf)) {
127 signal(SIGILL, SIG_DFL);
128 } else {
129 __lv_powerpc_canjump = 1;
130
131 __asm __volatile
132 ("mtspr 256, %0\n\t"
133 "vand %%v0, %%v0, %%v0"
134 :
135 : "r" (-1));
136
137 signal(SIGILL, SIG_DFL);
138 util_cpu_caps.has_altivec = 1;
139 }
140 #endif /* !PIPE_OS_APPLE */
141 }
142 #endif /* PIPE_ARCH_PPC */
143
144
145 #if defined(PIPE_ARCH_X86) || defined (PIPE_ARCH_X86_64)
has_cpuid(void)146 static int has_cpuid(void)
147 {
148 #if defined(PIPE_ARCH_X86)
149 #if defined(PIPE_OS_GCC)
150 int a, c;
151
152 __asm __volatile
153 ("pushf\n"
154 "popl %0\n"
155 "movl %0, %1\n"
156 "xorl $0x200000, %0\n"
157 "push %0\n"
158 "popf\n"
159 "pushf\n"
160 "popl %0\n"
161 : "=a" (a), "=c" (c)
162 :
163 : "cc");
164
165 return a != c;
166 #else
167 /* FIXME */
168 return 1;
169 #endif
170 #elif defined(PIPE_ARCH_X86_64)
171 return 1;
172 #else
173 return 0;
174 #endif
175 }
176
177
178 /**
179 * @sa cpuid.h included in gcc-4.3 onwards.
180 * @sa http://msdn.microsoft.com/en-us/library/hskdteyh.aspx
181 */
182 static inline void
cpuid(uint32_t ax,uint32_t * p)183 cpuid(uint32_t ax, uint32_t *p)
184 {
185 #if defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86)
186 __asm __volatile (
187 "xchgl %%ebx, %1\n\t"
188 "cpuid\n\t"
189 "xchgl %%ebx, %1"
190 : "=a" (p[0]),
191 "=S" (p[1]),
192 "=c" (p[2]),
193 "=d" (p[3])
194 : "0" (ax)
195 );
196 #elif defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64)
197 __asm __volatile (
198 "cpuid\n\t"
199 : "=a" (p[0]),
200 "=b" (p[1]),
201 "=c" (p[2]),
202 "=d" (p[3])
203 : "0" (ax)
204 );
205 #elif defined(PIPE_CC_MSVC)
206 __cpuid(p, ax);
207 #else
208 p[0] = 0;
209 p[1] = 0;
210 p[2] = 0;
211 p[3] = 0;
212 #endif
213 }
214
215 /**
216 * @sa cpuid.h included in gcc-4.4 onwards.
217 * @sa http://msdn.microsoft.com/en-us/library/hskdteyh%28v=vs.90%29.aspx
218 */
219 static inline void
cpuid_count(uint32_t ax,uint32_t cx,uint32_t * p)220 cpuid_count(uint32_t ax, uint32_t cx, uint32_t *p)
221 {
222 #if defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86)
223 __asm __volatile (
224 "xchgl %%ebx, %1\n\t"
225 "cpuid\n\t"
226 "xchgl %%ebx, %1"
227 : "=a" (p[0]),
228 "=S" (p[1]),
229 "=c" (p[2]),
230 "=d" (p[3])
231 : "0" (ax), "2" (cx)
232 );
233 #elif defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64)
234 __asm __volatile (
235 "cpuid\n\t"
236 : "=a" (p[0]),
237 "=b" (p[1]),
238 "=c" (p[2]),
239 "=d" (p[3])
240 : "0" (ax), "2" (cx)
241 );
242 #elif defined(PIPE_CC_MSVC)
243 __cpuidex(p, ax, cx);
244 #else
245 p[0] = 0;
246 p[1] = 0;
247 p[2] = 0;
248 p[3] = 0;
249 #endif
250 }
251
252
xgetbv(void)253 static inline uint64_t xgetbv(void)
254 {
255 #if defined(PIPE_CC_GCC)
256 uint32_t eax, edx;
257
258 __asm __volatile (
259 ".byte 0x0f, 0x01, 0xd0" // xgetbv isn't supported on gcc < 4.4
260 : "=a"(eax),
261 "=d"(edx)
262 : "c"(0)
263 );
264
265 return ((uint64_t)edx << 32) | eax;
266 #elif defined(PIPE_CC_MSVC) && defined(_MSC_FULL_VER) && defined(_XCR_XFEATURE_ENABLED_MASK)
267 return _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
268 #else
269 return 0;
270 #endif
271 }
272
273
274 #if defined(PIPE_ARCH_X86)
sse2_has_daz(void)275 PIPE_ALIGN_STACK static inline boolean sse2_has_daz(void)
276 {
277 struct {
278 uint32_t pad1[7];
279 uint32_t mxcsr_mask;
280 uint32_t pad2[128-8];
281 } PIPE_ALIGN_VAR(16) fxarea;
282
283 fxarea.mxcsr_mask = 0;
284 #if defined(PIPE_CC_GCC)
285 __asm __volatile ("fxsave %0" : "+m" (fxarea));
286 #elif defined(PIPE_CC_MSVC) || defined(PIPE_CC_ICL)
287 _fxsave(&fxarea);
288 #else
289 fxarea.mxcsr_mask = 0;
290 #endif
291 return !!(fxarea.mxcsr_mask & (1 << 6));
292 }
293 #endif
294
295 #endif /* X86 or X86_64 */
296
297 void
util_cpu_detect(void)298 util_cpu_detect(void)
299 {
300 static boolean util_cpu_detect_initialized = FALSE;
301
302 if(util_cpu_detect_initialized)
303 return;
304
305 memset(&util_cpu_caps, 0, sizeof util_cpu_caps);
306
307 /* Count the number of CPUs in system */
308 #if defined(PIPE_OS_WINDOWS)
309 {
310 SYSTEM_INFO system_info;
311 GetSystemInfo(&system_info);
312 util_cpu_caps.nr_cpus = system_info.dwNumberOfProcessors;
313 }
314 #elif defined(PIPE_OS_UNIX) && defined(_SC_NPROCESSORS_ONLN)
315 util_cpu_caps.nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
316 if (util_cpu_caps.nr_cpus == ~0u)
317 util_cpu_caps.nr_cpus = 1;
318 #elif defined(PIPE_OS_BSD)
319 {
320 int mib[2], ncpu;
321 int len;
322
323 mib[0] = CTL_HW;
324 mib[1] = HW_NCPU;
325
326 len = sizeof (ncpu);
327 sysctl(mib, 2, &ncpu, &len, NULL, 0);
328 util_cpu_caps.nr_cpus = ncpu;
329 }
330 #else
331 util_cpu_caps.nr_cpus = 1;
332 #endif
333
334 /* Make the fallback cacheline size nonzero so that it can be
335 * safely passed to align().
336 */
337 util_cpu_caps.cacheline = sizeof(void *);
338
339 #if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
340 if (has_cpuid()) {
341 uint32_t regs[4];
342 uint32_t regs2[4];
343
344 util_cpu_caps.cacheline = 32;
345
346 /* Get max cpuid level */
347 cpuid(0x00000000, regs);
348
349 if (regs[0] >= 0x00000001) {
350 unsigned int cacheline;
351
352 cpuid (0x00000001, regs2);
353
354 util_cpu_caps.x86_cpu_type = (regs2[0] >> 8) & 0xf;
355 if (util_cpu_caps.x86_cpu_type == 0xf)
356 util_cpu_caps.x86_cpu_type = 8 + ((regs2[0] >> 20) & 255); /* use extended family (P4, IA64) */
357
358 /* general feature flags */
359 util_cpu_caps.has_tsc = (regs2[3] >> 4) & 1; /* 0x0000010 */
360 util_cpu_caps.has_mmx = (regs2[3] >> 23) & 1; /* 0x0800000 */
361 util_cpu_caps.has_sse = (regs2[3] >> 25) & 1; /* 0x2000000 */
362 util_cpu_caps.has_sse2 = (regs2[3] >> 26) & 1; /* 0x4000000 */
363 util_cpu_caps.has_sse3 = (regs2[2] >> 0) & 1; /* 0x0000001 */
364 util_cpu_caps.has_ssse3 = (regs2[2] >> 9) & 1; /* 0x0000020 */
365 util_cpu_caps.has_sse4_1 = (regs2[2] >> 19) & 1;
366 util_cpu_caps.has_sse4_2 = (regs2[2] >> 20) & 1;
367 util_cpu_caps.has_popcnt = (regs2[2] >> 23) & 1;
368 util_cpu_caps.has_avx = ((regs2[2] >> 28) & 1) && // AVX
369 ((regs2[2] >> 27) & 1) && // OSXSAVE
370 ((xgetbv() & 6) == 6); // XMM & YMM
371 util_cpu_caps.has_f16c = ((regs2[2] >> 29) & 1) && util_cpu_caps.has_avx;
372 util_cpu_caps.has_fma = ((regs2[2] >> 12) & 1) && util_cpu_caps.has_avx;
373 util_cpu_caps.has_mmx2 = util_cpu_caps.has_sse; /* SSE cpus supports mmxext too */
374 #if defined(PIPE_ARCH_X86_64)
375 util_cpu_caps.has_daz = 1;
376 #else
377 util_cpu_caps.has_daz = util_cpu_caps.has_sse3 ||
378 (util_cpu_caps.has_sse2 && sse2_has_daz());
379 #endif
380
381 cacheline = ((regs2[1] >> 8) & 0xFF) * 8;
382 if (cacheline > 0)
383 util_cpu_caps.cacheline = cacheline;
384 }
385 if (util_cpu_caps.has_avx && regs[0] >= 0x00000007) {
386 uint32_t regs7[4];
387 cpuid_count(0x00000007, 0x00000000, regs7);
388 util_cpu_caps.has_avx2 = (regs7[1] >> 5) & 1;
389 }
390
391 // check for avx512
392 if (((regs2[2] >> 27) & 1) && // OSXSAVE
393 (xgetbv() & (0x7 << 5)) && // OPMASK: upper-256 enabled by OS
394 ((xgetbv() & 6) == 6)) { // XMM/YMM enabled by OS
395 uint32_t regs3[4];
396 cpuid(0x00000007, regs3);
397 util_cpu_caps.has_avx512f = (regs3[1] >> 16) & 1;
398 util_cpu_caps.has_avx512dq = (regs3[1] >> 17) & 1;
399 util_cpu_caps.has_avx512ifma = (regs3[1] >> 21) & 1;
400 util_cpu_caps.has_avx512pf = (regs3[1] >> 26) & 1;
401 util_cpu_caps.has_avx512er = (regs3[1] >> 27) & 1;
402 util_cpu_caps.has_avx512cd = (regs3[1] >> 28) & 1;
403 util_cpu_caps.has_avx512bw = (regs3[1] >> 30) & 1;
404 util_cpu_caps.has_avx512vl = (regs3[1] >> 31) & 1;
405 util_cpu_caps.has_avx512vbmi = (regs3[2] >> 1) & 1;
406 }
407
408 if (regs[1] == 0x756e6547 && regs[2] == 0x6c65746e && regs[3] == 0x49656e69) {
409 /* GenuineIntel */
410 util_cpu_caps.has_intel = 1;
411 }
412
413 cpuid(0x80000000, regs);
414
415 if (regs[0] >= 0x80000001) {
416
417 cpuid(0x80000001, regs2);
418
419 util_cpu_caps.has_mmx |= (regs2[3] >> 23) & 1;
420 util_cpu_caps.has_mmx2 |= (regs2[3] >> 22) & 1;
421 util_cpu_caps.has_3dnow = (regs2[3] >> 31) & 1;
422 util_cpu_caps.has_3dnow_ext = (regs2[3] >> 30) & 1;
423
424 util_cpu_caps.has_xop = util_cpu_caps.has_avx &&
425 ((regs2[2] >> 11) & 1);
426 }
427
428 if (regs[0] >= 0x80000006) {
429 /* should we really do this if the clflush size above worked? */
430 unsigned int cacheline;
431 cpuid(0x80000006, regs2);
432 cacheline = regs2[2] & 0xFF;
433 if (cacheline > 0)
434 util_cpu_caps.cacheline = cacheline;
435 }
436
437 if (!util_cpu_caps.has_sse) {
438 util_cpu_caps.has_sse2 = 0;
439 util_cpu_caps.has_sse3 = 0;
440 util_cpu_caps.has_ssse3 = 0;
441 util_cpu_caps.has_sse4_1 = 0;
442 }
443 }
444 #endif /* PIPE_ARCH_X86 || PIPE_ARCH_X86_64 */
445
446 #if defined(PIPE_ARCH_PPC)
447 check_os_altivec_support();
448 #endif /* PIPE_ARCH_PPC */
449
450 #ifdef DEBUG
451 if (debug_get_option_dump_cpu()) {
452 debug_printf("util_cpu_caps.nr_cpus = %u\n", util_cpu_caps.nr_cpus);
453
454 debug_printf("util_cpu_caps.x86_cpu_type = %u\n", util_cpu_caps.x86_cpu_type);
455 debug_printf("util_cpu_caps.cacheline = %u\n", util_cpu_caps.cacheline);
456
457 debug_printf("util_cpu_caps.has_tsc = %u\n", util_cpu_caps.has_tsc);
458 debug_printf("util_cpu_caps.has_mmx = %u\n", util_cpu_caps.has_mmx);
459 debug_printf("util_cpu_caps.has_mmx2 = %u\n", util_cpu_caps.has_mmx2);
460 debug_printf("util_cpu_caps.has_sse = %u\n", util_cpu_caps.has_sse);
461 debug_printf("util_cpu_caps.has_sse2 = %u\n", util_cpu_caps.has_sse2);
462 debug_printf("util_cpu_caps.has_sse3 = %u\n", util_cpu_caps.has_sse3);
463 debug_printf("util_cpu_caps.has_ssse3 = %u\n", util_cpu_caps.has_ssse3);
464 debug_printf("util_cpu_caps.has_sse4_1 = %u\n", util_cpu_caps.has_sse4_1);
465 debug_printf("util_cpu_caps.has_sse4_2 = %u\n", util_cpu_caps.has_sse4_2);
466 debug_printf("util_cpu_caps.has_avx = %u\n", util_cpu_caps.has_avx);
467 debug_printf("util_cpu_caps.has_avx2 = %u\n", util_cpu_caps.has_avx2);
468 debug_printf("util_cpu_caps.has_f16c = %u\n", util_cpu_caps.has_f16c);
469 debug_printf("util_cpu_caps.has_popcnt = %u\n", util_cpu_caps.has_popcnt);
470 debug_printf("util_cpu_caps.has_3dnow = %u\n", util_cpu_caps.has_3dnow);
471 debug_printf("util_cpu_caps.has_3dnow_ext = %u\n", util_cpu_caps.has_3dnow_ext);
472 debug_printf("util_cpu_caps.has_xop = %u\n", util_cpu_caps.has_xop);
473 debug_printf("util_cpu_caps.has_altivec = %u\n", util_cpu_caps.has_altivec);
474 debug_printf("util_cpu_caps.has_daz = %u\n", util_cpu_caps.has_daz);
475 debug_printf("util_cpu_caps.has_avx512f = %u\n", util_cpu_caps.has_avx512f);
476 debug_printf("util_cpu_caps.has_avx512dq = %u\n", util_cpu_caps.has_avx512dq);
477 debug_printf("util_cpu_caps.has_avx512ifma = %u\n", util_cpu_caps.has_avx512ifma);
478 debug_printf("util_cpu_caps.has_avx512pf = %u\n", util_cpu_caps.has_avx512pf);
479 debug_printf("util_cpu_caps.has_avx512er = %u\n", util_cpu_caps.has_avx512er);
480 debug_printf("util_cpu_caps.has_avx512cd = %u\n", util_cpu_caps.has_avx512cd);
481 debug_printf("util_cpu_caps.has_avx512bw = %u\n", util_cpu_caps.has_avx512bw);
482 debug_printf("util_cpu_caps.has_avx512vl = %u\n", util_cpu_caps.has_avx512vl);
483 debug_printf("util_cpu_caps.has_avx512vbmi = %u\n", util_cpu_caps.has_avx512vbmi);
484 }
485 #endif
486
487 util_cpu_detect_initialized = TRUE;
488 }
489