• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*===---- immintrin.h - Intel intrinsics -----------------------------------===
2  *
3  * Permission is hereby granted, free of charge, to any person obtaining a copy
4  * of this software and associated documentation files (the "Software"), to deal
5  * in the Software without restriction, including without limitation the rights
6  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7  * copies of the Software, and to permit persons to whom the Software is
8  * furnished to do so, subject to the following conditions:
9  *
10  * The above copyright notice and this permission notice shall be included in
11  * all copies or substantial portions of the Software.
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19  * THE SOFTWARE.
20  *
21  *===-----------------------------------------------------------------------===
22  */
23 
24 #ifndef __IMMINTRIN_H
25 #define __IMMINTRIN_H
26 
27 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__MMX__)
28 #include <mmintrin.h>
29 #endif
30 
31 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE__)
32 #include <xmmintrin.h>
33 #endif
34 
35 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE2__)
36 #include <emmintrin.h>
37 #endif
38 
39 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE3__)
40 #include <pmmintrin.h>
41 #endif
42 
43 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSSE3__)
44 #include <tmmintrin.h>
45 #endif
46 
47 #if !defined(_MSC_VER) || __has_feature(modules) || \
48     (defined(__SSE4_2__) || defined(__SSE4_1__))
49 #include <smmintrin.h>
50 #endif
51 
52 #if !defined(_MSC_VER) || __has_feature(modules) || \
53     (defined(__AES__) || defined(__PCLMUL__))
54 #include <wmmintrin.h>
55 #endif
56 
57 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLFLUSHOPT__)
58 #include <clflushoptintrin.h>
59 #endif
60 
61 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX__)
62 #include <avxintrin.h>
63 #endif
64 
65 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX2__)
66 #include <avx2intrin.h>
67 
68 /* The 256-bit versions of functions in f16cintrin.h.
69    Intel documents these as being in immintrin.h, and
70    they depend on typedefs from avxintrin.h. */
71 
72 #define _mm256_cvtps_ph(a, imm) __extension__ ({ \
73  (__m128i)__builtin_ia32_vcvtps2ph256((__v8sf)(__m256)(a), (imm)); })
74 
75 static __inline __m256 __attribute__((__always_inline__, __nodebug__, __target__("f16c")))
_mm256_cvtph_ps(__m128i __a)76 _mm256_cvtph_ps(__m128i __a)
77 {
78   return (__m256)__builtin_ia32_vcvtph2ps256((__v8hi)__a);
79 }
80 #endif /* __AVX2__ */
81 
82 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__BMI__)
83 #include <bmiintrin.h>
84 #endif
85 
86 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__BMI2__)
87 #include <bmi2intrin.h>
88 #endif
89 
90 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__LZCNT__)
91 #include <lzcntintrin.h>
92 #endif
93 
94 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FMA__)
95 #include <fmaintrin.h>
96 #endif
97 
98 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512F__)
99 #include <avx512fintrin.h>
100 #endif
101 
102 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VL__)
103 #include <avx512vlintrin.h>
104 #endif
105 
106 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512BW__)
107 #include <avx512bwintrin.h>
108 #endif
109 
110 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512CD__)
111 #include <avx512cdintrin.h>
112 #endif
113 
114 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512DQ__)
115 #include <avx512dqintrin.h>
116 #endif
117 
118 #if !defined(_MSC_VER) || __has_feature(modules) || \
119     (defined(__AVX512VL__) && defined(__AVX512BW__))
120 #include <avx512vlbwintrin.h>
121 #endif
122 
123 #if !defined(_MSC_VER) || __has_feature(modules) || \
124     (defined(__AVX512VL__) && defined(__AVX512CD__))
125 #include <avx512vlcdintrin.h>
126 #endif
127 
128 #if !defined(_MSC_VER) || __has_feature(modules) || \
129     (defined(__AVX512VL__) && defined(__AVX512DQ__))
130 #include <avx512vldqintrin.h>
131 #endif
132 
133 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512ER__)
134 #include <avx512erintrin.h>
135 #endif
136 
137 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512IFMA__)
138 #include <avx512ifmaintrin.h>
139 #endif
140 
141 #if !defined(_MSC_VER) || __has_feature(modules) || \
142     (defined(__AVX512IFMA__) && defined(__AVX512VL__))
143 #include <avx512ifmavlintrin.h>
144 #endif
145 
146 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VBMI__)
147 #include <avx512vbmiintrin.h>
148 #endif
149 
150 #if !defined(_MSC_VER) || __has_feature(modules) || \
151     (defined(__AVX512VBMI__) && defined(__AVX512VL__))
152 #include <avx512vbmivlintrin.h>
153 #endif
154 
155 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512PF__)
156 #include <avx512pfintrin.h>
157 #endif
158 
159 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__PKU__)
160 #include <pkuintrin.h>
161 #endif
162 
163 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RDRND__)
164 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
_rdrand16_step(unsigned short * __p)165 _rdrand16_step(unsigned short *__p)
166 {
167   return __builtin_ia32_rdrand16_step(__p);
168 }
169 
170 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
_rdrand32_step(unsigned int * __p)171 _rdrand32_step(unsigned int *__p)
172 {
173   return __builtin_ia32_rdrand32_step(__p);
174 }
175 
176 /* __bit_scan_forward */
177 static __inline__ int __attribute__((__always_inline__, __nodebug__))
_bit_scan_forward(int __A)178 _bit_scan_forward(int __A) {
179   return __builtin_ctz(__A);
180 }
181 
182 /* __bit_scan_reverse */
183 static __inline__ int __attribute__((__always_inline__, __nodebug__))
_bit_scan_reverse(int __A)184 _bit_scan_reverse(int __A) {
185   return 31 - __builtin_clz(__A);
186 }
187 
188 #ifdef __x86_64__
189 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
_rdrand64_step(unsigned long long * __p)190 _rdrand64_step(unsigned long long *__p)
191 {
192   return __builtin_ia32_rdrand64_step(__p);
193 }
194 #endif
195 #endif /* __RDRND__ */
196 
197 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FSGSBASE__)
198 #ifdef __x86_64__
199 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_readfsbase_u32(void)200 _readfsbase_u32(void)
201 {
202   return __builtin_ia32_rdfsbase32();
203 }
204 
205 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_readfsbase_u64(void)206 _readfsbase_u64(void)
207 {
208   return __builtin_ia32_rdfsbase64();
209 }
210 
211 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_readgsbase_u32(void)212 _readgsbase_u32(void)
213 {
214   return __builtin_ia32_rdgsbase32();
215 }
216 
217 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_readgsbase_u64(void)218 _readgsbase_u64(void)
219 {
220   return __builtin_ia32_rdgsbase64();
221 }
222 
223 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_writefsbase_u32(unsigned int __V)224 _writefsbase_u32(unsigned int __V)
225 {
226   return __builtin_ia32_wrfsbase32(__V);
227 }
228 
229 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_writefsbase_u64(unsigned long long __V)230 _writefsbase_u64(unsigned long long __V)
231 {
232   return __builtin_ia32_wrfsbase64(__V);
233 }
234 
235 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_writegsbase_u32(unsigned int __V)236 _writegsbase_u32(unsigned int __V)
237 {
238   return __builtin_ia32_wrgsbase32(__V);
239 }
240 
241 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
_writegsbase_u64(unsigned long long __V)242 _writegsbase_u64(unsigned long long __V)
243 {
244   return __builtin_ia32_wrgsbase64(__V);
245 }
246 
247 #endif
248 #endif /* __FSGSBASE__ */
249 
250 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RTM__)
251 #include <rtmintrin.h>
252 #include <xtestintrin.h>
253 #endif
254 
255 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SHA__)
256 #include <shaintrin.h>
257 #endif
258 
259 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FXSR__)
260 #include <fxsrintrin.h>
261 #endif
262 
263 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVE__)
264 #include <xsaveintrin.h>
265 #endif
266 
267 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVEOPT__)
268 #include <xsaveoptintrin.h>
269 #endif
270 
271 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVEC__)
272 #include <xsavecintrin.h>
273 #endif
274 
275 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVES__)
276 #include <xsavesintrin.h>
277 #endif
278 
279 /* Some intrinsics inside adxintrin.h are available only on processors with ADX,
280  * whereas others are also available at all times. */
281 #include <adxintrin.h>
282 
283 #endif /* __IMMINTRIN_H */
284