• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1/****************************************************************************
2* Copyright (C) 2017 Intel Corporation.   All Rights Reserved.
3*
4* Permission is hereby granted, free of charge, to any person obtaining a
5* copy of this software and associated documentation files (the "Software"),
6* to deal in the Software without restriction, including without limitation
7* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8* and/or sell copies of the Software, and to permit persons to whom the
9* Software is furnished to do so, subject to the following conditions:
10*
11* The above copyright notice and this permission notice (including the next
12* paragraph) shall be included in all copies or substantial portions of the
13* Software.
14*
15* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21* IN THE SOFTWARE.
22****************************************************************************/
23#if !defined(__SIMD_LIB_AVX512_HPP__)
24#error Do not include this file directly, use "simdlib.hpp" instead.
25#endif
26
27//============================================================================
28// SIMD16 AVX512 (F) implementation for Core processors
29//
30//============================================================================
31
32#define SIMD_WRAPPER_1_(op, intrin)  \
33    static SIMDINLINE Float SIMDCALL op(Float a)   \
34    {\
35        return intrin(a);\
36    }
37
38#define SIMD_WRAPPER_1(op)  \
39    SIMD_WRAPPER_1_(op, _mm512_##op)
40
41#define SIMD_WRAPPER_2_(op, intrin)  \
42    static SIMDINLINE Float SIMDCALL op(Float a, Float b)   \
43    {\
44        return _mm512_##intrin(a, b);\
45    }
46#define SIMD_WRAPPER_2(op) SIMD_WRAPPER_2_(op, op)
47
48#define SIMD_WRAPPERI_2_(op, intrin)  \
49    static SIMDINLINE Float SIMDCALL op(Float a, Float b)   \
50    {\
51        return _mm512_castsi512_ps(_mm512_##intrin(\
52            _mm512_castps_si512(a), _mm512_castps_si512(b)));\
53    }
54
55#define SIMD_DWRAPPER_2(op)  \
56    static SIMDINLINE Double SIMDCALL op(Double a, Double b)   \
57    {\
58        return _mm512_##op(a, b);\
59    }
60
61#define SIMD_WRAPPER_2I_(op, intrin)  \
62    template<int ImmT>\
63    static SIMDINLINE Float SIMDCALL op(Float a, Float b)   \
64    {\
65        return _mm512_##intrin(a, b, ImmT);\
66    }
67#define SIMD_WRAPPER_2I(op)  SIMD_WRAPPER_2I_(op, op)
68
69#define SIMD_DWRAPPER_2I_(op, intrin)  \
70    template<int ImmT>\
71    static SIMDINLINE Double SIMDCALL op(Double a, Double b)   \
72    {\
73        return _mm512_##intrin(a, b, ImmT);\
74    }
75#define SIMD_DWRAPPER_2I(op)  SIMD_DWRAPPER_2I_(op, op)
76
77#define SIMD_WRAPPER_3(op)  \
78    static SIMDINLINE Float SIMDCALL op(Float a, Float b, Float c)   \
79    {\
80        return _mm512_##op(a, b, c);\
81    }
82
83#define SIMD_IWRAPPER_1(op)  \
84    static SIMDINLINE Integer SIMDCALL op(Integer a)   \
85    {\
86        return _mm512_##op(a);\
87    }
88#define SIMD_IWRAPPER_1_8(op)  \
89    static SIMDINLINE Integer SIMDCALL op(SIMD256Impl::Integer a)   \
90    {\
91        return _mm512_##op(a);\
92    }
93
94#define SIMD_IWRAPPER_1_4(op)  \
95    static SIMDINLINE Integer SIMDCALL op(SIMD128Impl::Integer a)   \
96    {\
97        return _mm512_##op(a);\
98    }
99
100#define SIMD_IWRAPPER_1I_(op, intrin)  \
101    template<int ImmT> \
102    static SIMDINLINE Integer SIMDCALL op(Integer a)   \
103    {\
104        return intrin(a, ImmT);\
105    }
106#define SIMD_IWRAPPER_1I(op) SIMD_IWRAPPER_1I_(op, _mm512_##op)
107
108#define SIMD_IWRAPPER_2_(op, intrin)  \
109    static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b)   \
110    {\
111        return _mm512_##intrin(a, b);\
112    }
113#define SIMD_IWRAPPER_2(op)  SIMD_IWRAPPER_2_(op, op)
114
115#define SIMD_IWRAPPER_2_CMP(op, cmp)  \
116    static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b)   \
117    {\
118        return cmp(a, b);\
119    }
120
121#define SIMD_IFWRAPPER_2(op, intrin)  \
122    static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b)   \
123    {\
124        return castps_si(_mm512_##intrin(castsi_ps(a), castsi_ps(b)) );\
125    }
126
127#define SIMD_IWRAPPER_2I_(op, intrin)  \
128    template<int ImmT>\
129    static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b)   \
130    {\
131        return _mm512_##intrin(a, b, ImmT);\
132    }
133#define SIMD_IWRAPPER_2I(op) SIMD_IWRAPPER_2I_(op, op)
134
135private:
136    static SIMDINLINE Integer vmask(__mmask32 m)
137    {
138        return _mm512_maskz_set1_epi16(m, -1);
139    }
140    static SIMDINLINE Integer vmask(__mmask64 m)
141    {
142        return _mm512_maskz_set1_epi8(m, -1);
143    }
144public:
145
146SIMD_IWRAPPER_2(add_epi8);                  // return a + b (int8)
147SIMD_IWRAPPER_2(adds_epu8);                 // return ((a + b) > 0xff) ? 0xff : (a + b) (uint8)
148SIMD_IWRAPPER_2(subs_epu8);                 // return (b > a) ? 0 : (a - b) (uint8)
149
150SIMD_WRAPPER_2(and_ps);                     // return a & b       (float treated as int)
151SIMD_WRAPPER_2(andnot_ps);                  // return (~a) & b    (float treated as int)
152SIMD_WRAPPER_2(or_ps);                      // return a | b       (float treated as int)
153SIMD_WRAPPER_2(xor_ps);                     // return a ^ b       (float treated as int)
154
155SIMD_IWRAPPER_1_8(cvtepu8_epi16);           // return (int16)a    (uint8 --> int16)
156
157template<CompareTypeInt CmpTypeT>
158static SIMDINLINE Integer SIMDCALL cmp_epi8(Integer a, Integer b)
159{
160    // Legacy vector mask generator
161    __mmask64 result = _mm512_cmp_epi8_mask(a, b, static_cast<const int>(CmpTypeT));
162    return vmask(result);
163}
164template<CompareTypeInt CmpTypeT>
165static SIMDINLINE Integer SIMDCALL cmp_epi16(Integer a, Integer b)
166{
167    // Legacy vector mask generator
168    __mmask32 result = _mm512_cmp_epi16_mask(a, b, static_cast<const int>(CmpTypeT));
169    return vmask(result);
170}
171
172SIMD_IWRAPPER_2_CMP(cmpeq_epi8,  cmp_epi8<CompareTypeInt::EQ>);    // return a == b (int8)
173SIMD_IWRAPPER_2_CMP(cmpeq_epi16, cmp_epi16<CompareTypeInt::EQ>);   // return a == b (int16)
174SIMD_IWRAPPER_2_CMP(cmpgt_epi8,  cmp_epi8<CompareTypeInt::GT>);    // return a > b (int8)
175SIMD_IWRAPPER_2_CMP(cmpgt_epi16, cmp_epi16<CompareTypeInt::GT>);   // return a > b (int16)
176
177SIMD_IWRAPPER_2(packs_epi16);               // See documentation for _mm512_packs_epi16
178SIMD_IWRAPPER_2(packs_epi32);               // See documentation for _mm512_packs_epi32
179SIMD_IWRAPPER_2(packus_epi16);              // See documentation for _mm512_packus_epi16
180SIMD_IWRAPPER_2(packus_epi32);              // See documentation for _mm512_packus_epi32
181
182SIMD_IWRAPPER_2(unpackhi_epi8);             // See documentation for _mm512_unpackhi_epi8
183SIMD_IWRAPPER_2(unpacklo_epi16);            // See documentation for _mm512_unpacklo_epi16
184SIMD_IWRAPPER_2(unpacklo_epi8);             // See documentation for _mm512_unpacklo_epi8
185
186SIMD_IWRAPPER_2(shuffle_epi8);
187
188static SIMDINLINE uint64_t SIMDCALL movemask_epi8(Integer a)
189{
190    __mmask64 m = _mm512_cmplt_epi8_mask(a, setzero_si());
191    return static_cast<uint64_t>(m);
192}
193
194
195
196#undef SIMD_WRAPPER_1_
197#undef SIMD_WRAPPER_1
198#undef SIMD_WRAPPER_2
199#undef SIMD_WRAPPER_2_
200#undef SIMD_WRAPPERI_2_
201#undef SIMD_DWRAPPER_2
202#undef SIMD_DWRAPPER_2I
203#undef SIMD_WRAPPER_2I_
204#undef SIMD_WRAPPER_3_
205#undef SIMD_WRAPPER_2I
206#undef SIMD_WRAPPER_3
207#undef SIMD_IWRAPPER_1
208#undef SIMD_IWRAPPER_2
209#undef SIMD_IFWRAPPER_2
210#undef SIMD_IWRAPPER_2I
211#undef SIMD_IWRAPPER_1
212#undef SIMD_IWRAPPER_1I
213#undef SIMD_IWRAPPER_1I_
214#undef SIMD_IWRAPPER_2
215#undef SIMD_IWRAPPER_2_
216#undef SIMD_IWRAPPER_2I
217
218