• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1/****************************************************************************
2* Copyright (C) 2017 Intel Corporation.   All Rights Reserved.
3*
4* Permission is hereby granted, free of charge, to any person obtaining a
5* copy of this software and associated documentation files (the "Software"),
6* to deal in the Software without restriction, including without limitation
7* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8* and/or sell copies of the Software, and to permit persons to whom the
9* Software is furnished to do so, subject to the following conditions:
10*
11* The above copyright notice and this permission notice (including the next
12* paragraph) shall be included in all copies or substantial portions of the
13* Software.
14*
15* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21* IN THE SOFTWARE.
22****************************************************************************/
23#if !defined(__SIMD_LIB_AVX512_HPP__)
24#error Do not include this file directly, use "simdlib.hpp" instead.
25#endif
26
27//============================================================================
28// SIMD256 AVX (512) implementation for Core processors
29//
30// Since this implementation inherits from the AVX (2) implementation,
31// the only operations below ones that replace AVX (2) operations.
32// These use native AVX512 instructions with masking to enable a larger
33// register set.
34//============================================================================
35
36#define SIMD_DWRAPPER_1_(op, intrin, mask)  \
37    static SIMDINLINE Double SIMDCALL op(Double a)   \
38    {\
39        return __conv(_mm512_maskz_##intrin((mask), __conv(a)));\
40    }
41#define SIMD_DWRAPPER_1(op)  SIMD_DWRAPPER_1_(op, op, __mmask8(0xf))
42
43#define SIMD_DWRAPPER_1I_(op, intrin, mask)  \
44    template<int ImmT> \
45    static SIMDINLINE Double SIMDCALL op(Double a)   \
46    {\
47        return __conv(_mm512_maskz_##intrin((mask), __conv(a), ImmT));\
48    }
49#define SIMD_DWRAPPER_1I(op)  SIMD_DWRAPPER_1I_(op, op, __mmask8(0xf))
50
51#define SIMD_DWRAPPER_2_(op, intrin, mask)  \
52    static SIMDINLINE Double SIMDCALL op(Double a, Double b)   \
53    {\
54        return __conv(_mm512_maskz_##intrin((mask), __conv(a), __conv(b)));\
55    }
56#define SIMD_DWRAPPER_2(op)  SIMD_DWRAPPER_2_(op, op, __mmask8(0xf))
57
58#define SIMD_IWRAPPER_1_(op, intrin, mask)  \
59    static SIMDINLINE Integer SIMDCALL op(Integer a)   \
60    {\
61        return __conv(_mm512_maskz_##intrin((mask), __conv(a)));\
62    }
63#define SIMD_IWRAPPER_1_8(op)   SIMD_IWRAPPER_1_(op, op, __mmask64(0xffffffffull))
64#define SIMD_IWRAPPER_1_16(op)  SIMD_IWRAPPER_1_(op, op, __mmask32(0xffff))
65#define SIMD_IWRAPPER_1_64(op)  SIMD_IWRAPPER_1_(op, op, __mmask8(0xf))
66
67#define SIMD_IWRAPPER_1I_(op, intrin, mask)  \
68    template<int ImmT> \
69    static SIMDINLINE Integer SIMDCALL op(Integer a)   \
70    {\
71        return __conv(_mm512_maskz_##intrin((mask), __conv(a), ImmT));\
72    }
73#define SIMD_IWRAPPER_1I_8(op)   SIMD_IWRAPPER_1I_(op, op, __mmask64(0xffffffffull))
74#define SIMD_IWRAPPER_1I_16(op)  SIMD_IWRAPPER_1I_(op, op, __mmask32(0xffff))
75#define SIMD_IWRAPPER_1I_64(op)  SIMD_IWRAPPER_1I_(op, op, __mmask8(0xf))
76
77#define SIMD_IWRAPPER_2_(op, intrin, mask)  \
78    static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b)   \
79    {\
80        return __conv(_mm512_maskz_##intrin((mask), __conv(a), __conv(b)));\
81    }
82#define SIMD_IWRAPPER_2_8(op)   SIMD_IWRAPPER_2_(op, op, __mmask64(0xffffffffull))
83#define SIMD_IWRAPPER_2_16(op)  SIMD_IWRAPPER_2_(op, op, __mmask32(0xffff))
84#define SIMD_IWRAPPER_2_64(op)  SIMD_IWRAPPER_2_(op, op, __mmask8(0xf))
85
86
87SIMD_IWRAPPER_2_8(add_epi8);    // return a + b (int8)
88SIMD_IWRAPPER_2_8(adds_epu8);   // return ((a + b) > 0xff) ? 0xff : (a + b) (uint8)
89SIMD_IWRAPPER_2_64(sub_epi64);  // return a - b (int64)
90SIMD_IWRAPPER_2_8(subs_epu8);   // return (b > a) ? 0 : (a - b) (uint8)
91SIMD_IWRAPPER_2_8(packs_epi16);     // int16 --> int8    See documentation for _mm256_packs_epi16 and _mm512_packs_epi16
92SIMD_IWRAPPER_2_16(packs_epi32);    // int32 --> int16   See documentation for _mm256_packs_epi32 and _mm512_packs_epi32
93SIMD_IWRAPPER_2_8(packus_epi16);    // uint16 --> uint8  See documentation for _mm256_packus_epi16 and _mm512_packus_epi16
94SIMD_IWRAPPER_2_16(packus_epi32);   // uint32 --> uint16 See documentation for _mm256_packus_epi32 and _mm512_packus_epi32
95SIMD_IWRAPPER_2_16(unpackhi_epi16);
96SIMD_IWRAPPER_2_64(unpackhi_epi64);
97SIMD_IWRAPPER_2_8(unpackhi_epi8);
98SIMD_IWRAPPER_2_16(unpacklo_epi16);
99SIMD_IWRAPPER_2_64(unpacklo_epi64);
100SIMD_IWRAPPER_2_8(unpacklo_epi8);
101
102static SIMDINLINE uint32_t SIMDCALL movemask_epi8(Integer a)
103{
104    __mmask64 m = 0xffffffffull;
105    return static_cast<uint32_t>(
106        _mm512_mask_test_epi8_mask(m, __conv(a), _mm512_set1_epi8(0x80)));
107}
108
109#undef SIMD_DWRAPPER_1_
110#undef SIMD_DWRAPPER_1
111#undef SIMD_DWRAPPER_1I_
112#undef SIMD_DWRAPPER_1I
113#undef SIMD_DWRAPPER_2_
114#undef SIMD_DWRAPPER_2
115#undef SIMD_DWRAPPER_2I
116#undef SIMD_IWRAPPER_1_
117#undef SIMD_IWRAPPER_1_8
118#undef SIMD_IWRAPPER_1_16
119#undef SIMD_IWRAPPER_1_64
120#undef SIMD_IWRAPPER_1I_
121#undef SIMD_IWRAPPER_1I_8
122#undef SIMD_IWRAPPER_1I_16
123#undef SIMD_IWRAPPER_1I_64
124#undef SIMD_IWRAPPER_2_
125#undef SIMD_IWRAPPER_2_8
126#undef SIMD_IWRAPPER_2_16
127#undef SIMD_IWRAPPER_2_64
128