• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2017 The Gemmlowp Authors. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //     http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 // simd_wrappers_neon.h: SSE SIMD wrappers
16 
17 #ifndef GEMMLOWP_INTERNAL_SIMD_WRAPPERS_SSE_H_
18 #define GEMMLOWP_INTERNAL_SIMD_WRAPPERS_SSE_H_
19 
20 #include <smmintrin.h>
21 
22 namespace gemmlowp {
23 
24 using Int32x4 = __m128i;
25 using Int16x8 = __m128i;
26 using Uint8x16 = __m128i;
27 
28 template <int ScalarCount>
29 struct RegisterType<std::int32_t, ScalarCount> {
30   using Type =
31       typename std::conditional<ScalarCount >= 4, Int32x4, std::int32_t>::type;
32 };
33 
34 template <int ScalarCount>
35 struct RegisterType<std::int16_t, ScalarCount> {
36   using Type =
37       typename std::conditional<ScalarCount >= 8, Int16x8, std::int16_t>::type;
38 };
39 
40 template <int ScalarCount>
41 struct RegisterType<std::uint8_t, ScalarCount> {
42   using Type = typename std::conditional<
43       ScalarCount >= 16, Uint8x16,
44       typename std::conditional<ScalarCount >= 4, std::uint32_t,
45                                 std::uint8_t>::type>::type;
46 };
47 
48 inline Int32x4 LoadInt32x4(const std::int32_t* src) {
49   return _mm_loadu_si128(reinterpret_cast<const Int32x4*>(src));
50 }
51 
52 inline Int32x4 LoadInt16x8(const std::int16_t* src) {
53   return _mm_loadu_si128(reinterpret_cast<const Int16x8*>(src));
54 }
55 
56 inline void StoreInt32x4(std::int32_t* dst, Int32x4 value) {
57   _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), value);
58 }
59 
60 inline void StoreInt16x8(std::int16_t* dst, Int16x8 value) {
61   _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), value);
62 }
63 
64 inline Uint8x16 LoadUint8x16(const std::uint8_t* src) {
65   return _mm_loadu_si128(reinterpret_cast<const Uint8x16*>(src));
66 }
67 
68 inline void StoreUint8x16(std::uint8_t* dst, Uint8x16 value) {
69   _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), value);
70 }
71 
72 template <int Lane>
73 std::int32_t GetLane(Int32x4 value) {
74   return _mm_extract_epi32(value, Lane);
75 }
76 
77 template <int Lane>
78 Int32x4 DupLane(Int32x4 value) {
79   return _mm_shuffle_epi32(value, _MM_SHUFFLE(Lane, Lane, Lane, Lane));
80 }
81 
82 inline Int32x4 Mul(Int32x4 a, std::int32_t b) {
83   return Mul(a, Dup<Int32x4>(b));
84 }
85 
86 inline Int32x4 Min(Int32x4 a, Int32x4 b) { return _mm_min_epi32(a, b); }
87 
88 inline Int32x4 Max(Int32x4 a, Int32x4 b) { return _mm_max_epi32(a, b); }
89 
90 inline Int32x4 SaturatingRoundingDoublingHighMul(Int32x4 a, std::int32_t b) {
91   return SaturatingRoundingDoublingHighMul(a, Dup<Int32x4>(b));
92 }
93 
94 template <int Lane>
95 Int32x4 MulByRhsLane(Int32x4 a, Int32x4 b) {
96   return Mul(a, DupLane<Lane>(b));
97 }
98 
99 inline void MulAdd(Int32x4 lhs, Int32x4 rhs, Int32x4* acc) {
100   *acc = Add(*acc, Mul(lhs, rhs));
101 }
102 
103 inline void MulAdd(Int32x4 lhs, std::int32_t rhs, Int32x4* acc) {
104   *acc = Add(*acc, Mul(lhs, rhs));
105 }
106 
107 template <int Lane>
108 inline void MulAddByRhsLane(Int32x4 lhs, Int32x4 rhs, Int32x4* acc) {
109   *acc = Add(*acc, MulByRhsLane<Lane>(lhs, rhs));
110 }
111 
112 template <>
113 struct LoadContiguousImpl<RegBlockUint8<8, 8>> {
114   static RegBlockUint8<8, 8> Run(const std::uint8_t* src) {
115     RegBlockUint8<8, 8> result;
116     for (int i = 0; i < 4; i++) {
117       result.buf.reg[i] = LoadUint8x16(src + 16 * i);
118     }
119     return result;
120   }
121 };
122 
123 template <>
124 struct LoadContiguousImpl<RegBlockInt32<8, 8>> {
125   static RegBlockInt32<8, 8> Run(const std::int32_t* src) {
126     RegBlockInt32<8, 8> result;
127     for (int i = 0; i < 16; i++) {
128       result.buf.reg[i] = LoadInt32x4(src + 4 * i);
129     }
130     return result;
131   }
132 };
133 
134 template <>
135 struct LoadContiguousImpl<RegBlockInt16<8, 8>> {
136   static RegBlockInt16<8, 8> Run(const std::int16_t* src) {
137     RegBlockInt16<8, 8> result;
138     for (int i = 0; i < 8; i++) {
139       result.buf.reg[i] = LoadInt16x8(src + 8 * i);
140     }
141     return result;
142   }
143 };
144 
145 }  // end namespace gemmlowp
146 
147 #include "simd_wrappers_common_neon_sse.h"
148 
149 #endif  // GEMMLOWP_INTERNAL_SIMD_WRAPPERS_SSE_H_
150