1 // Copyright 2017 The Gemmlowp Authors. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // simd_wrappers_neon.h: NEON specialization of simd_wrappers.h 16 17 #ifndef GEMMLOWP_INTERNAL_SIMD_WRAPPERS_NEON_H_ 18 #define GEMMLOWP_INTERNAL_SIMD_WRAPPERS_NEON_H_ 19 20 #include <arm_neon.h> 21 22 namespace gemmlowp { 23 24 using Int32x4 = int32x4_t; 25 using Uint8x8 = uint8x8_t; 26 27 template <int ScalarCount> 28 struct RegisterType<std::int32_t, ScalarCount> { 29 using Type = 30 typename std::conditional<ScalarCount >= 4, Int32x4, std::int32_t>::type; 31 }; 32 33 template <int ScalarCount> 34 struct RegisterType<std::uint8_t, ScalarCount> { 35 using Type = typename std::conditional< 36 ScalarCount >= 8, Uint8x8, 37 typename std::conditional<ScalarCount >= 4, std::uint32_t, 38 std::uint8_t>::type>::type; 39 }; 40 41 inline Int32x4 LoadInt32x4(const std::int32_t* src) { return vld1q_s32(src); } 42 43 inline void StoreInt32x4(std::int32_t* dst, Int32x4 value) { 44 vst1q_s32(dst, value); 45 } 46 47 template <int Lane> 48 std::int32_t GetLane(Int32x4 value) { 49 return vgetq_lane_s32(value, Lane); 50 } 51 52 template <int Lane> 53 Int32x4 DupLane(Int32x4 value) { 54 switch (Lane) { 55 case 0: 56 return vdupq_lane_s32(vget_low_s32(value), 0); 57 case 1: 58 return vdupq_lane_s32(vget_low_s32(value), 1); 59 case 2: 60 return vdupq_lane_s32(vget_high_s32(value), 0); 61 case 3: 62 return vdupq_lane_s32(vget_high_s32(value), 1); 63 default: 64 static_assert(Lane >= 0 && Lane <= 3, ""); 65 return vdupq_n_s32(0); 66 } 67 } 68 69 inline Int32x4 Mul(Int32x4 a, std::int32_t b) { return vmulq_n_s32(a, b); } 70 71 inline Int32x4 Min(Int32x4 a, Int32x4 b) { return vminq_s32(a, b); } 72 73 inline Int32x4 Max(Int32x4 a, Int32x4 b) { return vmaxq_s32(a, b); } 74 75 inline Int32x4 SaturatingRoundingDoublingHighMul(Int32x4 a, std::int32_t b) { 76 return vqrdmulhq_n_s32(a, b); 77 } 78 79 template <int Lane> 80 Int32x4 MulByRhsLane(Int32x4 a, Int32x4 b) { 81 switch (Lane) { 82 case 0: 83 return vmulq_lane_s32(a, vget_low_s32(b), 0); 84 case 1: 85 return vmulq_lane_s32(a, vget_low_s32(b), 1); 86 case 2: 87 return vmulq_lane_s32(a, vget_high_s32(b), 0); 88 case 3: 89 return vmulq_lane_s32(a, vget_high_s32(b), 1); 90 default: 91 static_assert(Lane >= 0 && Lane <= 3, ""); 92 return vdupq_n_s32(0); 93 } 94 } 95 96 inline void MulAdd(Int32x4 lhs, Int32x4 rhs, Int32x4* acc) { 97 *acc = vmlaq_s32(*acc, lhs, rhs); 98 } 99 100 inline void MulAdd(Int32x4 lhs, std::int32_t rhs, Int32x4* acc) { 101 *acc = vmlaq_n_s32(*acc, lhs, rhs); 102 } 103 104 template <int Lane> 105 inline void MulAddByRhsLane(Int32x4 lhs, Int32x4 rhs, Int32x4* acc) { 106 switch (Lane) { 107 case 0: 108 *acc = vmlaq_lane_s32(*acc, lhs, vget_low_s32(rhs), 0); 109 break; 110 case 1: 111 *acc = vmlaq_lane_s32(*acc, lhs, vget_low_s32(rhs), 1); 112 break; 113 case 2: 114 *acc = vmlaq_lane_s32(*acc, lhs, vget_high_s32(rhs), 0); 115 break; 116 case 3: 117 *acc = vmlaq_lane_s32(*acc, lhs, vget_high_s32(rhs), 1); 118 break; 119 default: 120 static_assert(Lane >= 0 && Lane <= 3, ""); 121 } 122 } 123 124 template <> 125 struct LoadContiguousImpl<RegBlockUint8<8, 8>> { 126 static RegBlockUint8<8, 8> Run(const std::uint8_t* src) { 127 RegBlockUint8<8, 8> result; 128 for (int i = 0; i < 8; i++) { 129 result.buf.reg[i] = vld1_u8(src + 8 * i); 130 } 131 return result; 132 } 133 }; 134 135 template <> 136 struct LoadContiguousImpl<RegBlockInt32<8, 8>> { 137 static RegBlockInt32<8, 8> Run(const std::int32_t* src) { 138 RegBlockInt32<8, 8> result; 139 for (int i = 0; i < 16; i++) { 140 result.buf.reg[i] = vld1q_s32(src + 4 * i); 141 } 142 return result; 143 } 144 }; 145 146 } // end namespace gemmlowp 147 148 #include "simd_wrappers_common_neon_sse.h" 149 150 #endif // GEMMLOWP_INTERNAL_SIMD_WRAPPERS_NEON_H_ 151