1 /*
2 * Copyright (c) 2017-2020 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #ifndef ARM_COMPUTE_NEASYMM_H
25 #define ARM_COMPUTE_NEASYMM_H
26
27 #include "src/core/NEON/NEMath.h"
28 #include "src/core/NEON/wrapper/intrinsics/intrinsics.h"
29 #include <arm_neon.h>
30
31 namespace arm_compute
32 {
33 using qasymm8x8_t = uint8x8_t; /**< 8 bit quantized asymmetric vector with 8 elements */
34 using qasymm8x8x2_t = uint8x8x2_t; /**< 8 bit quantized asymmetric vector with 16 elements */
35 using qasymm8x8x3_t = uint8x8x3_t; /**< 8 bit quantized asymmetric vector with 24 elements */
36 using qasymm8x8x4_t = uint8x8x4_t; /**< 8 bit quantized asymmetric vector with 32 elements */
37 using qasymm8x16_t = uint8x16_t; /**< 8 bit quantized asymmetric vector with 16 elements */
38
39 using qasymm8x8_signed_t = int8x8_t; /**< 8 bit quantized signed asymmetric vector with 8 elements */
40 using qasymm8x8x2_signed_t = int8x8x2_t; /**< 8 bit quantized signed asymmetric vector with 16 elements */
41 using qasymm8x8x3_signed_t = int8x8x3_t; /**< 8 bit quantized signed asymmetric vector with 24 elements */
42 using qasymm8x8x4_signed_t = int8x8x4_t; /**< 8 bit quantized signed asymmetric vector with 32 elements */
43 using qasymm8x16_signed_t = int8x16_t; /**< 8 bit quantized signed asymmetric vector with 16 elements */
44
45 /** Perform a multiply-accumulate on all 16 components of a QASYMM8 vector
46 *
47 * vd*vs + vo
48 *
49 * @param[in] vd Input vector value in QASYMM8 format
50 * @param[in] vs Vector multiplier in F32 format. The multiplier value must be duplicated across all four lanes.
51 * @param[in] vo Vector addend in F32 format. The addend value must be duplicated across all four lanes.
52 *
53 * @return A 16-component vector in QASYMM8 format, saturated to fit
54 */
55 uint8x16_t vmlaq_qasymm8(qasymm8x16_t vd, float32x4_t vs, float32x4_t vo);
56
57 /** Perform a multiply-accumulate on all 16 components of a QASYMM8_SIGNED vector
58 *
59 * vd*vs + vo
60 *
61 * @param[in] vd Input vector value in QASYMM8_SIGNED format
62 * @param[in] vs Vector multiplier in F32 format. The multiplier value must be duplicated across all four lanes.
63 * @param[in] vo Vector addend in F32 format. The addend value must be duplicated across all four lanes.
64 *
65 * @return A 16-component vector in QASYMM8_SIGNED format, saturated to fit
66 */
67 int8x16_t vmlaq_qasymm8_signed(qasymm8x16_signed_t vd, float32x4_t vs, float32x4_t vo);
68
69 /** Performs final quantization step on 16 elements
70 *
71 * @param[in] in_s32 Input to be quantized.
72 * @param[in] result_fixedpoint_multiplier Result multiplier parameter
73 * @param[in] result_shift Result shift parameter
74 * @param[in] result_offset_after_shift_s32 Result offset parameter
75 * @param[in] min_u8 Relu lower bound
76 * @param[in] max_u8 Relu upper bound
77 * @param[in] is_bounded_relu Specified if a fused bounded relu should be applied
78 *
79 * @return Quantized values
80 */
finalize_quantization(int32x4x4_t & in_s32,int result_fixedpoint_multiplier,int32_t result_shift,int32x4_t result_offset_after_shift_s32,uint8x16_t min_u8,uint8x16_t max_u8,bool is_bounded_relu)81 inline uint8x16_t finalize_quantization(int32x4x4_t &in_s32,
82 int result_fixedpoint_multiplier,
83 int32_t result_shift,
84 int32x4_t result_offset_after_shift_s32,
85 uint8x16_t min_u8,
86 uint8x16_t max_u8,
87 bool is_bounded_relu)
88 {
89 const static int32x4_t zero_s32 = vdupq_n_s32(0);
90
91 if(result_shift < 0)
92 {
93 in_s32.val[0] = vmulq_n_s32(in_s32.val[0], (1 << (-result_shift)));
94 in_s32.val[1] = vmulq_n_s32(in_s32.val[1], (1 << (-result_shift)));
95 in_s32.val[2] = vmulq_n_s32(in_s32.val[2], (1 << (-result_shift)));
96 in_s32.val[3] = vmulq_n_s32(in_s32.val[3], (1 << (-result_shift)));
97
98 in_s32.val[0] = vqrdmulhq_n_s32(in_s32.val[0], result_fixedpoint_multiplier);
99 in_s32.val[1] = vqrdmulhq_n_s32(in_s32.val[1], result_fixedpoint_multiplier);
100 in_s32.val[2] = vqrdmulhq_n_s32(in_s32.val[2], result_fixedpoint_multiplier);
101 in_s32.val[3] = vqrdmulhq_n_s32(in_s32.val[3], result_fixedpoint_multiplier);
102 }
103 else
104 {
105 // Fixed point multiplication with vector saturating rounding doubling multiply high with scalar
106 in_s32.val[0] = vqrdmulhq_n_s32(in_s32.val[0], result_fixedpoint_multiplier);
107 in_s32.val[1] = vqrdmulhq_n_s32(in_s32.val[1], result_fixedpoint_multiplier);
108 in_s32.val[2] = vqrdmulhq_n_s32(in_s32.val[2], result_fixedpoint_multiplier);
109 in_s32.val[3] = vqrdmulhq_n_s32(in_s32.val[3], result_fixedpoint_multiplier);
110
111 // Round to the nearest division by a power-of-two using result_shift_s32
112 in_s32.val[0] = rounding_divide_by_pow2(in_s32.val[0], result_shift);
113 in_s32.val[1] = rounding_divide_by_pow2(in_s32.val[1], result_shift);
114 in_s32.val[2] = rounding_divide_by_pow2(in_s32.val[2], result_shift);
115 in_s32.val[3] = rounding_divide_by_pow2(in_s32.val[3], result_shift);
116 }
117
118 // Add the offset terms
119 in_s32.val[0] = vaddq_s32(in_s32.val[0], result_offset_after_shift_s32);
120 in_s32.val[1] = vaddq_s32(in_s32.val[1], result_offset_after_shift_s32);
121 in_s32.val[2] = vaddq_s32(in_s32.val[2], result_offset_after_shift_s32);
122 in_s32.val[3] = vaddq_s32(in_s32.val[3], result_offset_after_shift_s32);
123
124 // Saturate negative values
125 in_s32.val[0] = vmaxq_s32(in_s32.val[0], zero_s32);
126 in_s32.val[1] = vmaxq_s32(in_s32.val[1], zero_s32);
127 in_s32.val[2] = vmaxq_s32(in_s32.val[2], zero_s32);
128 in_s32.val[3] = vmaxq_s32(in_s32.val[3], zero_s32);
129
130 // Convert S32 to S16
131 const int16x8x2_t in_s16 =
132 {
133 {
134 vcombine_s16(vqmovn_s32(in_s32.val[0]), vqmovn_s32(in_s32.val[1])),
135 vcombine_s16(vqmovn_s32(in_s32.val[2]), vqmovn_s32(in_s32.val[3]))
136 }
137 };
138
139 // Convert S16 to U8
140 uint8x16_t out_u8 = vcombine_u8(vqmovun_s16(in_s16.val[0]), vqmovun_s16(in_s16.val[1]));
141
142 if(is_bounded_relu)
143 {
144 out_u8 = vmaxq_u8(out_u8, min_u8);
145 out_u8 = vminq_u8(out_u8, max_u8);
146 }
147
148 return out_u8;
149 }
150
151 /** Performs final quantization step on 16 elements
152 *
153 * @param[in] in_s32 Input to be quantized.
154 * @param[in] result_fixedpoint_multiplier Result multiplier parameter
155 * @param[in] result_shift Result shift parameter
156 * @param[in] result_offset_after_shift_s32 Result offset parameter
157 * @param[in] min_s8 Relu lower bound
158 * @param[in] max_s8 Relu upper bound
159 * @param[in] is_bounded_relu Specified if a fused bounded relu should be applied
160 *
161 * @return Quantized values
162 */
finalize_quantization(int32x4x4_t & in_s32,int result_fixedpoint_multiplier,int32_t result_shift,int32x4_t result_offset_after_shift_s32,int8x16_t min_s8,int8x16_t max_s8,bool is_bounded_relu)163 inline int8x16_t finalize_quantization(int32x4x4_t &in_s32,
164 int result_fixedpoint_multiplier,
165 int32_t result_shift,
166 int32x4_t result_offset_after_shift_s32,
167 int8x16_t min_s8,
168 int8x16_t max_s8,
169 bool is_bounded_relu)
170 {
171 if(result_shift < 0)
172 {
173 in_s32.val[0] = vmulq_n_s32(in_s32.val[0], (1 << (-result_shift)));
174 in_s32.val[1] = vmulq_n_s32(in_s32.val[1], (1 << (-result_shift)));
175 in_s32.val[2] = vmulq_n_s32(in_s32.val[2], (1 << (-result_shift)));
176 in_s32.val[3] = vmulq_n_s32(in_s32.val[3], (1 << (-result_shift)));
177
178 in_s32.val[0] = vqrdmulhq_n_s32(in_s32.val[0], result_fixedpoint_multiplier);
179 in_s32.val[1] = vqrdmulhq_n_s32(in_s32.val[1], result_fixedpoint_multiplier);
180 in_s32.val[2] = vqrdmulhq_n_s32(in_s32.val[2], result_fixedpoint_multiplier);
181 in_s32.val[3] = vqrdmulhq_n_s32(in_s32.val[3], result_fixedpoint_multiplier);
182 }
183 else
184 {
185 // Fixed point multiplication with vector saturating rounding doubling multiply high with scalar
186 in_s32.val[0] = vqrdmulhq_n_s32(in_s32.val[0], result_fixedpoint_multiplier);
187 in_s32.val[1] = vqrdmulhq_n_s32(in_s32.val[1], result_fixedpoint_multiplier);
188 in_s32.val[2] = vqrdmulhq_n_s32(in_s32.val[2], result_fixedpoint_multiplier);
189 in_s32.val[3] = vqrdmulhq_n_s32(in_s32.val[3], result_fixedpoint_multiplier);
190
191 // Round to the nearest division by a power-of-two using result_shift_s32
192 in_s32.val[0] = rounding_divide_by_pow2(in_s32.val[0], result_shift);
193 in_s32.val[1] = rounding_divide_by_pow2(in_s32.val[1], result_shift);
194 in_s32.val[2] = rounding_divide_by_pow2(in_s32.val[2], result_shift);
195 in_s32.val[3] = rounding_divide_by_pow2(in_s32.val[3], result_shift);
196 }
197
198 // Add the offset terms
199 in_s32.val[0] = vaddq_s32(in_s32.val[0], result_offset_after_shift_s32);
200 in_s32.val[1] = vaddq_s32(in_s32.val[1], result_offset_after_shift_s32);
201 in_s32.val[2] = vaddq_s32(in_s32.val[2], result_offset_after_shift_s32);
202 in_s32.val[3] = vaddq_s32(in_s32.val[3], result_offset_after_shift_s32);
203
204 // Convert S32 to S16
205 const int16x8x2_t in_s16 =
206 {
207 {
208 vcombine_s16(vqmovn_s32(in_s32.val[0]), vqmovn_s32(in_s32.val[1])),
209 vcombine_s16(vqmovn_s32(in_s32.val[2]), vqmovn_s32(in_s32.val[3]))
210 }
211 };
212
213 // Convert S16 to S8
214 int8x16_t out_s8 = vcombine_s8(vqmovn_s16(in_s16.val[0]), vqmovn_s16(in_s16.val[1]));
215
216 if(is_bounded_relu)
217 {
218 out_s8 = vmaxq_s8(out_s8, min_s8);
219 out_s8 = vminq_s8(out_s8, max_s8);
220 }
221
222 return out_s8;
223 }
224
225 /** Performs final quantization step on 16 elements for symmetric quantization
226 *
227 * @param[in] in_s32 Input to be quantized.
228 * @param[in] result_fixedpoint_multiplier Result multiplier parameter
229 * @param[in] result_shift Result shift parameter
230 * @param[in] result_offset_after_shift_s32 Result offset parameter
231 * @param[in] min_s8 Relu lower bound
232 * @param[in] max_s8 Relu upper bound
233 * @param[in] is_bounded_relu Specified if a fused bounded relu should be applied
234 *
235 * @return Quantized values
236 */
finalize_quantization_symm(int32x4x4_t & in_s32,const int32x4x4_t & result_fixedpoint_multiplier,const int32x4x4_t & result_shift,const int32x4_t & result_offset_after_shift_s32,const int8x16_t & min_s8,const int8x16_t & max_s8,const bool is_bounded_relu)237 inline int8x16_t finalize_quantization_symm(int32x4x4_t &in_s32,
238 const int32x4x4_t &result_fixedpoint_multiplier,
239 const int32x4x4_t &result_shift,
240 const int32x4_t &result_offset_after_shift_s32,
241 const int8x16_t &min_s8,
242 const int8x16_t &max_s8,
243 const bool is_bounded_relu)
244 {
245 const static int32x4_t one_s32 = vdupq_n_s32(1);
246
247 // Fixed point multiplication with vector saturating rounding doubling multiply high with scalar
248 int32x4x4_t res_shift_gt0 =
249 {
250 vqrdmulhq_s32(in_s32.val[0], result_fixedpoint_multiplier.val[0]),
251 vqrdmulhq_s32(in_s32.val[1], result_fixedpoint_multiplier.val[1]),
252 vqrdmulhq_s32(in_s32.val[2], result_fixedpoint_multiplier.val[2]),
253 vqrdmulhq_s32(in_s32.val[3], result_fixedpoint_multiplier.val[3]),
254 };
255 // Round to the nearest division by a power-of-two using result_shift_s32
256 res_shift_gt0.val[0] = rounding_divide_by_pow2(res_shift_gt0.val[0], result_shift.val[0]);
257 res_shift_gt0.val[1] = rounding_divide_by_pow2(res_shift_gt0.val[1], result_shift.val[1]);
258 res_shift_gt0.val[2] = rounding_divide_by_pow2(res_shift_gt0.val[2], result_shift.val[2]);
259 res_shift_gt0.val[3] = rounding_divide_by_pow2(res_shift_gt0.val[3], result_shift.val[3]);
260
261 int32x4x4_t res_shift_lt0 =
262 {
263 vmulq_s32(in_s32.val[0], vshlq_s32(one_s32, vnegq_s32(result_shift.val[0]))),
264 vmulq_s32(in_s32.val[1], vshlq_s32(one_s32, vnegq_s32(result_shift.val[1]))),
265 vmulq_s32(in_s32.val[2], vshlq_s32(one_s32, vnegq_s32(result_shift.val[2]))),
266 vmulq_s32(in_s32.val[3], vshlq_s32(one_s32, vnegq_s32(result_shift.val[3]))),
267 };
268 res_shift_lt0.val[0] = vqrdmulhq_s32(res_shift_lt0.val[0], result_fixedpoint_multiplier.val[0]);
269 res_shift_lt0.val[1] = vqrdmulhq_s32(res_shift_lt0.val[1], result_fixedpoint_multiplier.val[1]);
270 res_shift_lt0.val[2] = vqrdmulhq_s32(res_shift_lt0.val[2], result_fixedpoint_multiplier.val[2]);
271 res_shift_lt0.val[3] = vqrdmulhq_s32(res_shift_lt0.val[3], result_fixedpoint_multiplier.val[3]);
272
273 // Select result depending on shift value
274 const uint32x4x4_t mask_lt0 =
275 {
276 #ifdef __aarch64__
277 vcltzq_s32(result_shift.val[0]),
278 vcltzq_s32(result_shift.val[1]),
279 vcltzq_s32(result_shift.val[2]),
280 vcltzq_s32(result_shift.val[3]),
281 #else //__aarch64__
282 vcltq_s32(result_shift.val[0], vdupq_n_s32(0)),
283 vcltq_s32(result_shift.val[1], vdupq_n_s32(0)),
284 vcltq_s32(result_shift.val[2], vdupq_n_s32(0)),
285 vcltq_s32(result_shift.val[3], vdupq_n_s32(0)),
286 #endif //__aarch64__
287 };
288
289 in_s32.val[0] = vbslq_s32(mask_lt0.val[0], res_shift_lt0.val[0], res_shift_gt0.val[0]);
290 in_s32.val[1] = vbslq_s32(mask_lt0.val[1], res_shift_lt0.val[1], res_shift_gt0.val[1]);
291 in_s32.val[2] = vbslq_s32(mask_lt0.val[2], res_shift_lt0.val[2], res_shift_gt0.val[2]);
292 in_s32.val[3] = vbslq_s32(mask_lt0.val[3], res_shift_lt0.val[3], res_shift_gt0.val[3]);
293
294 // Add the offset terms
295 in_s32.val[0] = vaddq_s32(in_s32.val[0], result_offset_after_shift_s32);
296 in_s32.val[1] = vaddq_s32(in_s32.val[1], result_offset_after_shift_s32);
297 in_s32.val[2] = vaddq_s32(in_s32.val[2], result_offset_after_shift_s32);
298 in_s32.val[3] = vaddq_s32(in_s32.val[3], result_offset_after_shift_s32);
299
300 // Convert S32 to S16
301 const int16x8x2_t in_s16 =
302 {
303 {
304 vcombine_s16(vqmovn_s32(in_s32.val[0]), vqmovn_s32(in_s32.val[1])),
305 vcombine_s16(vqmovn_s32(in_s32.val[2]), vqmovn_s32(in_s32.val[3]))
306 }
307 };
308
309 // Convert S16 to S8
310 int8x16_t out_s8 = vcombine_s8(vqmovn_s16(in_s16.val[0]), vqmovn_s16(in_s16.val[1]));
311
312 if(is_bounded_relu)
313 {
314 out_s8 = vmaxq_s8(out_s8, min_s8);
315 out_s8 = vminq_s8(out_s8, max_s8);
316 }
317
318 return out_s8;
319 }
320
321 /** Performs final quantization step on single element
322 *
323 * @param[in] in_value Input to be quantized.
324 * @param[in] result_fixedpoint_multiplier Result multiplier parameter
325 * @param[in] result_shift Result shift parameter
326 * @param[in] result_offset_after_shift_s32 Result offset parameter
327 * @param[in] min_u8 Relu lower bound
328 * @param[in] max_u8 Relu upper bound
329 * @param[in] is_bounded_relu Specified if a fused bounded relu should be applied
330 *
331 * @return Quantized value
332 */
finalize_quantization(int32_t in_value,int result_fixedpoint_multiplier,int32_t result_shift,int32_t result_offset_after_shift_s32,uint8_t min_u8,uint8_t max_u8,bool is_bounded_relu)333 inline uint8_t finalize_quantization(int32_t in_value, int result_fixedpoint_multiplier,
334 int32_t result_shift, int32_t result_offset_after_shift_s32,
335 uint8_t min_u8, uint8_t max_u8, bool is_bounded_relu)
336 {
337 int32x4_t in_s32 = vdupq_n_s32(in_value);
338
339 if(result_shift < 0)
340 {
341 in_value = vgetq_lane_s32(vqrdmulhq_n_s32(vmulq_n_s32(in_s32, (1 << (-result_shift))), result_fixedpoint_multiplier), 0);
342 }
343 else
344 {
345 // Fixed point multiplication with vector saturating rounding doubling multiply high with scalar
346 in_value = vgetq_lane_s32(vqrdmulhq_n_s32(in_s32, result_fixedpoint_multiplier), 0);
347 // Shift value by result_shift_s32
348 in_value = rounding_divide_by_pow2(in_value, result_shift);
349 }
350
351 // Add the offset term
352 in_value += result_offset_after_shift_s32;
353
354 // Bound the result
355 uint8_t out_u8 = static_cast<uint8_t>(std::max<int32_t>(0, std::min<int32_t>(255, in_value)));
356 if(is_bounded_relu)
357 {
358 out_u8 = static_cast<uint8_t>(std::max(min_u8, std::min(max_u8, out_u8)));
359 }
360
361 return out_u8;
362 }
363
364 /** Performs final quantization step on single element
365 *
366 * @param[in] in_value Input to be quantized.
367 * @param[in] result_fixedpoint_multiplier Result multiplier parameter
368 * @param[in] result_shift Result shift parameter
369 * @param[in] result_offset_after_shift_s32 Result offset parameter
370 * @param[in] min_s8 Relu lower bound
371 * @param[in] max_s8 Relu upper bound
372 * @param[in] is_bounded_relu Specified if a fused bounded relu should be applied
373 *
374 * @return Quantized value
375 */
finalize_quantization(int32_t in_value,int result_fixedpoint_multiplier,int32_t result_shift,int32_t result_offset_after_shift_s32,int8_t min_s8,int8_t max_s8,bool is_bounded_relu)376 inline int8_t finalize_quantization(int32_t in_value, int result_fixedpoint_multiplier,
377 int32_t result_shift, int32_t result_offset_after_shift_s32,
378 int8_t min_s8, int8_t max_s8, bool is_bounded_relu)
379 {
380 int32x4_t in_s32 = vdupq_n_s32(in_value);
381
382 if(result_shift < 0)
383 {
384 in_value = vgetq_lane_s32(vqrdmulhq_n_s32(vmulq_n_s32(in_s32, (1 << (-result_shift))), result_fixedpoint_multiplier), 0);
385 }
386 else
387 {
388 // Fixed point multiplication with vector saturating rounding doubling multiply high with scalar
389 in_value = vgetq_lane_s32(vqrdmulhq_n_s32(in_s32, result_fixedpoint_multiplier), 0);
390
391 // Shift value by result_shift_s32
392 in_value = rounding_divide_by_pow2(in_value, result_shift);
393 }
394
395 // Add the offset term
396 in_value += result_offset_after_shift_s32;
397
398 // Bound the result
399 int8_t out_s8 = static_cast<int8_t>(std::max<int32_t>(-128, std::min<int32_t>(127, in_value)));
400 if(is_bounded_relu)
401 {
402 out_s8 = static_cast<int8_t>(std::max(min_s8, std::min(max_s8, out_s8)));
403 }
404
405 return out_s8;
406 }
407
408 /** Dequantize a neon vector holding 8 quantized values.
409 *
410 * @param[in] qv Input values to be dequantized.
411 * @param[in] qi Quantization information to be used in the computation.
412 *
413 * @return Dequantized values in a neon vector
414 */
vdequantize(const uint8x8_t & qv,const UniformQuantizationInfo & qi)415 inline float32x4x2_t vdequantize(const uint8x8_t &qv, const UniformQuantizationInfo &qi)
416 {
417 const float scale = qi.scale;
418 const int offset = qi.offset;
419 const int32x4_t voffset = vdupq_n_s32(offset);
420 const float32x4_t vscale = vdupq_n_f32(scale);
421 const float32x4x2_t vdequantized_input =
422 {
423 {
424 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(qv)))), voffset)), vscale),
425 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(qv)))), voffset)), vscale),
426 }
427 };
428 return vdequantized_input;
429 }
430
431 /** Dequantize a neon vector holding 8 singed quantized values.
432 *
433 * @param[in] qv Input values to be dequantized.
434 * @param[in] qi Quantization information to be used in the computation.
435 *
436 * @return Dequantized values in a neon vector
437 */
vdequantize(const int8x8_t & qv,const UniformQuantizationInfo & qi)438 inline float32x4x2_t vdequantize(const int8x8_t &qv, const UniformQuantizationInfo &qi)
439 {
440 const float scale = qi.scale;
441 const int offset = qi.offset;
442 const int32x4_t voffset = vdupq_n_s32(offset);
443 const float32x4_t vscale = vdupq_n_f32(scale);
444 const float32x4x2_t vdequantized_input =
445 {
446 {
447 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(qv))), voffset)), vscale),
448 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(qv))), voffset)), vscale),
449 }
450 };
451 return vdequantized_input;
452 }
453
454 /** Dequantize a neon vector holding 16 quantized values.
455 *
456 * @param[in] qv Input values to be dequantized.
457 * @param[in] qi Quantization information to be used in the computation.
458 *
459 * @return Dequantized values in a neon vector
460 */
vdequantize(const uint8x16_t & qv,const UniformQuantizationInfo & qi)461 inline float32x4x4_t vdequantize(const uint8x16_t &qv, const UniformQuantizationInfo &qi)
462 {
463 const float scale = qi.scale;
464 const int offset = qi.offset;
465 const int32x4_t voffset = vdupq_n_s32(offset);
466 const float32x4_t vscale = vdupq_n_f32(scale);
467 const float32x4x4_t vdequantized_input =
468 {
469 {
470 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(qv))))), voffset)), vscale),
471 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_low_u8(qv))))), voffset)), vscale),
472 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_high_u8(qv))))), voffset)), vscale),
473 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_high_u8(qv))))), voffset)), vscale),
474 }
475 };
476 return vdequantized_input;
477 }
478
479 /** Dequantize a neon vector holding 16 signed quantized values.
480 *
481 * @param[in] qv Input values to be dequantized.
482 * @param[in] qi Quantization information to be used in the computation.
483 *
484 * @return Dequantized values in a neon vector
485 */
vdequantize(const int8x16_t & qv,const UniformQuantizationInfo & qi)486 inline float32x4x4_t vdequantize(const int8x16_t &qv, const UniformQuantizationInfo &qi)
487 {
488 const float scale = qi.scale;
489 const int offset = qi.offset;
490 const int32x4_t voffset = vdupq_n_s32(offset);
491 const float32x4_t vscale = vdupq_n_f32(scale);
492 const float32x4x4_t vdequantized_input =
493 {
494 {
495 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(qv)))), voffset)), vscale),
496 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_low_s8(qv)))), voffset)), vscale),
497 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_high_s8(qv)))), voffset)), vscale),
498 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_high_s8(qv)))), voffset)), vscale),
499 }
500 };
501 return vdequantized_input;
502 }
503
504 /** Dequantize following an asymmetric quantization scheme a neon vector holding 16 quantized values.
505 *
506 * @param[in] qv Input values to be dequantized.
507 * @param[in] scale Quantization scaling factor.
508 * @param[in] offset Zero quantization offset.
509 *
510 * @return Dequantized values in a neon vector
511 */
vdequantize(const uint8x16_t & qv,float scale,int32_t offset)512 inline float32x4x4_t vdequantize(const uint8x16_t &qv, float scale, int32_t offset)
513 {
514 const int32x4_t voffset = vdupq_n_s32(offset);
515 const float32x4_t vscale = vdupq_n_f32(scale);
516 const float32x4x4_t vdequantized_input =
517 {
518 {
519 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(qv))))), voffset)), vscale),
520 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_low_u8(qv))))), voffset)), vscale),
521 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_high_u8(qv))))), voffset)), vscale),
522 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(vmovl_u8(vget_high_u8(qv))))), voffset)), vscale),
523 }
524 };
525 return vdequantized_input;
526 }
527
528 /** Dequantize a vector of 16 values stored as signed asymmetric.
529 *
530 * @param[in] qv Input values to be dequantized.
531 * @param[in] scale Quantization scaling factor.
532 * @param[in] offset Zero quantization offset.
533 *
534 * @return Dequantized values in a neon vector
535 */
vdequantize(const int8x16_t & qv,float scale,int32_t offset)536 inline float32x4x4_t vdequantize(const int8x16_t &qv, float scale, int32_t offset)
537 {
538 const int32x4_t voffset = vdupq_n_s32(offset);
539 const float32x4_t vscale = vdupq_n_f32(scale);
540 const float32x4x4_t vdequantized_input =
541 {
542 {
543 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(qv)))), voffset)), vscale),
544 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_low_s8(qv)))), voffset)), vscale),
545 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_high_s8(qv)))), voffset)), vscale),
546 vmulq_f32(vcvtq_f32_s32(vsubq_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_high_s8(qv)))), voffset)), vscale),
547 }
548 };
549 return vdequantized_input;
550 }
551
552 /** Dequantize following symmetric quantization scheme a neon vector holding 16 quantized values.
553 *
554 * @param[in] qv Input values to be dequantized.
555 * @param[in] vscale Vector containing quantization scaling factors.
556 *
557 * @return Dequantized values in a neon vector
558 */
vdequantize(const int8x16_t & qv,const float32x4x4_t vscale)559 inline float32x4x4_t vdequantize(const int8x16_t &qv, const float32x4x4_t vscale)
560 {
561 const float32x4x4_t vdequantized_input =
562 {
563 {
564 vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(qv))))), vscale.val[0]),
565 vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_low_s8(qv))))), vscale.val[1]),
566 vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_high_s8(qv))))), vscale.val[2]),
567 vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_high_s8(qv))))), vscale.val[3]),
568 }
569 };
570 return vdequantized_input;
571 }
572
573 /** Dequantize following a symmetric quantization scheme a neon vector holding 16 quantized values.
574 *
575 * @param[in] qv Input values to be dequantized.
576 * @param[in] scale Quantization scaling factor.
577 *
578 * @return Dequantized values in a neon vector
579 */
vdequantize(const int8x16_t & qv,float scale)580 inline float32x4x4_t vdequantize(const int8x16_t &qv, float scale)
581 {
582 const float32x4_t vscale = vdupq_n_f32(scale);
583 const float32x4x4_t vdequantized_input =
584 {
585 {
586 vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(qv))))), vscale),
587 vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_low_s8(qv))))), vscale),
588 vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_high_s8(qv))))), vscale),
589 vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(vmovl_s8(vget_high_s8(qv))))), vscale),
590 }
591 };
592 return vdequantized_input;
593 }
594
595 /** Quantize a neon vector holding 8 floating point values.
596 *
597 * @param[in] qv Input values to be quantized.
598 * @param[in] qi Quantization information to be used in the computation.
599 *
600 * @return A neon vector holding the quantized values
601 */
vquantize(const float32x4x2_t & qv,const UniformQuantizationInfo & qi)602 inline uint8x8_t vquantize(const float32x4x2_t &qv, const UniformQuantizationInfo &qi)
603 {
604 const float scale = qi.scale;
605 const int offset = qi.offset;
606 const float32x4_t voffset = vdupq_n_f32(offset);
607 const float32x4_t vinvscale = vdupq_n_f32(1.f / scale);
608 const int32x4x4_t rf =
609 {
610 {
611 #ifdef __aarch64__
612 vcvtnq_s32_f32(vmlaq_f32(voffset, qv.val[0], vinvscale)),
613 vcvtnq_s32_f32(vmlaq_f32(voffset, qv.val[1], vinvscale)),
614 #else //__aarch64__
615 vcvtq_s32_f32(vmlaq_f32(voffset, qv.val[0], vinvscale)),
616 vcvtq_s32_f32(vmlaq_f32(voffset, qv.val[1], vinvscale)),
617 #endif //__aarch64__
618 }
619 };
620 return vqmovun_s16(vcombine_s16(vqmovn_s32(rf.val[0]), vqmovn_s32(rf.val[1])));
621 }
622
623 /** Quantize a neon vector holding 8 floating point values.
624 *
625 * @param[in] qv Input values to be quantized.
626 * @param[in] qi Quantization information to be used in the computation.
627 *
628 * @return A neon vector holding the singed quantized values
629 */
vquantize_signed(const float32x4x2_t & qv,const UniformQuantizationInfo & qi)630 inline int8x8_t vquantize_signed(const float32x4x2_t &qv, const UniformQuantizationInfo &qi)
631 {
632 const float scale = qi.scale;
633 const int offset = qi.offset;
634 const float32x4_t voffset = vdupq_n_f32(offset);
635 const float32x4_t vinvscale = vdupq_n_f32(1.f / scale);
636 const int32x4x4_t rf =
637 {
638 {
639 #ifdef __aarch64__
640 vcvtnq_s32_f32(vmlaq_f32(voffset, qv.val[0], vinvscale)),
641 vcvtnq_s32_f32(vmlaq_f32(voffset, qv.val[1], vinvscale)),
642 #else //__aarch64__
643 vcvtq_s32_f32(vmlaq_f32(voffset, qv.val[0], vinvscale)),
644 vcvtq_s32_f32(vmlaq_f32(voffset, qv.val[1], vinvscale)),
645 #endif //__aarch64__
646 }
647 };
648 return vqmovn_s16(vcombine_s16(vqmovn_s32(rf.val[0]), vqmovn_s32(rf.val[1])));
649 }
650
vquantize_internal(const float32x4x4_t & qv,float scale,int32_t offset)651 inline int32x4x4_t vquantize_internal(const float32x4x4_t &qv, float scale, int32_t offset)
652 {
653 const int32x4_t voffset = vdupq_n_s32(offset);
654 const float32x4_t vinvscale = vdupq_n_f32(1.f / scale);
655 const int32x4x4_t rf =
656 {
657 {
658 #ifdef __aarch64__
659 vaddq_s32(vcvtaq_s32_f32(vmulq_f32(qv.val[0], vinvscale)), voffset),
660 vaddq_s32(vcvtaq_s32_f32(vmulq_f32(qv.val[1], vinvscale)), voffset),
661 vaddq_s32(vcvtaq_s32_f32(vmulq_f32(qv.val[2], vinvscale)), voffset),
662 vaddq_s32(vcvtaq_s32_f32(vmulq_f32(qv.val[3], vinvscale)), voffset),
663 #else //__aarch64__
664 vaddq_s32(vcvtq_s32_f32(vmulq_f32(qv.val[0], vinvscale)), voffset),
665 vaddq_s32(vcvtq_s32_f32(vmulq_f32(qv.val[1], vinvscale)), voffset),
666 vaddq_s32(vcvtq_s32_f32(vmulq_f32(qv.val[2], vinvscale)), voffset),
667 vaddq_s32(vcvtq_s32_f32(vmulq_f32(qv.val[3], vinvscale)), voffset),
668 #endif //__aarch64__
669 }
670 };
671 return rf;
672 }
673
674 /** Quantize a neon vector holding 16 floating point values.
675 *
676 * @param[in] qv Input values to be quantized.
677 * @param[in] qi Quantization information to be used in the computation.
678 *
679 * @return A neon vector holding the quantized values
680 */
vquantize(const float32x4x4_t & qv,const UniformQuantizationInfo & qi)681 inline uint8x16_t vquantize(const float32x4x4_t &qv, const UniformQuantizationInfo &qi)
682 {
683 auto rf = vquantize_internal(qv, qi.scale, qi.offset);
684 const uint8x8_t pa = vqmovun_s16(vcombine_s16(vqmovn_s32(rf.val[0]), vqmovn_s32(rf.val[1])));
685 const uint8x8_t pb = vqmovun_s16(vcombine_s16(vqmovn_s32(rf.val[2]), vqmovn_s32(rf.val[3])));
686 return vcombine_u8(pa, pb);
687 }
688
689 /** Signed quantize a neon vector holding 16 floating point values.
690 *
691 * @param[in] qv Input values to be quantized.
692 * @param[in] qi Quantization information to be used in the computation.
693 *
694 * @return A neon vector holding the quantized values
695 */
vquantize_signed(const float32x4x4_t & qv,const UniformQuantizationInfo & qi)696 inline int8x16_t vquantize_signed(const float32x4x4_t &qv, const UniformQuantizationInfo &qi)
697 {
698 auto rf = vquantize_internal(qv, qi.scale, qi.offset);
699 const int8x8_t pa = vqmovn_s16(vcombine_s16(vqmovn_s32(rf.val[0]), vqmovn_s32(rf.val[1])));
700 const int8x8_t pb = vqmovn_s16(vcombine_s16(vqmovn_s32(rf.val[2]), vqmovn_s32(rf.val[3])));
701 return vcombine_s8(pa, pb);
702 }
703
704 /** Quantize to QASYMM16 a neon vector holding 16 floating point values.
705 *
706 * @param[in] qv Input values to be quantized.
707 * @param[in] qi Quantization information to be used in the computation.
708 *
709 * @return A neon vector holding the quantized values
710 */
vquantize_qasymm16(const float32x4x4_t & qv,const UniformQuantizationInfo & qi)711 inline uint16x8x2_t vquantize_qasymm16(const float32x4x4_t &qv, const UniformQuantizationInfo &qi)
712 {
713 auto rf = vquantize_internal(qv, qi.scale, qi.offset);
714 const uint16x8_t pa = vcombine_u16(vqmovun_s32(rf.val[0]), vqmovun_s32(rf.val[1]));
715 const uint16x8_t pb = vcombine_u16(vqmovun_s32(rf.val[2]), vqmovun_s32(rf.val[3]));
716 return { pa, pb };
717 }
718 } // namespace arm_compute
719 #include "src/core/NEON/NEAsymm.inl"
720 #endif // ARM_COMPUTE_NEASYMM_H
721