1 // Copyright (c) Facebook, Inc. and its affiliates.
2 // All rights reserved.
3 //
4 // Copyright 2019 Google LLC
5 //
6 // This source code is licensed under the BSD-style license found in the
7 // LICENSE file in the root directory of this source tree.
8
9 #pragma once
10
11 #if defined(__cplusplus) && (__cplusplus >= 201103L)
12 #include <cstdint>
13 #include <cstddef>
14 #include <cassert>
15 #include <cmath>
16 #else
17 #include <stdint.h>
18 #include <stddef.h>
19 #include <assert.h>
20 #include <math.h>
21 #endif
22
23 #include <fp16.h>
24
25 #include <xnnpack/common.h>
26 #include <xnnpack/params.h>
27 #include <xnnpack/scalar-utils.h>
28
29
xnn_qu8_requantize_q31(int32_t n,union xnn_qu8_requantization_params params)30 static inline uint8_t xnn_qu8_requantize_q31(
31 int32_t n,
32 union xnn_qu8_requantization_params params)
33 {
34 const int64_t product = (int64_t) n * (int64_t) params.q31.multiplier;
35 const int32_t q31product = (int32_t) (uint32_t) ((uint64_t) (product + INT64_C(0x40000000)) >> 31);
36 const int32_t remainder = (q31product & params.q31.remainder_mask) - (int32_t) (n < 0);
37 n = asr_s32(q31product, params.q31.shift) + (int32_t) (remainder > params.q31.remainder_threshold);
38 if (n < params.q31.min_less_zero_point) {
39 n = params.q31.min_less_zero_point;
40 }
41 if (n > params.q31.max_less_zero_point) {
42 n = params.q31.max_less_zero_point;
43 }
44
45 return (uint8_t) (n + params.q31.zero_point);
46 }
47
xnn_qs8_requantize_q31(int32_t n,union xnn_qs8_requantization_params params)48 static inline uint8_t xnn_qs8_requantize_q31(
49 int32_t n,
50 union xnn_qs8_requantization_params params)
51 {
52 const int64_t product = (int64_t) n * (int64_t) params.q31.multiplier;
53 const int32_t q31product = (int32_t) (uint32_t) ((uint64_t) (product + INT64_C(0x40000000)) >> 31);
54 const int32_t remainder = (q31product & params.q31.remainder_mask) - (int32_t) (n < 0);
55 n = asr_s32(q31product, params.q31.shift) + (int32_t) (remainder > params.q31.remainder_threshold);
56 if (n < params.q31.min_less_zero_point) {
57 n = params.q31.min_less_zero_point;
58 }
59 if (n > params.q31.max_less_zero_point) {
60 n = params.q31.max_less_zero_point;
61 }
62
63 return (int8_t) (n + params.q31.zero_point);
64 }
65
xnn_qu8_requantize_precise(int32_t value,float scale,uint8_t zero_point,uint8_t qmin,uint8_t qmax)66 inline static uint8_t xnn_qu8_requantize_precise(
67 int32_t value,
68 float scale,
69 uint8_t zero_point,
70 uint8_t qmin,
71 uint8_t qmax)
72 {
73 assert(scale < 1.0f);
74 assert(scale >= 0x1.0p-32f);
75
76 const uint32_t scale_bits = fp32_to_bits(scale);
77 const uint32_t multiplier = (scale_bits & UINT32_C(0x007FFFFF)) | UINT32_C(0x00800000);
78 const uint32_t shift = 127 + 23 - (scale_bits >> 23);
79 assert(shift >= 24);
80 assert(shift < 56);
81
82 // Compute absolute value of input as unsigned 32-bit int.
83 // All further computations will work with unsigned values to avoid undefined behaviour on signed operations.
84 const uint32_t abs_value = (value >= 0) ? (uint32_t) value : -(uint32_t) value;
85
86 // Compute full 64-bit product of 32-bit factors
87 const uint64_t product = (uint64_t) abs_value * (uint64_t) multiplier;
88
89 // Shift the full 64-bit product right with rounding.
90 // Rounding is performed towards closest integer, with midpoints rounded up (same as away from zero).
91 const uint64_t rounding = UINT64_C(1) << (shift - 1);
92 const uint32_t abs_scaled_value = (uint32_t) ((product + rounding) >> shift);
93
94 // Copy the sign of input to scaled absolute input value.
95 const int32_t scaled_value = (int32_t) (value >= 0 ? abs_scaled_value : -abs_scaled_value);
96
97 // Clamp scaled value with zero point between smin and smax.
98 int32_t clamped_value = scaled_value;
99 const int32_t smin = (int32_t) (uint32_t) qmin - (int32_t) (uint32_t) zero_point;
100 if (clamped_value < smin) {
101 clamped_value = smin;
102 }
103 const int32_t smax = (int32_t) (uint32_t) qmax - (int32_t) (uint32_t) zero_point;
104 if (clamped_value > smax) {
105 clamped_value = smax;
106 }
107
108 // Add zero point to clamped value.
109 const int32_t biased_value = clamped_value + (int32_t) (uint32_t) zero_point;
110
111 return biased_value;
112 }
113
xnn_qs8_requantize_precise(int32_t value,float scale,int8_t zero_point,int8_t qmin,int8_t qmax)114 inline static int8_t xnn_qs8_requantize_precise(
115 int32_t value,
116 float scale,
117 int8_t zero_point,
118 int8_t qmin,
119 int8_t qmax)
120 {
121 assert(scale < 1.0f);
122 assert(scale >= 0x1.0p-32f);
123
124 const uint32_t scale_bits = fp32_to_bits(scale);
125 const uint32_t multiplier = (scale_bits & UINT32_C(0x007FFFFF)) | UINT32_C(0x00800000);
126 const uint32_t shift = 127 + 23 - (scale_bits >> 23);
127 assert(shift >= 24);
128 assert(shift < 56);
129
130 // Compute absolute value of input as unsigned 32-bit int.
131 // All further computations will work with unsigned values to avoid undefined behaviour on signed operations.
132 const uint32_t abs_value = (value >= 0) ? (uint32_t) value : -(uint32_t) value;
133
134 // Compute full 64-bit product of 32-bit factors
135 const uint64_t product = (uint64_t) abs_value * (uint64_t) multiplier;
136
137 // Shift the full 64-bit product right with rounding.
138 // Rounding is performed towards closest integer, with midpoints rounded up (same as away from zero).
139 const uint64_t rounding = UINT64_C(1) << (shift - 1);
140 const uint32_t abs_scaled_value = (uint32_t) ((product + rounding) >> shift);
141
142 // Copy the sign of input to scaled absolute input value.
143 const int32_t scaled_value = (int32_t) (value >= 0 ? abs_scaled_value : -abs_scaled_value);
144
145 // Clamp scaled value with zero point between smin and smax.
146 int32_t clamped_value = scaled_value;
147 const int32_t smin = (int32_t) qmin - (int32_t) zero_point;
148 if (clamped_value < smin) {
149 clamped_value = smin;
150 }
151 const int32_t smax = (int32_t) qmax - (int32_t) zero_point;
152 if (clamped_value > smax) {
153 clamped_value = smax;
154 }
155
156 // Add zero point to clamped value.
157 const int32_t biased_value = clamped_value + (int32_t) zero_point;
158
159 return biased_value;
160 }
161
xnn_qu8_quantize_avgpool(int32_t n,union xnn_qu8_avgpool_params params)162 static inline uint8_t xnn_qu8_quantize_avgpool(
163 int32_t n,
164 union xnn_qu8_avgpool_params params)
165 {
166 const int64_t product = (int64_t) n * (int64_t) params.scalar.multiplier;
167 const int64_t adjusted_product = product - (int64_t) (n < 0);
168
169 n = (int32_t) asr_s64(adjusted_product + params.scalar.rounding, params.scalar.right_shift);
170 if (n < params.scalar.output_min_less_zero_point) {
171 n = params.scalar.output_min_less_zero_point;
172 }
173 if (n > params.scalar.output_max_less_zero_point) {
174 n = params.scalar.output_max_less_zero_point;
175 }
176
177 return (uint8_t) (n + params.scalar.output_zero_point);
178 }
179
xnn_qs8_quantize_avgpool(int32_t n,union xnn_qs8_avgpool_params params)180 static inline int8_t xnn_qs8_quantize_avgpool(
181 int32_t n,
182 union xnn_qs8_avgpool_params params)
183 {
184 const int64_t product = (int64_t) n * (int64_t) params.scalar.multiplier;
185 const int64_t adjusted_product = product - (int64_t) (n < 0);
186
187 n = (int32_t) asr_s64(adjusted_product + params.scalar.rounding, params.scalar.shift);
188 if (n < params.scalar.output_min_less_zero_point) {
189 n = params.scalar.output_min_less_zero_point;
190 }
191 if (n > params.scalar.output_max_less_zero_point) {
192 n = params.scalar.output_max_less_zero_point;
193 }
194
195 return (int8_t) (n + params.scalar.output_zero_point);
196 }
197
xnn_qu8_quantize_add(uint8_t a,uint8_t b,union xnn_qu8_add_params params)198 static inline uint8_t xnn_qu8_quantize_add(
199 uint8_t a, uint8_t b,
200 union xnn_qu8_add_params params)
201 {
202 // Multiply by factors and accumulate products.
203 int32_t acc = params.scalar.zero_point_product +
204 (int32_t) ((uint32_t) a * params.scalar.a_multiplier) +
205 (int32_t) ((uint32_t) b * params.scalar.b_multiplier);
206
207 // Shift right and round.
208 const int32_t rem = (acc & params.scalar.remainder_mask) - (int32_t) (acc < 0);
209 acc = asr_s32(acc, params.scalar.shift) + (int32_t) (rem > params.scalar.remainder_threshold);
210
211 // Clamp and add output zero point.
212 int32_t y = acc + params.scalar.y_zero_point;
213 if (y >= params.scalar.y_max) {
214 y = params.scalar.y_max;
215 }
216 if (y <= params.scalar.y_min) {
217 y = params.scalar.y_min;
218 }
219 return (uint8_t) y;
220 }
221
xnn_qs8_quantize_add(int8_t x,int8_t y,union xnn_qs8_add_params params)222 static inline int8_t xnn_qs8_quantize_add(
223 int8_t x, int8_t y,
224 union xnn_qs8_add_params params)
225 {
226 // Multiply by factors and accumulate products.
227 int32_t acc = params.scalar.zero_point_product +
228 (int32_t) ((int32_t) x * params.scalar.x_multiplier) +
229 (int32_t) ((int32_t) y * params.scalar.y_multiplier);
230
231 // Shift right and round.
232 const int32_t rem = (acc & params.scalar.remainder_mask) - (int32_t) (acc < 0);
233 acc = asr_s32(acc, params.scalar.shift) + (int32_t) (rem > params.scalar.remainder_threshold);
234
235 // Clamp and add output zero point.
236 int32_t out = acc + params.scalar.output_zero_point;
237 if (out >= params.scalar.output_max) {
238 out = params.scalar.output_max;
239 }
240 if (out <= params.scalar.output_min) {
241 out = params.scalar.output_min;
242 }
243 return (int8_t) out;
244 }
245