1 // Quantized calculation utilities.
2 // TODO(vddang): Replace this with tensorflow/lite/kernels/internal/tensor_utils(common).h
3 // after TFLite module has been synced.
4
5 #ifndef ANDROID_PACKAGES_MODULES_NEURALNETWORKS_COMMON_QUANTUTILS_H
6 #define ANDROID_PACKAGES_MODULES_NEURALNETWORKS_COMMON_QUANTUTILS_H
7
8 #pragma clang diagnostic push
9 #pragma clang diagnostic ignored "-Wsign-compare"
10 #include <public/gemmlowp.h>
11 #pragma clang diagnostic pop
12
13 #include <limits>
14 #include <memory>
15
16 #include "LegacyUtils.h"
17 #include "OperationsExecutionUtils.h"
18
19 namespace android {
20 namespace nn {
21
MultiplyByQuantizedMultiplier(int32_t x,int32_t quantized_multiplier,int shift)22 inline int32_t MultiplyByQuantizedMultiplier(int32_t x, int32_t quantized_multiplier, int shift) {
23 using gemmlowp::RoundingDivideByPOT;
24 using gemmlowp::SaturatingRoundingDoublingHighMul;
25 int left_shift = shift > 0 ? shift : 0;
26 int right_shift = shift > 0 ? 0 : -shift;
27 return RoundingDivideByPOT(
28 SaturatingRoundingDoublingHighMul(x * (1 << left_shift), quantized_multiplier),
29 right_shift);
30 }
31
32 template <typename T>
MatrixBatchVectorMultiplyAccumulate(const int8_t * input,const int32_t * bias,const int8_t * input_to_gate_weights,int32_t multiplier,int32_t shift,int32_t n_batch,int32_t n_input,int32_t n_output,int32_t output_zp,T * output)33 void MatrixBatchVectorMultiplyAccumulate(const int8_t* input, const int32_t* bias,
34 const int8_t* input_to_gate_weights, int32_t multiplier,
35 int32_t shift, int32_t n_batch, int32_t n_input,
36 int32_t n_output, int32_t output_zp, T* output) {
37 const int16_t output_max = std::numeric_limits<T>::max();
38 const int16_t output_min = std::numeric_limits<T>::min();
39 for (int batch = 0; batch < n_batch; ++batch) {
40 for (int row = 0; row < n_output; ++row) {
41 int32_t acc = bias[row];
42 for (int col = 0; col < n_input; ++col) {
43 int8_t input_val = input[batch * n_input + col];
44 int8_t weights_val = input_to_gate_weights[row * n_input + col];
45 acc += input_val * weights_val;
46 }
47 acc = MultiplyByQuantizedMultiplier(acc, multiplier, shift);
48 acc += output_zp;
49 acc += output[batch * n_output + row];
50 if (acc > output_max) {
51 acc = output_max;
52 }
53 if (acc < output_min) {
54 acc = output_min;
55 }
56 output[batch * n_output + row] = static_cast<T>(acc);
57 }
58 }
59 }
60
61 template <typename T>
CountLeadingZeros(T integer_input)62 int CountLeadingZeros(T integer_input) {
63 static_assert(std::is_unsigned<T>::value, "Only unsigned integer types handled.");
64 #if defined(__GNUC__)
65 return integer_input ? __builtin_clz(integer_input) : std::numeric_limits<T>::digits;
66 #else
67 if (integer_input == 0) {
68 return std::numeric_limits<T>::digits;
69 }
70
71 const T one_in_leading_positive = static_cast<T>(1) << (std::numeric_limits<T>::digits - 1);
72 int leading_zeros = 0;
73 while (integer_input < one_in_leading_positive) {
74 integer_input <<= 1;
75 ++leading_zeros;
76 }
77 return leading_zeros;
78 #endif
79 }
80
GetInvSqrtQuantizedMultiplierExp(int32_t input,int reverse_shift,int32_t * output_inv_sqrt,int * output_shift)81 inline bool GetInvSqrtQuantizedMultiplierExp(int32_t input, int reverse_shift,
82 int32_t* output_inv_sqrt, int* output_shift) {
83 NN_RET_CHECK_GE(input, 0);
84 if (input <= 1) {
85 // Handle the input value 1 separately to avoid overflow in that case
86 // in the general computation below. Also handle 0 as if it
87 // were a 1. 0 is an invalid input here (divide by zero) and 1 is a valid
88 // but rare/unrealistic input value. We can expect both to occur in some
89 // incompletely trained models, but probably not in fully trained models.
90 *output_inv_sqrt = std::numeric_limits<std::int32_t>::max();
91 *output_shift = 0;
92 return true;
93 }
94
95 *output_shift = 11;
96 while (input >= (1 << 29)) {
97 input /= 4;
98 ++*output_shift;
99 }
100 const unsigned max_left_shift_bits = CountLeadingZeros(static_cast<uint32_t>(input)) - 1;
101 const unsigned max_left_shift_bit_pairs = max_left_shift_bits / 2;
102 const unsigned left_shift_bit_pairs = max_left_shift_bit_pairs - 1;
103 *output_shift -= left_shift_bit_pairs;
104 input <<= 2 * left_shift_bit_pairs;
105 NN_RET_CHECK_GE(input, (1 << 27));
106 NN_RET_CHECK_LT(input, (1 << 29));
107 using gemmlowp::FixedPoint;
108 using gemmlowp::Rescale;
109 using gemmlowp::SaturatingRoundingMultiplyByPOT;
110 // Using 3 integer bits gives us enough room for the internal arithmetic in
111 // this Newton-Raphson iteration.
112 using F3 = FixedPoint<int32_t, 3>;
113 using F0 = FixedPoint<int32_t, 0>;
114 const F3 fixedpoint_input = F3::FromRaw(input >> 1);
115 const F3 fixedpoint_half_input = SaturatingRoundingMultiplyByPOT<-1>(fixedpoint_input);
116 const F3 fixedpoint_half_three =
117 GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(F3, (1 << 28) + (1 << 27), 1.5);
118 // Newton-Raphson iteration
119 // Naive unoptimized starting guess: x = 1
120 F3 x = F3::One();
121 // Naive unoptimized number of iterations: 5
122 for (int i = 0; i < 5; i++) {
123 const F3 x3 = Rescale<3>(x * x * x);
124 x = Rescale<3>(fixedpoint_half_three * x - fixedpoint_half_input * x3);
125 }
126 const F0 fixedpoint_half_sqrt_2 =
127 GEMMLOWP_CHECKED_FIXEDPOINT_CONSTANT(F0, 1518500250, std::sqrt(2.) / 2.);
128 x = x * fixedpoint_half_sqrt_2;
129 *output_inv_sqrt = x.raw();
130 if (*output_shift < 0) {
131 *output_inv_sqrt <<= -*output_shift;
132 *output_shift = 0;
133 }
134 // Convert right shift (right is positive) to left shift.
135 *output_shift *= reverse_shift;
136 return true;
137 }
138
139 void ApplyLayerNorm(const int16_t* input, const int16_t* layer_norm_weights, const int32_t* bias,
140 int32_t layer_norm_scale_a, int32_t layer_norm_scale_b, int32_t variance_limit,
141 int n_batch, int n_input, int16_t* output);
142
143 void MatrixScalarMultiplyAccumulate(const int8_t* matrix, int32_t scalar, int32_t n_row,
144 int32_t n_col, int32_t* output);
145
146 bool PrecomputeZeroPointTimesWeightWithBias(int32_t zero_point, const int8_t* weight_tensor,
147 const Shape& weight_shape, const int32_t* bias_tensor,
148 std::unique_ptr<int32_t[]>* output);
149
150 void ApplySigmoid(const int16_t* input, int32_t n_batch, int32_t n_input, int16_t* output);
151
152 template <int IntegerBits>
ApplyTanh(const int16_t * input,int32_t n_batch,int32_t n_input,int16_t * output)153 void ApplyTanh(const int16_t* input, int32_t n_batch, int32_t n_input, int16_t* output) {
154 using FX = gemmlowp::FixedPoint<std::int16_t, IntegerBits>;
155 using F0 = gemmlowp::FixedPoint<std::int16_t, 0>;
156 for (int batch = 0; batch < n_batch; ++batch) {
157 for (int i = 0; i < n_input; ++i) {
158 const int index = batch * n_input + i;
159 FX tanh_input = FX::FromRaw(input[index]);
160 F0 tanh_output = gemmlowp::tanh(tanh_input);
161 output[index] = tanh_output.raw();
162 }
163 }
164 }
165
ApplyTanh(int32_t integer_bits,const int16_t * input,int32_t n_batch,int32_t n_input,int16_t * output)166 inline void ApplyTanh(int32_t integer_bits, const int16_t* input, int32_t n_batch, int32_t n_input,
167 int16_t* output) {
168 assert(integer_bits <= 6);
169 #define DISPATCH_TANH(i) \
170 case i: \
171 ApplyTanh<i>(input, n_batch, n_input, output); \
172 break;
173 switch (integer_bits) {
174 DISPATCH_TANH(0);
175 DISPATCH_TANH(1);
176 DISPATCH_TANH(2);
177 DISPATCH_TANH(3);
178 DISPATCH_TANH(4);
179 DISPATCH_TANH(5);
180 DISPATCH_TANH(6);
181 default:
182 return;
183 }
184 #undef DISPATCH_TANH
185 }
186
187 void CwiseMul(const int16_t* input_1, const int16_t* input_2, int n_batch, int n_input, int shift,
188 int16_t* output);
189 void CwiseMul(const int16_t* input_1, const int16_t* input_2, int32_t multiplier, int32_t shift,
190 int32_t n_batch, int32_t n_input, int32_t output_zp, int8_t* output);
191
192 bool CheckedLog2(const float x, int* log2_result);
193
194 void CwiseAdd(const int16_t* input_1, const int16_t* input_2, int n_batch, int n_input,
195 int16_t* output);
196
Sub1Vector(const int16_t * vector,int v_size,int16_t * result)197 inline void Sub1Vector(const int16_t* vector, int v_size, int16_t* result) {
198 static const int16_t kOne = 32767;
199 for (int v = 0; v < v_size; v++) {
200 *result++ = kOne - *vector++;
201 }
202 }
203
204 void CwiseClipping(int16_t* input, const int16_t clipping_value, int32_t n_batch, int32_t n_input);
205
206 void CwiseClipping(int8_t* input, const int8_t clipping_value, int32_t n_batch, int32_t n_input);
207
208 void VectorBatchVectorCwiseProductAccumulate(const int16_t* vector, int v_size,
209 const int16_t* batch_vector, int n_batch,
210 int32_t multiplier, int shift, int16_t* result);
211
212 } // namespace nn
213 } // namespace android
214
215 #endif // ANDROID_PACKAGES_MODULES_NEURALNETWORKS_COMMON_QUANTUTILS_H
216