1 // Copyright (c) Facebook, Inc. and its affiliates.
2 // All rights reserved.
3 //
4 // Copyright 2019 Google LLC
5 //
6 // This source code is licensed under the BSD-style license found in the
7 // LICENSE file in the root directory of this source tree.
8
9 #include <assert.h>
10 #include <stdint.h>
11 #include <stddef.h>
12
13 #include <fp16/bitcasts.h>
14
15 #include <xnnpack/scalar-utils.h>
16 #include <xnnpack/requantization-stubs.h>
17
18
xnn_qu8_requantize_precise__scalar_unsigned64(size_t n,const int32_t * input,float scale,uint8_t zero_point,uint8_t qmin,uint8_t qmax,uint8_t * output)19 void xnn_qu8_requantize_precise__scalar_unsigned64(
20 size_t n,
21 const int32_t* input,
22 float scale,
23 uint8_t zero_point,
24 uint8_t qmin,
25 uint8_t qmax,
26 uint8_t* output)
27 {
28 assert(n % 4 == 0);
29 assert(scale < 1.0f);
30 assert(scale >= 0x1.0p-32f);
31
32 const uint32_t scale_bits = fp32_to_bits(scale);
33 const uint32_t multiplier = (scale_bits & UINT32_C(0x007FFFFF)) | UINT32_C(0x00800000);
34 const uint32_t shift = 127 + 23 - (scale_bits >> 23);
35 assert(shift >= 24);
36 assert(shift < 56);
37
38 const uint64_t rounding = UINT64_C(1) << (shift - 1);
39 const int32_t smin = (int32_t)(uint32_t) qmin - (int32_t)(uint32_t) zero_point;
40 const int32_t smax = (int32_t)(uint32_t) qmax - (int32_t)(uint32_t) zero_point;
41 for (; n != 0; n -= 4) {
42 const int32_t x = input[0];
43 const int32_t y = input[1];
44 const int32_t z = input[2];
45 const int32_t w = input[3];
46 input += 4;
47
48 // Compute absolute value of input as unsigned 32-bit int.
49 // All further computations will work with unsigned values to avoid undefined behaviour on signed operations.
50 const uint32_t x_abs = (x >= 0) ? (uint32_t) x : -(uint32_t) x;
51 const uint32_t y_abs = (y >= 0) ? (uint32_t) y : -(uint32_t) y;
52 const uint32_t z_abs = (z >= 0) ? (uint32_t) z : -(uint32_t) z;
53 const uint32_t w_abs = (w >= 0) ? (uint32_t) w : -(uint32_t) w;
54
55 // Compute full 64-bit product of 32-bit factors.
56 const uint64_t x_product = (uint64_t) x_abs * (uint64_t) multiplier;
57 const uint64_t y_product = (uint64_t) y_abs * (uint64_t) multiplier;
58 const uint64_t z_product = (uint64_t) z_abs * (uint64_t) multiplier;
59 const uint64_t w_product = (uint64_t) w_abs * (uint64_t) multiplier;
60
61 // Shift the full 64-bit product right with rounding.
62 // Rounding is performed towards closest integer, with midpoints rounded up (same as away from zero).
63 //
64 // Note that although rounding is precomputed, it is dependent on shift value, and on processors with 64-bit
65 // "right shift with rounding" instruction each line below can be represented by just one such instruction
66 // (e.g. VRSHL.U64 on ARM NEON, URSHL in ARM64 Advanced SIMD).
67 const uint32_t x_abs_scaled = (uint32_t)((x_product + rounding) >> shift);
68 const uint32_t y_abs_scaled = (uint32_t)((y_product + rounding) >> shift);
69 const uint32_t z_abs_scaled = (uint32_t)((z_product + rounding) >> shift);
70 const uint32_t w_abs_scaled = (uint32_t)((w_product + rounding) >> shift);
71
72 // Copy the sign of input to scaled absolute input value.
73 //
74 // On x86 processors with SSSE3 instruction set, this operation nicely maps to PSIGND instruction.
75 const int32_t x_scaled = (int32_t)(x >= 0 ? x_abs_scaled : -x_abs_scaled);
76 const int32_t y_scaled = (int32_t)(y >= 0 ? y_abs_scaled : -y_abs_scaled);
77 const int32_t z_scaled = (int32_t)(z >= 0 ? z_abs_scaled : -z_abs_scaled);
78 const int32_t w_scaled = (int32_t)(w >= 0 ? w_abs_scaled : -w_abs_scaled);
79
80 // Clamp scaled value with zero point between (qmin - zero point) and (qmax - zero point).
81 const int32_t x_clamped = x_scaled < smin ? smin : x_scaled > smax ? smax : x_scaled;
82 const int32_t y_clamped = y_scaled < smin ? smin : y_scaled > smax ? smax : y_scaled;
83 const int32_t z_clamped = z_scaled < smin ? smin : z_scaled > smax ? smax : z_scaled;
84 const int32_t w_clamped = w_scaled < smin ? smin : w_scaled > smax ? smax : w_scaled;
85
86 // Add zero point to clamped value.
87 // The result is guaranteed to be in [qmin, qmax] range.
88 //
89 // This addition can not be safely done before clamping, because scaled values are in [-2147483520, 2147483519]
90 // range, so addition of zero point (which can be up to 255) can overflow signed 32-bit integer.
91 const int32_t x_biased = x_clamped + zero_point;
92 const int32_t y_biased = y_clamped + zero_point;
93 const int32_t z_biased = z_clamped + zero_point;
94 const int32_t w_biased = w_clamped + zero_point;
95
96 output[0] = (uint8_t) x_biased;
97 output[1] = (uint8_t) y_biased;
98 output[2] = (uint8_t) z_biased;
99 output[3] = (uint8_t) w_biased;
100 output += 4;
101 }
102 }
103