1 // Copyright (c) Facebook, Inc. and its affiliates.
2 // All rights reserved.
3 //
4 // Copyright 2019 Google LLC
5 //
6 // This source code is licensed under the BSD-style license found in the
7 // LICENSE file in the root directory of this source tree.
8
9 #include <assert.h>
10 #include <stdint.h>
11 #include <stddef.h>
12
13 #include <wasm_simd128.h>
14
15 #include <fp16/bitcasts.h>
16
17 #include <xnnpack/requantization-stubs.h>
18
19
xnn_qs8_requantize_gemmlowp__wasmsimd(size_t n,const int32_t * input,float scale,int8_t zero_point,int8_t qmin,int8_t qmax,int8_t * output)20 void xnn_qs8_requantize_gemmlowp__wasmsimd(
21 size_t n,
22 const int32_t* input,
23 float scale,
24 int8_t zero_point,
25 int8_t qmin,
26 int8_t qmax,
27 int8_t* output)
28 {
29 assert(n % 16 == 0);
30 assert(scale < 1.0f);
31 assert(scale >= 0x1.0p-32f);
32
33 // Compute requantization parameters.
34 const uint32_t scale_bits = fp32_to_bits(scale);
35
36 // Multiplier is in [0x40000000, 0x7FFFFF80] range.
37 const int32_t multiplier = (int32_t) (((scale_bits & UINT32_C(0x007FFFFF)) | UINT32_C(0x00800000)) << 7);
38 assert(multiplier >= INT32_C(0x40000000));
39 assert(multiplier <= INT32_C(0x7FFFFF80));
40 const int64_t twice_multiplier = INT64_C(2) * (int64_t) multiplier;
41
42 // Shift is in [0, 31] range.
43 const int32_t shift = 127 + 31 - 32 - (fp32_to_bits(scale) >> 23);
44 assert(shift >= 0);
45 assert(shift < 32);
46
47 const v128_t vmultiplier = wasm_i64x2_make(twice_multiplier, twice_multiplier);
48 const v128_t vzero_point = wasm_i16x8_splat((int16_t) zero_point);
49
50 const v128_t vqmin = wasm_i8x16_splat(qmin);
51 const v128_t vqmax = wasm_i8x16_splat(qmax);
52 const uint32_t remainder_mask = (UINT32_C(1) << shift) - UINT32_C(1);
53 const v128_t vremainder_mask = wasm_i32x4_splat((int32_t) remainder_mask);
54 const v128_t vthreshold = wasm_i32x4_splat((int32_t) (remainder_mask >> 1));
55 const v128_t vtwice_q31rounding = wasm_i64x2_splat(INT64_C(0x80000000));
56 for (; n != 0; n -= 16) {
57 const v128_t x = wasm_v128_load(input);
58 const v128_t y = wasm_v128_load(input + 4);
59 const v128_t z = wasm_v128_load(input + 8);
60 const v128_t w = wasm_v128_load(input + 12);
61 input += 16;
62
63 const v128_t x_sign = wasm_i32x4_shr(x, 31);
64 const v128_t y_sign = wasm_i32x4_shr(y, 31);
65 const v128_t z_sign = wasm_i32x4_shr(z, 31);
66 const v128_t w_sign = wasm_i32x4_shr(w, 31);
67
68 const v128_t x_lo = wasm_v32x4_shuffle(x, x_sign, 0, 4, 1, 5);
69 const v128_t y_lo = wasm_v32x4_shuffle(y, y_sign, 0, 4, 1, 5);
70 const v128_t z_lo = wasm_v32x4_shuffle(z, z_sign, 0, 4, 1, 5);
71 const v128_t w_lo = wasm_v32x4_shuffle(w, w_sign, 0, 4, 1, 5);
72
73 const v128_t x_hi = wasm_v32x4_shuffle(x, x_sign, 2, 6, 3, 7);
74 const v128_t y_hi = wasm_v32x4_shuffle(y, y_sign, 2, 6, 3, 7);
75 const v128_t z_hi = wasm_v32x4_shuffle(z, z_sign, 2, 6, 3, 7);
76 const v128_t w_hi = wasm_v32x4_shuffle(w, w_sign, 2, 6, 3, 7);
77
78 const v128_t x_product_lo = wasm_i64x2_add(wasm_i64x2_mul(x_lo, vmultiplier), vtwice_q31rounding);
79 const v128_t y_product_lo = wasm_i64x2_add(wasm_i64x2_mul(y_lo, vmultiplier), vtwice_q31rounding);
80 const v128_t z_product_lo = wasm_i64x2_add(wasm_i64x2_mul(z_lo, vmultiplier), vtwice_q31rounding);
81 const v128_t w_product_lo = wasm_i64x2_add(wasm_i64x2_mul(w_lo, vmultiplier), vtwice_q31rounding);
82
83 const v128_t x_product_hi = wasm_i64x2_add(wasm_i64x2_mul(x_hi, vmultiplier), vtwice_q31rounding);
84 const v128_t y_product_hi = wasm_i64x2_add(wasm_i64x2_mul(y_hi, vmultiplier), vtwice_q31rounding);
85 const v128_t z_product_hi = wasm_i64x2_add(wasm_i64x2_mul(z_hi, vmultiplier), vtwice_q31rounding);
86 const v128_t w_product_hi = wasm_i64x2_add(wasm_i64x2_mul(w_hi, vmultiplier), vtwice_q31rounding);
87
88 const v128_t x_q31product = wasm_v32x4_shuffle(x_product_lo, x_product_hi, 1, 3, 5, 7);
89 const v128_t y_q31product = wasm_v32x4_shuffle(y_product_lo, y_product_hi, 1, 3, 5, 7);
90 const v128_t z_q31product = wasm_v32x4_shuffle(z_product_lo, z_product_hi, 1, 3, 5, 7);
91 const v128_t w_q31product = wasm_v32x4_shuffle(w_product_lo, w_product_hi, 1, 3, 5, 7);
92
93 const v128_t x_remainder =
94 wasm_i32x4_add(wasm_v128_and(x_q31product, vremainder_mask), wasm_i32x4_shr(x_q31product, 31));
95 const v128_t y_remainder =
96 wasm_i32x4_add(wasm_v128_and(y_q31product, vremainder_mask), wasm_i32x4_shr(y_q31product, 31));
97 const v128_t z_remainder =
98 wasm_i32x4_add(wasm_v128_and(z_q31product, vremainder_mask), wasm_i32x4_shr(z_q31product, 31));
99 const v128_t w_remainder =
100 wasm_i32x4_add(wasm_v128_and(w_q31product, vremainder_mask), wasm_i32x4_shr(w_q31product, 31));
101
102 const v128_t x_scaled =
103 wasm_i32x4_sub(wasm_i32x4_shr(x_q31product, shift), wasm_i32x4_gt(x_remainder, vthreshold));
104 const v128_t y_scaled =
105 wasm_i32x4_sub(wasm_i32x4_shr(y_q31product, shift), wasm_i32x4_gt(y_remainder, vthreshold));
106 const v128_t z_scaled =
107 wasm_i32x4_sub(wasm_i32x4_shr(z_q31product, shift), wasm_i32x4_gt(z_remainder, vthreshold));
108 const v128_t w_scaled =
109 wasm_i32x4_sub(wasm_i32x4_shr(w_q31product, shift), wasm_i32x4_gt(w_remainder, vthreshold));
110
111 const v128_t xy_packed = wasm_i16x8_add_sat(wasm_i16x8_narrow_i32x4(x_scaled, y_scaled), vzero_point);
112 const v128_t zw_packed = wasm_i16x8_add_sat(wasm_i16x8_narrow_i32x4(z_scaled, w_scaled), vzero_point);
113 const v128_t xyzw_packed = wasm_i8x16_narrow_i16x8(xy_packed, zw_packed);
114 const v128_t xyzw_clamped = wasm_i8x16_min(wasm_i8x16_max(xyzw_packed, vqmin), vqmax);
115
116 // 12x v128.shuffle
117 // 8x i32x4.lt
118 // 8x i64x2.add
119 // 8x i64x2.mul
120 // 4x v128.and
121 // 4x i32x4.add
122 // 4x i32x4.sub
123 // 4x i32x4.gt
124 // 4x i32x4.shr_s
125 // 2x i16x8.narrow_i32x4_s
126 // 2x i16x8.add_saturate_s
127 // 1x i8x16.narrow_i16x8_s
128 // 1x i8x16.max_s
129 // 1x i8x16.min_s
130 // ---------------------
131 // 63 instructions total
132
133 wasm_v128_store(output, xyzw_clamped);
134 output += 16;
135 }
136 }
137