1// Copyright 2020 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 7$assert NR % 8 == 0 8$assert 8 <= NR <= 32 9$assert REQUANTIZATION in ["FP32", "RNDNU"] 10#include <assert.h> 11 12#include <arm_neon.h> 13 14#include <xnnpack/igemm.h> 15$if REQUANTIZATION == "FP32": 16 #include <xnnpack/intrinsics-polyfill.h> 17#include <xnnpack/math.h> 18 19 20$PARAMS_STRUCT = "fp32_neonv8" if REQUANTIZATION == "FP32" else REQUANTIZATION.lower() + "_neon" 21void xnn_qu8_igemm_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x${NR}c4__neondot( 22 size_t mr, 23 size_t nc, 24 size_t kc, 25 size_t ks, 26 const uint8_t** restrict a, 27 const void* restrict w, 28 uint8_t* restrict c, 29 size_t cm_stride, 30 size_t cn_stride, 31 size_t a_offset, 32 const uint8_t* zero, 33 const union xnn_qu8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS 34{ 35 assert(mr != 0); 36 assert(mr <= ${MR}); 37 assert(nc != 0); 38 assert(kc != 0); 39 assert(ks != 0); 40 assert(ks % (${MR} * sizeof(void*)) == 0); 41 assert(a_offset % sizeof(uint8_t) == 0); 42 assert(a != NULL); 43 assert(w != NULL); 44 assert(c != NULL); 45 46 kc = round_up_po2(kc, 4 * sizeof(uint8_t)); 47 uint8_t* c0 = c; 48 $for M in range(1, MR): 49 uint8_t* c${M} = (uint8_t*) ((uintptr_t) c${M-1} + cm_stride); 50 $if M % 2 == 0: 51 if XNN_UNPREDICTABLE(mr <= ${M}) { 52 c${M} = c${M-1}; 53 } 54 $elif M + 1 == MR: 55 if XNN_UNPREDICTABLE(mr != ${M+1}) { 56 c${M} = c${M-1}; 57 } 58 $else: 59 if XNN_UNPREDICTABLE(mr < ${M+1}) { 60 c${M} = c${M-1}; 61 } 62 63 const uint8x8_t va_zero_point = vld1_dup_u8(¶ms->${PARAMS_STRUCT}.kernel_zero_point[0]); 64 65 do { 66 // Initialize accumulators with bias. ${NR} bias values are loaded from the 67 // weight matrix, at the start of the group of ${NR} columns. 68 $for N in range(0, NR, 4): 69 uint32x4_t vpacc0x${ABC[N:N+4]} = vld1q_u32(w); w = (const void*) ((const uint32_t*) w + 4); 70 $for M in range(1, MR): 71 $for N in range(0, NR, 4): 72 uint32x4_t vpacc${M}x${ABC[N:N+4]} = vpacc0x${ABC[N:N+4]}; 73 $for M in range(0, MR): 74 uint32x2_t vnacc${M} = vmov_n_u32(0); 75 76 size_t p = ks; 77 do { 78 $for M in range(MR): 79 const uint8_t* restrict a${M} = a[${M}]; 80 if XNN_UNPREDICTABLE(a${M} != zero) { 81 a${M} = (const uint8_t*) ((uintptr_t) a${M} + a_offset); 82 } 83 a += ${MR}; 84 85 // Inner accumulation loop along the ${NR} columns. 86 size_t k = kc; 87 // 2x partial unrolled loop to load 8 bytes at a time. 88 while (k >= 8 * sizeof(uint8_t)) { 89 // Load a ${MR}x8 block of activations. 90 $for M in range(MR): 91 const uint8x8_t va${M}x01234567 = vld1_u8(a${M}); a${M} += 8; 92 93 // Load a 8x${NR} block of weights. 94 $for K in range(0, 8, 4): 95 $for N in range(0, NR, 4): 96 const uint8x16_t vb${ABC[K:K+4]}x${ABC[N:N+4]} = vld1q_u8(w); w = (const void*) ((const uint8_t*) w + 16); 97 98 // Multiply-accumulate: ${MR}x8 * 8x${NR} --> ${MR}x${NR}. 99 $for M in range(MR): 100 vnacc${M} = vdot_u32(vnacc${M}, va_zero_point, va${M}x01234567); 101 $for K in range(0, 8, 4): 102 $for N in range(0, NR, 4): 103 vpacc${M}x${ABC[N:N+4]} = vdotq_lane_u32(vpacc${M}x${ABC[N:N+4]}, vb${ABC[K:K+4]}x${ABC[N:N+4]}, va${M}x01234567, ${K//4}); 104 105 k -= 8 * sizeof(uint8_t); 106 } 107 // Handle up to 4 final positions of `k` 108 if XNN_UNLIKELY(k != 0) { 109 // Load a ${MR}x4 block of activations. 110 $for M in range(MR): 111 const uint8x8_t va${M}x01234567 = vreinterpret_u8_u32(vld1_lane_u32((const void*) a${M}, vmov_n_u32(0), 0)); a${M} += 4; 112 113 // Load a 4x${NR} block of weights. 114 $for N in range(0, NR, 4): 115 const uint8x16_t vb0123x${ABC[N:N+4]} = vld1q_u8(w); w = (const void*) ((const uint8_t*) w + 16); 116 117 // Multiply-accumulate: ${MR}x4 * 4x${NR} --> ${MR}x${NR}. 118 $for M in range(MR): 119 vnacc${M} = vdot_u32(vnacc${M}, va_zero_point, va${M}x01234567); 120 $for N in range(0, NR, 4): 121 vpacc${M}x${ABC[N:N+4]} = vdotq_lane_u32(vpacc${M}x${ABC[N:N+4]}, vb0123x${ABC[N:N+4]}, va${M}x01234567, 0); 122 } 123 p -= ${MR} * sizeof(void*); 124 } while (p != 0); 125 126 // Subtract zero point from accumulators. 127 $for M in range(0, MR): 128 vnacc${M} = vpadd_u32(vnacc${M}, vnacc${M}); 129 const uint32x4_t vnacc${M}x0123 = vcombine_u32(vnacc${M}, vnacc${M}); 130 $for N in range(0, NR, 4): 131 int32x4_t vacc${M}x${ABC[N:N+4]} = vreinterpretq_s32_u32(vsubq_u32(vpacc${M}x${ABC[N:N+4]}, vnacc${M}x0123)); 132 133 $if REQUANTIZATION == "RNDNU": 134 const int32x4_t vright_pre_shift = vld1q_dup_s32(¶ms->${PARAMS_STRUCT}.right_pre_shift); 135 const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->${PARAMS_STRUCT}.multiplier); 136 const int32x4_t vright_post_shift = vld1q_dup_s32(¶ms->${PARAMS_STRUCT}.right_post_shift); 137 138 $for M in range(MR): 139 $for N in range(0, NR, 4): 140 vacc${M}x${ABC[N:N+4]} = vshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_pre_shift); 141 142 $for M in range(MR): 143 $for N in range(0, NR, 4): 144 vacc${M}x${ABC[N:N+4]} = vqdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier); 145 146 $for M in range(MR): 147 $for N in range(0, NR, 4): 148 vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_post_shift); 149 $elif REQUANTIZATION == "FP32": 150 $for M in range(MR): 151 $for N in range(0, NR, 4): 152 float32x4_t vfpacc${M}x${ABC[N:N+4]} = vcvtq_f32_s32(vacc${M}x${ABC[N:N+4]}); 153 154 const float32x4_t vscale = vld1q_dup_f32(¶ms->${PARAMS_STRUCT}.scale); 155 $for M in range(MR): 156 $for N in range(0, NR, 4): 157 vfpacc${M}x${ABC[N:N+4]} = vmulq_f32(vfpacc${M}x${ABC[N:N+4]}, vscale); 158 159 $for M in range(MR): 160 $for N in range(0, NR, 4): 161 vacc${M}x${ABC[N:N+4]} = vcvtnq_s32_f32(vfpacc${M}x${ABC[N:N+4]}); 162 163 const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->${PARAMS_STRUCT}.output_zero_point); 164#if XNN_ARCH_ARM64 165 $for M in range(MR): 166 $for N in range(0, NR, 8): 167 const int16x8_t vacc${M}x${ABC[N:N+8]} = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc${M}x${ABC[N:N+4]}), vacc${M}x${ABC[N+4:N+8]}), voutput_zero_point); 168 169 $for M in range(MR): 170 $for N in range(0, NR, 16): 171 $if N + 8 < NR: 172 uint8x16_t vout${M}x${ABC[N:N+16]} = vqmovun_high_s16(vqmovun_s16(vacc${M}x${ABC[N:N+8]}), vacc${M}x${ABC[N+8:N+16]}); 173 $elif M % 2 == 1: 174 uint8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vqmovun_high_s16(vqmovun_s16(vacc${M-1}x${ABC[N:N+8]}), vacc${M}x${ABC[N:N+8]}); 175 $elif M + 1 == MR: 176 uint8x8_t vout${M}x${ABC[N:N+8]} = vqmovun_s16(vacc${M}x${ABC[N:N+8]}); 177#else 178 $for M in range(MR): 179 $for N in range(0, NR, 8): 180 const int16x8_t vacc${M}x${ABC[N:N+8]} = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc${M}x${ABC[N:N+4]}), vqmovn_s32(vacc${M}x${ABC[N+4:N+8]})), voutput_zero_point); 181 182 $for M in range(MR): 183 $for N in range(0, NR, 16): 184 $if N + 8 < NR: 185 uint8x16_t vout${M}x${ABC[N:N+16]} = vcombine_u8(vqmovun_s16(vacc${M}x${ABC[N:N+8]}), vqmovun_s16(vacc${M}x${ABC[N+8:N+16]})); 186 $elif M % 2 == 1: 187 uint8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vcombine_u8(vqmovun_s16(vacc${M-1}x${ABC[N:N+8]}), vqmovun_s16(vacc${M}x${ABC[N:N+8]})); 188 $elif M + 1 == MR: 189 uint8x8_t vout${M}x${ABC[N:N+8]} = vqmovun_s16(vacc${M}x${ABC[N:N+8]}); 190#endif 191 $if NR == 8 and MR == 1: 192 const uint8x8_t voutput_min = vld1_dup_u8(¶ms->${PARAMS_STRUCT}.output_min); 193 const uint8x8_t voutput_max = vld1_dup_u8(¶ms->${PARAMS_STRUCT}.output_max); 194 $else: 195 const uint8x16_t voutput_min = vld1q_dup_u8(¶ms->${PARAMS_STRUCT}.output_min); 196 const uint8x16_t voutput_max = vld1q_dup_u8(¶ms->${PARAMS_STRUCT}.output_max); 197 198 $for M in range(MR): 199 $for N in range(0, NR, 16): 200 $if N + 8 < NR: 201 vout${M}x${ABC[N:N+16]} = vmaxq_u8(vout${M}x${ABC[N:N+16]}, voutput_min); 202 $elif M % 2 == 1: 203 vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vmaxq_u8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]}, voutput_min); 204 $elif M + 1 == MR: 205 $if NR == 8 and MR == 1: 206 vout${M}x${ABC[N:N+8]} = vmax_u8(vout${M}x${ABC[N:N+8]}, voutput_min); 207 $else: 208 vout${M}x${ABC[N:N+8]} = vmax_u8(vout${M}x${ABC[N:N+8]}, vget_low_u8(voutput_min)); 209 210 $for M in range(MR): 211 $for N in range(0, NR, 16): 212 $if N + 8 < NR: 213 vout${M}x${ABC[N:N+16]} = vminq_u8(vout${M}x${ABC[N:N+16]}, voutput_max); 214 $elif M % 2 == 1: 215 vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vminq_u8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]}, voutput_max); 216 $elif M + 1 == MR: 217 $if NR == 8 and MR == 1: 218 vout${M}x${ABC[N:N+8]} = vmin_u8(vout${M}x${ABC[N:N+8]}, voutput_max); 219 $else: 220 vout${M}x${ABC[N:N+8]} = vmin_u8(vout${M}x${ABC[N:N+8]}, vget_low_u8(voutput_max)); 221 222 if (nc >= ${NR}) { 223 $for M in reversed(range(MR)): 224 $for N in range(0, NR, 16): 225 $if N + 8 < NR: 226 vst1q_u8(c${M} + ${N}, vout${M}x${ABC[N:N+16]}); 227 $elif M % 2 == 1: 228 vst1_u8(c${M} + ${N}, vget_high_u8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]})); 229 vst1_u8(c${M-1} + ${N}, vget_low_u8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]})); 230 $elif M + 1 == MR: 231 vst1_u8(c${M} + ${N}, vout${M}x${ABC[N:N+8]}); 232 233 $for M in reversed(range(MR)): 234 c${M} = (uint8_t*) ((uintptr_t) c${M} + cn_stride); 235 236 a = (const uint8_t**restrict) ((uintptr_t) a - ks); 237 238 nc -= ${NR}; 239 } else { 240 $if NR == 32: 241 if (nc & 16) { 242 $for M in reversed(range(MR)): 243 vst1q_u8(c${M}, vout${M}x${ABC[0:16]}); c${M} += 16; 244 245 $for M in reversed(range(MR)): 246 vout${M}x${ABC[0:16]} = vout${M}x${ABC[16:32]}; 247 } 248 $if NR >= 16: 249 $for M in reversed(range(MR)): 250 $if M % 2 == 1: 251 uint8x16_t vout${M-1}x01234567_${M}x01234567 = vcombine_u8(vget_low_u8(vout${M-1}x0123456789ABCDEF), vget_low_u8(vout${M}x0123456789ABCDEF)); 252 $elif M + 1 == MR: 253 uint8x8_t vout${M}x01234567 = vget_low_u8(vout${M}x0123456789ABCDEF); 254 if (nc & 8) { 255 $for M in reversed(range(MR)): 256 $if M % 2 == 1: 257 vst1_u8(c${M}, vget_high_u8(vout${M-1}x01234567_${M}x01234567)); c${M} += 8; 258 vst1_u8(c${M-1}, vget_low_u8(vout${M-1}x01234567_${M}x01234567)); c${M-1} += 8; 259 $elif M + 1 == MR: 260 vst1_u8(c${M}, vout${M}x01234567); c${M} += 8; // This line 261 $for M in reversed(range(MR)): 262 $if M % 2 == 1: 263 vout${M-1}x01234567_${M}x01234567 = vcombine_u8(vget_high_u8(vout${M-1}x0123456789ABCDEF), vget_high_u8(vout${M}x0123456789ABCDEF)); 264 $elif M + 1 == MR: 265 vout${M}x01234567 = vget_high_u8(vout${M}x0123456789ABCDEF); 266 } 267 if (nc & 4) { 268 $for M in reversed(range(MR)): 269 $if M % 2 == 1: 270 vst1q_lane_u32((void*) c${M}, vreinterpretq_u32_u8(vout${M-1}x01234567_${M}x01234567), 2); c${M} += 4; 271 vst1q_lane_u32((void*) c${M-1}, vreinterpretq_u32_u8(vout${M-1}x01234567_${M}x01234567), 0); c${M-1} += 4; 272 $elif M + 1 == MR: 273 vst1_lane_u32((void*) c${M}, vreinterpret_u32_u8(vout${M}x01234567), 0); c${M} += 4; 274 $for M in reversed(range(MR)): 275 $if M % 2 == 1: 276 vout${M-1}x01234567_${M}x01234567 = vextq_u8(vout${M-1}x01234567_${M}x01234567, vout${M-1}x01234567_${M}x01234567, 4); 277 $elif M + 1 == MR: 278 vout${M}x01234567 = vext_u8(vout${M}x01234567, vout${M}x01234567, 4); 279 } 280 if (nc & 2) { 281 $for M in reversed(range(MR)): 282 $if M % 2 == 1: 283 vst1q_lane_u16((void*) c${M}, vreinterpretq_u16_u8(vout${M-1}x01234567_${M}x01234567), 4); c${M} += 2; 284 vst1q_lane_u16((void*) c${M-1}, vreinterpretq_u16_u8(vout${M-1}x01234567_${M}x01234567), 0); c${M-1} += 2; 285 $elif M + 1 == MR: 286 vst1_lane_u16((void*) c${M}, vreinterpret_u16_u8(vout${M}x01234567), 0); c${M} += 2; 287 $for M in reversed(range(MR)): 288 $if M % 2 == 1: 289 vout${M-1}x01234567_${M}x01234567 = vextq_u8(vout${M-1}x01234567_${M}x01234567, vout${M-1}x01234567_${M}x01234567, 2); 290 $elif M + 1 == MR: 291 vout${M}x01234567 = vext_u8(vout${M}x01234567, vout${M}x01234567, 2); 292 } 293 if (nc & 1) { 294 $for M in reversed(range(MR)): 295 $if M % 2 == 1: 296 vst1q_lane_u8(c${M}, vout${M-1}x01234567_${M}x01234567, 8); 297 vst1q_lane_u8(c${M-1}, vout${M-1}x01234567_${M}x01234567, 0); 298 $elif M + 1 == MR: 299 vst1_lane_u8(c${M}, vout${M}x01234567, 0); 300 } 301 302 nc = 0; 303 } 304 } while (nc != 0); 305} 306