1// Copyright 2020 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert DATATYPE in ["QS8", "QU8"] 7$assert BATCH_TILE % (16 if LD128 else 8) == 0 8$assert BATCH_TILE >= 8 9$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 10#include <assert.h> 11 12#include <arm_neon.h> 13 14#include <xnnpack/vadd.h> 15 16 17$XINT8_T = {"QS8": "int8_t", "QU8": "uint8_t"}[DATATYPE] 18$XINT8X8_T = {"QS8": "int8x8_t", "QU8": "uint8x8_t"}[DATATYPE] 19$XINT8X16_T = {"QS8": "int8x16_t", "QU8": "uint8x16_t"}[DATATYPE] 20$VLD1_X8 = {"QS8": "vld1_s8", "QU8": "vld1_u8"}[DATATYPE] 21$VLD1Q_X8 = {"QS8": "vld1q_s8", "QU8": "vld1q_u8"}[DATATYPE] 22$VLD1_DUP_X8 = {"QS8": "vld1_dup_s8", "QU8": "vld1_dup_u8"}[DATATYPE] 23$VLD1Q_DUP_X8 = {"QS8": "vld1q_dup_s8", "QU8": "vld1q_dup_u8"}[DATATYPE] 24$VST1_LANE_X8 = {"QS8": "vst1_lane_s8", "QU8": "vst1_lane_u8"}[DATATYPE] 25$VST1_X8 = {"QS8": "vst1_s8", "QU8": "vst1_u8"}[DATATYPE] 26$VST1Q_X8 = {"QS8": "vst1q_s8", "QU8": "vst1q_u8"}[DATATYPE] 27$VMIN_X8 = {"QS8": "vmin_s8", "QU8": "vmin_u8"}[DATATYPE] 28$VMAX_X8 = {"QS8": "vmax_s8", "QU8": "vmax_u8"}[DATATYPE] 29$VMINQ_X8 = {"QS8": "vminq_s8", "QU8": "vminq_u8"}[DATATYPE] 30$VMAXQ_X8 = {"QS8": "vmaxq_s8", "QU8": "vmaxq_u8"}[DATATYPE] 31$VQMOVXN_S16 = {"QS8": "vqmovn_s16", "QU8": "vqmovun_s16"}[DATATYPE] 32$VEXT_X8 = {"QS8": "vext_s8", "QU8": "vext_u8"}[DATATYPE] 33$VGET_LOW_X8 = {"QS8": "vget_low_s8", "QU8": "vget_low_u8"}[DATATYPE] 34$VCOMBINE_X8 = {"QS8": "vcombine_s8", "QU8": "vcombine_u8"}[DATATYPE] 35$VREINTERPRET_U32_X8 = {"QS8": "vreinterpret_u32_s8", "QU8": "vreinterpret_u32_u8"}[DATATYPE] 36$VREINTERPRET_U16_X8 = {"QS8": "vreinterpret_u16_s8", "QU8": "vreinterpret_u16_u8"}[DATATYPE] 37void xnn_${DATATYPE.lower()}_vaddc_minmax_ukernel__neon_${"ld128" if LD128 else "ld64"}_x${BATCH_TILE}( 38 size_t n, 39 const ${XINT8_T}* input_a, 40 const ${XINT8_T}* input_b, 41 ${XINT8_T}* output, 42 const union xnn_${DATATYPE.lower()}_add_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS 43{ 44 $if LD128: 45 #if XNN_ARCH_ARM64 46 const ${XINT8X16_T} va_zero_point = ${VLD1Q_DUP_X8}(¶ms->neon.a_zero_point); 47 #else 48 const ${XINT8X8_T} va_zero_point = ${VLD1_DUP_X8}(¶ms->neon.a_zero_point); 49 #endif 50 $else: 51 const ${XINT8X8_T} va_zero_point = ${VLD1_DUP_X8}(¶ms->neon.a_zero_point); 52 const int32x4_t va_multiplier = vld1q_dup_s32(¶ms->neon.a_multiplier); 53 const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift); 54 const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point); 55 $if BATCH_TILE >= 16: 56 const ${XINT8X16_T} voutput_min = ${VLD1Q_DUP_X8}(¶ms->neon.output_min); 57 const ${XINT8X16_T} voutput_max = ${VLD1Q_DUP_X8}(¶ms->neon.output_max); 58 $else: 59 const ${XINT8X8_T} voutput_min = ${VLD1_DUP_X8}(¶ms->neon.output_min); 60 const ${XINT8X8_T} voutput_max = ${VLD1_DUP_X8}(¶ms->neon.output_max); 61 62 const int32_t vxb = (int32_t) *input_b - (int32_t) params->neon.b_zero_point; 63 const int32_t vb = params->neon.b_multiplier; 64 const int32x4_t vbias = vdupq_n_s32(vxb * vb); 65 66 for (; n >= ${BATCH_TILE} * sizeof(${XINT8_T}); n -= ${BATCH_TILE} * sizeof(${XINT8_T})) { 67 $if LD128: 68 $for N in range(0, BATCH_TILE, 16): 69 const ${XINT8X16_T} va${ABC[N:N+16]} = ${VLD1Q_X8}(input_a); input_a += 16; 70 71 #if XNN_ARCH_ARM64 72 $for N in range(0, BATCH_TILE, 16): 73 $if DATATYPE == "QU8": 74 const int16x8_t vxa${ABC[N:N+8]} = vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(va${ABC[N:N+16]}), vget_low_u8(va_zero_point))); 75 const int16x8_t vxa${ABC[N+8:N+16]} = vreinterpretq_s16_u16(vsubl_high_u8(va${ABC[N:N+16]}, va_zero_point)); 76 $else: 77 const int16x8_t vxa${ABC[N:N+8]} = vsubl_s8(vget_low_s8(va${ABC[N:N+16]}), vget_low_s8(va_zero_point)); 78 const int16x8_t vxa${ABC[N+8:N+16]} = vsubl_high_s8(va${ABC[N:N+16]}, va_zero_point); 79 #else // !XNN_ARCH_ARM64 80 $for N in range(0, BATCH_TILE, 16): 81 $if DATATYPE == "QU8": 82 const int16x8_t vxa${ABC[N:N+8]} = vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(va${ABC[N:N+16]}), va_zero_point)); 83 const int16x8_t vxa${ABC[N+8:N+16]} = vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(va${ABC[N:N+16]}), va_zero_point)); 84 $else: 85 const int16x8_t vxa${ABC[N:N+8]} = vsubl_s8(vget_low_s8(va${ABC[N:N+16]}), va_zero_point); 86 const int16x8_t vxa${ABC[N+8:N+16]} = vsubl_s8(vget_high_s8(va${ABC[N:N+16]}), va_zero_point); 87 #endif // XNN_ARCH_ARM64 88 $else: 89 $for N in range(0, BATCH_TILE, 8): 90 const ${XINT8X8_T} va${ABC[N:N+8]} = ${VLD1_X8}(input_a); input_a += 8; 91 92 $for N in range(0, BATCH_TILE, 8): 93 $if DATATYPE == "QU8": 94 const int16x8_t vxa${ABC[N:N+8]} = vreinterpretq_s16_u16(vsubl_u8(va${ABC[N:N+8]}, va_zero_point)); 95 $else: 96 const int16x8_t vxa${ABC[N:N+8]} = vsubl_s8(va${ABC[N:N+8]}, va_zero_point); 97 98 $for N in range(0, BATCH_TILE, 8): 99 int32x4_t vacc${ABC[N:N+4]} = vmlaq_s32(vbias, vmovl_s16(vget_low_s16(vxa${ABC[N:N+8]})), va_multiplier); 100 int32x4_t vacc${ABC[N+4:N+8]} = vmlaq_s32(vbias, vmovl_s16(vget_high_s16(vxa${ABC[N:N+8]})), va_multiplier); 101 102 $for N in range(0, BATCH_TILE, 4): 103 vacc${ABC[N:N+4]} = vrshlq_s32(vacc${ABC[N:N+4]}, vright_shift); 104 105 $for N in range(0, BATCH_TILE, 8): 106 const int16x8_t vacc${ABC[N:N+8]} = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc${ABC[N:N+4]}), vqmovn_s32(vacc${ABC[N+4:N+8]})), voutput_zero_point); 107 108 $for N in range(0, BATCH_TILE, 16): 109 $if N + 8 < BATCH_TILE: 110 ${XINT8X16_T} vout${ABC[N:N+16]} = ${VCOMBINE_X8}(${VQMOVXN_S16}(vacc${ABC[N:N+8]}), ${VQMOVXN_S16}(vacc${ABC[N+8:N+16]})); 111 $else: 112 ${XINT8X8_T} vout${ABC[N:N+8]} = ${VQMOVXN_S16}(vacc${ABC[N:N+8]}); 113 114 $for N in range(0, BATCH_TILE, 16): 115 $if N + 8 < BATCH_TILE: 116 vout${ABC[N:N+16]} = ${VMAXQ_X8}(vout${ABC[N:N+16]}, voutput_min); 117 $elif BATCH_TILE >= 16: 118 vout${ABC[N:N+8]} = ${VMAX_X8}(vout${ABC[N:N+8]}, ${VGET_LOW_X8}(voutput_min)); 119 $else: 120 vout${ABC[N:N+8]} = ${VMAX_X8}(vout${ABC[N:N+8]}, voutput_min); 121 122 $for N in range(0, BATCH_TILE, 16): 123 $if N + 8 < BATCH_TILE: 124 vout${ABC[N:N+16]} = ${VMINQ_X8}(vout${ABC[N:N+16]}, voutput_max); 125 $elif BATCH_TILE >= 16: 126 vout${ABC[N:N+8]} = ${VMIN_X8}(vout${ABC[N:N+8]}, ${VGET_LOW_X8}(voutput_max)); 127 $else: 128 vout${ABC[N:N+8]} = ${VMIN_X8}(vout${ABC[N:N+8]}, voutput_max); 129 130 $for N in range(0, BATCH_TILE, 16): 131 $if N + 8 < BATCH_TILE: 132 ${VST1Q_X8}(output, vout${ABC[N:N+16]}); output += 16; 133 $else: 134 ${VST1_X8}(output, vout${ABC[N:N+8]}); output += 8; 135 } 136 if XNN_UNLIKELY(n != 0) { 137 ${"do " if BATCH_TILE > 8 else ""}{ 138 $if BATCH_TILE > 8: 139 const ${XINT8X8_T} va${ABC[0:8]} = ${VLD1_X8}(input_a); input_a += 8; 140 $else: 141 const ${XINT8X8_T} va${ABC[0:8]} = ${VLD1_X8}(input_a); 142 143 $if LD128: 144 $if DATATYPE == "QU8": 145 #if XNN_ARCH_ARM64 146 const int16x8_t vxa${ABC[0:8]} = vreinterpretq_s16_u16(vsubl_u8(va${ABC[0:8]}, vget_low_u8(va_zero_point))); 147 #else // !XNN_ARCH_ARM64 148 const int16x8_t vxa${ABC[0:8]} = vreinterpretq_s16_u16(vsubl_u8(va${ABC[0:8]}, va_zero_point)); 149 #endif 150 $else: 151 #if XNN_ARCH_ARM64 152 const int16x8_t vxa${ABC[0:8]} = vsubl_s8(va${ABC[0:8]}, vget_low_s8(va_zero_point)); 153 #else // !XNN_ARCH_ARM64 154 const int16x8_t vxa${ABC[0:8]} = vsubl_s8(va${ABC[0:8]}, va_zero_point); 155 #endif 156 $else: 157 $if DATATYPE == "QU8": 158 const int16x8_t vxa${ABC[0:8]} = vreinterpretq_s16_u16(vsubl_u8(va${ABC[0:8]}, va_zero_point)); 159 $else: 160 const int16x8_t vxa${ABC[0:8]} = vsubl_s8(va${ABC[0:8]}, va_zero_point); 161 162 int32x4_t vacc${ABC[0:4]} = vmlaq_s32(vbias, vmovl_s16(vget_low_s16(vxa${ABC[0:8]})), va_multiplier); 163 int32x4_t vacc${ABC[4:8]} = vmlaq_s32(vbias, vmovl_s16(vget_high_s16(vxa${ABC[0:8]})), va_multiplier); 164 165 vacc${ABC[0:4]} = vrshlq_s32(vacc${ABC[0:4]}, vright_shift); 166 vacc${ABC[4:8]} = vrshlq_s32(vacc${ABC[4:8]}, vright_shift); 167 168 const int16x8_t vacc${ABC[0:8]} = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc${ABC[0:4]}), vqmovn_s32(vacc${ABC[4:8]})), voutput_zero_point); 169 170 ${XINT8X8_T} vout${ABC[0:8]} = ${VQMOVXN_S16}(vacc${ABC[0:8]}); 171 $if BATCH_TILE >= 16: 172 vout${ABC[0:8]} = ${VMAX_X8}(vout${ABC[0:8]}, ${VGET_LOW_X8}(voutput_min)); 173 vout${ABC[0:8]} = ${VMIN_X8}(vout${ABC[0:8]}, ${VGET_LOW_X8}(voutput_max)); 174 $else: 175 vout${ABC[0:8]} = ${VMAX_X8}(vout${ABC[0:8]}, voutput_min); 176 vout${ABC[0:8]} = ${VMIN_X8}(vout${ABC[0:8]}, voutput_max); 177 178 $if BATCH_TILE > 8: 179 if XNN_LIKELY(n >= (8 * sizeof(${XINT8_T}))) { 180 ${VST1_X8}(output, vout${ABC[0:8]}); output += 8; 181 n -= 8 * sizeof(${XINT8_T}); 182 } else { 183 if (n & (4 * sizeof(${XINT8_T}))) { 184 vst1_lane_u32((void*) output, ${VREINTERPRET_U32_X8}(vout${ABC[0:8]}), 0); output += 4; 185 vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 4); 186 } 187 if (n & (2 * sizeof(${XINT8_T}))) { 188 vst1_lane_u16((void*) output, ${VREINTERPRET_U16_X8}(vout${ABC[0:8]}), 0); output += 2; 189 vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 2); 190 } 191 if (n & (1 * sizeof(${XINT8_T}))) { 192 ${VST1_LANE_X8}(output, vout${ABC[0:8]}, 0); 193 } 194 n = 0; 195 } 196 $else: 197 if (n & (4 * sizeof(${XINT8_T}))) { 198 vst1_lane_u32((void*) output, ${VREINTERPRET_U32_X8}(vout${ABC[0:8]}), 0); output += 4; 199 vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 4); 200 } 201 if (n & (2 * sizeof(${XINT8_T}))) { 202 vst1_lane_u16((void*) output, ${VREINTERPRET_U16_X8}(vout${ABC[0:8]}), 0); output += 2; 203 vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 2); 204 } 205 if (n & (1 * sizeof(${XINT8_T}))) { 206 ${VST1_LANE_X8}(output, vout${ABC[0:8]}, 0); 207 } 208 }${" while (n != 0);" if BATCH_TILE > 8 else ""} 209 } 210} 211