1// Copyright 2020 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert REQUANTIZATION == "FP32" 7$assert DATATYPE in ["QC8", "QS8", "QU8"] 8$assert MR <= 4 9#include <assert.h> 10 11#include <immintrin.h> 12 13#include <xnnpack/igemm.h> 14#include <xnnpack/intrinsics-polyfill.h> 15#include <xnnpack/math.h> 16 17 18$PARAMS_UNION = "xnn_qs8_minmax_params" if DATATYPE == "QC8" else "xnn_%s_conv_minmax_params" % DATATYPE.lower() 19$PARAMS_STRUCT = "avx2" if DATATYPE == "QC8" else "fp32_avx2" 20$XINT8_T = "uint8_t" if DATATYPE == "QU8" else "int8_t" 21void xnn_${DATATYPE.lower()}_igemm_minmax_fp32_ukernel_${MR}x8c8__avx2( 22 size_t mr, 23 size_t nc, 24 size_t kc, 25 size_t ks, 26 const ${XINT8_T}** restrict a, 27 const void* restrict w, 28 ${XINT8_T}* restrict c, 29 size_t cm_stride, 30 size_t cn_stride, 31 size_t a_offset, 32 const ${XINT8_T}* zero, 33 const union ${PARAMS_UNION} params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS 34{ 35 assert(mr != 0); 36 assert(mr <= ${MR}); 37 assert(nc != 0); 38 assert(kc != 0); 39 assert(ks != 0); 40 assert(ks % (${MR} * sizeof(void*)) == 0); 41 assert(a_offset % sizeof(${XINT8_T}) == 0); 42 assert(a != NULL); 43 assert(w != NULL); 44 assert(c != NULL); 45 46 kc = round_up_po2(kc, 8); 47 ${XINT8_T}* c0 = c; 48 $for M in range(1, MR): 49 ${XINT8_T}* c${M} = (${XINT8_T}*) ((uintptr_t) c${M-1} + cm_stride); 50 $if M % 2 == 0: 51 if XNN_UNPREDICTABLE(mr <= ${M}) { 52 c${M} = c${M-1}; 53 } 54 $elif M + 1 == MR: 55 if XNN_UNPREDICTABLE(mr != ${M+1}) { 56 c${M} = c${M-1}; 57 } 58 $else: 59 if XNN_UNPREDICTABLE(mr < ${M+1}) { 60 c${M} = c${M-1}; 61 } 62 63 do { 64 const __m128i vbias0x0 = _mm_loadu_si32(w); 65 const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1); 66 __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1); 67 $for N in range(2, 8, 2): 68 const __m128i vbias0x${N} = _mm_loadu_si32((const int32_t*) w + ${N}); 69 const __m128i vbias0x${N+1} = _mm_loadu_si32((const int32_t*) w + ${N+1}); 70 __m256i vacc0x${N}${N+1} = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x${N}), vbias0x${N+1}, 1); 71 $for M in range(1, MR): 72 $for N in range(0, 8, 2): 73 __m256i vacc${M}x${N}${N+1} = vacc0x${N}${N+1}; 74 w = (const void*) ((const int32_t*) w + 8); 75 76 size_t p = ks; 77 $if DATATYPE == "QU8": 78 const __m256i vb_zero_point = _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.kernel_zero_point); 79 do { 80 $for M in range(MR): 81 const ${XINT8_T}* restrict a${M} = a[${M}]; 82 if XNN_UNPREDICTABLE(a${M} != zero) { 83 a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M} + a_offset); 84 } 85 a += ${MR}; 86 87 size_t k = 0; 88 while (k < kc) { 89 $for M in range(MR): 90 const __m128i va${M} = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a${M})); 91 $if DATATYPE == "QU8": 92 const __m256i vxa${M} = _mm256_cvtepu8_epi16(va${M}); 93 $else: 94 const __m256i vxa${M} = _mm256_cvtepi8_epi16(va${M}); 95 a${M} += 8; 96 97 $for N in range(0, 8, 2): 98 $if N == 0: 99 const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) w); 100 $else: 101 const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) ((const ${XINT8_T}*) w + ${N * 8})); 102 $if DATATYPE == "QU8": 103 const __m256i vxb${N}${N+1} = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb${N}${N+1}), vb_zero_point); 104 $else: 105 const __m256i vxb${N}${N+1} = _mm256_cvtepi8_epi16(vb${N}${N+1}); 106 107 $for M in range(MR): 108 vacc${M}x${N}${N+1} = _mm256_add_epi32(vacc${M}x${N}${N+1}, _mm256_madd_epi16(vxa${M}, vxb${N}${N+1})); 109 110 w = (const void*) ((const ${XINT8_T}*) w + 64); 111 k += 8 * sizeof(${XINT8_T}); 112 } 113 p -= ${MR} * sizeof(void*); 114 } while (p != 0); 115 116 $for M in range(MR): 117 const __m256i vacc${M}x0213 = _mm256_hadd_epi32(vacc${M}x01, vacc${M}x23); 118 const __m256i vacc${M}x4657 = _mm256_hadd_epi32(vacc${M}x45, vacc${M}x67); 119 120 $for M in range(MR): 121 const __m256i vacc${M}x02461357 = _mm256_hadd_epi32(vacc${M}x0213, vacc${M}x4657); 122 123 const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0); 124 $for M in range(MR): 125 __m256i vacc${M}x01234567 = _mm256_permutevar8x32_epi32(vacc${M}x02461357, vpermute_mask); 126 127 $for M in range(MR): 128 __m256 vscaled${M}x01234567 = _mm256_cvtepi32_ps(vacc${M}x01234567); 129 130 $if DATATYPE == "QC8": 131 const __m256 vscale01234567 = _mm256_load_ps(w); 132 w = (const void*) ((const float*) w + 8); 133 $for M in range(MR): 134 vscaled${M}x01234567 = _mm256_mul_ps(vscaled${M}x01234567, vscale01234567); 135 $else: 136 const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale); 137 $for M in range(MR): 138 vscaled${M}x01234567 = _mm256_mul_ps(vscaled${M}x01234567, vscale); 139 140 const __m256 voutput_max_less_zero_point = _mm256_load_ps(params->${PARAMS_STRUCT}.output_max_less_zero_point); 141 $for M in range(MR): 142 vscaled${M}x01234567 = _mm256_min_ps(vscaled${M}x01234567, voutput_max_less_zero_point); 143 144 $for M in range(MR): 145 vacc${M}x01234567 = _mm256_cvtps_epi32(vscaled${M}x01234567); 146 147 const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_zero_point); 148 $for M in range(0, MR, 2): 149 __m256i vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc${M}x01234567, vacc${min(M+1, MR-1)}x01234567), voutput_zero_point); 150 151 $for M in range(0, MR, 2): 152 vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_permute4x64_epi64(vacc${M}${min(M+1, MR-1)}x01234567, _MM_SHUFFLE(3, 1, 2, 0)); 153 154 $if DATATYPE == "QU8": 155 $if MR > 2: 156 __m256i vout = _mm256_packus_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567); 157 $else: 158 __m256i vout = _mm256_packus_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567); 159 160 vout = _mm256_max_epu8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_min)); 161 $else: 162 $if MR > 2: 163 __m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567); 164 $else: 165 __m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567); 166 167 vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_min)); 168 169 __m128i vout_lo = _mm256_castsi256_si128(vout); 170 __m128i vout_hi = _mm256_extracti128_si256(vout, 1); 171 172 if (nc >= 8) { 173 $if MR > 3: 174 _mm_storeh_pi((__m64*) c3, _mm_castsi128_ps(vout_hi)); 175 $if MR > 2: 176 _mm_storeh_pi((__m64*) c2, _mm_castsi128_ps(vout_lo)); 177 $if MR > 1: 178 _mm_storel_epi64((__m128i*) c1, vout_hi); 179 _mm_storel_epi64((__m128i*) c0, vout_lo); 180 181 $for M in reversed(range(MR)): 182 c${M} = (${XINT8_T}*) ((uintptr_t) c${M} + cn_stride); 183 184 a = (const ${XINT8_T}**restrict) ((uintptr_t) a - ks); 185 186 nc -= 8; 187 } else { 188 if (nc & 4) { 189 $if MR > 3: 190 *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout_hi, 2); 191 $if MR > 2: 192 *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout_lo, 2); 193 $if MR > 1: 194 _mm_storeu_si32(c1, vout_hi); 195 _mm_storeu_si32(c0, vout_lo); 196 197 $for M in reversed(range(MR)): 198 c${M} += 4; 199 200 vout_lo = _mm_srli_epi64(vout_lo, 32); 201 vout_hi = _mm_srli_epi64(vout_hi, 32); 202 } 203 if (nc & 2) { 204 $if MR > 3: 205 *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout_hi, 4); 206 $if MR > 2: 207 *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout_lo, 4); 208 $if MR > 1: 209 *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout_hi, 0); 210 *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0); 211 212 $for M in reversed(range(MR)): 213 c${M} += 2; 214 215 vout_lo = _mm_srli_epi32(vout_lo, 16); 216 vout_hi = _mm_srli_epi32(vout_hi, 16); 217 } 218 if (nc & 1) { 219 $if MR > 3: 220 *c3 = (${XINT8_T}) _mm_extract_epi8(vout_hi, 8); 221 $if MR > 2: 222 *c2 = (${XINT8_T}) _mm_extract_epi8(vout_lo, 8); 223 $if MR > 1: 224 *c1 = (${XINT8_T}) _mm_extract_epi8(vout_hi, 0); 225 *c0 = (${XINT8_T}) _mm_extract_epi8(vout_lo, 0); 226 } 227 228 nc = 0; 229 } 230 } while (nc != 0); 231} 232