1// Copyright 2020 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert REQUANTIZATION == "FP32" 7$assert DATATYPE in ["QC8", "QS8", "QU8"] 8$assert VARIANT in ["LD128", "EXTENDED"] 9$assert MR <= 4 10#include <assert.h> 11 12#include <immintrin.h> 13 14#include <xnnpack/gemm.h> 15#include <xnnpack/intrinsics-polyfill.h> 16#include <xnnpack/math.h> 17 18 19$PARAMS_STRUCT = "avx2" if DATATYPE == "QC8" else "fp32_avx2" 20$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else "" 21$PARAMS_UNION = "xnn_qs8_minmax_params" if DATATYPE == "QC8" else "xnn_%s_conv_minmax_params" % DATATYPE.lower() 22$XINT8_T = "uint8_t" if DATATYPE == "QU8" else "int8_t" 23void xnn_${DATATYPE.lower()}_gemm${GEMM_SUFFIX}_minmax_fp32_ukernel_${MR}x8c8__avx2( 24 size_t mr, 25 size_t nc, 26 size_t kc, 27 const ${XINT8_T}* restrict a, 28 size_t a_stride, 29 const void* restrict w, 30 ${XINT8_T}* restrict c, 31 size_t cm_stride, 32 size_t cn_stride, 33 const union ${PARAMS_UNION} params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS 34{ 35 assert(mr != 0); 36 assert(mr <= ${MR}); 37 assert(nc != 0); 38 assert(kc != 0); 39 assert(kc % sizeof(${XINT8_T}) == 0); 40 assert(a != NULL); 41 assert(w != NULL); 42 assert(c != NULL); 43 44 kc = round_up_po2(kc, 8); 45 const ${XINT8_T}* a0 = a; 46 ${XINT8_T}* c0 = c; 47 $for M in range(1, MR): 48 const ${XINT8_T}* a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M-1} + a_stride); 49 ${XINT8_T}* c${M} = (${XINT8_T}*) ((uintptr_t) c${M-1} + cm_stride); 50 $if M % 2 == 0: 51 if XNN_UNPREDICTABLE(mr <= ${M}) { 52 a${M} = a${M-1}; 53 c${M} = c${M-1}; 54 } 55 $elif M + 1 == MR: 56 if XNN_UNPREDICTABLE(mr != ${M+1}) { 57 a${M} = a${M-1}; 58 c${M} = c${M-1}; 59 } 60 $else: 61 if XNN_UNPREDICTABLE(mr < ${M+1}) { 62 a${M} = a${M-1}; 63 c${M} = c${M-1}; 64 } 65 66 do { 67 const __m128i vbias0x0 = _mm_loadu_si32(w); 68 const __m128i vbias0x1 = _mm_loadu_si32((const int32_t*) w + 1); 69 __m256i vacc0x01 = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x0), vbias0x1, 1); 70 $for N in range(2, 8, 2): 71 const __m128i vbias0x${N} = _mm_loadu_si32((const int32_t*) w + ${N}); 72 const __m128i vbias0x${N+1} = _mm_loadu_si32((const int32_t*) w + ${N+1}); 73 __m256i vacc0x${N}${N+1} = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x${N}), vbias0x${N+1}, 1); 74 $for M in range(1, MR): 75 $for N in range(0, 8, 2): 76 __m256i vacc${M}x${N}${N+1} = vacc0x${N}${N+1}; 77 w = (const void*) ((const int32_t*) w + 8); 78 79 size_t k = 0; 80 $if DATATYPE == "QU8": 81 const __m256i vb_zero_point = _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.kernel_zero_point); 82 while (k < kc) { 83 $for M in range(MR): 84 const __m128i va${M} = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a${M})); 85 $if DATATYPE == "QU8": 86 const __m256i vxa${M} = _mm256_cvtepu8_epi16(va${M}); 87 $else: 88 const __m256i vxa${M} = _mm256_cvtepi8_epi16(va${M}); 89 a${M} += 8; 90 91 $for N in range(0, 8, 2): 92 $if VARIANT == "EXTENDED": 93 $if N == 0: 94 const __m256i vxb${N}${N+1} = _mm256_load_si256((const __m256i*) w); 95 $else: 96 const __m256i vxb${N}${N+1} = _mm256_load_si256((const __m256i*) ((const int16_t*) w + ${N * 8})); 97 $else: 98 $if N == 0: 99 const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) w); 100 $else: 101 const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) ((const ${XINT8_T}*) w + ${N * 8})); 102 $if DATATYPE == "QU8": 103 const __m256i vxb${N}${N+1} = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb${N}${N+1}), vb_zero_point); 104 $else: 105 const __m256i vxb${N}${N+1} = _mm256_cvtepi8_epi16(vb${N}${N+1}); 106 107 $for M in range(MR): 108 vacc${M}x${N}${N+1} = _mm256_add_epi32(vacc${M}x${N}${N+1}, _mm256_madd_epi16(vxa${M}, vxb${N}${N+1})); 109 110 $if VARIANT == "EXTENDED": 111 w = (const void*) ((const int16_t*) w + 64); 112 $else: 113 w = (const void*) ((const ${XINT8_T}*) w + 64); 114 k += 8 * sizeof(${XINT8_T}); 115 } 116 117 $for M in range(MR): 118 const __m256i vacc${M}x0213 = _mm256_hadd_epi32(vacc${M}x01, vacc${M}x23); 119 const __m256i vacc${M}x4657 = _mm256_hadd_epi32(vacc${M}x45, vacc${M}x67); 120 121 $for M in range(MR): 122 const __m256i vacc${M}x02461357 = _mm256_hadd_epi32(vacc${M}x0213, vacc${M}x4657); 123 124 const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0); 125 $for M in range(MR): 126 __m256i vacc${M}x01234567 = _mm256_permutevar8x32_epi32(vacc${M}x02461357, vpermute_mask); 127 128 $for M in range(MR): 129 __m256 vscaled${M}x01234567 = _mm256_cvtepi32_ps(vacc${M}x01234567); 130 131 $if DATATYPE == "QC8": 132 const __m256 vscale01234567 = _mm256_load_ps(w); 133 w = (const void*) ((const float*) w + 8); 134 $for M in range(MR): 135 vscaled${M}x01234567 = _mm256_mul_ps(vscaled${M}x01234567, vscale01234567); 136 $else: 137 const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale); 138 $for M in range(MR): 139 vscaled${M}x01234567 = _mm256_mul_ps(vscaled${M}x01234567, vscale); 140 141 const __m256 voutput_max_less_zero_point = _mm256_load_ps(params->${PARAMS_STRUCT}.output_max_less_zero_point); 142 $for M in range(MR): 143 vscaled${M}x01234567 = _mm256_min_ps(vscaled${M}x01234567, voutput_max_less_zero_point); 144 145 $for M in range(MR): 146 vacc${M}x01234567 = _mm256_cvtps_epi32(vscaled${M}x01234567); 147 148 const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_zero_point); 149 $for M in range(0, MR, 2): 150 __m256i vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc${M}x01234567, vacc${min(M+1, MR-1)}x01234567), voutput_zero_point); 151 152 $for M in range(0, MR, 2): 153 vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_permute4x64_epi64(vacc${M}${min(M+1, MR-1)}x01234567, _MM_SHUFFLE(3, 1, 2, 0)); 154 155 $if DATATYPE == "QU8": 156 $if MR > 2: 157 __m256i vout = _mm256_packus_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567); 158 $else: 159 __m256i vout = _mm256_packus_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567); 160 161 vout = _mm256_max_epu8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_min)); 162 $else: 163 $if MR > 2: 164 __m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567); 165 $else: 166 __m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567); 167 168 vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_min)); 169 170 __m128i vout_lo = _mm256_castsi256_si128(vout); 171 __m128i vout_hi = _mm256_extracti128_si256(vout, 1); 172 173 if (nc >= 8) { 174 _mm_storel_epi64((__m128i*) c0, vout_lo); 175 $if MR > 1: 176 _mm_storel_epi64((__m128i*) c1, vout_hi); 177 $if MR > 2: 178 _mm_storeh_pi((__m64*) c2, _mm_castsi128_ps(vout_lo)); 179 $if MR > 3: 180 _mm_storeh_pi((__m64*) c3, _mm_castsi128_ps(vout_hi)); 181 182 $for M in range(MR): 183 c${M} = (${XINT8_T}*) ((uintptr_t) c${M} + cn_stride); 184 185 $for M in range(MR): 186 a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M} - kc); 187 188 nc -= 8; 189 } else { 190 if (nc & 4) { 191 _mm_storeu_si32(c0, vout_lo); 192 $if MR > 1: 193 _mm_storeu_si32(c1, vout_hi); 194 $if MR > 2: 195 *((uint32_t*) c2) = (uint32_t) _mm_extract_epi32(vout_lo, 2); 196 $if MR > 3: 197 *((uint32_t*) c3) = (uint32_t) _mm_extract_epi32(vout_hi, 2); 198 199 $for M in range(MR): 200 c${M} += 4; 201 202 vout_lo = _mm_srli_epi64(vout_lo, 32); 203 vout_hi = _mm_srli_epi64(vout_hi, 32); 204 } 205 if (nc & 2) { 206 *((uint16_t*) c0) = (uint16_t) _mm_extract_epi16(vout_lo, 0); 207 $if MR > 1: 208 *((uint16_t*) c1) = (uint16_t) _mm_extract_epi16(vout_hi, 0); 209 $if MR > 2: 210 *((uint16_t*) c2) = (uint16_t) _mm_extract_epi16(vout_lo, 4); 211 $if MR > 3: 212 *((uint16_t*) c3) = (uint16_t) _mm_extract_epi16(vout_hi, 4); 213 214 $for M in range(MR): 215 c${M} += 2; 216 217 vout_lo = _mm_srli_epi32(vout_lo, 16); 218 vout_hi = _mm_srli_epi32(vout_hi, 16); 219 } 220 if (nc & 1) { 221 *c0 = (${XINT8_T}) _mm_extract_epi8(vout_lo, 0); 222 $if MR > 1: 223 *c1 = (${XINT8_T}) _mm_extract_epi8(vout_hi, 0); 224 $if MR > 2: 225 *c2 = (${XINT8_T}) _mm_extract_epi8(vout_lo, 8); 226 $if MR > 3: 227 *c3 = (${XINT8_T}) _mm_extract_epi8(vout_hi, 8); 228 } 229 230 nc = 0; 231 } 232 } while (nc != 0); 233} 234