1// Copyright 2021 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert BATCH_TILE % 16 == 0 7$assert BATCH_TILE >= 16 8$SIMD_TILE = BATCH_TILE // 16 9#include <assert.h> 10 11#include <immintrin.h> 12 13#include <xnnpack/common.h> 14#include <xnnpack/intrinsics-polyfill.h> 15#include <xnnpack/vcvt.h> 16 17 18void xnn_f16_f32_vcvt_ukernel__avx512skx_x${BATCH_TILE}( 19 size_t n, 20 const void* input, 21 float* output, 22 const union xnn_f16_f32_cvt_params params[restrict XNN_MIN_ELEMENTS(1)]) 23{ 24 assert(n != 0); 25 assert(n % sizeof(uint16_t) == 0); 26 assert(input != NULL); 27 assert(output != NULL); 28 29 const uint16_t* i = (const uint16_t*) input; 30 $if BATCH_TILE > 16: 31 for (; n >= ${BATCH_TILE} * sizeof(uint16_t); n -= ${BATCH_TILE} * sizeof(uint16_t)) { 32 const __m512 vacc0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i*) i)); 33 $for N in range(1, SIMD_TILE): 34 const __m512 vacc${N} = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i*) (i + ${N * 16}))); 35 i += ${BATCH_TILE}; 36 37 _mm512_storeu_ps(output, vacc0); 38 $for N in range(1, SIMD_TILE): 39 _mm512_storeu_ps(output + ${N * 16}, vacc${N}); 40 output += ${BATCH_TILE}; 41 } 42 for (; n >= 16 * sizeof(uint16_t); n -= 16 * sizeof(uint16_t)) { 43 const __m512 vacc = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i*) i)); 44 i += 16; 45 46 _mm512_storeu_ps(output, vacc); 47 output += 16; 48 } 49 if XNN_UNLIKELY(n != 0) { 50 assert(n >= 1 * sizeof(uint16_t)); 51 assert(n <= 15 * sizeof(uint16_t)); 52 53 // Prepare mask for valid 32-bit elements (depends on n). 54 n >>= 1 /* log2(sizeof(uint16_t)) */; 55 const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1))); 56 57 const __m512 vacc = _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(vmask, i)); 58 59 _mm512_mask_storeu_ps(output, vmask, vacc); 60 } 61} 62