1// Copyright 2019 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert BATCH_TILE % 4 == 0 7$assert BATCH_TILE >= 4 8$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 9$assert OP in ["ADD", "DIV", "RDIV", "MAX", "MIN", "MUL", "SUB", "RSUB", "SQRDIFF"] 10$assert ACTIVATION in ["LINEAR", "MINMAX"] 11#include <assert.h> 12 13#include <xmmintrin.h> 14 15#include <xnnpack/common.h> 16#include <xnnpack/intrinsics-polyfill.h> 17#include <xnnpack/vbinary.h> 18 19 20$_MM_OP_PS = { 21$ "ADD": lambda x: "_mm_add_ps(%s, vb)" % x, 22$ "DIV": lambda x: "_mm_div_ps(%s, vb)" % x, 23$ "RDIV": lambda x: "_mm_div_ps(vb, %s)" % x, 24$ "MAX": lambda x: "_mm_max_ps(%s, vb)" % x, 25$ "MIN": lambda x: "_mm_min_ps(%s, vb)" % x, 26$ "MUL": lambda x: "_mm_mul_ps(%s, vb)" % x, 27$ "SUB": lambda x: "_mm_sub_ps(%s, vb)" % x, 28$ "RSUB": lambda x: "_mm_sub_ps(vb, %s)" % x, 29$ "SQRDIFF": lambda x: "_mm_sub_ps(%s, vb)" % x, 30$}[OP] 31$SUFFIX = {"LINEAR": "", "MINMAX": "_minmax"}[ACTIVATION] 32$PARAMS = {"LINEAR": "xnn_f32_default_params", "MINMAX": "xnn_f32_minmax_params"}[ACTIVATION] 33void xnn_f32_v${OP.lower()}c${SUFFIX}_ukernel__sse_x${BATCH_TILE}( 34 size_t n, 35 const float* a, 36 const float* b, 37 float* y, 38 const union ${PARAMS} params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS 39{ 40 assert(n != 0); 41 assert(n % sizeof(float) == 0); 42 assert(a != NULL); 43 assert(b != NULL); 44 assert(y != NULL); 45 46 $if ACTIVATION == "MINMAX": 47 const __m128 vy_min = _mm_load_ps(params->sse.min); 48 const __m128 vy_max = _mm_load_ps(params->sse.max); 49 50 const __m128 vb = _mm_load1_ps(b); 51 for (; n >= ${BATCH_TILE} * sizeof(float); n -= ${BATCH_TILE} * sizeof(float)) { 52 const __m128 va${ABC[0:4]} = _mm_loadu_ps(a); 53 $for N in range(4, BATCH_TILE, 4): 54 const __m128 va${ABC[N:N+4]} = _mm_loadu_ps(a + ${N}); 55 a += ${BATCH_TILE}; 56 57 $for N in range(0, BATCH_TILE, 4): 58 __m128 vy${ABC[N:N+4]} = ${_MM_OP_PS("va" + ABC[N:N+4])}; 59 60 $if OP == "SQRDIFF": 61 $for N in range(0, BATCH_TILE, 4): 62 vy${ABC[N:N+4]} = _mm_mul_ps(vy${ABC[N:N+4]}, vy${ABC[N:N+4]}); 63 64 $if ACTIVATION == "MINMAX": 65 $for N in range(0, BATCH_TILE, 4): 66 vy${ABC[N:N+4]} = _mm_max_ps(vy${ABC[N:N+4]}, vy_min); 67 68 $for N in range(0, BATCH_TILE, 4): 69 vy${ABC[N:N+4]} = _mm_min_ps(vy${ABC[N:N+4]}, vy_max); 70 71 _mm_storeu_ps(y, vy${ABC[0:4]}); 72 $for N in range(4, BATCH_TILE, 4): 73 _mm_storeu_ps(y + ${N}, vy${ABC[N:N+4]}); 74 y += ${BATCH_TILE}; 75 } 76 $if BATCH_TILE > 4: 77 for (; n >= 4 * sizeof(float); n -= 4 * sizeof(float)) { 78 const __m128 va0123 = _mm_loadu_ps(a); 79 a += 4; 80 81 __m128 vy0123 = ${_MM_OP_PS("va0123")}; 82 $if OP == "SQRDIFF": 83 vy0123 = _mm_mul_ps(vy0123, vy0123); 84 $if ACTIVATION == "MINMAX": 85 vy0123 = _mm_max_ps(vy0123, vy_min); 86 vy0123 = _mm_min_ps(vy0123, vy_max); 87 _mm_storeu_ps(y, vy0123); 88 y += 4; 89 } 90 if XNN_UNLIKELY(n != 0) { 91 const __m128 va0123 = _mm_loadu_ps(a); 92 93 __m128 vy0123 = ${_MM_OP_PS("va0123")}; 94 $if OP == "SQRDIFF": 95 vy0123 = _mm_mul_ps(vy0123, vy0123); 96 $if ACTIVATION == "MINMAX": 97 vy0123 = _mm_max_ps(vy0123, vy_min); 98 vy0123 = _mm_min_ps(vy0123, vy_max); 99 if (n & (2 * sizeof(float))) { 100 _mm_storel_pi((__m64*) y, vy0123); 101 vy0123 = _mm_movehl_ps(vy0123, vy0123); 102 y += 2; 103 } 104 if (n & (1 * sizeof(float))) { 105 _mm_store_ss(y, vy0123); 106 } 107 } 108} 109