1// Copyright 2019 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert BATCH_TILE % 4 == 0 7$assert BATCH_TILE >= 4 8$ABC = "0123456789ABCDEFGHIJKLMN" 9$assert OP in ["ADD", "DIV", "RDIV", "MAX", "MIN", "MUL", "SUB", "RSUB"] 10#include <assert.h> 11 12#include <arm_neon.h> 13 14#include <xnnpack/common.h> 15#include <xnnpack/vbinary.h> 16 17 18$VOPQ_F32 = { 19$ "ADD": lambda x: "vaddq_f32(%s, vb)" % x, 20$ "DIV": lambda x: "vdivq_f32(%s, vb)" % x, 21$ "RDIV": lambda x: "vdivq_f32(vb, %s)" % x, 22$ "MAX": lambda x: "vmaxq_f32(%s, vb)" % x, 23$ "MIN": lambda x: "vminq_f32(%s, vb)" % x, 24$ "MUL": lambda x: "vmulq_f32(%s, vb)" % x, 25$ "SUB": lambda x: "vsubq_f32(%s, vb)" % x, 26$ "RSUB": lambda x: "vsubq_f32(vb, %s)" % x, 27$}[OP] 28void xnn_f32_v${OP.lower()}c_ukernel__neon_x${BATCH_TILE}( 29 size_t n, 30 const float* a, 31 const float* b, 32 float* y, 33 const union xnn_f32_output_params params[restrict static 1]) 34{ 35 assert(n != 0); 36 assert(n % sizeof(float) == 0); 37 38 const float32x4_t vy_min = vld1q_dup_f32(¶ms->scalar.min); 39 const float32x4_t vy_max = vld1q_dup_f32(¶ms->scalar.max); 40 41 const float32x4_t vb = vld1q_dup_f32(b); 42 for (; n >= ${BATCH_TILE} * sizeof(float); n -= ${BATCH_TILE} * sizeof(float)) { 43 $for N in range(0, BATCH_TILE, 4): 44 const float32x4_t va${ABC[N:N+4]} = vld1q_f32(a); a += 4; 45 46 $for N in range(0, BATCH_TILE, 4): 47 float32x4_t vy${ABC[N:N+4]} = ${VOPQ_F32("va" + ABC[N:N+4])}; 48 49 $for N in range(0, BATCH_TILE, 4): 50 vy${ABC[N:N+4]} = vmaxq_f32(vy${ABC[N:N+4]}, vy_min); 51 52 $for N in range(0, BATCH_TILE, 4): 53 vy${ABC[N:N+4]} = vminq_f32(vy${ABC[N:N+4]}, vy_max); 54 55 $for N in range(0, BATCH_TILE, 4): 56 vst1q_f32(y, vy${ABC[N:N+4]}); y += 4; 57 } 58 $if BATCH_TILE >= 4: 59 for (; n >= 4 * sizeof(float); n -= 4 * sizeof(float)) { 60 const float32x4_t va0123 = vld1q_f32(a); a += 4; 61 62 float32x4_t vy0123 = ${VOPQ_F32("va0123")}; 63 vy0123 = vmaxq_f32(vy0123, vy_min); 64 vy0123 = vminq_f32(vy0123, vy_max); 65 vst1q_f32(y, vy0123); y += 4; 66 } 67 if XNN_UNLIKELY(n != 0) { 68 const float32x4_t va0123 = vld1q_f32(a); 69 70 float32x4_t vy0123 = ${VOPQ_F32("va0123")}; 71 vy0123 = vmaxq_f32(vy0123, vy_min); 72 vy0123 = vminq_f32(vy0123, vy_max); 73 74 float32x2_t vy01 = vget_low_f32(vy0123); 75 if (n & (2 * sizeof(float))) { 76 vst1_f32(y, vy01); y += 2; 77 vy01 = vget_high_f32(vy0123); 78 } 79 if (n & (1 * sizeof(float))) { 80 vst1_lane_f32(y, vy01, 0); 81 } 82 } 83} 84