1// Copyright 2019 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert MR % 4 == 0 7$assert NR % 4 == 0 8$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 9#include <assert.h> 10 11#include <arm_neon.h> 12 13#include <xnnpack/common.h> 14#include <xnnpack/ppmm.h> 15 16 17void xnn_f32_ppmm_minmax_ukernel_${MR}x${NR}__${"neonfma" if FMA else "neon"}( 18 size_t mr, 19 size_t nc, 20 size_t kc, 21 const float*restrict a, 22 const float*restrict w, 23 float*restrict c, 24 size_t cm_stride, 25 size_t cn_stride, 26 const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) 27{ 28 assert(mr != 0); 29 assert(mr <= ${MR}); 30 assert(nc != 0); 31 assert(kc != 0); 32 assert(kc % sizeof(float) == 0); 33 34 float* c0 = c; 35 $for M in range(1, MR): 36 float* c${M} = (float*) ((uintptr_t) c${M-1} + cm_stride); 37 $if M % 2 == 0: 38 if XNN_UNPREDICTABLE(mr <= ${M}) { 39 c${M} = c${M-1}; 40 } 41 $elif M + 1 == MR: 42 if XNN_UNPREDICTABLE(mr != ${M+1}) { 43 c${M} = c${M-1}; 44 } 45 $else: 46 if XNN_UNPREDICTABLE(mr < ${M+1}) { 47 c${M} = c${M-1}; 48 } 49 50 do { 51 $for N in range(0, NR, 4): 52 float32x4_t vacc0x${ABC[N:N+4]} = vld1q_f32(w); w += 4; 53 $for M in range(1, MR): 54 $for N in range(0, NR, 4): 55 float32x4_t vacc${M}x${ABC[N:N+4]} = vacc0x${ABC[N:N+4]}; 56 57 size_t k = kc; 58 do { 59 $for M in range(0, MR, 4): 60 const float32x4_t va${ABC[M:M+4]} = vld1q_f32(a); a += 4; 61 62 $for N in range(0, NR, 4): 63 const float32x4_t vb${ABC[N:N+4]} = vld1q_f32(w); w += 4; 64 65 $if FMA: 66 #if XNN_ARCH_ARM64 67 $for N in range(0, NR, 4): 68 $for M in range(MR): 69 vacc${M}x${ABC[N:N+4]} = vfmaq_laneq_f32(vacc${M}x${ABC[N:N+4]}, vb${ABC[N:N+4]}, va${ABC[M&-4:4+M&-4]}, ${M % 4}); 70 #else 71 $for M in range(MR): 72 $VGET_PART_F32 = "vget_low_f32" if M % 4 < 2 else "vget_high_f32" 73 $MMMM = str(M) * 4 74 const float32x4_t va${MMMM} = vdupq_lane_f32(${VGET_PART_F32}(va${ABC[M&-4:4+M&-4]}), ${M % 2}); 75 76 $for N in range(0, NR, 4): 77 $for M in range(MR): 78 $MMMM = str(M) * 4 79 vacc${M}x${ABC[N:N+4]} = vfmaq_f32(vacc${M}x${ABC[N:N+4]}, va${MMMM}, vb${ABC[N:N+4]}); 80 #endif 81 $else: 82 $for N in range(0, NR, 4): 83 $for M in range(MR): 84 $VGET_PART_F32 = "vget_low_f32" if M % 4 < 2 else "vget_high_f32" 85 vacc${M}x${ABC[N:N+4]} = vmlaq_lane_f32(vacc${M}x${ABC[N:N+4]}, vb${ABC[N:N+4]}, ${VGET_PART_F32}(va${ABC[M&-4:4+M&-4]}), ${M % 2}); 86 87 k -= sizeof(float); 88 } while (k != 0); 89 90 const float32x4_t vmax = vld1q_dup_f32(¶ms->scalar.max); 91 $for N in range(0, NR, 4): 92 $for M in range(MR): 93 vacc${M}x${ABC[N:N+4]} = vminq_f32(vacc${M}x${ABC[N:N+4]}, vmax); 94 95 const float32x4_t vmin = vld1q_dup_f32(¶ms->scalar.min); 96 $for N in range(0, NR, 4): 97 $for M in range(MR): 98 vacc${M}x${ABC[N:N+4]} = vmaxq_f32(vacc${M}x${ABC[N:N+4]}, vmin); 99 100 if XNN_LIKELY(nc >= ${NR}) { 101 $for M in reversed(range(MR)): 102 vst1q_f32(c${M}, vacc${M}x${ABC[0:4]}); 103 $for N in range(4, NR, 4): 104 vst1q_f32(c${M} + ${N}, vacc${M}x${ABC[N:N+4]}); 105 c${M} = (float*) ((uintptr_t) c${M} + cn_stride); 106 107 a = (const float*) ((uintptr_t) a - kc * ${MR}); 108 109 nc -= ${NR}; 110 } else { 111 $for LOG2N in reversed(range(NR.bit_length())): 112 $if NR != 1 << LOG2N: 113 $if LOG2N == 1: 114 $for M in reversed(range(MR)): 115 float32x2_t vacc${M}x01 = vget_low_f32(vacc${M}x0123); 116 if (nc & ${1 << LOG2N}) { 117 $if LOG2N >= 2: 118 $for M in reversed(range(MR)): 119 $for N in range(0, 1 << LOG2N, 4): 120 vst1q_f32(c${M}, vacc${M}x${ABC[N:N+4]}); c${M} += 4; 121 122 $for M in reversed(range(MR)): 123 $for N in range(0, 1 << (LOG2N - 1), 4): 124 vacc${M}x${ABC[N:N+4]} = vacc${M}x${ABC[N + (1 << LOG2N):N + (1 << LOG2N)+4]}; 125 $elif LOG2N == 1: 126 $for M in reversed(range(MR)): 127 vst1_f32(c${M}, vacc${M}x01); c${M} += 2; 128 129 $for M in reversed(range(MR)): 130 vacc${M}x01 = vget_high_f32(vacc${M}x0123); 131 $elif LOG2N == 0: 132 $for M in reversed(range(MR)): 133 vst1_lane_f32(c${M}, vacc${M}x01, 0); 134 } 135 136 nc = 0; 137 } 138 } while (nc != 0); 139} 140