1// Copyright 2019 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert NR % 4 == 0 7$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 8$VMULADDQ_F32 = "vfmaq_f32" if FMA else "vmlaq_f32" 9#include <assert.h> 10 11#include <arm_neon.h> 12 13#include <xnnpack/igemm.h> 14 15 16void xnn_f32_igemm_minmax_ukernel_${MR}x${NR}s4__${"neonfma" if FMA else "neon"}( 17 size_t mr, 18 size_t nc, 19 size_t kc, 20 size_t ks, 21 const float**restrict a, 22 const float*restrict w, 23 float*restrict c, 24 size_t cm_stride, 25 size_t cn_stride, 26 size_t a_offset, 27 const float* zero, 28 const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) 29{ 30 assert(mr != 0); 31 assert(mr <= ${MR}); 32 assert(nc != 0); 33 assert(kc != 0); 34 assert(kc % sizeof(float) == 0); 35 assert(ks != 0); 36 assert(ks % (${MR} * sizeof(void*)) == 0); 37 assert(a_offset % sizeof(float) == 0); 38 assert(a != NULL); 39 assert(w != NULL); 40 assert(c != NULL); 41 42 float* c0 = c; 43 $for M in range(1, MR): 44 float* c${M} = (float*) ((uintptr_t) c${M-1} + cm_stride); 45 $if M % 2 == 0: 46 if XNN_UNPREDICTABLE(mr <= ${M}) { 47 c${M} = c${M-1}; 48 } 49 $elif M + 1 == MR: 50 if XNN_UNPREDICTABLE(mr != ${M+1}) { 51 c${M} = c${M-1}; 52 } 53 $else: 54 if XNN_UNPREDICTABLE(mr < ${M+1}) { 55 c${M} = c${M-1}; 56 } 57 58 do { 59 $for N in range(0, NR, 4): 60 float32x4_t vacc0x${ABC[N:N+4]} = vld1q_f32(w); w += 4; 61 $for M in range(1, MR): 62 $for N in range(0, NR, 4): 63 float32x4_t vacc${M}x${ABC[N:N+4]} = vacc0x${ABC[N:N+4]}; 64 65 size_t p = ks; 66 do { 67 $for M in range(MR): 68 const float* restrict a${M} = a[${M}]; 69 assert(a${M} != NULL); 70 if XNN_UNPREDICTABLE(a${M} != zero) { 71 a${M} = (const float*) ((uintptr_t) a${M} + a_offset); 72 } 73 a += ${MR}; 74 75 size_t k = kc; 76 while (k >= 4 * sizeof(float)) { 77 $for M in range(MR): 78 float32x4_t va${M} = vld1q_f32(a${M}); a${M} += 4; 79 80 $for L in range(4): 81 82 $for N in range(0, NR, 4): 83 const float32x4_t vb${ABC[N:N+4]}c${L} = vld1q_f32(w + ${L * NR + N}); 84 85 $for N in range(0, NR, 4): 86 $for M in range(MR): 87 vacc${M}x${ABC[N:N+4]} = ${VMULADDQ_F32}(vacc${M}x${ABC[N:N+4]}, va${M}, vb${ABC[N:N+4]}c${L}); 88 89 $if L + 1 != 4: 90 $for M in range(MR): 91 va${M} = vextq_f32(va${M}, va${M}, 1); 92 93 w += ${4 * NR}; 94 k -= 4 * sizeof(float); 95 } 96 if XNN_UNLIKELY(k != 0) { 97 do { 98 $for M in range(MR): 99 const float32x4_t va${M} = vld1q_dup_f32(a${M}); a${M} += 1; 100 101 $for N in range(0, NR, 4): 102 const float32x4_t vb${ABC[N:N+4]} = vld1q_f32(w); w += 4; 103 104 $for N in range(0, NR, 4): 105 $for M in range(MR): 106 vacc${M}x${ABC[N:N+4]} = ${VMULADDQ_F32}(vacc${M}x${ABC[N:N+4]}, va${M}, vb${ABC[N:N+4]}); 107 108 k -= sizeof(float); 109 } while (k != 0); 110 } 111 112 p -= ${MR} * sizeof(void*); 113 } while (p != 0); 114 115 const float32x4_t vmax = vld1q_dup_f32(¶ms->scalar.max); 116 $for N in range(0, NR, 4): 117 $for M in range(MR): 118 vacc${M}x${ABC[N:N+4]} = vminq_f32(vacc${M}x${ABC[N:N+4]}, vmax); 119 120 const float32x4_t vmin = vld1q_dup_f32(¶ms->scalar.min); 121 $for N in range(0, NR, 4): 122 $for M in range(MR): 123 vacc${M}x${ABC[N:N+4]} = vmaxq_f32(vacc${M}x${ABC[N:N+4]}, vmin); 124 125 if XNN_LIKELY(nc >= ${NR}) { 126 $for M in reversed(range(MR)): 127 vst1q_f32(c${M}, vacc${M}x${ABC[0:4]}); 128 $for N in range(4, NR, 4): 129 vst1q_f32(c${M} + ${N}, vacc${M}x${ABC[N:N+4]}); 130 c${M} = (float*) ((uintptr_t) c${M} + cn_stride); 131 132 a = (const float**restrict) ((uintptr_t) a - ks); 133 nc -= ${NR}; 134 } else { 135 $for LOG2N in reversed(range(NR.bit_length())): 136 $if LOG2N == 1: 137 $for M in reversed(range(MR)): 138 float32x2_t vacc${M}x${ABC[0:2]} = vget_low_f32(vacc${M}x${ABC[0:4]}); 139 $if 1 << LOG2N != NR: 140 if (nc & ${1 << LOG2N}) { 141 $if LOG2N >= 2: 142 $for N in range(0, 1 << LOG2N, 4): 143 $for M in reversed(range(MR)): 144 vst1q_f32(c${M}, vacc${M}x${ABC[N:N+4]}); c${M} += 4; 145 146 $for M in reversed(range(MR)): 147 $for N in range(0, 1 << (LOG2N - 1), 4): 148 vacc${M}x${ABC[N:N+4]} = vacc${M}x${ABC[N + (1 << LOG2N):N + (1 << LOG2N)+4]}; 149 $elif LOG2N == 1: 150 $for M in reversed(range(MR)): 151 vst1_f32(c${M}, vacc${M}x${ABC[0:2]}); c${M} += 2; 152 153 $for M in reversed(range(MR)): 154 vacc${M}x${ABC[0:2]} = vget_high_f32(vacc${M}x${ABC[0:4]}); 155 $elif LOG2N == 0: 156 $for M in reversed(range(MR)): 157 vst1_lane_f32(c${M}, vacc${M}x${ABC[0:2]}, 0); 158 } 159 160 nc = 0; 161 } 162 } while (nc != 0); 163} 164