1// Copyright 2019 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert NR == 2 7#include <assert.h> 8 9#include <arm_neon.h> 10 11#include <xnnpack/common.h> 12#include <xnnpack/igemm.h> 13 14 15void xnn_f32_igemm_ukernel_${MR}x${NR}__${"neonfma" if FMA else "neon"}_lane_ld64( 16 size_t mr, 17 size_t nc, 18 size_t kc, 19 size_t ks, 20 const float**restrict a, 21 const float*restrict w, 22 float*restrict c, 23 size_t cm_stride, 24 size_t cn_stride, 25 size_t a_offset, 26 const float* zero, 27 const union xnn_f32_output_params params[restrict static 1]) 28{ 29 assert(mr != 0); 30 assert(mr <= ${MR}); 31 assert(nc != 0); 32 assert(kc != 0); 33 assert(kc % sizeof(float) == 0); 34 assert(ks != 0); 35 assert(ks % (${MR} * sizeof(void*)) == 0); 36 assert(a_offset % sizeof(float) == 0); 37 assert(a != NULL); 38 assert(w != NULL); 39 assert(c != NULL); 40 41 float* c0 = c; 42 $for M in range(1, MR): 43 float* c${M} = (float*) ((uintptr_t) c${M-1} + cm_stride); 44 $if M % 2 == 0: 45 if XNN_UNPREDICTABLE(mr <= ${M}) { 46 c${M} = c${M-1}; 47 } 48 $elif M + 1 == MR: 49 if XNN_UNPREDICTABLE(mr != ${M+1}) { 50 c${M} = c${M-1}; 51 } 52 $else: 53 if XNN_UNPREDICTABLE(mr < ${M+1}) { 54 c${M} = c${M-1}; 55 } 56 57 do { 58 float32x2_t vacc0x01 = vld1_f32(w); w += 2; 59 $for M in range(1, MR): 60 float32x2_t vacc${M}x01 = vacc0x01; 61 62 size_t p = ks; 63 do { 64 $for M in range(MR): 65 const float* restrict a${M} = a[${M}]; 66 assert(a${M} != NULL); 67 if XNN_UNPREDICTABLE(a${M} != zero) { 68 a${M} = (const float*) ((uintptr_t) a${M} + a_offset); 69 } 70 a += ${MR}; 71 72 size_t k = kc; 73 for (; k >= 2 * sizeof(float); k -= 2 * sizeof(float)) { 74 $for M in range(MR): 75 const float32x2_t va${M} = vld1_f32(a${M}); a${M} += 2; 76 77 $for L in range(2): 78 const float32x2_t vb01c${L} = vld1_f32(w); w += 2; 79 80 $if FMA: 81 #if XNN_ARCH_ARM64 82 $for M in range(MR): 83 vacc${M}x01 = vfma_lane_f32(vacc${M}x01, vb01c${L}, va${M}, ${L}); 84 #else 85 $for M in range(MR): 86 const float32x2_t va${M}c${L} = vdup_lane_f32(va${M}, ${L}); 87 $for M in range(MR): 88 vacc${M}x01 = vfma_f32(vacc${M}x01, va${M}c${L}, vb01c${L}); 89 #endif 90 $else: 91 $for M in range(MR): 92 vacc${M}x01 = vmla_lane_f32(vacc${M}x01, vb01c${L}, va${M}, ${L}); 93 } 94 if XNN_UNLIKELY(k != 0) { 95 $for M in range(MR): 96 const float32x2_t va${M} = vld1_dup_f32(a${M}); 97 98 const float32x2_t vb01 = vld1_f32(w); w += 2; 99 100 $for M in range(MR): 101 $if FMA: 102 vacc${M}x01 = vfma_f32(vacc${M}x01, va${M}, vb01); 103 $else: 104 vacc${M}x01 = vmla_f32(vacc${M}x01, va${M}, vb01); 105 } 106 p -= ${MR} * sizeof(void*); 107 } while (p != 0); 108 109 const float32x2_t vmax = vld1_dup_f32(¶ms->scalar.max); 110 $for N in range(0, NR, 4): 111 $for M in range(MR): 112 vacc${M}x01 = vmin_f32(vacc${M}x01, vmax); 113 114 const float32x2_t vmin = vld1_dup_f32(¶ms->scalar.min); 115 $for N in range(0, NR, 4): 116 $for M in range(MR): 117 vacc${M}x01 = vmax_f32(vacc${M}x01, vmin); 118 119 if XNN_LIKELY(nc >= ${NR}) { 120 $for M in reversed(range(MR)): 121 vst1_f32(c${M}, vacc${M}x01); 122 c${M} = (float*) ((uintptr_t) c${M} + cn_stride); 123 124 a = (const float**restrict) ((uintptr_t) a - ks); 125 nc -= ${NR}; 126 } else { 127 assert(nc == 1); 128 $for M in reversed(range(MR)): 129 vst1_lane_f32(c${M}, vacc${M}x01, 0); 130 131 nc = 0; 132 } 133 } while (nc != 0); 134} 135