// Copyright 2019 Google LLC // // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. $assert NR == 2 #include #include #include #include void xnn_f32_gemm${"inc" if INC else ""}_ukernel_${MR}x${NR}__${"neonfma" if FMA else "neon"}_lane_ld64( size_t mr, size_t nc, size_t kc, const float* restrict a, size_t a_stride, const float*restrict w, float*restrict c, size_t cm_stride, size_t cn_stride, $if INC: const float*restrict acc, const union xnn_f32_output_params params[restrict static 1]) { assert(mr != 0); assert(mr <= ${MR}); assert(nc != 0); assert(kc != 0); assert(kc % sizeof(float) == 0); assert(a != NULL); assert(w != NULL); assert(c != NULL); $if INC: assert(acc != NULL); const float* a0 = a; float* c0 = c; $for M in range(1, MR): const float* a${M} = (const float*) ((uintptr_t) a${M-1} + a_stride); float* c${M} = (float*) ((uintptr_t) c${M-1} + cm_stride); $if M % 2 == 0: if XNN_UNPREDICTABLE(mr <= ${M}) { a${M} = a${M-1}; c${M} = c${M-1}; } $elif M + 1 == MR: if XNN_UNPREDICTABLE(mr != ${M+1}) { a${M} = a${M-1}; c${M} = c${M-1}; } $else: if XNN_UNPREDICTABLE(mr < ${M+1}) { a${M} = a${M-1}; c${M} = c${M-1}; } do { $if INC: $for M in range(0, MR): float32x2_t vacc${M}x01 = vld1_f32(w); w += 2; $else: float32x2_t vacc0x01 = vld1_f32(w); w += 2; $for M in range(1, MR): float32x2_t vacc${M}x01 = vacc0x01; size_t k = kc; for (; k >= 2 * sizeof(float); k -= 2 * sizeof(float)) { $for M in range(MR): const float32x2_t va${M} = vld1_f32(a${M}); a${M} += 2; $for L in range(2): const float32x2_t vb01c${L} = vld1_f32(w); w += 2; $if FMA: #if XNN_ARCH_ARM64 $for M in range(MR): vacc${M}x01 = vfma_lane_f32(vacc${M}x01, vb01c${L}, va${M}, ${L}); #else $for M in range(MR): const float32x2_t va${M}c${L} = vdup_lane_f32(va${M}, ${L}); $for M in range(MR): vacc${M}x01 = vfma_f32(vacc${M}x01, va${M}c${L}, vb01c${L}); #endif $else: $for M in range(MR): vacc${M}x01 = vmla_lane_f32(vacc${M}x01, vb01c${L}, va${M}, ${L}); } if XNN_UNLIKELY(k != 0) { $for M in range(MR): const float32x2_t va${M} = vld1_dup_f32(a${M}); a${M} += 1; const float32x2_t vb01 = vld1_f32(w); w += 2; $for M in range(MR): $if FMA: vacc${M}x01 = vfma_f32(vacc${M}x01, va${M}, vb01); $else: vacc${M}x01 = vmla_f32(vacc${M}x01, va${M}, vb01); } const float32x2_t vmax = vld1_dup_f32(¶ms->scalar.max); $for N in range(0, NR, 4): $for M in range(MR): vacc${M}x01 = vmin_f32(vacc${M}x01, vmax); const float32x2_t vmin = vld1_dup_f32(¶ms->scalar.min); $for N in range(0, NR, 4): $for M in range(MR): vacc${M}x01 = vmax_f32(vacc${M}x01, vmin); if XNN_LIKELY(nc >= ${NR}) { $for M in range(MR): vst1_f32(c${M}, vacc${M}x01); c${M} = (float*) ((uintptr_t) c${M} + cn_stride); $for M in range(MR): a${M} = (const float*) ((uintptr_t) a${M} - kc); nc -= ${NR}; } else { assert(nc == 1); $for M in range(MR): vst1_lane_f32(c${M}, vacc${M}x01, 0); nc = 0; } } while (nc != 0); }