• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Auto-generated file. Do not edit!
2//   Template: src/f32-gemm/1x8-aarch64-neonfma-cortex-a53.S.in
3//   Generator: tools/xngen
4//
5// Copyright 2019 Google LLC
6//
7// This source code is licensed under the BSD-style license found in the
8// LICENSE file in the root directory of this source tree.
9
10#include <xnnpack/assembly.h>
11
12# void xnn_f32_gemm_ukernel_1x8__aarch64_neonfma_cortex_a53(
13#     size_t mr,                (x0) - unused.  mr = 1
14#     size_t nc,                x1
15#     size_t kc,                x2 / x0
16#     const uint8_t*restrict a, x3
17#     size_t a_stride,          (x4) - unused
18#     const void*restrict w,    x5
19#     uint8_t*restrict c,       x6
20#     size_t cm_stride,         (x7) - unused
21#     size_t cn_stride,         [sp] -> x14
22#     const union xnn_f32_output_params params[restrict static 1])  [sp + 8] -> x8
23
24# d8-d15 need to be preserved if used.
25# x19-30 need to be preserved if used.
26
27# A pointer
28# x3  a0
29
30# C pointer
31# x6  c0
32
33# Clamp v4 v5
34
35# A53 based on A57/A75 but with LD64
36
37BEGIN_FUNCTION xnn_f32_gemm_ukernel_1x8__aarch64_neonfma_cortex_a53
38
39        # Load cn_stride, params pointer
40        LDP x14, x8, [sp]
41
42        # Load clamping_params values
43        LD2R {v4.4s, v5.4s}, [x8]
440:
45        # Load initial bias from w into accumulators
46        LDP q16, q17, [x5], 32
47
48        MOVI v18.4s, 0  // second set of C for pipelining FMLA
49        PRFM PLDL1KEEP, [x5]
50        MOVI v19.4s, 0
51        PRFM PLDL1KEEP, [x5, 64]
52        PRFM PLDL1KEEP, [x5, 128]
53        PRFM PLDL1KEEP, [x5, 192]
54
55        # Is there at least 8 floats (32 bytes) for prologue + epilogue?
56        SUBS x0, x2, 32  // k = kc - 32
57
58        B.LO 3f
59
60        # 16 prologue
61        # Read first block of 1 A and B.
62        LDP q20, q21, [x5], 32
63        LDP q22, q23, [x5], 32
64        LDP q24, q25, [x5], 32
65        LDP q26, q27, [x5], 32
66        LDR q0, [x3], 16
67
68        # Is there at least 32.  yes do main loop
69        SUBS x0, x0, 32
70        B.LO 2f
71
72        # Main loop - 8 floats of A (32 bytes)
731:
74        # First block of 4.  FMA for first 4, loads for 2nd block of 4.
75        FMLA v16.4s, v20.4s, v0.s[0]
76        LDR q1, [x3], 16
77        FMLA v17.4s, v21.4s, v0.s[0]
78        LDR q20, [x5], 16
79        FMLA v18.4s, v22.4s, v0.s[1]
80        LDR q21, [x5], 16
81        FMLA v19.4s, v23.4s, v0.s[1]
82        LDR q22, [x5], 16
83        FMLA v16.4s, v24.4s, v0.s[2]
84        LDR q23, [x5], 16
85        FMLA v17.4s, v25.4s, v0.s[2]
86        LDR q24, [x5], 16
87        FMLA v18.4s, v26.4s, v0.s[3]
88        LDR q25, [x5], 16
89        FMLA v19.4s, v27.4s, v0.s[3]
90        LDR q26, [x5], 16
91        LDR q27, [x5], 16
92
93        # Second block of 4.  FMA for second 4, loads for 1st block of 4.
94        FMLA v16.4s, v20.4s, v1.s[0]
95        LDR q0, [x3], 16
96        FMLA v17.4s, v21.4s, v1.s[0]
97        LDR q20, [x5], 16
98        FMLA v18.4s, v22.4s, v1.s[1]
99        LDR q21, [x5], 16
100        FMLA v19.4s, v23.4s, v1.s[1]
101        LDR q22, [x5], 16
102        FMLA v16.4s, v24.4s, v1.s[2]
103        LDR q23, [x5], 16
104        FMLA v17.4s, v25.4s, v1.s[2]
105        LDR q24, [x5], 16
106        FMLA v18.4s, v26.4s, v1.s[3]
107        LDR q25, [x5], 16
108        FMLA v19.4s, v27.4s, v1.s[3]
109        LDR q26, [x5], 16
110        SUBS x0, x0, 32
111        LDR q27, [x5], 16
112        B.HS 1b
113
1142:
115        # Epilogue
116
117        # First block of 4.  FMA for first 4, loads for 2nd block of 4.
118        FMLA v16.4s, v20.4s, v0.s[0]
119        LDR q1, [x3], 16
120        FMLA v17.4s, v21.4s, v0.s[0]
121        LDR q20, [x5], 16
122        FMLA v18.4s, v22.4s, v0.s[1]
123        LDR q21, [x5], 16
124        FMLA v19.4s, v23.4s, v0.s[1]
125        LDR q22, [x5], 16
126        FMLA v16.4s, v24.4s, v0.s[2]
127        LDR q23, [x5], 16
128        FMLA v17.4s, v25.4s, v0.s[2]
129        LDR q24, [x5], 16
130        FMLA v18.4s, v26.4s, v0.s[3]
131        LDR q25, [x5], 16
132        FMLA v19.4s, v27.4s, v0.s[3]
133        LDR q26, [x5], 16
134
135        # Second block of 4.  no loads
136        FMLA v16.4s, v20.4s, v1.s[0]
137        LDR q27, [x5], 16
138        FMLA v17.4s, v21.4s, v1.s[0]
139        FMLA v18.4s, v22.4s, v1.s[1]
140        FMLA v19.4s, v23.4s, v1.s[1]
141        FMLA v16.4s, v24.4s, v1.s[2]
142        FMLA v17.4s, v25.4s, v1.s[2]
143        FMLA v18.4s, v26.4s, v1.s[3]
144        FMLA v19.4s, v27.4s, v1.s[3]
145
1463:
147        # Is there a remainder?- 4 floats of A (16 bytes)
148        TBNZ x0, 4, 5f
149        # Is there a remainder?- 2 floats of A (8 bytes)
150        TBNZ x0, 3, 6f
151        # Is there a remainder?- 1 floats of A (4 bytes)
152        TBNZ x0, 2, 8f
153
1544:
155        FADD v16.4s, v16.4s, v18.4s
156        FADD v17.4s, v17.4s, v19.4s
157
158        # Clamp
159        FMIN v16.4s, v16.4s, v4.4s
160        SUBS x1, x1, 8
161        FMIN v17.4s, v17.4s, v4.4s
162        FMAX v16.4s, v16.4s, v5.4s
163        FMAX v17.4s, v17.4s, v5.4s
164
165        # Store full 1 x 8
166        B.LO 9f
167
168        ST1 {v16.16b, v17.16b}, [x6], x14
169        SUB  x3,  x3, x2 // a0 -= kc
170
171        B.HI 0b
172
173        RET
174
1755:
176        # Remainder- 4 floats of A (16 bytes)
177        LDR q20, [x5], 16
178        LDR q21, [x5], 16
179        LDR q0, [x3], 16
180        FMLA v16.4s, v20.4s, v0.s[0]
181        FMLA v17.4s, v21.4s, v0.s[0]
182        LDR q22, [x5], 16
183        LDR q23, [x5], 16
184        LDR q24, [x5], 16
185        LDR q25, [x5], 16
186        LDR q26, [x5], 16
187        LDR q27, [x5], 16
188        FMLA v18.4s, v22.4s, v0.s[1]
189        FMLA v19.4s, v23.4s, v0.s[1]
190        FMLA v16.4s, v24.4s, v0.s[2]
191        FMLA v17.4s, v25.4s, v0.s[2]
192        FMLA v18.4s, v26.4s, v0.s[3]
193        FMLA v19.4s, v27.4s, v0.s[3]
194
195        TBZ x0, 3, 7f
1966:
197        # Remainder- 2 floats of A (8 bytes)
198        LDR q20, [x5], 16
199        LDR q21, [x5], 16
200        LDR d0, [x3], 8
201        FMLA v16.4s, v20.4s, v0.s[0]
202        FMLA v17.4s, v21.4s, v0.s[0]
203        LDR q22, [x5], 16
204        LDR q23, [x5], 16
205        FMLA v18.4s, v22.4s, v0.s[1]
206        FMLA v19.4s, v23.4s, v0.s[1]
2077:
208        TBZ x0, 2, 4b
2098:
210        # Remainder- 1 float of A (4 bytes)
211        LDR q20, [x5], 16
212        LDR q21, [x5], 16
213        LDR s0, [x3], 4
214        FMLA v16.4s, v20.4s, v0.s[0]
215        FMLA v17.4s, v21.4s, v0.s[0]
216        B 4b
217
218        # Store odd channels
2199:
220        TBZ x1, 2, 10f
221        STR q16, [x6], 16
222        MOV v16.16b, v17.16b
223
22410:
225        TBZ x1, 1, 11f
226        STR d16, [x6], 8
227        DUP d16, v16.d[1]
228
22911:
230        TBZ x1, 0, 12f
231        STR s16, [x6]
23212:
233        RET
234
235END_FUNCTION xnn_f32_gemm_ukernel_1x8__aarch64_neonfma_cortex_a53
236
237#ifdef __ELF__
238.section ".note.GNU-stack","",%progbits
239#endif
240