1 // Copyright 2022 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <xnnpack/aarch64-assembler.h>
7 #include <xnnpack/allocator.h>
8 #include <xnnpack/common.h>
9
10 #include "assembler-helpers.h"
11 #include <gtest/gtest.h>
12
13 namespace xnnpack {
14 namespace aarch64 {
15
TEST(AArch64Assembler,Initialization)16 TEST(AArch64Assembler, Initialization) {
17 xnn_code_buffer b;
18 xnn_allocate_code_memory(&b, XNN_DEFAULT_CODE_BUFFER_SIZE);
19 Assembler a(&b);
20 ASSERT_EQ(xnn_status_success, xnn_release_code_memory(&b));
21 }
22
TEST(AArch64Assembler,BaseInstructionEncoding)23 TEST(AArch64Assembler, BaseInstructionEncoding) {
24 xnn_code_buffer b;
25 xnn_allocate_code_memory(&b, XNN_DEFAULT_CODE_BUFFER_SIZE);
26 Assembler a(&b);
27
28 CHECK_ENCODING(0x91008041, a.add(x1, x2, 32));
29 CHECK_ENCODING(0x913FFC41, a.add(x1, x2, 4095));
30 EXPECT_ERROR(Error::kInvalidOperand, a.add(x1, x2, 4096));
31
32 CHECK_ENCODING(0x8B040069, a.add(x9, x3, x4));
33
34 CHECK_ENCODING(0xF100081F, a.cmp(x0, 2));
35 EXPECT_ERROR(Error::kInvalidOperand, a.cmp(x0, 4096));
36
37 CHECK_ENCODING(0xEB0C02DF, a.cmp(x22, x12));
38
39 CHECK_ENCODING(0x9A8F322E, a.csel(x14, x17, x15, kLO));
40
41 CHECK_ENCODING(0xA9403FEE, a.ldp(x14, x15, mem[sp]));
42 CHECK_ENCODING(0xA8C13FEE, a.ldp(x14, x15, mem[sp], 16));
43 CHECK_ENCODING(0xA9413FEE, a.ldp(x14, x15, mem[sp, 16]));
44 CHECK_ENCODING(0xA9603FEE, a.ldp(x14, x15, mem[sp, -512]));
45 CHECK_ENCODING(0xA95FBFEE, a.ldp(x14, x15, mem[sp, 504]));
46 EXPECT_ERROR(Error::kInvalidOperand, a.ldp(x14, x15, mem[sp], 15));
47 EXPECT_ERROR(Error::kInvalidOperand, a.ldp(x14, x15, mem[sp], -520));
48 EXPECT_ERROR(Error::kInvalidOperand, a.ldp(x14, x15, mem[sp], 512));
49 EXPECT_ERROR(Error::kInvalidOperand, a.ldp(x14, x15, mem[sp, 16], 16));
50
51 CHECK_ENCODING(0xF9400BE8, a.ldr(x8, mem[sp, 16]));
52 CHECK_ENCODING(0xF97FFFE8, a.ldr(x8, mem[sp, 32760]));
53 EXPECT_ERROR(Error::kInvalidOperand, a.ldr(x8, mem[sp, -8]));
54 EXPECT_ERROR(Error::kInvalidOperand, a.ldr(x8, mem[sp, 7]));
55 EXPECT_ERROR(Error::kInvalidOperand, a.ldr(x8, mem[sp, 32768]));
56 EXPECT_ERROR(Error::kInvalidOperand, a.ldr(x8, MemOperand(sp, 16, AddressingMode::kPostIndex)));
57
58 CHECK_ENCODING(0xF8408488, a.ldr(x8, mem[x4], 8));
59 CHECK_ENCODING(0xF84FF488, a.ldr(x8, mem[x4], 255));
60 CHECK_ENCODING(0xF8500488, a.ldr(x8, mem[x4], -256));
61 EXPECT_ERROR(Error::kInvalidOperand, a.ldr(x8, mem[x4], 256));
62 EXPECT_ERROR(Error::kInvalidOperand, a.ldr(x8, mem[x4], -257));
63
64 CHECK_ENCODING(0xAA0303E9, a.mov(x9, x3));
65
66 CHECK_ENCODING(0xF98000A0, a.prfm(kPLDL1KEEP, mem[x5]));
67 CHECK_ENCODING(0xF98020A0, a.prfm(kPLDL1KEEP, mem[x5, 64]));
68 EXPECT_ERROR(Error::kInvalidOperand, a.prfm(kPLDL1KEEP, mem[x5, -8]));
69 EXPECT_ERROR(Error::kInvalidOperand, a.prfm(kPLDL1KEEP, mem[x5, 32761]));
70
71 CHECK_ENCODING(0xD65F03C0, a.ret());
72
73 CHECK_ENCODING(0xCB020083, a.sub(x3, x4, x2));
74
75 CHECK_ENCODING(0xA90457F4, a.stp(x20, x21, mem[sp, 64]));
76 CHECK_ENCODING(0xA98457F4, a.stp(x20, x21, mem[sp, 64]++));
77 CHECK_ENCODING(0xA91FD7F4, a.stp(x20, x21, mem[sp, 504]));
78 CHECK_ENCODING(0xA92057F4, a.stp(x20, x21, mem[sp, -512]));
79 EXPECT_ERROR(Error::kInvalidOperand, a.stp(x20, x21, mem[sp, 3]));
80 EXPECT_ERROR(Error::kInvalidOperand, a.stp(x20, x21, mem[sp, 512]));
81 EXPECT_ERROR(Error::kInvalidOperand, a.stp(x20, x21, mem[sp, -520]));
82
83 CHECK_ENCODING(0xF1008040, a.subs(x0, x2, 32));
84 CHECK_ENCODING(0xF13FFC40, a.subs(x0, x2, 4095));
85 EXPECT_ERROR(Error::kInvalidOperand, a.subs(x0, x2, -32));
86 EXPECT_ERROR(Error::kInvalidOperand, a.subs(x0, x2, 4096));
87
88 CHECK_ENCODING(0xF240043F, a.tst(x1, 3));
89 CHECK_ENCODING(0xF2400C3F, a.tst(x1, 15));
90 CHECK_ENCODING(0xF240103F, a.tst(x1, 31));
91 EXPECT_ERROR(Error::kUnimplemented, a.tst(x1, 32));
92
93 ASSERT_EQ(xnn_status_success, xnn_release_code_memory(&b));
94 }
95
TEST(AArch64Assembler,SIMDInstructionEncoding)96 TEST(AArch64Assembler, SIMDInstructionEncoding) {
97 xnn_code_buffer b;
98 xnn_allocate_code_memory(&b, XNN_DEFAULT_CODE_BUFFER_SIZE);
99 Assembler a(&b);
100
101 CHECK_ENCODING(0x5E180610, a.dup(d16, v16.d()[1]));
102 EXPECT_ERROR(Error::kInvalidOperand, a.dup(d16, v16.d()[2]));
103 EXPECT_ERROR(Error::kInvalidOperand, a.dup(d16, v16.s()[1]));
104
105 CHECK_ENCODING(0x4E25D690, a.fadd(v16.v4s(), v20.v4s(), v5.v4s()));
106 EXPECT_ERROR(Error::kInvalidOperand, a.fadd(v16.v4s(), v20.v4s(), v5.v2s()));
107
108 CHECK_ENCODING(0x4E30F7E3, a.fmax(v3.v4s(), v31.v4s(), v16.v4s()));
109 EXPECT_ERROR(Error::kInvalidOperand, a.fmax(v3.v8h(), v31.v4s(), v16.v4s()));
110
111 CHECK_ENCODING(0x4EB1F7C2, a.fmin(v2.v4s(), v30.v4s(), v17.v4s()));
112 EXPECT_ERROR(Error::kInvalidOperand, a.fmin(v2.v4s(), v30.v16b(), v17.v4s()));
113
114 CHECK_ENCODING(0x4F801290, a.fmla(v16.v4s(), v20.v4s(), v0.s()[0]));
115 EXPECT_ERROR(Error::kInvalidOperand, a.fmla(v16.v4s(), v20.v2s(), v0.s()[0]));
116 EXPECT_ERROR(Error::kInvalidOperand, a.fmla(v16.v2d(), v20.v2d(), v0.s()[0]));
117 EXPECT_ERROR(Error::kInvalidLaneIndex, a.fmla(v16.v4s(), v20.v4s(), v0.s()[4]));
118
119 CHECK_ENCODING(0x0CDF7060, a.ld1({v0.v8b()}, mem[x3], 8));
120 EXPECT_ERROR(Error::kInvalidOperand, a.ld1({v0.v8b()}, mem[x3], 16));
121 EXPECT_ERROR(Error::kInvalidOperand, a.ld1({v0.v16b()}, mem[x3], 8));
122
123 CHECK_ENCODING(0x0CDFA060, a.ld1({v0.v8b(), v1.v8b()}, mem[x3], 16));
124 EXPECT_ERROR(Error::kInvalidOperand, a.ld1({v0.v8b(), v1.v8b()}, mem[x3], 32));
125 EXPECT_ERROR(Error::kInvalidOperand, a.ld1({v0.v16b(), v1.v16b()}, mem[x3], 16));
126 EXPECT_ERROR(Error::kInvalidOperand, a.ld1({v0.v8b(), v2.v8b()}, mem[x3], 16));
127
128 CHECK_ENCODING(0x4CDF61F0, a.ld1({v16.v16b(), v17.v16b(), v18.v16b()}, mem[x15], 48));
129 EXPECT_ERROR(Error::kInvalidOperand, a.ld1({v16.v8b(), v17.v16b(), v18.v16b()}, mem[x15], 48));
130 EXPECT_ERROR(Error::kInvalidOperand, a.ld1({v16.v16b(), v17.v16b(), v18.v8b()}, mem[x15], 48));
131 EXPECT_ERROR(Error::kInvalidOperand, a.ld1({v16.v16b(), v17.v16b(), v18.v16b()}, mem[x15], 24));
132 EXPECT_ERROR(Error::kInvalidOperand, a.ld1({v16.v8b(), v17.v8b(), v18.v8b()}, mem[x15], 48));
133
134 CHECK_ENCODING(0x6D433FEE, a.ldp(d14, d15, mem[sp, 48]));
135 CHECK_ENCODING(0x6DC33FEE, a.ldp(d14, d15, mem[sp, 48]++));
136 CHECK_ENCODING(0x6CC427E8, a.ldp(d8, d9, mem[sp], 64));
137 EXPECT_ERROR(Error::kInvalidOperand, a.ldp(d14, d15, mem[sp, 7]));
138
139 CHECK_ENCODING(0xACC154B4, a.ldp(q20, q21, mem[x5], 32));
140 CHECK_ENCODING(0xACE054B4, a.ldp(q20, q21, mem[x5], -1024));
141 CHECK_ENCODING(0xACDFD4B4, a.ldp(q20, q21, mem[x5], 1008));
142 EXPECT_ERROR(Error::kInvalidOperand, a.ldp(q20, q21, mem[x5], 15));
143 EXPECT_ERROR(Error::kInvalidOperand, a.ldp(q20, q21, mem[x5], -1040));
144 EXPECT_ERROR(Error::kInvalidOperand, a.ldp(q20, q21, mem[x5], 1024));
145
146 CHECK_ENCODING(0xFC408460, a.ldr(d0, mem[x3], 8));
147 CHECK_ENCODING(0xBC404460, a.ldr(s0, mem[x3], 4));
148
149 CHECK_ENCODING(0x3CC10460, a.ldr(q0, mem[x3], 16));
150 CHECK_ENCODING(0x3CCFF460, a.ldr(q0, mem[x3], 255));
151 CHECK_ENCODING(0x3CD00460, a.ldr(q0, mem[x3], -256));
152 EXPECT_ERROR(Error::kInvalidOperand, a.ldr(q0, mem[x3], -257));
153 EXPECT_ERROR(Error::kInvalidOperand, a.ldr(q0, mem[x3], 256));
154 EXPECT_ERROR(Error::kInvalidOperand, a.ldr(q0, mem[x3, 16], 16));
155
156 CHECK_ENCODING(0x4D40C904, a.ld1r({v4.v4s()}, mem[x8]));
157 EXPECT_ERROR(Error::kInvalidOperand, a.ld1r({v4.v4s(), v5.v4s()}, mem[x8]));
158 EXPECT_ERROR(Error::kInvalidOperand, a.ld1r({v4.v4s()}, mem[x8, 16]));
159
160 CHECK_ENCODING(0x4D60C902, a.ld2r({v2.v4s(), v3.v4s()}, mem[x8]));
161 EXPECT_ERROR(Error::kInvalidOperand, a.ld2r({v2.v4s(), v3.v4s()}, mem[x8, 16]));
162 EXPECT_ERROR(Error::kInvalidOperand, a.ld2r({v2.v4s(), v4.v4s()}, mem[x8, 16]));
163 EXPECT_ERROR(Error::kInvalidOperand, a.ld2r({v2.v4s(), v3.v8b()}, mem[x8]));
164
165 CHECK_ENCODING(0x4EB21E50, a.mov(v16.v16b(), v18.v16b()));
166 CHECK_ENCODING(0x0EB21E50, a.mov(v16.v8b(), v18.v8b()));
167 EXPECT_ERROR(Error::kInvalidOperand, a.mov(v16.v16b(), v18.v8b()));
168
169 CHECK_ENCODING(0x4F000405, a.movi(v5.v4s(), 0));
170 CHECK_ENCODING(0x4F008405, a.movi(v5.v8h(), 0));
171 CHECK_ENCODING(0x4F00E405, a.movi(v5.v16b(), 0));
172 EXPECT_ERROR(Error::kUnimplemented, a.movi(v5.v16b(), 0xFF));
173
174 CHECK_ENCODING(0x4C82746F, a.st1({v15.v8h()}, mem[x3], x2));
175
176 CHECK_ENCODING(0x4C95AA8F, a.st1({v15.v4s(), v16.v4s()}, mem[x20], x21));
177 EXPECT_ERROR(Error::kInvalidOperand, a.st1({v15.v4s(), v17.v4s()}, mem[x20], x21));
178 EXPECT_ERROR(Error::kInvalidOperand, a.st1({v15.v4s(), v16.v8h()}, mem[x20], x21));
179
180 CHECK_ENCODING(0x4C8E60D0, a.st1({v16.v16b(), v17.v16b(), v18.v16b() }, mem[x6], x14));
181 EXPECT_ERROR(Error::kInvalidOperand, a.st1({v15.v16b(), v17.v16b(), v18.v16b()}, mem[x6], x14));
182 EXPECT_ERROR(Error::kInvalidOperand, a.st1({v16.v16b(), v17.v16b(), v18.v4s()}, mem[x6], x14));
183
184 CHECK_ENCODING(0x4C812FB4, a.st1({v20.v2d(), v21.v2d(), v22.v2d(), v23.v2d()}, mem[x29], x1));
185 EXPECT_ERROR(Error::kInvalidOperand, a.st1({v20.v2d(), v21.v2d(), v22.v2d(), v23.v2s()}, mem[x29], x1));
186 EXPECT_ERROR(Error::kInvalidOperand, a.st1({v20.v2d(), v21.v2d(), v22.v2d(), v27.v2d()}, mem[x29], x1));
187
188 CHECK_ENCODING(0x6D012FEA, a.stp(d10, d11, mem[sp, 16]));
189 CHECK_ENCODING(0x6D202FEA, a.stp(d10, d11, mem[sp, -512]));
190 CHECK_ENCODING(0x6D1FAFEA, a.stp(d10, d11, mem[sp, 504]));
191 EXPECT_ERROR(Error::kInvalidOperand, a.stp(d10, d11, mem[sp, -520]));
192 EXPECT_ERROR(Error::kInvalidOperand, a.stp(d10, d11, mem[sp, 512]));
193
194 CHECK_ENCODING(0x6D812FEA, a.stp(d10, d11, mem[sp, 16]++));
195
196 CHECK_ENCODING(0xAD0075BC, a.stp(q28, q29, mem[x13]));
197 CHECK_ENCODING(0xAD80F5BC, a.stp(q28, q29, mem[x13, 16]++));
198 EXPECT_ERROR(Error::kInvalidOperand, a.stp(q28, q28, mem[x13, 7]));
199
200 CHECK_ENCODING(0xAC8144D0, a.stp(q16, q17, mem[x6], 32));
201 CHECK_ENCODING(0xAC9FC4D0, a.stp(q16, q17, mem[x6], 1008));
202 CHECK_ENCODING(0xACA044D0, a.stp(q16, q17, mem[x6], -1024));
203 EXPECT_ERROR(Error::kInvalidOperand, a.stp(q16, q17, mem[x6], 34));
204 EXPECT_ERROR(Error::kInvalidOperand, a.stp(q16, q17, mem[x6], 1024));
205 EXPECT_ERROR(Error::kInvalidOperand, a.stp(q16, q17, mem[x6], -1040));
206
207 CHECK_ENCODING(0xFC0084D0, a.str(d16, mem[x6], 8));
208 CHECK_ENCODING(0x3C8104D0, a.str(q16, mem[x6], 16));
209 CHECK_ENCODING(0x3C8FF4D0, a.str(q16, mem[x6], 255));
210 CHECK_ENCODING(0x3C9004D0, a.str(q16, mem[x6], -256));
211 EXPECT_ERROR(Error::kInvalidOperand, a.str(q16, mem[x6], 256));
212 EXPECT_ERROR(Error::kInvalidOperand, a.str(q16, mem[x6], -257));
213
214 CHECK_ENCODING(0xBD0000D0, a.str(s16, mem[x6]));
215 CHECK_ENCODING(0xBD3FFCD0, a.str(s16, mem[x6, 16380]));
216 EXPECT_ERROR(Error::kInvalidOperand, a.str(s16, mem[x6, 3]));
217 EXPECT_ERROR(Error::kInvalidOperand, a.str(s16, mem[x6, -4]));
218 EXPECT_ERROR(Error::kInvalidOperand, a.str(s16, mem[x6, 16384]));
219
220 CHECK_ENCODING(0xBC0044D0, a.str(s16, mem[x6], 4));
221 CHECK_ENCODING(0xBC0FF4D0, a.str(s16, mem[x6], 255));
222 CHECK_ENCODING(0xBC1004D0, a.str(s16, mem[x6], -256));
223 EXPECT_ERROR(Error::kInvalidOperand, a.str(s16, mem[x6], 256));
224 EXPECT_ERROR(Error::kInvalidOperand, a.str(s16, mem[x6], -257));
225
226 ASSERT_EQ(xnn_status_success, xnn_release_code_memory(&b));
227 }
228
TEST(AArch64Assembler,Label)229 TEST(AArch64Assembler, Label) {
230 xnn_code_buffer b;
231 xnn_allocate_code_memory(&b, XNN_DEFAULT_CODE_BUFFER_SIZE);
232 Assembler a(&b);
233
234 Label l1;
235 a.movi(v0.v4s(), 0);
236
237 // Branch to unbound label.
238 auto b1 = a.offset<uint32_t*>();
239 a.b_eq(l1);
240
241 a.movi(v1.v4s(), 0);
242
243 auto b2 = a.offset<uint32_t*>();
244 a.b_ne(l1);
245
246 a.movi(v2.v4s(), 0);
247
248 a.bind(l1);
249
250 // Check that b1 and b2 are both patched after binding l1.
251 EXPECT_INSTR(0x54000080, *b1);
252 EXPECT_INSTR(0x54000041, *b2);
253
254 a.movi(v3, 0);
255
256 // Branch to bound label.
257 auto b3 = a.offset<uint32_t*>();
258 a.b_hi(l1);
259 auto b4 = a.offset<uint32_t*>();
260 a.b_hs(l1);
261 auto b5 = a.offset<uint32_t*>();
262 a.b_lo(l1);
263
264 EXPECT_INSTR(0x54FFFFE8, *b3);
265 EXPECT_INSTR(0x54FFFFC2, *b4);
266 EXPECT_INSTR(0x54FFFFA3, *b5);
267
268 // Binding a bound label is an error.
269 a.bind(l1);
270 EXPECT_ERROR(Error::kLabelAlreadyBound, a.bind(l1));
271
272 // Check for bind failure due to too many users of label.
273 Label lfail;
274 a.reset();
275 // Arbitrary high number of users that we probably won't support.
276 for (int i = 0; i < 1000; i++) {
277 a.b_eq(lfail);
278 }
279 EXPECT_EQ(Error::kLabelHasTooManyUsers, a.error());
280
281 ASSERT_EQ(xnn_status_success, xnn_release_code_memory(&b));
282 }
283
TEST(AArch64Assembler,Tbnz)284 TEST(AArch64Assembler, Tbnz) {
285 xnn_code_buffer b;
286 xnn_allocate_code_memory(&b, XNN_DEFAULT_CODE_BUFFER_SIZE);
287 Assembler a(&b);
288
289 Label l1;
290 a.movi(v0.v4s(), 0);
291
292 // Branch to unbound label.
293 auto b1 = a.offset<uint32_t*>();
294 a.tbnz(x0, 4, l1);
295
296 a.movi(v1.v4s(), 0);
297 a.bind(l1);
298
299 EXPECT_INSTR(0x37200040, *b1);
300
301 a.movi(v2.v4s(), 0);
302
303 // Branch to bound label.
304 auto b2 = a.offset<uint32_t*>();
305 a.tbnz(x1, 6, l1);
306
307 EXPECT_INSTR(0x3737FFE1, *b2);
308
309 ASSERT_EQ(xnn_status_success, xnn_release_code_memory(&b));
310 }
311
TEST(AArch64Assembler,Tbz)312 TEST(AArch64Assembler, Tbz) {
313 xnn_code_buffer b;
314 xnn_allocate_code_memory(&b, XNN_DEFAULT_CODE_BUFFER_SIZE);
315 Assembler a(&b);
316
317 Label l1;
318 a.movi(v0.v4s(), 0);
319
320 // Branch to unbound label.
321 auto b1 = a.offset<uint32_t*>();
322 a.tbz(x0, 4, l1);
323
324 a.movi(v1.v4s(), 0);
325 a.bind(l1);
326
327 EXPECT_INSTR(0x36200040, *b1);
328
329 a.movi(v2.v4s(), 0);
330
331 // Branch to bound label.
332 auto b2 = a.offset<uint32_t*>();
333 a.tbz(x1, 6, l1);
334
335 EXPECT_INSTR(0x3637FFE1, *b2);
336
337 ASSERT_EQ(xnn_status_success, xnn_release_code_memory(&b));
338 }
339
TEST(AArch64Assembler,UnconditionalBranch)340 TEST(AArch64Assembler, UnconditionalBranch) {
341 xnn_code_buffer b;
342 xnn_allocate_code_memory(&b, XNN_DEFAULT_CODE_BUFFER_SIZE);
343 Assembler a(&b);
344
345 Label l1;
346 a.movi(v0.v4s(), 0);
347
348 // Branch to unbound label.
349 auto b1 = a.offset<uint32_t*>();
350 a.b(l1);
351
352 a.movi(v1.v4s(), 0);
353 a.bind(l1);
354
355 EXPECT_INSTR(0x14000002, *b1);
356
357 a.movi(v2.v4s(), 0);
358
359 // Branch to bound label.
360 auto b2 = a.offset<uint32_t*>();
361 a.b(l1);
362
363 EXPECT_INSTR(0x17FFFFFF, *b2);
364
365 ASSERT_EQ(xnn_status_success, xnn_release_code_memory(&b));
366 }
367
368 } // namespace aarch64
369 } // namespace xnnpack
370