1 // Copyright 2018, VIXL authors
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 //
7 // * Redistributions of source code must retain the above copyright notice,
8 // this list of conditions and the following disclaimer.
9 // * Redistributions in binary form must reproduce the above copyright notice,
10 // this list of conditions and the following disclaimer in the documentation
11 // and/or other materials provided with the distribution.
12 // * Neither the name of Arm Limited nor the names of its contributors may be
13 // used to endorse or promote products derived from this software without
14 // specific prior written permission.
15 //
16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
17 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27 #include "cpu-features.h"
28 #include "globals-vixl.h"
29 #include "utils-vixl.h"
30 #include "decoder-aarch64.h"
31
32 #include "cpu-features-auditor-aarch64.h"
33
34 namespace vixl {
35 namespace aarch64 {
36
37
38 const CPUFeaturesAuditor::FormToVisitorFnMap*
GetFormToVisitorFnMap()39 CPUFeaturesAuditor::GetFormToVisitorFnMap() {
40 static const FormToVisitorFnMap form_to_visitor = {
41 DEFAULT_FORM_TO_VISITOR_MAP(CPUFeaturesAuditor),
42 SIM_AUD_VISITOR_MAP(CPUFeaturesAuditor),
43 {"fcmla_asimdelem_c_h"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
44 {"fcmla_asimdelem_c_s"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
45 {"fmlal2_asimdelem_lh"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
46 {"fmlal_asimdelem_lh"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
47 {"fmla_asimdelem_rh_h"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
48 {"fmla_asimdelem_r_sd"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
49 {"fmlsl2_asimdelem_lh"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
50 {"fmlsl_asimdelem_lh"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
51 {"fmls_asimdelem_rh_h"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
52 {"fmls_asimdelem_r_sd"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
53 {"fmulx_asimdelem_rh_h"_h,
54 &CPUFeaturesAuditor::VisitNEONByIndexedElement},
55 {"fmulx_asimdelem_r_sd"_h,
56 &CPUFeaturesAuditor::VisitNEONByIndexedElement},
57 {"fmul_asimdelem_rh_h"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
58 {"fmul_asimdelem_r_sd"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
59 {"sdot_asimdelem_d"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
60 {"smlal_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
61 {"smlsl_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
62 {"smull_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
63 {"sqdmlal_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
64 {"sqdmlsl_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
65 {"sqdmull_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
66 {"udot_asimdelem_d"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
67 {"umlal_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
68 {"umlsl_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
69 {"umull_asimdelem_l"_h, &CPUFeaturesAuditor::VisitNEONByIndexedElement},
70 };
71 return &form_to_visitor;
72 }
73
74 // Every instruction must update last_instruction_, even if only to clear it,
75 // and every instruction must also update seen_ once it has been fully handled.
76 // This scope makes that simple, and allows early returns in the decode logic.
77 class CPUFeaturesAuditor::RecordInstructionFeaturesScope {
78 public:
RecordInstructionFeaturesScope(CPUFeaturesAuditor * auditor)79 explicit RecordInstructionFeaturesScope(CPUFeaturesAuditor* auditor)
80 : auditor_(auditor) {
81 auditor_->last_instruction_ = CPUFeatures::None();
82 }
~RecordInstructionFeaturesScope()83 ~RecordInstructionFeaturesScope() {
84 auditor_->seen_.Combine(auditor_->last_instruction_);
85 }
86
Record(const CPUFeatures & features)87 void Record(const CPUFeatures& features) {
88 auditor_->last_instruction_.Combine(features);
89 }
90
Record(CPUFeatures::Feature feature0,CPUFeatures::Feature feature1=CPUFeatures::kNone,CPUFeatures::Feature feature2=CPUFeatures::kNone,CPUFeatures::Feature feature3=CPUFeatures::kNone)91 void Record(CPUFeatures::Feature feature0,
92 CPUFeatures::Feature feature1 = CPUFeatures::kNone,
93 CPUFeatures::Feature feature2 = CPUFeatures::kNone,
94 CPUFeatures::Feature feature3 = CPUFeatures::kNone) {
95 auditor_->last_instruction_.Combine(feature0, feature1, feature2, feature3);
96 }
97
98 // If exactly one of a or b is known to be available, record it. Otherwise,
99 // record both. This is intended for encodings that can be provided by two
100 // different features.
RecordOneOrBothOf(CPUFeatures::Feature a,CPUFeatures::Feature b)101 void RecordOneOrBothOf(CPUFeatures::Feature a, CPUFeatures::Feature b) {
102 bool hint_a = auditor_->available_.Has(a);
103 bool hint_b = auditor_->available_.Has(b);
104 if (hint_a && !hint_b) {
105 Record(a);
106 } else if (hint_b && !hint_a) {
107 Record(b);
108 } else {
109 Record(a, b);
110 }
111 }
112
113 private:
114 CPUFeaturesAuditor* auditor_;
115 };
116
LoadStoreHelper(const Instruction * instr)117 void CPUFeaturesAuditor::LoadStoreHelper(const Instruction* instr) {
118 RecordInstructionFeaturesScope scope(this);
119 switch (instr->Mask(LoadStoreMask)) {
120 case LDR_b:
121 case LDR_q:
122 case STR_b:
123 case STR_q:
124 scope.Record(CPUFeatures::kNEON);
125 return;
126 case LDR_h:
127 case LDR_s:
128 case LDR_d:
129 case STR_h:
130 case STR_s:
131 case STR_d:
132 scope.RecordOneOrBothOf(CPUFeatures::kFP, CPUFeatures::kNEON);
133 return;
134 default:
135 // No special CPU features.
136 return;
137 }
138 }
139
LoadStorePairHelper(const Instruction * instr)140 void CPUFeaturesAuditor::LoadStorePairHelper(const Instruction* instr) {
141 RecordInstructionFeaturesScope scope(this);
142 switch (instr->Mask(LoadStorePairMask)) {
143 case LDP_q:
144 case STP_q:
145 scope.Record(CPUFeatures::kNEON);
146 return;
147 case LDP_s:
148 case LDP_d:
149 case STP_s:
150 case STP_d: {
151 scope.RecordOneOrBothOf(CPUFeatures::kFP, CPUFeatures::kNEON);
152 return;
153 }
154 default:
155 // No special CPU features.
156 return;
157 }
158 }
159
VisitAddSubExtended(const Instruction * instr)160 void CPUFeaturesAuditor::VisitAddSubExtended(const Instruction* instr) {
161 RecordInstructionFeaturesScope scope(this);
162 USE(instr);
163 }
164
VisitAddSubImmediate(const Instruction * instr)165 void CPUFeaturesAuditor::VisitAddSubImmediate(const Instruction* instr) {
166 RecordInstructionFeaturesScope scope(this);
167 USE(instr);
168 }
169
VisitAddSubShifted(const Instruction * instr)170 void CPUFeaturesAuditor::VisitAddSubShifted(const Instruction* instr) {
171 RecordInstructionFeaturesScope scope(this);
172 USE(instr);
173 }
174
VisitAddSubWithCarry(const Instruction * instr)175 void CPUFeaturesAuditor::VisitAddSubWithCarry(const Instruction* instr) {
176 RecordInstructionFeaturesScope scope(this);
177 USE(instr);
178 }
179
VisitRotateRightIntoFlags(const Instruction * instr)180 void CPUFeaturesAuditor::VisitRotateRightIntoFlags(const Instruction* instr) {
181 RecordInstructionFeaturesScope scope(this);
182 switch (instr->Mask(RotateRightIntoFlagsMask)) {
183 case RMIF:
184 scope.Record(CPUFeatures::kFlagM);
185 return;
186 }
187 }
188
VisitEvaluateIntoFlags(const Instruction * instr)189 void CPUFeaturesAuditor::VisitEvaluateIntoFlags(const Instruction* instr) {
190 RecordInstructionFeaturesScope scope(this);
191 switch (instr->Mask(EvaluateIntoFlagsMask)) {
192 case SETF8:
193 case SETF16:
194 scope.Record(CPUFeatures::kFlagM);
195 return;
196 }
197 }
198
VisitAtomicMemory(const Instruction * instr)199 void CPUFeaturesAuditor::VisitAtomicMemory(const Instruction* instr) {
200 RecordInstructionFeaturesScope scope(this);
201 switch (instr->Mask(AtomicMemoryMask)) {
202 case LDAPRB:
203 case LDAPRH:
204 case LDAPR_w:
205 case LDAPR_x:
206 scope.Record(CPUFeatures::kRCpc);
207 return;
208 default:
209 // Everything else belongs to the Atomics extension.
210 scope.Record(CPUFeatures::kAtomics);
211 return;
212 }
213 }
214
VisitBitfield(const Instruction * instr)215 void CPUFeaturesAuditor::VisitBitfield(const Instruction* instr) {
216 RecordInstructionFeaturesScope scope(this);
217 USE(instr);
218 }
219
VisitCompareBranch(const Instruction * instr)220 void CPUFeaturesAuditor::VisitCompareBranch(const Instruction* instr) {
221 RecordInstructionFeaturesScope scope(this);
222 USE(instr);
223 }
224
VisitConditionalBranch(const Instruction * instr)225 void CPUFeaturesAuditor::VisitConditionalBranch(const Instruction* instr) {
226 RecordInstructionFeaturesScope scope(this);
227 USE(instr);
228 }
229
VisitConditionalCompareImmediate(const Instruction * instr)230 void CPUFeaturesAuditor::VisitConditionalCompareImmediate(
231 const Instruction* instr) {
232 RecordInstructionFeaturesScope scope(this);
233 USE(instr);
234 }
235
VisitConditionalCompareRegister(const Instruction * instr)236 void CPUFeaturesAuditor::VisitConditionalCompareRegister(
237 const Instruction* instr) {
238 RecordInstructionFeaturesScope scope(this);
239 USE(instr);
240 }
241
VisitConditionalSelect(const Instruction * instr)242 void CPUFeaturesAuditor::VisitConditionalSelect(const Instruction* instr) {
243 RecordInstructionFeaturesScope scope(this);
244 USE(instr);
245 }
246
VisitCrypto2RegSHA(const Instruction * instr)247 void CPUFeaturesAuditor::VisitCrypto2RegSHA(const Instruction* instr) {
248 RecordInstructionFeaturesScope scope(this);
249 USE(instr);
250 }
251
VisitCrypto3RegSHA(const Instruction * instr)252 void CPUFeaturesAuditor::VisitCrypto3RegSHA(const Instruction* instr) {
253 RecordInstructionFeaturesScope scope(this);
254 USE(instr);
255 }
256
VisitCryptoAES(const Instruction * instr)257 void CPUFeaturesAuditor::VisitCryptoAES(const Instruction* instr) {
258 RecordInstructionFeaturesScope scope(this);
259 USE(instr);
260 }
261
VisitDataProcessing1Source(const Instruction * instr)262 void CPUFeaturesAuditor::VisitDataProcessing1Source(const Instruction* instr) {
263 RecordInstructionFeaturesScope scope(this);
264 switch (instr->Mask(DataProcessing1SourceMask)) {
265 case PACIA:
266 case PACIB:
267 case PACDA:
268 case PACDB:
269 case AUTIA:
270 case AUTIB:
271 case AUTDA:
272 case AUTDB:
273 case PACIZA:
274 case PACIZB:
275 case PACDZA:
276 case PACDZB:
277 case AUTIZA:
278 case AUTIZB:
279 case AUTDZA:
280 case AUTDZB:
281 case XPACI:
282 case XPACD:
283 scope.Record(CPUFeatures::kPAuth);
284 return;
285 default:
286 // No special CPU features.
287 return;
288 }
289 }
290
VisitDataProcessing2Source(const Instruction * instr)291 void CPUFeaturesAuditor::VisitDataProcessing2Source(const Instruction* instr) {
292 RecordInstructionFeaturesScope scope(this);
293 switch (instr->Mask(DataProcessing2SourceMask)) {
294 case CRC32B:
295 case CRC32H:
296 case CRC32W:
297 case CRC32X:
298 case CRC32CB:
299 case CRC32CH:
300 case CRC32CW:
301 case CRC32CX:
302 scope.Record(CPUFeatures::kCRC32);
303 return;
304 case PACGA:
305 scope.Record(CPUFeatures::kPAuth, CPUFeatures::kPAuthGeneric);
306 return;
307 default:
308 // No special CPU features.
309 return;
310 }
311 }
312
VisitLoadStoreRCpcUnscaledOffset(const Instruction * instr)313 void CPUFeaturesAuditor::VisitLoadStoreRCpcUnscaledOffset(
314 const Instruction* instr) {
315 RecordInstructionFeaturesScope scope(this);
316 switch (instr->Mask(LoadStoreRCpcUnscaledOffsetMask)) {
317 case LDAPURB:
318 case LDAPURSB_w:
319 case LDAPURSB_x:
320 case LDAPURH:
321 case LDAPURSH_w:
322 case LDAPURSH_x:
323 case LDAPUR_w:
324 case LDAPURSW:
325 case LDAPUR_x:
326
327 // These stores don't actually have RCpc semantics but they're included with
328 // the RCpc extensions.
329 case STLURB:
330 case STLURH:
331 case STLUR_w:
332 case STLUR_x:
333 scope.Record(CPUFeatures::kRCpc, CPUFeatures::kRCpcImm);
334 return;
335 }
336 }
337
VisitLoadStorePAC(const Instruction * instr)338 void CPUFeaturesAuditor::VisitLoadStorePAC(const Instruction* instr) {
339 RecordInstructionFeaturesScope scope(this);
340 USE(instr);
341 scope.Record(CPUFeatures::kPAuth);
342 }
343
VisitDataProcessing3Source(const Instruction * instr)344 void CPUFeaturesAuditor::VisitDataProcessing3Source(const Instruction* instr) {
345 RecordInstructionFeaturesScope scope(this);
346 USE(instr);
347 }
348
VisitException(const Instruction * instr)349 void CPUFeaturesAuditor::VisitException(const Instruction* instr) {
350 RecordInstructionFeaturesScope scope(this);
351 USE(instr);
352 }
353
VisitExtract(const Instruction * instr)354 void CPUFeaturesAuditor::VisitExtract(const Instruction* instr) {
355 RecordInstructionFeaturesScope scope(this);
356 USE(instr);
357 }
358
VisitFPCompare(const Instruction * instr)359 void CPUFeaturesAuditor::VisitFPCompare(const Instruction* instr) {
360 RecordInstructionFeaturesScope scope(this);
361 // All of these instructions require FP.
362 scope.Record(CPUFeatures::kFP);
363 switch (instr->Mask(FPCompareMask)) {
364 case FCMP_h:
365 case FCMP_h_zero:
366 case FCMPE_h:
367 case FCMPE_h_zero:
368 scope.Record(CPUFeatures::kFPHalf);
369 return;
370 default:
371 // No special CPU features.
372 return;
373 }
374 }
375
VisitFPConditionalCompare(const Instruction * instr)376 void CPUFeaturesAuditor::VisitFPConditionalCompare(const Instruction* instr) {
377 RecordInstructionFeaturesScope scope(this);
378 // All of these instructions require FP.
379 scope.Record(CPUFeatures::kFP);
380 switch (instr->Mask(FPConditionalCompareMask)) {
381 case FCCMP_h:
382 case FCCMPE_h:
383 scope.Record(CPUFeatures::kFPHalf);
384 return;
385 default:
386 // No special CPU features.
387 return;
388 }
389 }
390
VisitFPConditionalSelect(const Instruction * instr)391 void CPUFeaturesAuditor::VisitFPConditionalSelect(const Instruction* instr) {
392 RecordInstructionFeaturesScope scope(this);
393 // All of these instructions require FP.
394 scope.Record(CPUFeatures::kFP);
395 if (instr->Mask(FPConditionalSelectMask) == FCSEL_h) {
396 scope.Record(CPUFeatures::kFPHalf);
397 }
398 }
399
VisitFPDataProcessing1Source(const Instruction * instr)400 void CPUFeaturesAuditor::VisitFPDataProcessing1Source(
401 const Instruction* instr) {
402 RecordInstructionFeaturesScope scope(this);
403 // All of these instructions require FP.
404 scope.Record(CPUFeatures::kFP);
405 switch (instr->Mask(FPDataProcessing1SourceMask)) {
406 case FMOV_h:
407 case FABS_h:
408 case FNEG_h:
409 case FSQRT_h:
410 case FRINTN_h:
411 case FRINTP_h:
412 case FRINTM_h:
413 case FRINTZ_h:
414 case FRINTA_h:
415 case FRINTX_h:
416 case FRINTI_h:
417 scope.Record(CPUFeatures::kFPHalf);
418 return;
419 case FRINT32X_s:
420 case FRINT32X_d:
421 case FRINT32Z_s:
422 case FRINT32Z_d:
423 case FRINT64X_s:
424 case FRINT64X_d:
425 case FRINT64Z_s:
426 case FRINT64Z_d:
427 scope.Record(CPUFeatures::kFrintToFixedSizedInt);
428 return;
429 default:
430 // No special CPU features.
431 // This category includes some half-precision FCVT instructions that do
432 // not require FPHalf.
433 return;
434 }
435 }
436
VisitFPDataProcessing2Source(const Instruction * instr)437 void CPUFeaturesAuditor::VisitFPDataProcessing2Source(
438 const Instruction* instr) {
439 RecordInstructionFeaturesScope scope(this);
440 // All of these instructions require FP.
441 scope.Record(CPUFeatures::kFP);
442 switch (instr->Mask(FPDataProcessing2SourceMask)) {
443 case FMUL_h:
444 case FDIV_h:
445 case FADD_h:
446 case FSUB_h:
447 case FMAX_h:
448 case FMIN_h:
449 case FMAXNM_h:
450 case FMINNM_h:
451 case FNMUL_h:
452 scope.Record(CPUFeatures::kFPHalf);
453 return;
454 default:
455 // No special CPU features.
456 return;
457 }
458 }
459
VisitFPDataProcessing3Source(const Instruction * instr)460 void CPUFeaturesAuditor::VisitFPDataProcessing3Source(
461 const Instruction* instr) {
462 RecordInstructionFeaturesScope scope(this);
463 // All of these instructions require FP.
464 scope.Record(CPUFeatures::kFP);
465 switch (instr->Mask(FPDataProcessing3SourceMask)) {
466 case FMADD_h:
467 case FMSUB_h:
468 case FNMADD_h:
469 case FNMSUB_h:
470 scope.Record(CPUFeatures::kFPHalf);
471 return;
472 default:
473 // No special CPU features.
474 return;
475 }
476 }
477
VisitFPFixedPointConvert(const Instruction * instr)478 void CPUFeaturesAuditor::VisitFPFixedPointConvert(const Instruction* instr) {
479 RecordInstructionFeaturesScope scope(this);
480 // All of these instructions require FP.
481 scope.Record(CPUFeatures::kFP);
482 switch (instr->Mask(FPFixedPointConvertMask)) {
483 case FCVTZS_wh_fixed:
484 case FCVTZS_xh_fixed:
485 case FCVTZU_wh_fixed:
486 case FCVTZU_xh_fixed:
487 case SCVTF_hw_fixed:
488 case SCVTF_hx_fixed:
489 case UCVTF_hw_fixed:
490 case UCVTF_hx_fixed:
491 scope.Record(CPUFeatures::kFPHalf);
492 return;
493 default:
494 // No special CPU features.
495 return;
496 }
497 }
498
VisitFPImmediate(const Instruction * instr)499 void CPUFeaturesAuditor::VisitFPImmediate(const Instruction* instr) {
500 RecordInstructionFeaturesScope scope(this);
501 // All of these instructions require FP.
502 scope.Record(CPUFeatures::kFP);
503 if (instr->Mask(FPImmediateMask) == FMOV_h_imm) {
504 scope.Record(CPUFeatures::kFPHalf);
505 }
506 }
507
VisitFPIntegerConvert(const Instruction * instr)508 void CPUFeaturesAuditor::VisitFPIntegerConvert(const Instruction* instr) {
509 RecordInstructionFeaturesScope scope(this);
510 // All of these instructions require FP.
511 scope.Record(CPUFeatures::kFP);
512 switch (instr->Mask(FPIntegerConvertMask)) {
513 case FCVTAS_wh:
514 case FCVTAS_xh:
515 case FCVTAU_wh:
516 case FCVTAU_xh:
517 case FCVTMS_wh:
518 case FCVTMS_xh:
519 case FCVTMU_wh:
520 case FCVTMU_xh:
521 case FCVTNS_wh:
522 case FCVTNS_xh:
523 case FCVTNU_wh:
524 case FCVTNU_xh:
525 case FCVTPS_wh:
526 case FCVTPS_xh:
527 case FCVTPU_wh:
528 case FCVTPU_xh:
529 case FCVTZS_wh:
530 case FCVTZS_xh:
531 case FCVTZU_wh:
532 case FCVTZU_xh:
533 case FMOV_hw:
534 case FMOV_hx:
535 case FMOV_wh:
536 case FMOV_xh:
537 case SCVTF_hw:
538 case SCVTF_hx:
539 case UCVTF_hw:
540 case UCVTF_hx:
541 scope.Record(CPUFeatures::kFPHalf);
542 return;
543 case FMOV_d1_x:
544 case FMOV_x_d1:
545 scope.Record(CPUFeatures::kNEON);
546 return;
547 case FJCVTZS:
548 scope.Record(CPUFeatures::kJSCVT);
549 return;
550 default:
551 // No special CPU features.
552 return;
553 }
554 }
555
VisitLoadLiteral(const Instruction * instr)556 void CPUFeaturesAuditor::VisitLoadLiteral(const Instruction* instr) {
557 RecordInstructionFeaturesScope scope(this);
558 switch (instr->Mask(LoadLiteralMask)) {
559 case LDR_s_lit:
560 case LDR_d_lit:
561 scope.RecordOneOrBothOf(CPUFeatures::kFP, CPUFeatures::kNEON);
562 return;
563 case LDR_q_lit:
564 scope.Record(CPUFeatures::kNEON);
565 return;
566 default:
567 // No special CPU features.
568 return;
569 }
570 }
571
VisitLoadStoreExclusive(const Instruction * instr)572 void CPUFeaturesAuditor::VisitLoadStoreExclusive(const Instruction* instr) {
573 RecordInstructionFeaturesScope scope(this);
574 switch (instr->Mask(LoadStoreExclusiveMask)) {
575 case CAS_w:
576 case CASA_w:
577 case CASL_w:
578 case CASAL_w:
579 case CAS_x:
580 case CASA_x:
581 case CASL_x:
582 case CASAL_x:
583 case CASB:
584 case CASAB:
585 case CASLB:
586 case CASALB:
587 case CASH:
588 case CASAH:
589 case CASLH:
590 case CASALH:
591 case CASP_w:
592 case CASPA_w:
593 case CASPL_w:
594 case CASPAL_w:
595 case CASP_x:
596 case CASPA_x:
597 case CASPL_x:
598 case CASPAL_x:
599 scope.Record(CPUFeatures::kAtomics);
600 return;
601 case STLLRB:
602 case LDLARB:
603 case STLLRH:
604 case LDLARH:
605 case STLLR_w:
606 case LDLAR_w:
607 case STLLR_x:
608 case LDLAR_x:
609 scope.Record(CPUFeatures::kLORegions);
610 return;
611 default:
612 // No special CPU features.
613 return;
614 }
615 }
616
VisitLoadStorePairNonTemporal(const Instruction * instr)617 void CPUFeaturesAuditor::VisitLoadStorePairNonTemporal(
618 const Instruction* instr) {
619 LoadStorePairHelper(instr);
620 }
621
VisitLoadStorePairOffset(const Instruction * instr)622 void CPUFeaturesAuditor::VisitLoadStorePairOffset(const Instruction* instr) {
623 LoadStorePairHelper(instr);
624 }
625
VisitLoadStorePairPostIndex(const Instruction * instr)626 void CPUFeaturesAuditor::VisitLoadStorePairPostIndex(const Instruction* instr) {
627 LoadStorePairHelper(instr);
628 }
629
VisitLoadStorePairPreIndex(const Instruction * instr)630 void CPUFeaturesAuditor::VisitLoadStorePairPreIndex(const Instruction* instr) {
631 LoadStorePairHelper(instr);
632 }
633
VisitLoadStorePostIndex(const Instruction * instr)634 void CPUFeaturesAuditor::VisitLoadStorePostIndex(const Instruction* instr) {
635 LoadStoreHelper(instr);
636 }
637
VisitLoadStorePreIndex(const Instruction * instr)638 void CPUFeaturesAuditor::VisitLoadStorePreIndex(const Instruction* instr) {
639 LoadStoreHelper(instr);
640 }
641
VisitLoadStoreRegisterOffset(const Instruction * instr)642 void CPUFeaturesAuditor::VisitLoadStoreRegisterOffset(
643 const Instruction* instr) {
644 LoadStoreHelper(instr);
645 }
646
VisitLoadStoreUnscaledOffset(const Instruction * instr)647 void CPUFeaturesAuditor::VisitLoadStoreUnscaledOffset(
648 const Instruction* instr) {
649 LoadStoreHelper(instr);
650 }
651
VisitLoadStoreUnsignedOffset(const Instruction * instr)652 void CPUFeaturesAuditor::VisitLoadStoreUnsignedOffset(
653 const Instruction* instr) {
654 LoadStoreHelper(instr);
655 }
656
VisitLogicalImmediate(const Instruction * instr)657 void CPUFeaturesAuditor::VisitLogicalImmediate(const Instruction* instr) {
658 RecordInstructionFeaturesScope scope(this);
659 USE(instr);
660 }
661
VisitLogicalShifted(const Instruction * instr)662 void CPUFeaturesAuditor::VisitLogicalShifted(const Instruction* instr) {
663 RecordInstructionFeaturesScope scope(this);
664 USE(instr);
665 }
666
VisitMoveWideImmediate(const Instruction * instr)667 void CPUFeaturesAuditor::VisitMoveWideImmediate(const Instruction* instr) {
668 RecordInstructionFeaturesScope scope(this);
669 USE(instr);
670 }
671
VisitNEON2RegMisc(const Instruction * instr)672 void CPUFeaturesAuditor::VisitNEON2RegMisc(const Instruction* instr) {
673 RecordInstructionFeaturesScope scope(this);
674 // All of these instructions require NEON.
675 scope.Record(CPUFeatures::kNEON);
676 switch (instr->Mask(NEON2RegMiscFPMask)) {
677 case NEON_FABS:
678 case NEON_FNEG:
679 case NEON_FSQRT:
680 case NEON_FCVTL:
681 case NEON_FCVTN:
682 case NEON_FCVTXN:
683 case NEON_FRINTI:
684 case NEON_FRINTX:
685 case NEON_FRINTA:
686 case NEON_FRINTM:
687 case NEON_FRINTN:
688 case NEON_FRINTP:
689 case NEON_FRINTZ:
690 case NEON_FCVTNS:
691 case NEON_FCVTNU:
692 case NEON_FCVTPS:
693 case NEON_FCVTPU:
694 case NEON_FCVTMS:
695 case NEON_FCVTMU:
696 case NEON_FCVTZS:
697 case NEON_FCVTZU:
698 case NEON_FCVTAS:
699 case NEON_FCVTAU:
700 case NEON_SCVTF:
701 case NEON_UCVTF:
702 case NEON_FRSQRTE:
703 case NEON_FRECPE:
704 case NEON_FCMGT_zero:
705 case NEON_FCMGE_zero:
706 case NEON_FCMEQ_zero:
707 case NEON_FCMLE_zero:
708 case NEON_FCMLT_zero:
709 scope.Record(CPUFeatures::kFP);
710 return;
711 case NEON_FRINT32X:
712 case NEON_FRINT32Z:
713 case NEON_FRINT64X:
714 case NEON_FRINT64Z:
715 scope.Record(CPUFeatures::kFP, CPUFeatures::kFrintToFixedSizedInt);
716 return;
717 default:
718 // No additional features.
719 return;
720 }
721 }
722
VisitNEON2RegMiscFP16(const Instruction * instr)723 void CPUFeaturesAuditor::VisitNEON2RegMiscFP16(const Instruction* instr) {
724 RecordInstructionFeaturesScope scope(this);
725 // All of these instructions require NEONHalf.
726 scope.Record(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kNEONHalf);
727 USE(instr);
728 }
729
VisitNEON3Different(const Instruction * instr)730 void CPUFeaturesAuditor::VisitNEON3Different(const Instruction* instr) {
731 RecordInstructionFeaturesScope scope(this);
732 // All of these instructions require NEON.
733 scope.Record(CPUFeatures::kNEON);
734 USE(instr);
735 }
736
VisitNEON3Same(const Instruction * instr)737 void CPUFeaturesAuditor::VisitNEON3Same(const Instruction* instr) {
738 RecordInstructionFeaturesScope scope(this);
739 // All of these instructions require NEON.
740 scope.Record(CPUFeatures::kNEON);
741 if (instr->Mask(NEON3SameFPFMask) == NEON3SameFPFixed) {
742 scope.Record(CPUFeatures::kFP);
743 }
744 switch (instr->Mask(NEON3SameFHMMask)) {
745 case NEON_FMLAL:
746 case NEON_FMLAL2:
747 case NEON_FMLSL:
748 case NEON_FMLSL2:
749 scope.Record(CPUFeatures::kFP, CPUFeatures::kNEONHalf, CPUFeatures::kFHM);
750 return;
751 default:
752 // No additional features.
753 return;
754 }
755 }
756
VisitNEON3SameExtra(const Instruction * instr)757 void CPUFeaturesAuditor::VisitNEON3SameExtra(const Instruction* instr) {
758 RecordInstructionFeaturesScope scope(this);
759 // All of these instructions require NEON.
760 scope.Record(CPUFeatures::kNEON);
761 if ((instr->Mask(NEON3SameExtraFCMLAMask) == NEON_FCMLA) ||
762 (instr->Mask(NEON3SameExtraFCADDMask) == NEON_FCADD)) {
763 scope.Record(CPUFeatures::kFP, CPUFeatures::kFcma);
764 if (instr->GetNEONSize() == 1) scope.Record(CPUFeatures::kNEONHalf);
765 } else {
766 switch (instr->Mask(NEON3SameExtraMask)) {
767 case NEON_SDOT:
768 case NEON_UDOT:
769 scope.Record(CPUFeatures::kDotProduct);
770 return;
771 case NEON_SQRDMLAH:
772 case NEON_SQRDMLSH:
773 scope.Record(CPUFeatures::kRDM);
774 return;
775 default:
776 // No additional features.
777 return;
778 }
779 }
780 }
781
VisitNEON3SameFP16(const Instruction * instr)782 void CPUFeaturesAuditor::VisitNEON3SameFP16(const Instruction* instr) {
783 RecordInstructionFeaturesScope scope(this);
784 // All of these instructions require NEON FP16 support.
785 scope.Record(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kNEONHalf);
786 USE(instr);
787 }
788
VisitNEONAcrossLanes(const Instruction * instr)789 void CPUFeaturesAuditor::VisitNEONAcrossLanes(const Instruction* instr) {
790 RecordInstructionFeaturesScope scope(this);
791 // All of these instructions require NEON.
792 scope.Record(CPUFeatures::kNEON);
793 if (instr->Mask(NEONAcrossLanesFP16FMask) == NEONAcrossLanesFP16Fixed) {
794 // FMAXV_H, FMINV_H, FMAXNMV_H, FMINNMV_H
795 scope.Record(CPUFeatures::kFP, CPUFeatures::kNEONHalf);
796 } else if (instr->Mask(NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
797 // FMAXV, FMINV, FMAXNMV, FMINNMV
798 scope.Record(CPUFeatures::kFP);
799 }
800 }
801
VisitNEONByIndexedElement(const Instruction * instr)802 void CPUFeaturesAuditor::VisitNEONByIndexedElement(const Instruction* instr) {
803 RecordInstructionFeaturesScope scope(this);
804 // All of these instructions require NEON.
805 scope.Record(CPUFeatures::kNEON);
806 switch (instr->Mask(NEONByIndexedElementMask)) {
807 case NEON_SDOT_byelement:
808 case NEON_UDOT_byelement:
809 scope.Record(CPUFeatures::kDotProduct);
810 return;
811 case NEON_SQRDMLAH_byelement:
812 case NEON_SQRDMLSH_byelement:
813 scope.Record(CPUFeatures::kRDM);
814 return;
815 default:
816 // Fall through to check other instructions.
817 break;
818 }
819 switch (instr->Mask(NEONByIndexedElementFPLongMask)) {
820 case NEON_FMLAL_H_byelement:
821 case NEON_FMLAL2_H_byelement:
822 case NEON_FMLSL_H_byelement:
823 case NEON_FMLSL2_H_byelement:
824 scope.Record(CPUFeatures::kFP, CPUFeatures::kNEONHalf, CPUFeatures::kFHM);
825 return;
826 default:
827 // Fall through to check other instructions.
828 break;
829 }
830 switch (instr->Mask(NEONByIndexedElementFPMask)) {
831 case NEON_FMLA_H_byelement:
832 case NEON_FMLS_H_byelement:
833 case NEON_FMUL_H_byelement:
834 case NEON_FMULX_H_byelement:
835 scope.Record(CPUFeatures::kNEONHalf);
836 VIXL_FALLTHROUGH();
837 case NEON_FMLA_byelement:
838 case NEON_FMLS_byelement:
839 case NEON_FMUL_byelement:
840 case NEON_FMULX_byelement:
841 scope.Record(CPUFeatures::kFP);
842 return;
843 default:
844 switch (instr->Mask(NEONByIndexedElementFPComplexMask)) {
845 case NEON_FCMLA_byelement:
846 scope.Record(CPUFeatures::kFP, CPUFeatures::kFcma);
847 if (instr->GetNEONSize() == 1) scope.Record(CPUFeatures::kNEONHalf);
848 return;
849 }
850 // No additional features.
851 return;
852 }
853 }
854
VisitNEONCopy(const Instruction * instr)855 void CPUFeaturesAuditor::VisitNEONCopy(const Instruction* instr) {
856 RecordInstructionFeaturesScope scope(this);
857 // All of these instructions require NEON.
858 scope.Record(CPUFeatures::kNEON);
859 USE(instr);
860 }
861
VisitNEONExtract(const Instruction * instr)862 void CPUFeaturesAuditor::VisitNEONExtract(const Instruction* instr) {
863 RecordInstructionFeaturesScope scope(this);
864 // All of these instructions require NEON.
865 scope.Record(CPUFeatures::kNEON);
866 USE(instr);
867 }
868
VisitNEONLoadStoreMultiStruct(const Instruction * instr)869 void CPUFeaturesAuditor::VisitNEONLoadStoreMultiStruct(
870 const Instruction* instr) {
871 RecordInstructionFeaturesScope scope(this);
872 // All of these instructions require NEON.
873 scope.Record(CPUFeatures::kNEON);
874 USE(instr);
875 }
876
VisitNEONLoadStoreMultiStructPostIndex(const Instruction * instr)877 void CPUFeaturesAuditor::VisitNEONLoadStoreMultiStructPostIndex(
878 const Instruction* instr) {
879 RecordInstructionFeaturesScope scope(this);
880 // All of these instructions require NEON.
881 scope.Record(CPUFeatures::kNEON);
882 USE(instr);
883 }
884
VisitNEONLoadStoreSingleStruct(const Instruction * instr)885 void CPUFeaturesAuditor::VisitNEONLoadStoreSingleStruct(
886 const Instruction* instr) {
887 RecordInstructionFeaturesScope scope(this);
888 // All of these instructions require NEON.
889 scope.Record(CPUFeatures::kNEON);
890 USE(instr);
891 }
892
VisitNEONLoadStoreSingleStructPostIndex(const Instruction * instr)893 void CPUFeaturesAuditor::VisitNEONLoadStoreSingleStructPostIndex(
894 const Instruction* instr) {
895 RecordInstructionFeaturesScope scope(this);
896 // All of these instructions require NEON.
897 scope.Record(CPUFeatures::kNEON);
898 USE(instr);
899 }
900
VisitNEONModifiedImmediate(const Instruction * instr)901 void CPUFeaturesAuditor::VisitNEONModifiedImmediate(const Instruction* instr) {
902 RecordInstructionFeaturesScope scope(this);
903 // All of these instructions require NEON.
904 scope.Record(CPUFeatures::kNEON);
905 if (instr->GetNEONCmode() == 0xf) {
906 // FMOV (vector, immediate), double-, single- or half-precision.
907 scope.Record(CPUFeatures::kFP);
908 if (instr->ExtractBit(11)) scope.Record(CPUFeatures::kNEONHalf);
909 }
910 }
911
VisitNEONPerm(const Instruction * instr)912 void CPUFeaturesAuditor::VisitNEONPerm(const Instruction* instr) {
913 RecordInstructionFeaturesScope scope(this);
914 // All of these instructions require NEON.
915 scope.Record(CPUFeatures::kNEON);
916 USE(instr);
917 }
918
VisitNEONScalar2RegMisc(const Instruction * instr)919 void CPUFeaturesAuditor::VisitNEONScalar2RegMisc(const Instruction* instr) {
920 RecordInstructionFeaturesScope scope(this);
921 // All of these instructions require NEON.
922 scope.Record(CPUFeatures::kNEON);
923 switch (instr->Mask(NEONScalar2RegMiscFPMask)) {
924 case NEON_FRECPE_scalar:
925 case NEON_FRECPX_scalar:
926 case NEON_FRSQRTE_scalar:
927 case NEON_FCMGT_zero_scalar:
928 case NEON_FCMGE_zero_scalar:
929 case NEON_FCMEQ_zero_scalar:
930 case NEON_FCMLE_zero_scalar:
931 case NEON_FCMLT_zero_scalar:
932 case NEON_SCVTF_scalar:
933 case NEON_UCVTF_scalar:
934 case NEON_FCVTNS_scalar:
935 case NEON_FCVTNU_scalar:
936 case NEON_FCVTPS_scalar:
937 case NEON_FCVTPU_scalar:
938 case NEON_FCVTMS_scalar:
939 case NEON_FCVTMU_scalar:
940 case NEON_FCVTZS_scalar:
941 case NEON_FCVTZU_scalar:
942 case NEON_FCVTAS_scalar:
943 case NEON_FCVTAU_scalar:
944 case NEON_FCVTXN_scalar:
945 scope.Record(CPUFeatures::kFP);
946 return;
947 default:
948 // No additional features.
949 return;
950 }
951 }
952
VisitNEONScalar2RegMiscFP16(const Instruction * instr)953 void CPUFeaturesAuditor::VisitNEONScalar2RegMiscFP16(const Instruction* instr) {
954 RecordInstructionFeaturesScope scope(this);
955 // All of these instructions require NEONHalf.
956 scope.Record(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kNEONHalf);
957 USE(instr);
958 }
959
VisitNEONScalar3Diff(const Instruction * instr)960 void CPUFeaturesAuditor::VisitNEONScalar3Diff(const Instruction* instr) {
961 RecordInstructionFeaturesScope scope(this);
962 // All of these instructions require NEON.
963 scope.Record(CPUFeatures::kNEON);
964 USE(instr);
965 }
966
VisitNEONScalar3Same(const Instruction * instr)967 void CPUFeaturesAuditor::VisitNEONScalar3Same(const Instruction* instr) {
968 RecordInstructionFeaturesScope scope(this);
969 // All of these instructions require NEON.
970 scope.Record(CPUFeatures::kNEON);
971 if (instr->Mask(NEONScalar3SameFPFMask) == NEONScalar3SameFPFixed) {
972 scope.Record(CPUFeatures::kFP);
973 }
974 }
975
VisitNEONScalar3SameExtra(const Instruction * instr)976 void CPUFeaturesAuditor::VisitNEONScalar3SameExtra(const Instruction* instr) {
977 RecordInstructionFeaturesScope scope(this);
978 // All of these instructions require NEON and RDM.
979 scope.Record(CPUFeatures::kNEON, CPUFeatures::kRDM);
980 USE(instr);
981 }
982
VisitNEONScalar3SameFP16(const Instruction * instr)983 void CPUFeaturesAuditor::VisitNEONScalar3SameFP16(const Instruction* instr) {
984 RecordInstructionFeaturesScope scope(this);
985 // All of these instructions require NEONHalf.
986 scope.Record(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::kNEONHalf);
987 USE(instr);
988 }
989
VisitNEONScalarByIndexedElement(const Instruction * instr)990 void CPUFeaturesAuditor::VisitNEONScalarByIndexedElement(
991 const Instruction* instr) {
992 RecordInstructionFeaturesScope scope(this);
993 // All of these instructions require NEON.
994 scope.Record(CPUFeatures::kNEON);
995 switch (instr->Mask(NEONScalarByIndexedElementMask)) {
996 case NEON_SQRDMLAH_byelement_scalar:
997 case NEON_SQRDMLSH_byelement_scalar:
998 scope.Record(CPUFeatures::kRDM);
999 return;
1000 default:
1001 switch (instr->Mask(NEONScalarByIndexedElementFPMask)) {
1002 case NEON_FMLA_H_byelement_scalar:
1003 case NEON_FMLS_H_byelement_scalar:
1004 case NEON_FMUL_H_byelement_scalar:
1005 case NEON_FMULX_H_byelement_scalar:
1006 scope.Record(CPUFeatures::kNEONHalf);
1007 VIXL_FALLTHROUGH();
1008 case NEON_FMLA_byelement_scalar:
1009 case NEON_FMLS_byelement_scalar:
1010 case NEON_FMUL_byelement_scalar:
1011 case NEON_FMULX_byelement_scalar:
1012 scope.Record(CPUFeatures::kFP);
1013 return;
1014 }
1015 // No additional features.
1016 return;
1017 }
1018 }
1019
VisitNEONScalarCopy(const Instruction * instr)1020 void CPUFeaturesAuditor::VisitNEONScalarCopy(const Instruction* instr) {
1021 RecordInstructionFeaturesScope scope(this);
1022 // All of these instructions require NEON.
1023 scope.Record(CPUFeatures::kNEON);
1024 USE(instr);
1025 }
1026
VisitNEONScalarPairwise(const Instruction * instr)1027 void CPUFeaturesAuditor::VisitNEONScalarPairwise(const Instruction* instr) {
1028 RecordInstructionFeaturesScope scope(this);
1029 // All of these instructions require NEON.
1030 scope.Record(CPUFeatures::kNEON);
1031 switch (instr->Mask(NEONScalarPairwiseMask)) {
1032 case NEON_FMAXNMP_h_scalar:
1033 case NEON_FADDP_h_scalar:
1034 case NEON_FMAXP_h_scalar:
1035 case NEON_FMINNMP_h_scalar:
1036 case NEON_FMINP_h_scalar:
1037 scope.Record(CPUFeatures::kNEONHalf);
1038 VIXL_FALLTHROUGH();
1039 case NEON_FADDP_scalar:
1040 case NEON_FMAXP_scalar:
1041 case NEON_FMAXNMP_scalar:
1042 case NEON_FMINP_scalar:
1043 case NEON_FMINNMP_scalar:
1044 scope.Record(CPUFeatures::kFP);
1045 return;
1046 default:
1047 // No additional features.
1048 return;
1049 }
1050 }
1051
VisitNEONScalarShiftImmediate(const Instruction * instr)1052 void CPUFeaturesAuditor::VisitNEONScalarShiftImmediate(
1053 const Instruction* instr) {
1054 RecordInstructionFeaturesScope scope(this);
1055 // All of these instructions require NEON.
1056 scope.Record(CPUFeatures::kNEON);
1057 switch (instr->Mask(NEONScalarShiftImmediateMask)) {
1058 case NEON_FCVTZS_imm_scalar:
1059 case NEON_FCVTZU_imm_scalar:
1060 case NEON_SCVTF_imm_scalar:
1061 case NEON_UCVTF_imm_scalar:
1062 scope.Record(CPUFeatures::kFP);
1063 // If immh is 0b001x then the data type is FP16, and requires kNEONHalf.
1064 if ((instr->GetImmNEONImmh() & 0xe) == 0x2) {
1065 scope.Record(CPUFeatures::kNEONHalf);
1066 }
1067 return;
1068 default:
1069 // No additional features.
1070 return;
1071 }
1072 }
1073
VisitNEONShiftImmediate(const Instruction * instr)1074 void CPUFeaturesAuditor::VisitNEONShiftImmediate(const Instruction* instr) {
1075 RecordInstructionFeaturesScope scope(this);
1076 // All of these instructions require NEON.
1077 scope.Record(CPUFeatures::kNEON);
1078 switch (instr->Mask(NEONShiftImmediateMask)) {
1079 case NEON_SCVTF_imm:
1080 case NEON_UCVTF_imm:
1081 case NEON_FCVTZS_imm:
1082 case NEON_FCVTZU_imm:
1083 scope.Record(CPUFeatures::kFP);
1084 // If immh is 0b001x then the data type is FP16, and requires kNEONHalf.
1085 if ((instr->GetImmNEONImmh() & 0xe) == 0x2) {
1086 scope.Record(CPUFeatures::kNEONHalf);
1087 }
1088 return;
1089 default:
1090 // No additional features.
1091 return;
1092 }
1093 }
1094
VisitNEONTable(const Instruction * instr)1095 void CPUFeaturesAuditor::VisitNEONTable(const Instruction* instr) {
1096 RecordInstructionFeaturesScope scope(this);
1097 // All of these instructions require NEON.
1098 scope.Record(CPUFeatures::kNEON);
1099 USE(instr);
1100 }
1101
VisitPCRelAddressing(const Instruction * instr)1102 void CPUFeaturesAuditor::VisitPCRelAddressing(const Instruction* instr) {
1103 RecordInstructionFeaturesScope scope(this);
1104 USE(instr);
1105 }
1106
1107 // Most SVE visitors require only SVE.
1108 #define VIXL_SIMPLE_SVE_VISITOR_LIST(V) \
1109 V(SVE32BitGatherLoad_ScalarPlus32BitUnscaledOffsets) \
1110 V(SVE32BitGatherLoad_VectorPlusImm) \
1111 V(SVE32BitGatherLoadHalfwords_ScalarPlus32BitScaledOffsets) \
1112 V(SVE32BitGatherLoadWords_ScalarPlus32BitScaledOffsets) \
1113 V(SVE32BitGatherPrefetch_ScalarPlus32BitScaledOffsets) \
1114 V(SVE32BitGatherPrefetch_VectorPlusImm) \
1115 V(SVE32BitScatterStore_ScalarPlus32BitScaledOffsets) \
1116 V(SVE32BitScatterStore_ScalarPlus32BitUnscaledOffsets) \
1117 V(SVE32BitScatterStore_VectorPlusImm) \
1118 V(SVE64BitGatherLoad_ScalarPlus32BitUnpackedScaledOffsets) \
1119 V(SVE64BitGatherLoad_ScalarPlus64BitScaledOffsets) \
1120 V(SVE64BitGatherLoad_ScalarPlus64BitUnscaledOffsets) \
1121 V(SVE64BitGatherLoad_ScalarPlusUnpacked32BitUnscaledOffsets) \
1122 V(SVE64BitGatherLoad_VectorPlusImm) \
1123 V(SVE64BitGatherPrefetch_ScalarPlus64BitScaledOffsets) \
1124 V(SVE64BitGatherPrefetch_ScalarPlusUnpacked32BitScaledOffsets) \
1125 V(SVE64BitGatherPrefetch_VectorPlusImm) \
1126 V(SVE64BitScatterStore_ScalarPlus64BitScaledOffsets) \
1127 V(SVE64BitScatterStore_ScalarPlus64BitUnscaledOffsets) \
1128 V(SVE64BitScatterStore_ScalarPlusUnpacked32BitScaledOffsets) \
1129 V(SVE64BitScatterStore_ScalarPlusUnpacked32BitUnscaledOffsets) \
1130 V(SVE64BitScatterStore_VectorPlusImm) \
1131 V(SVEAddressGeneration) \
1132 V(SVEBitwiseLogicalUnpredicated) \
1133 V(SVEBitwiseShiftUnpredicated) \
1134 V(SVEFFRInitialise) \
1135 V(SVEFFRWriteFromPredicate) \
1136 V(SVEFPAccumulatingReduction) \
1137 V(SVEFPArithmeticUnpredicated) \
1138 V(SVEFPCompareVectors) \
1139 V(SVEFPCompareWithZero) \
1140 V(SVEFPComplexAddition) \
1141 V(SVEFPComplexMulAdd) \
1142 V(SVEFPComplexMulAddIndex) \
1143 V(SVEFPFastReduction) \
1144 V(SVEFPMulIndex) \
1145 V(SVEFPMulAdd) \
1146 V(SVEFPMulAddIndex) \
1147 V(SVEFPUnaryOpUnpredicated) \
1148 V(SVEIncDecByPredicateCount) \
1149 V(SVEIndexGeneration) \
1150 V(SVEIntArithmeticUnpredicated) \
1151 V(SVEIntCompareSignedImm) \
1152 V(SVEIntCompareUnsignedImm) \
1153 V(SVEIntCompareVectors) \
1154 V(SVEIntMulAddPredicated) \
1155 V(SVEIntMulAddUnpredicated) \
1156 V(SVEIntReduction) \
1157 V(SVEIntUnaryArithmeticPredicated) \
1158 V(SVEMovprfx) \
1159 V(SVEMulIndex) \
1160 V(SVEPermuteVectorExtract) \
1161 V(SVEPermuteVectorInterleaving) \
1162 V(SVEPredicateCount) \
1163 V(SVEPredicateLogical) \
1164 V(SVEPropagateBreak) \
1165 V(SVEStackFrameAdjustment) \
1166 V(SVEStackFrameSize) \
1167 V(SVEVectorSelect) \
1168 V(SVEBitwiseLogical_Predicated) \
1169 V(SVEBitwiseLogicalWithImm_Unpredicated) \
1170 V(SVEBitwiseShiftByImm_Predicated) \
1171 V(SVEBitwiseShiftByVector_Predicated) \
1172 V(SVEBitwiseShiftByWideElements_Predicated) \
1173 V(SVEBroadcastBitmaskImm) \
1174 V(SVEBroadcastFPImm_Unpredicated) \
1175 V(SVEBroadcastGeneralRegister) \
1176 V(SVEBroadcastIndexElement) \
1177 V(SVEBroadcastIntImm_Unpredicated) \
1178 V(SVECompressActiveElements) \
1179 V(SVEConditionallyBroadcastElementToVector) \
1180 V(SVEConditionallyExtractElementToSIMDFPScalar) \
1181 V(SVEConditionallyExtractElementToGeneralRegister) \
1182 V(SVEConditionallyTerminateScalars) \
1183 V(SVEConstructivePrefix_Unpredicated) \
1184 V(SVEContiguousFirstFaultLoad_ScalarPlusScalar) \
1185 V(SVEContiguousLoad_ScalarPlusImm) \
1186 V(SVEContiguousLoad_ScalarPlusScalar) \
1187 V(SVEContiguousNonFaultLoad_ScalarPlusImm) \
1188 V(SVEContiguousNonTemporalLoad_ScalarPlusImm) \
1189 V(SVEContiguousNonTemporalLoad_ScalarPlusScalar) \
1190 V(SVEContiguousNonTemporalStore_ScalarPlusImm) \
1191 V(SVEContiguousNonTemporalStore_ScalarPlusScalar) \
1192 V(SVEContiguousPrefetch_ScalarPlusImm) \
1193 V(SVEContiguousPrefetch_ScalarPlusScalar) \
1194 V(SVEContiguousStore_ScalarPlusImm) \
1195 V(SVEContiguousStore_ScalarPlusScalar) \
1196 V(SVECopySIMDFPScalarRegisterToVector_Predicated) \
1197 V(SVECopyFPImm_Predicated) \
1198 V(SVECopyGeneralRegisterToVector_Predicated) \
1199 V(SVECopyIntImm_Predicated) \
1200 V(SVEElementCount) \
1201 V(SVEExtractElementToSIMDFPScalarRegister) \
1202 V(SVEExtractElementToGeneralRegister) \
1203 V(SVEFPArithmetic_Predicated) \
1204 V(SVEFPArithmeticWithImm_Predicated) \
1205 V(SVEFPConvertPrecision) \
1206 V(SVEFPConvertToInt) \
1207 V(SVEFPExponentialAccelerator) \
1208 V(SVEFPRoundToIntegralValue) \
1209 V(SVEFPTrigMulAddCoefficient) \
1210 V(SVEFPTrigSelectCoefficient) \
1211 V(SVEFPUnaryOp) \
1212 V(SVEIncDecRegisterByElementCount) \
1213 V(SVEIncDecVectorByElementCount) \
1214 V(SVEInsertSIMDFPScalarRegister) \
1215 V(SVEInsertGeneralRegister) \
1216 V(SVEIntAddSubtractImm_Unpredicated) \
1217 V(SVEIntAddSubtractVectors_Predicated) \
1218 V(SVEIntCompareScalarCountAndLimit) \
1219 V(SVEIntConvertToFP) \
1220 V(SVEIntDivideVectors_Predicated) \
1221 V(SVEIntMinMaxImm_Unpredicated) \
1222 V(SVEIntMinMaxDifference_Predicated) \
1223 V(SVEIntMulImm_Unpredicated) \
1224 V(SVEIntMulVectors_Predicated) \
1225 V(SVELoadAndBroadcastElement) \
1226 V(SVELoadAndBroadcastQOWord_ScalarPlusImm) \
1227 V(SVELoadAndBroadcastQOWord_ScalarPlusScalar) \
1228 V(SVELoadMultipleStructures_ScalarPlusImm) \
1229 V(SVELoadMultipleStructures_ScalarPlusScalar) \
1230 V(SVELoadPredicateRegister) \
1231 V(SVELoadVectorRegister) \
1232 V(SVEPartitionBreakCondition) \
1233 V(SVEPermutePredicateElements) \
1234 V(SVEPredicateFirstActive) \
1235 V(SVEPredicateInitialize) \
1236 V(SVEPredicateNextActive) \
1237 V(SVEPredicateReadFromFFR_Predicated) \
1238 V(SVEPredicateReadFromFFR_Unpredicated) \
1239 V(SVEPredicateTest) \
1240 V(SVEPredicateZero) \
1241 V(SVEPropagateBreakToNextPartition) \
1242 V(SVEReversePredicateElements) \
1243 V(SVEReverseVectorElements) \
1244 V(SVEReverseWithinElements) \
1245 V(SVESaturatingIncDecRegisterByElementCount) \
1246 V(SVESaturatingIncDecVectorByElementCount) \
1247 V(SVEStoreMultipleStructures_ScalarPlusImm) \
1248 V(SVEStoreMultipleStructures_ScalarPlusScalar) \
1249 V(SVEStorePredicateRegister) \
1250 V(SVEStoreVectorRegister) \
1251 V(SVETableLookup) \
1252 V(SVEUnpackPredicateElements) \
1253 V(SVEUnpackVectorElements) \
1254 V(SVEVectorSplice)
1255
1256 #define VIXL_DEFINE_SIMPLE_SVE_VISITOR(NAME) \
1257 void CPUFeaturesAuditor::Visit##NAME(const Instruction* instr) { \
1258 RecordInstructionFeaturesScope scope(this); \
1259 scope.Record(CPUFeatures::kSVE); \
1260 USE(instr); \
1261 }
VIXL_SIMPLE_SVE_VISITOR_LIST(VIXL_DEFINE_SIMPLE_SVE_VISITOR)1262 VIXL_SIMPLE_SVE_VISITOR_LIST(VIXL_DEFINE_SIMPLE_SVE_VISITOR)
1263 #undef VIXL_DEFINE_SIMPLE_SVE_VISITOR
1264 #undef VIXL_SIMPLE_SVE_VISITOR_LIST
1265
1266 void CPUFeaturesAuditor::VisitSystem(const Instruction* instr) {
1267 RecordInstructionFeaturesScope scope(this);
1268 if (instr->Mask(SystemHintFMask) == SystemHintFixed) {
1269 CPUFeatures required;
1270 switch (instr->GetInstructionBits()) {
1271 case PACIA1716:
1272 case PACIB1716:
1273 case AUTIA1716:
1274 case AUTIB1716:
1275 case PACIAZ:
1276 case PACIASP:
1277 case PACIBZ:
1278 case PACIBSP:
1279 case AUTIAZ:
1280 case AUTIASP:
1281 case AUTIBZ:
1282 case AUTIBSP:
1283 case XPACLRI:
1284 required.Combine(CPUFeatures::kPAuth);
1285 break;
1286 default:
1287 switch (instr->GetImmHint()) {
1288 case ESB:
1289 required.Combine(CPUFeatures::kRAS);
1290 break;
1291 case BTI:
1292 case BTI_j:
1293 case BTI_c:
1294 case BTI_jc:
1295 required.Combine(CPUFeatures::kBTI);
1296 break;
1297 default:
1298 break;
1299 }
1300 break;
1301 }
1302
1303 // These are all HINT instructions, and behave as NOPs if the corresponding
1304 // features are not implemented, so we record the corresponding features
1305 // only if they are available.
1306 if (available_.Has(required)) scope.Record(required);
1307 } else if (instr->Mask(SystemSysMask) == SYS) {
1308 switch (instr->GetSysOp()) {
1309 // DC instruction variants.
1310 case CVAP:
1311 scope.Record(CPUFeatures::kDCPoP);
1312 break;
1313 case CVADP:
1314 scope.Record(CPUFeatures::kDCCVADP);
1315 break;
1316 case IVAU:
1317 case CVAC:
1318 case CVAU:
1319 case CIVAC:
1320 // No special CPU features.
1321 break;
1322 }
1323 } else if (instr->Mask(SystemPStateFMask) == SystemPStateFixed) {
1324 switch (instr->Mask(SystemPStateMask)) {
1325 case CFINV:
1326 scope.Record(CPUFeatures::kFlagM);
1327 break;
1328 case AXFLAG:
1329 case XAFLAG:
1330 scope.Record(CPUFeatures::kAXFlag);
1331 break;
1332 }
1333 } else if (instr->Mask(SystemSysRegFMask) == SystemSysRegFixed) {
1334 if (instr->Mask(SystemSysRegMask) == MRS) {
1335 switch (instr->GetImmSystemRegister()) {
1336 case RNDR:
1337 case RNDRRS:
1338 scope.Record(CPUFeatures::kRNG);
1339 break;
1340 }
1341 }
1342 }
1343 }
1344
VisitTestBranch(const Instruction * instr)1345 void CPUFeaturesAuditor::VisitTestBranch(const Instruction* instr) {
1346 RecordInstructionFeaturesScope scope(this);
1347 USE(instr);
1348 }
1349
VisitUnallocated(const Instruction * instr)1350 void CPUFeaturesAuditor::VisitUnallocated(const Instruction* instr) {
1351 RecordInstructionFeaturesScope scope(this);
1352 USE(instr);
1353 }
1354
VisitUnconditionalBranch(const Instruction * instr)1355 void CPUFeaturesAuditor::VisitUnconditionalBranch(const Instruction* instr) {
1356 RecordInstructionFeaturesScope scope(this);
1357 USE(instr);
1358 }
1359
VisitUnconditionalBranchToRegister(const Instruction * instr)1360 void CPUFeaturesAuditor::VisitUnconditionalBranchToRegister(
1361 const Instruction* instr) {
1362 RecordInstructionFeaturesScope scope(this);
1363 switch (instr->Mask(UnconditionalBranchToRegisterMask)) {
1364 case BRAAZ:
1365 case BRABZ:
1366 case BLRAAZ:
1367 case BLRABZ:
1368 case RETAA:
1369 case RETAB:
1370 case BRAA:
1371 case BRAB:
1372 case BLRAA:
1373 case BLRAB:
1374 scope.Record(CPUFeatures::kPAuth);
1375 return;
1376 default:
1377 // No additional features.
1378 return;
1379 }
1380 }
1381
VisitReserved(const Instruction * instr)1382 void CPUFeaturesAuditor::VisitReserved(const Instruction* instr) {
1383 RecordInstructionFeaturesScope scope(this);
1384 USE(instr);
1385 }
1386
VisitUnimplemented(const Instruction * instr)1387 void CPUFeaturesAuditor::VisitUnimplemented(const Instruction* instr) {
1388 RecordInstructionFeaturesScope scope(this);
1389 USE(instr);
1390 }
1391
Visit(Metadata * metadata,const Instruction * instr)1392 void CPUFeaturesAuditor::Visit(Metadata* metadata, const Instruction* instr) {
1393 VIXL_ASSERT(metadata->count("form") > 0);
1394 const std::string& form = (*metadata)["form"];
1395 uint32_t form_hash = Hash(form.c_str());
1396 const FormToVisitorFnMap* fv = CPUFeaturesAuditor::GetFormToVisitorFnMap();
1397 FormToVisitorFnMap::const_iterator it = fv->find(form_hash);
1398 if (it == fv->end()) {
1399 RecordInstructionFeaturesScope scope(this);
1400 std::map<uint32_t, const CPUFeatures> features = {
1401 {"adclb_z_zzz"_h, CPUFeatures::kSVE2},
1402 {"adclt_z_zzz"_h, CPUFeatures::kSVE2},
1403 {"addhnb_z_zz"_h, CPUFeatures::kSVE2},
1404 {"addhnt_z_zz"_h, CPUFeatures::kSVE2},
1405 {"addp_z_p_zz"_h, CPUFeatures::kSVE2},
1406 {"bcax_z_zzz"_h, CPUFeatures::kSVE2},
1407 {"bdep_z_zz"_h,
1408 CPUFeatures(CPUFeatures::kSVE2, CPUFeatures::kSVEBitPerm)},
1409 {"bext_z_zz"_h,
1410 CPUFeatures(CPUFeatures::kSVE2, CPUFeatures::kSVEBitPerm)},
1411 {"bgrp_z_zz"_h,
1412 CPUFeatures(CPUFeatures::kSVE2, CPUFeatures::kSVEBitPerm)},
1413 {"bsl1n_z_zzz"_h, CPUFeatures::kSVE2},
1414 {"bsl2n_z_zzz"_h, CPUFeatures::kSVE2},
1415 {"bsl_z_zzz"_h, CPUFeatures::kSVE2},
1416 {"cadd_z_zz"_h, CPUFeatures::kSVE2},
1417 {"cdot_z_zzz"_h, CPUFeatures::kSVE2},
1418 {"cdot_z_zzzi_d"_h, CPUFeatures::kSVE2},
1419 {"cdot_z_zzzi_s"_h, CPUFeatures::kSVE2},
1420 {"cmla_z_zzz"_h, CPUFeatures::kSVE2},
1421 {"cmla_z_zzzi_h"_h, CPUFeatures::kSVE2},
1422 {"cmla_z_zzzi_s"_h, CPUFeatures::kSVE2},
1423 {"eor3_z_zzz"_h, CPUFeatures::kSVE2},
1424 {"eorbt_z_zz"_h, CPUFeatures::kSVE2},
1425 {"eortb_z_zz"_h, CPUFeatures::kSVE2},
1426 {"ext_z_zi_con"_h, CPUFeatures::kSVE2},
1427 {"faddp_z_p_zz"_h, CPUFeatures::kSVE2},
1428 {"fcvtlt_z_p_z_h2s"_h, CPUFeatures::kSVE2},
1429 {"fcvtlt_z_p_z_s2d"_h, CPUFeatures::kSVE2},
1430 {"fcvtnt_z_p_z_d2s"_h, CPUFeatures::kSVE2},
1431 {"fcvtnt_z_p_z_s2h"_h, CPUFeatures::kSVE2},
1432 {"fcvtx_z_p_z_d2s"_h, CPUFeatures::kSVE2},
1433 {"fcvtxnt_z_p_z_d2s"_h, CPUFeatures::kSVE2},
1434 {"flogb_z_p_z"_h, CPUFeatures::kSVE2},
1435 {"fmaxnmp_z_p_zz"_h, CPUFeatures::kSVE2},
1436 {"fmaxp_z_p_zz"_h, CPUFeatures::kSVE2},
1437 {"fminnmp_z_p_zz"_h, CPUFeatures::kSVE2},
1438 {"fminp_z_p_zz"_h, CPUFeatures::kSVE2},
1439 {"fmlalb_z_zzz"_h, CPUFeatures::kSVE2},
1440 {"fmlalb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1441 {"fmlalt_z_zzz"_h, CPUFeatures::kSVE2},
1442 {"fmlalt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1443 {"fmlslb_z_zzz"_h, CPUFeatures::kSVE2},
1444 {"fmlslb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1445 {"fmlslt_z_zzz"_h, CPUFeatures::kSVE2},
1446 {"fmlslt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1447 {"histcnt_z_p_zz"_h, CPUFeatures::kSVE2},
1448 {"histseg_z_zz"_h, CPUFeatures::kSVE2},
1449 {"ldnt1b_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1450 {"ldnt1b_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1451 {"ldnt1d_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1452 {"ldnt1h_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1453 {"ldnt1h_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1454 {"ldnt1sb_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1455 {"ldnt1sb_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1456 {"ldnt1sh_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1457 {"ldnt1sh_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1458 {"ldnt1sw_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1459 {"ldnt1w_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1460 {"ldnt1w_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1461 {"match_p_p_zz"_h, CPUFeatures::kSVE2},
1462 {"mla_z_zzzi_d"_h, CPUFeatures::kSVE2},
1463 {"mla_z_zzzi_h"_h, CPUFeatures::kSVE2},
1464 {"mla_z_zzzi_s"_h, CPUFeatures::kSVE2},
1465 {"mls_z_zzzi_d"_h, CPUFeatures::kSVE2},
1466 {"mls_z_zzzi_h"_h, CPUFeatures::kSVE2},
1467 {"mls_z_zzzi_s"_h, CPUFeatures::kSVE2},
1468 {"mul_z_zz"_h, CPUFeatures::kSVE2},
1469 {"mul_z_zzi_d"_h, CPUFeatures::kSVE2},
1470 {"mul_z_zzi_h"_h, CPUFeatures::kSVE2},
1471 {"mul_z_zzi_s"_h, CPUFeatures::kSVE2},
1472 {"nbsl_z_zzz"_h, CPUFeatures::kSVE2},
1473 {"nmatch_p_p_zz"_h, CPUFeatures::kSVE2},
1474 {"pmul_z_zz"_h, CPUFeatures::kSVE2},
1475 {"pmullb_z_zz"_h, CPUFeatures::kSVE2},
1476 {"pmullt_z_zz"_h, CPUFeatures::kSVE2},
1477 {"raddhnb_z_zz"_h, CPUFeatures::kSVE2},
1478 {"raddhnt_z_zz"_h, CPUFeatures::kSVE2},
1479 {"rshrnb_z_zi"_h, CPUFeatures::kSVE2},
1480 {"rshrnt_z_zi"_h, CPUFeatures::kSVE2},
1481 {"rsubhnb_z_zz"_h, CPUFeatures::kSVE2},
1482 {"rsubhnt_z_zz"_h, CPUFeatures::kSVE2},
1483 {"saba_z_zzz"_h, CPUFeatures::kSVE2},
1484 {"sabalb_z_zzz"_h, CPUFeatures::kSVE2},
1485 {"sabalt_z_zzz"_h, CPUFeatures::kSVE2},
1486 {"sabdlb_z_zz"_h, CPUFeatures::kSVE2},
1487 {"sabdlt_z_zz"_h, CPUFeatures::kSVE2},
1488 {"sadalp_z_p_z"_h, CPUFeatures::kSVE2},
1489 {"saddlb_z_zz"_h, CPUFeatures::kSVE2},
1490 {"saddlbt_z_zz"_h, CPUFeatures::kSVE2},
1491 {"saddlt_z_zz"_h, CPUFeatures::kSVE2},
1492 {"saddwb_z_zz"_h, CPUFeatures::kSVE2},
1493 {"saddwt_z_zz"_h, CPUFeatures::kSVE2},
1494 {"sbclb_z_zzz"_h, CPUFeatures::kSVE2},
1495 {"sbclt_z_zzz"_h, CPUFeatures::kSVE2},
1496 {"shadd_z_p_zz"_h, CPUFeatures::kSVE2},
1497 {"shrnb_z_zi"_h, CPUFeatures::kSVE2},
1498 {"shrnt_z_zi"_h, CPUFeatures::kSVE2},
1499 {"shsub_z_p_zz"_h, CPUFeatures::kSVE2},
1500 {"shsubr_z_p_zz"_h, CPUFeatures::kSVE2},
1501 {"sli_z_zzi"_h, CPUFeatures::kSVE2},
1502 {"smaxp_z_p_zz"_h, CPUFeatures::kSVE2},
1503 {"sminp_z_p_zz"_h, CPUFeatures::kSVE2},
1504 {"smlalb_z_zzz"_h, CPUFeatures::kSVE2},
1505 {"smlalb_z_zzzi_d"_h, CPUFeatures::kSVE2},
1506 {"smlalb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1507 {"smlalt_z_zzz"_h, CPUFeatures::kSVE2},
1508 {"smlalt_z_zzzi_d"_h, CPUFeatures::kSVE2},
1509 {"smlalt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1510 {"smlslb_z_zzz"_h, CPUFeatures::kSVE2},
1511 {"smlslb_z_zzzi_d"_h, CPUFeatures::kSVE2},
1512 {"smlslb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1513 {"smlslt_z_zzz"_h, CPUFeatures::kSVE2},
1514 {"smlslt_z_zzzi_d"_h, CPUFeatures::kSVE2},
1515 {"smlslt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1516 {"smulh_z_zz"_h, CPUFeatures::kSVE2},
1517 {"smullb_z_zz"_h, CPUFeatures::kSVE2},
1518 {"smullb_z_zzi_d"_h, CPUFeatures::kSVE2},
1519 {"smullb_z_zzi_s"_h, CPUFeatures::kSVE2},
1520 {"smullt_z_zz"_h, CPUFeatures::kSVE2},
1521 {"smullt_z_zzi_d"_h, CPUFeatures::kSVE2},
1522 {"smullt_z_zzi_s"_h, CPUFeatures::kSVE2},
1523 {"splice_z_p_zz_con"_h, CPUFeatures::kSVE2},
1524 {"sqabs_z_p_z"_h, CPUFeatures::kSVE2},
1525 {"sqadd_z_p_zz"_h, CPUFeatures::kSVE2},
1526 {"sqcadd_z_zz"_h, CPUFeatures::kSVE2},
1527 {"sqdmlalb_z_zzz"_h, CPUFeatures::kSVE2},
1528 {"sqdmlalb_z_zzzi_d"_h, CPUFeatures::kSVE2},
1529 {"sqdmlalb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1530 {"sqdmlalbt_z_zzz"_h, CPUFeatures::kSVE2},
1531 {"sqdmlalt_z_zzz"_h, CPUFeatures::kSVE2},
1532 {"sqdmlalt_z_zzzi_d"_h, CPUFeatures::kSVE2},
1533 {"sqdmlalt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1534 {"sqdmlslb_z_zzz"_h, CPUFeatures::kSVE2},
1535 {"sqdmlslb_z_zzzi_d"_h, CPUFeatures::kSVE2},
1536 {"sqdmlslb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1537 {"sqdmlslbt_z_zzz"_h, CPUFeatures::kSVE2},
1538 {"sqdmlslt_z_zzz"_h, CPUFeatures::kSVE2},
1539 {"sqdmlslt_z_zzzi_d"_h, CPUFeatures::kSVE2},
1540 {"sqdmlslt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1541 {"sqdmulh_z_zz"_h, CPUFeatures::kSVE2},
1542 {"sqdmulh_z_zzi_d"_h, CPUFeatures::kSVE2},
1543 {"sqdmulh_z_zzi_h"_h, CPUFeatures::kSVE2},
1544 {"sqdmulh_z_zzi_s"_h, CPUFeatures::kSVE2},
1545 {"sqdmullb_z_zz"_h, CPUFeatures::kSVE2},
1546 {"sqdmullb_z_zzi_d"_h, CPUFeatures::kSVE2},
1547 {"sqdmullb_z_zzi_s"_h, CPUFeatures::kSVE2},
1548 {"sqdmullt_z_zz"_h, CPUFeatures::kSVE2},
1549 {"sqdmullt_z_zzi_d"_h, CPUFeatures::kSVE2},
1550 {"sqdmullt_z_zzi_s"_h, CPUFeatures::kSVE2},
1551 {"sqneg_z_p_z"_h, CPUFeatures::kSVE2},
1552 {"sqrdcmlah_z_zzz"_h, CPUFeatures::kSVE2},
1553 {"sqrdcmlah_z_zzzi_h"_h, CPUFeatures::kSVE2},
1554 {"sqrdcmlah_z_zzzi_s"_h, CPUFeatures::kSVE2},
1555 {"sqrdmlah_z_zzz"_h, CPUFeatures::kSVE2},
1556 {"sqrdmlah_z_zzzi_d"_h, CPUFeatures::kSVE2},
1557 {"sqrdmlah_z_zzzi_h"_h, CPUFeatures::kSVE2},
1558 {"sqrdmlah_z_zzzi_s"_h, CPUFeatures::kSVE2},
1559 {"sqrdmlsh_z_zzz"_h, CPUFeatures::kSVE2},
1560 {"sqrdmlsh_z_zzzi_d"_h, CPUFeatures::kSVE2},
1561 {"sqrdmlsh_z_zzzi_h"_h, CPUFeatures::kSVE2},
1562 {"sqrdmlsh_z_zzzi_s"_h, CPUFeatures::kSVE2},
1563 {"sqrdmulh_z_zz"_h, CPUFeatures::kSVE2},
1564 {"sqrdmulh_z_zzi_d"_h, CPUFeatures::kSVE2},
1565 {"sqrdmulh_z_zzi_h"_h, CPUFeatures::kSVE2},
1566 {"sqrdmulh_z_zzi_s"_h, CPUFeatures::kSVE2},
1567 {"sqrshl_z_p_zz"_h, CPUFeatures::kSVE2},
1568 {"sqrshlr_z_p_zz"_h, CPUFeatures::kSVE2},
1569 {"sqrshrnb_z_zi"_h, CPUFeatures::kSVE2},
1570 {"sqrshrnt_z_zi"_h, CPUFeatures::kSVE2},
1571 {"sqrshrunb_z_zi"_h, CPUFeatures::kSVE2},
1572 {"sqrshrunt_z_zi"_h, CPUFeatures::kSVE2},
1573 {"sqshl_z_p_zi"_h, CPUFeatures::kSVE2},
1574 {"sqshl_z_p_zz"_h, CPUFeatures::kSVE2},
1575 {"sqshlr_z_p_zz"_h, CPUFeatures::kSVE2},
1576 {"sqshlu_z_p_zi"_h, CPUFeatures::kSVE2},
1577 {"sqshrnb_z_zi"_h, CPUFeatures::kSVE2},
1578 {"sqshrnt_z_zi"_h, CPUFeatures::kSVE2},
1579 {"sqshrunb_z_zi"_h, CPUFeatures::kSVE2},
1580 {"sqshrunt_z_zi"_h, CPUFeatures::kSVE2},
1581 {"sqsub_z_p_zz"_h, CPUFeatures::kSVE2},
1582 {"sqsubr_z_p_zz"_h, CPUFeatures::kSVE2},
1583 {"sqxtnb_z_zz"_h, CPUFeatures::kSVE2},
1584 {"sqxtnt_z_zz"_h, CPUFeatures::kSVE2},
1585 {"sqxtunb_z_zz"_h, CPUFeatures::kSVE2},
1586 {"sqxtunt_z_zz"_h, CPUFeatures::kSVE2},
1587 {"srhadd_z_p_zz"_h, CPUFeatures::kSVE2},
1588 {"sri_z_zzi"_h, CPUFeatures::kSVE2},
1589 {"srshl_z_p_zz"_h, CPUFeatures::kSVE2},
1590 {"srshlr_z_p_zz"_h, CPUFeatures::kSVE2},
1591 {"srshr_z_p_zi"_h, CPUFeatures::kSVE2},
1592 {"srsra_z_zi"_h, CPUFeatures::kSVE2},
1593 {"sshllb_z_zi"_h, CPUFeatures::kSVE2},
1594 {"sshllt_z_zi"_h, CPUFeatures::kSVE2},
1595 {"ssra_z_zi"_h, CPUFeatures::kSVE2},
1596 {"ssublb_z_zz"_h, CPUFeatures::kSVE2},
1597 {"ssublbt_z_zz"_h, CPUFeatures::kSVE2},
1598 {"ssublt_z_zz"_h, CPUFeatures::kSVE2},
1599 {"ssubltb_z_zz"_h, CPUFeatures::kSVE2},
1600 {"ssubwb_z_zz"_h, CPUFeatures::kSVE2},
1601 {"ssubwt_z_zz"_h, CPUFeatures::kSVE2},
1602 {"stnt1b_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1603 {"stnt1b_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1604 {"stnt1d_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1605 {"stnt1h_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1606 {"stnt1h_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1607 {"stnt1w_z_p_ar_d_64_unscaled"_h, CPUFeatures::kSVE2},
1608 {"stnt1w_z_p_ar_s_x32_unscaled"_h, CPUFeatures::kSVE2},
1609 {"subhnb_z_zz"_h, CPUFeatures::kSVE2},
1610 {"subhnt_z_zz"_h, CPUFeatures::kSVE2},
1611 {"suqadd_z_p_zz"_h, CPUFeatures::kSVE2},
1612 {"tbl_z_zz_2"_h, CPUFeatures::kSVE2},
1613 {"tbx_z_zz"_h, CPUFeatures::kSVE2},
1614 {"uaba_z_zzz"_h, CPUFeatures::kSVE2},
1615 {"uabalb_z_zzz"_h, CPUFeatures::kSVE2},
1616 {"uabalt_z_zzz"_h, CPUFeatures::kSVE2},
1617 {"uabdlb_z_zz"_h, CPUFeatures::kSVE2},
1618 {"uabdlt_z_zz"_h, CPUFeatures::kSVE2},
1619 {"uadalp_z_p_z"_h, CPUFeatures::kSVE2},
1620 {"uaddlb_z_zz"_h, CPUFeatures::kSVE2},
1621 {"uaddlt_z_zz"_h, CPUFeatures::kSVE2},
1622 {"uaddwb_z_zz"_h, CPUFeatures::kSVE2},
1623 {"uaddwt_z_zz"_h, CPUFeatures::kSVE2},
1624 {"uhadd_z_p_zz"_h, CPUFeatures::kSVE2},
1625 {"uhsub_z_p_zz"_h, CPUFeatures::kSVE2},
1626 {"uhsubr_z_p_zz"_h, CPUFeatures::kSVE2},
1627 {"umaxp_z_p_zz"_h, CPUFeatures::kSVE2},
1628 {"uminp_z_p_zz"_h, CPUFeatures::kSVE2},
1629 {"umlalb_z_zzz"_h, CPUFeatures::kSVE2},
1630 {"umlalb_z_zzzi_d"_h, CPUFeatures::kSVE2},
1631 {"umlalb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1632 {"umlalt_z_zzz"_h, CPUFeatures::kSVE2},
1633 {"umlalt_z_zzzi_d"_h, CPUFeatures::kSVE2},
1634 {"umlalt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1635 {"umlslb_z_zzz"_h, CPUFeatures::kSVE2},
1636 {"umlslb_z_zzzi_d"_h, CPUFeatures::kSVE2},
1637 {"umlslb_z_zzzi_s"_h, CPUFeatures::kSVE2},
1638 {"umlslt_z_zzz"_h, CPUFeatures::kSVE2},
1639 {"umlslt_z_zzzi_d"_h, CPUFeatures::kSVE2},
1640 {"umlslt_z_zzzi_s"_h, CPUFeatures::kSVE2},
1641 {"umulh_z_zz"_h, CPUFeatures::kSVE2},
1642 {"umullb_z_zz"_h, CPUFeatures::kSVE2},
1643 {"umullb_z_zzi_d"_h, CPUFeatures::kSVE2},
1644 {"umullb_z_zzi_s"_h, CPUFeatures::kSVE2},
1645 {"umullt_z_zz"_h, CPUFeatures::kSVE2},
1646 {"umullt_z_zzi_d"_h, CPUFeatures::kSVE2},
1647 {"umullt_z_zzi_s"_h, CPUFeatures::kSVE2},
1648 {"uqadd_z_p_zz"_h, CPUFeatures::kSVE2},
1649 {"uqrshl_z_p_zz"_h, CPUFeatures::kSVE2},
1650 {"uqrshlr_z_p_zz"_h, CPUFeatures::kSVE2},
1651 {"uqrshrnb_z_zi"_h, CPUFeatures::kSVE2},
1652 {"uqrshrnt_z_zi"_h, CPUFeatures::kSVE2},
1653 {"uqshl_z_p_zi"_h, CPUFeatures::kSVE2},
1654 {"uqshl_z_p_zz"_h, CPUFeatures::kSVE2},
1655 {"uqshlr_z_p_zz"_h, CPUFeatures::kSVE2},
1656 {"uqshrnb_z_zi"_h, CPUFeatures::kSVE2},
1657 {"uqshrnt_z_zi"_h, CPUFeatures::kSVE2},
1658 {"uqsub_z_p_zz"_h, CPUFeatures::kSVE2},
1659 {"uqsubr_z_p_zz"_h, CPUFeatures::kSVE2},
1660 {"uqxtnb_z_zz"_h, CPUFeatures::kSVE2},
1661 {"uqxtnt_z_zz"_h, CPUFeatures::kSVE2},
1662 {"urecpe_z_p_z"_h, CPUFeatures::kSVE2},
1663 {"urhadd_z_p_zz"_h, CPUFeatures::kSVE2},
1664 {"urshl_z_p_zz"_h, CPUFeatures::kSVE2},
1665 {"urshlr_z_p_zz"_h, CPUFeatures::kSVE2},
1666 {"urshr_z_p_zi"_h, CPUFeatures::kSVE2},
1667 {"ursqrte_z_p_z"_h, CPUFeatures::kSVE2},
1668 {"ursra_z_zi"_h, CPUFeatures::kSVE2},
1669 {"ushllb_z_zi"_h, CPUFeatures::kSVE2},
1670 {"ushllt_z_zi"_h, CPUFeatures::kSVE2},
1671 {"usqadd_z_p_zz"_h, CPUFeatures::kSVE2},
1672 {"usra_z_zi"_h, CPUFeatures::kSVE2},
1673 {"usublb_z_zz"_h, CPUFeatures::kSVE2},
1674 {"usublt_z_zz"_h, CPUFeatures::kSVE2},
1675 {"usubwb_z_zz"_h, CPUFeatures::kSVE2},
1676 {"usubwt_z_zz"_h, CPUFeatures::kSVE2},
1677 {"whilege_p_p_rr"_h, CPUFeatures::kSVE2},
1678 {"whilegt_p_p_rr"_h, CPUFeatures::kSVE2},
1679 {"whilehi_p_p_rr"_h, CPUFeatures::kSVE2},
1680 {"whilehs_p_p_rr"_h, CPUFeatures::kSVE2},
1681 {"whilerw_p_rr"_h, CPUFeatures::kSVE2},
1682 {"whilewr_p_rr"_h, CPUFeatures::kSVE2},
1683 {"xar_z_zzi"_h, CPUFeatures::kSVE2},
1684 {"smmla_z_zzz"_h,
1685 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEI8MM)},
1686 {"ummla_z_zzz"_h,
1687 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEI8MM)},
1688 {"usmmla_z_zzz"_h,
1689 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEI8MM)},
1690 {"fmmla_z_zzz_s"_h,
1691 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF32MM)},
1692 {"fmmla_z_zzz_d"_h,
1693 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1694 {"smmla_asimdsame2_g"_h,
1695 CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kI8MM)},
1696 {"ummla_asimdsame2_g"_h,
1697 CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kI8MM)},
1698 {"usmmla_asimdsame2_g"_h,
1699 CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kI8MM)},
1700 {"ld1row_z_p_bi_u32"_h,
1701 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1702 {"ld1row_z_p_br_contiguous"_h,
1703 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1704 {"ld1rod_z_p_bi_u64"_h,
1705 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1706 {"ld1rod_z_p_br_contiguous"_h,
1707 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1708 {"ld1rob_z_p_bi_u8"_h,
1709 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1710 {"ld1rob_z_p_br_contiguous"_h,
1711 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1712 {"ld1roh_z_p_bi_u16"_h,
1713 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1714 {"ld1roh_z_p_br_contiguous"_h,
1715 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEF64MM)},
1716 {"usdot_asimdsame2_d"_h,
1717 CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kI8MM)},
1718 {"sudot_asimdelem_d"_h,
1719 CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kI8MM)},
1720 {"usdot_asimdelem_d"_h,
1721 CPUFeatures(CPUFeatures::kNEON, CPUFeatures::kI8MM)},
1722 {"usdot_z_zzz_s"_h,
1723 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEI8MM)},
1724 {"usdot_z_zzzi_s"_h,
1725 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEI8MM)},
1726 {"sudot_z_zzzi_s"_h,
1727 CPUFeatures(CPUFeatures::kSVE, CPUFeatures::kSVEI8MM)},
1728 };
1729
1730 if (features.count(form_hash) > 0) {
1731 scope.Record(features[form_hash]);
1732 }
1733 } else {
1734 (it->second)(this, instr);
1735 }
1736 }
1737
1738 } // namespace aarch64
1739 } // namespace vixl
1740