1 //===- SveEmitter.cpp - Generate arm_sve.h for use with clang -*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This tablegen backend is responsible for emitting arm_sve.h, which includes
10 // a declaration and definition of each function specified by the ARM C/C++
11 // Language Extensions (ACLE).
12 //
13 // For details, visit:
14 // https://developer.arm.com/architectures/system-architectures/software-standards/acle
15 //
16 // Each SVE instruction is implemented in terms of 1 or more functions which
17 // are suffixed with the element type of the input vectors. Functions may be
18 // implemented in terms of generic vector operations such as +, *, -, etc. or
19 // by calling a __builtin_-prefixed function which will be handled by clang's
20 // CodeGen library.
21 //
22 // See also the documentation in include/clang/Basic/arm_sve.td.
23 //
24 //===----------------------------------------------------------------------===//
25
26 #include "llvm/ADT/STLExtras.h"
27 #include "llvm/ADT/StringMap.h"
28 #include "llvm/ADT/ArrayRef.h"
29 #include "llvm/ADT/StringExtras.h"
30 #include "llvm/TableGen/Record.h"
31 #include "llvm/TableGen/Error.h"
32 #include <string>
33 #include <sstream>
34 #include <set>
35 #include <cctype>
36 #include <tuple>
37
38 using namespace llvm;
39
40 enum ClassKind {
41 ClassNone,
42 ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix
43 ClassG, // Overloaded name without type suffix
44 };
45
46 using TypeSpec = std::string;
47
48 namespace {
49
50 class ImmCheck {
51 unsigned Arg;
52 unsigned Kind;
53 unsigned ElementSizeInBits;
54
55 public:
ImmCheck(unsigned Arg,unsigned Kind,unsigned ElementSizeInBits=0)56 ImmCheck(unsigned Arg, unsigned Kind, unsigned ElementSizeInBits = 0)
57 : Arg(Arg), Kind(Kind), ElementSizeInBits(ElementSizeInBits) {}
58 ImmCheck(const ImmCheck &Other) = default;
59 ~ImmCheck() = default;
60
getArg() const61 unsigned getArg() const { return Arg; }
getKind() const62 unsigned getKind() const { return Kind; }
getElementSizeInBits() const63 unsigned getElementSizeInBits() const { return ElementSizeInBits; }
64 };
65
66 class SVEType {
67 TypeSpec TS;
68 bool Float, Signed, Immediate, Void, Constant, Pointer, BFloat;
69 bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp;
70 unsigned Bitwidth, ElementBitwidth, NumVectors;
71
72 public:
SVEType()73 SVEType() : SVEType(TypeSpec(), 'v') {}
74
SVEType(TypeSpec TS,char CharMod)75 SVEType(TypeSpec TS, char CharMod)
76 : TS(TS), Float(false), Signed(true), Immediate(false), Void(false),
77 Constant(false), Pointer(false), BFloat(false), DefaultType(false),
78 IsScalable(true), Predicate(false), PredicatePattern(false),
79 PrefetchOp(false), Bitwidth(128), ElementBitwidth(~0U), NumVectors(1) {
80 if (!TS.empty())
81 applyTypespec();
82 applyModifier(CharMod);
83 }
84
isPointer() const85 bool isPointer() const { return Pointer; }
isVoidPointer() const86 bool isVoidPointer() const { return Pointer && Void; }
isSigned() const87 bool isSigned() const { return Signed; }
isImmediate() const88 bool isImmediate() const { return Immediate; }
isScalar() const89 bool isScalar() const { return NumVectors == 0; }
isVector() const90 bool isVector() const { return NumVectors > 0; }
isScalableVector() const91 bool isScalableVector() const { return isVector() && IsScalable; }
isChar() const92 bool isChar() const { return ElementBitwidth == 8; }
isVoid() const93 bool isVoid() const { return Void & !Pointer; }
isDefault() const94 bool isDefault() const { return DefaultType; }
isFloat() const95 bool isFloat() const { return Float && !BFloat; }
isBFloat() const96 bool isBFloat() const { return BFloat && !Float; }
isFloatingPoint() const97 bool isFloatingPoint() const { return Float || BFloat; }
isInteger() const98 bool isInteger() const { return !isFloatingPoint() && !Predicate; }
isScalarPredicate() const99 bool isScalarPredicate() const {
100 return !isFloatingPoint() && Predicate && NumVectors == 0;
101 }
isPredicateVector() const102 bool isPredicateVector() const { return Predicate; }
isPredicatePattern() const103 bool isPredicatePattern() const { return PredicatePattern; }
isPrefetchOp() const104 bool isPrefetchOp() const { return PrefetchOp; }
isConstant() const105 bool isConstant() const { return Constant; }
getElementSizeInBits() const106 unsigned getElementSizeInBits() const { return ElementBitwidth; }
getNumVectors() const107 unsigned getNumVectors() const { return NumVectors; }
108
getNumElements() const109 unsigned getNumElements() const {
110 assert(ElementBitwidth != ~0U);
111 return Bitwidth / ElementBitwidth;
112 }
getSizeInBits() const113 unsigned getSizeInBits() const {
114 return Bitwidth;
115 }
116
117 /// Return the string representation of a type, which is an encoded
118 /// string for passing to the BUILTIN() macro in Builtins.def.
119 std::string builtin_str() const;
120
121 /// Return the C/C++ string representation of a type for use in the
122 /// arm_sve.h header file.
123 std::string str() const;
124
125 private:
126 /// Creates the type based on the typespec string in TS.
127 void applyTypespec();
128
129 /// Applies a prototype modifier to the type.
130 void applyModifier(char Mod);
131 };
132
133
134 class SVEEmitter;
135
136 /// The main grunt class. This represents an instantiation of an intrinsic with
137 /// a particular typespec and prototype.
138 class Intrinsic {
139 /// The unmangled name.
140 std::string Name;
141
142 /// The name of the corresponding LLVM IR intrinsic.
143 std::string LLVMName;
144
145 /// Intrinsic prototype.
146 std::string Proto;
147
148 /// The base type spec for this intrinsic.
149 TypeSpec BaseTypeSpec;
150
151 /// The base class kind. Most intrinsics use ClassS, which has full type
152 /// info for integers (_s32/_u32), or ClassG which is used for overloaded
153 /// intrinsics.
154 ClassKind Class;
155
156 /// The architectural #ifdef guard.
157 std::string Guard;
158
159 // The merge suffix such as _m, _x or _z.
160 std::string MergeSuffix;
161
162 /// The types of return value [0] and parameters [1..].
163 std::vector<SVEType> Types;
164
165 /// The "base type", which is VarType('d', BaseTypeSpec).
166 SVEType BaseType;
167
168 uint64_t Flags;
169
170 SmallVector<ImmCheck, 2> ImmChecks;
171
172 public:
173 Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
174 StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
175 uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,
176 ClassKind Class, SVEEmitter &Emitter, StringRef Guard);
177
178 ~Intrinsic()=default;
179
getName() const180 std::string getName() const { return Name; }
getLLVMName() const181 std::string getLLVMName() const { return LLVMName; }
getProto() const182 std::string getProto() const { return Proto; }
getBaseTypeSpec() const183 TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }
getBaseType() const184 SVEType getBaseType() const { return BaseType; }
185
getGuard() const186 StringRef getGuard() const { return Guard; }
getClassKind() const187 ClassKind getClassKind() const { return Class; }
188
getReturnType() const189 SVEType getReturnType() const { return Types[0]; }
getTypes() const190 ArrayRef<SVEType> getTypes() const { return Types; }
getParamType(unsigned I) const191 SVEType getParamType(unsigned I) const { return Types[I + 1]; }
getNumParams() const192 unsigned getNumParams() const { return Proto.size() - 1; }
193
getFlags() const194 uint64_t getFlags() const { return Flags; }
isFlagSet(uint64_t Flag) const195 bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}
196
getImmChecks() const197 ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }
198
199 /// Return the type string for a BUILTIN() macro in Builtins.def.
200 std::string getBuiltinTypeStr();
201
202 /// Return the name, mangled with type information. The name is mangled for
203 /// ClassS, so will add type suffixes such as _u32/_s32.
getMangledName() const204 std::string getMangledName() const { return mangleName(ClassS); }
205
206 /// Returns true if the intrinsic is overloaded, in that it should also generate
207 /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
208 /// 'svld1_u32(..)'.
isOverloadedIntrinsic(StringRef Name)209 static bool isOverloadedIntrinsic(StringRef Name) {
210 auto BrOpen = Name.find("[");
211 auto BrClose = Name.find(']');
212 return BrOpen != std::string::npos && BrClose != std::string::npos;
213 }
214
215 /// Return true if the intrinsic takes a splat operand.
hasSplat() const216 bool hasSplat() const {
217 // These prototype modifiers are described in arm_sve.td.
218 return Proto.find_first_of("ajfrKLR@") != std::string::npos;
219 }
220
221 /// Return the parameter index of the splat operand.
getSplatIdx() const222 unsigned getSplatIdx() const {
223 // These prototype modifiers are described in arm_sve.td.
224 auto Idx = Proto.find_first_of("ajfrKLR@");
225 assert(Idx != std::string::npos && Idx > 0 &&
226 "Prototype has no splat operand");
227 return Idx - 1;
228 }
229
230 /// Emits the intrinsic declaration to the ostream.
231 void emitIntrinsic(raw_ostream &OS) const;
232
233 private:
getMergeSuffix() const234 std::string getMergeSuffix() const { return MergeSuffix; }
235 std::string mangleName(ClassKind LocalCK) const;
236 std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
237 std::string Proto) const;
238 };
239
240 class SVEEmitter {
241 private:
242 // The reinterpret builtins are generated separately because they
243 // need the cross product of all types (121 functions in total),
244 // which is inconvenient to specify in the arm_sve.td file or
245 // generate in CGBuiltin.cpp.
246 struct ReinterpretTypeInfo {
247 const char *Suffix;
248 const char *Type;
249 const char *BuiltinType;
250 };
251 SmallVector<ReinterpretTypeInfo, 12> Reinterprets = {
252 {"s8", "svint8_t", "q16Sc"}, {"s16", "svint16_t", "q8Ss"},
253 {"s32", "svint32_t", "q4Si"}, {"s64", "svint64_t", "q2SWi"},
254 {"u8", "svuint8_t", "q16Uc"}, {"u16", "svuint16_t", "q8Us"},
255 {"u32", "svuint32_t", "q4Ui"}, {"u64", "svuint64_t", "q2UWi"},
256 {"f16", "svfloat16_t", "q8h"}, {"bf16", "svbfloat16_t", "q8y"},
257 {"f32", "svfloat32_t", "q4f"}, {"f64", "svfloat64_t", "q2d"}};
258
259 RecordKeeper &Records;
260 llvm::StringMap<uint64_t> EltTypes;
261 llvm::StringMap<uint64_t> MemEltTypes;
262 llvm::StringMap<uint64_t> FlagTypes;
263 llvm::StringMap<uint64_t> MergeTypes;
264 llvm::StringMap<uint64_t> ImmCheckTypes;
265
266 public:
SVEEmitter(RecordKeeper & R)267 SVEEmitter(RecordKeeper &R) : Records(R) {
268 for (auto *RV : Records.getAllDerivedDefinitions("EltType"))
269 EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
270 for (auto *RV : Records.getAllDerivedDefinitions("MemEltType"))
271 MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
272 for (auto *RV : Records.getAllDerivedDefinitions("FlagType"))
273 FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
274 for (auto *RV : Records.getAllDerivedDefinitions("MergeType"))
275 MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
276 for (auto *RV : Records.getAllDerivedDefinitions("ImmCheckType"))
277 ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
278 }
279
280 /// Returns the enum value for the immcheck type
getEnumValueForImmCheck(StringRef C) const281 unsigned getEnumValueForImmCheck(StringRef C) const {
282 auto It = ImmCheckTypes.find(C);
283 if (It != ImmCheckTypes.end())
284 return It->getValue();
285 llvm_unreachable("Unsupported imm check");
286 }
287
288 /// Returns the enum value for the flag type
getEnumValueForFlag(StringRef C) const289 uint64_t getEnumValueForFlag(StringRef C) const {
290 auto Res = FlagTypes.find(C);
291 if (Res != FlagTypes.end())
292 return Res->getValue();
293 llvm_unreachable("Unsupported flag");
294 }
295
296 // Returns the SVETypeFlags for a given value and mask.
encodeFlag(uint64_t V,StringRef MaskName) const297 uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {
298 auto It = FlagTypes.find(MaskName);
299 if (It != FlagTypes.end()) {
300 uint64_t Mask = It->getValue();
301 unsigned Shift = llvm::countTrailingZeros(Mask);
302 return (V << Shift) & Mask;
303 }
304 llvm_unreachable("Unsupported flag");
305 }
306
307 // Returns the SVETypeFlags for the given element type.
encodeEltType(StringRef EltName)308 uint64_t encodeEltType(StringRef EltName) {
309 auto It = EltTypes.find(EltName);
310 if (It != EltTypes.end())
311 return encodeFlag(It->getValue(), "EltTypeMask");
312 llvm_unreachable("Unsupported EltType");
313 }
314
315 // Returns the SVETypeFlags for the given memory element type.
encodeMemoryElementType(uint64_t MT)316 uint64_t encodeMemoryElementType(uint64_t MT) {
317 return encodeFlag(MT, "MemEltTypeMask");
318 }
319
320 // Returns the SVETypeFlags for the given merge type.
encodeMergeType(uint64_t MT)321 uint64_t encodeMergeType(uint64_t MT) {
322 return encodeFlag(MT, "MergeTypeMask");
323 }
324
325 // Returns the SVETypeFlags for the given splat operand.
encodeSplatOperand(unsigned SplatIdx)326 unsigned encodeSplatOperand(unsigned SplatIdx) {
327 assert(SplatIdx < 7 && "SplatIdx out of encodable range");
328 return encodeFlag(SplatIdx + 1, "SplatOperandMask");
329 }
330
331 // Returns the SVETypeFlags value for the given SVEType.
332 uint64_t encodeTypeFlags(const SVEType &T);
333
334 /// Emit arm_sve.h.
335 void createHeader(raw_ostream &o);
336
337 /// Emit all the __builtin prototypes and code needed by Sema.
338 void createBuiltins(raw_ostream &o);
339
340 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
341 void createCodeGenMap(raw_ostream &o);
342
343 /// Emit all the range checks for the immediates.
344 void createRangeChecks(raw_ostream &o);
345
346 /// Create the SVETypeFlags used in CGBuiltins
347 void createTypeFlags(raw_ostream &o);
348
349 /// Create intrinsic and add it to \p Out
350 void createIntrinsic(Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);
351 };
352
353 } // end anonymous namespace
354
355
356 //===----------------------------------------------------------------------===//
357 // Type implementation
358 //===----------------------------------------------------------------------===//
359
builtin_str() const360 std::string SVEType::builtin_str() const {
361 std::string S;
362 if (isVoid())
363 return "v";
364
365 if (isVoidPointer())
366 S += "v";
367 else if (!isFloatingPoint())
368 switch (ElementBitwidth) {
369 case 1: S += "b"; break;
370 case 8: S += "c"; break;
371 case 16: S += "s"; break;
372 case 32: S += "i"; break;
373 case 64: S += "Wi"; break;
374 case 128: S += "LLLi"; break;
375 default: llvm_unreachable("Unhandled case!");
376 }
377 else if (isFloat())
378 switch (ElementBitwidth) {
379 case 16: S += "h"; break;
380 case 32: S += "f"; break;
381 case 64: S += "d"; break;
382 default: llvm_unreachable("Unhandled case!");
383 }
384 else if (isBFloat()) {
385 assert(ElementBitwidth == 16 && "Not a valid BFloat.");
386 S += "y";
387 }
388
389 if (!isFloatingPoint()) {
390 if ((isChar() || isPointer()) && !isVoidPointer()) {
391 // Make chars and typed pointers explicitly signed.
392 if (Signed)
393 S = "S" + S;
394 else if (!Signed)
395 S = "U" + S;
396 } else if (!isVoidPointer() && !Signed) {
397 S = "U" + S;
398 }
399 }
400
401 // Constant indices are "int", but have the "constant expression" modifier.
402 if (isImmediate()) {
403 assert(!isFloat() && "fp immediates are not supported");
404 S = "I" + S;
405 }
406
407 if (isScalar()) {
408 if (Constant) S += "C";
409 if (Pointer) S += "*";
410 return S;
411 }
412
413 assert(isScalableVector() && "Unsupported type");
414 return "q" + utostr(getNumElements() * NumVectors) + S;
415 }
416
str() const417 std::string SVEType::str() const {
418 if (isPredicatePattern())
419 return "enum svpattern";
420
421 if (isPrefetchOp())
422 return "enum svprfop";
423
424 std::string S;
425 if (Void)
426 S += "void";
427 else {
428 if (isScalableVector())
429 S += "sv";
430 if (!Signed && !isFloatingPoint())
431 S += "u";
432
433 if (Float)
434 S += "float";
435 else if (isScalarPredicate() || isPredicateVector())
436 S += "bool";
437 else if (isBFloat())
438 S += "bfloat";
439 else
440 S += "int";
441
442 if (!isScalarPredicate() && !isPredicateVector())
443 S += utostr(ElementBitwidth);
444 if (!isScalableVector() && isVector())
445 S += "x" + utostr(getNumElements());
446 if (NumVectors > 1)
447 S += "x" + utostr(NumVectors);
448 if (!isScalarPredicate())
449 S += "_t";
450 }
451
452 if (Constant)
453 S += " const";
454 if (Pointer)
455 S += " *";
456
457 return S;
458 }
applyTypespec()459 void SVEType::applyTypespec() {
460 for (char I : TS) {
461 switch (I) {
462 case 'P':
463 Predicate = true;
464 break;
465 case 'U':
466 Signed = false;
467 break;
468 case 'c':
469 ElementBitwidth = 8;
470 break;
471 case 's':
472 ElementBitwidth = 16;
473 break;
474 case 'i':
475 ElementBitwidth = 32;
476 break;
477 case 'l':
478 ElementBitwidth = 64;
479 break;
480 case 'h':
481 Float = true;
482 ElementBitwidth = 16;
483 break;
484 case 'f':
485 Float = true;
486 ElementBitwidth = 32;
487 break;
488 case 'd':
489 Float = true;
490 ElementBitwidth = 64;
491 break;
492 case 'b':
493 BFloat = true;
494 Float = false;
495 ElementBitwidth = 16;
496 break;
497 default:
498 llvm_unreachable("Unhandled type code!");
499 }
500 }
501 assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
502 }
503
applyModifier(char Mod)504 void SVEType::applyModifier(char Mod) {
505 switch (Mod) {
506 case '2':
507 NumVectors = 2;
508 break;
509 case '3':
510 NumVectors = 3;
511 break;
512 case '4':
513 NumVectors = 4;
514 break;
515 case 'v':
516 Void = true;
517 break;
518 case 'd':
519 DefaultType = true;
520 break;
521 case 'c':
522 Constant = true;
523 LLVM_FALLTHROUGH;
524 case 'p':
525 Pointer = true;
526 Bitwidth = ElementBitwidth;
527 NumVectors = 0;
528 break;
529 case 'e':
530 Signed = false;
531 ElementBitwidth /= 2;
532 break;
533 case 'h':
534 ElementBitwidth /= 2;
535 break;
536 case 'q':
537 ElementBitwidth /= 4;
538 break;
539 case 'b':
540 Signed = false;
541 Float = false;
542 BFloat = false;
543 ElementBitwidth /= 4;
544 break;
545 case 'o':
546 ElementBitwidth *= 4;
547 break;
548 case 'P':
549 Signed = true;
550 Float = false;
551 BFloat = false;
552 Predicate = true;
553 Bitwidth = 16;
554 ElementBitwidth = 1;
555 break;
556 case 's':
557 case 'a':
558 Bitwidth = ElementBitwidth;
559 NumVectors = 0;
560 break;
561 case 'R':
562 ElementBitwidth /= 2;
563 NumVectors = 0;
564 break;
565 case 'r':
566 ElementBitwidth /= 4;
567 NumVectors = 0;
568 break;
569 case '@':
570 Signed = false;
571 Float = false;
572 BFloat = false;
573 ElementBitwidth /= 4;
574 NumVectors = 0;
575 break;
576 case 'K':
577 Signed = true;
578 Float = false;
579 BFloat = false;
580 Bitwidth = ElementBitwidth;
581 NumVectors = 0;
582 break;
583 case 'L':
584 Signed = false;
585 Float = false;
586 BFloat = false;
587 Bitwidth = ElementBitwidth;
588 NumVectors = 0;
589 break;
590 case 'u':
591 Predicate = false;
592 Signed = false;
593 Float = false;
594 BFloat = false;
595 break;
596 case 'x':
597 Predicate = false;
598 Signed = true;
599 Float = false;
600 BFloat = false;
601 break;
602 case 'i':
603 Predicate = false;
604 Float = false;
605 BFloat = false;
606 ElementBitwidth = Bitwidth = 64;
607 NumVectors = 0;
608 Signed = false;
609 Immediate = true;
610 break;
611 case 'I':
612 Predicate = false;
613 Float = false;
614 BFloat = false;
615 ElementBitwidth = Bitwidth = 32;
616 NumVectors = 0;
617 Signed = true;
618 Immediate = true;
619 PredicatePattern = true;
620 break;
621 case 'J':
622 Predicate = false;
623 Float = false;
624 BFloat = false;
625 ElementBitwidth = Bitwidth = 32;
626 NumVectors = 0;
627 Signed = true;
628 Immediate = true;
629 PrefetchOp = true;
630 break;
631 case 'k':
632 Predicate = false;
633 Signed = true;
634 Float = false;
635 BFloat = false;
636 ElementBitwidth = Bitwidth = 32;
637 NumVectors = 0;
638 break;
639 case 'l':
640 Predicate = false;
641 Signed = true;
642 Float = false;
643 BFloat = false;
644 ElementBitwidth = Bitwidth = 64;
645 NumVectors = 0;
646 break;
647 case 'm':
648 Predicate = false;
649 Signed = false;
650 Float = false;
651 BFloat = false;
652 ElementBitwidth = Bitwidth = 32;
653 NumVectors = 0;
654 break;
655 case 'n':
656 Predicate = false;
657 Signed = false;
658 Float = false;
659 BFloat = false;
660 ElementBitwidth = Bitwidth = 64;
661 NumVectors = 0;
662 break;
663 case 'w':
664 ElementBitwidth = 64;
665 break;
666 case 'j':
667 ElementBitwidth = Bitwidth = 64;
668 NumVectors = 0;
669 break;
670 case 'f':
671 Signed = false;
672 ElementBitwidth = Bitwidth = 64;
673 NumVectors = 0;
674 break;
675 case 'g':
676 Signed = false;
677 Float = false;
678 BFloat = false;
679 ElementBitwidth = 64;
680 break;
681 case 't':
682 Signed = true;
683 Float = false;
684 BFloat = false;
685 ElementBitwidth = 32;
686 break;
687 case 'z':
688 Signed = false;
689 Float = false;
690 BFloat = false;
691 ElementBitwidth = 32;
692 break;
693 case 'O':
694 Predicate = false;
695 Float = true;
696 ElementBitwidth = 16;
697 break;
698 case 'M':
699 Predicate = false;
700 Float = true;
701 BFloat = false;
702 ElementBitwidth = 32;
703 break;
704 case 'N':
705 Predicate = false;
706 Float = true;
707 ElementBitwidth = 64;
708 break;
709 case 'Q':
710 Constant = true;
711 Pointer = true;
712 Void = true;
713 NumVectors = 0;
714 break;
715 case 'S':
716 Constant = true;
717 Pointer = true;
718 ElementBitwidth = Bitwidth = 8;
719 NumVectors = 0;
720 Signed = true;
721 break;
722 case 'W':
723 Constant = true;
724 Pointer = true;
725 ElementBitwidth = Bitwidth = 8;
726 NumVectors = 0;
727 Signed = false;
728 break;
729 case 'T':
730 Constant = true;
731 Pointer = true;
732 ElementBitwidth = Bitwidth = 16;
733 NumVectors = 0;
734 Signed = true;
735 break;
736 case 'X':
737 Constant = true;
738 Pointer = true;
739 ElementBitwidth = Bitwidth = 16;
740 NumVectors = 0;
741 Signed = false;
742 break;
743 case 'Y':
744 Constant = true;
745 Pointer = true;
746 ElementBitwidth = Bitwidth = 32;
747 NumVectors = 0;
748 Signed = false;
749 break;
750 case 'U':
751 Constant = true;
752 Pointer = true;
753 ElementBitwidth = Bitwidth = 32;
754 NumVectors = 0;
755 Signed = true;
756 break;
757 case 'A':
758 Pointer = true;
759 ElementBitwidth = Bitwidth = 8;
760 NumVectors = 0;
761 Signed = true;
762 break;
763 case 'B':
764 Pointer = true;
765 ElementBitwidth = Bitwidth = 16;
766 NumVectors = 0;
767 Signed = true;
768 break;
769 case 'C':
770 Pointer = true;
771 ElementBitwidth = Bitwidth = 32;
772 NumVectors = 0;
773 Signed = true;
774 break;
775 case 'D':
776 Pointer = true;
777 ElementBitwidth = Bitwidth = 64;
778 NumVectors = 0;
779 Signed = true;
780 break;
781 case 'E':
782 Pointer = true;
783 ElementBitwidth = Bitwidth = 8;
784 NumVectors = 0;
785 Signed = false;
786 break;
787 case 'F':
788 Pointer = true;
789 ElementBitwidth = Bitwidth = 16;
790 NumVectors = 0;
791 Signed = false;
792 break;
793 case 'G':
794 Pointer = true;
795 ElementBitwidth = Bitwidth = 32;
796 NumVectors = 0;
797 Signed = false;
798 break;
799 default:
800 llvm_unreachable("Unhandled character!");
801 }
802 }
803
804
805 //===----------------------------------------------------------------------===//
806 // Intrinsic implementation
807 //===----------------------------------------------------------------------===//
808
Intrinsic(StringRef Name,StringRef Proto,uint64_t MergeTy,StringRef MergeSuffix,uint64_t MemoryElementTy,StringRef LLVMName,uint64_t Flags,ArrayRef<ImmCheck> Checks,TypeSpec BT,ClassKind Class,SVEEmitter & Emitter,StringRef Guard)809 Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
810 StringRef MergeSuffix, uint64_t MemoryElementTy,
811 StringRef LLVMName, uint64_t Flags,
812 ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,
813 SVEEmitter &Emitter, StringRef Guard)
814 : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),
815 BaseTypeSpec(BT), Class(Class), Guard(Guard.str()),
816 MergeSuffix(MergeSuffix.str()), BaseType(BT, 'd'), Flags(Flags),
817 ImmChecks(Checks.begin(), Checks.end()) {
818 // Types[0] is the return value.
819 for (unsigned I = 0; I < Proto.size(); ++I) {
820 SVEType T(BaseTypeSpec, Proto[I]);
821 Types.push_back(T);
822
823 // Add range checks for immediates
824 if (I > 0) {
825 if (T.isPredicatePattern())
826 ImmChecks.emplace_back(
827 I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31"));
828 else if (T.isPrefetchOp())
829 ImmChecks.emplace_back(
830 I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13"));
831 }
832 }
833
834 // Set flags based on properties
835 this->Flags |= Emitter.encodeTypeFlags(BaseType);
836 this->Flags |= Emitter.encodeMemoryElementType(MemoryElementTy);
837 this->Flags |= Emitter.encodeMergeType(MergeTy);
838 if (hasSplat())
839 this->Flags |= Emitter.encodeSplatOperand(getSplatIdx());
840 }
841
getBuiltinTypeStr()842 std::string Intrinsic::getBuiltinTypeStr() {
843 std::string S = getReturnType().builtin_str();
844 for (unsigned I = 0; I < getNumParams(); ++I)
845 S += getParamType(I).builtin_str();
846
847 return S;
848 }
849
replaceTemplatedArgs(std::string Name,TypeSpec TS,std::string Proto) const850 std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
851 std::string Proto) const {
852 std::string Ret = Name;
853 while (Ret.find('{') != std::string::npos) {
854 size_t Pos = Ret.find('{');
855 size_t End = Ret.find('}');
856 unsigned NumChars = End - Pos + 1;
857 assert(NumChars == 3 && "Unexpected template argument");
858
859 SVEType T;
860 char C = Ret[Pos+1];
861 switch(C) {
862 default:
863 llvm_unreachable("Unknown predication specifier");
864 case 'd':
865 T = SVEType(TS, 'd');
866 break;
867 case '0':
868 case '1':
869 case '2':
870 case '3':
871 T = SVEType(TS, Proto[C - '0']);
872 break;
873 }
874
875 // Replace templated arg with the right suffix (e.g. u32)
876 std::string TypeCode;
877 if (T.isInteger())
878 TypeCode = T.isSigned() ? 's' : 'u';
879 else if (T.isPredicateVector())
880 TypeCode = 'b';
881 else if (T.isBFloat())
882 TypeCode = "bf";
883 else
884 TypeCode = 'f';
885 Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits()));
886 }
887
888 return Ret;
889 }
890
mangleName(ClassKind LocalCK) const891 std::string Intrinsic::mangleName(ClassKind LocalCK) const {
892 std::string S = getName();
893
894 if (LocalCK == ClassG) {
895 // Remove the square brackets and everything in between.
896 while (S.find("[") != std::string::npos) {
897 auto Start = S.find("[");
898 auto End = S.find(']');
899 S.erase(Start, (End-Start)+1);
900 }
901 } else {
902 // Remove the square brackets.
903 while (S.find("[") != std::string::npos) {
904 auto BrPos = S.find('[');
905 if (BrPos != std::string::npos)
906 S.erase(BrPos, 1);
907 BrPos = S.find(']');
908 if (BrPos != std::string::npos)
909 S.erase(BrPos, 1);
910 }
911 }
912
913 // Replace all {d} like expressions with e.g. 'u32'
914 return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()) +
915 getMergeSuffix();
916 }
917
emitIntrinsic(raw_ostream & OS) const918 void Intrinsic::emitIntrinsic(raw_ostream &OS) const {
919 // Use the preprocessor to
920 if (getClassKind() != ClassG || getProto().size() <= 1) {
921 OS << "#define " << mangleName(getClassKind())
922 << "(...) __builtin_sve_" << mangleName(ClassS)
923 << "(__VA_ARGS__)\n";
924 } else {
925 std::string FullName = mangleName(ClassS);
926 std::string ProtoName = mangleName(ClassG);
927
928 OS << "__aio __attribute__((__clang_arm_builtin_alias("
929 << "__builtin_sve_" << FullName << ")))\n";
930
931 OS << getTypes()[0].str() << " " << ProtoName << "(";
932 for (unsigned I = 0; I < getTypes().size() - 1; ++I) {
933 if (I != 0)
934 OS << ", ";
935 OS << getTypes()[I + 1].str();
936 }
937 OS << ");\n";
938 }
939 }
940
941 //===----------------------------------------------------------------------===//
942 // SVEEmitter implementation
943 //===----------------------------------------------------------------------===//
encodeTypeFlags(const SVEType & T)944 uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
945 if (T.isFloat()) {
946 switch (T.getElementSizeInBits()) {
947 case 16:
948 return encodeEltType("EltTyFloat16");
949 case 32:
950 return encodeEltType("EltTyFloat32");
951 case 64:
952 return encodeEltType("EltTyFloat64");
953 default:
954 llvm_unreachable("Unhandled float element bitwidth!");
955 }
956 }
957
958 if (T.isBFloat()) {
959 assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
960 return encodeEltType("EltTyBFloat16");
961 }
962
963 if (T.isPredicateVector()) {
964 switch (T.getElementSizeInBits()) {
965 case 8:
966 return encodeEltType("EltTyBool8");
967 case 16:
968 return encodeEltType("EltTyBool16");
969 case 32:
970 return encodeEltType("EltTyBool32");
971 case 64:
972 return encodeEltType("EltTyBool64");
973 default:
974 llvm_unreachable("Unhandled predicate element bitwidth!");
975 }
976 }
977
978 switch (T.getElementSizeInBits()) {
979 case 8:
980 return encodeEltType("EltTyInt8");
981 case 16:
982 return encodeEltType("EltTyInt16");
983 case 32:
984 return encodeEltType("EltTyInt32");
985 case 64:
986 return encodeEltType("EltTyInt64");
987 default:
988 llvm_unreachable("Unhandled integer element bitwidth!");
989 }
990 }
991
createIntrinsic(Record * R,SmallVectorImpl<std::unique_ptr<Intrinsic>> & Out)992 void SVEEmitter::createIntrinsic(
993 Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {
994 StringRef Name = R->getValueAsString("Name");
995 StringRef Proto = R->getValueAsString("Prototype");
996 StringRef Types = R->getValueAsString("Types");
997 StringRef Guard = R->getValueAsString("ArchGuard");
998 StringRef LLVMName = R->getValueAsString("LLVMIntrinsic");
999 uint64_t Merge = R->getValueAsInt("Merge");
1000 StringRef MergeSuffix = R->getValueAsString("MergeSuffix");
1001 uint64_t MemEltType = R->getValueAsInt("MemEltType");
1002 std::vector<Record*> FlagsList = R->getValueAsListOfDefs("Flags");
1003 std::vector<Record*> ImmCheckList = R->getValueAsListOfDefs("ImmChecks");
1004
1005 int64_t Flags = 0;
1006 for (auto FlagRec : FlagsList)
1007 Flags |= FlagRec->getValueAsInt("Value");
1008
1009 // Create a dummy TypeSpec for non-overloaded builtins.
1010 if (Types.empty()) {
1011 assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&
1012 "Expect TypeSpec for overloaded builtin!");
1013 Types = "i";
1014 }
1015
1016 // Extract type specs from string
1017 SmallVector<TypeSpec, 8> TypeSpecs;
1018 TypeSpec Acc;
1019 for (char I : Types) {
1020 Acc.push_back(I);
1021 if (islower(I)) {
1022 TypeSpecs.push_back(TypeSpec(Acc));
1023 Acc.clear();
1024 }
1025 }
1026
1027 // Remove duplicate type specs.
1028 llvm::sort(TypeSpecs);
1029 TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()),
1030 TypeSpecs.end());
1031
1032 // Create an Intrinsic for each type spec.
1033 for (auto TS : TypeSpecs) {
1034 // Collate a list of range/option checks for the immediates.
1035 SmallVector<ImmCheck, 2> ImmChecks;
1036 for (auto *R : ImmCheckList) {
1037 int64_t Arg = R->getValueAsInt("Arg");
1038 int64_t EltSizeArg = R->getValueAsInt("EltSizeArg");
1039 int64_t Kind = R->getValueAsDef("Kind")->getValueAsInt("Value");
1040 assert(Arg >= 0 && Kind >= 0 && "Arg and Kind must be nonnegative");
1041
1042 unsigned ElementSizeInBits = 0;
1043 if (EltSizeArg >= 0)
1044 ElementSizeInBits =
1045 SVEType(TS, Proto[EltSizeArg + /* offset by return arg */ 1])
1046 .getElementSizeInBits();
1047 ImmChecks.push_back(ImmCheck(Arg, Kind, ElementSizeInBits));
1048 }
1049
1050 Out.push_back(std::make_unique<Intrinsic>(
1051 Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, ImmChecks,
1052 TS, ClassS, *this, Guard));
1053
1054 // Also generate the short-form (e.g. svadd_m) for the given type-spec.
1055 if (Intrinsic::isOverloadedIntrinsic(Name))
1056 Out.push_back(std::make_unique<Intrinsic>(
1057 Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags,
1058 ImmChecks, TS, ClassG, *this, Guard));
1059 }
1060 }
1061
createHeader(raw_ostream & OS)1062 void SVEEmitter::createHeader(raw_ostream &OS) {
1063 OS << "/*===---- arm_sve.h - ARM SVE intrinsics "
1064 "-----------------------------------===\n"
1065 " *\n"
1066 " *\n"
1067 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1068 "Exceptions.\n"
1069 " * See https://llvm.org/LICENSE.txt for license information.\n"
1070 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1071 " *\n"
1072 " *===-----------------------------------------------------------------"
1073 "------===\n"
1074 " */\n\n";
1075
1076 OS << "#ifndef __ARM_SVE_H\n";
1077 OS << "#define __ARM_SVE_H\n\n";
1078
1079 OS << "#if !defined(__ARM_FEATURE_SVE)\n";
1080 OS << "#error \"SVE support not enabled\"\n";
1081 OS << "#else\n\n";
1082
1083 OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1084 OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";
1085 OS << "#endif\n";
1086
1087 OS << "#include <stdint.h>\n\n";
1088 OS << "#ifdef __cplusplus\n";
1089 OS << "extern \"C\" {\n";
1090 OS << "#else\n";
1091 OS << "#include <stdbool.h>\n";
1092 OS << "#endif\n\n";
1093
1094 OS << "typedef __fp16 float16_t;\n";
1095 OS << "typedef float float32_t;\n";
1096 OS << "typedef double float64_t;\n";
1097
1098 OS << "typedef __SVInt8_t svint8_t;\n";
1099 OS << "typedef __SVInt16_t svint16_t;\n";
1100 OS << "typedef __SVInt32_t svint32_t;\n";
1101 OS << "typedef __SVInt64_t svint64_t;\n";
1102 OS << "typedef __SVUint8_t svuint8_t;\n";
1103 OS << "typedef __SVUint16_t svuint16_t;\n";
1104 OS << "typedef __SVUint32_t svuint32_t;\n";
1105 OS << "typedef __SVUint64_t svuint64_t;\n";
1106 OS << "typedef __SVFloat16_t svfloat16_t;\n\n";
1107
1108 OS << "#if defined(__ARM_FEATURE_SVE_BF16) && "
1109 "!defined(__ARM_FEATURE_BF16_SCALAR_ARITHMETIC)\n";
1110 OS << "#error \"__ARM_FEATURE_BF16_SCALAR_ARITHMETIC must be defined when "
1111 "__ARM_FEATURE_SVE_BF16 is defined\"\n";
1112 OS << "#endif\n\n";
1113
1114 OS << "#if defined(__ARM_FEATURE_SVE_BF16)\n";
1115 OS << "typedef __SVBFloat16_t svbfloat16_t;\n";
1116 OS << "#endif\n\n";
1117
1118 OS << "#if defined(__ARM_FEATURE_BF16_SCALAR_ARITHMETIC)\n";
1119 OS << "#include <arm_bf16.h>\n";
1120 OS << "typedef __bf16 bfloat16_t;\n";
1121 OS << "#endif\n\n";
1122
1123 OS << "typedef __SVFloat32_t svfloat32_t;\n";
1124 OS << "typedef __SVFloat64_t svfloat64_t;\n";
1125 OS << "typedef __clang_svint8x2_t svint8x2_t;\n";
1126 OS << "typedef __clang_svint16x2_t svint16x2_t;\n";
1127 OS << "typedef __clang_svint32x2_t svint32x2_t;\n";
1128 OS << "typedef __clang_svint64x2_t svint64x2_t;\n";
1129 OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";
1130 OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";
1131 OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";
1132 OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";
1133 OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";
1134 OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";
1135 OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";
1136 OS << "typedef __clang_svint8x3_t svint8x3_t;\n";
1137 OS << "typedef __clang_svint16x3_t svint16x3_t;\n";
1138 OS << "typedef __clang_svint32x3_t svint32x3_t;\n";
1139 OS << "typedef __clang_svint64x3_t svint64x3_t;\n";
1140 OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";
1141 OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";
1142 OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";
1143 OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";
1144 OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";
1145 OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";
1146 OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";
1147 OS << "typedef __clang_svint8x4_t svint8x4_t;\n";
1148 OS << "typedef __clang_svint16x4_t svint16x4_t;\n";
1149 OS << "typedef __clang_svint32x4_t svint32x4_t;\n";
1150 OS << "typedef __clang_svint64x4_t svint64x4_t;\n";
1151 OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";
1152 OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";
1153 OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";
1154 OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";
1155 OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";
1156 OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";
1157 OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
1158 OS << "typedef __SVBool_t svbool_t;\n\n";
1159
1160 OS << "#ifdef __ARM_FEATURE_SVE_BF16\n";
1161 OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
1162 OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
1163 OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
1164 OS << "#endif\n";
1165
1166 OS << "enum svpattern\n";
1167 OS << "{\n";
1168 OS << " SV_POW2 = 0,\n";
1169 OS << " SV_VL1 = 1,\n";
1170 OS << " SV_VL2 = 2,\n";
1171 OS << " SV_VL3 = 3,\n";
1172 OS << " SV_VL4 = 4,\n";
1173 OS << " SV_VL5 = 5,\n";
1174 OS << " SV_VL6 = 6,\n";
1175 OS << " SV_VL7 = 7,\n";
1176 OS << " SV_VL8 = 8,\n";
1177 OS << " SV_VL16 = 9,\n";
1178 OS << " SV_VL32 = 10,\n";
1179 OS << " SV_VL64 = 11,\n";
1180 OS << " SV_VL128 = 12,\n";
1181 OS << " SV_VL256 = 13,\n";
1182 OS << " SV_MUL4 = 29,\n";
1183 OS << " SV_MUL3 = 30,\n";
1184 OS << " SV_ALL = 31\n";
1185 OS << "};\n\n";
1186
1187 OS << "enum svprfop\n";
1188 OS << "{\n";
1189 OS << " SV_PLDL1KEEP = 0,\n";
1190 OS << " SV_PLDL1STRM = 1,\n";
1191 OS << " SV_PLDL2KEEP = 2,\n";
1192 OS << " SV_PLDL2STRM = 3,\n";
1193 OS << " SV_PLDL3KEEP = 4,\n";
1194 OS << " SV_PLDL3STRM = 5,\n";
1195 OS << " SV_PSTL1KEEP = 8,\n";
1196 OS << " SV_PSTL1STRM = 9,\n";
1197 OS << " SV_PSTL2KEEP = 10,\n";
1198 OS << " SV_PSTL2STRM = 11,\n";
1199 OS << " SV_PSTL3KEEP = 12,\n";
1200 OS << " SV_PSTL3STRM = 13\n";
1201 OS << "};\n\n";
1202
1203 OS << "/* Function attributes */\n";
1204 OS << "#define __aio static inline __attribute__((__always_inline__, "
1205 "__nodebug__, __overloadable__))\n\n";
1206
1207 // Add reinterpret functions.
1208 for (auto ShortForm : { false, true } )
1209 for (const ReinterpretTypeInfo &From : Reinterprets)
1210 for (const ReinterpretTypeInfo &To : Reinterprets) {
1211 const bool IsBFloat = StringRef(From.Suffix).equals("bf16") ||
1212 StringRef(To.Suffix).equals("bf16");
1213 if (IsBFloat)
1214 OS << "#if defined(__ARM_FEATURE_SVE_BF16)\n";
1215 if (ShortForm) {
1216 OS << "__aio " << From.Type << " svreinterpret_" << From.Suffix;
1217 OS << "(" << To.Type << " op) {\n";
1218 OS << " return __builtin_sve_reinterpret_" << From.Suffix << "_"
1219 << To.Suffix << "(op);\n";
1220 OS << "}\n\n";
1221 } else
1222 OS << "#define svreinterpret_" << From.Suffix << "_" << To.Suffix
1223 << "(...) __builtin_sve_reinterpret_" << From.Suffix << "_"
1224 << To.Suffix << "(__VA_ARGS__)\n";
1225 if (IsBFloat)
1226 OS << "#endif /* #if defined(__ARM_FEATURE_SVE_BF16) */\n";
1227 }
1228
1229 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1230 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1231 for (auto *R : RV)
1232 createIntrinsic(R, Defs);
1233
1234 // Sort intrinsics in header file by following order/priority:
1235 // - Architectural guard (i.e. does it require SVE2 or SVE2_AES)
1236 // - Class (is intrinsic overloaded or not)
1237 // - Intrinsic name
1238 std::stable_sort(
1239 Defs.begin(), Defs.end(), [](const std::unique_ptr<Intrinsic> &A,
1240 const std::unique_ptr<Intrinsic> &B) {
1241 auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
1242 return std::make_tuple(I->getGuard(), (unsigned)I->getClassKind(), I->getName());
1243 };
1244 return ToTuple(A) < ToTuple(B);
1245 });
1246
1247 StringRef InGuard = "";
1248 for (auto &I : Defs) {
1249 // Emit #endif/#if pair if needed.
1250 if (I->getGuard() != InGuard) {
1251 if (!InGuard.empty())
1252 OS << "#endif //" << InGuard << "\n";
1253 InGuard = I->getGuard();
1254 if (!InGuard.empty())
1255 OS << "\n#if " << InGuard << "\n";
1256 }
1257
1258 // Actually emit the intrinsic declaration.
1259 I->emitIntrinsic(OS);
1260 }
1261
1262 if (!InGuard.empty())
1263 OS << "#endif //" << InGuard << "\n";
1264
1265 OS << "#if defined(__ARM_FEATURE_SVE_BF16)\n";
1266 OS << "#define svcvtnt_bf16_x svcvtnt_bf16_m\n";
1267 OS << "#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m\n";
1268 OS << "#endif /*__ARM_FEATURE_SVE_BF16 */\n\n";
1269
1270 OS << "#if defined(__ARM_FEATURE_SVE2)\n";
1271 OS << "#define svcvtnt_f16_x svcvtnt_f16_m\n";
1272 OS << "#define svcvtnt_f16_f32_x svcvtnt_f16_f32_m\n";
1273 OS << "#define svcvtnt_f32_x svcvtnt_f32_m\n";
1274 OS << "#define svcvtnt_f32_f64_x svcvtnt_f32_f64_m\n\n";
1275
1276 OS << "#define svcvtxnt_f32_x svcvtxnt_f32_m\n";
1277 OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";
1278
1279 OS << "#endif /*__ARM_FEATURE_SVE2 */\n\n";
1280
1281 OS << "#ifdef __cplusplus\n";
1282 OS << "} // extern \"C\"\n";
1283 OS << "#endif\n\n";
1284 OS << "#endif /*__ARM_FEATURE_SVE */\n\n";
1285 OS << "#endif /* __ARM_SVE_H */\n";
1286 }
1287
createBuiltins(raw_ostream & OS)1288 void SVEEmitter::createBuiltins(raw_ostream &OS) {
1289 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1290 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1291 for (auto *R : RV)
1292 createIntrinsic(R, Defs);
1293
1294 // The mappings must be sorted based on BuiltinID.
1295 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1296 const std::unique_ptr<Intrinsic> &B) {
1297 return A->getMangledName() < B->getMangledName();
1298 });
1299
1300 OS << "#ifdef GET_SVE_BUILTINS\n";
1301 for (auto &Def : Defs) {
1302 // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1303 // declarations only live in the header file.
1304 if (Def->getClassKind() != ClassG)
1305 OS << "BUILTIN(__builtin_sve_" << Def->getMangledName() << ", \""
1306 << Def->getBuiltinTypeStr() << "\", \"n\")\n";
1307 }
1308
1309 // Add reinterpret builtins
1310 for (const ReinterpretTypeInfo &From : Reinterprets)
1311 for (const ReinterpretTypeInfo &To : Reinterprets)
1312 OS << "BUILTIN(__builtin_sve_reinterpret_" << From.Suffix << "_"
1313 << To.Suffix << +", \"" << From.BuiltinType << To.BuiltinType
1314 << "\", \"n\")\n";
1315
1316 OS << "#endif\n\n";
1317 }
1318
createCodeGenMap(raw_ostream & OS)1319 void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
1320 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1321 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1322 for (auto *R : RV)
1323 createIntrinsic(R, Defs);
1324
1325 // The mappings must be sorted based on BuiltinID.
1326 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1327 const std::unique_ptr<Intrinsic> &B) {
1328 return A->getMangledName() < B->getMangledName();
1329 });
1330
1331 OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";
1332 for (auto &Def : Defs) {
1333 // Builtins only exist for non-overloaded intrinsics, overloaded
1334 // declarations only live in the header file.
1335 if (Def->getClassKind() == ClassG)
1336 continue;
1337
1338 uint64_t Flags = Def->getFlags();
1339 auto FlagString = std::to_string(Flags);
1340
1341 std::string LLVMName = Def->getLLVMName();
1342 std::string Builtin = Def->getMangledName();
1343 if (!LLVMName.empty())
1344 OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1345 << "),\n";
1346 else
1347 OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
1348 }
1349 OS << "#endif\n\n";
1350 }
1351
createRangeChecks(raw_ostream & OS)1352 void SVEEmitter::createRangeChecks(raw_ostream &OS) {
1353 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1354 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1355 for (auto *R : RV)
1356 createIntrinsic(R, Defs);
1357
1358 // The mappings must be sorted based on BuiltinID.
1359 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1360 const std::unique_ptr<Intrinsic> &B) {
1361 return A->getMangledName() < B->getMangledName();
1362 });
1363
1364
1365 OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";
1366
1367 // Ensure these are only emitted once.
1368 std::set<std::string> Emitted;
1369
1370 for (auto &Def : Defs) {
1371 if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
1372 Def->getImmChecks().empty())
1373 continue;
1374
1375 OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";
1376 for (auto &Check : Def->getImmChecks())
1377 OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
1378 << Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n";
1379 OS << " break;\n";
1380
1381 Emitted.insert(Def->getMangledName());
1382 }
1383
1384 OS << "#endif\n\n";
1385 }
1386
1387 /// Create the SVETypeFlags used in CGBuiltins
createTypeFlags(raw_ostream & OS)1388 void SVEEmitter::createTypeFlags(raw_ostream &OS) {
1389 OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";
1390 for (auto &KV : FlagTypes)
1391 OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";
1392 OS << "#endif\n\n";
1393
1394 OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";
1395 for (auto &KV : EltTypes)
1396 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1397 OS << "#endif\n\n";
1398
1399 OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";
1400 for (auto &KV : MemEltTypes)
1401 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1402 OS << "#endif\n\n";
1403
1404 OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";
1405 for (auto &KV : MergeTypes)
1406 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1407 OS << "#endif\n\n";
1408
1409 OS << "#ifdef LLVM_GET_SVE_IMMCHECKTYPES\n";
1410 for (auto &KV : ImmCheckTypes)
1411 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1412 OS << "#endif\n\n";
1413 }
1414
1415 namespace clang {
EmitSveHeader(RecordKeeper & Records,raw_ostream & OS)1416 void EmitSveHeader(RecordKeeper &Records, raw_ostream &OS) {
1417 SVEEmitter(Records).createHeader(OS);
1418 }
1419
EmitSveBuiltins(RecordKeeper & Records,raw_ostream & OS)1420 void EmitSveBuiltins(RecordKeeper &Records, raw_ostream &OS) {
1421 SVEEmitter(Records).createBuiltins(OS);
1422 }
1423
EmitSveBuiltinCG(RecordKeeper & Records,raw_ostream & OS)1424 void EmitSveBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
1425 SVEEmitter(Records).createCodeGenMap(OS);
1426 }
1427
EmitSveRangeChecks(RecordKeeper & Records,raw_ostream & OS)1428 void EmitSveRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
1429 SVEEmitter(Records).createRangeChecks(OS);
1430 }
1431
EmitSveTypeFlags(RecordKeeper & Records,raw_ostream & OS)1432 void EmitSveTypeFlags(RecordKeeper &Records, raw_ostream &OS) {
1433 SVEEmitter(Records).createTypeFlags(OS);
1434 }
1435
1436 } // End namespace clang
1437