1 //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Populates the VFABI attribute with the scalar-to-vector mappings
10 // from the TargetLibraryInfo.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
15 #include "llvm/ADT/Statistic.h"
16 #include "llvm/Analysis/VectorUtils.h"
17 #include "llvm/IR/InstIterator.h"
18 #include "llvm/Transforms/Utils.h"
19 #include "llvm/Transforms/Utils/ModuleUtils.h"
20
21 using namespace llvm;
22
23 #define DEBUG_TYPE "inject-tli-mappings"
24
25 STATISTIC(NumCallInjected,
26 "Number of calls in which the mappings have been injected.");
27
28 STATISTIC(NumVFDeclAdded,
29 "Number of function declarations that have been added.");
30 STATISTIC(NumCompUsedAdded,
31 "Number of `@llvm.compiler.used` operands that have been added.");
32
33 /// Helper function to map the TLI name to a strings that holds
34 /// scalar-to-vector mapping.
35 ///
36 /// _ZGV<isa><mask><vlen><vparams>_<scalarname>(<vectorname>)
37 ///
38 /// where:
39 ///
40 /// <isa> = "_LLVM_"
41 /// <mask> = "N". Note: TLI does not support masked interfaces.
42 /// <vlen> = Number of concurrent lanes, stored in the `VectorizationFactor`
43 /// field of the `VecDesc` struct.
44 /// <vparams> = "v", as many as are the number of parameters of CI.
45 /// <scalarname> = the name of the scalar function called by CI.
46 /// <vectorname> = the name of the vector function mapped by the TLI.
mangleTLIName(StringRef VectorName,const CallInst & CI,unsigned VF)47 static std::string mangleTLIName(StringRef VectorName, const CallInst &CI,
48 unsigned VF) {
49 SmallString<256> Buffer;
50 llvm::raw_svector_ostream Out(Buffer);
51 Out << "_ZGV" << VFABI::_LLVM_ << "N" << VF;
52 for (unsigned I = 0; I < CI.getNumArgOperands(); ++I)
53 Out << "v";
54 Out << "_" << CI.getCalledFunction()->getName() << "(" << VectorName << ")";
55 return Out.str();
56 }
57
58 /// A helper function for converting Scalar types to vector types.
59 /// If the incoming type is void, we return void. If the VF is 1, we return
60 /// the scalar type.
ToVectorTy(Type * Scalar,unsigned VF,bool isScalable=false)61 static Type *ToVectorTy(Type *Scalar, unsigned VF, bool isScalable = false) {
62 if (Scalar->isVoidTy() || VF == 1)
63 return Scalar;
64 return VectorType::get(Scalar, {VF, isScalable});
65 }
66
67 /// A helper function that adds the vector function declaration that
68 /// vectorizes the CallInst CI with a vectorization factor of VF
69 /// lanes. The TLI assumes that all parameters and the return type of
70 /// CI (other than void) need to be widened to a VectorType of VF
71 /// lanes.
addVariantDeclaration(CallInst & CI,const unsigned VF,const StringRef VFName)72 static void addVariantDeclaration(CallInst &CI, const unsigned VF,
73 const StringRef VFName) {
74 Module *M = CI.getModule();
75
76 // Add function declaration.
77 Type *RetTy = ToVectorTy(CI.getType(), VF);
78 SmallVector<Type *, 4> Tys;
79 for (Value *ArgOperand : CI.arg_operands())
80 Tys.push_back(ToVectorTy(ArgOperand->getType(), VF));
81 assert(!CI.getFunctionType()->isVarArg() &&
82 "VarArg functions are not supported.");
83 FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false);
84 Function *VectorF =
85 Function::Create(FTy, Function::ExternalLinkage, VFName, M);
86 VectorF->copyAttributesFrom(CI.getCalledFunction());
87 ++NumVFDeclAdded;
88 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
89 << "` of type " << *(VectorF->getType()) << "\n");
90
91 // Make function declaration (without a body) "sticky" in the IR by
92 // listing it in the @llvm.compiler.used intrinsic.
93 assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` "
94 "only on declarations.");
95 appendToCompilerUsed(*M, {VectorF});
96 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
97 << "` to `@llvm.compiler.used`.\n");
98 ++NumCompUsedAdded;
99 }
100
addMappingsFromTLI(const TargetLibraryInfo & TLI,CallInst & CI)101 static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
102 // This is needed to make sure we don't query the TLI for calls to
103 // bitcast of function pointers, like `%call = call i32 (i32*, ...)
104 // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
105 // as such calls make the `isFunctionVectorizable` raise an
106 // exception.
107 if (CI.isNoBuiltin() || !CI.getCalledFunction())
108 return;
109
110 const std::string ScalarName = CI.getCalledFunction()->getName();
111 // Nothing to be done if the TLI thinks the function is not
112 // vectorizable.
113 if (!TLI.isFunctionVectorizable(ScalarName))
114 return;
115 SmallVector<std::string, 8> Mappings;
116 VFABI::getVectorVariantNames(CI, Mappings);
117 Module *M = CI.getModule();
118 const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(),
119 Mappings.end());
120 // All VFs in the TLI are powers of 2.
121 for (unsigned VF = 2, WidestVF = TLI.getWidestVF(ScalarName); VF <= WidestVF;
122 VF *= 2) {
123 const std::string TLIName = TLI.getVectorizedFunction(ScalarName, VF);
124 if (!TLIName.empty()) {
125 std::string MangledName = mangleTLIName(TLIName, CI, VF);
126 if (!OriginalSetOfMappings.count(MangledName)) {
127 Mappings.push_back(MangledName);
128 ++NumCallInjected;
129 }
130 Function *VariantF = M->getFunction(TLIName);
131 if (!VariantF)
132 addVariantDeclaration(CI, VF, TLIName);
133 }
134 }
135
136 VFABI::setVectorVariantNames(&CI, Mappings);
137 }
138
runImpl(const TargetLibraryInfo & TLI,Function & F)139 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
140 for (auto &I : instructions(F))
141 if (auto CI = dyn_cast<CallInst>(&I))
142 addMappingsFromTLI(TLI, *CI);
143 // Even if the pass adds IR attributes, the analyses are preserved.
144 return false;
145 }
146
147 ////////////////////////////////////////////////////////////////////////////////
148 // New pass manager implementation.
149 ////////////////////////////////////////////////////////////////////////////////
run(Function & F,FunctionAnalysisManager & AM)150 PreservedAnalyses InjectTLIMappings::run(Function &F,
151 FunctionAnalysisManager &AM) {
152 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
153 runImpl(TLI, F);
154 // Even if the pass adds IR attributes, the analyses are preserved.
155 return PreservedAnalyses::all();
156 }
157
158 ////////////////////////////////////////////////////////////////////////////////
159 // Legacy PM Implementation.
160 ////////////////////////////////////////////////////////////////////////////////
runOnFunction(Function & F)161 bool InjectTLIMappingsLegacy::runOnFunction(Function &F) {
162 const TargetLibraryInfo &TLI =
163 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
164 return runImpl(TLI, F);
165 }
166
getAnalysisUsage(AnalysisUsage & AU) const167 void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
168 AU.setPreservesCFG();
169 AU.addRequired<TargetLibraryInfoWrapperPass>();
170 AU.addPreserved<TargetLibraryInfoWrapperPass>();
171 }
172
173 ////////////////////////////////////////////////////////////////////////////////
174 // Legacy Pass manager initialization
175 ////////////////////////////////////////////////////////////////////////////////
176 char InjectTLIMappingsLegacy::ID = 0;
177
178 INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE,
179 "Inject TLI Mappings", false, false)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)180 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
181 INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings",
182 false, false)
183
184 FunctionPass *llvm::createInjectTLIMappingsLegacyPass() {
185 return new InjectTLIMappingsLegacy();
186 }
187