1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #include "CGLoopInfo.h"
11 #include "clang/AST/ASTContext.h"
12 #include "clang/AST/Attr.h"
13 #include "clang/Sema/LoopHint.h"
14 #include "llvm/IR/BasicBlock.h"
15 #include "llvm/IR/Constants.h"
16 #include "llvm/IR/InstrTypes.h"
17 #include "llvm/IR/Instructions.h"
18 #include "llvm/IR/Metadata.h"
19 using namespace clang::CodeGen;
20 using namespace llvm;
21
createMetadata(LLVMContext & Ctx,const LoopAttributes & Attrs,llvm::DebugLoc Location)22 static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
23 llvm::DebugLoc Location) {
24
25 if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
26 Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
27 Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
28 Attrs.UnrollEnable == LoopAttributes::Unspecified &&
29 Attrs.DistributeEnable == LoopAttributes::Unspecified &&
30 !Location)
31 return nullptr;
32
33 SmallVector<Metadata *, 4> Args;
34 // Reserve operand 0 for loop id self reference.
35 auto TempNode = MDNode::getTemporary(Ctx, None);
36 Args.push_back(TempNode.get());
37
38 // If we have a valid debug location for the loop, add it.
39 if (Location)
40 Args.push_back(Location.getAsMDNode());
41
42 // Setting vectorize.width
43 if (Attrs.VectorizeWidth > 0) {
44 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"),
45 ConstantAsMetadata::get(ConstantInt::get(
46 Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))};
47 Args.push_back(MDNode::get(Ctx, Vals));
48 }
49
50 // Setting interleave.count
51 if (Attrs.InterleaveCount > 0) {
52 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"),
53 ConstantAsMetadata::get(ConstantInt::get(
54 Type::getInt32Ty(Ctx), Attrs.InterleaveCount))};
55 Args.push_back(MDNode::get(Ctx, Vals));
56 }
57
58 // Setting interleave.count
59 if (Attrs.UnrollCount > 0) {
60 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
61 ConstantAsMetadata::get(ConstantInt::get(
62 Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
63 Args.push_back(MDNode::get(Ctx, Vals));
64 }
65
66 // Setting vectorize.enable
67 if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
68 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
69 ConstantAsMetadata::get(ConstantInt::get(
70 Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable ==
71 LoopAttributes::Enable)))};
72 Args.push_back(MDNode::get(Ctx, Vals));
73 }
74
75 // Setting unroll.full or unroll.disable
76 if (Attrs.UnrollEnable != LoopAttributes::Unspecified) {
77 std::string Name;
78 if (Attrs.UnrollEnable == LoopAttributes::Enable)
79 Name = "llvm.loop.unroll.enable";
80 else if (Attrs.UnrollEnable == LoopAttributes::Full)
81 Name = "llvm.loop.unroll.full";
82 else
83 Name = "llvm.loop.unroll.disable";
84 Metadata *Vals[] = {MDString::get(Ctx, Name)};
85 Args.push_back(MDNode::get(Ctx, Vals));
86 }
87
88 if (Attrs.DistributeEnable != LoopAttributes::Unspecified) {
89 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
90 ConstantAsMetadata::get(ConstantInt::get(
91 Type::getInt1Ty(Ctx), (Attrs.DistributeEnable ==
92 LoopAttributes::Enable)))};
93 Args.push_back(MDNode::get(Ctx, Vals));
94 }
95
96 // Set the first operand to itself.
97 MDNode *LoopID = MDNode::get(Ctx, Args);
98 LoopID->replaceOperandWith(0, LoopID);
99 return LoopID;
100 }
101
LoopAttributes(bool IsParallel)102 LoopAttributes::LoopAttributes(bool IsParallel)
103 : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
104 UnrollEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
105 InterleaveCount(0), UnrollCount(0),
106 DistributeEnable(LoopAttributes::Unspecified) {}
107
clear()108 void LoopAttributes::clear() {
109 IsParallel = false;
110 VectorizeWidth = 0;
111 InterleaveCount = 0;
112 UnrollCount = 0;
113 VectorizeEnable = LoopAttributes::Unspecified;
114 UnrollEnable = LoopAttributes::Unspecified;
115 }
116
LoopInfo(BasicBlock * Header,const LoopAttributes & Attrs,llvm::DebugLoc Location)117 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
118 llvm::DebugLoc Location)
119 : LoopID(nullptr), Header(Header), Attrs(Attrs) {
120 LoopID = createMetadata(Header->getContext(), Attrs, Location);
121 }
122
push(BasicBlock * Header,llvm::DebugLoc Location)123 void LoopInfoStack::push(BasicBlock *Header, llvm::DebugLoc Location) {
124 Active.push_back(LoopInfo(Header, StagedAttrs, Location));
125 // Clear the attributes so nested loops do not inherit them.
126 StagedAttrs.clear();
127 }
128
push(BasicBlock * Header,clang::ASTContext & Ctx,ArrayRef<const clang::Attr * > Attrs,llvm::DebugLoc Location)129 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
130 ArrayRef<const clang::Attr *> Attrs,
131 llvm::DebugLoc Location) {
132
133 // Identify loop hint attributes from Attrs.
134 for (const auto *Attr : Attrs) {
135 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
136 const OpenCLUnrollHintAttr *OpenCLHint =
137 dyn_cast<OpenCLUnrollHintAttr>(Attr);
138
139 // Skip non loop hint attributes
140 if (!LH && !OpenCLHint) {
141 continue;
142 }
143
144 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
145 LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
146 unsigned ValueInt = 1;
147 // Translate opencl_unroll_hint attribute argument to
148 // equivalent LoopHintAttr enums.
149 // OpenCL v2.0 s6.11.5:
150 // 0 - full unroll (no argument).
151 // 1 - disable unroll.
152 // other positive integer n - unroll by n.
153 if (OpenCLHint) {
154 ValueInt = OpenCLHint->getUnrollHint();
155 if (ValueInt == 0) {
156 State = LoopHintAttr::Full;
157 } else if (ValueInt != 1) {
158 Option = LoopHintAttr::UnrollCount;
159 State = LoopHintAttr::Numeric;
160 }
161 } else if (LH) {
162 auto *ValueExpr = LH->getValue();
163 if (ValueExpr) {
164 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
165 ValueInt = ValueAPS.getSExtValue();
166 }
167
168 Option = LH->getOption();
169 State = LH->getState();
170 }
171 switch (State) {
172 case LoopHintAttr::Disable:
173 switch (Option) {
174 case LoopHintAttr::Vectorize:
175 // Disable vectorization by specifying a width of 1.
176 setVectorizeWidth(1);
177 break;
178 case LoopHintAttr::Interleave:
179 // Disable interleaving by speciyfing a count of 1.
180 setInterleaveCount(1);
181 break;
182 case LoopHintAttr::Unroll:
183 setUnrollState(LoopAttributes::Disable);
184 break;
185 case LoopHintAttr::Distribute:
186 setDistributeState(false);
187 break;
188 case LoopHintAttr::UnrollCount:
189 case LoopHintAttr::VectorizeWidth:
190 case LoopHintAttr::InterleaveCount:
191 llvm_unreachable("Options cannot be disabled.");
192 break;
193 }
194 break;
195 case LoopHintAttr::Enable:
196 switch (Option) {
197 case LoopHintAttr::Vectorize:
198 case LoopHintAttr::Interleave:
199 setVectorizeEnable(true);
200 break;
201 case LoopHintAttr::Unroll:
202 setUnrollState(LoopAttributes::Enable);
203 break;
204 case LoopHintAttr::Distribute:
205 setDistributeState(true);
206 break;
207 case LoopHintAttr::UnrollCount:
208 case LoopHintAttr::VectorizeWidth:
209 case LoopHintAttr::InterleaveCount:
210 llvm_unreachable("Options cannot enabled.");
211 break;
212 }
213 break;
214 case LoopHintAttr::AssumeSafety:
215 switch (Option) {
216 case LoopHintAttr::Vectorize:
217 case LoopHintAttr::Interleave:
218 // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
219 setParallel(true);
220 setVectorizeEnable(true);
221 break;
222 case LoopHintAttr::Unroll:
223 case LoopHintAttr::UnrollCount:
224 case LoopHintAttr::VectorizeWidth:
225 case LoopHintAttr::InterleaveCount:
226 case LoopHintAttr::Distribute:
227 llvm_unreachable("Options cannot be used to assume mem safety.");
228 break;
229 }
230 break;
231 case LoopHintAttr::Full:
232 switch (Option) {
233 case LoopHintAttr::Unroll:
234 setUnrollState(LoopAttributes::Full);
235 break;
236 case LoopHintAttr::Vectorize:
237 case LoopHintAttr::Interleave:
238 case LoopHintAttr::UnrollCount:
239 case LoopHintAttr::VectorizeWidth:
240 case LoopHintAttr::InterleaveCount:
241 case LoopHintAttr::Distribute:
242 llvm_unreachable("Options cannot be used with 'full' hint.");
243 break;
244 }
245 break;
246 case LoopHintAttr::Numeric:
247 switch (Option) {
248 case LoopHintAttr::VectorizeWidth:
249 setVectorizeWidth(ValueInt);
250 break;
251 case LoopHintAttr::InterleaveCount:
252 setInterleaveCount(ValueInt);
253 break;
254 case LoopHintAttr::UnrollCount:
255 setUnrollCount(ValueInt);
256 break;
257 case LoopHintAttr::Unroll:
258 case LoopHintAttr::Vectorize:
259 case LoopHintAttr::Interleave:
260 case LoopHintAttr::Distribute:
261 llvm_unreachable("Options cannot be assigned a value.");
262 break;
263 }
264 break;
265 }
266 }
267
268 /// Stage the attributes.
269 push(Header, Location);
270 }
271
pop()272 void LoopInfoStack::pop() {
273 assert(!Active.empty() && "No active loops to pop");
274 Active.pop_back();
275 }
276
InsertHelper(Instruction * I) const277 void LoopInfoStack::InsertHelper(Instruction *I) const {
278 if (!hasInfo())
279 return;
280
281 const LoopInfo &L = getInfo();
282 if (!L.getLoopID())
283 return;
284
285 if (TerminatorInst *TI = dyn_cast<TerminatorInst>(I)) {
286 for (unsigned i = 0, ie = TI->getNumSuccessors(); i < ie; ++i)
287 if (TI->getSuccessor(i) == L.getHeader()) {
288 TI->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
289 break;
290 }
291 return;
292 }
293
294 if (L.getAttributes().IsParallel && I->mayReadOrWriteMemory())
295 I->setMetadata("llvm.mem.parallel_loop_access", L.getLoopID());
296 }
297