1 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file This file declares the API for the instruction selector.
11 /// This class is responsible for selecting machine instructions.
12 /// It's implemented by the target. It's used by the InstructionSelect pass.
13 //
14 //===----------------------------------------------------------------------===//
15
16 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
17 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
18
19 #include "llvm/ADT/SmallVector.h"
20 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
21 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
22 #include "llvm/CodeGen/GlobalISel/Utils.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineOperand.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
26 #include "llvm/CodeGen/TargetInstrInfo.h"
27 #include "llvm/CodeGen/TargetOpcodes.h"
28 #include "llvm/CodeGen/TargetRegisterInfo.h"
29 #include "llvm/IR/Constants.h"
30 #include "llvm/Support/Debug.h"
31 #include "llvm/Support/ErrorHandling.h"
32 #include "llvm/Support/raw_ostream.h"
33 #include <cassert>
34 #include <cstddef>
35 #include <cstdint>
36
37 namespace llvm {
38
39 /// GlobalISel PatFrag Predicates
40 enum {
41 GIPFP_I64_Invalid = 0,
42 GIPFP_APInt_Invalid = 0,
43 GIPFP_APFloat_Invalid = 0,
44 GIPFP_MI_Invalid = 0,
45 };
46
47 template <class TgtInstructionSelector, class PredicateBitset,
48 class ComplexMatcherMemFn, class CustomRendererFn>
executeMatchTable(TgtInstructionSelector & ISel,NewMIVector & OutMIs,MatcherState & State,const ISelInfoTy<PredicateBitset,ComplexMatcherMemFn,CustomRendererFn> & ISelInfo,const int64_t * MatchTable,const TargetInstrInfo & TII,MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,const RegisterBankInfo & RBI,const PredicateBitset & AvailableFeatures,CodeGenCoverage & CoverageInfo)49 bool InstructionSelector::executeMatchTable(
50 TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
51 const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
52 &ISelInfo,
53 const int64_t *MatchTable, const TargetInstrInfo &TII,
54 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
55 const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
56 CodeGenCoverage &CoverageInfo) const {
57
58 uint64_t CurrentIdx = 0;
59 SmallVector<uint64_t, 4> OnFailResumeAt;
60
61 enum RejectAction { RejectAndGiveUp, RejectAndResume };
62 auto handleReject = [&]() -> RejectAction {
63 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64 dbgs() << CurrentIdx << ": Rejected\n");
65 if (OnFailResumeAt.empty())
66 return RejectAndGiveUp;
67 CurrentIdx = OnFailResumeAt.pop_back_val();
68 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70 << OnFailResumeAt.size() << " try-blocks remain)\n");
71 return RejectAndResume;
72 };
73
74 while (true) {
75 assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76 int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77 switch (MatcherOpcode) {
78 case GIM_Try: {
79 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80 dbgs() << CurrentIdx << ": Begin try-block\n");
81 OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82 break;
83 }
84
85 case GIM_RecordInsn: {
86 int64_t NewInsnID = MatchTable[CurrentIdx++];
87 int64_t InsnID = MatchTable[CurrentIdx++];
88 int64_t OpIdx = MatchTable[CurrentIdx++];
89
90 // As an optimisation we require that MIs[0] is always the root. Refuse
91 // any attempt to modify it.
92 assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
94 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95 if (!MO.isReg()) {
96 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97 dbgs() << CurrentIdx << ": Not a register\n");
98 if (handleReject() == RejectAndGiveUp)
99 return false;
100 break;
101 }
102 if (TRI.isPhysicalRegister(MO.getReg())) {
103 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104 dbgs() << CurrentIdx << ": Is a physical register\n");
105 if (handleReject() == RejectAndGiveUp)
106 return false;
107 break;
108 }
109
110 MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111 if ((size_t)NewInsnID < State.MIs.size())
112 State.MIs[NewInsnID] = NewMI;
113 else {
114 assert((size_t)NewInsnID == State.MIs.size() &&
115 "Expected to store MIs in order");
116 State.MIs.push_back(NewMI);
117 }
118 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119 dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121 << ")\n");
122 break;
123 }
124
125 case GIM_CheckFeatures: {
126 int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128 dbgs() << CurrentIdx
129 << ": GIM_CheckFeatures(ExpectedBitsetID="
130 << ExpectedBitsetID << ")\n");
131 if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132 ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133 if (handleReject() == RejectAndGiveUp)
134 return false;
135 }
136 break;
137 }
138
139 case GIM_CheckOpcode: {
140 int64_t InsnID = MatchTable[CurrentIdx++];
141 int64_t Expected = MatchTable[CurrentIdx++];
142
143 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144 unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
146 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148 << "], ExpectedOpcode=" << Expected
149 << ") // Got=" << Opcode << "\n");
150 if (Opcode != Expected) {
151 if (handleReject() == RejectAndGiveUp)
152 return false;
153 }
154 break;
155 }
156
157 case GIM_SwitchOpcode: {
158 int64_t InsnID = MatchTable[CurrentIdx++];
159 int64_t LowerBound = MatchTable[CurrentIdx++];
160 int64_t UpperBound = MatchTable[CurrentIdx++];
161 int64_t Default = MatchTable[CurrentIdx++];
162
163 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164 const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
166 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168 << LowerBound << ", " << UpperBound << "), Default=" << Default
169 << ", JumpTable...) // Got=" << Opcode << "\n";
170 });
171 if (Opcode < LowerBound || UpperBound <= Opcode) {
172 CurrentIdx = Default;
173 break;
174 }
175 CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176 if (!CurrentIdx) {
177 CurrentIdx = Default;
178 break;
179 }
180 OnFailResumeAt.push_back(Default);
181 break;
182 }
183
184 case GIM_SwitchType: {
185 int64_t InsnID = MatchTable[CurrentIdx++];
186 int64_t OpIdx = MatchTable[CurrentIdx++];
187 int64_t LowerBound = MatchTable[CurrentIdx++];
188 int64_t UpperBound = MatchTable[CurrentIdx++];
189 int64_t Default = MatchTable[CurrentIdx++];
190
191 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
194 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197 << UpperBound << "), Default=" << Default
198 << ", JumpTable...) // Got=";
199 if (!MO.isReg())
200 dbgs() << "Not a VReg\n";
201 else
202 dbgs() << MRI.getType(MO.getReg()) << "\n";
203 });
204 if (!MO.isReg()) {
205 CurrentIdx = Default;
206 break;
207 }
208 const LLT Ty = MRI.getType(MO.getReg());
209 const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210 if (TyI == ISelInfo.TypeIDMap.end()) {
211 CurrentIdx = Default;
212 break;
213 }
214 const int64_t TypeID = TyI->second;
215 if (TypeID < LowerBound || UpperBound <= TypeID) {
216 CurrentIdx = Default;
217 break;
218 }
219 CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220 if (!CurrentIdx) {
221 CurrentIdx = Default;
222 break;
223 }
224 OnFailResumeAt.push_back(Default);
225 break;
226 }
227
228 case GIM_CheckNumOperands: {
229 int64_t InsnID = MatchTable[CurrentIdx++];
230 int64_t Expected = MatchTable[CurrentIdx++];
231 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233 << InsnID << "], Expected=" << Expected << ")\n");
234 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235 if (State.MIs[InsnID]->getNumOperands() != Expected) {
236 if (handleReject() == RejectAndGiveUp)
237 return false;
238 }
239 break;
240 }
241 case GIM_CheckI64ImmPredicate: {
242 int64_t InsnID = MatchTable[CurrentIdx++];
243 int64_t Predicate = MatchTable[CurrentIdx++];
244 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245 dbgs()
246 << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247 << InsnID << "], Predicate=" << Predicate << ")\n");
248 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250 "Expected G_CONSTANT");
251 assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252 int64_t Value = 0;
253 if (State.MIs[InsnID]->getOperand(1).isCImm())
254 Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255 else if (State.MIs[InsnID]->getOperand(1).isImm())
256 Value = State.MIs[InsnID]->getOperand(1).getImm();
257 else
258 llvm_unreachable("Expected Imm or CImm operand");
259
260 if (!testImmPredicate_I64(Predicate, Value))
261 if (handleReject() == RejectAndGiveUp)
262 return false;
263 break;
264 }
265 case GIM_CheckAPIntImmPredicate: {
266 int64_t InsnID = MatchTable[CurrentIdx++];
267 int64_t Predicate = MatchTable[CurrentIdx++];
268 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269 dbgs()
270 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271 << InsnID << "], Predicate=" << Predicate << ")\n");
272 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274 "Expected G_CONSTANT");
275 assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276 APInt Value;
277 if (State.MIs[InsnID]->getOperand(1).isCImm())
278 Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279 else
280 llvm_unreachable("Expected Imm or CImm operand");
281
282 if (!testImmPredicate_APInt(Predicate, Value))
283 if (handleReject() == RejectAndGiveUp)
284 return false;
285 break;
286 }
287 case GIM_CheckAPFloatImmPredicate: {
288 int64_t InsnID = MatchTable[CurrentIdx++];
289 int64_t Predicate = MatchTable[CurrentIdx++];
290 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291 dbgs()
292 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293 << InsnID << "], Predicate=" << Predicate << ")\n");
294 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296 "Expected G_FCONSTANT");
297 assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298 assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299 APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
301 if (!testImmPredicate_APFloat(Predicate, Value))
302 if (handleReject() == RejectAndGiveUp)
303 return false;
304 break;
305 }
306 case GIM_CheckCxxInsnPredicate: {
307 int64_t InsnID = MatchTable[CurrentIdx++];
308 int64_t Predicate = MatchTable[CurrentIdx++];
309 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310 dbgs()
311 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312 << InsnID << "], Predicate=" << Predicate << ")\n");
313 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314 assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
316 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317 if (handleReject() == RejectAndGiveUp)
318 return false;
319 break;
320 }
321 case GIM_CheckAtomicOrdering: {
322 int64_t InsnID = MatchTable[CurrentIdx++];
323 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326 << InsnID << "], " << (uint64_t)Ordering << ")\n");
327 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328 if (!State.MIs[InsnID]->hasOneMemOperand())
329 if (handleReject() == RejectAndGiveUp)
330 return false;
331
332 for (const auto &MMO : State.MIs[InsnID]->memoperands())
333 if (MMO->getOrdering() != Ordering)
334 if (handleReject() == RejectAndGiveUp)
335 return false;
336 break;
337 }
338 case GIM_CheckAtomicOrderingOrStrongerThan: {
339 int64_t InsnID = MatchTable[CurrentIdx++];
340 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342 dbgs() << CurrentIdx
343 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344 << InsnID << "], " << (uint64_t)Ordering << ")\n");
345 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346 if (!State.MIs[InsnID]->hasOneMemOperand())
347 if (handleReject() == RejectAndGiveUp)
348 return false;
349
350 for (const auto &MMO : State.MIs[InsnID]->memoperands())
351 if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352 if (handleReject() == RejectAndGiveUp)
353 return false;
354 break;
355 }
356 case GIM_CheckAtomicOrderingWeakerThan: {
357 int64_t InsnID = MatchTable[CurrentIdx++];
358 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360 dbgs() << CurrentIdx
361 << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362 << InsnID << "], " << (uint64_t)Ordering << ")\n");
363 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364 if (!State.MIs[InsnID]->hasOneMemOperand())
365 if (handleReject() == RejectAndGiveUp)
366 return false;
367
368 for (const auto &MMO : State.MIs[InsnID]->memoperands())
369 if (!isStrongerThan(Ordering, MMO->getOrdering()))
370 if (handleReject() == RejectAndGiveUp)
371 return false;
372 break;
373 }
374 case GIM_CheckMemorySizeEqualTo: {
375 int64_t InsnID = MatchTable[CurrentIdx++];
376 int64_t MMOIdx = MatchTable[CurrentIdx++];
377 uint64_t Size = MatchTable[CurrentIdx++];
378
379 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380 dbgs() << CurrentIdx
381 << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382 << "]->memoperands() + " << MMOIdx
383 << ", Size=" << Size << ")\n");
384 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
386 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387 if (handleReject() == RejectAndGiveUp)
388 return false;
389 break;
390 }
391
392 MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
394 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395 dbgs() << MMO->getSize() << " bytes vs " << Size
396 << " bytes\n");
397 if (MMO->getSize() != Size)
398 if (handleReject() == RejectAndGiveUp)
399 return false;
400
401 break;
402 }
403 case GIM_CheckMemorySizeEqualToLLT:
404 case GIM_CheckMemorySizeLessThanLLT:
405 case GIM_CheckMemorySizeGreaterThanLLT: {
406 int64_t InsnID = MatchTable[CurrentIdx++];
407 int64_t MMOIdx = MatchTable[CurrentIdx++];
408 int64_t OpIdx = MatchTable[CurrentIdx++];
409
410 DEBUG_WITH_TYPE(
411 TgtInstructionSelector::getName(),
412 dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414 ? "EqualTo"
415 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416 ? "GreaterThan"
417 : "LessThan")
418 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419 << ", OpIdx=" << OpIdx << ")\n");
420 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
422 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423 if (!MO.isReg()) {
424 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425 dbgs() << CurrentIdx << ": Not a register\n");
426 if (handleReject() == RejectAndGiveUp)
427 return false;
428 break;
429 }
430
431 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432 if (handleReject() == RejectAndGiveUp)
433 return false;
434 break;
435 }
436
437 MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
439 unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441 MMO->getSize() * 8 != Size) {
442 if (handleReject() == RejectAndGiveUp)
443 return false;
444 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445 MMO->getSize() * 8 >= Size) {
446 if (handleReject() == RejectAndGiveUp)
447 return false;
448 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449 MMO->getSize() * 8 <= Size)
450 if (handleReject() == RejectAndGiveUp)
451 return false;
452
453 break;
454 }
455 case GIM_CheckType: {
456 int64_t InsnID = MatchTable[CurrentIdx++];
457 int64_t OpIdx = MatchTable[CurrentIdx++];
458 int64_t TypeID = MatchTable[CurrentIdx++];
459 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461 << "]->getOperand(" << OpIdx
462 << "), TypeID=" << TypeID << ")\n");
463 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465 if (!MO.isReg() ||
466 MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467 if (handleReject() == RejectAndGiveUp)
468 return false;
469 }
470 break;
471 }
472 case GIM_CheckPointerToAny: {
473 int64_t InsnID = MatchTable[CurrentIdx++];
474 int64_t OpIdx = MatchTable[CurrentIdx++];
475 int64_t SizeInBits = MatchTable[CurrentIdx++];
476
477 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479 << InsnID << "]->getOperand(" << OpIdx
480 << "), SizeInBits=" << SizeInBits << ")\n");
481 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482 // iPTR must be looked up in the target.
483 if (SizeInBits == 0) {
484 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485 SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486 }
487
488 assert(SizeInBits != 0 && "Pointer size must be known");
489
490 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491 if (MO.isReg()) {
492 const LLT &Ty = MRI.getType(MO.getReg());
493 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494 if (handleReject() == RejectAndGiveUp)
495 return false;
496 } else if (handleReject() == RejectAndGiveUp)
497 return false;
498
499 break;
500 }
501 case GIM_CheckRegBankForClass: {
502 int64_t InsnID = MatchTable[CurrentIdx++];
503 int64_t OpIdx = MatchTable[CurrentIdx++];
504 int64_t RCEnum = MatchTable[CurrentIdx++];
505 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507 << InsnID << "]->getOperand(" << OpIdx
508 << "), RCEnum=" << RCEnum << ")\n");
509 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511 if (!MO.isReg() ||
512 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513 RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514 if (handleReject() == RejectAndGiveUp)
515 return false;
516 }
517 break;
518 }
519
520 case GIM_CheckComplexPattern: {
521 int64_t InsnID = MatchTable[CurrentIdx++];
522 int64_t OpIdx = MatchTable[CurrentIdx++];
523 int64_t RendererID = MatchTable[CurrentIdx++];
524 int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527 << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528 << "]->getOperand(" << OpIdx
529 << "), ComplexPredicateID=" << ComplexPredicateID
530 << ")\n");
531 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532 // FIXME: Use std::invoke() when it's available.
533 ComplexRendererFns Renderer =
534 (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535 State.MIs[InsnID]->getOperand(OpIdx));
536 if (Renderer.hasValue())
537 State.Renderers[RendererID] = Renderer.getValue();
538 else
539 if (handleReject() == RejectAndGiveUp)
540 return false;
541 break;
542 }
543
544 case GIM_CheckConstantInt: {
545 int64_t InsnID = MatchTable[CurrentIdx++];
546 int64_t OpIdx = MatchTable[CurrentIdx++];
547 int64_t Value = MatchTable[CurrentIdx++];
548 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550 << InsnID << "]->getOperand(" << OpIdx
551 << "), Value=" << Value << ")\n");
552 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554 if (MO.isReg()) {
555 // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556 LLT Ty = MRI.getType(MO.getReg());
557 Value = SignExtend64(Value, Ty.getSizeInBits());
558
559 if (!isOperandImmEqual(MO, Value, MRI)) {
560 if (handleReject() == RejectAndGiveUp)
561 return false;
562 }
563 } else if (handleReject() == RejectAndGiveUp)
564 return false;
565
566 break;
567 }
568
569 case GIM_CheckLiteralInt: {
570 int64_t InsnID = MatchTable[CurrentIdx++];
571 int64_t OpIdx = MatchTable[CurrentIdx++];
572 int64_t Value = MatchTable[CurrentIdx++];
573 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575 << InsnID << "]->getOperand(" << OpIdx
576 << "), Value=" << Value << ")\n");
577 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579 if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580 if (handleReject() == RejectAndGiveUp)
581 return false;
582 }
583 break;
584 }
585
586 case GIM_CheckIntrinsicID: {
587 int64_t InsnID = MatchTable[CurrentIdx++];
588 int64_t OpIdx = MatchTable[CurrentIdx++];
589 int64_t Value = MatchTable[CurrentIdx++];
590 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592 << InsnID << "]->getOperand(" << OpIdx
593 << "), Value=" << Value << ")\n");
594 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
597 if (handleReject() == RejectAndGiveUp)
598 return false;
599 break;
600 }
601
602 case GIM_CheckIsMBB: {
603 int64_t InsnID = MatchTable[CurrentIdx++];
604 int64_t OpIdx = MatchTable[CurrentIdx++];
605 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607 << "]->getOperand(" << OpIdx << "))\n");
608 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610 if (handleReject() == RejectAndGiveUp)
611 return false;
612 }
613 break;
614 }
615
616 case GIM_CheckIsSafeToFold: {
617 int64_t InsnID = MatchTable[CurrentIdx++];
618 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620 << InsnID << "])\n");
621 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622 if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623 if (handleReject() == RejectAndGiveUp)
624 return false;
625 }
626 break;
627 }
628 case GIM_CheckIsSameOperand: {
629 int64_t InsnID = MatchTable[CurrentIdx++];
630 int64_t OpIdx = MatchTable[CurrentIdx++];
631 int64_t OtherInsnID = MatchTable[CurrentIdx++];
632 int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635 << InsnID << "][" << OpIdx << "], MIs["
636 << OtherInsnID << "][" << OtherOpIdx << "])\n");
637 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639 if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640 State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641 if (handleReject() == RejectAndGiveUp)
642 return false;
643 }
644 break;
645 }
646 case GIM_Reject:
647 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648 dbgs() << CurrentIdx << ": GIM_Reject\n");
649 if (handleReject() == RejectAndGiveUp)
650 return false;
651 break;
652
653 case GIR_MutateOpcode: {
654 int64_t OldInsnID = MatchTable[CurrentIdx++];
655 uint64_t NewInsnID = MatchTable[CurrentIdx++];
656 int64_t NewOpcode = MatchTable[CurrentIdx++];
657 if (NewInsnID >= OutMIs.size())
658 OutMIs.resize(NewInsnID + 1);
659
660 OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661 State.MIs[OldInsnID]);
662 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665 << NewInsnID << "], MIs[" << OldInsnID << "], "
666 << NewOpcode << ")\n");
667 break;
668 }
669
670 case GIR_BuildMI: {
671 uint64_t NewInsnID = MatchTable[CurrentIdx++];
672 int64_t Opcode = MatchTable[CurrentIdx++];
673 if (NewInsnID >= OutMIs.size())
674 OutMIs.resize(NewInsnID + 1);
675
676 OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677 State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680 << NewInsnID << "], " << Opcode << ")\n");
681 break;
682 }
683
684 case GIR_Copy: {
685 int64_t NewInsnID = MatchTable[CurrentIdx++];
686 int64_t OldInsnID = MatchTable[CurrentIdx++];
687 int64_t OpIdx = MatchTable[CurrentIdx++];
688 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691 dbgs()
692 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694 break;
695 }
696
697 case GIR_CopyOrAddZeroReg: {
698 int64_t NewInsnID = MatchTable[CurrentIdx++];
699 int64_t OldInsnID = MatchTable[CurrentIdx++];
700 int64_t OpIdx = MatchTable[CurrentIdx++];
701 int64_t ZeroReg = MatchTable[CurrentIdx++];
702 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704 if (isOperandImmEqual(MO, 0, MRI))
705 OutMIs[NewInsnID].addReg(ZeroReg);
706 else
707 OutMIs[NewInsnID].add(MO);
708 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710 << NewInsnID << "], MIs[" << OldInsnID << "], "
711 << OpIdx << ", " << ZeroReg << ")\n");
712 break;
713 }
714
715 case GIR_CopySubReg: {
716 int64_t NewInsnID = MatchTable[CurrentIdx++];
717 int64_t OldInsnID = MatchTable[CurrentIdx++];
718 int64_t OpIdx = MatchTable[CurrentIdx++];
719 int64_t SubRegIdx = MatchTable[CurrentIdx++];
720 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722 0, SubRegIdx);
723 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725 << NewInsnID << "], MIs[" << OldInsnID << "], "
726 << OpIdx << ", " << SubRegIdx << ")\n");
727 break;
728 }
729
730 case GIR_AddImplicitDef: {
731 int64_t InsnID = MatchTable[CurrentIdx++];
732 int64_t RegNum = MatchTable[CurrentIdx++];
733 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734 OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737 << InsnID << "], " << RegNum << ")\n");
738 break;
739 }
740
741 case GIR_AddImplicitUse: {
742 int64_t InsnID = MatchTable[CurrentIdx++];
743 int64_t RegNum = MatchTable[CurrentIdx++];
744 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745 OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748 << InsnID << "], " << RegNum << ")\n");
749 break;
750 }
751
752 case GIR_AddRegister: {
753 int64_t InsnID = MatchTable[CurrentIdx++];
754 int64_t RegNum = MatchTable[CurrentIdx++];
755 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756 OutMIs[InsnID].addReg(RegNum);
757 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758 dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759 << InsnID << "], " << RegNum << ")\n");
760 break;
761 }
762
763 case GIR_AddTempRegister: {
764 int64_t InsnID = MatchTable[CurrentIdx++];
765 int64_t TempRegID = MatchTable[CurrentIdx++];
766 uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771 << InsnID << "], TempRegisters[" << TempRegID
772 << "], " << TempRegFlags << ")\n");
773 break;
774 }
775
776 case GIR_AddImm: {
777 int64_t InsnID = MatchTable[CurrentIdx++];
778 int64_t Imm = MatchTable[CurrentIdx++];
779 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780 OutMIs[InsnID].addImm(Imm);
781 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783 << "], " << Imm << ")\n");
784 break;
785 }
786
787 case GIR_ComplexRenderer: {
788 int64_t InsnID = MatchTable[CurrentIdx++];
789 int64_t RendererID = MatchTable[CurrentIdx++];
790 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791 for (const auto &RenderOpFn : State.Renderers[RendererID])
792 RenderOpFn(OutMIs[InsnID]);
793 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795 << InsnID << "], " << RendererID << ")\n");
796 break;
797 }
798 case GIR_ComplexSubOperandRenderer: {
799 int64_t InsnID = MatchTable[CurrentIdx++];
800 int64_t RendererID = MatchTable[CurrentIdx++];
801 int64_t RenderOpID = MatchTable[CurrentIdx++];
802 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805 dbgs() << CurrentIdx
806 << ": GIR_ComplexSubOperandRenderer(OutMIs["
807 << InsnID << "], " << RendererID << ", "
808 << RenderOpID << ")\n");
809 break;
810 }
811
812 case GIR_CopyConstantAsSImm: {
813 int64_t NewInsnID = MatchTable[CurrentIdx++];
814 int64_t OldInsnID = MatchTable[CurrentIdx++];
815 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818 OutMIs[NewInsnID].addImm(
819 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820 } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
821 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822 else
823 llvm_unreachable("Expected Imm or CImm operand");
824 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827 break;
828 }
829
830 // TODO: Needs a test case once we have a pattern that uses this.
831 case GIR_CopyFConstantAsFPImm: {
832 int64_t NewInsnID = MatchTable[CurrentIdx++];
833 int64_t OldInsnID = MatchTable[CurrentIdx++];
834 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836 if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837 OutMIs[NewInsnID].addFPImm(
838 State.MIs[OldInsnID]->getOperand(1).getFPImm());
839 else
840 llvm_unreachable("Expected FPImm operand");
841 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842 dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844 break;
845 }
846
847 case GIR_CustomRenderer: {
848 int64_t InsnID = MatchTable[CurrentIdx++];
849 int64_t OldInsnID = MatchTable[CurrentIdx++];
850 int64_t RendererFnID = MatchTable[CurrentIdx++];
851 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854 << InsnID << "], MIs[" << OldInsnID << "], "
855 << RendererFnID << ")\n");
856 (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857 *State.MIs[OldInsnID]);
858 break;
859 }
860 case GIR_ConstrainOperandRC: {
861 int64_t InsnID = MatchTable[CurrentIdx++];
862 int64_t OpIdx = MatchTable[CurrentIdx++];
863 int64_t RCEnum = MatchTable[CurrentIdx++];
864 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865 constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866 *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869 << InsnID << "], " << OpIdx << ", " << RCEnum
870 << ")\n");
871 break;
872 }
873
874 case GIR_ConstrainSelectedInstOperands: {
875 int64_t InsnID = MatchTable[CurrentIdx++];
876 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878 RBI);
879 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880 dbgs() << CurrentIdx
881 << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882 << InsnID << "])\n");
883 break;
884 }
885
886 case GIR_MergeMemOperands: {
887 int64_t InsnID = MatchTable[CurrentIdx++];
888 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
890 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892 << InsnID << "]");
893 int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894 while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895 GIU_MergeMemOperands_EndOfList) {
896 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897 dbgs() << ", MIs[" << MergeInsnID << "]");
898 for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899 OutMIs[InsnID].addMemOperand(MMO);
900 }
901 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902 break;
903 }
904
905 case GIR_EraseFromParent: {
906 int64_t InsnID = MatchTable[CurrentIdx++];
907 assert(State.MIs[InsnID] &&
908 "Attempted to erase an undefined instruction");
909 State.MIs[InsnID]->eraseFromParent();
910 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912 << InsnID << "])\n");
913 break;
914 }
915
916 case GIR_MakeTempReg: {
917 int64_t TempRegID = MatchTable[CurrentIdx++];
918 int64_t TypeID = MatchTable[CurrentIdx++];
919
920 State.TempRegisters[TempRegID] =
921 MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
922 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
923 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
924 << "] = GIR_MakeTempReg(" << TypeID << ")\n");
925 break;
926 }
927
928 case GIR_Coverage: {
929 int64_t RuleID = MatchTable[CurrentIdx++];
930 CoverageInfo.setCovered(RuleID);
931
932 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
933 dbgs()
934 << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
935 break;
936 }
937
938 case GIR_Done:
939 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
940 dbgs() << CurrentIdx << ": GIR_Done\n");
941 return true;
942
943 default:
944 llvm_unreachable("Unexpected command");
945 }
946 }
947 }
948
949 } // end namespace llvm
950
951 #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
952