• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file This file declares the API for the instruction selector.
11 /// This class is responsible for selecting machine instructions.
12 /// It's implemented by the target. It's used by the InstructionSelect pass.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
17 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
18 
19 #include "llvm/ADT/SmallVector.h"
20 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
21 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
22 #include "llvm/CodeGen/GlobalISel/Utils.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineOperand.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
26 #include "llvm/CodeGen/TargetInstrInfo.h"
27 #include "llvm/CodeGen/TargetOpcodes.h"
28 #include "llvm/CodeGen/TargetRegisterInfo.h"
29 #include "llvm/IR/Constants.h"
30 #include "llvm/Support/Debug.h"
31 #include "llvm/Support/ErrorHandling.h"
32 #include "llvm/Support/raw_ostream.h"
33 #include <cassert>
34 #include <cstddef>
35 #include <cstdint>
36 
37 namespace llvm {
38 
39 /// GlobalISel PatFrag Predicates
40 enum {
41   GIPFP_I64_Invalid = 0,
42   GIPFP_APInt_Invalid = 0,
43   GIPFP_APFloat_Invalid = 0,
44   GIPFP_MI_Invalid = 0,
45 };
46 
47 template <class TgtInstructionSelector, class PredicateBitset,
48           class ComplexMatcherMemFn, class CustomRendererFn>
executeMatchTable(TgtInstructionSelector & ISel,NewMIVector & OutMIs,MatcherState & State,const ISelInfoTy<PredicateBitset,ComplexMatcherMemFn,CustomRendererFn> & ISelInfo,const int64_t * MatchTable,const TargetInstrInfo & TII,MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,const RegisterBankInfo & RBI,const PredicateBitset & AvailableFeatures,CodeGenCoverage & CoverageInfo)49 bool InstructionSelector::executeMatchTable(
50     TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
51     const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
52         &ISelInfo,
53     const int64_t *MatchTable, const TargetInstrInfo &TII,
54     MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
55     const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
56     CodeGenCoverage &CoverageInfo) const {
57 
58   uint64_t CurrentIdx = 0;
59   SmallVector<uint64_t, 4> OnFailResumeAt;
60 
61   enum RejectAction { RejectAndGiveUp, RejectAndResume };
62   auto handleReject = [&]() -> RejectAction {
63     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64                     dbgs() << CurrentIdx << ": Rejected\n");
65     if (OnFailResumeAt.empty())
66       return RejectAndGiveUp;
67     CurrentIdx = OnFailResumeAt.pop_back_val();
68     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69                     dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70                            << OnFailResumeAt.size() << " try-blocks remain)\n");
71     return RejectAndResume;
72   };
73 
74   while (true) {
75     assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76     int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77     switch (MatcherOpcode) {
78     case GIM_Try: {
79       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80                       dbgs() << CurrentIdx << ": Begin try-block\n");
81       OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82       break;
83     }
84 
85     case GIM_RecordInsn: {
86       int64_t NewInsnID = MatchTable[CurrentIdx++];
87       int64_t InsnID = MatchTable[CurrentIdx++];
88       int64_t OpIdx = MatchTable[CurrentIdx++];
89 
90       // As an optimisation we require that MIs[0] is always the root. Refuse
91       // any attempt to modify it.
92       assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93 
94       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95       if (!MO.isReg()) {
96         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97                         dbgs() << CurrentIdx << ": Not a register\n");
98         if (handleReject() == RejectAndGiveUp)
99           return false;
100         break;
101       }
102       if (TRI.isPhysicalRegister(MO.getReg())) {
103         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104                         dbgs() << CurrentIdx << ": Is a physical register\n");
105         if (handleReject() == RejectAndGiveUp)
106           return false;
107         break;
108       }
109 
110       MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111       if ((size_t)NewInsnID < State.MIs.size())
112         State.MIs[NewInsnID] = NewMI;
113       else {
114         assert((size_t)NewInsnID == State.MIs.size() &&
115                "Expected to store MIs in order");
116         State.MIs.push_back(NewMI);
117       }
118       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119                       dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120                              << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121                              << ")\n");
122       break;
123     }
124 
125     case GIM_CheckFeatures: {
126       int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128                       dbgs() << CurrentIdx
129                              << ": GIM_CheckFeatures(ExpectedBitsetID="
130                              << ExpectedBitsetID << ")\n");
131       if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132           ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133         if (handleReject() == RejectAndGiveUp)
134           return false;
135       }
136       break;
137     }
138 
139     case GIM_CheckOpcode: {
140       int64_t InsnID = MatchTable[CurrentIdx++];
141       int64_t Expected = MatchTable[CurrentIdx++];
142 
143       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144       unsigned Opcode = State.MIs[InsnID]->getOpcode();
145 
146       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147                       dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148                              << "], ExpectedOpcode=" << Expected
149                              << ") // Got=" << Opcode << "\n");
150       if (Opcode != Expected) {
151         if (handleReject() == RejectAndGiveUp)
152           return false;
153       }
154       break;
155     }
156 
157     case GIM_SwitchOpcode: {
158       int64_t InsnID = MatchTable[CurrentIdx++];
159       int64_t LowerBound = MatchTable[CurrentIdx++];
160       int64_t UpperBound = MatchTable[CurrentIdx++];
161       int64_t Default = MatchTable[CurrentIdx++];
162 
163       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164       const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165 
166       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167         dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168                << LowerBound << ", " << UpperBound << "), Default=" << Default
169                << ", JumpTable...) // Got=" << Opcode << "\n";
170       });
171       if (Opcode < LowerBound || UpperBound <= Opcode) {
172         CurrentIdx = Default;
173         break;
174       }
175       CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176       if (!CurrentIdx) {
177         CurrentIdx = Default;
178 	break;
179       }
180       OnFailResumeAt.push_back(Default);
181       break;
182     }
183 
184     case GIM_SwitchType: {
185       int64_t InsnID = MatchTable[CurrentIdx++];
186       int64_t OpIdx = MatchTable[CurrentIdx++];
187       int64_t LowerBound = MatchTable[CurrentIdx++];
188       int64_t UpperBound = MatchTable[CurrentIdx++];
189       int64_t Default = MatchTable[CurrentIdx++];
190 
191       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193 
194       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195         dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196                << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197                << UpperBound << "), Default=" << Default
198                << ", JumpTable...) // Got=";
199         if (!MO.isReg())
200           dbgs() << "Not a VReg\n";
201         else
202           dbgs() << MRI.getType(MO.getReg()) << "\n";
203       });
204       if (!MO.isReg()) {
205         CurrentIdx = Default;
206         break;
207       }
208       const LLT Ty = MRI.getType(MO.getReg());
209       const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210       if (TyI == ISelInfo.TypeIDMap.end()) {
211         CurrentIdx = Default;
212         break;
213       }
214       const int64_t TypeID = TyI->second;
215       if (TypeID < LowerBound || UpperBound <= TypeID) {
216         CurrentIdx = Default;
217         break;
218       }
219       CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220       if (!CurrentIdx) {
221         CurrentIdx = Default;
222         break;
223       }
224       OnFailResumeAt.push_back(Default);
225       break;
226     }
227 
228     case GIM_CheckNumOperands: {
229       int64_t InsnID = MatchTable[CurrentIdx++];
230       int64_t Expected = MatchTable[CurrentIdx++];
231       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232                       dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233                              << InsnID << "], Expected=" << Expected << ")\n");
234       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235       if (State.MIs[InsnID]->getNumOperands() != Expected) {
236         if (handleReject() == RejectAndGiveUp)
237           return false;
238       }
239       break;
240     }
241     case GIM_CheckI64ImmPredicate: {
242       int64_t InsnID = MatchTable[CurrentIdx++];
243       int64_t Predicate = MatchTable[CurrentIdx++];
244       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245                       dbgs()
246                           << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247                           << InsnID << "], Predicate=" << Predicate << ")\n");
248       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250              "Expected G_CONSTANT");
251       assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252       int64_t Value = 0;
253       if (State.MIs[InsnID]->getOperand(1).isCImm())
254         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255       else if (State.MIs[InsnID]->getOperand(1).isImm())
256         Value = State.MIs[InsnID]->getOperand(1).getImm();
257       else
258         llvm_unreachable("Expected Imm or CImm operand");
259 
260       if (!testImmPredicate_I64(Predicate, Value))
261         if (handleReject() == RejectAndGiveUp)
262           return false;
263       break;
264     }
265     case GIM_CheckAPIntImmPredicate: {
266       int64_t InsnID = MatchTable[CurrentIdx++];
267       int64_t Predicate = MatchTable[CurrentIdx++];
268       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269                       dbgs()
270                           << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271                           << InsnID << "], Predicate=" << Predicate << ")\n");
272       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274              "Expected G_CONSTANT");
275       assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276       APInt Value;
277       if (State.MIs[InsnID]->getOperand(1).isCImm())
278         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279       else
280         llvm_unreachable("Expected Imm or CImm operand");
281 
282       if (!testImmPredicate_APInt(Predicate, Value))
283         if (handleReject() == RejectAndGiveUp)
284           return false;
285       break;
286     }
287     case GIM_CheckAPFloatImmPredicate: {
288       int64_t InsnID = MatchTable[CurrentIdx++];
289       int64_t Predicate = MatchTable[CurrentIdx++];
290       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291                       dbgs()
292                           << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293                           << InsnID << "], Predicate=" << Predicate << ")\n");
294       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296              "Expected G_FCONSTANT");
297       assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298       assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299       APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300 
301       if (!testImmPredicate_APFloat(Predicate, Value))
302         if (handleReject() == RejectAndGiveUp)
303           return false;
304       break;
305     }
306     case GIM_CheckCxxInsnPredicate: {
307       int64_t InsnID = MatchTable[CurrentIdx++];
308       int64_t Predicate = MatchTable[CurrentIdx++];
309       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310                       dbgs()
311                           << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312                           << InsnID << "], Predicate=" << Predicate << ")\n");
313       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314       assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315 
316       if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317         if (handleReject() == RejectAndGiveUp)
318           return false;
319       break;
320     }
321     case GIM_CheckAtomicOrdering: {
322       int64_t InsnID = MatchTable[CurrentIdx++];
323       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325                       dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
327       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328       if (!State.MIs[InsnID]->hasOneMemOperand())
329         if (handleReject() == RejectAndGiveUp)
330           return false;
331 
332       for (const auto &MMO : State.MIs[InsnID]->memoperands())
333         if (MMO->getOrdering() != Ordering)
334           if (handleReject() == RejectAndGiveUp)
335             return false;
336       break;
337     }
338     case GIM_CheckAtomicOrderingOrStrongerThan: {
339       int64_t InsnID = MatchTable[CurrentIdx++];
340       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342                       dbgs() << CurrentIdx
343                              << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
345       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346       if (!State.MIs[InsnID]->hasOneMemOperand())
347         if (handleReject() == RejectAndGiveUp)
348           return false;
349 
350       for (const auto &MMO : State.MIs[InsnID]->memoperands())
351         if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352           if (handleReject() == RejectAndGiveUp)
353             return false;
354       break;
355     }
356     case GIM_CheckAtomicOrderingWeakerThan: {
357       int64_t InsnID = MatchTable[CurrentIdx++];
358       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360                       dbgs() << CurrentIdx
361                              << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
363       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364       if (!State.MIs[InsnID]->hasOneMemOperand())
365         if (handleReject() == RejectAndGiveUp)
366           return false;
367 
368       for (const auto &MMO : State.MIs[InsnID]->memoperands())
369         if (!isStrongerThan(Ordering, MMO->getOrdering()))
370           if (handleReject() == RejectAndGiveUp)
371             return false;
372       break;
373     }
374     case GIM_CheckMemorySizeEqualTo: {
375       int64_t InsnID = MatchTable[CurrentIdx++];
376       int64_t MMOIdx = MatchTable[CurrentIdx++];
377       uint64_t Size = MatchTable[CurrentIdx++];
378 
379       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380                       dbgs() << CurrentIdx
381                              << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382                              << "]->memoperands() + " << MMOIdx
383                              << ", Size=" << Size << ")\n");
384       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385 
386       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387         if (handleReject() == RejectAndGiveUp)
388           return false;
389         break;
390       }
391 
392       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393 
394       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395                       dbgs() << MMO->getSize() << " bytes vs " << Size
396                              << " bytes\n");
397       if (MMO->getSize() != Size)
398         if (handleReject() == RejectAndGiveUp)
399           return false;
400 
401       break;
402     }
403     case GIM_CheckMemorySizeEqualToLLT:
404     case GIM_CheckMemorySizeLessThanLLT:
405     case GIM_CheckMemorySizeGreaterThanLLT: {
406       int64_t InsnID = MatchTable[CurrentIdx++];
407       int64_t MMOIdx = MatchTable[CurrentIdx++];
408       int64_t OpIdx = MatchTable[CurrentIdx++];
409 
410       DEBUG_WITH_TYPE(
411           TgtInstructionSelector::getName(),
412           dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413                  << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414                          ? "EqualTo"
415                          : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416                                ? "GreaterThan"
417                                : "LessThan")
418                  << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419                  << ", OpIdx=" << OpIdx << ")\n");
420       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421 
422       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423       if (!MO.isReg()) {
424         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425                         dbgs() << CurrentIdx << ": Not a register\n");
426         if (handleReject() == RejectAndGiveUp)
427           return false;
428         break;
429       }
430 
431       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432         if (handleReject() == RejectAndGiveUp)
433           return false;
434         break;
435       }
436 
437       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438 
439       unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440       if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441           MMO->getSize() * 8 != Size) {
442         if (handleReject() == RejectAndGiveUp)
443           return false;
444       } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445                  MMO->getSize() * 8 >= Size) {
446         if (handleReject() == RejectAndGiveUp)
447           return false;
448       } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449                  MMO->getSize() * 8 <= Size)
450         if (handleReject() == RejectAndGiveUp)
451           return false;
452 
453       break;
454     }
455     case GIM_CheckType: {
456       int64_t InsnID = MatchTable[CurrentIdx++];
457       int64_t OpIdx = MatchTable[CurrentIdx++];
458       int64_t TypeID = MatchTable[CurrentIdx++];
459       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460                       dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461                              << "]->getOperand(" << OpIdx
462                              << "), TypeID=" << TypeID << ")\n");
463       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465       if (!MO.isReg() ||
466           MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467         if (handleReject() == RejectAndGiveUp)
468           return false;
469       }
470       break;
471     }
472     case GIM_CheckPointerToAny: {
473       int64_t InsnID = MatchTable[CurrentIdx++];
474       int64_t OpIdx = MatchTable[CurrentIdx++];
475       int64_t SizeInBits = MatchTable[CurrentIdx++];
476 
477       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478                       dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479                              << InsnID << "]->getOperand(" << OpIdx
480                              << "), SizeInBits=" << SizeInBits << ")\n");
481       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482       // iPTR must be looked up in the target.
483       if (SizeInBits == 0) {
484         MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485         SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486       }
487 
488       assert(SizeInBits != 0 && "Pointer size must be known");
489 
490       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491       if (MO.isReg()) {
492         const LLT &Ty = MRI.getType(MO.getReg());
493         if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494           if (handleReject() == RejectAndGiveUp)
495             return false;
496       } else if (handleReject() == RejectAndGiveUp)
497         return false;
498 
499       break;
500     }
501     case GIM_CheckRegBankForClass: {
502       int64_t InsnID = MatchTable[CurrentIdx++];
503       int64_t OpIdx = MatchTable[CurrentIdx++];
504       int64_t RCEnum = MatchTable[CurrentIdx++];
505       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506                       dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507                              << InsnID << "]->getOperand(" << OpIdx
508                              << "), RCEnum=" << RCEnum << ")\n");
509       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511       if (!MO.isReg() ||
512           &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513               RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514         if (handleReject() == RejectAndGiveUp)
515           return false;
516       }
517       break;
518     }
519 
520     case GIM_CheckComplexPattern: {
521       int64_t InsnID = MatchTable[CurrentIdx++];
522       int64_t OpIdx = MatchTable[CurrentIdx++];
523       int64_t RendererID = MatchTable[CurrentIdx++];
524       int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526                       dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527                              << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528                              << "]->getOperand(" << OpIdx
529                              << "), ComplexPredicateID=" << ComplexPredicateID
530                              << ")\n");
531       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532       // FIXME: Use std::invoke() when it's available.
533       ComplexRendererFns Renderer =
534           (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535               State.MIs[InsnID]->getOperand(OpIdx));
536       if (Renderer.hasValue())
537         State.Renderers[RendererID] = Renderer.getValue();
538       else
539         if (handleReject() == RejectAndGiveUp)
540           return false;
541       break;
542     }
543 
544     case GIM_CheckConstantInt: {
545       int64_t InsnID = MatchTable[CurrentIdx++];
546       int64_t OpIdx = MatchTable[CurrentIdx++];
547       int64_t Value = MatchTable[CurrentIdx++];
548       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549                       dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550                              << InsnID << "]->getOperand(" << OpIdx
551                              << "), Value=" << Value << ")\n");
552       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554       if (MO.isReg()) {
555         // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556         LLT Ty = MRI.getType(MO.getReg());
557         Value = SignExtend64(Value, Ty.getSizeInBits());
558 
559         if (!isOperandImmEqual(MO, Value, MRI)) {
560           if (handleReject() == RejectAndGiveUp)
561             return false;
562         }
563       } else if (handleReject() == RejectAndGiveUp)
564         return false;
565 
566       break;
567     }
568 
569     case GIM_CheckLiteralInt: {
570       int64_t InsnID = MatchTable[CurrentIdx++];
571       int64_t OpIdx = MatchTable[CurrentIdx++];
572       int64_t Value = MatchTable[CurrentIdx++];
573       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574                       dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575                              << InsnID << "]->getOperand(" << OpIdx
576                              << "), Value=" << Value << ")\n");
577       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579       if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580         if (handleReject() == RejectAndGiveUp)
581           return false;
582       }
583       break;
584     }
585 
586     case GIM_CheckIntrinsicID: {
587       int64_t InsnID = MatchTable[CurrentIdx++];
588       int64_t OpIdx = MatchTable[CurrentIdx++];
589       int64_t Value = MatchTable[CurrentIdx++];
590       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591                       dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592                              << InsnID << "]->getOperand(" << OpIdx
593                              << "), Value=" << Value << ")\n");
594       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596       if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
597         if (handleReject() == RejectAndGiveUp)
598           return false;
599       break;
600     }
601 
602     case GIM_CheckIsMBB: {
603       int64_t InsnID = MatchTable[CurrentIdx++];
604       int64_t OpIdx = MatchTable[CurrentIdx++];
605       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606                       dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607                              << "]->getOperand(" << OpIdx << "))\n");
608       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609       if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610         if (handleReject() == RejectAndGiveUp)
611           return false;
612       }
613       break;
614     }
615 
616     case GIM_CheckIsSafeToFold: {
617       int64_t InsnID = MatchTable[CurrentIdx++];
618       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619                       dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620                              << InsnID << "])\n");
621       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622       if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623         if (handleReject() == RejectAndGiveUp)
624           return false;
625       }
626       break;
627     }
628     case GIM_CheckIsSameOperand: {
629       int64_t InsnID = MatchTable[CurrentIdx++];
630       int64_t OpIdx = MatchTable[CurrentIdx++];
631       int64_t OtherInsnID = MatchTable[CurrentIdx++];
632       int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634                       dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635                              << InsnID << "][" << OpIdx << "], MIs["
636                              << OtherInsnID << "][" << OtherOpIdx << "])\n");
637       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638       assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639       if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640               State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641         if (handleReject() == RejectAndGiveUp)
642           return false;
643       }
644       break;
645     }
646     case GIM_Reject:
647       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648                       dbgs() << CurrentIdx << ": GIM_Reject\n");
649       if (handleReject() == RejectAndGiveUp)
650         return false;
651       break;
652 
653     case GIR_MutateOpcode: {
654       int64_t OldInsnID = MatchTable[CurrentIdx++];
655       uint64_t NewInsnID = MatchTable[CurrentIdx++];
656       int64_t NewOpcode = MatchTable[CurrentIdx++];
657       if (NewInsnID >= OutMIs.size())
658         OutMIs.resize(NewInsnID + 1);
659 
660       OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661                                               State.MIs[OldInsnID]);
662       OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664                       dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665                              << NewInsnID << "], MIs[" << OldInsnID << "], "
666                              << NewOpcode << ")\n");
667       break;
668     }
669 
670     case GIR_BuildMI: {
671       uint64_t NewInsnID = MatchTable[CurrentIdx++];
672       int64_t Opcode = MatchTable[CurrentIdx++];
673       if (NewInsnID >= OutMIs.size())
674         OutMIs.resize(NewInsnID + 1);
675 
676       OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677                                   State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679                       dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680                              << NewInsnID << "], " << Opcode << ")\n");
681       break;
682     }
683 
684     case GIR_Copy: {
685       int64_t NewInsnID = MatchTable[CurrentIdx++];
686       int64_t OldInsnID = MatchTable[CurrentIdx++];
687       int64_t OpIdx = MatchTable[CurrentIdx++];
688       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689       OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691                       dbgs()
692                           << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693                           << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694       break;
695     }
696 
697     case GIR_CopyOrAddZeroReg: {
698       int64_t NewInsnID = MatchTable[CurrentIdx++];
699       int64_t OldInsnID = MatchTable[CurrentIdx++];
700       int64_t OpIdx = MatchTable[CurrentIdx++];
701       int64_t ZeroReg = MatchTable[CurrentIdx++];
702       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703       MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704       if (isOperandImmEqual(MO, 0, MRI))
705         OutMIs[NewInsnID].addReg(ZeroReg);
706       else
707         OutMIs[NewInsnID].add(MO);
708       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709                       dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710                              << NewInsnID << "], MIs[" << OldInsnID << "], "
711                              << OpIdx << ", " << ZeroReg << ")\n");
712       break;
713     }
714 
715     case GIR_CopySubReg: {
716       int64_t NewInsnID = MatchTable[CurrentIdx++];
717       int64_t OldInsnID = MatchTable[CurrentIdx++];
718       int64_t OpIdx = MatchTable[CurrentIdx++];
719       int64_t SubRegIdx = MatchTable[CurrentIdx++];
720       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721       OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722                                0, SubRegIdx);
723       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724                       dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725                              << NewInsnID << "], MIs[" << OldInsnID << "], "
726                              << OpIdx << ", " << SubRegIdx << ")\n");
727       break;
728     }
729 
730     case GIR_AddImplicitDef: {
731       int64_t InsnID = MatchTable[CurrentIdx++];
732       int64_t RegNum = MatchTable[CurrentIdx++];
733       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734       OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736                       dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737                              << InsnID << "], " << RegNum << ")\n");
738       break;
739     }
740 
741     case GIR_AddImplicitUse: {
742       int64_t InsnID = MatchTable[CurrentIdx++];
743       int64_t RegNum = MatchTable[CurrentIdx++];
744       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745       OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747                       dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748                              << InsnID << "], " << RegNum << ")\n");
749       break;
750     }
751 
752     case GIR_AddRegister: {
753       int64_t InsnID = MatchTable[CurrentIdx++];
754       int64_t RegNum = MatchTable[CurrentIdx++];
755       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756       OutMIs[InsnID].addReg(RegNum);
757       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758                       dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759                              << InsnID << "], " << RegNum << ")\n");
760       break;
761     }
762 
763     case GIR_AddTempRegister: {
764       int64_t InsnID = MatchTable[CurrentIdx++];
765       int64_t TempRegID = MatchTable[CurrentIdx++];
766       uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768       OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770                       dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771                              << InsnID << "], TempRegisters[" << TempRegID
772                              << "], " << TempRegFlags << ")\n");
773       break;
774     }
775 
776     case GIR_AddImm: {
777       int64_t InsnID = MatchTable[CurrentIdx++];
778       int64_t Imm = MatchTable[CurrentIdx++];
779       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780       OutMIs[InsnID].addImm(Imm);
781       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782                       dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783                              << "], " << Imm << ")\n");
784       break;
785     }
786 
787     case GIR_ComplexRenderer: {
788       int64_t InsnID = MatchTable[CurrentIdx++];
789       int64_t RendererID = MatchTable[CurrentIdx++];
790       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791       for (const auto &RenderOpFn : State.Renderers[RendererID])
792         RenderOpFn(OutMIs[InsnID]);
793       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794                       dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795                              << InsnID << "], " << RendererID << ")\n");
796       break;
797     }
798     case GIR_ComplexSubOperandRenderer: {
799       int64_t InsnID = MatchTable[CurrentIdx++];
800       int64_t RendererID = MatchTable[CurrentIdx++];
801       int64_t RenderOpID = MatchTable[CurrentIdx++];
802       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803       State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805                       dbgs() << CurrentIdx
806                              << ": GIR_ComplexSubOperandRenderer(OutMIs["
807                              << InsnID << "], " << RendererID << ", "
808                              << RenderOpID << ")\n");
809       break;
810     }
811 
812     case GIR_CopyConstantAsSImm: {
813       int64_t NewInsnID = MatchTable[CurrentIdx++];
814       int64_t OldInsnID = MatchTable[CurrentIdx++];
815       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817       if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818         OutMIs[NewInsnID].addImm(
819             State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820       } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
821         OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822       else
823         llvm_unreachable("Expected Imm or CImm operand");
824       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825                       dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827       break;
828     }
829 
830     // TODO: Needs a test case once we have a pattern that uses this.
831     case GIR_CopyFConstantAsFPImm: {
832       int64_t NewInsnID = MatchTable[CurrentIdx++];
833       int64_t OldInsnID = MatchTable[CurrentIdx++];
834       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836       if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837         OutMIs[NewInsnID].addFPImm(
838             State.MIs[OldInsnID]->getOperand(1).getFPImm());
839       else
840         llvm_unreachable("Expected FPImm operand");
841       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842                       dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844       break;
845     }
846 
847     case GIR_CustomRenderer: {
848       int64_t InsnID = MatchTable[CurrentIdx++];
849       int64_t OldInsnID = MatchTable[CurrentIdx++];
850       int64_t RendererFnID = MatchTable[CurrentIdx++];
851       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853                       dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854                              << InsnID << "], MIs[" << OldInsnID << "], "
855                              << RendererFnID << ")\n");
856       (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857                                                      *State.MIs[OldInsnID]);
858       break;
859     }
860     case GIR_ConstrainOperandRC: {
861       int64_t InsnID = MatchTable[CurrentIdx++];
862       int64_t OpIdx = MatchTable[CurrentIdx++];
863       int64_t RCEnum = MatchTable[CurrentIdx++];
864       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865       constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866                                     *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868                       dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869                              << InsnID << "], " << OpIdx << ", " << RCEnum
870                              << ")\n");
871       break;
872     }
873 
874     case GIR_ConstrainSelectedInstOperands: {
875       int64_t InsnID = MatchTable[CurrentIdx++];
876       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877       constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878                                        RBI);
879       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880                       dbgs() << CurrentIdx
881                              << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882                              << InsnID << "])\n");
883       break;
884     }
885 
886     case GIR_MergeMemOperands: {
887       int64_t InsnID = MatchTable[CurrentIdx++];
888       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889 
890       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891                       dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892                              << InsnID << "]");
893       int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894       while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895              GIU_MergeMemOperands_EndOfList) {
896         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897                         dbgs() << ", MIs[" << MergeInsnID << "]");
898         for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899           OutMIs[InsnID].addMemOperand(MMO);
900       }
901       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902       break;
903     }
904 
905     case GIR_EraseFromParent: {
906       int64_t InsnID = MatchTable[CurrentIdx++];
907       assert(State.MIs[InsnID] &&
908              "Attempted to erase an undefined instruction");
909       State.MIs[InsnID]->eraseFromParent();
910       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911                       dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912                              << InsnID << "])\n");
913       break;
914     }
915 
916     case GIR_MakeTempReg: {
917       int64_t TempRegID = MatchTable[CurrentIdx++];
918       int64_t TypeID = MatchTable[CurrentIdx++];
919 
920       State.TempRegisters[TempRegID] =
921           MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
922       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
923                       dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
924                              << "] = GIR_MakeTempReg(" << TypeID << ")\n");
925       break;
926     }
927 
928     case GIR_Coverage: {
929       int64_t RuleID = MatchTable[CurrentIdx++];
930       CoverageInfo.setCovered(RuleID);
931 
932       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
933                       dbgs()
934                           << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
935       break;
936     }
937 
938     case GIR_Done:
939       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
940                       dbgs() << CurrentIdx << ": GIR_Done\n");
941       return true;
942 
943     default:
944       llvm_unreachable("Unexpected command");
945     }
946   }
947 }
948 
949 } // end namespace llvm
950 
951 #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
952