1 //===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file implements GIMatchTableExecutor's `executeMatchTable`
10 /// function. This is implemented in a separate file because the function is
11 /// quite large.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
16 #define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
17
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
20 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
21 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22 #include "llvm/CodeGen/GlobalISel/Utils.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineOperand.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
26 #include "llvm/CodeGen/RegisterBankInfo.h"
27 #include "llvm/CodeGen/TargetInstrInfo.h"
28 #include "llvm/CodeGen/TargetOpcodes.h"
29 #include "llvm/CodeGen/TargetRegisterInfo.h"
30 #include "llvm/IR/Constants.h"
31 #include "llvm/IR/DataLayout.h"
32 #include "llvm/IR/Type.h"
33 #include "llvm/Support/CodeGenCoverage.h"
34 #include "llvm/Support/Debug.h"
35 #include "llvm/Support/ErrorHandling.h"
36 #include "llvm/Support/LEB128.h"
37 #include "llvm/Support/raw_ostream.h"
38 #include <cassert>
39 #include <cstddef>
40 #include <cstdint>
41
42 namespace llvm {
43
44 template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn,
45 class CustomRendererFn>
executeMatchTable(TgtExecutor & Exec,MatcherState & State,const ExecInfoTy<PredicateBitset,ComplexMatcherMemFn,CustomRendererFn> & ExecInfo,MachineIRBuilder & Builder,const uint8_t * MatchTable,const TargetInstrInfo & TII,MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,const RegisterBankInfo & RBI,const PredicateBitset & AvailableFeatures,CodeGenCoverage * CoverageInfo)46 bool GIMatchTableExecutor::executeMatchTable(
47 TgtExecutor &Exec, MatcherState &State,
48 const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
49 &ExecInfo,
50 MachineIRBuilder &Builder, const uint8_t *MatchTable,
51 const TargetInstrInfo &TII, MachineRegisterInfo &MRI,
52 const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI,
53 const PredicateBitset &AvailableFeatures,
54 CodeGenCoverage *CoverageInfo) const {
55
56 uint64_t CurrentIdx = 0;
57 SmallVector<uint64_t, 4> OnFailResumeAt;
58 NewMIVector OutMIs;
59
60 GISelChangeObserver *Observer = Builder.getObserver();
61 // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
62 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
63
64 const uint16_t Flags = State.MIs[0]->getFlags();
65
66 enum RejectAction { RejectAndGiveUp, RejectAndResume };
67 auto handleReject = [&]() -> RejectAction {
68 DEBUG_WITH_TYPE(TgtExecutor::getName(),
69 dbgs() << CurrentIdx << ": Rejected\n");
70 if (OnFailResumeAt.empty())
71 return RejectAndGiveUp;
72 CurrentIdx = OnFailResumeAt.pop_back_val();
73 DEBUG_WITH_TYPE(TgtExecutor::getName(),
74 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
75 << OnFailResumeAt.size() << " try-blocks remain)\n");
76 return RejectAndResume;
77 };
78
79 const auto propagateFlags = [&]() {
80 for (auto MIB : OutMIs) {
81 // Set the NoFPExcept flag when no original matched instruction could
82 // raise an FP exception, but the new instruction potentially might.
83 uint16_t MIBFlags = Flags;
84 if (NoFPException && MIB->mayRaiseFPException())
85 MIBFlags |= MachineInstr::NoFPExcept;
86 if (Observer)
87 Observer->changingInstr(*MIB);
88 MIB.setMIFlags(MIBFlags);
89 if (Observer)
90 Observer->changedInstr(*MIB);
91 }
92 };
93
94 // If the index is >= 0, it's an index in the type objects generated by
95 // TableGen. If the index is <0, it's an index in the recorded types object.
96 const auto getTypeFromIdx = [&](int64_t Idx) -> LLT {
97 if (Idx >= 0)
98 return ExecInfo.TypeObjects[Idx];
99 return State.RecordedTypes[1 - Idx];
100 };
101
102 const auto readULEB = [&]() {
103 return fastDecodeULEB128(MatchTable, CurrentIdx);
104 };
105
106 // Convenience function to return a signed value. This avoids
107 // us forgetting to first cast to int8_t before casting to a
108 // wider signed int type.
109 // if we casted uint8 directly to a wider type we'd lose
110 // negative values.
111 const auto readS8 = [&]() { return (int8_t)MatchTable[CurrentIdx++]; };
112
113 const auto readU16 = [&]() {
114 auto V = readBytesAs<uint16_t>(MatchTable + CurrentIdx);
115 CurrentIdx += 2;
116 return V;
117 };
118
119 const auto readU32 = [&]() {
120 auto V = readBytesAs<uint32_t>(MatchTable + CurrentIdx);
121 CurrentIdx += 4;
122 return V;
123 };
124
125 const auto readU64 = [&]() {
126 auto V = readBytesAs<uint64_t>(MatchTable + CurrentIdx);
127 CurrentIdx += 8;
128 return V;
129 };
130
131 const auto eraseImpl = [&](MachineInstr *MI) {
132 // If we're erasing the insertion point, ensure we don't leave a dangling
133 // pointer in the builder.
134 if (Builder.getInsertPt() == MI)
135 Builder.setInsertPt(*MI->getParent(), ++MI->getIterator());
136 if (Observer)
137 Observer->erasingInstr(*MI);
138 MI->eraseFromParent();
139 };
140
141 while (true) {
142 assert(CurrentIdx != ~0u && "Invalid MatchTable index");
143 uint8_t MatcherOpcode = MatchTable[CurrentIdx++];
144 switch (MatcherOpcode) {
145 case GIM_Try: {
146 DEBUG_WITH_TYPE(TgtExecutor::getName(),
147 dbgs() << CurrentIdx << ": Begin try-block\n");
148 OnFailResumeAt.push_back(readU32());
149 break;
150 }
151
152 case GIM_RecordInsn:
153 case GIM_RecordInsnIgnoreCopies: {
154 uint64_t NewInsnID = readULEB();
155 uint64_t InsnID = readULEB();
156 uint64_t OpIdx = readULEB();
157
158 // As an optimisation we require that MIs[0] is always the root. Refuse
159 // any attempt to modify it.
160 assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
161
162 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
163 if (!MO.isReg()) {
164 DEBUG_WITH_TYPE(TgtExecutor::getName(),
165 dbgs() << CurrentIdx << ": Not a register\n");
166 if (handleReject() == RejectAndGiveUp)
167 return false;
168 break;
169 }
170 if (MO.getReg().isPhysical()) {
171 DEBUG_WITH_TYPE(TgtExecutor::getName(),
172 dbgs() << CurrentIdx << ": Is a physical register\n");
173 if (handleReject() == RejectAndGiveUp)
174 return false;
175 break;
176 }
177
178 MachineInstr *NewMI;
179 if (MatcherOpcode == GIM_RecordInsnIgnoreCopies)
180 NewMI = getDefIgnoringCopies(MO.getReg(), MRI);
181 else
182 NewMI = MRI.getVRegDef(MO.getReg());
183
184 if ((size_t)NewInsnID < State.MIs.size())
185 State.MIs[NewInsnID] = NewMI;
186 else {
187 assert((size_t)NewInsnID == State.MIs.size() &&
188 "Expected to store MIs in order");
189 State.MIs.push_back(NewMI);
190 }
191 DEBUG_WITH_TYPE(TgtExecutor::getName(),
192 dbgs() << CurrentIdx << ": MIs[" << NewInsnID
193 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
194 << ")\n");
195 break;
196 }
197
198 case GIM_CheckFeatures: {
199 uint16_t ExpectedBitsetID = readU16();
200 DEBUG_WITH_TYPE(TgtExecutor::getName(),
201 dbgs() << CurrentIdx
202 << ": GIM_CheckFeatures(ExpectedBitsetID="
203 << ExpectedBitsetID << ")\n");
204 if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) !=
205 ExecInfo.FeatureBitsets[ExpectedBitsetID]) {
206 if (handleReject() == RejectAndGiveUp)
207 return false;
208 }
209 break;
210 }
211 case GIM_CheckOpcode:
212 case GIM_CheckOpcodeIsEither: {
213 uint64_t InsnID = readULEB();
214 uint16_t Expected0 = readU16();
215 uint16_t Expected1 = -1;
216 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
217 Expected1 = readU16();
218
219 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
220 unsigned Opcode = State.MIs[InsnID]->getOpcode();
221
222 DEBUG_WITH_TYPE(TgtExecutor::getName(),
223 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
224 << "], ExpectedOpcode=" << Expected0;
225 if (MatcherOpcode == GIM_CheckOpcodeIsEither) dbgs()
226 << " || " << Expected1;
227 dbgs() << ") // Got=" << Opcode << "\n";);
228
229 if (Opcode != Expected0 && Opcode != Expected1) {
230 if (handleReject() == RejectAndGiveUp)
231 return false;
232 }
233 break;
234 }
235 case GIM_SwitchOpcode: {
236 uint64_t InsnID = readULEB();
237 uint16_t LowerBound = readU16();
238 uint16_t UpperBound = readU16();
239 uint32_t Default = readU32();
240
241 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
242 const int64_t Opcode = State.MIs[InsnID]->getOpcode();
243
244 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
245 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
246 << LowerBound << ", " << UpperBound << "), Default=" << Default
247 << ", JumpTable...) // Got=" << Opcode << "\n";
248 });
249 if (Opcode < LowerBound || UpperBound <= Opcode) {
250 CurrentIdx = Default;
251 break;
252 }
253 const auto EntryIdx = (Opcode - LowerBound);
254 // Each entry is 4 bytes
255 CurrentIdx =
256 readBytesAs<uint32_t>(MatchTable + CurrentIdx + (EntryIdx * 4));
257 if (!CurrentIdx) {
258 CurrentIdx = Default;
259 break;
260 }
261 OnFailResumeAt.push_back(Default);
262 break;
263 }
264
265 case GIM_SwitchType: {
266 uint64_t InsnID = readULEB();
267 uint64_t OpIdx = readULEB();
268 uint16_t LowerBound = readU16();
269 uint16_t UpperBound = readU16();
270 int64_t Default = readU32();
271
272 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
274
275 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
276 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
277 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
278 << UpperBound << "), Default=" << Default
279 << ", JumpTable...) // Got=";
280 if (!MO.isReg())
281 dbgs() << "Not a VReg\n";
282 else
283 dbgs() << MRI.getType(MO.getReg()) << "\n";
284 });
285 if (!MO.isReg()) {
286 CurrentIdx = Default;
287 break;
288 }
289 const LLT Ty = MRI.getType(MO.getReg());
290 const auto TyI = ExecInfo.TypeIDMap.find(Ty);
291 if (TyI == ExecInfo.TypeIDMap.end()) {
292 CurrentIdx = Default;
293 break;
294 }
295 const int64_t TypeID = TyI->second;
296 if (TypeID < LowerBound || UpperBound <= TypeID) {
297 CurrentIdx = Default;
298 break;
299 }
300 const auto NumEntry = (TypeID - LowerBound);
301 // Each entry is 4 bytes
302 CurrentIdx =
303 readBytesAs<uint32_t>(MatchTable + CurrentIdx + (NumEntry * 4));
304 if (!CurrentIdx) {
305 CurrentIdx = Default;
306 break;
307 }
308 OnFailResumeAt.push_back(Default);
309 break;
310 }
311
312 case GIM_CheckNumOperandsGE:
313 case GIM_CheckNumOperandsLE: {
314 uint64_t InsnID = readULEB();
315 uint64_t Expected = readULEB();
316 const bool IsLE = (MatcherOpcode == GIM_CheckNumOperandsLE);
317 DEBUG_WITH_TYPE(TgtExecutor::getName(),
318 dbgs() << CurrentIdx << ": GIM_CheckNumOperands"
319 << (IsLE ? "LE" : "GE") << "(MIs[" << InsnID
320 << "], Expected=" << Expected << ")\n");
321 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
322 const unsigned NumOps = State.MIs[InsnID]->getNumOperands();
323 if (IsLE ? (NumOps > Expected) : (NumOps < Expected)) {
324 if (handleReject() == RejectAndGiveUp)
325 return false;
326 }
327 break;
328 }
329 case GIM_CheckNumOperands: {
330 uint64_t InsnID = readULEB();
331 uint64_t Expected = readULEB();
332 DEBUG_WITH_TYPE(TgtExecutor::getName(),
333 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
334 << InsnID << "], Expected=" << Expected << ")\n");
335 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
336 if (State.MIs[InsnID]->getNumOperands() != Expected) {
337 if (handleReject() == RejectAndGiveUp)
338 return false;
339 }
340 break;
341 }
342 case GIM_CheckI64ImmPredicate:
343 case GIM_CheckImmOperandPredicate: {
344 uint64_t InsnID = readULEB();
345 unsigned OpIdx =
346 MatcherOpcode == GIM_CheckImmOperandPredicate ? readULEB() : 1;
347 uint16_t Predicate = readU16();
348 DEBUG_WITH_TYPE(TgtExecutor::getName(),
349 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["
350 << InsnID << "]->getOperand(" << OpIdx
351 << "), Predicate=" << Predicate << ")\n");
352 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
353 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() ||
354 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) &&
355 "Expected immediate operand");
356 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
357 int64_t Value = 0;
358 if (State.MIs[InsnID]->getOperand(OpIdx).isCImm())
359 Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue();
360 else if (State.MIs[InsnID]->getOperand(OpIdx).isImm())
361 Value = State.MIs[InsnID]->getOperand(OpIdx).getImm();
362 else
363 llvm_unreachable("Expected Imm or CImm operand");
364
365 if (!testImmPredicate_I64(Predicate, Value))
366 if (handleReject() == RejectAndGiveUp)
367 return false;
368 break;
369 }
370 case GIM_CheckAPIntImmPredicate: {
371 uint64_t InsnID = readULEB();
372 uint16_t Predicate = readU16();
373 DEBUG_WITH_TYPE(TgtExecutor::getName(),
374 dbgs()
375 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
376 << InsnID << "], Predicate=" << Predicate << ")\n");
377 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
378 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
379 "Expected G_CONSTANT");
380 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
381 if (!State.MIs[InsnID]->getOperand(1).isCImm())
382 llvm_unreachable("Expected Imm or CImm operand");
383
384 const APInt &Value =
385 State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
386 if (!testImmPredicate_APInt(Predicate, Value))
387 if (handleReject() == RejectAndGiveUp)
388 return false;
389 break;
390 }
391 case GIM_CheckAPFloatImmPredicate: {
392 uint64_t InsnID = readULEB();
393 uint16_t Predicate = readU16();
394 DEBUG_WITH_TYPE(TgtExecutor::getName(),
395 dbgs()
396 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
397 << InsnID << "], Predicate=" << Predicate << ")\n");
398 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
399 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
400 "Expected G_FCONSTANT");
401 assert(State.MIs[InsnID]->getOperand(1).isFPImm() &&
402 "Expected FPImm operand");
403 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
404 const APFloat &Value =
405 State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
406
407 if (!testImmPredicate_APFloat(Predicate, Value))
408 if (handleReject() == RejectAndGiveUp)
409 return false;
410 break;
411 }
412 case GIM_CheckIsBuildVectorAllOnes:
413 case GIM_CheckIsBuildVectorAllZeros: {
414 uint64_t InsnID = readULEB();
415
416 DEBUG_WITH_TYPE(TgtExecutor::getName(),
417 dbgs() << CurrentIdx
418 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
419 << InsnID << "])\n");
420 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
422 const MachineInstr *MI = State.MIs[InsnID];
423 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
424 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
425 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
426
427 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
428 if (!isBuildVectorAllOnes(*MI, MRI)) {
429 if (handleReject() == RejectAndGiveUp)
430 return false;
431 }
432 } else {
433 if (!isBuildVectorAllZeros(*MI, MRI)) {
434 if (handleReject() == RejectAndGiveUp)
435 return false;
436 }
437 }
438
439 break;
440 }
441 case GIM_CheckSimplePredicate: {
442 // Note: we don't check for invalid here because this is purely a hook to
443 // allow some executors (such as the combiner) to check arbitrary,
444 // contextless predicates, such as whether a rule is enabled or not.
445 uint16_t Predicate = readU16();
446 DEBUG_WITH_TYPE(TgtExecutor::getName(),
447 dbgs() << CurrentIdx
448 << ": GIM_CheckSimplePredicate(Predicate="
449 << Predicate << ")\n");
450 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
451 if (!testSimplePredicate(Predicate)) {
452 if (handleReject() == RejectAndGiveUp)
453 return false;
454 }
455 break;
456 }
457 case GIM_CheckCxxInsnPredicate: {
458 uint64_t InsnID = readULEB();
459 uint16_t Predicate = readU16();
460 DEBUG_WITH_TYPE(TgtExecutor::getName(),
461 dbgs()
462 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
463 << InsnID << "], Predicate=" << Predicate << ")\n");
464 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
465 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
466
467 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State))
468 if (handleReject() == RejectAndGiveUp)
469 return false;
470 break;
471 }
472 case GIM_CheckHasNoUse: {
473 uint64_t InsnID = readULEB();
474
475 DEBUG_WITH_TYPE(TgtExecutor::getName(),
476 dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs["
477 << InsnID << "]\n");
478
479 const MachineInstr *MI = State.MIs[InsnID];
480 assert(MI && "Used insn before defined");
481 assert(MI->getNumDefs() > 0 && "No defs");
482 const Register Res = MI->getOperand(0).getReg();
483
484 if (!MRI.use_nodbg_empty(Res)) {
485 if (handleReject() == RejectAndGiveUp)
486 return false;
487 }
488 break;
489 }
490 case GIM_CheckHasOneUse: {
491 uint64_t InsnID = readULEB();
492
493 DEBUG_WITH_TYPE(TgtExecutor::getName(),
494 dbgs() << CurrentIdx << ": GIM_CheckHasOneUse(MIs["
495 << InsnID << "]\n");
496
497 const MachineInstr *MI = State.MIs[InsnID];
498 assert(MI && "Used insn before defined");
499 assert(MI->getNumDefs() > 0 && "No defs");
500 const Register Res = MI->getOperand(0).getReg();
501
502 if (!MRI.hasOneNonDBGUse(Res)) {
503 if (handleReject() == RejectAndGiveUp)
504 return false;
505 }
506 break;
507 }
508 case GIM_CheckAtomicOrdering: {
509 uint64_t InsnID = readULEB();
510 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
511 DEBUG_WITH_TYPE(TgtExecutor::getName(),
512 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
513 << InsnID << "], " << (uint64_t)Ordering << ")\n");
514 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
515 if (!State.MIs[InsnID]->hasOneMemOperand())
516 if (handleReject() == RejectAndGiveUp)
517 return false;
518
519 for (const auto &MMO : State.MIs[InsnID]->memoperands())
520 if (MMO->getMergedOrdering() != Ordering)
521 if (handleReject() == RejectAndGiveUp)
522 return false;
523 break;
524 }
525 case GIM_CheckAtomicOrderingOrStrongerThan: {
526 uint64_t InsnID = readULEB();
527 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
528 DEBUG_WITH_TYPE(TgtExecutor::getName(),
529 dbgs() << CurrentIdx
530 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
531 << InsnID << "], " << (uint64_t)Ordering << ")\n");
532 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
533 if (!State.MIs[InsnID]->hasOneMemOperand())
534 if (handleReject() == RejectAndGiveUp)
535 return false;
536
537 for (const auto &MMO : State.MIs[InsnID]->memoperands())
538 if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering))
539 if (handleReject() == RejectAndGiveUp)
540 return false;
541 break;
542 }
543 case GIM_CheckAtomicOrderingWeakerThan: {
544 uint64_t InsnID = readULEB();
545 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
546 DEBUG_WITH_TYPE(TgtExecutor::getName(),
547 dbgs() << CurrentIdx
548 << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
549 << InsnID << "], " << (uint64_t)Ordering << ")\n");
550 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
551 if (!State.MIs[InsnID]->hasOneMemOperand())
552 if (handleReject() == RejectAndGiveUp)
553 return false;
554
555 for (const auto &MMO : State.MIs[InsnID]->memoperands())
556 if (!isStrongerThan(Ordering, MMO->getMergedOrdering()))
557 if (handleReject() == RejectAndGiveUp)
558 return false;
559 break;
560 }
561 case GIM_CheckMemoryAddressSpace: {
562 uint64_t InsnID = readULEB();
563 uint64_t MMOIdx = readULEB();
564 // This accepts a list of possible address spaces.
565 const uint64_t NumAddrSpace = MatchTable[CurrentIdx++];
566
567 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
568 if (handleReject() == RejectAndGiveUp)
569 return false;
570 break;
571 }
572
573 // Need to still jump to the end of the list of address spaces if we find
574 // a match earlier.
575 const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
576
577 const MachineMemOperand *MMO =
578 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
579 const unsigned MMOAddrSpace = MMO->getAddrSpace();
580
581 bool Success = false;
582 for (unsigned I = 0; I != NumAddrSpace; ++I) {
583 uint64_t AddrSpace = readULEB();
584 DEBUG_WITH_TYPE(TgtExecutor::getName(),
585 dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
586 << AddrSpace << '\n');
587
588 if (AddrSpace == MMOAddrSpace) {
589 Success = true;
590 break;
591 }
592 }
593
594 CurrentIdx = LastIdx;
595 if (!Success && handleReject() == RejectAndGiveUp)
596 return false;
597 break;
598 }
599 case GIM_CheckMemoryAlignment: {
600 uint64_t InsnID = readULEB();
601 uint64_t MMOIdx = readULEB();
602 uint64_t MinAlign = MatchTable[CurrentIdx++];
603
604 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
605
606 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
607 if (handleReject() == RejectAndGiveUp)
608 return false;
609 break;
610 }
611
612 MachineMemOperand *MMO =
613 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
614 DEBUG_WITH_TYPE(TgtExecutor::getName(),
615 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
616 << "(MIs[" << InsnID << "]->memoperands() + "
617 << MMOIdx << ")->getAlignment() >= " << MinAlign
618 << ")\n");
619 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
620 return false;
621
622 break;
623 }
624 case GIM_CheckMemorySizeEqualTo: {
625 uint64_t InsnID = readULEB();
626 uint64_t MMOIdx = readULEB();
627 uint32_t Size = readU32();
628
629 DEBUG_WITH_TYPE(TgtExecutor::getName(),
630 dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs["
631 << InsnID << "]->memoperands() + " << MMOIdx
632 << ", Size=" << Size << ")\n");
633 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
634
635 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
636 if (handleReject() == RejectAndGiveUp)
637 return false;
638 break;
639 }
640
641 MachineMemOperand *MMO =
642 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
643
644 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize()
645 << " bytes vs " << Size
646 << " bytes\n");
647 if (MMO->getSize() != Size)
648 if (handleReject() == RejectAndGiveUp)
649 return false;
650
651 break;
652 }
653 case GIM_CheckMemorySizeEqualToLLT:
654 case GIM_CheckMemorySizeLessThanLLT:
655 case GIM_CheckMemorySizeGreaterThanLLT: {
656 uint64_t InsnID = readULEB();
657 uint64_t MMOIdx = readULEB();
658 uint64_t OpIdx = readULEB();
659
660 DEBUG_WITH_TYPE(
661 TgtExecutor::getName(),
662 dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
663 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo"
664 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
665 ? "GreaterThan"
666 : "LessThan")
667 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
668 << ", OpIdx=" << OpIdx << ")\n");
669 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
670
671 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
672 if (!MO.isReg()) {
673 DEBUG_WITH_TYPE(TgtExecutor::getName(),
674 dbgs() << CurrentIdx << ": Not a register\n");
675 if (handleReject() == RejectAndGiveUp)
676 return false;
677 break;
678 }
679
680 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
681 if (handleReject() == RejectAndGiveUp)
682 return false;
683 break;
684 }
685
686 MachineMemOperand *MMO =
687 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
688
689 const TypeSize Size = MRI.getType(MO.getReg()).getSizeInBits();
690 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
691 MMO->getSizeInBits() != Size) {
692 if (handleReject() == RejectAndGiveUp)
693 return false;
694 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
695 TypeSize::isKnownGE(MMO->getSizeInBits().getValue(), Size)) {
696 if (handleReject() == RejectAndGiveUp)
697 return false;
698 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
699 TypeSize::isKnownLE(MMO->getSizeInBits().getValue(), Size))
700 if (handleReject() == RejectAndGiveUp)
701 return false;
702
703 break;
704 }
705 case GIM_RootCheckType:
706 case GIM_CheckType: {
707 uint64_t InsnID = (MatcherOpcode == GIM_RootCheckType) ? 0 : readULEB();
708 uint64_t OpIdx = readULEB();
709 int TypeID = readS8();
710 DEBUG_WITH_TYPE(TgtExecutor::getName(),
711 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
712 << "]->getOperand(" << OpIdx
713 << "), TypeID=" << TypeID << ")\n");
714 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
715 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
716 if (!MO.isReg() || MRI.getType(MO.getReg()) != getTypeFromIdx(TypeID)) {
717 if (handleReject() == RejectAndGiveUp)
718 return false;
719 }
720 break;
721 }
722 case GIM_CheckPointerToAny: {
723 uint64_t InsnID = readULEB();
724 uint64_t OpIdx = readULEB();
725 uint64_t SizeInBits = readULEB();
726
727 DEBUG_WITH_TYPE(TgtExecutor::getName(),
728 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
729 << InsnID << "]->getOperand(" << OpIdx
730 << "), SizeInBits=" << SizeInBits << ")\n");
731 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
732 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
733 const LLT Ty = MRI.getType(MO.getReg());
734
735 // iPTR must be looked up in the target.
736 if (SizeInBits == 0) {
737 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
738 const unsigned AddrSpace = Ty.getAddressSpace();
739 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
740 }
741
742 assert(SizeInBits != 0 && "Pointer size must be known");
743
744 if (MO.isReg()) {
745 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
746 if (handleReject() == RejectAndGiveUp)
747 return false;
748 } else if (handleReject() == RejectAndGiveUp)
749 return false;
750
751 break;
752 }
753 case GIM_RecordNamedOperand: {
754 uint64_t InsnID = readULEB();
755 uint64_t OpIdx = readULEB();
756 uint64_t StoreIdx = readULEB();
757
758 DEBUG_WITH_TYPE(TgtExecutor::getName(),
759 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
760 << InsnID << "]->getOperand(" << OpIdx
761 << "), StoreIdx=" << StoreIdx << ")\n");
762 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
763 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
764 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx);
765 break;
766 }
767 case GIM_RecordRegType: {
768 uint64_t InsnID = readULEB();
769 uint64_t OpIdx = readULEB();
770 int TypeIdx = readS8();
771
772 DEBUG_WITH_TYPE(TgtExecutor::getName(),
773 dbgs() << CurrentIdx << ": GIM_RecordRegType(MIs["
774 << InsnID << "]->getOperand(" << OpIdx
775 << "), TypeIdx=" << TypeIdx << ")\n");
776 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
777 assert(TypeIdx < 0 && "Temp types always have negative indexes!");
778 // Indexes start at -1.
779 TypeIdx = 1 - TypeIdx;
780 const auto &Op = State.MIs[InsnID]->getOperand(OpIdx);
781 if (State.RecordedTypes.size() <= (uint64_t)TypeIdx)
782 State.RecordedTypes.resize(TypeIdx + 1, LLT());
783 State.RecordedTypes[TypeIdx] = MRI.getType(Op.getReg());
784 break;
785 }
786
787 case GIM_RootCheckRegBankForClass:
788 case GIM_CheckRegBankForClass: {
789 uint64_t InsnID =
790 (MatcherOpcode == GIM_RootCheckRegBankForClass) ? 0 : readULEB();
791 uint64_t OpIdx = readULEB();
792 uint16_t RCEnum = readU16();
793 DEBUG_WITH_TYPE(TgtExecutor::getName(),
794 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
795 << InsnID << "]->getOperand(" << OpIdx
796 << "), RCEnum=" << RCEnum << ")\n");
797 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
798 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
799 if (!MO.isReg() ||
800 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum),
801 MRI.getType(MO.getReg())) !=
802 RBI.getRegBank(MO.getReg(), MRI, TRI)) {
803 if (handleReject() == RejectAndGiveUp)
804 return false;
805 }
806 break;
807 }
808
809 case GIM_CheckComplexPattern: {
810 uint64_t InsnID = readULEB();
811 uint64_t OpIdx = readULEB();
812 uint16_t RendererID = readU16();
813 uint16_t ComplexPredicateID = readU16();
814 DEBUG_WITH_TYPE(TgtExecutor::getName(),
815 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
816 << "] = GIM_CheckComplexPattern(MIs[" << InsnID
817 << "]->getOperand(" << OpIdx
818 << "), ComplexPredicateID=" << ComplexPredicateID
819 << ")\n");
820 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
821 // FIXME: Use std::invoke() when it's available.
822 ComplexRendererFns Renderer =
823 (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])(
824 State.MIs[InsnID]->getOperand(OpIdx));
825 if (Renderer)
826 State.Renderers[RendererID] = *Renderer;
827 else if (handleReject() == RejectAndGiveUp)
828 return false;
829 break;
830 }
831
832 case GIM_CheckConstantInt:
833 case GIM_CheckConstantInt8: {
834 const bool IsInt8 = (MatcherOpcode == GIM_CheckConstantInt8);
835
836 uint64_t InsnID = readULEB();
837 uint64_t OpIdx = readULEB();
838 uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64();
839 DEBUG_WITH_TYPE(TgtExecutor::getName(),
840 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
841 << InsnID << "]->getOperand(" << OpIdx
842 << "), Value=" << Value << ")\n");
843 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
844 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
845 if (MO.isReg()) {
846 // isOperandImmEqual() will sign-extend to 64-bits, so should we.
847 LLT Ty = MRI.getType(MO.getReg());
848 // If the type is > 64 bits, it can't be a constant int, so we bail
849 // early because SignExtend64 will assert otherwise.
850 if (Ty.getScalarSizeInBits() > 64) {
851 if (handleReject() == RejectAndGiveUp)
852 return false;
853 break;
854 }
855
856 Value = SignExtend64(Value, Ty.getScalarSizeInBits());
857 if (!isOperandImmEqual(MO, Value, MRI, /*Splat=*/true)) {
858 if (handleReject() == RejectAndGiveUp)
859 return false;
860 }
861 } else if (handleReject() == RejectAndGiveUp)
862 return false;
863
864 break;
865 }
866
867 case GIM_CheckLiteralInt: {
868 uint64_t InsnID = readULEB();
869 uint64_t OpIdx = readULEB();
870 int64_t Value = readU64();
871 DEBUG_WITH_TYPE(TgtExecutor::getName(),
872 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
873 << InsnID << "]->getOperand(" << OpIdx
874 << "), Value=" << Value << ")\n");
875 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
876 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
877 if (MO.isImm() && MO.getImm() == Value)
878 break;
879
880 if (MO.isCImm() && MO.getCImm()->equalsInt(Value))
881 break;
882
883 if (handleReject() == RejectAndGiveUp)
884 return false;
885
886 break;
887 }
888
889 case GIM_CheckIntrinsicID: {
890 uint64_t InsnID = readULEB();
891 uint64_t OpIdx = readULEB();
892 uint16_t Value = readU16();
893 DEBUG_WITH_TYPE(TgtExecutor::getName(),
894 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
895 << InsnID << "]->getOperand(" << OpIdx
896 << "), Value=" << Value << ")\n");
897 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
898 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
899 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
900 if (handleReject() == RejectAndGiveUp)
901 return false;
902 break;
903 }
904 case GIM_CheckCmpPredicate: {
905 uint64_t InsnID = readULEB();
906 uint64_t OpIdx = readULEB();
907 uint16_t Value = readU16();
908 DEBUG_WITH_TYPE(TgtExecutor::getName(),
909 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
910 << InsnID << "]->getOperand(" << OpIdx
911 << "), Value=" << Value << ")\n");
912 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
913 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
914 if (!MO.isPredicate() || MO.getPredicate() != Value)
915 if (handleReject() == RejectAndGiveUp)
916 return false;
917 break;
918 }
919 case GIM_CheckIsMBB: {
920 uint64_t InsnID = readULEB();
921 uint64_t OpIdx = readULEB();
922 DEBUG_WITH_TYPE(TgtExecutor::getName(),
923 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
924 << "]->getOperand(" << OpIdx << "))\n");
925 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
926 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
927 if (handleReject() == RejectAndGiveUp)
928 return false;
929 }
930 break;
931 }
932 case GIM_CheckIsImm: {
933 uint64_t InsnID = readULEB();
934 uint64_t OpIdx = readULEB();
935 DEBUG_WITH_TYPE(TgtExecutor::getName(),
936 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
937 << "]->getOperand(" << OpIdx << "))\n");
938 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
939 if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
940 if (handleReject() == RejectAndGiveUp)
941 return false;
942 }
943 break;
944 }
945 case GIM_CheckIsSafeToFold: {
946 uint64_t NumInsn = MatchTable[CurrentIdx++];
947 DEBUG_WITH_TYPE(TgtExecutor::getName(),
948 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(N = "
949 << NumInsn << ")\n");
950 MachineInstr &Root = *State.MIs[0];
951 for (unsigned K = 1, E = NumInsn + 1; K < E; ++K) {
952 if (!isObviouslySafeToFold(*State.MIs[K], Root)) {
953 if (handleReject() == RejectAndGiveUp)
954 return false;
955 }
956 }
957 break;
958 }
959 case GIM_CheckIsSameOperand:
960 case GIM_CheckIsSameOperandIgnoreCopies: {
961 uint64_t InsnID = readULEB();
962 uint64_t OpIdx = readULEB();
963 uint64_t OtherInsnID = readULEB();
964 uint64_t OtherOpIdx = readULEB();
965 DEBUG_WITH_TYPE(TgtExecutor::getName(),
966 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
967 << InsnID << "][" << OpIdx << "], MIs["
968 << OtherInsnID << "][" << OtherOpIdx << "])\n");
969 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
970 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
971
972 MachineOperand &Op = State.MIs[InsnID]->getOperand(OpIdx);
973 MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx);
974
975 if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) {
976 if (Op.isReg() && OtherOp.isReg()) {
977 if (getSrcRegIgnoringCopies(Op.getReg(), MRI) ==
978 getSrcRegIgnoringCopies(OtherOp.getReg(), MRI))
979 break;
980 }
981 }
982
983 if (!Op.isIdenticalTo(OtherOp)) {
984 if (handleReject() == RejectAndGiveUp)
985 return false;
986 }
987 break;
988 }
989 case GIM_CheckCanReplaceReg: {
990 uint64_t OldInsnID = readULEB();
991 uint64_t OldOpIdx = readULEB();
992 uint64_t NewInsnID = readULEB();
993 uint64_t NewOpIdx = readULEB();
994
995 DEBUG_WITH_TYPE(TgtExecutor::getName(),
996 dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs["
997 << OldInsnID << "][" << OldOpIdx << "] = MIs["
998 << NewInsnID << "][" << NewOpIdx << "])\n");
999
1000 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1001 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg();
1002 if (!canReplaceReg(Old, New, MRI)) {
1003 if (handleReject() == RejectAndGiveUp)
1004 return false;
1005 }
1006 break;
1007 }
1008 case GIM_MIFlags: {
1009 uint64_t InsnID = readULEB();
1010 uint32_t Flags = readU32();
1011
1012 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1013 dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID
1014 << "], " << Flags << ")\n");
1015 if ((State.MIs[InsnID]->getFlags() & Flags) != Flags) {
1016 if (handleReject() == RejectAndGiveUp)
1017 return false;
1018 }
1019 break;
1020 }
1021 case GIM_MIFlagsNot: {
1022 uint64_t InsnID = readULEB();
1023 uint32_t Flags = readU32();
1024
1025 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1026 dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID
1027 << "], " << Flags << ")\n");
1028 if ((State.MIs[InsnID]->getFlags() & Flags)) {
1029 if (handleReject() == RejectAndGiveUp)
1030 return false;
1031 }
1032 break;
1033 }
1034 case GIM_Reject:
1035 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1036 dbgs() << CurrentIdx << ": GIM_Reject\n");
1037 if (handleReject() == RejectAndGiveUp)
1038 return false;
1039 break;
1040 case GIR_MutateOpcode: {
1041 uint64_t OldInsnID = readULEB();
1042 uint64_t NewInsnID = readULEB();
1043 uint16_t NewOpcode = readU16();
1044 if (NewInsnID >= OutMIs.size())
1045 OutMIs.resize(NewInsnID + 1);
1046
1047 MachineInstr *OldMI = State.MIs[OldInsnID];
1048 if (Observer)
1049 Observer->changingInstr(*OldMI);
1050 OutMIs[NewInsnID] = MachineInstrBuilder(*OldMI->getMF(), OldMI);
1051 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
1052 if (Observer)
1053 Observer->changedInstr(*OldMI);
1054 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1055 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
1056 << NewInsnID << "], MIs[" << OldInsnID << "], "
1057 << NewOpcode << ")\n");
1058 break;
1059 }
1060
1061 case GIR_BuildRootMI:
1062 case GIR_BuildMI: {
1063 uint64_t NewInsnID = (MatcherOpcode == GIR_BuildRootMI) ? 0 : readULEB();
1064 uint16_t Opcode = readU16();
1065 if (NewInsnID >= OutMIs.size())
1066 OutMIs.resize(NewInsnID + 1);
1067
1068 OutMIs[NewInsnID] = Builder.buildInstr(Opcode);
1069 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1070 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
1071 << NewInsnID << "], " << Opcode << ")\n");
1072 break;
1073 }
1074
1075 case GIR_BuildConstant: {
1076 uint64_t TempRegID = readULEB();
1077 uint64_t Imm = readU64();
1078 Builder.buildConstant(State.TempRegisters[TempRegID], Imm);
1079 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1080 dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg["
1081 << TempRegID << "], Imm=" << Imm << ")\n");
1082 break;
1083 }
1084
1085 case GIR_RootToRootCopy:
1086 case GIR_Copy: {
1087 uint64_t NewInsnID =
1088 (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB();
1089 uint64_t OldInsnID =
1090 (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB();
1091 uint64_t OpIdx = readULEB();
1092 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1093 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
1094 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1095 dbgs()
1096 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
1097 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
1098 break;
1099 }
1100
1101 case GIR_CopyRemaining: {
1102 uint64_t NewInsnID = readULEB();
1103 uint64_t OldInsnID = readULEB();
1104 uint64_t OpIdx = readULEB();
1105 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1106 MachineInstr &OldMI = *State.MIs[OldInsnID];
1107 MachineInstrBuilder &NewMI = OutMIs[NewInsnID];
1108 for (const auto &Op : drop_begin(OldMI.operands(), OpIdx))
1109 NewMI.add(Op);
1110 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1111 dbgs() << CurrentIdx << ": GIR_CopyRemaining(OutMIs["
1112 << NewInsnID << "], MIs[" << OldInsnID
1113 << "], /*start=*/" << OpIdx << ")\n");
1114 break;
1115 }
1116
1117 case GIR_CopyOrAddZeroReg: {
1118 uint64_t NewInsnID = readULEB();
1119 uint64_t OldInsnID = readULEB();
1120 uint64_t OpIdx = readULEB();
1121 uint16_t ZeroReg = readU16();
1122 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1123 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
1124 if (isOperandImmEqual(MO, 0, MRI))
1125 OutMIs[NewInsnID].addReg(ZeroReg);
1126 else
1127 OutMIs[NewInsnID].add(MO);
1128 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1129 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
1130 << NewInsnID << "], MIs[" << OldInsnID << "], "
1131 << OpIdx << ", " << ZeroReg << ")\n");
1132 break;
1133 }
1134
1135 case GIR_CopySubReg: {
1136 uint64_t NewInsnID = readULEB();
1137 uint64_t OldInsnID = readULEB();
1138 uint64_t OpIdx = readULEB();
1139 uint16_t SubRegIdx = readU16();
1140 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1141 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
1142 0, SubRegIdx);
1143 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1144 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
1145 << NewInsnID << "], MIs[" << OldInsnID << "], "
1146 << OpIdx << ", " << SubRegIdx << ")\n");
1147 break;
1148 }
1149
1150 case GIR_AddImplicitDef: {
1151 uint64_t InsnID = readULEB();
1152 uint16_t RegNum = readU16();
1153 uint16_t Flags = readU16();
1154 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1155 Flags |= RegState::Implicit;
1156 OutMIs[InsnID].addDef(RegNum, Flags);
1157 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1158 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
1159 << InsnID << "], " << RegNum << ")\n");
1160 break;
1161 }
1162
1163 case GIR_AddImplicitUse: {
1164 uint64_t InsnID = readULEB();
1165 uint16_t RegNum = readU16();
1166 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1167 OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
1168 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1169 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
1170 << InsnID << "], " << RegNum << ")\n");
1171 break;
1172 }
1173
1174 case GIR_AddRegister: {
1175 uint64_t InsnID = readULEB();
1176 uint16_t RegNum = readU16();
1177 uint16_t RegFlags = readU16();
1178 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1179 OutMIs[InsnID].addReg(RegNum, RegFlags);
1180 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1181 dbgs()
1182 << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID
1183 << "], " << RegNum << ", " << RegFlags << ")\n");
1184 break;
1185 }
1186 case GIR_AddIntrinsicID: {
1187 uint64_t InsnID = readULEB();
1188 uint16_t Value = readU16();
1189 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1190 OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value);
1191 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1192 dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs["
1193 << InsnID << "], " << Value << ")\n");
1194 break;
1195 }
1196 case GIR_SetImplicitDefDead: {
1197 uint64_t InsnID = readULEB();
1198 uint64_t OpIdx = readULEB();
1199 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1200 dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs["
1201 << InsnID << "], OpIdx=" << OpIdx << ")\n");
1202 MachineInstr *MI = OutMIs[InsnID];
1203 assert(MI && "Modifying undefined instruction");
1204 MI->getOperand(MI->getNumExplicitOperands() + OpIdx).setIsDead();
1205 break;
1206 }
1207 case GIR_SetMIFlags: {
1208 uint64_t InsnID = readULEB();
1209 uint32_t Flags = readU32();
1210
1211 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1212 dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs["
1213 << InsnID << "], " << Flags << ")\n");
1214 MachineInstr *MI = OutMIs[InsnID];
1215 MI->setFlags(MI->getFlags() | Flags);
1216 break;
1217 }
1218 case GIR_UnsetMIFlags: {
1219 uint64_t InsnID = readULEB();
1220 uint32_t Flags = readU32();
1221
1222 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1223 dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs["
1224 << InsnID << "], " << Flags << ")\n");
1225 MachineInstr *MI = OutMIs[InsnID];
1226 MI->setFlags(MI->getFlags() & ~Flags);
1227 break;
1228 }
1229 case GIR_CopyMIFlags: {
1230 uint64_t InsnID = readULEB();
1231 uint64_t OldInsnID = readULEB();
1232
1233 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1234 dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs["
1235 << InsnID << "], MIs[" << OldInsnID << "])\n");
1236 MachineInstr *MI = OutMIs[InsnID];
1237 MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags());
1238 break;
1239 }
1240 case GIR_AddSimpleTempRegister:
1241 case GIR_AddTempRegister:
1242 case GIR_AddTempSubRegister: {
1243 uint64_t InsnID = readULEB();
1244 uint64_t TempRegID = readULEB();
1245 uint16_t TempRegFlags = 0;
1246 if (MatcherOpcode != GIR_AddSimpleTempRegister)
1247 TempRegFlags = readU16();
1248 uint16_t SubReg = 0;
1249 if (MatcherOpcode == GIR_AddTempSubRegister)
1250 SubReg = readU16();
1251
1252 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1253
1254 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags,
1255 SubReg);
1256 DEBUG_WITH_TYPE(
1257 TgtExecutor::getName(),
1258 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID
1259 << "], TempRegisters[" << TempRegID << "]";
1260 if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
1261 dbgs() << ", " << TempRegFlags << ")\n");
1262 break;
1263 }
1264
1265 case GIR_AddImm8:
1266 case GIR_AddImm: {
1267 const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8);
1268 uint64_t InsnID = readULEB();
1269 uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64();
1270 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1271 OutMIs[InsnID].addImm(Imm);
1272 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1273 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
1274 << "], " << Imm << ")\n");
1275 break;
1276 }
1277
1278 case GIR_AddCImm: {
1279 uint64_t InsnID = readULEB();
1280 int TypeID = readS8();
1281 uint64_t Imm = readU64();
1282 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1283
1284 unsigned Width = ExecInfo.TypeObjects[TypeID].getScalarSizeInBits();
1285 LLVMContext &Ctx = MF->getFunction().getContext();
1286 OutMIs[InsnID].addCImm(
1287 ConstantInt::get(IntegerType::get(Ctx, Width), Imm, /*signed*/ true));
1288 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1289 dbgs() << CurrentIdx << ": GIR_AddCImm(OutMIs[" << InsnID
1290 << "], TypeID=" << TypeID << ", Imm=" << Imm
1291 << ")\n");
1292 break;
1293 }
1294
1295 case GIR_ComplexRenderer: {
1296 uint64_t InsnID = readULEB();
1297 uint16_t RendererID = readU16();
1298 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1299 for (const auto &RenderOpFn : State.Renderers[RendererID])
1300 RenderOpFn(OutMIs[InsnID]);
1301 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1302 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
1303 << InsnID << "], " << RendererID << ")\n");
1304 break;
1305 }
1306 case GIR_ComplexSubOperandRenderer: {
1307 uint64_t InsnID = readULEB();
1308 uint16_t RendererID = readU16();
1309 uint64_t RenderOpID = readULEB();
1310 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1311 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
1312 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1313 dbgs() << CurrentIdx
1314 << ": GIR_ComplexSubOperandRenderer(OutMIs["
1315 << InsnID << "], " << RendererID << ", "
1316 << RenderOpID << ")\n");
1317 break;
1318 }
1319 case GIR_ComplexSubOperandSubRegRenderer: {
1320 uint64_t InsnID = readULEB();
1321 uint16_t RendererID = readU16();
1322 uint64_t RenderOpID = readULEB();
1323 uint16_t SubRegIdx = readU16();
1324 MachineInstrBuilder &MI = OutMIs[InsnID];
1325 assert(MI && "Attempted to add to undefined instruction");
1326 State.Renderers[RendererID][RenderOpID](MI);
1327 MI->getOperand(MI->getNumOperands() - 1).setSubReg(SubRegIdx);
1328 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1329 dbgs() << CurrentIdx
1330 << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs["
1331 << InsnID << "], " << RendererID << ", "
1332 << RenderOpID << ", " << SubRegIdx << ")\n");
1333 break;
1334 }
1335
1336 case GIR_CopyConstantAsSImm: {
1337 uint64_t NewInsnID = readULEB();
1338 uint64_t OldInsnID = readULEB();
1339 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1340 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
1341 "Expected G_CONSTANT");
1342 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
1343 OutMIs[NewInsnID].addImm(
1344 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
1345 } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
1346 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
1347 else
1348 llvm_unreachable("Expected Imm or CImm operand");
1349 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1350 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
1351 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1352 break;
1353 }
1354
1355 // TODO: Needs a test case once we have a pattern that uses this.
1356 case GIR_CopyFConstantAsFPImm: {
1357 uint64_t NewInsnID = readULEB();
1358 uint64_t OldInsnID = readULEB();
1359 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1360 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
1361 "Expected G_FCONSTANT");
1362 if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
1363 OutMIs[NewInsnID].addFPImm(
1364 State.MIs[OldInsnID]->getOperand(1).getFPImm());
1365 else
1366 llvm_unreachable("Expected FPImm operand");
1367 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1368 dbgs()
1369 << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
1370 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1371 break;
1372 }
1373
1374 case GIR_CustomRenderer: {
1375 uint64_t InsnID = readULEB();
1376 uint64_t OldInsnID = readULEB();
1377 uint16_t RendererFnID = readU16();
1378 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1379 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1380 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
1381 << InsnID << "], MIs[" << OldInsnID << "], "
1382 << RendererFnID << ")\n");
1383 (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1384 OutMIs[InsnID], *State.MIs[OldInsnID],
1385 -1); // Not a source operand of the old instruction.
1386 break;
1387 }
1388 case GIR_DoneWithCustomAction: {
1389 uint16_t FnID = readU16();
1390 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1391 dbgs() << CurrentIdx << ": GIR_DoneWithCustomAction(FnID="
1392 << FnID << ")\n");
1393 assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID");
1394 if (runCustomAction(FnID, State, OutMIs)) {
1395 propagateFlags();
1396 return true;
1397 }
1398
1399 if (handleReject() == RejectAndGiveUp)
1400 return false;
1401 break;
1402 }
1403 case GIR_CustomOperandRenderer: {
1404 uint64_t InsnID = readULEB();
1405 uint64_t OldInsnID = readULEB();
1406 uint64_t OpIdx = readULEB();
1407 uint16_t RendererFnID = readU16();
1408 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1409
1410 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1411 dbgs() << CurrentIdx
1412 << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID
1413 << "], MIs[" << OldInsnID << "]->getOperand("
1414 << OpIdx << "), " << RendererFnID << ")\n");
1415 (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1416 OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx);
1417 break;
1418 }
1419 case GIR_ConstrainOperandRC: {
1420 uint64_t InsnID = readULEB();
1421 uint64_t OpIdx = readULEB();
1422 uint16_t RCEnum = readU16();
1423 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1424 MachineInstr &I = *OutMIs[InsnID].getInstr();
1425 MachineFunction &MF = *I.getParent()->getParent();
1426 MachineRegisterInfo &MRI = MF.getRegInfo();
1427 const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum);
1428 MachineOperand &MO = I.getOperand(OpIdx);
1429 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO);
1430 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1431 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1432 << InsnID << "], " << OpIdx << ", " << RCEnum
1433 << ")\n");
1434 break;
1435 }
1436
1437 case GIR_RootConstrainSelectedInstOperands:
1438 case GIR_ConstrainSelectedInstOperands: {
1439 uint64_t InsnID = (MatcherOpcode == GIR_RootConstrainSelectedInstOperands)
1440 ? 0
1441 : readULEB();
1442 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1443 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
1444 RBI);
1445 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1446 dbgs() << CurrentIdx
1447 << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1448 << InsnID << "])\n");
1449 break;
1450 }
1451 case GIR_MergeMemOperands: {
1452 uint64_t InsnID = readULEB();
1453 uint64_t NumInsn = MatchTable[CurrentIdx++];
1454 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1455
1456 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1457 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1458 << InsnID << "]");
1459 for (unsigned K = 0; K < NumInsn; ++K) {
1460 uint64_t NextID = readULEB();
1461 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1462 dbgs() << ", MIs[" << NextID << "]");
1463 for (const auto &MMO : State.MIs[NextID]->memoperands())
1464 OutMIs[InsnID].addMemOperand(MMO);
1465 }
1466 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n");
1467 break;
1468 }
1469 case GIR_EraseFromParent: {
1470 uint64_t InsnID = readULEB();
1471 MachineInstr *MI = State.MIs[InsnID];
1472 assert(MI && "Attempted to erase an undefined instruction");
1473 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1474 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1475 << InsnID << "])\n");
1476 eraseImpl(MI);
1477 break;
1478 }
1479 case GIR_EraseRootFromParent_Done: {
1480 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1481 dbgs()
1482 << CurrentIdx << ": GIR_EraseRootFromParent_Done\n");
1483 eraseImpl(State.MIs[0]);
1484 propagateFlags();
1485 return true;
1486 }
1487 case GIR_MakeTempReg: {
1488 uint64_t TempRegID = readULEB();
1489 int TypeID = readS8();
1490
1491 State.TempRegisters[TempRegID] =
1492 MRI.createGenericVirtualRegister(getTypeFromIdx(TypeID));
1493 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1494 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1495 << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1496 break;
1497 }
1498 case GIR_ReplaceReg: {
1499 uint64_t OldInsnID = readULEB();
1500 uint64_t OldOpIdx = readULEB();
1501 uint64_t NewInsnID = readULEB();
1502 uint64_t NewOpIdx = readULEB();
1503
1504 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1505 dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs["
1506 << OldInsnID << "][" << OldOpIdx << "] = MIs["
1507 << NewInsnID << "][" << NewOpIdx << "])\n");
1508
1509 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1510 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg();
1511 if (Observer)
1512 Observer->changingAllUsesOfReg(MRI, Old);
1513 MRI.replaceRegWith(Old, New);
1514 if (Observer)
1515 Observer->finishedChangingAllUsesOfReg();
1516 break;
1517 }
1518 case GIR_ReplaceRegWithTempReg: {
1519 uint64_t OldInsnID = readULEB();
1520 uint64_t OldOpIdx = readULEB();
1521 uint64_t TempRegID = readULEB();
1522
1523 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1524 dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs["
1525 << OldInsnID << "][" << OldOpIdx << "] = TempRegs["
1526 << TempRegID << "])\n");
1527
1528 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1529 Register New = State.TempRegisters[TempRegID];
1530 if (Observer)
1531 Observer->changingAllUsesOfReg(MRI, Old);
1532 MRI.replaceRegWith(Old, New);
1533 if (Observer)
1534 Observer->finishedChangingAllUsesOfReg();
1535 break;
1536 }
1537 case GIR_Coverage: {
1538 uint32_t RuleID = readU32();
1539 assert(CoverageInfo);
1540 CoverageInfo->setCovered(RuleID);
1541
1542 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx
1543 << ": GIR_Coverage("
1544 << RuleID << ")");
1545 break;
1546 }
1547
1548 case GIR_Done:
1549 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1550 dbgs() << CurrentIdx << ": GIR_Done\n");
1551 propagateFlags();
1552 return true;
1553 default:
1554 llvm_unreachable("Unexpected command");
1555 }
1556 }
1557 }
1558
1559 } // end namespace llvm
1560
1561 #endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
1562