1 //===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file implements GIMatchTableExecutor's `executeMatchTable`
10 /// function. This is implemented in a separate file because the function is
11 /// quite large.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
16 #define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
17
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
20 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
21 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22 #include "llvm/CodeGen/GlobalISel/Utils.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineOperand.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
26 #include "llvm/CodeGen/RegisterBankInfo.h"
27 #include "llvm/CodeGen/TargetInstrInfo.h"
28 #include "llvm/CodeGen/TargetOpcodes.h"
29 #include "llvm/CodeGen/TargetRegisterInfo.h"
30 #include "llvm/IR/Constants.h"
31 #include "llvm/IR/DataLayout.h"
32 #include "llvm/IR/Type.h"
33 #include "llvm/Support/CodeGenCoverage.h"
34 #include "llvm/Support/Debug.h"
35 #include "llvm/Support/ErrorHandling.h"
36 #include "llvm/Support/LEB128.h"
37 #include "llvm/Support/raw_ostream.h"
38 #include <cassert>
39 #include <cstddef>
40 #include <cstdint>
41
42 namespace llvm {
43
44 template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn,
45 class CustomRendererFn>
executeMatchTable(TgtExecutor & Exec,MatcherState & State,const ExecInfoTy<PredicateBitset,ComplexMatcherMemFn,CustomRendererFn> & ExecInfo,MachineIRBuilder & Builder,const uint8_t * MatchTable,const TargetInstrInfo & TII,MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,const RegisterBankInfo & RBI,const PredicateBitset & AvailableFeatures,CodeGenCoverage * CoverageInfo)46 bool GIMatchTableExecutor::executeMatchTable(
47 TgtExecutor &Exec, MatcherState &State,
48 const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
49 &ExecInfo,
50 MachineIRBuilder &Builder, const uint8_t *MatchTable,
51 const TargetInstrInfo &TII, MachineRegisterInfo &MRI,
52 const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI,
53 const PredicateBitset &AvailableFeatures,
54 CodeGenCoverage *CoverageInfo) const {
55
56 uint64_t CurrentIdx = 0;
57 SmallVector<uint64_t, 4> OnFailResumeAt;
58 NewMIVector OutMIs;
59
60 GISelChangeObserver *Observer = Builder.getObserver();
61 // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
62 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
63
64 const uint16_t Flags = State.MIs[0]->getFlags();
65
66 enum RejectAction { RejectAndGiveUp, RejectAndResume };
67 auto handleReject = [&]() -> RejectAction {
68 DEBUG_WITH_TYPE(TgtExecutor::getName(),
69 dbgs() << CurrentIdx << ": Rejected\n");
70 if (OnFailResumeAt.empty())
71 return RejectAndGiveUp;
72 CurrentIdx = OnFailResumeAt.pop_back_val();
73 DEBUG_WITH_TYPE(TgtExecutor::getName(),
74 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
75 << OnFailResumeAt.size() << " try-blocks remain)\n");
76 return RejectAndResume;
77 };
78
79 const auto propagateFlags = [&]() {
80 for (auto MIB : OutMIs) {
81 // Set the NoFPExcept flag when no original matched instruction could
82 // raise an FP exception, but the new instruction potentially might.
83 uint16_t MIBFlags = Flags;
84 if (NoFPException && MIB->mayRaiseFPException())
85 MIBFlags |= MachineInstr::NoFPExcept;
86 if (Observer)
87 Observer->changingInstr(*MIB);
88 MIB.setMIFlags(MIBFlags);
89 if (Observer)
90 Observer->changedInstr(*MIB);
91 }
92 };
93
94 // If the index is >= 0, it's an index in the type objects generated by
95 // TableGen. If the index is <0, it's an index in the recorded types object.
96 const auto getTypeFromIdx = [&](int64_t Idx) -> LLT {
97 if (Idx >= 0)
98 return ExecInfo.TypeObjects[Idx];
99 return State.RecordedTypes[1 - Idx];
100 };
101
102 const auto readULEB = [&]() {
103 return fastDecodeULEB128(MatchTable, CurrentIdx);
104 };
105
106 // Convenience function to return a signed value. This avoids
107 // us forgetting to first cast to int8_t before casting to a
108 // wider signed int type.
109 // if we casted uint8 directly to a wider type we'd lose
110 // negative values.
111 const auto readS8 = [&]() { return (int8_t)MatchTable[CurrentIdx++]; };
112
113 const auto readU16 = [&]() {
114 auto V = readBytesAs<uint16_t>(MatchTable + CurrentIdx);
115 CurrentIdx += 2;
116 return V;
117 };
118
119 const auto readU32 = [&]() {
120 auto V = readBytesAs<uint32_t>(MatchTable + CurrentIdx);
121 CurrentIdx += 4;
122 return V;
123 };
124
125 const auto readU64 = [&]() {
126 auto V = readBytesAs<uint64_t>(MatchTable + CurrentIdx);
127 CurrentIdx += 8;
128 return V;
129 };
130
131 const auto eraseImpl = [&](MachineInstr *MI) {
132 // If we're erasing the insertion point, ensure we don't leave a dangling
133 // pointer in the builder.
134 if (Builder.getInsertPt() == MI)
135 Builder.setInsertPt(*MI->getParent(), ++MI->getIterator());
136 if (Observer)
137 Observer->erasingInstr(*MI);
138 MI->eraseFromParent();
139 };
140
141 while (true) {
142 assert(CurrentIdx != ~0u && "Invalid MatchTable index");
143 uint8_t MatcherOpcode = MatchTable[CurrentIdx++];
144 switch (MatcherOpcode) {
145 case GIM_Try: {
146 DEBUG_WITH_TYPE(TgtExecutor::getName(),
147 dbgs() << CurrentIdx << ": Begin try-block\n");
148 OnFailResumeAt.push_back(readU32());
149 break;
150 }
151
152 case GIM_RecordInsn:
153 case GIM_RecordInsnIgnoreCopies: {
154 uint64_t NewInsnID = readULEB();
155 uint64_t InsnID = readULEB();
156 uint64_t OpIdx = readULEB();
157
158 // As an optimisation we require that MIs[0] is always the root. Refuse
159 // any attempt to modify it.
160 assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
161
162 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
163 if (!MO.isReg()) {
164 DEBUG_WITH_TYPE(TgtExecutor::getName(),
165 dbgs() << CurrentIdx << ": Not a register\n");
166 if (handleReject() == RejectAndGiveUp)
167 return false;
168 break;
169 }
170 if (MO.getReg().isPhysical()) {
171 DEBUG_WITH_TYPE(TgtExecutor::getName(),
172 dbgs() << CurrentIdx << ": Is a physical register\n");
173 if (handleReject() == RejectAndGiveUp)
174 return false;
175 break;
176 }
177
178 MachineInstr *NewMI;
179 if (MatcherOpcode == GIM_RecordInsnIgnoreCopies)
180 NewMI = getDefIgnoringCopies(MO.getReg(), MRI);
181 else
182 NewMI = MRI.getVRegDef(MO.getReg());
183
184 if ((size_t)NewInsnID < State.MIs.size())
185 State.MIs[NewInsnID] = NewMI;
186 else {
187 assert((size_t)NewInsnID == State.MIs.size() &&
188 "Expected to store MIs in order");
189 State.MIs.push_back(NewMI);
190 }
191 DEBUG_WITH_TYPE(TgtExecutor::getName(),
192 dbgs() << CurrentIdx << ": MIs[" << NewInsnID
193 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
194 << ")\n");
195 break;
196 }
197
198 case GIM_CheckFeatures: {
199 uint16_t ExpectedBitsetID = readU16();
200 DEBUG_WITH_TYPE(TgtExecutor::getName(),
201 dbgs() << CurrentIdx
202 << ": GIM_CheckFeatures(ExpectedBitsetID="
203 << ExpectedBitsetID << ")\n");
204 if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) !=
205 ExecInfo.FeatureBitsets[ExpectedBitsetID]) {
206 if (handleReject() == RejectAndGiveUp)
207 return false;
208 }
209 break;
210 }
211 case GIM_CheckOpcode:
212 case GIM_CheckOpcodeIsEither: {
213 uint64_t InsnID = readULEB();
214 uint16_t Expected0 = readU16();
215 uint16_t Expected1 = -1;
216 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
217 Expected1 = readU16();
218
219 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
220 unsigned Opcode = State.MIs[InsnID]->getOpcode();
221
222 DEBUG_WITH_TYPE(TgtExecutor::getName(),
223 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
224 << "], ExpectedOpcode=" << Expected0;
225 if (MatcherOpcode == GIM_CheckOpcodeIsEither) dbgs()
226 << " || " << Expected1;
227 dbgs() << ") // Got=" << Opcode << "\n";);
228
229 if (Opcode != Expected0 && Opcode != Expected1) {
230 if (handleReject() == RejectAndGiveUp)
231 return false;
232 }
233 break;
234 }
235 case GIM_SwitchOpcode: {
236 uint64_t InsnID = readULEB();
237 uint16_t LowerBound = readU16();
238 uint16_t UpperBound = readU16();
239 uint32_t Default = readU32();
240
241 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
242 const int64_t Opcode = State.MIs[InsnID]->getOpcode();
243
244 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
245 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
246 << LowerBound << ", " << UpperBound << "), Default=" << Default
247 << ", JumpTable...) // Got=" << Opcode << "\n";
248 });
249 if (Opcode < LowerBound || UpperBound <= Opcode) {
250 CurrentIdx = Default;
251 break;
252 }
253 const auto EntryIdx = (Opcode - LowerBound);
254 // Each entry is 4 bytes
255 CurrentIdx =
256 readBytesAs<uint32_t>(MatchTable + CurrentIdx + (EntryIdx * 4));
257 if (!CurrentIdx) {
258 CurrentIdx = Default;
259 break;
260 }
261 OnFailResumeAt.push_back(Default);
262 break;
263 }
264
265 case GIM_SwitchType: {
266 uint64_t InsnID = readULEB();
267 uint64_t OpIdx = readULEB();
268 uint16_t LowerBound = readU16();
269 uint16_t UpperBound = readU16();
270 int64_t Default = readU32();
271
272 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
274
275 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
276 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
277 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
278 << UpperBound << "), Default=" << Default
279 << ", JumpTable...) // Got=";
280 if (!MO.isReg())
281 dbgs() << "Not a VReg\n";
282 else
283 dbgs() << MRI.getType(MO.getReg()) << "\n";
284 });
285 if (!MO.isReg()) {
286 CurrentIdx = Default;
287 break;
288 }
289 const LLT Ty = MRI.getType(MO.getReg());
290 const auto TyI = ExecInfo.TypeIDMap.find(Ty);
291 if (TyI == ExecInfo.TypeIDMap.end()) {
292 CurrentIdx = Default;
293 break;
294 }
295 const int64_t TypeID = TyI->second;
296 if (TypeID < LowerBound || UpperBound <= TypeID) {
297 CurrentIdx = Default;
298 break;
299 }
300 const auto NumEntry = (TypeID - LowerBound);
301 // Each entry is 4 bytes
302 CurrentIdx =
303 readBytesAs<uint32_t>(MatchTable + CurrentIdx + (NumEntry * 4));
304 if (!CurrentIdx) {
305 CurrentIdx = Default;
306 break;
307 }
308 OnFailResumeAt.push_back(Default);
309 break;
310 }
311
312 case GIM_CheckNumOperands: {
313 uint64_t InsnID = readULEB();
314 uint64_t Expected = readULEB();
315 DEBUG_WITH_TYPE(TgtExecutor::getName(),
316 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
317 << InsnID << "], Expected=" << Expected << ")\n");
318 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
319 if (State.MIs[InsnID]->getNumOperands() != Expected) {
320 if (handleReject() == RejectAndGiveUp)
321 return false;
322 }
323 break;
324 }
325 case GIM_CheckI64ImmPredicate:
326 case GIM_CheckImmOperandPredicate: {
327 uint64_t InsnID = readULEB();
328 unsigned OpIdx =
329 MatcherOpcode == GIM_CheckImmOperandPredicate ? readULEB() : 1;
330 uint16_t Predicate = readU16();
331 DEBUG_WITH_TYPE(TgtExecutor::getName(),
332 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["
333 << InsnID << "]->getOperand(" << OpIdx
334 << "), Predicate=" << Predicate << ")\n");
335 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
336 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() ||
337 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) &&
338 "Expected immediate operand");
339 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
340 int64_t Value = 0;
341 if (State.MIs[InsnID]->getOperand(OpIdx).isCImm())
342 Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue();
343 else if (State.MIs[InsnID]->getOperand(OpIdx).isImm())
344 Value = State.MIs[InsnID]->getOperand(OpIdx).getImm();
345 else
346 llvm_unreachable("Expected Imm or CImm operand");
347
348 if (!testImmPredicate_I64(Predicate, Value))
349 if (handleReject() == RejectAndGiveUp)
350 return false;
351 break;
352 }
353 case GIM_CheckAPIntImmPredicate: {
354 uint64_t InsnID = readULEB();
355 uint16_t Predicate = readU16();
356 DEBUG_WITH_TYPE(TgtExecutor::getName(),
357 dbgs()
358 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
359 << InsnID << "], Predicate=" << Predicate << ")\n");
360 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
361 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
362 "Expected G_CONSTANT");
363 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
364 if (!State.MIs[InsnID]->getOperand(1).isCImm())
365 llvm_unreachable("Expected Imm or CImm operand");
366
367 const APInt &Value =
368 State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
369 if (!testImmPredicate_APInt(Predicate, Value))
370 if (handleReject() == RejectAndGiveUp)
371 return false;
372 break;
373 }
374 case GIM_CheckAPFloatImmPredicate: {
375 uint64_t InsnID = readULEB();
376 uint16_t Predicate = readU16();
377 DEBUG_WITH_TYPE(TgtExecutor::getName(),
378 dbgs()
379 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
380 << InsnID << "], Predicate=" << Predicate << ")\n");
381 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
382 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
383 "Expected G_FCONSTANT");
384 assert(State.MIs[InsnID]->getOperand(1).isFPImm() &&
385 "Expected FPImm operand");
386 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
387 const APFloat &Value =
388 State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
389
390 if (!testImmPredicate_APFloat(Predicate, Value))
391 if (handleReject() == RejectAndGiveUp)
392 return false;
393 break;
394 }
395 case GIM_CheckIsBuildVectorAllOnes:
396 case GIM_CheckIsBuildVectorAllZeros: {
397 uint64_t InsnID = readULEB();
398
399 DEBUG_WITH_TYPE(TgtExecutor::getName(),
400 dbgs() << CurrentIdx
401 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
402 << InsnID << "])\n");
403 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
404
405 const MachineInstr *MI = State.MIs[InsnID];
406 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
407 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
408 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
409
410 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
411 if (!isBuildVectorAllOnes(*MI, MRI)) {
412 if (handleReject() == RejectAndGiveUp)
413 return false;
414 }
415 } else {
416 if (!isBuildVectorAllZeros(*MI, MRI)) {
417 if (handleReject() == RejectAndGiveUp)
418 return false;
419 }
420 }
421
422 break;
423 }
424 case GIM_CheckSimplePredicate: {
425 // Note: we don't check for invalid here because this is purely a hook to
426 // allow some executors (such as the combiner) to check arbitrary,
427 // contextless predicates, such as whether a rule is enabled or not.
428 uint16_t Predicate = readU16();
429 DEBUG_WITH_TYPE(TgtExecutor::getName(),
430 dbgs() << CurrentIdx
431 << ": GIM_CheckSimplePredicate(Predicate="
432 << Predicate << ")\n");
433 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
434 if (!testSimplePredicate(Predicate)) {
435 if (handleReject() == RejectAndGiveUp)
436 return false;
437 }
438 break;
439 }
440 case GIM_CheckCxxInsnPredicate: {
441 uint64_t InsnID = readULEB();
442 uint16_t Predicate = readU16();
443 DEBUG_WITH_TYPE(TgtExecutor::getName(),
444 dbgs()
445 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
446 << InsnID << "], Predicate=" << Predicate << ")\n");
447 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
448 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
449
450 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State))
451 if (handleReject() == RejectAndGiveUp)
452 return false;
453 break;
454 }
455 case GIM_CheckHasNoUse: {
456 uint64_t InsnID = readULEB();
457
458 DEBUG_WITH_TYPE(TgtExecutor::getName(),
459 dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs["
460 << InsnID << "]\n");
461
462 const MachineInstr *MI = State.MIs[InsnID];
463 assert(MI && "Used insn before defined");
464 assert(MI->getNumDefs() > 0 && "No defs");
465 const Register Res = MI->getOperand(0).getReg();
466
467 if (!MRI.use_nodbg_empty(Res)) {
468 if (handleReject() == RejectAndGiveUp)
469 return false;
470 }
471
472 break;
473 }
474 case GIM_CheckAtomicOrdering: {
475 uint64_t InsnID = readULEB();
476 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
477 DEBUG_WITH_TYPE(TgtExecutor::getName(),
478 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
479 << InsnID << "], " << (uint64_t)Ordering << ")\n");
480 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
481 if (!State.MIs[InsnID]->hasOneMemOperand())
482 if (handleReject() == RejectAndGiveUp)
483 return false;
484
485 for (const auto &MMO : State.MIs[InsnID]->memoperands())
486 if (MMO->getMergedOrdering() != Ordering)
487 if (handleReject() == RejectAndGiveUp)
488 return false;
489 break;
490 }
491 case GIM_CheckAtomicOrderingOrStrongerThan: {
492 uint64_t InsnID = readULEB();
493 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
494 DEBUG_WITH_TYPE(TgtExecutor::getName(),
495 dbgs() << CurrentIdx
496 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
497 << InsnID << "], " << (uint64_t)Ordering << ")\n");
498 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
499 if (!State.MIs[InsnID]->hasOneMemOperand())
500 if (handleReject() == RejectAndGiveUp)
501 return false;
502
503 for (const auto &MMO : State.MIs[InsnID]->memoperands())
504 if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering))
505 if (handleReject() == RejectAndGiveUp)
506 return false;
507 break;
508 }
509 case GIM_CheckAtomicOrderingWeakerThan: {
510 uint64_t InsnID = readULEB();
511 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
512 DEBUG_WITH_TYPE(TgtExecutor::getName(),
513 dbgs() << CurrentIdx
514 << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
515 << InsnID << "], " << (uint64_t)Ordering << ")\n");
516 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
517 if (!State.MIs[InsnID]->hasOneMemOperand())
518 if (handleReject() == RejectAndGiveUp)
519 return false;
520
521 for (const auto &MMO : State.MIs[InsnID]->memoperands())
522 if (!isStrongerThan(Ordering, MMO->getMergedOrdering()))
523 if (handleReject() == RejectAndGiveUp)
524 return false;
525 break;
526 }
527 case GIM_CheckMemoryAddressSpace: {
528 uint64_t InsnID = readULEB();
529 uint64_t MMOIdx = readULEB();
530 // This accepts a list of possible address spaces.
531 const uint64_t NumAddrSpace = MatchTable[CurrentIdx++];
532
533 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
534 if (handleReject() == RejectAndGiveUp)
535 return false;
536 break;
537 }
538
539 // Need to still jump to the end of the list of address spaces if we find
540 // a match earlier.
541 const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
542
543 const MachineMemOperand *MMO =
544 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
545 const unsigned MMOAddrSpace = MMO->getAddrSpace();
546
547 bool Success = false;
548 for (unsigned I = 0; I != NumAddrSpace; ++I) {
549 uint64_t AddrSpace = readULEB();
550 DEBUG_WITH_TYPE(TgtExecutor::getName(),
551 dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
552 << AddrSpace << '\n');
553
554 if (AddrSpace == MMOAddrSpace) {
555 Success = true;
556 break;
557 }
558 }
559
560 CurrentIdx = LastIdx;
561 if (!Success && handleReject() == RejectAndGiveUp)
562 return false;
563 break;
564 }
565 case GIM_CheckMemoryAlignment: {
566 uint64_t InsnID = readULEB();
567 uint64_t MMOIdx = readULEB();
568 uint64_t MinAlign = MatchTable[CurrentIdx++];
569
570 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
571
572 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
573 if (handleReject() == RejectAndGiveUp)
574 return false;
575 break;
576 }
577
578 MachineMemOperand *MMO =
579 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
580 DEBUG_WITH_TYPE(TgtExecutor::getName(),
581 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
582 << "(MIs[" << InsnID << "]->memoperands() + "
583 << MMOIdx << ")->getAlignment() >= " << MinAlign
584 << ")\n");
585 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
586 return false;
587
588 break;
589 }
590 case GIM_CheckMemorySizeEqualTo: {
591 uint64_t InsnID = readULEB();
592 uint64_t MMOIdx = readULEB();
593 uint32_t Size = readU32();
594
595 DEBUG_WITH_TYPE(TgtExecutor::getName(),
596 dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs["
597 << InsnID << "]->memoperands() + " << MMOIdx
598 << ", Size=" << Size << ")\n");
599 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
600
601 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
602 if (handleReject() == RejectAndGiveUp)
603 return false;
604 break;
605 }
606
607 MachineMemOperand *MMO =
608 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
609
610 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize()
611 << " bytes vs " << Size
612 << " bytes\n");
613 if (MMO->getSize() != Size)
614 if (handleReject() == RejectAndGiveUp)
615 return false;
616
617 break;
618 }
619 case GIM_CheckMemorySizeEqualToLLT:
620 case GIM_CheckMemorySizeLessThanLLT:
621 case GIM_CheckMemorySizeGreaterThanLLT: {
622 uint64_t InsnID = readULEB();
623 uint64_t MMOIdx = readULEB();
624 uint64_t OpIdx = readULEB();
625
626 DEBUG_WITH_TYPE(
627 TgtExecutor::getName(),
628 dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
629 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo"
630 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
631 ? "GreaterThan"
632 : "LessThan")
633 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
634 << ", OpIdx=" << OpIdx << ")\n");
635 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
636
637 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
638 if (!MO.isReg()) {
639 DEBUG_WITH_TYPE(TgtExecutor::getName(),
640 dbgs() << CurrentIdx << ": Not a register\n");
641 if (handleReject() == RejectAndGiveUp)
642 return false;
643 break;
644 }
645
646 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
647 if (handleReject() == RejectAndGiveUp)
648 return false;
649 break;
650 }
651
652 MachineMemOperand *MMO =
653 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
654
655 unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
656 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
657 MMO->getSizeInBits().getValue() != Size) {
658 if (handleReject() == RejectAndGiveUp)
659 return false;
660 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
661 MMO->getSizeInBits().getValue() >= Size) {
662 if (handleReject() == RejectAndGiveUp)
663 return false;
664 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
665 MMO->getSizeInBits().getValue() <= Size)
666 if (handleReject() == RejectAndGiveUp)
667 return false;
668
669 break;
670 }
671 case GIM_RootCheckType:
672 case GIM_CheckType: {
673 uint64_t InsnID = (MatcherOpcode == GIM_RootCheckType) ? 0 : readULEB();
674 uint64_t OpIdx = readULEB();
675 int TypeID = readS8();
676 DEBUG_WITH_TYPE(TgtExecutor::getName(),
677 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
678 << "]->getOperand(" << OpIdx
679 << "), TypeID=" << TypeID << ")\n");
680 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
681 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
682 if (!MO.isReg() || MRI.getType(MO.getReg()) != getTypeFromIdx(TypeID)) {
683 if (handleReject() == RejectAndGiveUp)
684 return false;
685 }
686 break;
687 }
688 case GIM_CheckPointerToAny: {
689 uint64_t InsnID = readULEB();
690 uint64_t OpIdx = readULEB();
691 uint64_t SizeInBits = readULEB();
692
693 DEBUG_WITH_TYPE(TgtExecutor::getName(),
694 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
695 << InsnID << "]->getOperand(" << OpIdx
696 << "), SizeInBits=" << SizeInBits << ")\n");
697 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
698 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
699 const LLT Ty = MRI.getType(MO.getReg());
700
701 // iPTR must be looked up in the target.
702 if (SizeInBits == 0) {
703 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
704 const unsigned AddrSpace = Ty.getAddressSpace();
705 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
706 }
707
708 assert(SizeInBits != 0 && "Pointer size must be known");
709
710 if (MO.isReg()) {
711 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
712 if (handleReject() == RejectAndGiveUp)
713 return false;
714 } else if (handleReject() == RejectAndGiveUp)
715 return false;
716
717 break;
718 }
719 case GIM_RecordNamedOperand: {
720 uint64_t InsnID = readULEB();
721 uint64_t OpIdx = readULEB();
722 uint64_t StoreIdx = readULEB();
723
724 DEBUG_WITH_TYPE(TgtExecutor::getName(),
725 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
726 << InsnID << "]->getOperand(" << OpIdx
727 << "), StoreIdx=" << StoreIdx << ")\n");
728 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
729 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
730 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx);
731 break;
732 }
733 case GIM_RecordRegType: {
734 uint64_t InsnID = readULEB();
735 uint64_t OpIdx = readULEB();
736 int TypeIdx = readS8();
737
738 DEBUG_WITH_TYPE(TgtExecutor::getName(),
739 dbgs() << CurrentIdx << ": GIM_RecordRegType(MIs["
740 << InsnID << "]->getOperand(" << OpIdx
741 << "), TypeIdx=" << TypeIdx << ")\n");
742 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
743 assert(TypeIdx < 0 && "Temp types always have negative indexes!");
744 // Indexes start at -1.
745 TypeIdx = 1 - TypeIdx;
746 const auto &Op = State.MIs[InsnID]->getOperand(OpIdx);
747 if (State.RecordedTypes.size() <= (uint64_t)TypeIdx)
748 State.RecordedTypes.resize(TypeIdx + 1, LLT());
749 State.RecordedTypes[TypeIdx] = MRI.getType(Op.getReg());
750 break;
751 }
752
753 case GIM_RootCheckRegBankForClass:
754 case GIM_CheckRegBankForClass: {
755 uint64_t InsnID =
756 (MatcherOpcode == GIM_RootCheckRegBankForClass) ? 0 : readULEB();
757 uint64_t OpIdx = readULEB();
758 uint16_t RCEnum = readU16();
759 DEBUG_WITH_TYPE(TgtExecutor::getName(),
760 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
761 << InsnID << "]->getOperand(" << OpIdx
762 << "), RCEnum=" << RCEnum << ")\n");
763 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
764 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
765 if (!MO.isReg() ||
766 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum),
767 MRI.getType(MO.getReg())) !=
768 RBI.getRegBank(MO.getReg(), MRI, TRI)) {
769 if (handleReject() == RejectAndGiveUp)
770 return false;
771 }
772 break;
773 }
774
775 case GIM_CheckComplexPattern: {
776 uint64_t InsnID = readULEB();
777 uint64_t OpIdx = readULEB();
778 uint16_t RendererID = readU16();
779 uint16_t ComplexPredicateID = readU16();
780 DEBUG_WITH_TYPE(TgtExecutor::getName(),
781 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
782 << "] = GIM_CheckComplexPattern(MIs[" << InsnID
783 << "]->getOperand(" << OpIdx
784 << "), ComplexPredicateID=" << ComplexPredicateID
785 << ")\n");
786 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
787 // FIXME: Use std::invoke() when it's available.
788 ComplexRendererFns Renderer =
789 (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])(
790 State.MIs[InsnID]->getOperand(OpIdx));
791 if (Renderer)
792 State.Renderers[RendererID] = *Renderer;
793 else if (handleReject() == RejectAndGiveUp)
794 return false;
795 break;
796 }
797
798 case GIM_CheckConstantInt:
799 case GIM_CheckConstantInt8: {
800 const bool IsInt8 = (MatcherOpcode == GIM_CheckConstantInt8);
801
802 uint64_t InsnID = readULEB();
803 uint64_t OpIdx = readULEB();
804 uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64();
805 DEBUG_WITH_TYPE(TgtExecutor::getName(),
806 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
807 << InsnID << "]->getOperand(" << OpIdx
808 << "), Value=" << Value << ")\n");
809 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
810 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
811 if (MO.isReg()) {
812 // isOperandImmEqual() will sign-extend to 64-bits, so should we.
813 LLT Ty = MRI.getType(MO.getReg());
814 // If the type is > 64 bits, it can't be a constant int, so we bail
815 // early because SignExtend64 will assert otherwise.
816 if (Ty.getScalarSizeInBits() > 64) {
817 if (handleReject() == RejectAndGiveUp)
818 return false;
819 break;
820 }
821
822 Value = SignExtend64(Value, Ty.getScalarSizeInBits());
823 if (!isOperandImmEqual(MO, Value, MRI, /*Splat=*/true)) {
824 if (handleReject() == RejectAndGiveUp)
825 return false;
826 }
827 } else if (handleReject() == RejectAndGiveUp)
828 return false;
829
830 break;
831 }
832
833 case GIM_CheckLiteralInt: {
834 uint64_t InsnID = readULEB();
835 uint64_t OpIdx = readULEB();
836 int64_t Value = readU64();
837 DEBUG_WITH_TYPE(TgtExecutor::getName(),
838 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
839 << InsnID << "]->getOperand(" << OpIdx
840 << "), Value=" << Value << ")\n");
841 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
842 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
843 if (MO.isImm() && MO.getImm() == Value)
844 break;
845
846 if (MO.isCImm() && MO.getCImm()->equalsInt(Value))
847 break;
848
849 if (handleReject() == RejectAndGiveUp)
850 return false;
851
852 break;
853 }
854
855 case GIM_CheckIntrinsicID: {
856 uint64_t InsnID = readULEB();
857 uint64_t OpIdx = readULEB();
858 uint16_t Value = readU16();
859 DEBUG_WITH_TYPE(TgtExecutor::getName(),
860 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
861 << InsnID << "]->getOperand(" << OpIdx
862 << "), Value=" << Value << ")\n");
863 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
864 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
865 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
866 if (handleReject() == RejectAndGiveUp)
867 return false;
868 break;
869 }
870 case GIM_CheckCmpPredicate: {
871 uint64_t InsnID = readULEB();
872 uint64_t OpIdx = readULEB();
873 uint16_t Value = readU16();
874 DEBUG_WITH_TYPE(TgtExecutor::getName(),
875 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
876 << InsnID << "]->getOperand(" << OpIdx
877 << "), Value=" << Value << ")\n");
878 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
879 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
880 if (!MO.isPredicate() || MO.getPredicate() != Value)
881 if (handleReject() == RejectAndGiveUp)
882 return false;
883 break;
884 }
885 case GIM_CheckIsMBB: {
886 uint64_t InsnID = readULEB();
887 uint64_t OpIdx = readULEB();
888 DEBUG_WITH_TYPE(TgtExecutor::getName(),
889 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
890 << "]->getOperand(" << OpIdx << "))\n");
891 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
892 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
893 if (handleReject() == RejectAndGiveUp)
894 return false;
895 }
896 break;
897 }
898 case GIM_CheckIsImm: {
899 uint64_t InsnID = readULEB();
900 uint64_t OpIdx = readULEB();
901 DEBUG_WITH_TYPE(TgtExecutor::getName(),
902 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
903 << "]->getOperand(" << OpIdx << "))\n");
904 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
905 if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
906 if (handleReject() == RejectAndGiveUp)
907 return false;
908 }
909 break;
910 }
911 case GIM_CheckIsSafeToFold: {
912 uint64_t NumInsn = MatchTable[CurrentIdx++];
913 DEBUG_WITH_TYPE(TgtExecutor::getName(),
914 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(N = "
915 << NumInsn << ")\n");
916 MachineInstr &Root = *State.MIs[0];
917 for (unsigned K = 1, E = NumInsn + 1; K < E; ++K) {
918 if (!isObviouslySafeToFold(*State.MIs[K], Root)) {
919 if (handleReject() == RejectAndGiveUp)
920 return false;
921 }
922 }
923 break;
924 }
925 case GIM_CheckIsSameOperand:
926 case GIM_CheckIsSameOperandIgnoreCopies: {
927 uint64_t InsnID = readULEB();
928 uint64_t OpIdx = readULEB();
929 uint64_t OtherInsnID = readULEB();
930 uint64_t OtherOpIdx = readULEB();
931 DEBUG_WITH_TYPE(TgtExecutor::getName(),
932 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
933 << InsnID << "][" << OpIdx << "], MIs["
934 << OtherInsnID << "][" << OtherOpIdx << "])\n");
935 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
936 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
937
938 MachineOperand &Op = State.MIs[InsnID]->getOperand(OpIdx);
939 MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx);
940
941 if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) {
942 if (Op.isReg() && OtherOp.isReg()) {
943 if (getSrcRegIgnoringCopies(Op.getReg(), MRI) ==
944 getSrcRegIgnoringCopies(OtherOp.getReg(), MRI))
945 break;
946 }
947 }
948
949 if (!Op.isIdenticalTo(OtherOp)) {
950 if (handleReject() == RejectAndGiveUp)
951 return false;
952 }
953 break;
954 }
955 case GIM_CheckCanReplaceReg: {
956 uint64_t OldInsnID = readULEB();
957 uint64_t OldOpIdx = readULEB();
958 uint64_t NewInsnID = readULEB();
959 uint64_t NewOpIdx = readULEB();
960
961 DEBUG_WITH_TYPE(TgtExecutor::getName(),
962 dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs["
963 << OldInsnID << "][" << OldOpIdx << "] = MIs["
964 << NewInsnID << "][" << NewOpIdx << "])\n");
965
966 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
967 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg();
968 if (!canReplaceReg(Old, New, MRI)) {
969 if (handleReject() == RejectAndGiveUp)
970 return false;
971 }
972 break;
973 }
974 case GIM_MIFlags: {
975 uint64_t InsnID = readULEB();
976 uint32_t Flags = readU32();
977
978 DEBUG_WITH_TYPE(TgtExecutor::getName(),
979 dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID
980 << "], " << Flags << ")\n");
981 if ((State.MIs[InsnID]->getFlags() & Flags) != Flags) {
982 if (handleReject() == RejectAndGiveUp)
983 return false;
984 }
985 break;
986 }
987 case GIM_MIFlagsNot: {
988 uint64_t InsnID = readULEB();
989 uint32_t Flags = readU32();
990
991 DEBUG_WITH_TYPE(TgtExecutor::getName(),
992 dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID
993 << "], " << Flags << ")\n");
994 if ((State.MIs[InsnID]->getFlags() & Flags)) {
995 if (handleReject() == RejectAndGiveUp)
996 return false;
997 }
998 break;
999 }
1000 case GIM_Reject:
1001 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1002 dbgs() << CurrentIdx << ": GIM_Reject\n");
1003 if (handleReject() == RejectAndGiveUp)
1004 return false;
1005 break;
1006 case GIR_MutateOpcode: {
1007 uint64_t OldInsnID = readULEB();
1008 uint64_t NewInsnID = readULEB();
1009 uint16_t NewOpcode = readU16();
1010 if (NewInsnID >= OutMIs.size())
1011 OutMIs.resize(NewInsnID + 1);
1012
1013 MachineInstr *OldMI = State.MIs[OldInsnID];
1014 if (Observer)
1015 Observer->changingInstr(*OldMI);
1016 OutMIs[NewInsnID] = MachineInstrBuilder(*OldMI->getMF(), OldMI);
1017 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
1018 if (Observer)
1019 Observer->changedInstr(*OldMI);
1020 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1021 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
1022 << NewInsnID << "], MIs[" << OldInsnID << "], "
1023 << NewOpcode << ")\n");
1024 break;
1025 }
1026
1027 case GIR_BuildRootMI:
1028 case GIR_BuildMI: {
1029 uint64_t NewInsnID = (MatcherOpcode == GIR_BuildRootMI) ? 0 : readULEB();
1030 uint16_t Opcode = readU16();
1031 if (NewInsnID >= OutMIs.size())
1032 OutMIs.resize(NewInsnID + 1);
1033
1034 OutMIs[NewInsnID] = Builder.buildInstr(Opcode);
1035 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1036 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
1037 << NewInsnID << "], " << Opcode << ")\n");
1038 break;
1039 }
1040
1041 case GIR_BuildConstant: {
1042 uint64_t TempRegID = readULEB();
1043 uint64_t Imm = readU64();
1044 Builder.buildConstant(State.TempRegisters[TempRegID], Imm);
1045 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1046 dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg["
1047 << TempRegID << "], Imm=" << Imm << ")\n");
1048 break;
1049 }
1050
1051 case GIR_RootToRootCopy:
1052 case GIR_Copy: {
1053 uint64_t NewInsnID =
1054 (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB();
1055 uint64_t OldInsnID =
1056 (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB();
1057 uint64_t OpIdx = readULEB();
1058 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1059 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
1060 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1061 dbgs()
1062 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
1063 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
1064 break;
1065 }
1066
1067 case GIR_CopyOrAddZeroReg: {
1068 uint64_t NewInsnID = readULEB();
1069 uint64_t OldInsnID = readULEB();
1070 uint64_t OpIdx = readULEB();
1071 uint16_t ZeroReg = readU16();
1072 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1073 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
1074 if (isOperandImmEqual(MO, 0, MRI))
1075 OutMIs[NewInsnID].addReg(ZeroReg);
1076 else
1077 OutMIs[NewInsnID].add(MO);
1078 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1079 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
1080 << NewInsnID << "], MIs[" << OldInsnID << "], "
1081 << OpIdx << ", " << ZeroReg << ")\n");
1082 break;
1083 }
1084
1085 case GIR_CopySubReg: {
1086 uint64_t NewInsnID = readULEB();
1087 uint64_t OldInsnID = readULEB();
1088 uint64_t OpIdx = readULEB();
1089 uint16_t SubRegIdx = readU16();
1090 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1091 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
1092 0, SubRegIdx);
1093 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1094 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
1095 << NewInsnID << "], MIs[" << OldInsnID << "], "
1096 << OpIdx << ", " << SubRegIdx << ")\n");
1097 break;
1098 }
1099
1100 case GIR_AddImplicitDef: {
1101 uint64_t InsnID = readULEB();
1102 uint16_t RegNum = readU16();
1103 uint16_t Flags = readU16();
1104 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1105 Flags |= RegState::Implicit;
1106 OutMIs[InsnID].addDef(RegNum, Flags);
1107 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1108 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
1109 << InsnID << "], " << RegNum << ")\n");
1110 break;
1111 }
1112
1113 case GIR_AddImplicitUse: {
1114 uint64_t InsnID = readULEB();
1115 uint16_t RegNum = readU16();
1116 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1117 OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
1118 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1119 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
1120 << InsnID << "], " << RegNum << ")\n");
1121 break;
1122 }
1123
1124 case GIR_AddRegister: {
1125 uint64_t InsnID = readULEB();
1126 uint16_t RegNum = readU16();
1127 uint16_t RegFlags = readU16();
1128 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1129 OutMIs[InsnID].addReg(RegNum, RegFlags);
1130 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1131 dbgs()
1132 << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID
1133 << "], " << RegNum << ", " << RegFlags << ")\n");
1134 break;
1135 }
1136 case GIR_AddIntrinsicID: {
1137 uint64_t InsnID = readULEB();
1138 uint16_t Value = readU16();
1139 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1140 OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value);
1141 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1142 dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs["
1143 << InsnID << "], " << Value << ")\n");
1144 break;
1145 }
1146 case GIR_SetImplicitDefDead: {
1147 uint64_t InsnID = readULEB();
1148 uint64_t OpIdx = readULEB();
1149 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1150 dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs["
1151 << InsnID << "], OpIdx=" << OpIdx << ")\n");
1152 MachineInstr *MI = OutMIs[InsnID];
1153 assert(MI && "Modifying undefined instruction");
1154 MI->getOperand(MI->getNumExplicitOperands() + OpIdx).setIsDead();
1155 break;
1156 }
1157 case GIR_SetMIFlags: {
1158 uint64_t InsnID = readULEB();
1159 uint32_t Flags = readU32();
1160
1161 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1162 dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs["
1163 << InsnID << "], " << Flags << ")\n");
1164 MachineInstr *MI = OutMIs[InsnID];
1165 MI->setFlags(MI->getFlags() | Flags);
1166 break;
1167 }
1168 case GIR_UnsetMIFlags: {
1169 uint64_t InsnID = readULEB();
1170 uint32_t Flags = readU32();
1171
1172 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1173 dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs["
1174 << InsnID << "], " << Flags << ")\n");
1175 MachineInstr *MI = OutMIs[InsnID];
1176 MI->setFlags(MI->getFlags() & ~Flags);
1177 break;
1178 }
1179 case GIR_CopyMIFlags: {
1180 uint64_t InsnID = readULEB();
1181 uint64_t OldInsnID = readULEB();
1182
1183 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1184 dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs["
1185 << InsnID << "], MIs[" << OldInsnID << "])\n");
1186 MachineInstr *MI = OutMIs[InsnID];
1187 MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags());
1188 break;
1189 }
1190 case GIR_AddSimpleTempRegister:
1191 case GIR_AddTempRegister:
1192 case GIR_AddTempSubRegister: {
1193 uint64_t InsnID = readULEB();
1194 uint64_t TempRegID = readULEB();
1195 uint16_t TempRegFlags = 0;
1196 if (MatcherOpcode != GIR_AddSimpleTempRegister)
1197 TempRegFlags = readU16();
1198 uint16_t SubReg = 0;
1199 if (MatcherOpcode == GIR_AddTempSubRegister)
1200 SubReg = readU16();
1201
1202 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1203
1204 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags,
1205 SubReg);
1206 DEBUG_WITH_TYPE(
1207 TgtExecutor::getName(),
1208 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID
1209 << "], TempRegisters[" << TempRegID << "]";
1210 if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
1211 dbgs() << ", " << TempRegFlags << ")\n");
1212 break;
1213 }
1214
1215 case GIR_AddImm8:
1216 case GIR_AddImm: {
1217 const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8);
1218 uint64_t InsnID = readULEB();
1219 uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64();
1220 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1221 OutMIs[InsnID].addImm(Imm);
1222 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1223 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
1224 << "], " << Imm << ")\n");
1225 break;
1226 }
1227
1228 case GIR_AddCImm: {
1229 uint64_t InsnID = readULEB();
1230 int TypeID = readS8();
1231 uint64_t Imm = readU64();
1232 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1233
1234 unsigned Width = ExecInfo.TypeObjects[TypeID].getScalarSizeInBits();
1235 LLVMContext &Ctx = MF->getFunction().getContext();
1236 OutMIs[InsnID].addCImm(
1237 ConstantInt::get(IntegerType::get(Ctx, Width), Imm, /*signed*/ true));
1238 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1239 dbgs() << CurrentIdx << ": GIR_AddCImm(OutMIs[" << InsnID
1240 << "], TypeID=" << TypeID << ", Imm=" << Imm
1241 << ")\n");
1242 break;
1243 }
1244
1245 case GIR_ComplexRenderer: {
1246 uint64_t InsnID = readULEB();
1247 uint16_t RendererID = readU16();
1248 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1249 for (const auto &RenderOpFn : State.Renderers[RendererID])
1250 RenderOpFn(OutMIs[InsnID]);
1251 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1252 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
1253 << InsnID << "], " << RendererID << ")\n");
1254 break;
1255 }
1256 case GIR_ComplexSubOperandRenderer: {
1257 uint64_t InsnID = readULEB();
1258 uint16_t RendererID = readU16();
1259 uint64_t RenderOpID = readULEB();
1260 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1261 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
1262 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1263 dbgs() << CurrentIdx
1264 << ": GIR_ComplexSubOperandRenderer(OutMIs["
1265 << InsnID << "], " << RendererID << ", "
1266 << RenderOpID << ")\n");
1267 break;
1268 }
1269 case GIR_ComplexSubOperandSubRegRenderer: {
1270 uint64_t InsnID = readULEB();
1271 uint16_t RendererID = readU16();
1272 uint64_t RenderOpID = readULEB();
1273 uint16_t SubRegIdx = readU16();
1274 MachineInstrBuilder &MI = OutMIs[InsnID];
1275 assert(MI && "Attempted to add to undefined instruction");
1276 State.Renderers[RendererID][RenderOpID](MI);
1277 MI->getOperand(MI->getNumOperands() - 1).setSubReg(SubRegIdx);
1278 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1279 dbgs() << CurrentIdx
1280 << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs["
1281 << InsnID << "], " << RendererID << ", "
1282 << RenderOpID << ", " << SubRegIdx << ")\n");
1283 break;
1284 }
1285
1286 case GIR_CopyConstantAsSImm: {
1287 uint64_t NewInsnID = readULEB();
1288 uint64_t OldInsnID = readULEB();
1289 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1290 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
1291 "Expected G_CONSTANT");
1292 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
1293 OutMIs[NewInsnID].addImm(
1294 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
1295 } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
1296 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
1297 else
1298 llvm_unreachable("Expected Imm or CImm operand");
1299 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1300 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
1301 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1302 break;
1303 }
1304
1305 // TODO: Needs a test case once we have a pattern that uses this.
1306 case GIR_CopyFConstantAsFPImm: {
1307 uint64_t NewInsnID = readULEB();
1308 uint64_t OldInsnID = readULEB();
1309 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1310 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
1311 "Expected G_FCONSTANT");
1312 if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
1313 OutMIs[NewInsnID].addFPImm(
1314 State.MIs[OldInsnID]->getOperand(1).getFPImm());
1315 else
1316 llvm_unreachable("Expected FPImm operand");
1317 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1318 dbgs()
1319 << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
1320 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1321 break;
1322 }
1323
1324 case GIR_CustomRenderer: {
1325 uint64_t InsnID = readULEB();
1326 uint64_t OldInsnID = readULEB();
1327 uint16_t RendererFnID = readU16();
1328 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1329 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1330 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
1331 << InsnID << "], MIs[" << OldInsnID << "], "
1332 << RendererFnID << ")\n");
1333 (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1334 OutMIs[InsnID], *State.MIs[OldInsnID],
1335 -1); // Not a source operand of the old instruction.
1336 break;
1337 }
1338 case GIR_CustomAction: {
1339 uint16_t FnID = readU16();
1340 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1341 dbgs() << CurrentIdx << ": GIR_CustomAction(FnID=" << FnID
1342 << ")\n");
1343 assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID");
1344 runCustomAction(FnID, State, OutMIs);
1345 break;
1346 }
1347 case GIR_CustomOperandRenderer: {
1348 uint64_t InsnID = readULEB();
1349 uint64_t OldInsnID = readULEB();
1350 uint64_t OpIdx = readULEB();
1351 uint16_t RendererFnID = readU16();
1352 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1353
1354 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1355 dbgs() << CurrentIdx
1356 << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID
1357 << "], MIs[" << OldInsnID << "]->getOperand("
1358 << OpIdx << "), " << RendererFnID << ")\n");
1359 (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1360 OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx);
1361 break;
1362 }
1363 case GIR_ConstrainOperandRC: {
1364 uint64_t InsnID = readULEB();
1365 uint64_t OpIdx = readULEB();
1366 uint16_t RCEnum = readU16();
1367 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1368 MachineInstr &I = *OutMIs[InsnID].getInstr();
1369 MachineFunction &MF = *I.getParent()->getParent();
1370 MachineRegisterInfo &MRI = MF.getRegInfo();
1371 const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum);
1372 MachineOperand &MO = I.getOperand(OpIdx);
1373 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO);
1374 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1375 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1376 << InsnID << "], " << OpIdx << ", " << RCEnum
1377 << ")\n");
1378 break;
1379 }
1380
1381 case GIR_RootConstrainSelectedInstOperands:
1382 case GIR_ConstrainSelectedInstOperands: {
1383 uint64_t InsnID = (MatcherOpcode == GIR_RootConstrainSelectedInstOperands)
1384 ? 0
1385 : readULEB();
1386 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1387 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
1388 RBI);
1389 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1390 dbgs() << CurrentIdx
1391 << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1392 << InsnID << "])\n");
1393 break;
1394 }
1395 case GIR_MergeMemOperands: {
1396 uint64_t InsnID = readULEB();
1397 uint64_t NumInsn = MatchTable[CurrentIdx++];
1398 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1399
1400 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1401 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1402 << InsnID << "]");
1403 for (unsigned K = 0; K < NumInsn; ++K) {
1404 uint64_t NextID = readULEB();
1405 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1406 dbgs() << ", MIs[" << NextID << "]");
1407 for (const auto &MMO : State.MIs[NextID]->memoperands())
1408 OutMIs[InsnID].addMemOperand(MMO);
1409 }
1410 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n");
1411 break;
1412 }
1413 case GIR_EraseFromParent: {
1414 uint64_t InsnID = readULEB();
1415 MachineInstr *MI = State.MIs[InsnID];
1416 assert(MI && "Attempted to erase an undefined instruction");
1417 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1418 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1419 << InsnID << "])\n");
1420 eraseImpl(MI);
1421 break;
1422 }
1423 case GIR_EraseRootFromParent_Done: {
1424 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1425 dbgs()
1426 << CurrentIdx << ": GIR_EraseRootFromParent_Done\n");
1427 eraseImpl(State.MIs[0]);
1428 propagateFlags();
1429 return true;
1430 }
1431 case GIR_MakeTempReg: {
1432 uint64_t TempRegID = readULEB();
1433 int TypeID = readS8();
1434
1435 State.TempRegisters[TempRegID] =
1436 MRI.createGenericVirtualRegister(getTypeFromIdx(TypeID));
1437 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1438 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1439 << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1440 break;
1441 }
1442 case GIR_ReplaceReg: {
1443 uint64_t OldInsnID = readULEB();
1444 uint64_t OldOpIdx = readULEB();
1445 uint64_t NewInsnID = readULEB();
1446 uint64_t NewOpIdx = readULEB();
1447
1448 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1449 dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs["
1450 << OldInsnID << "][" << OldOpIdx << "] = MIs["
1451 << NewInsnID << "][" << NewOpIdx << "])\n");
1452
1453 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1454 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg();
1455 if (Observer)
1456 Observer->changingAllUsesOfReg(MRI, Old);
1457 MRI.replaceRegWith(Old, New);
1458 if (Observer)
1459 Observer->finishedChangingAllUsesOfReg();
1460 break;
1461 }
1462 case GIR_ReplaceRegWithTempReg: {
1463 uint64_t OldInsnID = readULEB();
1464 uint64_t OldOpIdx = readULEB();
1465 uint64_t TempRegID = readULEB();
1466
1467 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1468 dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs["
1469 << OldInsnID << "][" << OldOpIdx << "] = TempRegs["
1470 << TempRegID << "])\n");
1471
1472 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();
1473 Register New = State.TempRegisters[TempRegID];
1474 if (Observer)
1475 Observer->changingAllUsesOfReg(MRI, Old);
1476 MRI.replaceRegWith(Old, New);
1477 if (Observer)
1478 Observer->finishedChangingAllUsesOfReg();
1479 break;
1480 }
1481 case GIR_Coverage: {
1482 uint32_t RuleID = readU32();
1483 assert(CoverageInfo);
1484 CoverageInfo->setCovered(RuleID);
1485
1486 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx
1487 << ": GIR_Coverage("
1488 << RuleID << ")");
1489 break;
1490 }
1491
1492 case GIR_Done:
1493 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1494 dbgs() << CurrentIdx << ": GIR_Done\n");
1495 propagateFlags();
1496 return true;
1497 default:
1498 llvm_unreachable("Unexpected command");
1499 }
1500 }
1501 }
1502
1503 } // end namespace llvm
1504
1505 #endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
1506