1 //===- subzero/src/IceInstX86BaseImpl.h - Generic X86 instructions -*- C++ -*=//
2 //
3 // The Subzero Code Generator
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief Implements the InstX86Base class and its descendants.
12 ///
13 //===----------------------------------------------------------------------===//
14
15 #ifndef SUBZERO_SRC_ICEINSTX86BASEIMPL_H
16 #define SUBZERO_SRC_ICEINSTX86BASEIMPL_H
17
18 #include "IceInstX86Base.h"
19
20 #include "IceAssemblerX86Base.h"
21 #include "IceCfg.h"
22 #include "IceCfgNode.h"
23 #include "IceDefs.h"
24 #include "IceInst.h"
25 #include "IceOperand.h"
26 #include "IceTargetLowering.h"
27 #include "IceTargetLoweringX86Base.h"
28
29 namespace Ice {
30
31 namespace X86NAMESPACE {
32
33 template <typename TraitsType>
getWidthString(Type Ty)34 const char *InstImpl<TraitsType>::InstX86Base::getWidthString(Type Ty) {
35 return Traits::TypeAttributes[Ty].WidthString;
36 }
37
38 template <typename TraitsType>
getFldString(Type Ty)39 const char *InstImpl<TraitsType>::InstX86Base::getFldString(Type Ty) {
40 return Traits::TypeAttributes[Ty].FldString;
41 }
42
43 template <typename TraitsType>
44 typename InstImpl<TraitsType>::Cond::BrCond
getOppositeCondition(BrCond Cond)45 InstImpl<TraitsType>::InstX86Base::getOppositeCondition(BrCond Cond) {
46 return Traits::InstBrAttributes[Cond].Opposite;
47 }
48
49 template <typename TraitsType>
InstX86FakeRMW(Cfg * Func,Operand * Data,Operand * Addr,InstArithmetic::OpKind Op,Variable * Beacon)50 InstImpl<TraitsType>::InstX86FakeRMW::InstX86FakeRMW(Cfg *Func, Operand *Data,
51 Operand *Addr,
52 InstArithmetic::OpKind Op,
53 Variable *Beacon)
54 : InstX86Base(Func, InstX86Base::FakeRMW, 3, nullptr), Op(Op) {
55 this->addSource(Data);
56 this->addSource(Addr);
57 this->addSource(Beacon);
58 }
59
60 template <typename TraitsType>
InstX86GetIP(Cfg * Func,Variable * Dest)61 InstImpl<TraitsType>::InstX86GetIP::InstX86GetIP(Cfg *Func, Variable *Dest)
62 : InstX86Base(Func, InstX86Base::GetIP, 0, Dest) {}
63
64 template <typename TraitsType>
InstX86Mul(Cfg * Func,Variable * Dest,Variable * Source1,Operand * Source2)65 InstImpl<TraitsType>::InstX86Mul::InstX86Mul(Cfg *Func, Variable *Dest,
66 Variable *Source1,
67 Operand *Source2)
68 : InstX86Base(Func, InstX86Base::Mul, 2, Dest) {
69 this->addSource(Source1);
70 this->addSource(Source2);
71 }
72
73 template <typename TraitsType>
InstX86Shld(Cfg * Func,Variable * Dest,Variable * Source1,Operand * Source2)74 InstImpl<TraitsType>::InstX86Shld::InstX86Shld(Cfg *Func, Variable *Dest,
75 Variable *Source1,
76 Operand *Source2)
77 : InstX86Base(Func, InstX86Base::Shld, 3, Dest) {
78 this->addSource(Dest);
79 this->addSource(Source1);
80 this->addSource(Source2);
81 }
82
83 template <typename TraitsType>
InstX86Shrd(Cfg * Func,Variable * Dest,Variable * Source1,Operand * Source2)84 InstImpl<TraitsType>::InstX86Shrd::InstX86Shrd(Cfg *Func, Variable *Dest,
85 Variable *Source1,
86 Operand *Source2)
87 : InstX86Base(Func, InstX86Base::Shrd, 3, Dest) {
88 this->addSource(Dest);
89 this->addSource(Source1);
90 this->addSource(Source2);
91 }
92
93 template <typename TraitsType>
InstX86Label(Cfg * Func,TargetLowering * Target)94 InstImpl<TraitsType>::InstX86Label::InstX86Label(Cfg *Func,
95 TargetLowering *Target)
96 : InstX86Base(Func, InstX86Base::Label, 0, nullptr),
97 LabelNumber(Target->makeNextLabelNumber()) {
98 if (BuildDefs::dump()) {
99 Name = GlobalString::createWithString(
100 Func->getContext(), ".L" + Func->getFunctionName() + "$local$__" +
101 std::to_string(LabelNumber));
102 } else {
103 Name = GlobalString::createWithoutString(Func->getContext());
104 }
105 }
106
107 template <typename TraitsType>
InstX86Br(Cfg * Func,const CfgNode * TargetTrue,const CfgNode * TargetFalse,const InstX86Label * Label,BrCond Condition,Mode Kind)108 InstImpl<TraitsType>::InstX86Br::InstX86Br(Cfg *Func, const CfgNode *TargetTrue,
109 const CfgNode *TargetFalse,
110 const InstX86Label *Label,
111 BrCond Condition, Mode Kind)
112 : InstX86Base(Func, InstX86Base::Br, 0, nullptr), Condition(Condition),
113 TargetTrue(TargetTrue), TargetFalse(TargetFalse), Label(Label),
114 Kind(Kind) {}
115
116 template <typename TraitsType>
optimizeBranch(const CfgNode * NextNode)117 bool InstImpl<TraitsType>::InstX86Br::optimizeBranch(const CfgNode *NextNode) {
118 // If there is no next block, then there can be no fallthrough to optimize.
119 if (NextNode == nullptr)
120 return false;
121 // Intra-block conditional branches can't be optimized.
122 if (Label)
123 return false;
124 // If there is no fallthrough node, such as a non-default case label for a
125 // switch instruction, then there is no opportunity to optimize.
126 if (getTargetFalse() == nullptr)
127 return false;
128
129 // Unconditional branch to the next node can be removed.
130 if (Condition == Cond::Br_None && getTargetFalse() == NextNode) {
131 assert(getTargetTrue() == nullptr);
132 this->setDeleted();
133 return true;
134 }
135 // If the fallthrough is to the next node, set fallthrough to nullptr to
136 // indicate.
137 if (getTargetFalse() == NextNode) {
138 TargetFalse = nullptr;
139 return true;
140 }
141 // If TargetTrue is the next node, and TargetFalse is not nullptr (which was
142 // already tested above), then invert the branch condition, swap the targets,
143 // and set new fallthrough to nullptr.
144 if (getTargetTrue() == NextNode) {
145 assert(Condition != Cond::Br_None);
146 Condition = this->getOppositeCondition(Condition);
147 TargetTrue = getTargetFalse();
148 TargetFalse = nullptr;
149 return true;
150 }
151 return false;
152 }
153
154 template <typename TraitsType>
repointEdges(CfgNode * OldNode,CfgNode * NewNode)155 bool InstImpl<TraitsType>::InstX86Br::repointEdges(CfgNode *OldNode,
156 CfgNode *NewNode) {
157 bool Found = false;
158 if (TargetFalse == OldNode) {
159 TargetFalse = NewNode;
160 Found = true;
161 }
162 if (TargetTrue == OldNode) {
163 TargetTrue = NewNode;
164 Found = true;
165 }
166 return Found;
167 }
168
169 template <typename TraitsType>
InstX86Jmp(Cfg * Func,Operand * Target)170 InstImpl<TraitsType>::InstX86Jmp::InstX86Jmp(Cfg *Func, Operand *Target)
171 : InstX86Base(Func, InstX86Base::Jmp, 1, nullptr) {
172 this->addSource(Target);
173 }
174
175 template <typename TraitsType>
InstX86Call(Cfg * Func,Variable * Dest,Operand * CallTarget)176 InstImpl<TraitsType>::InstX86Call::InstX86Call(Cfg *Func, Variable *Dest,
177 Operand *CallTarget)
178 : InstX86Base(Func, InstX86Base::Call, 1, Dest) {
179 this->HasSideEffects = true;
180 this->addSource(CallTarget);
181 }
182
183 template <typename TraitsType>
InstX86Movmsk(Cfg * Func,Variable * Dest,Operand * Source)184 InstImpl<TraitsType>::InstX86Movmsk::InstX86Movmsk(Cfg *Func, Variable *Dest,
185 Operand *Source)
186 : InstX86Base(Func, InstX86Base::Movmsk, 1, Dest) {
187 this->addSource(Source);
188 }
189
190 template <typename TraitsType>
InstX86Cmov(Cfg * Func,Variable * Dest,Operand * Source,BrCond Condition)191 InstImpl<TraitsType>::InstX86Cmov::InstX86Cmov(Cfg *Func, Variable *Dest,
192 Operand *Source,
193 BrCond Condition)
194 : InstX86Base(Func, InstX86Base::Cmov, 2, Dest), Condition(Condition) {
195 // The final result is either the original Dest, or Source, so mark both as
196 // sources.
197 this->addSource(Dest);
198 this->addSource(Source);
199 }
200
201 template <typename TraitsType>
InstX86Cmpps(Cfg * Func,Variable * Dest,Operand * Source,CmppsCond Condition)202 InstImpl<TraitsType>::InstX86Cmpps::InstX86Cmpps(Cfg *Func, Variable *Dest,
203 Operand *Source,
204 CmppsCond Condition)
205 : InstX86Base(Func, InstX86Base::Cmpps, 2, Dest), Condition(Condition) {
206 this->addSource(Dest);
207 this->addSource(Source);
208 }
209
210 template <typename TraitsType>
InstX86Cmpxchg(Cfg * Func,Operand * DestOrAddr,Variable * Eax,Variable * Desired,bool Locked)211 InstImpl<TraitsType>::InstX86Cmpxchg::InstX86Cmpxchg(Cfg *Func,
212 Operand *DestOrAddr,
213 Variable *Eax,
214 Variable *Desired,
215 bool Locked)
216 : InstImpl<TraitsType>::InstX86BaseLockable(
217 Func, InstX86Base::Cmpxchg, 3, llvm::dyn_cast<Variable>(DestOrAddr),
218 Locked) {
219 constexpr uint16_t Encoded_rAX = 0;
220 (void)Encoded_rAX;
221 assert(Traits::getEncodedGPR(Eax->getRegNum()) == Encoded_rAX);
222 this->addSource(DestOrAddr);
223 this->addSource(Eax);
224 this->addSource(Desired);
225 }
226
227 template <typename TraitsType>
InstX86Cmpxchg8b(Cfg * Func,X86OperandMem * Addr,Variable * Edx,Variable * Eax,Variable * Ecx,Variable * Ebx,bool Locked)228 InstImpl<TraitsType>::InstX86Cmpxchg8b::InstX86Cmpxchg8b(
229 Cfg *Func, X86OperandMem *Addr, Variable *Edx, Variable *Eax, Variable *Ecx,
230 Variable *Ebx, bool Locked)
231 : InstImpl<TraitsType>::InstX86BaseLockable(Func, InstX86Base::Cmpxchg, 5,
232 nullptr, Locked) {
233 assert(Edx->getRegNum() == RegisterSet::Reg_edx);
234 assert(Eax->getRegNum() == RegisterSet::Reg_eax);
235 assert(Ecx->getRegNum() == RegisterSet::Reg_ecx);
236 assert(Ebx->getRegNum() == RegisterSet::Reg_ebx);
237 this->addSource(Addr);
238 this->addSource(Edx);
239 this->addSource(Eax);
240 this->addSource(Ecx);
241 this->addSource(Ebx);
242 }
243
244 template <typename TraitsType>
InstX86Cvt(Cfg * Func,Variable * Dest,Operand * Source,CvtVariant Variant)245 InstImpl<TraitsType>::InstX86Cvt::InstX86Cvt(Cfg *Func, Variable *Dest,
246 Operand *Source,
247 CvtVariant Variant)
248 : InstX86Base(Func, InstX86Base::Cvt, 1, Dest), Variant(Variant) {
249 this->addSource(Source);
250 }
251
252 template <typename TraitsType>
InstX86Icmp(Cfg * Func,Operand * Src0,Operand * Src1)253 InstImpl<TraitsType>::InstX86Icmp::InstX86Icmp(Cfg *Func, Operand *Src0,
254 Operand *Src1)
255 : InstX86Base(Func, InstX86Base::Icmp, 2, nullptr) {
256 this->addSource(Src0);
257 this->addSource(Src1);
258 }
259
260 template <typename TraitsType>
InstX86Ucomiss(Cfg * Func,Operand * Src0,Operand * Src1)261 InstImpl<TraitsType>::InstX86Ucomiss::InstX86Ucomiss(Cfg *Func, Operand *Src0,
262 Operand *Src1)
263 : InstX86Base(Func, InstX86Base::Ucomiss, 2, nullptr) {
264 this->addSource(Src0);
265 this->addSource(Src1);
266 }
267
268 template <typename TraitsType>
InstX86UD2(Cfg * Func)269 InstImpl<TraitsType>::InstX86UD2::InstX86UD2(Cfg *Func)
270 : InstX86Base(Func, InstX86Base::UD2, 0, nullptr) {}
271
272 template <typename TraitsType>
InstX86Int3(Cfg * Func)273 InstImpl<TraitsType>::InstX86Int3::InstX86Int3(Cfg *Func)
274 : InstX86Base(Func, InstX86Base::Int3, 0, nullptr) {}
275
276 template <typename TraitsType>
InstX86Test(Cfg * Func,Operand * Src1,Operand * Src2)277 InstImpl<TraitsType>::InstX86Test::InstX86Test(Cfg *Func, Operand *Src1,
278 Operand *Src2)
279 : InstX86Base(Func, InstX86Base::Test, 2, nullptr) {
280 this->addSource(Src1);
281 this->addSource(Src2);
282 }
283
284 template <typename TraitsType>
InstX86Mfence(Cfg * Func)285 InstImpl<TraitsType>::InstX86Mfence::InstX86Mfence(Cfg *Func)
286 : InstX86Base(Func, InstX86Base::Mfence, 0, nullptr) {
287 this->HasSideEffects = true;
288 }
289
290 template <typename TraitsType>
InstX86Store(Cfg * Func,Operand * Value,X86Operand * Mem)291 InstImpl<TraitsType>::InstX86Store::InstX86Store(Cfg *Func, Operand *Value,
292 X86Operand *Mem)
293 : InstX86Base(Func, InstX86Base::Store, 2, nullptr) {
294 this->addSource(Value);
295 this->addSource(Mem);
296 }
297
298 template <typename TraitsType>
InstX86StoreP(Cfg * Func,Variable * Value,X86OperandMem * Mem)299 InstImpl<TraitsType>::InstX86StoreP::InstX86StoreP(Cfg *Func, Variable *Value,
300 X86OperandMem *Mem)
301 : InstX86Base(Func, InstX86Base::StoreP, 2, nullptr) {
302 this->addSource(Value);
303 this->addSource(Mem);
304 }
305
306 template <typename TraitsType>
InstX86StoreQ(Cfg * Func,Operand * Value,X86OperandMem * Mem)307 InstImpl<TraitsType>::InstX86StoreQ::InstX86StoreQ(Cfg *Func, Operand *Value,
308 X86OperandMem *Mem)
309 : InstX86Base(Func, InstX86Base::StoreQ, 2, nullptr) {
310 this->addSource(Value);
311 this->addSource(Mem);
312 }
313
314 template <typename TraitsType>
InstX86StoreD(Cfg * Func,Operand * Value,X86OperandMem * Mem)315 InstImpl<TraitsType>::InstX86StoreD::InstX86StoreD(Cfg *Func, Operand *Value,
316 X86OperandMem *Mem)
317 : InstX86Base(Func, InstX86Base::StoreD, 2, nullptr) {
318 this->addSource(Value);
319 this->addSource(Mem);
320 }
321
322 template <typename TraitsType>
InstX86Nop(Cfg * Func,NopVariant Variant)323 InstImpl<TraitsType>::InstX86Nop::InstX86Nop(Cfg *Func, NopVariant Variant)
324 : InstX86Base(Func, InstX86Base::Nop, 0, nullptr), Variant(Variant) {}
325
326 template <typename TraitsType>
InstX86Fld(Cfg * Func,Operand * Src)327 InstImpl<TraitsType>::InstX86Fld::InstX86Fld(Cfg *Func, Operand *Src)
328 : InstX86Base(Func, InstX86Base::Fld, 1, nullptr) {
329 this->addSource(Src);
330 }
331
332 template <typename TraitsType>
InstX86Fstp(Cfg * Func,Variable * Dest)333 InstImpl<TraitsType>::InstX86Fstp::InstX86Fstp(Cfg *Func, Variable *Dest)
334 : InstX86Base(Func, InstX86Base::Fstp, 0, Dest) {}
335
336 template <typename TraitsType>
InstX86Pop(Cfg * Func,Variable * Dest)337 InstImpl<TraitsType>::InstX86Pop::InstX86Pop(Cfg *Func, Variable *Dest)
338 : InstX86Base(Func, InstX86Base::Pop, 0, Dest) {
339 // A pop instruction affects the stack pointer and so it should not be
340 // allowed to be automatically dead-code eliminated. (The corresponding push
341 // instruction doesn't need this treatment because it has no dest variable
342 // and therefore won't be dead-code eliminated.) This is needed for
343 // late-stage liveness analysis (e.g. asm-verbose mode).
344 this->HasSideEffects = true;
345 }
346
347 template <typename TraitsType>
InstX86Push(Cfg * Func,Operand * Source)348 InstImpl<TraitsType>::InstX86Push::InstX86Push(Cfg *Func, Operand *Source)
349 : InstX86Base(Func, InstX86Base::Push, 1, nullptr) {
350 this->addSource(Source);
351 }
352
353 template <typename TraitsType>
InstX86Push(Cfg * Func,InstX86Label * L)354 InstImpl<TraitsType>::InstX86Push::InstX86Push(Cfg *Func, InstX86Label *L)
355 : InstX86Base(Func, InstX86Base::Push, 0, nullptr), Label(L) {}
356
357 template <typename TraitsType>
InstX86Ret(Cfg * Func,Variable * Source)358 InstImpl<TraitsType>::InstX86Ret::InstX86Ret(Cfg *Func, Variable *Source)
359 : InstX86Base(Func, InstX86Base::Ret, Source ? 1 : 0, nullptr) {
360 if (Source)
361 this->addSource(Source);
362 }
363
364 template <typename TraitsType>
InstX86Setcc(Cfg * Func,Variable * Dest,BrCond Cond)365 InstImpl<TraitsType>::InstX86Setcc::InstX86Setcc(Cfg *Func, Variable *Dest,
366 BrCond Cond)
367 : InstX86Base(Func, InstX86Base::Setcc, 0, Dest), Condition(Cond) {}
368
369 template <typename TraitsType>
InstX86Xadd(Cfg * Func,Operand * Dest,Variable * Source,bool Locked)370 InstImpl<TraitsType>::InstX86Xadd::InstX86Xadd(Cfg *Func, Operand *Dest,
371 Variable *Source, bool Locked)
372 : InstImpl<TraitsType>::InstX86BaseLockable(
373 Func, InstX86Base::Xadd, 2, llvm::dyn_cast<Variable>(Dest), Locked) {
374 this->addSource(Dest);
375 this->addSource(Source);
376 }
377
378 template <typename TraitsType>
InstX86Xchg(Cfg * Func,Operand * Dest,Variable * Source)379 InstImpl<TraitsType>::InstX86Xchg::InstX86Xchg(Cfg *Func, Operand *Dest,
380 Variable *Source)
381 : InstX86Base(Func, InstX86Base::Xchg, 2, llvm::dyn_cast<Variable>(Dest)) {
382 this->addSource(Dest);
383 this->addSource(Source);
384 }
385
386 template <typename TraitsType>
InstX86IacaStart(Cfg * Func)387 InstImpl<TraitsType>::InstX86IacaStart::InstX86IacaStart(Cfg *Func)
388 : InstX86Base(Func, InstX86Base::IacaStart, 0, nullptr) {
389 assert(getFlags().getAllowIacaMarks());
390 }
391
392 template <typename TraitsType>
InstX86IacaEnd(Cfg * Func)393 InstImpl<TraitsType>::InstX86IacaEnd::InstX86IacaEnd(Cfg *Func)
394 : InstX86Base(Func, InstX86Base::IacaEnd, 0, nullptr) {
395 assert(getFlags().getAllowIacaMarks());
396 }
397
398 // ======================== Dump routines ======================== //
399
400 template <typename TraitsType>
dump(const Cfg * Func)401 void InstImpl<TraitsType>::InstX86Base::dump(const Cfg *Func) const {
402 if (!BuildDefs::dump())
403 return;
404 Ostream &Str = Func->getContext()->getStrDump();
405 Str << "[" << Traits::TargetName << "] ";
406 Inst::dump(Func);
407 }
408
409 template <typename TraitsType>
dump(const Cfg * Func)410 void InstImpl<TraitsType>::InstX86FakeRMW::dump(const Cfg *Func) const {
411 if (!BuildDefs::dump())
412 return;
413 Ostream &Str = Func->getContext()->getStrDump();
414 Type Ty = getData()->getType();
415 Str << "rmw " << InstArithmetic::getOpName(getOp()) << " " << Ty << " *";
416 getAddr()->dump(Func);
417 Str << ", ";
418 getData()->dump(Func);
419 Str << ", beacon=";
420 getBeacon()->dump(Func);
421 }
422
423 template <typename TraitsType>
emit(const Cfg * Func)424 void InstImpl<TraitsType>::InstX86GetIP::emit(const Cfg *Func) const {
425 if (!BuildDefs::dump())
426 return;
427 const auto *Dest = this->getDest();
428 assert(Dest->hasReg());
429 Ostream &Str = Func->getContext()->getStrEmit();
430 Str << "\t"
431 "call"
432 "\t";
433 auto *Target = static_cast<TargetLowering *>(Func->getTarget());
434 Target->emitWithoutPrefix(Target->createGetIPForRegister(Dest));
435 }
436
437 template <typename TraitsType>
emitIAS(const Cfg * Func)438 void InstImpl<TraitsType>::InstX86GetIP::emitIAS(const Cfg *Func) const {
439 const auto *Dest = this->getDest();
440 Assembler *Asm = Func->getAssembler<Assembler>();
441 assert(Dest->hasReg());
442 Asm->call(static_cast<TargetLowering *>(Func->getTarget())
443 ->createGetIPForRegister(Dest));
444 }
445
446 template <typename TraitsType>
dump(const Cfg * Func)447 void InstImpl<TraitsType>::InstX86GetIP::dump(const Cfg *Func) const {
448 if (!BuildDefs::dump())
449 return;
450 Ostream &Str = Func->getContext()->getStrDump();
451 this->getDest()->dump(Func);
452 Str << " = call getIP";
453 }
454
455 template <typename TraitsType>
emit(const Cfg * Func)456 void InstImpl<TraitsType>::InstX86Label::emit(const Cfg *Func) const {
457 if (!BuildDefs::dump())
458 return;
459 Ostream &Str = Func->getContext()->getStrEmit();
460 Str << getLabelName() << ":";
461 }
462
463 template <typename TraitsType>
emitIAS(const Cfg * Func)464 void InstImpl<TraitsType>::InstX86Label::emitIAS(const Cfg *Func) const {
465 Assembler *Asm = Func->getAssembler<Assembler>();
466 Asm->bindLocalLabel(LabelNumber);
467 if (OffsetReloc != nullptr) {
468 Asm->bindRelocOffset(OffsetReloc);
469 }
470 }
471
472 template <typename TraitsType>
dump(const Cfg * Func)473 void InstImpl<TraitsType>::InstX86Label::dump(const Cfg *Func) const {
474 if (!BuildDefs::dump())
475 return;
476 Ostream &Str = Func->getContext()->getStrDump();
477 Str << getLabelName() << ":";
478 }
479
480 template <typename TraitsType>
emit(const Cfg * Func)481 void InstImpl<TraitsType>::InstX86Br::emit(const Cfg *Func) const {
482 if (!BuildDefs::dump())
483 return;
484 Ostream &Str = Func->getContext()->getStrEmit();
485 Str << "\t";
486
487 if (Condition == Cond::Br_None) {
488 Str << "jmp";
489 } else {
490 Str << Traits::InstBrAttributes[Condition].EmitString;
491 }
492
493 if (Label) {
494 Str << "\t" << Label->getLabelName();
495 } else {
496 if (Condition == Cond::Br_None) {
497 Str << "\t" << getTargetFalse()->getAsmName();
498 } else {
499 Str << "\t" << getTargetTrue()->getAsmName();
500 if (getTargetFalse()) {
501 Str << "\n\t"
502 "jmp\t"
503 << getTargetFalse()->getAsmName();
504 }
505 }
506 }
507 }
508
509 template <typename TraitsType>
emitIAS(const Cfg * Func)510 void InstImpl<TraitsType>::InstX86Br::emitIAS(const Cfg *Func) const {
511 Assembler *Asm = Func->getAssembler<Assembler>();
512 if (Label) {
513 auto *L = Asm->getOrCreateLocalLabel(Label->getLabelNumber());
514 if (Condition == Cond::Br_None) {
515 Asm->jmp(L, isNear());
516 } else {
517 Asm->j(Condition, L, isNear());
518 }
519 } else {
520 if (Condition == Cond::Br_None) {
521 auto *L = Asm->getOrCreateCfgNodeLabel(getTargetFalse()->getIndex());
522 assert(!getTargetTrue());
523 Asm->jmp(L, isNear());
524 } else {
525 auto *L = Asm->getOrCreateCfgNodeLabel(getTargetTrue()->getIndex());
526 Asm->j(Condition, L, isNear());
527 if (getTargetFalse()) {
528 auto *L2 = Asm->getOrCreateCfgNodeLabel(getTargetFalse()->getIndex());
529 Asm->jmp(L2, isNear());
530 }
531 }
532 }
533 }
534
535 template <typename TraitsType>
dump(const Cfg * Func)536 void InstImpl<TraitsType>::InstX86Br::dump(const Cfg *Func) const {
537 if (!BuildDefs::dump())
538 return;
539 Ostream &Str = Func->getContext()->getStrDump();
540 Str << "br ";
541
542 if (Condition == Cond::Br_None) {
543 if (Label) {
544 Str << "label %" << Label->getLabelName();
545 } else {
546 Str << "label %" << getTargetFalse()->getName();
547 }
548 return;
549 }
550
551 Str << Traits::InstBrAttributes[Condition].DisplayString;
552 if (Label) {
553 Str << ", label %" << Label->getLabelName();
554 } else {
555 Str << ", label %" << getTargetTrue()->getName();
556 if (getTargetFalse()) {
557 Str << ", label %" << getTargetFalse()->getName();
558 }
559 }
560
561 Str << " // (" << (isNear() ? "near" : "far") << " jump)";
562 }
563
564 template <typename TraitsType>
emit(const Cfg * Func)565 void InstImpl<TraitsType>::InstX86Jmp::emit(const Cfg *Func) const {
566 if (!BuildDefs::dump())
567 return;
568 Ostream &Str = Func->getContext()->getStrEmit();
569 assert(this->getSrcSize() == 1);
570 const Operand *Src = this->getSrc(0);
571 if (Traits::Is64Bit) {
572 if (const auto *CR = llvm::dyn_cast<ConstantRelocatable>(Src)) {
573 Str << "\t"
574 "jmp"
575 "\t"
576 << CR->getName();
577 return;
578 }
579 }
580 Str << "\t"
581 "jmp"
582 "\t*";
583 getJmpTarget()->emit(Func);
584 }
585
586 template <typename TraitsType>
emitIAS(const Cfg * Func)587 void InstImpl<TraitsType>::InstX86Jmp::emitIAS(const Cfg *Func) const {
588 // Note: Adapted (mostly copied) from
589 // InstImpl<TraitsType>::InstX86Call::emitIAS().
590 Assembler *Asm = Func->getAssembler<Assembler>();
591 Operand *Target = getJmpTarget();
592 if (const auto *Var = llvm::dyn_cast<Variable>(Target)) {
593 if (Var->hasReg()) {
594 Asm->jmp(Traits::getEncodedGPR(Var->getRegNum()));
595 } else {
596 // The jmp instruction with a memory operand should be possible to
597 // encode, but it isn't a valid sandboxed instruction, and there
598 // shouldn't be a register allocation issue to jump through a scratch
599 // register, so we don't really need to bother implementing it.
600 llvm::report_fatal_error("Assembler can't jmp to memory operand");
601 }
602 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Target)) {
603 (void)Mem;
604 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
605 llvm::report_fatal_error("Assembler can't jmp to memory operand");
606 } else if (const auto *CR = llvm::dyn_cast<ConstantRelocatable>(Target)) {
607 Asm->jmp(CR);
608 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Target)) {
609 // NaCl trampoline calls refer to an address within the sandbox directly.
610 // This is usually only needed for non-IRT builds and otherwise not very
611 // portable or stable. Usually this is only done for "calls" and not jumps.
612 Asm->jmp(AssemblerImmediate(Imm->getValue()));
613 } else {
614 llvm::report_fatal_error("Unexpected operand type");
615 }
616 }
617
618 template <typename TraitsType>
dump(const Cfg * Func)619 void InstImpl<TraitsType>::InstX86Jmp::dump(const Cfg *Func) const {
620 if (!BuildDefs::dump())
621 return;
622 Ostream &Str = Func->getContext()->getStrDump();
623 Str << "jmp ";
624 getJmpTarget()->dump(Func);
625 }
626
627 template <typename TraitsType>
emit(const Cfg * Func)628 void InstImpl<TraitsType>::InstX86Call::emit(const Cfg *Func) const {
629 if (!BuildDefs::dump())
630 return;
631 Ostream &Str = Func->getContext()->getStrEmit();
632 assert(this->getSrcSize() == 1);
633 Str << "\t"
634 "call\t";
635 Operand *CallTarget = getCallTarget();
636 auto *Target = InstX86Base::getTarget(Func);
637 if (const auto *CI = llvm::dyn_cast<ConstantInteger32>(CallTarget)) {
638 // Emit without a leading '$'.
639 Str << CI->getValue();
640 } else if (const auto DirectCallTarget =
641 llvm::dyn_cast<ConstantRelocatable>(CallTarget)) {
642 DirectCallTarget->emitWithoutPrefix(Target);
643 } else {
644 Str << "*";
645 CallTarget->emit(Func);
646 }
647 }
648
649 template <typename TraitsType>
emitIAS(const Cfg * Func)650 void InstImpl<TraitsType>::InstX86Call::emitIAS(const Cfg *Func) const {
651 Assembler *Asm = Func->getAssembler<Assembler>();
652 Operand *CallTarget = getCallTarget();
653 auto *Target = InstX86Base::getTarget(Func);
654 if (const auto *Var = llvm::dyn_cast<Variable>(CallTarget)) {
655 if (Var->hasReg()) {
656 Asm->call(Traits::getEncodedGPR(Var->getRegNum()));
657 } else {
658 Asm->call(Target->stackVarToAsmOperand(Var));
659 }
660 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(CallTarget)) {
661 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
662 Asm->call(Mem->toAsmAddress(Asm, Target));
663 } else if (const auto *CR = llvm::dyn_cast<ConstantRelocatable>(CallTarget)) {
664 Asm->call(CR);
665 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(CallTarget)) {
666 Asm->call(AssemblerImmediate(Imm->getValue()));
667 } else {
668 llvm_unreachable("Unexpected operand type");
669 }
670 }
671
672 template <typename TraitsType>
dump(const Cfg * Func)673 void InstImpl<TraitsType>::InstX86Call::dump(const Cfg *Func) const {
674 if (!BuildDefs::dump())
675 return;
676 Ostream &Str = Func->getContext()->getStrDump();
677 if (this->getDest()) {
678 this->dumpDest(Func);
679 Str << " = ";
680 }
681 Str << "call ";
682 getCallTarget()->dump(Func);
683 }
684
685 // The this->Opcode parameter needs to be char* and not std::string because of
686 // template issues.
687 template <typename TraitsType>
emitTwoAddress(const Cfg * Func,const char * Opcode,const char * Suffix)688 void InstImpl<TraitsType>::InstX86Base::emitTwoAddress(
689 const Cfg *Func, const char *Opcode, const char *Suffix) const {
690 if (!BuildDefs::dump())
691 return;
692 Ostream &Str = Func->getContext()->getStrEmit();
693 assert(getSrcSize() == 2);
694 Operand *Dest = getDest();
695 if (Dest == nullptr)
696 Dest = getSrc(0);
697 assert(Dest == getSrc(0));
698 Operand *Src1 = getSrc(1);
699 Str << "\t" << Opcode << Suffix
700 << InstX86Base::getWidthString(Dest->getType()) << "\t";
701 Src1->emit(Func);
702 Str << ", ";
703 Dest->emit(Func);
704 }
705
706 template <typename TraitsType>
emitIASOpTyGPR(const Cfg * Func,Type Ty,const Operand * Op,const GPREmitterOneOp & Emitter)707 void InstImpl<TraitsType>::emitIASOpTyGPR(const Cfg *Func, Type Ty,
708 const Operand *Op,
709 const GPREmitterOneOp &Emitter) {
710 auto *Target = InstX86Base::getTarget(Func);
711 Assembler *Asm = Func->getAssembler<Assembler>();
712 if (const auto *Var = llvm::dyn_cast<Variable>(Op)) {
713 if (Var->hasReg()) {
714 // We cheat a little and use GPRRegister even for byte operations.
715 GPRRegister VarReg = Traits::getEncodedGPR(Var->getRegNum());
716 (Asm->*(Emitter.Reg))(Ty, VarReg);
717 } else {
718 Address StackAddr(Target->stackVarToAsmOperand(Var));
719 (Asm->*(Emitter.Addr))(Ty, StackAddr);
720 }
721 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Op)) {
722 Mem->emitSegmentOverride(Asm);
723 (Asm->*(Emitter.Addr))(Ty, Mem->toAsmAddress(Asm, Target));
724 } else {
725 llvm_unreachable("Unexpected operand type");
726 }
727 }
728
729 template <typename TraitsType>
730 template <bool VarCanBeByte, bool SrcCanBeByte>
emitIASRegOpTyGPR(const Cfg * Func,bool IsLea,Type Ty,const Variable * Var,const Operand * Src,const GPREmitterRegOp & Emitter)731 void InstImpl<TraitsType>::emitIASRegOpTyGPR(const Cfg *Func, bool IsLea,
732 Type Ty, const Variable *Var,
733 const Operand *Src,
734 const GPREmitterRegOp &Emitter) {
735 auto *Target = InstX86Base::getTarget(Func);
736 Assembler *Asm = Func->getAssembler<Assembler>();
737 assert(Var->hasReg());
738 // We cheat a little and use GPRRegister even for byte operations.
739 GPRRegister VarReg = VarCanBeByte ? Traits::getEncodedGPR(Var->getRegNum())
740 : Traits::getEncodedGPR(Var->getRegNum());
741 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
742 if (SrcVar->hasReg()) {
743 GPRRegister SrcReg = SrcCanBeByte
744 ? Traits::getEncodedGPR(SrcVar->getRegNum())
745 : Traits::getEncodedGPR(SrcVar->getRegNum());
746 (Asm->*(Emitter.GPRGPR))(Ty, VarReg, SrcReg);
747 } else {
748 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar);
749 (Asm->*(Emitter.GPRAddr))(Ty, VarReg, SrcStackAddr);
750 }
751 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) {
752 Mem->emitSegmentOverride(Asm);
753 (Asm->*(Emitter.GPRAddr))(Ty, VarReg,
754 Mem->toAsmAddress(Asm, Target, IsLea));
755 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src)) {
756 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue()));
757 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger64>(Src)) {
758 assert(Traits::Is64Bit);
759 assert(Utils::IsInt(32, Imm->getValue()));
760 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue()));
761 } else if (const auto *Reloc = llvm::dyn_cast<ConstantRelocatable>(Src)) {
762 const auto FixupKind = (Reloc->getName().hasStdString() &&
763 Reloc->getName().toString() == GlobalOffsetTable)
764 ? Traits::FK_GotPC
765 : Traits::TargetLowering::getAbsFixup();
766 AssemblerFixup *Fixup = Asm->createFixup(FixupKind, Reloc);
767 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Fixup));
768 } else if (const auto *Split = llvm::dyn_cast<VariableSplit>(Src)) {
769 (Asm->*(Emitter.GPRAddr))(Ty, VarReg, Split->toAsmAddress(Func));
770 } else {
771 llvm_unreachable("Unexpected operand type");
772 }
773 }
774
775 template <typename TraitsType>
emitIASAddrOpTyGPR(const Cfg * Func,Type Ty,const Address & Addr,const Operand * Src,const GPREmitterAddrOp & Emitter)776 void InstImpl<TraitsType>::emitIASAddrOpTyGPR(const Cfg *Func, Type Ty,
777 const Address &Addr,
778 const Operand *Src,
779 const GPREmitterAddrOp &Emitter) {
780 Assembler *Asm = Func->getAssembler<Assembler>();
781 // Src can only be Reg or AssemblerImmediate.
782 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
783 assert(SrcVar->hasReg());
784 GPRRegister SrcReg = Traits::getEncodedGPR(SrcVar->getRegNum());
785 (Asm->*(Emitter.AddrGPR))(Ty, Addr, SrcReg);
786 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src)) {
787 (Asm->*(Emitter.AddrImm))(Ty, Addr, AssemblerImmediate(Imm->getValue()));
788 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger64>(Src)) {
789 assert(Traits::Is64Bit);
790 assert(Utils::IsInt(32, Imm->getValue()));
791 (Asm->*(Emitter.AddrImm))(Ty, Addr, AssemblerImmediate(Imm->getValue()));
792 } else if (const auto *Reloc = llvm::dyn_cast<ConstantRelocatable>(Src)) {
793 const auto FixupKind = (Reloc->getName().hasStdString() &&
794 Reloc->getName().toString() == GlobalOffsetTable)
795 ? Traits::FK_GotPC
796 : Traits::TargetLowering::getAbsFixup();
797 AssemblerFixup *Fixup = Asm->createFixup(FixupKind, Reloc);
798 (Asm->*(Emitter.AddrImm))(Ty, Addr, AssemblerImmediate(Fixup));
799 } else {
800 llvm_unreachable("Unexpected operand type");
801 }
802 }
803
804 template <typename TraitsType>
emitIASAsAddrOpTyGPR(const Cfg * Func,Type Ty,const Operand * Op0,const Operand * Op1,const GPREmitterAddrOp & Emitter)805 void InstImpl<TraitsType>::emitIASAsAddrOpTyGPR(
806 const Cfg *Func, Type Ty, const Operand *Op0, const Operand *Op1,
807 const GPREmitterAddrOp &Emitter) {
808 auto *Target = InstX86Base::getTarget(Func);
809 if (const auto *Op0Var = llvm::dyn_cast<Variable>(Op0)) {
810 assert(!Op0Var->hasReg());
811 Address StackAddr(Target->stackVarToAsmOperand(Op0Var));
812 emitIASAddrOpTyGPR(Func, Ty, StackAddr, Op1, Emitter);
813 } else if (const auto *Op0Mem = llvm::dyn_cast<X86OperandMem>(Op0)) {
814 Assembler *Asm = Func->getAssembler<Assembler>();
815 Op0Mem->emitSegmentOverride(Asm);
816 emitIASAddrOpTyGPR(Func, Ty, Op0Mem->toAsmAddress(Asm, Target), Op1,
817 Emitter);
818 } else if (const auto *Split = llvm::dyn_cast<VariableSplit>(Op0)) {
819 emitIASAddrOpTyGPR(Func, Ty, Split->toAsmAddress(Func), Op1, Emitter);
820 } else {
821 llvm_unreachable("Unexpected operand type");
822 }
823 }
824
825 template <typename TraitsType>
emitIASGPRShift(const Cfg * Func,Type Ty,const Variable * Var,const Operand * Src,const GPREmitterShiftOp & Emitter)826 void InstImpl<TraitsType>::emitIASGPRShift(const Cfg *Func, Type Ty,
827 const Variable *Var,
828 const Operand *Src,
829 const GPREmitterShiftOp &Emitter) {
830 Assembler *Asm = Func->getAssembler<Assembler>();
831 // Technically, the Dest Var can be mem as well, but we only use Reg. We can
832 // extend this to check Dest if we decide to use that form.
833 assert(Var->hasReg());
834 // We cheat a little and use GPRRegister even for byte operations.
835 GPRRegister VarReg = Traits::getEncodedGPR(Var->getRegNum());
836 // Src must be reg == ECX or an Imm8. This is asserted by the assembler.
837 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
838 assert(SrcVar->hasReg());
839 GPRRegister SrcReg = Traits::getEncodedGPR(SrcVar->getRegNum());
840 (Asm->*(Emitter.GPRGPR))(Ty, VarReg, SrcReg);
841 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src)) {
842 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue()));
843 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger64>(Src)) {
844 assert(Traits::Is64Bit);
845 assert(Utils::IsInt(32, Imm->getValue()));
846 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue()));
847 } else {
848 llvm_unreachable("Unexpected operand type");
849 }
850 }
851
852 template <typename TraitsType>
emitIASGPRShiftDouble(const Cfg * Func,const Variable * Dest,const Operand * Src1Op,const Operand * Src2Op,const GPREmitterShiftD & Emitter)853 void InstImpl<TraitsType>::emitIASGPRShiftDouble(
854 const Cfg *Func, const Variable *Dest, const Operand *Src1Op,
855 const Operand *Src2Op, const GPREmitterShiftD &Emitter) {
856 Assembler *Asm = Func->getAssembler<Assembler>();
857 // Dest can be reg or mem, but we only use the reg variant.
858 assert(Dest->hasReg());
859 GPRRegister DestReg = Traits::getEncodedGPR(Dest->getRegNum());
860 // SrcVar1 must be reg.
861 const auto *SrcVar1 = llvm::cast<Variable>(Src1Op);
862 assert(SrcVar1->hasReg());
863 GPRRegister SrcReg = Traits::getEncodedGPR(SrcVar1->getRegNum());
864 Type Ty = SrcVar1->getType();
865 // Src2 can be the implicit CL register or an immediate.
866 if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src2Op)) {
867 (Asm->*(Emitter.GPRGPRImm))(Ty, DestReg, SrcReg,
868 AssemblerImmediate(Imm->getValue()));
869 } else {
870 assert(llvm::cast<Variable>(Src2Op)->getRegNum() == RegisterSet::Reg_cl);
871 (Asm->*(Emitter.GPRGPR))(Ty, DestReg, SrcReg);
872 }
873 }
874
875 template <typename TraitsType>
emitIASXmmShift(const Cfg * Func,Type Ty,const Variable * Var,const Operand * Src,const XmmEmitterShiftOp & Emitter)876 void InstImpl<TraitsType>::emitIASXmmShift(const Cfg *Func, Type Ty,
877 const Variable *Var,
878 const Operand *Src,
879 const XmmEmitterShiftOp &Emitter) {
880 auto *Target = InstX86Base::getTarget(Func);
881 Assembler *Asm = Func->getAssembler<Assembler>();
882 assert(Var->hasReg());
883 XmmRegister VarReg = Traits::getEncodedXmm(Var->getRegNum());
884 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
885 if (SrcVar->hasReg()) {
886 XmmRegister SrcReg = Traits::getEncodedXmm(SrcVar->getRegNum());
887 (Asm->*(Emitter.XmmXmm))(Ty, VarReg, SrcReg);
888 } else {
889 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar);
890 (Asm->*(Emitter.XmmAddr))(Ty, VarReg, SrcStackAddr);
891 }
892 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) {
893 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
894 (Asm->*(Emitter.XmmAddr))(Ty, VarReg, Mem->toAsmAddress(Asm, Target));
895 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src)) {
896 (Asm->*(Emitter.XmmImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue()));
897 } else {
898 llvm_unreachable("Unexpected operand type");
899 }
900 }
901
902 template <typename TraitsType>
emitIASRegOpTyXMM(const Cfg * Func,Type Ty,const Variable * Var,const Operand * Src,const XmmEmitterRegOp & Emitter)903 void InstImpl<TraitsType>::emitIASRegOpTyXMM(const Cfg *Func, Type Ty,
904 const Variable *Var,
905 const Operand *Src,
906 const XmmEmitterRegOp &Emitter) {
907 auto *Target = InstX86Base::getTarget(Func);
908 Assembler *Asm = Func->getAssembler<Assembler>();
909 assert(Var->hasReg());
910 XmmRegister VarReg = Traits::getEncodedXmm(Var->getRegNum());
911 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
912 if (SrcVar->hasReg()) {
913 XmmRegister SrcReg = Traits::getEncodedXmm(SrcVar->getRegNum());
914 (Asm->*(Emitter.XmmXmm))(Ty, VarReg, SrcReg);
915 } else {
916 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar);
917 (Asm->*(Emitter.XmmAddr))(Ty, VarReg, SrcStackAddr);
918 }
919 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) {
920 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
921 (Asm->*(Emitter.XmmAddr))(Ty, VarReg, Mem->toAsmAddress(Asm, Target));
922 } else if (const auto *Imm = llvm::dyn_cast<Constant>(Src)) {
923 (Asm->*(Emitter.XmmAddr))(Ty, VarReg,
924 Traits::Address::ofConstPool(Asm, Imm));
925 } else {
926 llvm_unreachable("Unexpected operand type");
927 }
928 }
929
930 template <typename TraitsType>
931 template <typename DReg_t, typename SReg_t, DReg_t (*destEnc)(RegNumT),
932 SReg_t (*srcEnc)(RegNumT)>
emitIASCastRegOp(const Cfg * Func,Type DestTy,const Variable * Dest,Type SrcTy,const Operand * Src,const CastEmitterRegOp<DReg_t,SReg_t> & Emitter)933 void InstImpl<TraitsType>::emitIASCastRegOp(
934 const Cfg *Func, Type DestTy, const Variable *Dest, Type SrcTy,
935 const Operand *Src, const CastEmitterRegOp<DReg_t, SReg_t> &Emitter) {
936 auto *Target = InstX86Base::getTarget(Func);
937 Assembler *Asm = Func->getAssembler<Assembler>();
938 assert(Dest->hasReg());
939 DReg_t DestReg = destEnc(Dest->getRegNum());
940 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
941 if (SrcVar->hasReg()) {
942 SReg_t SrcReg = srcEnc(SrcVar->getRegNum());
943 (Asm->*(Emitter.RegReg))(DestTy, DestReg, SrcTy, SrcReg);
944 } else {
945 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar);
946 (Asm->*(Emitter.RegAddr))(DestTy, DestReg, SrcTy, SrcStackAddr);
947 }
948 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) {
949 Mem->emitSegmentOverride(Asm);
950 (Asm->*(Emitter.RegAddr))(DestTy, DestReg, SrcTy,
951 Mem->toAsmAddress(Asm, Target));
952 } else {
953 llvm_unreachable("Unexpected operand type");
954 }
955 }
956
957 template <typename TraitsType>
958 template <typename DReg_t, typename SReg_t, DReg_t (*destEnc)(RegNumT),
959 SReg_t (*srcEnc)(RegNumT)>
emitIASThreeOpImmOps(const Cfg * Func,Type DispatchTy,const Variable * Dest,const Operand * Src0,const Operand * Src1,const ThreeOpImmEmitter<DReg_t,SReg_t> Emitter)960 void InstImpl<TraitsType>::emitIASThreeOpImmOps(
961 const Cfg *Func, Type DispatchTy, const Variable *Dest, const Operand *Src0,
962 const Operand *Src1, const ThreeOpImmEmitter<DReg_t, SReg_t> Emitter) {
963 auto *Target = InstX86Base::getTarget(Func);
964 Assembler *Asm = Func->getAssembler<Assembler>();
965 // This only handles Dest being a register, and Src1 being an immediate.
966 assert(Dest->hasReg());
967 DReg_t DestReg = destEnc(Dest->getRegNum());
968 AssemblerImmediate Imm(llvm::cast<ConstantInteger32>(Src1)->getValue());
969 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src0)) {
970 if (SrcVar->hasReg()) {
971 SReg_t SrcReg = srcEnc(SrcVar->getRegNum());
972 (Asm->*(Emitter.RegRegImm))(DispatchTy, DestReg, SrcReg, Imm);
973 } else {
974 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar);
975 (Asm->*(Emitter.RegAddrImm))(DispatchTy, DestReg, SrcStackAddr, Imm);
976 }
977 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src0)) {
978 Mem->emitSegmentOverride(Asm);
979 (Asm->*(Emitter.RegAddrImm))(DispatchTy, DestReg,
980 Mem->toAsmAddress(Asm, Target), Imm);
981 } else {
982 llvm_unreachable("Unexpected operand type");
983 }
984 }
985
986 template <typename TraitsType>
emitIASMovlikeXMM(const Cfg * Func,const Variable * Dest,const Operand * Src,const XmmEmitterMovOps Emitter)987 void InstImpl<TraitsType>::emitIASMovlikeXMM(const Cfg *Func,
988 const Variable *Dest,
989 const Operand *Src,
990 const XmmEmitterMovOps Emitter) {
991 auto *Target = InstX86Base::getTarget(Func);
992 Assembler *Asm = Func->getAssembler<Assembler>();
993 if (Dest->hasReg()) {
994 XmmRegister DestReg = Traits::getEncodedXmm(Dest->getRegNum());
995 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
996 if (SrcVar->hasReg()) {
997 (Asm->*(Emitter.XmmXmm))(DestReg,
998 Traits::getEncodedXmm(SrcVar->getRegNum()));
999 } else {
1000 Address StackAddr(Target->stackVarToAsmOperand(SrcVar));
1001 (Asm->*(Emitter.XmmAddr))(DestReg, StackAddr);
1002 }
1003 } else if (const auto *SrcMem = llvm::dyn_cast<X86OperandMem>(Src)) {
1004 assert(SrcMem->getSegmentRegister() == X86OperandMem::DefaultSegment);
1005 (Asm->*(Emitter.XmmAddr))(DestReg, SrcMem->toAsmAddress(Asm, Target));
1006 } else {
1007 llvm_unreachable("Unexpected operand type");
1008 }
1009 } else {
1010 Address StackAddr(Target->stackVarToAsmOperand(Dest));
1011 // Src must be a register in this case.
1012 const auto *SrcVar = llvm::cast<Variable>(Src);
1013 assert(SrcVar->hasReg());
1014 (Asm->*(Emitter.AddrXmm))(StackAddr,
1015 Traits::getEncodedXmm(SrcVar->getRegNum()));
1016 }
1017 }
1018
1019 template <typename TraitsType>
dump(const Cfg * Func)1020 void InstImpl<TraitsType>::InstX86Movmsk::dump(const Cfg *Func) const {
1021 if (!BuildDefs::dump())
1022 return;
1023 Ostream &Str = Func->getContext()->getStrDump();
1024 this->dumpDest(Func);
1025 Str << " = movmsk." << this->getSrc(0)->getType() << " ";
1026 this->dumpSources(Func);
1027 }
1028
1029 template <typename TraitsType>
emit(const Cfg * Func)1030 void InstImpl<TraitsType>::InstX86Movmsk::emit(const Cfg *Func) const {
1031 if (!BuildDefs::dump())
1032 return;
1033 Ostream &Str = Func->getContext()->getStrEmit();
1034 assert(this->getSrcSize() == 1);
1035 Type SrcTy = this->getSrc(0)->getType();
1036 assert(isVectorType(SrcTy));
1037 switch (SrcTy) {
1038 case IceType_v16i8:
1039 Str << "\t"
1040 "pmovmskb"
1041 "\t";
1042 break;
1043 case IceType_v4i32:
1044 case IceType_v4f32:
1045 Str << "\t"
1046 "movmskps"
1047 "\t";
1048 break;
1049 default:
1050 llvm_unreachable("Unexpected operand type");
1051 }
1052 this->getSrc(0)->emit(Func);
1053 Str << ", ";
1054 this->getDest()->emit(Func);
1055 }
1056
1057 template <typename TraitsType>
emitIAS(const Cfg * Func)1058 void InstImpl<TraitsType>::InstX86Movmsk::emitIAS(const Cfg *Func) const {
1059 assert(this->getSrcSize() == 1);
1060 Assembler *Asm = Func->getAssembler<Assembler>();
1061 const Variable *Dest = this->getDest();
1062 const Variable *Src = llvm::cast<Variable>(this->getSrc(0));
1063 const Type DestTy = Dest->getType();
1064 (void)DestTy;
1065 const Type SrcTy = Src->getType();
1066 assert(isVectorType(SrcTy));
1067 assert(isScalarIntegerType(DestTy));
1068 if (Traits::Is64Bit) {
1069 assert(DestTy == IceType_i32 || DestTy == IceType_i64);
1070 } else {
1071 assert(typeWidthInBytes(DestTy) <= 4);
1072 }
1073 XmmRegister SrcReg = Traits::getEncodedXmm(Src->getRegNum());
1074 GPRRegister DestReg = Traits::getEncodedGPR(Dest->getRegNum());
1075 Asm->movmsk(SrcTy, DestReg, SrcReg);
1076 }
1077
1078 template <typename TraitsType>
emit(const Cfg * Func)1079 void InstImpl<TraitsType>::InstX86Sqrt::emit(const Cfg *Func) const {
1080 if (!BuildDefs::dump())
1081 return;
1082 Ostream &Str = Func->getContext()->getStrEmit();
1083 assert(this->getSrcSize() == 1);
1084 Type Ty = this->getSrc(0)->getType();
1085 assert(isScalarFloatingType(Ty));
1086 Str << "\t"
1087 "sqrt"
1088 << Traits::TypeAttributes[Ty].SpSdString << "\t";
1089 this->getSrc(0)->emit(Func);
1090 Str << ", ";
1091 this->getDest()->emit(Func);
1092 }
1093
1094 template <typename TraitsType>
emit(const Cfg * Func)1095 void InstImpl<TraitsType>::InstX86Div::emit(const Cfg *Func) const {
1096 if (!BuildDefs::dump())
1097 return;
1098 Ostream &Str = Func->getContext()->getStrEmit();
1099 assert(this->getSrcSize() == 3);
1100 Operand *Src1 = this->getSrc(1);
1101 Str << "\t" << this->Opcode << this->getWidthString(Src1->getType()) << "\t";
1102 Src1->emit(Func);
1103 }
1104
1105 template <typename TraitsType>
emitIAS(const Cfg * Func)1106 void InstImpl<TraitsType>::InstX86Div::emitIAS(const Cfg *Func) const {
1107 assert(this->getSrcSize() == 3);
1108 const Operand *Src = this->getSrc(1);
1109 Type Ty = Src->getType();
1110 static GPREmitterOneOp Emitter = {&Assembler::div, &Assembler::div};
1111 emitIASOpTyGPR(Func, Ty, Src, Emitter);
1112 }
1113
1114 template <typename TraitsType>
emit(const Cfg * Func)1115 void InstImpl<TraitsType>::InstX86Idiv::emit(const Cfg *Func) const {
1116 if (!BuildDefs::dump())
1117 return;
1118 Ostream &Str = Func->getContext()->getStrEmit();
1119 assert(this->getSrcSize() == 3);
1120 Operand *Src1 = this->getSrc(1);
1121 Str << "\t" << this->Opcode << this->getWidthString(Src1->getType()) << "\t";
1122 Src1->emit(Func);
1123 }
1124
1125 template <typename TraitsType>
emitIAS(const Cfg * Func)1126 void InstImpl<TraitsType>::InstX86Idiv::emitIAS(const Cfg *Func) const {
1127 assert(this->getSrcSize() == 3);
1128 const Operand *Src = this->getSrc(1);
1129 Type Ty = Src->getType();
1130 static const GPREmitterOneOp Emitter = {&Assembler::idiv, &Assembler::idiv};
1131 emitIASOpTyGPR(Func, Ty, Src, Emitter);
1132 }
1133
1134 // pblendvb and blendvps take xmm0 as a final implicit argument.
1135 template <typename TraitsType>
emitVariableBlendInst(const char * Opcode,const Inst * Instr,const Cfg * Func)1136 void InstImpl<TraitsType>::emitVariableBlendInst(const char *Opcode,
1137 const Inst *Instr,
1138 const Cfg *Func) {
1139 if (!BuildDefs::dump())
1140 return;
1141 Ostream &Str = Func->getContext()->getStrEmit();
1142 assert(Instr->getSrcSize() == 3);
1143 assert(llvm::cast<Variable>(Instr->getSrc(2))->getRegNum() ==
1144 RegisterSet::Reg_xmm0);
1145 Str << "\t" << Opcode << "\t";
1146 Instr->getSrc(1)->emit(Func);
1147 Str << ", ";
1148 Instr->getDest()->emit(Func);
1149 }
1150
1151 template <typename TraitsType>
emitIASVariableBlendInst(const Inst * Instr,const Cfg * Func,const XmmEmitterRegOp & Emitter)1152 void InstImpl<TraitsType>::emitIASVariableBlendInst(
1153 const Inst *Instr, const Cfg *Func, const XmmEmitterRegOp &Emitter) {
1154 assert(Instr->getSrcSize() == 3);
1155 assert(llvm::cast<Variable>(Instr->getSrc(2))->getRegNum() ==
1156 RegisterSet::Reg_xmm0);
1157 const Variable *Dest = Instr->getDest();
1158 const Operand *Src = Instr->getSrc(1);
1159 emitIASRegOpTyXMM(Func, Dest->getType(), Dest, Src, Emitter);
1160 }
1161
1162 template <typename TraitsType>
emit(const Cfg * Func)1163 void InstImpl<TraitsType>::InstX86Blendvps::emit(const Cfg *Func) const {
1164 if (!BuildDefs::dump())
1165 return;
1166 emitVariableBlendInst(this->Opcode, this, Func);
1167 }
1168
1169 template <typename TraitsType>
emitIAS(const Cfg * Func)1170 void InstImpl<TraitsType>::InstX86Blendvps::emitIAS(const Cfg *Func) const {
1171 static const XmmEmitterRegOp Emitter = {&Assembler::blendvps,
1172 &Assembler::blendvps};
1173 emitIASVariableBlendInst(this, Func, Emitter);
1174 }
1175
1176 template <typename TraitsType>
emit(const Cfg * Func)1177 void InstImpl<TraitsType>::InstX86Pblendvb::emit(const Cfg *Func) const {
1178 if (!BuildDefs::dump())
1179 return;
1180 emitVariableBlendInst(this->Opcode, this, Func);
1181 }
1182
1183 template <typename TraitsType>
emitIAS(const Cfg * Func)1184 void InstImpl<TraitsType>::InstX86Pblendvb::emitIAS(const Cfg *Func) const {
1185 static const XmmEmitterRegOp Emitter = {&Assembler::pblendvb,
1186 &Assembler::pblendvb};
1187 emitIASVariableBlendInst(this, Func, Emitter);
1188 }
1189
1190 template <typename TraitsType>
emit(const Cfg * Func)1191 void InstImpl<TraitsType>::InstX86Imul::emit(const Cfg *Func) const {
1192 if (!BuildDefs::dump())
1193 return;
1194 Ostream &Str = Func->getContext()->getStrEmit();
1195 assert(this->getSrcSize() == 2);
1196 Variable *Dest = this->getDest();
1197 if (isByteSizedArithType(Dest->getType())) {
1198 // The 8-bit version of imul only allows the form "imul r/m8".
1199 const auto *Src0Var = llvm::dyn_cast<Variable>(this->getSrc(0));
1200 (void)Src0Var;
1201 assert(Src0Var->getRegNum() == RegisterSet::Reg_al);
1202 Str << "\t"
1203 "imulb\t";
1204 this->getSrc(1)->emit(Func);
1205 } else if (llvm::isa<Constant>(this->getSrc(1))) {
1206 Str << "\t"
1207 "imul"
1208 << this->getWidthString(Dest->getType()) << "\t";
1209 this->getSrc(1)->emit(Func);
1210 Str << ", ";
1211 this->getSrc(0)->emit(Func);
1212 Str << ", ";
1213 Dest->emit(Func);
1214 } else {
1215 this->emitTwoAddress(Func, this->Opcode);
1216 }
1217 }
1218
1219 template <typename TraitsType>
emitIAS(const Cfg * Func)1220 void InstImpl<TraitsType>::InstX86Imul::emitIAS(const Cfg *Func) const {
1221 assert(this->getSrcSize() == 2);
1222 const Variable *Var = this->getDest();
1223 Type Ty = Var->getType();
1224 const Operand *Src = this->getSrc(1);
1225 if (isByteSizedArithType(Ty)) {
1226 // The 8-bit version of imul only allows the form "imul r/m8".
1227 const auto *Src0Var = llvm::dyn_cast<Variable>(this->getSrc(0));
1228 (void)Src0Var;
1229 assert(Src0Var->getRegNum() == RegisterSet::Reg_al);
1230 static const GPREmitterOneOp Emitter = {&Assembler::imul, &Assembler::imul};
1231 emitIASOpTyGPR(Func, Ty, this->getSrc(1), Emitter);
1232 } else {
1233 // The two-address version is used when multiplying by a non-constant
1234 // or doing an 8-bit multiply.
1235 assert(Var == this->getSrc(0));
1236 static const GPREmitterRegOp Emitter = {&Assembler::imul, &Assembler::imul,
1237 &Assembler::imul};
1238 constexpr bool NotLea = false;
1239 emitIASRegOpTyGPR(Func, NotLea, Ty, Var, Src, Emitter);
1240 }
1241 }
1242
1243 template <typename TraitsType>
emit(const Cfg * Func)1244 void InstImpl<TraitsType>::InstX86ImulImm::emit(const Cfg *Func) const {
1245 if (!BuildDefs::dump())
1246 return;
1247 Ostream &Str = Func->getContext()->getStrEmit();
1248 assert(this->getSrcSize() == 2);
1249 Variable *Dest = this->getDest();
1250 assert(Dest->getType() == IceType_i16 || Dest->getType() == IceType_i32);
1251 assert(llvm::isa<Constant>(this->getSrc(1)));
1252 Str << "\t"
1253 "imul"
1254 << this->getWidthString(Dest->getType()) << "\t";
1255 this->getSrc(1)->emit(Func);
1256 Str << ", ";
1257 this->getSrc(0)->emit(Func);
1258 Str << ", ";
1259 Dest->emit(Func);
1260 }
1261
1262 template <typename TraitsType>
emitIAS(const Cfg * Func)1263 void InstImpl<TraitsType>::InstX86ImulImm::emitIAS(const Cfg *Func) const {
1264 assert(this->getSrcSize() == 2);
1265 const Variable *Dest = this->getDest();
1266 Type Ty = Dest->getType();
1267 assert(llvm::isa<Constant>(this->getSrc(1)));
1268 static const ThreeOpImmEmitter<GPRRegister, GPRRegister> Emitter = {
1269 &Assembler::imul, &Assembler::imul};
1270 emitIASThreeOpImmOps<GPRRegister, GPRRegister, Traits::getEncodedGPR,
1271 Traits::getEncodedGPR>(Func, Ty, Dest, this->getSrc(0),
1272 this->getSrc(1), Emitter);
1273 }
1274
1275 template <typename TraitsType>
emitIAS(const Cfg * Func)1276 void InstImpl<TraitsType>::InstX86Insertps::emitIAS(const Cfg *Func) const {
1277 assert(this->getSrcSize() == 3);
1278 assert(InstX86Base::getTarget(Func)->getInstructionSet() >= Traits::SSE4_1);
1279 const Variable *Dest = this->getDest();
1280 assert(Dest == this->getSrc(0));
1281 Type Ty = Dest->getType();
1282 static const ThreeOpImmEmitter<XmmRegister, XmmRegister> Emitter = {
1283 &Assembler::insertps, &Assembler::insertps};
1284 emitIASThreeOpImmOps<XmmRegister, XmmRegister, Traits::getEncodedXmm,
1285 Traits::getEncodedXmm>(Func, Ty, Dest, this->getSrc(1),
1286 this->getSrc(2), Emitter);
1287 }
1288
1289 template <typename TraitsType>
emit(const Cfg * Func)1290 void InstImpl<TraitsType>::InstX86Cbwdq::emit(const Cfg *Func) const {
1291 if (!BuildDefs::dump())
1292 return;
1293 Ostream &Str = Func->getContext()->getStrEmit();
1294 assert(this->getSrcSize() == 1);
1295 Operand *Src0 = this->getSrc(0);
1296 const auto DestReg = this->getDest()->getRegNum();
1297 const auto SrcReg = llvm::cast<Variable>(Src0)->getRegNum();
1298 (void)DestReg;
1299 (void)SrcReg;
1300 switch (Src0->getType()) {
1301 default:
1302 llvm_unreachable("unexpected source type!");
1303 break;
1304 case IceType_i8:
1305 assert(SrcReg == RegisterSet::Reg_al);
1306 assert(DestReg == RegisterSet::Reg_ax || DestReg == RegisterSet::Reg_ah);
1307 Str << "\t"
1308 "cbtw";
1309 break;
1310 case IceType_i16:
1311 assert(SrcReg == RegisterSet::Reg_ax);
1312 assert(DestReg == RegisterSet::Reg_dx);
1313 Str << "\t"
1314 "cwtd";
1315 break;
1316 case IceType_i32:
1317 assert(SrcReg == RegisterSet::Reg_eax);
1318 assert(DestReg == RegisterSet::Reg_edx);
1319 Str << "\t"
1320 "cltd";
1321 break;
1322 case IceType_i64:
1323 assert(Traits::Is64Bit);
1324 assert(SrcReg == Traits::getRaxOrDie());
1325 assert(DestReg == Traits::getRdxOrDie());
1326 Str << "\t"
1327 "cqo";
1328 break;
1329 }
1330 }
1331
1332 template <typename TraitsType>
emitIAS(const Cfg * Func)1333 void InstImpl<TraitsType>::InstX86Cbwdq::emitIAS(const Cfg *Func) const {
1334 Assembler *Asm = Func->getAssembler<Assembler>();
1335 assert(this->getSrcSize() == 1);
1336 Operand *Src0 = this->getSrc(0);
1337 const auto DestReg = this->getDest()->getRegNum();
1338 const auto SrcReg = llvm::cast<Variable>(Src0)->getRegNum();
1339 (void)DestReg;
1340 (void)SrcReg;
1341 switch (Src0->getType()) {
1342 default:
1343 llvm_unreachable("unexpected source type!");
1344 break;
1345 case IceType_i8:
1346 assert(SrcReg == RegisterSet::Reg_al);
1347 assert(DestReg == RegisterSet::Reg_ax || DestReg == RegisterSet::Reg_ah);
1348 Asm->cbw();
1349 break;
1350 case IceType_i16:
1351 assert(SrcReg == RegisterSet::Reg_ax);
1352 assert(DestReg == RegisterSet::Reg_dx);
1353 Asm->cwd();
1354 break;
1355 case IceType_i32:
1356 assert(SrcReg == RegisterSet::Reg_eax);
1357 assert(DestReg == RegisterSet::Reg_edx);
1358 Asm->cdq();
1359 break;
1360 case IceType_i64:
1361 assert(Traits::Is64Bit);
1362 assert(SrcReg == Traits::getRaxOrDie());
1363 assert(DestReg == Traits::getRdxOrDie());
1364 Asm->cqo();
1365 break;
1366 }
1367 }
1368
1369 template <typename TraitsType>
emit(const Cfg * Func)1370 void InstImpl<TraitsType>::InstX86Mul::emit(const Cfg *Func) const {
1371 if (!BuildDefs::dump())
1372 return;
1373 Ostream &Str = Func->getContext()->getStrEmit();
1374 assert(this->getSrcSize() == 2);
1375 assert(llvm::isa<Variable>(this->getSrc(0)));
1376 assert(llvm::cast<Variable>(this->getSrc(0))->getRegNum() ==
1377 RegisterSet::Reg_eax);
1378 assert(this->getDest()->getRegNum() == RegisterSet::Reg_eax); // TODO:
1379 // allow
1380 // edx?
1381 Str << "\t"
1382 "mul"
1383 << this->getWidthString(this->getDest()->getType()) << "\t";
1384 this->getSrc(1)->emit(Func);
1385 }
1386
1387 template <typename TraitsType>
emitIAS(const Cfg * Func)1388 void InstImpl<TraitsType>::InstX86Mul::emitIAS(const Cfg *Func) const {
1389 assert(this->getSrcSize() == 2);
1390 assert(llvm::isa<Variable>(this->getSrc(0)));
1391 assert(llvm::cast<Variable>(this->getSrc(0))->getRegNum() ==
1392 RegisterSet::Reg_eax);
1393 assert(this->getDest()->getRegNum() == RegisterSet::Reg_eax); // TODO:
1394 // allow
1395 // edx?
1396 const Operand *Src = this->getSrc(1);
1397 Type Ty = Src->getType();
1398 static const GPREmitterOneOp Emitter = {&Assembler::mul, &Assembler::mul};
1399 emitIASOpTyGPR(Func, Ty, Src, Emitter);
1400 }
1401
1402 template <typename TraitsType>
dump(const Cfg * Func)1403 void InstImpl<TraitsType>::InstX86Mul::dump(const Cfg *Func) const {
1404 if (!BuildDefs::dump())
1405 return;
1406 Ostream &Str = Func->getContext()->getStrDump();
1407 this->dumpDest(Func);
1408 Str << " = mul." << this->getDest()->getType() << " ";
1409 this->dumpSources(Func);
1410 }
1411
1412 template <typename TraitsType>
emit(const Cfg * Func)1413 void InstImpl<TraitsType>::InstX86Shld::emit(const Cfg *Func) const {
1414 if (!BuildDefs::dump())
1415 return;
1416 Ostream &Str = Func->getContext()->getStrEmit();
1417 Variable *Dest = this->getDest();
1418 assert(this->getSrcSize() == 3);
1419 assert(Dest == this->getSrc(0));
1420 Str << "\t"
1421 "shld"
1422 << this->getWidthString(Dest->getType()) << "\t";
1423 this->getSrc(2)->emit(Func);
1424 Str << ", ";
1425 this->getSrc(1)->emit(Func);
1426 Str << ", ";
1427 Dest->emit(Func);
1428 }
1429
1430 template <typename TraitsType>
emitIAS(const Cfg * Func)1431 void InstImpl<TraitsType>::InstX86Shld::emitIAS(const Cfg *Func) const {
1432 assert(this->getSrcSize() == 3);
1433 assert(this->getDest() == this->getSrc(0));
1434 const Variable *Dest = this->getDest();
1435 const Operand *Src1 = this->getSrc(1);
1436 const Operand *Src2 = this->getSrc(2);
1437 static const GPREmitterShiftD Emitter = {&Assembler::shld, &Assembler::shld};
1438 emitIASGPRShiftDouble(Func, Dest, Src1, Src2, Emitter);
1439 }
1440
1441 template <typename TraitsType>
dump(const Cfg * Func)1442 void InstImpl<TraitsType>::InstX86Shld::dump(const Cfg *Func) const {
1443 if (!BuildDefs::dump())
1444 return;
1445 Ostream &Str = Func->getContext()->getStrDump();
1446 this->dumpDest(Func);
1447 Str << " = shld." << this->getDest()->getType() << " ";
1448 this->dumpSources(Func);
1449 }
1450
1451 template <typename TraitsType>
emit(const Cfg * Func)1452 void InstImpl<TraitsType>::InstX86Shrd::emit(const Cfg *Func) const {
1453 if (!BuildDefs::dump())
1454 return;
1455 Ostream &Str = Func->getContext()->getStrEmit();
1456 Variable *Dest = this->getDest();
1457 assert(this->getSrcSize() == 3);
1458 assert(Dest == this->getSrc(0));
1459 Str << "\t"
1460 "shrd"
1461 << this->getWidthString(Dest->getType()) << "\t";
1462 this->getSrc(2)->emit(Func);
1463 Str << ", ";
1464 this->getSrc(1)->emit(Func);
1465 Str << ", ";
1466 Dest->emit(Func);
1467 }
1468
1469 template <typename TraitsType>
emitIAS(const Cfg * Func)1470 void InstImpl<TraitsType>::InstX86Shrd::emitIAS(const Cfg *Func) const {
1471 assert(this->getSrcSize() == 3);
1472 assert(this->getDest() == this->getSrc(0));
1473 const Variable *Dest = this->getDest();
1474 const Operand *Src1 = this->getSrc(1);
1475 const Operand *Src2 = this->getSrc(2);
1476 static const GPREmitterShiftD Emitter = {&Assembler::shrd, &Assembler::shrd};
1477 emitIASGPRShiftDouble(Func, Dest, Src1, Src2, Emitter);
1478 }
1479
1480 template <typename TraitsType>
dump(const Cfg * Func)1481 void InstImpl<TraitsType>::InstX86Shrd::dump(const Cfg *Func) const {
1482 if (!BuildDefs::dump())
1483 return;
1484 Ostream &Str = Func->getContext()->getStrDump();
1485 this->dumpDest(Func);
1486 Str << " = shrd." << this->getDest()->getType() << " ";
1487 this->dumpSources(Func);
1488 }
1489
1490 template <typename TraitsType>
emit(const Cfg * Func)1491 void InstImpl<TraitsType>::InstX86Cmov::emit(const Cfg *Func) const {
1492 if (!BuildDefs::dump())
1493 return;
1494 Ostream &Str = Func->getContext()->getStrEmit();
1495 Variable *Dest = this->getDest();
1496 Str << "\t";
1497 assert(Condition != Cond::Br_None);
1498 assert(this->getDest()->hasReg());
1499 Str << "cmov" << Traits::InstBrAttributes[Condition].DisplayString
1500 << this->getWidthString(Dest->getType()) << "\t";
1501 this->getSrc(1)->emit(Func);
1502 Str << ", ";
1503 Dest->emit(Func);
1504 }
1505
1506 template <typename TraitsType>
emitIAS(const Cfg * Func)1507 void InstImpl<TraitsType>::InstX86Cmov::emitIAS(const Cfg *Func) const {
1508 assert(Condition != Cond::Br_None);
1509 assert(this->getDest()->hasReg());
1510 assert(this->getSrcSize() == 2);
1511 Operand *Src = this->getSrc(1);
1512 Type SrcTy = Src->getType();
1513 assert(SrcTy == IceType_i16 || SrcTy == IceType_i32 || (Traits::Is64Bit));
1514 Assembler *Asm = Func->getAssembler<Assembler>();
1515 auto *Target = InstX86Base::getTarget(Func);
1516 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
1517 if (SrcVar->hasReg()) {
1518 Asm->cmov(SrcTy, Condition,
1519 Traits::getEncodedGPR(this->getDest()->getRegNum()),
1520 Traits::getEncodedGPR(SrcVar->getRegNum()));
1521 } else {
1522 Asm->cmov(SrcTy, Condition,
1523 Traits::getEncodedGPR(this->getDest()->getRegNum()),
1524 Target->stackVarToAsmOperand(SrcVar));
1525 }
1526 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) {
1527 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
1528 Asm->cmov(SrcTy, Condition,
1529 Traits::getEncodedGPR(this->getDest()->getRegNum()),
1530 Mem->toAsmAddress(Asm, Target));
1531 } else {
1532 llvm_unreachable("Unexpected operand type");
1533 }
1534 }
1535
1536 template <typename TraitsType>
dump(const Cfg * Func)1537 void InstImpl<TraitsType>::InstX86Cmov::dump(const Cfg *Func) const {
1538 if (!BuildDefs::dump())
1539 return;
1540 Ostream &Str = Func->getContext()->getStrDump();
1541 Str << "cmov" << Traits::InstBrAttributes[Condition].DisplayString << ".";
1542 Str << this->getDest()->getType() << " ";
1543 this->dumpDest(Func);
1544 Str << ", ";
1545 this->dumpSources(Func);
1546 }
1547
1548 template <typename TraitsType>
emit(const Cfg * Func)1549 void InstImpl<TraitsType>::InstX86Cmpps::emit(const Cfg *Func) const {
1550 if (!BuildDefs::dump())
1551 return;
1552 Ostream &Str = Func->getContext()->getStrEmit();
1553 assert(this->getSrcSize() == 2);
1554 assert(Condition < Cond::Cmpps_Invalid);
1555 Type DestTy = this->Dest->getType();
1556 Str << "\t"
1557 "cmp"
1558 << Traits::InstCmppsAttributes[Condition].EmitString
1559 << Traits::TypeAttributes[DestTy].PdPsString << "\t";
1560 this->getSrc(1)->emit(Func);
1561 Str << ", ";
1562 this->getDest()->emit(Func);
1563 }
1564
1565 template <typename TraitsType>
emitIAS(const Cfg * Func)1566 void InstImpl<TraitsType>::InstX86Cmpps::emitIAS(const Cfg *Func) const {
1567 Assembler *Asm = Func->getAssembler<Assembler>();
1568 assert(this->getSrcSize() == 2);
1569 assert(Condition < Cond::Cmpps_Invalid);
1570 // Assuming there isn't any load folding for cmpps, and vector constants are
1571 // not allowed in PNaCl.
1572 assert(llvm::isa<Variable>(this->getSrc(1)));
1573 auto *Target = InstX86Base::getTarget(Func);
1574 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(1));
1575 if (SrcVar->hasReg()) {
1576 Asm->cmpps(this->getDest()->getType(),
1577 Traits::getEncodedXmm(this->getDest()->getRegNum()),
1578 Traits::getEncodedXmm(SrcVar->getRegNum()), Condition);
1579 } else {
1580 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar);
1581 Asm->cmpps(this->getDest()->getType(),
1582 Traits::getEncodedXmm(this->getDest()->getRegNum()),
1583 SrcStackAddr, Condition);
1584 }
1585 }
1586
1587 template <typename TraitsType>
dump(const Cfg * Func)1588 void InstImpl<TraitsType>::InstX86Cmpps::dump(const Cfg *Func) const {
1589 if (!BuildDefs::dump())
1590 return;
1591 Ostream &Str = Func->getContext()->getStrDump();
1592 assert(Condition < Cond::Cmpps_Invalid);
1593 this->dumpDest(Func);
1594 Str << " = cmp" << Traits::InstCmppsAttributes[Condition].EmitString
1595 << "ps"
1596 "\t";
1597 this->dumpSources(Func);
1598 }
1599
1600 template <typename TraitsType>
emit(const Cfg * Func)1601 void InstImpl<TraitsType>::InstX86Cmpxchg::emit(const Cfg *Func) const {
1602 if (!BuildDefs::dump())
1603 return;
1604 Ostream &Str = Func->getContext()->getStrEmit();
1605 assert(this->getSrcSize() == 3);
1606 if (this->Locked) {
1607 Str << "\t"
1608 "lock";
1609 }
1610 Str << "\t"
1611 "cmpxchg"
1612 << this->getWidthString(this->getSrc(0)->getType()) << "\t";
1613 this->getSrc(2)->emit(Func);
1614 Str << ", ";
1615 this->getSrc(0)->emit(Func);
1616 }
1617
1618 template <typename TraitsType>
emitIAS(const Cfg * Func)1619 void InstImpl<TraitsType>::InstX86Cmpxchg::emitIAS(const Cfg *Func) const {
1620 assert(this->getSrcSize() == 3);
1621 Assembler *Asm = Func->getAssembler<Assembler>();
1622 Type Ty = this->getSrc(0)->getType();
1623 auto *Target = InstX86Base::getTarget(Func);
1624 const auto Mem = llvm::cast<X86OperandMem>(this->getSrc(0));
1625 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
1626 const Address Addr = Mem->toAsmAddress(Asm, Target);
1627 const auto *VarReg = llvm::cast<Variable>(this->getSrc(2));
1628 assert(VarReg->hasReg());
1629 const GPRRegister Reg = Traits::getEncodedGPR(VarReg->getRegNum());
1630 Asm->cmpxchg(Ty, Addr, Reg, this->Locked);
1631 }
1632
1633 template <typename TraitsType>
dump(const Cfg * Func)1634 void InstImpl<TraitsType>::InstX86Cmpxchg::dump(const Cfg *Func) const {
1635 if (!BuildDefs::dump())
1636 return;
1637 Ostream &Str = Func->getContext()->getStrDump();
1638 if (this->Locked) {
1639 Str << "lock ";
1640 }
1641 Str << "cmpxchg." << this->getSrc(0)->getType() << " ";
1642 this->dumpSources(Func);
1643 }
1644
1645 template <typename TraitsType>
emit(const Cfg * Func)1646 void InstImpl<TraitsType>::InstX86Cmpxchg8b::emit(const Cfg *Func) const {
1647 if (!BuildDefs::dump())
1648 return;
1649 Ostream &Str = Func->getContext()->getStrEmit();
1650 assert(this->getSrcSize() == 5);
1651 if (this->Locked) {
1652 Str << "\t"
1653 "lock";
1654 }
1655 Str << "\t"
1656 "cmpxchg8b\t";
1657 this->getSrc(0)->emit(Func);
1658 }
1659
1660 template <typename TraitsType>
emitIAS(const Cfg * Func)1661 void InstImpl<TraitsType>::InstX86Cmpxchg8b::emitIAS(const Cfg *Func) const {
1662 assert(this->getSrcSize() == 5);
1663 Assembler *Asm = Func->getAssembler<Assembler>();
1664 const auto Mem = llvm::cast<X86OperandMem>(this->getSrc(0));
1665 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
1666 auto *Target = InstX86Base::getTarget(Func);
1667 const Address Addr = Mem->toAsmAddress(Asm, Target);
1668 Asm->cmpxchg8b(Addr, this->Locked);
1669 }
1670
1671 template <typename TraitsType>
dump(const Cfg * Func)1672 void InstImpl<TraitsType>::InstX86Cmpxchg8b::dump(const Cfg *Func) const {
1673 if (!BuildDefs::dump())
1674 return;
1675 Ostream &Str = Func->getContext()->getStrDump();
1676 if (this->Locked) {
1677 Str << "lock ";
1678 }
1679 Str << "cmpxchg8b ";
1680 this->dumpSources(Func);
1681 }
1682
1683 template <typename TraitsType>
emit(const Cfg * Func)1684 void InstImpl<TraitsType>::InstX86Cvt::emit(const Cfg *Func) const {
1685 if (!BuildDefs::dump())
1686 return;
1687 Ostream &Str = Func->getContext()->getStrEmit();
1688 assert(this->getSrcSize() == 1);
1689 Str << "\t"
1690 "cvt";
1691 if (isTruncating())
1692 Str << "t";
1693 Str << Traits::TypeAttributes[this->getSrc(0)->getType()].CvtString << "2"
1694 << Traits::TypeAttributes[this->getDest()->getType()].CvtString << "\t";
1695 this->getSrc(0)->emit(Func);
1696 Str << ", ";
1697 this->getDest()->emit(Func);
1698 }
1699
1700 template <typename TraitsType>
emitIAS(const Cfg * Func)1701 void InstImpl<TraitsType>::InstX86Cvt::emitIAS(const Cfg *Func) const {
1702 assert(this->getSrcSize() == 1);
1703 const Variable *Dest = this->getDest();
1704 const Operand *Src = this->getSrc(0);
1705 Type DestTy = Dest->getType();
1706 Type SrcTy = Src->getType();
1707 switch (Variant) {
1708 case Si2ss: {
1709 assert(isScalarIntegerType(SrcTy));
1710 if (!Traits::Is64Bit) {
1711 assert(typeWidthInBytes(SrcTy) <= 4);
1712 } else {
1713 assert(SrcTy == IceType_i32 || SrcTy == IceType_i64);
1714 }
1715 assert(isScalarFloatingType(DestTy));
1716 static const CastEmitterRegOp<XmmRegister, GPRRegister> Emitter = {
1717 &Assembler::cvtsi2ss, &Assembler::cvtsi2ss};
1718 emitIASCastRegOp<XmmRegister, GPRRegister, Traits::getEncodedXmm,
1719 Traits::getEncodedGPR>(Func, DestTy, Dest, SrcTy, Src,
1720 Emitter);
1721 return;
1722 }
1723 case Tss2si: {
1724 assert(isScalarFloatingType(SrcTy));
1725 assert(isScalarIntegerType(DestTy));
1726 if (Traits::Is64Bit) {
1727 assert(DestTy == IceType_i32 || DestTy == IceType_i64);
1728 } else {
1729 assert(typeWidthInBytes(DestTy) <= 4);
1730 }
1731 static const CastEmitterRegOp<GPRRegister, XmmRegister> Emitter = {
1732 &Assembler::cvttss2si, &Assembler::cvttss2si};
1733 emitIASCastRegOp<GPRRegister, XmmRegister, Traits::getEncodedGPR,
1734 Traits::getEncodedXmm>(Func, DestTy, Dest, SrcTy, Src,
1735 Emitter);
1736 return;
1737 }
1738 case Ss2si: {
1739 assert(isScalarFloatingType(SrcTy));
1740 assert(isScalarIntegerType(DestTy));
1741 if (Traits::Is64Bit) {
1742 assert(DestTy == IceType_i32 || DestTy == IceType_i64);
1743 } else {
1744 assert(typeWidthInBytes(DestTy) <= 4);
1745 }
1746 static const CastEmitterRegOp<GPRRegister, XmmRegister> Emitter = {
1747 &Assembler::cvtss2si, &Assembler::cvtss2si};
1748 emitIASCastRegOp<GPRRegister, XmmRegister, Traits::getEncodedGPR,
1749 Traits::getEncodedXmm>(Func, DestTy, Dest, SrcTy, Src,
1750 Emitter);
1751 return;
1752 }
1753 case Float2float: {
1754 assert(isScalarFloatingType(SrcTy));
1755 assert(isScalarFloatingType(DestTy));
1756 assert(DestTy != SrcTy);
1757 static const XmmEmitterRegOp Emitter = {&Assembler::cvtfloat2float,
1758 &Assembler::cvtfloat2float};
1759 emitIASRegOpTyXMM(Func, SrcTy, Dest, Src, Emitter);
1760 return;
1761 }
1762 case Dq2ps: {
1763 assert(isVectorIntegerType(SrcTy));
1764 assert(isVectorFloatingType(DestTy));
1765 static const XmmEmitterRegOp Emitter = {&Assembler::cvtdq2ps,
1766 &Assembler::cvtdq2ps};
1767 emitIASRegOpTyXMM(Func, DestTy, Dest, Src, Emitter);
1768 return;
1769 }
1770 case Tps2dq: {
1771 assert(isVectorFloatingType(SrcTy));
1772 assert(isVectorIntegerType(DestTy));
1773 static const XmmEmitterRegOp Emitter = {&Assembler::cvttps2dq,
1774 &Assembler::cvttps2dq};
1775 emitIASRegOpTyXMM(Func, DestTy, Dest, Src, Emitter);
1776 return;
1777 }
1778 case Ps2dq: {
1779 assert(isVectorFloatingType(SrcTy));
1780 assert(isVectorIntegerType(DestTy));
1781 static const XmmEmitterRegOp Emitter = {&Assembler::cvtps2dq,
1782 &Assembler::cvtps2dq};
1783 emitIASRegOpTyXMM(Func, DestTy, Dest, Src, Emitter);
1784 return;
1785 }
1786 }
1787 }
1788
1789 template <typename TraitsType>
dump(const Cfg * Func)1790 void InstImpl<TraitsType>::InstX86Cvt::dump(const Cfg *Func) const {
1791 if (!BuildDefs::dump())
1792 return;
1793 Ostream &Str = Func->getContext()->getStrDump();
1794 this->dumpDest(Func);
1795 Str << " = cvt";
1796 if (isTruncating())
1797 Str << "t";
1798 Str << Traits::TypeAttributes[this->getSrc(0)->getType()].CvtString << "2"
1799 << Traits::TypeAttributes[this->getDest()->getType()].CvtString << " ";
1800 this->dumpSources(Func);
1801 }
1802
1803 template <typename TraitsType>
emit(const Cfg * Func)1804 void InstImpl<TraitsType>::InstX86Round::emit(const Cfg *Func) const {
1805 if (!BuildDefs::dump())
1806 return;
1807 Ostream &Str = Func->getContext()->getStrEmit();
1808 assert(this->getSrcSize() == 3);
1809 Str << "\t" << this->Opcode
1810 << Traits::TypeAttributes[this->getDest()->getType()].SpSdString << "\t";
1811 this->getSrc(1)->emit(Func);
1812 Str << ", ";
1813 this->getSrc(0)->emit(Func);
1814 Str << ", ";
1815 this->getDest()->emit(Func);
1816 }
1817
1818 template <typename TraitsType>
emitIAS(const Cfg * Func)1819 void InstImpl<TraitsType>::InstX86Round::emitIAS(const Cfg *Func) const {
1820 assert(this->getSrcSize() == 2);
1821 assert(InstX86Base::getTarget(Func)->getInstructionSet() >= Traits::SSE4_1);
1822 const Variable *Dest = this->getDest();
1823 Type Ty = Dest->getType();
1824 static const ThreeOpImmEmitter<XmmRegister, XmmRegister> Emitter = {
1825 &Assembler::round, &Assembler::round};
1826 emitIASThreeOpImmOps<XmmRegister, XmmRegister, Traits::getEncodedXmm,
1827 Traits::getEncodedXmm>(Func, Ty, Dest, this->getSrc(0),
1828 this->getSrc(1), Emitter);
1829 }
1830
1831 template <typename TraitsType>
emit(const Cfg * Func)1832 void InstImpl<TraitsType>::InstX86Icmp::emit(const Cfg *Func) const {
1833 if (!BuildDefs::dump())
1834 return;
1835 Ostream &Str = Func->getContext()->getStrEmit();
1836 assert(this->getSrcSize() == 2);
1837 Str << "\t"
1838 "cmp"
1839 << this->getWidthString(this->getSrc(0)->getType()) << "\t";
1840 this->getSrc(1)->emit(Func);
1841 Str << ", ";
1842 this->getSrc(0)->emit(Func);
1843 }
1844
1845 template <typename TraitsType>
emitIAS(const Cfg * Func)1846 void InstImpl<TraitsType>::InstX86Icmp::emitIAS(const Cfg *Func) const {
1847 assert(this->getSrcSize() == 2);
1848 const Operand *Src0 = this->getSrc(0);
1849 const Operand *Src1 = this->getSrc(1);
1850 Type Ty = Src0->getType();
1851 static const GPREmitterRegOp RegEmitter = {&Assembler::cmp, &Assembler::cmp,
1852 &Assembler::cmp};
1853 static const GPREmitterAddrOp AddrEmitter = {&Assembler::cmp,
1854 &Assembler::cmp};
1855 if (const auto *SrcVar0 = llvm::dyn_cast<Variable>(Src0)) {
1856 if (SrcVar0->hasReg()) {
1857 constexpr bool NotLea = false;
1858 emitIASRegOpTyGPR(Func, NotLea, Ty, SrcVar0, Src1, RegEmitter);
1859 return;
1860 }
1861 }
1862 emitIASAsAddrOpTyGPR(Func, Ty, Src0, Src1, AddrEmitter);
1863 }
1864
1865 template <typename TraitsType>
dump(const Cfg * Func)1866 void InstImpl<TraitsType>::InstX86Icmp::dump(const Cfg *Func) const {
1867 if (!BuildDefs::dump())
1868 return;
1869 Ostream &Str = Func->getContext()->getStrDump();
1870 Str << "cmp." << this->getSrc(0)->getType() << " ";
1871 this->dumpSources(Func);
1872 }
1873
1874 template <typename TraitsType>
emit(const Cfg * Func)1875 void InstImpl<TraitsType>::InstX86Ucomiss::emit(const Cfg *Func) const {
1876 if (!BuildDefs::dump())
1877 return;
1878 Ostream &Str = Func->getContext()->getStrEmit();
1879 assert(this->getSrcSize() == 2);
1880 Str << "\t"
1881 "ucomi"
1882 << Traits::TypeAttributes[this->getSrc(0)->getType()].SdSsString << "\t";
1883 this->getSrc(1)->emit(Func);
1884 Str << ", ";
1885 this->getSrc(0)->emit(Func);
1886 }
1887
1888 template <typename TraitsType>
emitIAS(const Cfg * Func)1889 void InstImpl<TraitsType>::InstX86Ucomiss::emitIAS(const Cfg *Func) const {
1890 assert(this->getSrcSize() == 2);
1891 // Currently src0 is always a variable by convention, to avoid having two
1892 // memory operands.
1893 assert(llvm::isa<Variable>(this->getSrc(0)));
1894 const auto *Src0Var = llvm::cast<Variable>(this->getSrc(0));
1895 Type Ty = Src0Var->getType();
1896 static const XmmEmitterRegOp Emitter = {&Assembler::ucomiss,
1897 &Assembler::ucomiss};
1898 emitIASRegOpTyXMM(Func, Ty, Src0Var, this->getSrc(1), Emitter);
1899 }
1900
1901 template <typename TraitsType>
dump(const Cfg * Func)1902 void InstImpl<TraitsType>::InstX86Ucomiss::dump(const Cfg *Func) const {
1903 if (!BuildDefs::dump())
1904 return;
1905 Ostream &Str = Func->getContext()->getStrDump();
1906 Str << "ucomiss." << this->getSrc(0)->getType() << " ";
1907 this->dumpSources(Func);
1908 }
1909
1910 template <typename TraitsType>
emit(const Cfg * Func)1911 void InstImpl<TraitsType>::InstX86UD2::emit(const Cfg *Func) const {
1912 if (!BuildDefs::dump())
1913 return;
1914 Ostream &Str = Func->getContext()->getStrEmit();
1915 assert(this->getSrcSize() == 0);
1916 Str << "\t"
1917 "ud2";
1918 }
1919
1920 template <typename TraitsType>
emitIAS(const Cfg * Func)1921 void InstImpl<TraitsType>::InstX86UD2::emitIAS(const Cfg *Func) const {
1922 Assembler *Asm = Func->getAssembler<Assembler>();
1923 Asm->ud2();
1924 }
1925
1926 template <typename TraitsType>
dump(const Cfg * Func)1927 void InstImpl<TraitsType>::InstX86UD2::dump(const Cfg *Func) const {
1928 if (!BuildDefs::dump())
1929 return;
1930 Ostream &Str = Func->getContext()->getStrDump();
1931 Str << "ud2";
1932 }
1933
1934 template <typename TraitsType>
emit(const Cfg * Func)1935 void InstImpl<TraitsType>::InstX86Int3::emit(const Cfg *Func) const {
1936 if (!BuildDefs::dump())
1937 return;
1938 Ostream &Str = Func->getContext()->getStrEmit();
1939 assert(this->getSrcSize() == 0);
1940 Str << "\t"
1941 "int 3";
1942 }
1943
1944 template <typename TraitsType>
emitIAS(const Cfg * Func)1945 void InstImpl<TraitsType>::InstX86Int3::emitIAS(const Cfg *Func) const {
1946 Assembler *Asm = Func->getAssembler<Assembler>();
1947 Asm->int3();
1948 }
1949
1950 template <typename TraitsType>
dump(const Cfg * Func)1951 void InstImpl<TraitsType>::InstX86Int3::dump(const Cfg *Func) const {
1952 if (!BuildDefs::dump())
1953 return;
1954 Ostream &Str = Func->getContext()->getStrDump();
1955 Str << "int 3";
1956 }
1957
1958 template <typename TraitsType>
emit(const Cfg * Func)1959 void InstImpl<TraitsType>::InstX86Test::emit(const Cfg *Func) const {
1960 if (!BuildDefs::dump())
1961 return;
1962 Ostream &Str = Func->getContext()->getStrEmit();
1963 assert(this->getSrcSize() == 2);
1964 Str << "\t"
1965 "test"
1966 << this->getWidthString(this->getSrc(0)->getType()) << "\t";
1967 this->getSrc(1)->emit(Func);
1968 Str << ", ";
1969 this->getSrc(0)->emit(Func);
1970 }
1971
1972 template <typename TraitsType>
emitIAS(const Cfg * Func)1973 void InstImpl<TraitsType>::InstX86Test::emitIAS(const Cfg *Func) const {
1974 assert(this->getSrcSize() == 2);
1975 const Operand *Src0 = this->getSrc(0);
1976 const Operand *Src1 = this->getSrc(1);
1977 Type Ty = Src0->getType();
1978 // The Reg/Addr form of test is not encodeable.
1979 static const GPREmitterRegOp RegEmitter = {&Assembler::test, nullptr,
1980 &Assembler::test};
1981 static const GPREmitterAddrOp AddrEmitter = {&Assembler::test,
1982 &Assembler::test};
1983 if (const auto *SrcVar0 = llvm::dyn_cast<Variable>(Src0)) {
1984 if (SrcVar0->hasReg()) {
1985 constexpr bool NotLea = false;
1986 emitIASRegOpTyGPR(Func, NotLea, Ty, SrcVar0, Src1, RegEmitter);
1987 return;
1988 }
1989 }
1990 emitIASAsAddrOpTyGPR(Func, Ty, Src0, Src1, AddrEmitter);
1991 }
1992
1993 template <typename TraitsType>
dump(const Cfg * Func)1994 void InstImpl<TraitsType>::InstX86Test::dump(const Cfg *Func) const {
1995 if (!BuildDefs::dump())
1996 return;
1997 Ostream &Str = Func->getContext()->getStrDump();
1998 Str << "test." << this->getSrc(0)->getType() << " ";
1999 this->dumpSources(Func);
2000 }
2001
2002 template <typename TraitsType>
emit(const Cfg * Func)2003 void InstImpl<TraitsType>::InstX86Mfence::emit(const Cfg *Func) const {
2004 if (!BuildDefs::dump())
2005 return;
2006 Ostream &Str = Func->getContext()->getStrEmit();
2007 assert(this->getSrcSize() == 0);
2008 Str << "\t"
2009 "mfence";
2010 }
2011
2012 template <typename TraitsType>
emitIAS(const Cfg * Func)2013 void InstImpl<TraitsType>::InstX86Mfence::emitIAS(const Cfg *Func) const {
2014 Assembler *Asm = Func->getAssembler<Assembler>();
2015 Asm->mfence();
2016 }
2017
2018 template <typename TraitsType>
dump(const Cfg * Func)2019 void InstImpl<TraitsType>::InstX86Mfence::dump(const Cfg *Func) const {
2020 if (!BuildDefs::dump())
2021 return;
2022 Ostream &Str = Func->getContext()->getStrDump();
2023 Str << "mfence";
2024 }
2025
2026 template <typename TraitsType>
emit(const Cfg * Func)2027 void InstImpl<TraitsType>::InstX86Store::emit(const Cfg *Func) const {
2028 if (!BuildDefs::dump())
2029 return;
2030 Ostream &Str = Func->getContext()->getStrEmit();
2031 assert(this->getSrcSize() == 2);
2032 Type Ty = this->getSrc(0)->getType();
2033 Str << "\t"
2034 "mov"
2035 << this->getWidthString(Ty) << Traits::TypeAttributes[Ty].SdSsString
2036 << "\t";
2037 this->getSrc(0)->emit(Func);
2038 Str << ", ";
2039 this->getSrc(1)->emit(Func);
2040 }
2041
2042 template <typename TraitsType>
emitIAS(const Cfg * Func)2043 void InstImpl<TraitsType>::InstX86Store::emitIAS(const Cfg *Func) const {
2044 assert(this->getSrcSize() == 2);
2045 const Operand *Dest = this->getSrc(1);
2046 const Operand *Src = this->getSrc(0);
2047 Type DestTy = Dest->getType();
2048 if (isScalarFloatingType(DestTy)) {
2049 // Src must be a register, since Dest is a Mem operand of some kind.
2050 const auto *SrcVar = llvm::cast<Variable>(Src);
2051 assert(SrcVar->hasReg());
2052 XmmRegister SrcReg = Traits::getEncodedXmm(SrcVar->getRegNum());
2053 Assembler *Asm = Func->getAssembler<Assembler>();
2054 auto *Target = InstX86Base::getTarget(Func);
2055 if (const auto *DestVar = llvm::dyn_cast<Variable>(Dest)) {
2056 assert(!DestVar->hasReg());
2057 Address StackAddr(Target->stackVarToAsmOperand(DestVar));
2058 Asm->movss(DestTy, StackAddr, SrcReg);
2059 } else {
2060 const auto DestMem = llvm::cast<X86OperandMem>(Dest);
2061 assert(DestMem->getSegmentRegister() == X86OperandMem::DefaultSegment);
2062 Asm->movss(DestTy, DestMem->toAsmAddress(Asm, Target), SrcReg);
2063 }
2064 return;
2065 } else {
2066 assert(isScalarIntegerType(DestTy));
2067 static const GPREmitterAddrOp GPRAddrEmitter = {&Assembler::mov,
2068 &Assembler::mov};
2069 emitIASAsAddrOpTyGPR(Func, DestTy, Dest, Src, GPRAddrEmitter);
2070 }
2071 }
2072
2073 template <typename TraitsType>
dump(const Cfg * Func)2074 void InstImpl<TraitsType>::InstX86Store::dump(const Cfg *Func) const {
2075 if (!BuildDefs::dump())
2076 return;
2077 Ostream &Str = Func->getContext()->getStrDump();
2078 Str << "mov." << this->getSrc(0)->getType() << " ";
2079 this->getSrc(1)->dump(Func);
2080 Str << ", ";
2081 this->getSrc(0)->dump(Func);
2082 }
2083
2084 template <typename TraitsType>
emit(const Cfg * Func)2085 void InstImpl<TraitsType>::InstX86StoreP::emit(const Cfg *Func) const {
2086 if (!BuildDefs::dump())
2087 return;
2088 Ostream &Str = Func->getContext()->getStrEmit();
2089 assert(this->getSrcSize() == 2);
2090 assert(isVectorType(this->getSrc(1)->getType()));
2091 Str << "\t"
2092 "movups\t";
2093 this->getSrc(0)->emit(Func);
2094 Str << ", ";
2095 this->getSrc(1)->emit(Func);
2096 }
2097
2098 template <typename TraitsType>
emitIAS(const Cfg * Func)2099 void InstImpl<TraitsType>::InstX86StoreP::emitIAS(const Cfg *Func) const {
2100 Assembler *Asm = Func->getAssembler<Assembler>();
2101 assert(this->getSrcSize() == 2);
2102 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(0));
2103 const auto DestMem = llvm::cast<X86OperandMem>(this->getSrc(1));
2104 assert(DestMem->getSegmentRegister() == X86OperandMem::DefaultSegment);
2105 assert(SrcVar->hasReg());
2106 auto *Target = InstX86Base::getTarget(Func);
2107 Asm->movups(DestMem->toAsmAddress(Asm, Target),
2108 Traits::getEncodedXmm(SrcVar->getRegNum()));
2109 }
2110
2111 template <typename TraitsType>
dump(const Cfg * Func)2112 void InstImpl<TraitsType>::InstX86StoreP::dump(const Cfg *Func) const {
2113 if (!BuildDefs::dump())
2114 return;
2115 Ostream &Str = Func->getContext()->getStrDump();
2116 Str << "storep." << this->getSrc(0)->getType() << " ";
2117 this->getSrc(1)->dump(Func);
2118 Str << ", ";
2119 this->getSrc(0)->dump(Func);
2120 }
2121
2122 template <typename TraitsType>
emit(const Cfg * Func)2123 void InstImpl<TraitsType>::InstX86StoreQ::emit(const Cfg *Func) const {
2124 if (!BuildDefs::dump())
2125 return;
2126 Ostream &Str = Func->getContext()->getStrEmit();
2127 assert(this->getSrcSize() == 2);
2128 assert(this->getSrc(1)->getType() == IceType_i64 ||
2129 this->getSrc(1)->getType() == IceType_f64 ||
2130 isVectorType(this->getSrc(1)->getType()));
2131 Str << "\t"
2132 "movq\t";
2133 this->getSrc(0)->emit(Func);
2134 Str << ", ";
2135 this->getSrc(1)->emit(Func);
2136 }
2137
2138 template <typename TraitsType>
emitIAS(const Cfg * Func)2139 void InstImpl<TraitsType>::InstX86StoreQ::emitIAS(const Cfg *Func) const {
2140 Assembler *Asm = Func->getAssembler<Assembler>();
2141 assert(this->getSrcSize() == 2);
2142 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(0));
2143 const auto DestMem = llvm::cast<X86OperandMem>(this->getSrc(1));
2144 assert(DestMem->getSegmentRegister() == X86OperandMem::DefaultSegment);
2145 assert(SrcVar->hasReg());
2146 auto *Target = InstX86Base::getTarget(Func);
2147 Asm->movq(DestMem->toAsmAddress(Asm, Target),
2148 Traits::getEncodedXmm(SrcVar->getRegNum()));
2149 }
2150
2151 template <typename TraitsType>
dump(const Cfg * Func)2152 void InstImpl<TraitsType>::InstX86StoreQ::dump(const Cfg *Func) const {
2153 if (!BuildDefs::dump())
2154 return;
2155 Ostream &Str = Func->getContext()->getStrDump();
2156 Str << "storeq." << this->getSrc(0)->getType() << " ";
2157 this->getSrc(1)->dump(Func);
2158 Str << ", ";
2159 this->getSrc(0)->dump(Func);
2160 }
2161
2162 template <typename TraitsType>
emit(const Cfg * Func)2163 void InstImpl<TraitsType>::InstX86StoreD::emit(const Cfg *Func) const {
2164 if (!BuildDefs::dump())
2165 return;
2166 Ostream &Str = Func->getContext()->getStrEmit();
2167 assert(this->getSrcSize() == 2);
2168 assert(this->getSrc(1)->getType() == IceType_i64 ||
2169 this->getSrc(1)->getType() == IceType_f64 ||
2170 isVectorType(this->getSrc(1)->getType()));
2171 Str << "\t"
2172 "movd\t";
2173 this->getSrc(0)->emit(Func);
2174 Str << ", ";
2175 this->getSrc(1)->emit(Func);
2176 }
2177
2178 template <typename TraitsType>
emitIAS(const Cfg * Func)2179 void InstImpl<TraitsType>::InstX86StoreD::emitIAS(const Cfg *Func) const {
2180 Assembler *Asm = Func->getAssembler<Assembler>();
2181 assert(this->getSrcSize() == 2);
2182 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(0));
2183 const auto DestMem = llvm::cast<X86OperandMem>(this->getSrc(1));
2184 assert(DestMem->getSegmentRegister() == X86OperandMem::DefaultSegment);
2185 assert(SrcVar->hasReg());
2186 auto *Target = InstX86Base::getTarget(Func);
2187 Asm->movd(SrcVar->getType(), DestMem->toAsmAddress(Asm, Target),
2188 Traits::getEncodedXmm(SrcVar->getRegNum()));
2189 }
2190
2191 template <typename TraitsType>
dump(const Cfg * Func)2192 void InstImpl<TraitsType>::InstX86StoreD::dump(const Cfg *Func) const {
2193 if (!BuildDefs::dump())
2194 return;
2195 Ostream &Str = Func->getContext()->getStrDump();
2196 Str << "stored." << this->getSrc(0)->getType() << " ";
2197 this->getSrc(1)->dump(Func);
2198 Str << ", ";
2199 this->getSrc(0)->dump(Func);
2200 }
2201
2202 template <typename TraitsType>
emit(const Cfg * Func)2203 void InstImpl<TraitsType>::InstX86Lea::emit(const Cfg *Func) const {
2204 if (!BuildDefs::dump())
2205 return;
2206 if (auto *Add = this->deoptLeaToAddOrNull(Func)) {
2207 Add->emit(Func);
2208 return;
2209 }
2210
2211 Ostream &Str = Func->getContext()->getStrEmit();
2212 assert(this->getSrcSize() == 1);
2213 assert(this->getDest()->hasReg());
2214 Str << "\t"
2215 "lea"
2216 << this->getWidthString(this->getDest()->getType()) << "\t";
2217 Operand *Src0 = this->getSrc(0);
2218 if (const auto *Src0Var = llvm::dyn_cast<Variable>(Src0)) {
2219 Type Ty = Src0Var->getType();
2220 // lea on x86-32 doesn't accept mem128 operands, so cast VSrc0 to an
2221 // acceptable type.
2222 Src0Var->asType(Func, isVectorType(Ty) ? IceType_i32 : Ty, RegNumT())
2223 ->emit(Func);
2224 } else {
2225 Src0->emit(Func);
2226 }
2227 Str << ", ";
2228 this->getDest()->emit(Func);
2229 }
2230
2231 template <typename TraitsType>
emit(const Cfg * Func)2232 void InstImpl<TraitsType>::InstX86Mov::emit(const Cfg *Func) const {
2233 if (!BuildDefs::dump())
2234 return;
2235 Ostream &Str = Func->getContext()->getStrEmit();
2236 assert(this->getSrcSize() == 1);
2237 Operand *Src = this->getSrc(0);
2238 Type SrcTy = Src->getType();
2239 Type DestTy = this->getDest()->getType();
2240 if (Traits::Is64Bit && DestTy == IceType_i64 &&
2241 llvm::isa<ConstantInteger64>(Src) &&
2242 !Utils::IsInt(32, llvm::cast<ConstantInteger64>(Src)->getValue())) {
2243 Str << "\t"
2244 "movabs"
2245 "\t";
2246 } else {
2247 Str << "\t"
2248 "mov"
2249 << (!isScalarFloatingType(DestTy)
2250 ? this->getWidthString(DestTy)
2251 : Traits::TypeAttributes[DestTy].SdSsString)
2252 << "\t";
2253 }
2254 // For an integer truncation operation, src is wider than dest. In this case,
2255 // we use a mov instruction whose data width matches the narrower dest.
2256 // TODO: This assert disallows usages such as copying a floating
2257 // point value between a vector and a scalar (which movss is used for). Clean
2258 // this up.
2259 assert(InstX86Base::getTarget(Func)->typeWidthInBytesOnStack(DestTy) ==
2260 InstX86Base::getTarget(Func)->typeWidthInBytesOnStack(SrcTy));
2261 const Operand *NewSrc = Src;
2262 if (auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
2263 RegNumT NewRegNum;
2264 if (SrcVar->hasReg())
2265 NewRegNum = Traits::getGprForType(DestTy, SrcVar->getRegNum());
2266 if (SrcTy != DestTy)
2267 NewSrc = SrcVar->asType(Func, DestTy, NewRegNum);
2268 }
2269 NewSrc->emit(Func);
2270 Str << ", ";
2271 this->getDest()->emit(Func);
2272 }
2273
2274 template <typename TraitsType>
emitIAS(const Cfg * Func)2275 void InstImpl<TraitsType>::InstX86Mov::emitIAS(const Cfg *Func) const {
2276 assert(this->getSrcSize() == 1);
2277 const Variable *Dest = this->getDest();
2278 const Operand *Src = this->getSrc(0);
2279 Type DestTy = Dest->getType();
2280 Type SrcTy = Src->getType();
2281 // Mov can be used for GPRs or XMM registers. Also, the type does not
2282 // necessarily match (Mov can be used for bitcasts). However, when the type
2283 // does not match, one of the operands must be a register. Thus, the strategy
2284 // is to find out if Src or Dest are a register, then use that register's
2285 // type to decide on which emitter set to use. The emitter set will include
2286 // reg-reg movs, but that case should be unused when the types don't match.
2287 static const XmmEmitterRegOp XmmRegEmitter = {&Assembler::movss,
2288 &Assembler::movss};
2289 static const GPREmitterRegOp GPRRegEmitter = {
2290 &Assembler::mov, &Assembler::mov, &Assembler::mov};
2291 static const GPREmitterAddrOp GPRAddrEmitter = {&Assembler::mov,
2292 &Assembler::mov};
2293 // For an integer truncation operation, src is wider than dest. In this case,
2294 // we use a mov instruction whose data width matches the narrower dest.
2295 // TODO: This assert disallows usages such as copying a floating
2296 // point value between a vector and a scalar (which movss is used for). Clean
2297 // this up.
2298 auto *Target = InstX86Base::getTarget(Func);
2299 assert(Target->typeWidthInBytesOnStack(this->getDest()->getType()) ==
2300 Target->typeWidthInBytesOnStack(Src->getType()));
2301 if (Dest->hasReg()) {
2302 if (isScalarFloatingType(DestTy)) {
2303 emitIASRegOpTyXMM(Func, DestTy, Dest, Src, XmmRegEmitter);
2304 return;
2305 } else {
2306 assert(isScalarIntegerType(DestTy));
2307 // Widen DestTy for truncation (see above note). We should only do this
2308 // when both Src and Dest are integer types.
2309 if (Traits::Is64Bit && DestTy == IceType_i64) {
2310 if (const auto *C64 = llvm::dyn_cast<ConstantInteger64>(Src)) {
2311 Func->getAssembler<Assembler>()->movabs(
2312 Traits::getEncodedGPR(Dest->getRegNum()), C64->getValue());
2313 return;
2314 }
2315 }
2316 if (isScalarIntegerType(SrcTy)) {
2317 SrcTy = DestTy;
2318 }
2319 constexpr bool NotLea = false;
2320 emitIASRegOpTyGPR(Func, NotLea, DestTy, Dest, Src, GPRRegEmitter);
2321 return;
2322 }
2323 } else {
2324 // Dest must be Stack and Src *could* be a register. Use Src's type to
2325 // decide on the emitters.
2326 Address StackAddr(Target->stackVarToAsmOperand(Dest));
2327 if (isScalarFloatingType(SrcTy)) {
2328 // Src must be a register.
2329 const auto *SrcVar = llvm::cast<Variable>(Src);
2330 assert(SrcVar->hasReg());
2331 Assembler *Asm = Func->getAssembler<Assembler>();
2332 Asm->movss(SrcTy, StackAddr, Traits::getEncodedXmm(SrcVar->getRegNum()));
2333 return;
2334 } else if (isVectorType(SrcTy)) {
2335 // Src must be a register
2336 const auto *SrcVar = llvm::cast<Variable>(Src);
2337 assert(SrcVar->hasReg());
2338 Assembler *Asm = Func->getAssembler<Assembler>();
2339 Asm->movups(StackAddr, Traits::getEncodedXmm(SrcVar->getRegNum()));
2340 } else {
2341 // Src can be a register or immediate.
2342 assert(isScalarIntegerType(SrcTy));
2343 emitIASAddrOpTyGPR(Func, SrcTy, StackAddr, Src, GPRAddrEmitter);
2344 return;
2345 }
2346 return;
2347 }
2348 }
2349
2350 template <typename TraitsType>
emit(const Cfg * Func)2351 void InstImpl<TraitsType>::InstX86Movd::emit(const Cfg *Func) const {
2352 if (!BuildDefs::dump())
2353 return;
2354 assert(this->getSrcSize() == 1);
2355 Variable *Dest = this->getDest();
2356 Operand *Src = this->getSrc(0);
2357
2358 if (Dest->getType() == IceType_i64 || Src->getType() == IceType_i64) {
2359 assert(Dest->getType() == IceType_f64 || Src->getType() == IceType_f64);
2360 assert(Dest->getType() != Src->getType());
2361 Ostream &Str = Func->getContext()->getStrEmit();
2362 Str << "\t"
2363 "movq"
2364 "\t";
2365 Src->emit(Func);
2366 Str << ", ";
2367 Dest->emit(Func);
2368 return;
2369 }
2370
2371 InstX86BaseUnaryopXmm<InstX86Base::Movd>::emit(Func);
2372 }
2373
2374 template <typename TraitsType>
emitIAS(const Cfg * Func)2375 void InstImpl<TraitsType>::InstX86Movd::emitIAS(const Cfg *Func) const {
2376 Assembler *Asm = Func->getAssembler<Assembler>();
2377 assert(this->getSrcSize() == 1);
2378 const Variable *Dest = this->getDest();
2379 auto *Target = InstX86Base::getTarget(Func);
2380 // For insert/extract element (one of Src/Dest is an Xmm vector and the other
2381 // is an int type).
2382 if (const auto *SrcVar = llvm::dyn_cast<Variable>(this->getSrc(0))) {
2383 if (SrcVar->getType() == IceType_i32 ||
2384 (Traits::Is64Bit && SrcVar->getType() == IceType_i64)) {
2385 assert(isVectorType(Dest->getType()) ||
2386 (isScalarFloatingType(Dest->getType()) &&
2387 typeWidthInBytes(SrcVar->getType()) ==
2388 typeWidthInBytes(Dest->getType())));
2389 assert(Dest->hasReg());
2390 XmmRegister DestReg = Traits::getEncodedXmm(Dest->getRegNum());
2391 if (SrcVar->hasReg()) {
2392 Asm->movd(SrcVar->getType(), DestReg,
2393 Traits::getEncodedGPR(SrcVar->getRegNum()));
2394 } else {
2395 Address StackAddr(Target->stackVarToAsmOperand(SrcVar));
2396 Asm->movd(SrcVar->getType(), DestReg, StackAddr);
2397 }
2398 } else {
2399 assert(isVectorType(SrcVar->getType()) ||
2400 (isScalarFloatingType(SrcVar->getType()) &&
2401 typeWidthInBytes(SrcVar->getType()) ==
2402 typeWidthInBytes(Dest->getType())));
2403 assert(SrcVar->hasReg());
2404 assert(Dest->getType() == IceType_i32 ||
2405 (Traits::Is64Bit && Dest->getType() == IceType_i64));
2406 XmmRegister SrcReg = Traits::getEncodedXmm(SrcVar->getRegNum());
2407 if (Dest->hasReg()) {
2408 Asm->movd(Dest->getType(), Traits::getEncodedGPR(Dest->getRegNum()),
2409 SrcReg);
2410 } else {
2411 Address StackAddr(Target->stackVarToAsmOperand(Dest));
2412 Asm->movd(Dest->getType(), StackAddr, SrcReg);
2413 }
2414 }
2415 } else {
2416 assert(Dest->hasReg());
2417 XmmRegister DestReg = Traits::getEncodedXmm(Dest->getRegNum());
2418 auto *Mem = llvm::cast<X86OperandMem>(this->getSrc(0));
2419 Asm->movd(Mem->getType(), DestReg, Mem->toAsmAddress(Asm, Target));
2420 }
2421 }
2422
2423 template <typename TraitsType>
emit(const Cfg * Func)2424 void InstImpl<TraitsType>::InstX86Movp::emit(const Cfg *Func) const {
2425 if (!BuildDefs::dump())
2426 return;
2427 // TODO(wala,stichnot): movups works with all vector operands, but there
2428 // exist other instructions (movaps, movdqa, movdqu) that may perform better,
2429 // depending on the data type and alignment of the operands.
2430 Ostream &Str = Func->getContext()->getStrEmit();
2431 assert(this->getSrcSize() == 1);
2432 Str << "\t"
2433 "movups\t";
2434 this->getSrc(0)->emit(Func);
2435 Str << ", ";
2436 this->getDest()->emit(Func);
2437 }
2438
2439 template <typename TraitsType>
emitIAS(const Cfg * Func)2440 void InstImpl<TraitsType>::InstX86Movp::emitIAS(const Cfg *Func) const {
2441 assert(this->getSrcSize() == 1);
2442 assert(isVectorType(this->getDest()->getType()));
2443 const Variable *Dest = this->getDest();
2444 const Operand *Src = this->getSrc(0);
2445 static const XmmEmitterMovOps Emitter = {
2446 &Assembler::movups, &Assembler::movups, &Assembler::movups};
2447 emitIASMovlikeXMM(Func, Dest, Src, Emitter);
2448 }
2449
2450 template <typename TraitsType>
emit(const Cfg * Func)2451 void InstImpl<TraitsType>::InstX86Movq::emit(const Cfg *Func) const {
2452 if (!BuildDefs::dump())
2453 return;
2454 Ostream &Str = Func->getContext()->getStrEmit();
2455 assert(this->getSrcSize() == 1);
2456 assert(this->getDest()->getType() == IceType_i64 ||
2457 this->getDest()->getType() == IceType_f64);
2458 Str << "\t"
2459 "movq"
2460 "\t";
2461 this->getSrc(0)->emit(Func);
2462 Str << ", ";
2463 this->getDest()->emit(Func);
2464 }
2465
2466 template <typename TraitsType>
emitIAS(const Cfg * Func)2467 void InstImpl<TraitsType>::InstX86Movq::emitIAS(const Cfg *Func) const {
2468 assert(this->getSrcSize() == 1);
2469 assert(this->getDest()->getType() == IceType_i64 ||
2470 this->getDest()->getType() == IceType_f64 ||
2471 isVectorType(this->getDest()->getType()));
2472 const Variable *Dest = this->getDest();
2473 const Operand *Src = this->getSrc(0);
2474 static const XmmEmitterMovOps Emitter = {&Assembler::movq, &Assembler::movq,
2475 &Assembler::movq};
2476 emitIASMovlikeXMM(Func, Dest, Src, Emitter);
2477 }
2478
2479 template <typename TraitsType>
emitIAS(const Cfg * Func)2480 void InstImpl<TraitsType>::InstX86MovssRegs::emitIAS(const Cfg *Func) const {
2481 // This is Binop variant is only intended to be used for reg-reg moves where
2482 // part of the Dest register is untouched.
2483 assert(this->getSrcSize() == 2);
2484 const Variable *Dest = this->getDest();
2485 assert(Dest == this->getSrc(0));
2486 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(1));
2487 assert(Dest->hasReg() && SrcVar->hasReg());
2488 Assembler *Asm = Func->getAssembler<Assembler>();
2489 Asm->movss(IceType_f32, Traits::getEncodedXmm(Dest->getRegNum()),
2490 Traits::getEncodedXmm(SrcVar->getRegNum()));
2491 }
2492
2493 template <typename TraitsType>
emitIAS(const Cfg * Func)2494 void InstImpl<TraitsType>::InstX86Movsx::emitIAS(const Cfg *Func) const {
2495 assert(this->getSrcSize() == 1);
2496 const Variable *Dest = this->getDest();
2497 const Operand *Src = this->getSrc(0);
2498 // Dest must be a > 8-bit register, but Src can be 8-bit. In practice we just
2499 // use the full register for Dest to avoid having an OperandSizeOverride
2500 // prefix. It also allows us to only dispatch on SrcTy.
2501 Type SrcTy = Src->getType();
2502 assert(typeWidthInBytes(Dest->getType()) > 1);
2503 assert(typeWidthInBytes(Dest->getType()) > typeWidthInBytes(SrcTy));
2504 constexpr bool NotLea = false;
2505 emitIASRegOpTyGPR<false, true>(Func, NotLea, SrcTy, Dest, Src, this->Emitter);
2506 }
2507
2508 template <typename TraitsType>
mayBeElided(const Variable * Dest,const Operand * SrcOpnd)2509 bool InstImpl<TraitsType>::InstX86Movzx::mayBeElided(
2510 const Variable *Dest, const Operand *SrcOpnd) const {
2511 assert(Traits::Is64Bit);
2512 const auto *Src = llvm::dyn_cast<Variable>(SrcOpnd);
2513
2514 // Src is not a Variable, so it does not have a register. Movzx can't be
2515 // elided.
2516 if (Src == nullptr)
2517 return false;
2518
2519 // Movzx to/from memory can't be elided.
2520 if (!Src->hasReg() || !Dest->hasReg())
2521 return false;
2522
2523 // Reg/reg move with different source and dest can't be elided.
2524 if (Traits::getEncodedGPR(Src->getRegNum()) !=
2525 Traits::getEncodedGPR(Dest->getRegNum()))
2526 return false;
2527
2528 // A must-keep movzx 32- to 64-bit is sometimes needed in x86-64 sandboxing.
2529 return !MustKeep;
2530 }
2531
2532 template <typename TraitsType>
emit(const Cfg * Func)2533 void InstImpl<TraitsType>::InstX86Movzx::emit(const Cfg *Func) const {
2534 if (!BuildDefs::dump())
2535 return;
2536 if (Traits::Is64Bit) {
2537 // There's no movzx %eXX, %rXX. To zero extend 32- to 64-bits, we emit a
2538 // mov %eXX, %eXX. The processor will still do a movzx[bw]q.
2539 assert(this->getSrcSize() == 1);
2540 const Operand *Src = this->getSrc(0);
2541 const Variable *Dest = this->Dest;
2542 if (Src->getType() == IceType_i32 && Dest->getType() == IceType_i64) {
2543 Ostream &Str = Func->getContext()->getStrEmit();
2544 if (mayBeElided(Dest, Src)) {
2545 Str << "\t/* elided movzx */";
2546 } else {
2547 Str << "\t"
2548 "mov"
2549 "\t";
2550 Src->emit(Func);
2551 Str << ", ";
2552 Dest->asType(Func, IceType_i32,
2553 Traits::getGprForType(IceType_i32, Dest->getRegNum()))
2554 ->emit(Func);
2555 Str << " /* movzx */";
2556 }
2557 return;
2558 }
2559 }
2560 InstX86BaseUnaryopGPR<InstX86Base::Movzx>::emit(Func);
2561 }
2562
2563 template <typename TraitsType>
emitIAS(const Cfg * Func)2564 void InstImpl<TraitsType>::InstX86Movzx::emitIAS(const Cfg *Func) const {
2565 assert(this->getSrcSize() == 1);
2566 const Variable *Dest = this->getDest();
2567 const Operand *Src = this->getSrc(0);
2568 Type SrcTy = Src->getType();
2569 assert(typeWidthInBytes(Dest->getType()) > 1);
2570 assert(typeWidthInBytes(Dest->getType()) > typeWidthInBytes(SrcTy));
2571 if (Traits::Is64Bit) {
2572 if (Src->getType() == IceType_i32 && Dest->getType() == IceType_i64 &&
2573 mayBeElided(Dest, Src)) {
2574 return;
2575 }
2576 }
2577 constexpr bool NotLea = false;
2578 emitIASRegOpTyGPR<false, true>(Func, NotLea, SrcTy, Dest, Src, this->Emitter);
2579 }
2580
2581 template <typename TraitsType>
emit(const Cfg * Func)2582 void InstImpl<TraitsType>::InstX86Nop::emit(const Cfg *Func) const {
2583 if (!BuildDefs::dump())
2584 return;
2585 Ostream &Str = Func->getContext()->getStrEmit();
2586 // TODO: Emit the right code for each variant.
2587 Str << "\t"
2588 "nop\t/* variant = "
2589 << Variant << " */";
2590 }
2591
2592 template <typename TraitsType>
emitIAS(const Cfg * Func)2593 void InstImpl<TraitsType>::InstX86Nop::emitIAS(const Cfg *Func) const {
2594 Assembler *Asm = Func->getAssembler<Assembler>();
2595 // TODO: Emit the right code for the variant.
2596 Asm->nop();
2597 }
2598
2599 template <typename TraitsType>
dump(const Cfg * Func)2600 void InstImpl<TraitsType>::InstX86Nop::dump(const Cfg *Func) const {
2601 if (!BuildDefs::dump())
2602 return;
2603 Ostream &Str = Func->getContext()->getStrDump();
2604 Str << "nop (variant = " << Variant << ")";
2605 }
2606
2607 template <typename TraitsType>
emit(const Cfg * Func)2608 void InstImpl<TraitsType>::InstX86Fld::emit(const Cfg *Func) const {
2609 if (!BuildDefs::dump())
2610 return;
2611 Ostream &Str = Func->getContext()->getStrEmit();
2612 assert(this->getSrcSize() == 1);
2613 Type Ty = this->getSrc(0)->getType();
2614 const auto *Var = llvm::dyn_cast<Variable>(this->getSrc(0));
2615 if (Var && Var->hasReg()) {
2616 // This is a physical xmm register, so we need to spill it to a temporary
2617 // stack slot. Function prolog emission guarantees that there is sufficient
2618 // space to do this.
2619 Str << "\t"
2620 "mov"
2621 << Traits::TypeAttributes[Ty].SdSsString << "\t";
2622 Var->emit(Func);
2623 Str << ", (%esp)\n"
2624 "\t"
2625 "fld"
2626 << this->getFldString(Ty)
2627 << "\t"
2628 "(%esp)";
2629 return;
2630 }
2631 Str << "\t"
2632 "fld"
2633 << this->getFldString(Ty) << "\t";
2634 this->getSrc(0)->emit(Func);
2635 }
2636
2637 template <typename TraitsType>
emitIAS(const Cfg * Func)2638 void InstImpl<TraitsType>::InstX86Fld::emitIAS(const Cfg *Func) const {
2639 Assembler *Asm = Func->getAssembler<Assembler>();
2640 assert(this->getSrcSize() == 1);
2641 const Operand *Src = this->getSrc(0);
2642 auto *Target = InstX86Base::getTarget(Func);
2643 Type Ty = Src->getType();
2644 if (const auto *Var = llvm::dyn_cast<Variable>(Src)) {
2645 if (Var->hasReg()) {
2646 // This is a physical xmm register, so we need to spill it to a temporary
2647 // stack slot. Function prolog emission guarantees that there is
2648 // sufficient space to do this.
2649 Address StackSlot =
2650 Address(RegisterSet::Encoded_Reg_esp, 0, AssemblerFixup::NoFixup);
2651 Asm->movss(Ty, StackSlot, Traits::getEncodedXmm(Var->getRegNum()));
2652 Asm->fld(Ty, StackSlot);
2653 } else {
2654 Address StackAddr(Target->stackVarToAsmOperand(Var));
2655 Asm->fld(Ty, StackAddr);
2656 }
2657 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) {
2658 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
2659 Asm->fld(Ty, Mem->toAsmAddress(Asm, Target));
2660 } else if (const auto *Imm = llvm::dyn_cast<Constant>(Src)) {
2661 Asm->fld(Ty, Traits::Address::ofConstPool(Asm, Imm));
2662 } else {
2663 llvm_unreachable("Unexpected operand type");
2664 }
2665 }
2666
2667 template <typename TraitsType>
dump(const Cfg * Func)2668 void InstImpl<TraitsType>::InstX86Fld::dump(const Cfg *Func) const {
2669 if (!BuildDefs::dump())
2670 return;
2671 Ostream &Str = Func->getContext()->getStrDump();
2672 Str << "fld." << this->getSrc(0)->getType() << " ";
2673 this->dumpSources(Func);
2674 }
2675
2676 template <typename TraitsType>
emit(const Cfg * Func)2677 void InstImpl<TraitsType>::InstX86Fstp::emit(const Cfg *Func) const {
2678 if (!BuildDefs::dump())
2679 return;
2680 Ostream &Str = Func->getContext()->getStrEmit();
2681 assert(this->getSrcSize() == 0);
2682 // TODO(jvoung,stichnot): Utilize this by setting Dest to nullptr to
2683 // "partially" delete the fstp if the Dest is unused. Even if Dest is unused,
2684 // the fstp should be kept for the SideEffects of popping the stack.
2685 if (!this->getDest()) {
2686 Str << "\t"
2687 "fstp\t"
2688 "st(0)";
2689 return;
2690 }
2691 Type Ty = this->getDest()->getType();
2692 if (!this->getDest()->hasReg()) {
2693 Str << "\t"
2694 "fstp"
2695 << this->getFldString(Ty) << "\t";
2696 this->getDest()->emit(Func);
2697 return;
2698 }
2699 // Dest is a physical (xmm) register, so st(0) needs to go through memory.
2700 // Hack this by using caller-reserved memory at the top of stack, spilling
2701 // st(0) there, and loading it into the xmm register.
2702 Str << "\t"
2703 "fstp"
2704 << this->getFldString(Ty)
2705 << "\t"
2706 "(%esp)\n";
2707 Str << "\t"
2708 "mov"
2709 << Traits::TypeAttributes[Ty].SdSsString
2710 << "\t"
2711 "(%esp), ";
2712 this->getDest()->emit(Func);
2713 }
2714
2715 template <typename TraitsType>
emitIAS(const Cfg * Func)2716 void InstImpl<TraitsType>::InstX86Fstp::emitIAS(const Cfg *Func) const {
2717 Assembler *Asm = Func->getAssembler<Assembler>();
2718 assert(this->getSrcSize() == 0);
2719 const Variable *Dest = this->getDest();
2720 // TODO(jvoung,stichnot): Utilize this by setting Dest to nullptr to
2721 // "partially" delete the fstp if the Dest is unused. Even if Dest is unused,
2722 // the fstp should be kept for the SideEffects of popping the stack.
2723 if (!Dest) {
2724 Asm->fstp(RegisterSet::getEncodedSTReg(0));
2725 return;
2726 }
2727 auto *Target = InstX86Base::getTarget(Func);
2728 Type Ty = Dest->getType();
2729 if (!Dest->hasReg()) {
2730 Address StackAddr(Target->stackVarToAsmOperand(Dest));
2731 Asm->fstp(Ty, StackAddr);
2732 } else {
2733 // Dest is a physical (xmm) register, so st(0) needs to go through memory.
2734 // Hack this by using caller-reserved memory at the top of stack, spilling
2735 // st(0) there, and loading it into the xmm register.
2736 Address StackSlot =
2737 Address(RegisterSet::Encoded_Reg_esp, 0, AssemblerFixup::NoFixup);
2738 Asm->fstp(Ty, StackSlot);
2739 Asm->movss(Ty, Traits::getEncodedXmm(Dest->getRegNum()), StackSlot);
2740 }
2741 }
2742
2743 template <typename TraitsType>
dump(const Cfg * Func)2744 void InstImpl<TraitsType>::InstX86Fstp::dump(const Cfg *Func) const {
2745 if (!BuildDefs::dump())
2746 return;
2747 Ostream &Str = Func->getContext()->getStrDump();
2748 this->dumpDest(Func);
2749 Str << " = fstp." << this->getDest()->getType() << ", st(0)";
2750 }
2751
2752 template <typename TraitsType>
emit(const Cfg * Func)2753 void InstImpl<TraitsType>::InstX86Pextr::emit(const Cfg *Func) const {
2754 if (!BuildDefs::dump())
2755 return;
2756 Ostream &Str = Func->getContext()->getStrEmit();
2757 assert(this->getSrcSize() == 2);
2758 // pextrb and pextrd are SSE4.1 instructions.
2759 Str << "\t" << this->Opcode
2760 << Traits::TypeAttributes[this->getSrc(0)->getType()].IntegralString
2761 << "\t";
2762 this->getSrc(1)->emit(Func);
2763 Str << ", ";
2764 this->getSrc(0)->emit(Func);
2765 Str << ", ";
2766 Variable *Dest = this->getDest();
2767 // pextrw must take a register dest. There is an SSE4.1 version that takes a
2768 // memory dest, but we aren't using it. For uniformity, just restrict them
2769 // all to have a register dest for now.
2770 assert(Dest->hasReg());
2771 Dest->asType(Func, IceType_i32, Dest->getRegNum())->emit(Func);
2772 }
2773
2774 template <typename TraitsType>
emitIAS(const Cfg * Func)2775 void InstImpl<TraitsType>::InstX86Pextr::emitIAS(const Cfg *Func) const {
2776 assert(this->getSrcSize() == 2);
2777 // pextrb and pextrd are SSE4.1 instructions.
2778 const Variable *Dest = this->getDest();
2779 Type DispatchTy = Traits::getInVectorElementType(this->getSrc(0)->getType());
2780 // pextrw must take a register dest. There is an SSE4.1 version that takes a
2781 // memory dest, but we aren't using it. For uniformity, just restrict them
2782 // all to have a register dest for now.
2783 assert(Dest->hasReg());
2784 // pextrw's Src(0) must be a register (both SSE4.1 and SSE2).
2785 assert(llvm::cast<Variable>(this->getSrc(0))->hasReg());
2786 static const ThreeOpImmEmitter<GPRRegister, XmmRegister> Emitter = {
2787 &Assembler::pextr, nullptr};
2788 emitIASThreeOpImmOps<GPRRegister, XmmRegister, Traits::getEncodedGPR,
2789 Traits::getEncodedXmm>(
2790 Func, DispatchTy, Dest, this->getSrc(0), this->getSrc(1), Emitter);
2791 }
2792
2793 template <typename TraitsType>
emit(const Cfg * Func)2794 void InstImpl<TraitsType>::InstX86Pinsr::emit(const Cfg *Func) const {
2795 if (!BuildDefs::dump())
2796 return;
2797 Ostream &Str = Func->getContext()->getStrEmit();
2798 assert(this->getSrcSize() == 3);
2799 Str << "\t" << this->Opcode
2800 << Traits::TypeAttributes[this->getDest()->getType()].IntegralString
2801 << "\t";
2802 this->getSrc(2)->emit(Func);
2803 Str << ", ";
2804 Operand *Src1 = this->getSrc(1);
2805 if (const auto *Src1Var = llvm::dyn_cast<Variable>(Src1)) {
2806 // If src1 is a register, it should always be r32.
2807 if (Src1Var->hasReg()) {
2808 const auto NewRegNum = Traits::getBaseReg(Src1Var->getRegNum());
2809 const Variable *NewSrc = Src1Var->asType(Func, IceType_i32, NewRegNum);
2810 NewSrc->emit(Func);
2811 } else {
2812 Src1Var->emit(Func);
2813 }
2814 } else {
2815 Src1->emit(Func);
2816 }
2817 Str << ", ";
2818 this->getDest()->emit(Func);
2819 }
2820
2821 template <typename TraitsType>
emitIAS(const Cfg * Func)2822 void InstImpl<TraitsType>::InstX86Pinsr::emitIAS(const Cfg *Func) const {
2823 assert(this->getSrcSize() == 3);
2824 assert(this->getDest() == this->getSrc(0));
2825 // pinsrb and pinsrd are SSE4.1 instructions.
2826 const Operand *Src0 = this->getSrc(1);
2827 Type DispatchTy = Src0->getType();
2828 // If src1 is a register, it should always be r32 (this should fall out from
2829 // the encodings for ByteRegs overlapping the encodings for r32), but we have
2830 // to make sure the register allocator didn't choose an 8-bit high register
2831 // like "ah".
2832 if (BuildDefs::asserts()) {
2833 if (auto *Src0Var = llvm::dyn_cast<Variable>(Src0)) {
2834 if (Src0Var->hasReg()) {
2835 const auto RegNum = Src0Var->getRegNum();
2836 const auto BaseRegNum = Traits::getBaseReg(RegNum);
2837 (void)BaseRegNum;
2838 assert(Traits::getEncodedGPR(RegNum) ==
2839 Traits::getEncodedGPR(BaseRegNum));
2840 }
2841 }
2842 }
2843 static const ThreeOpImmEmitter<XmmRegister, GPRRegister> Emitter = {
2844 &Assembler::pinsr, &Assembler::pinsr};
2845 emitIASThreeOpImmOps<XmmRegister, GPRRegister, Traits::getEncodedXmm,
2846 Traits::getEncodedGPR>(Func, DispatchTy, this->getDest(),
2847 Src0, this->getSrc(2), Emitter);
2848 }
2849
2850 template <typename TraitsType>
emitIAS(const Cfg * Func)2851 void InstImpl<TraitsType>::InstX86Pshufd::emitIAS(const Cfg *Func) const {
2852 assert(this->getSrcSize() == 2);
2853 const Variable *Dest = this->getDest();
2854 Type Ty = Dest->getType();
2855 static const ThreeOpImmEmitter<XmmRegister, XmmRegister> Emitter = {
2856 &Assembler::pshufd, &Assembler::pshufd};
2857 emitIASThreeOpImmOps<XmmRegister, XmmRegister, Traits::getEncodedXmm,
2858 Traits::getEncodedXmm>(Func, Ty, Dest, this->getSrc(0),
2859 this->getSrc(1), Emitter);
2860 }
2861
2862 template <typename TraitsType>
emitIAS(const Cfg * Func)2863 void InstImpl<TraitsType>::InstX86Shufps::emitIAS(const Cfg *Func) const {
2864 assert(this->getSrcSize() == 3);
2865 const Variable *Dest = this->getDest();
2866 assert(Dest == this->getSrc(0));
2867 Type Ty = Dest->getType();
2868 static const ThreeOpImmEmitter<XmmRegister, XmmRegister> Emitter = {
2869 &Assembler::shufps, &Assembler::shufps};
2870 emitIASThreeOpImmOps<XmmRegister, XmmRegister, Traits::getEncodedXmm,
2871 Traits::getEncodedXmm>(Func, Ty, Dest, this->getSrc(1),
2872 this->getSrc(2), Emitter);
2873 }
2874
2875 template <typename TraitsType>
emit(const Cfg * Func)2876 void InstImpl<TraitsType>::InstX86Pop::emit(const Cfg *Func) const {
2877 if (!BuildDefs::dump())
2878 return;
2879 Ostream &Str = Func->getContext()->getStrEmit();
2880 assert(this->getSrcSize() == 0);
2881 Str << "\t"
2882 "pop\t";
2883 this->getDest()->emit(Func);
2884 }
2885
2886 template <typename TraitsType>
emitIAS(const Cfg * Func)2887 void InstImpl<TraitsType>::InstX86Pop::emitIAS(const Cfg *Func) const {
2888 assert(this->getSrcSize() == 0);
2889 Assembler *Asm = Func->getAssembler<Assembler>();
2890 if (this->getDest()->hasReg()) {
2891 Asm->popl(Traits::getEncodedGPR(this->getDest()->getRegNum()));
2892 } else {
2893 auto *Target = InstX86Base::getTarget(Func);
2894 Asm->popl(Target->stackVarToAsmOperand(this->getDest()));
2895 }
2896 }
2897
2898 template <typename TraitsType>
dump(const Cfg * Func)2899 void InstImpl<TraitsType>::InstX86Pop::dump(const Cfg *Func) const {
2900 if (!BuildDefs::dump())
2901 return;
2902 Ostream &Str = Func->getContext()->getStrDump();
2903 this->dumpDest(Func);
2904 Str << " = pop." << this->getDest()->getType() << " ";
2905 }
2906
2907 template <typename TraitsType>
emit(const Cfg * Func)2908 void InstImpl<TraitsType>::InstX86Push::emit(const Cfg *Func) const {
2909 if (!BuildDefs::dump())
2910 return;
2911 Ostream &Str = Func->getContext()->getStrEmit();
2912 Str << "\t"
2913 "push"
2914 "\t";
2915 assert(this->getSrcSize() == 1);
2916 const Operand *Src = this->getSrc(0);
2917 Src->emit(Func);
2918 }
2919
2920 template <typename TraitsType>
emitIAS(const Cfg * Func)2921 void InstImpl<TraitsType>::InstX86Push::emitIAS(const Cfg *Func) const {
2922 Assembler *Asm = Func->getAssembler<Assembler>();
2923
2924 assert(this->getSrcSize() == 1);
2925 const Operand *Src = this->getSrc(0);
2926
2927 if (const auto *Var = llvm::dyn_cast<Variable>(Src)) {
2928 Asm->pushl(Traits::getEncodedGPR(Var->getRegNum()));
2929 } else if (const auto *Const32 = llvm::dyn_cast<ConstantInteger32>(Src)) {
2930 Asm->pushl(AssemblerImmediate(Const32->getValue()));
2931 } else if (auto *CR = llvm::dyn_cast<ConstantRelocatable>(Src)) {
2932 Asm->pushl(CR);
2933 } else {
2934 llvm_unreachable("Unexpected operand type");
2935 }
2936 }
2937
2938 template <typename TraitsType>
dump(const Cfg * Func)2939 void InstImpl<TraitsType>::InstX86Push::dump(const Cfg *Func) const {
2940 if (!BuildDefs::dump())
2941 return;
2942 Ostream &Str = Func->getContext()->getStrDump();
2943 Str << "push." << this->getSrc(0)->getType() << " ";
2944 this->dumpSources(Func);
2945 }
2946
2947 template <typename TraitsType>
emit(const Cfg * Func)2948 void InstImpl<TraitsType>::InstX86Ret::emit(const Cfg *Func) const {
2949 if (!BuildDefs::dump())
2950 return;
2951 Ostream &Str = Func->getContext()->getStrEmit();
2952 Str << "\t"
2953 "ret";
2954 }
2955
2956 template <typename TraitsType>
emitIAS(const Cfg * Func)2957 void InstImpl<TraitsType>::InstX86Ret::emitIAS(const Cfg *Func) const {
2958 Assembler *Asm = Func->getAssembler<Assembler>();
2959 Asm->ret();
2960 }
2961
2962 template <typename TraitsType>
dump(const Cfg * Func)2963 void InstImpl<TraitsType>::InstX86Ret::dump(const Cfg *Func) const {
2964 if (!BuildDefs::dump())
2965 return;
2966 Ostream &Str = Func->getContext()->getStrDump();
2967 Type Ty =
2968 (this->getSrcSize() == 0 ? IceType_void : this->getSrc(0)->getType());
2969 Str << "ret." << Ty << " ";
2970 this->dumpSources(Func);
2971 }
2972
2973 template <typename TraitsType>
emit(const Cfg * Func)2974 void InstImpl<TraitsType>::InstX86Setcc::emit(const Cfg *Func) const {
2975 if (!BuildDefs::dump())
2976 return;
2977 Ostream &Str = Func->getContext()->getStrEmit();
2978 Str << "\t"
2979 "set"
2980 << Traits::InstBrAttributes[Condition].DisplayString << "\t";
2981 this->Dest->emit(Func);
2982 }
2983
2984 template <typename TraitsType>
emitIAS(const Cfg * Func)2985 void InstImpl<TraitsType>::InstX86Setcc::emitIAS(const Cfg *Func) const {
2986 assert(Condition != Cond::Br_None);
2987 assert(this->getDest()->getType() == IceType_i1);
2988 assert(this->getSrcSize() == 0);
2989 Assembler *Asm = Func->getAssembler<Assembler>();
2990 auto *Target = InstX86Base::getTarget(Func);
2991 if (this->getDest()->hasReg())
2992 Asm->setcc(Condition,
2993 Traits::getEncodedByteReg(this->getDest()->getRegNum()));
2994 else
2995 Asm->setcc(Condition, Target->stackVarToAsmOperand(this->getDest()));
2996 return;
2997 }
2998
2999 template <typename TraitsType>
dump(const Cfg * Func)3000 void InstImpl<TraitsType>::InstX86Setcc::dump(const Cfg *Func) const {
3001 if (!BuildDefs::dump())
3002 return;
3003 Ostream &Str = Func->getContext()->getStrDump();
3004 Str << "setcc." << Traits::InstBrAttributes[Condition].DisplayString << " ";
3005 this->dumpDest(Func);
3006 }
3007
3008 template <typename TraitsType>
emit(const Cfg * Func)3009 void InstImpl<TraitsType>::InstX86Xadd::emit(const Cfg *Func) const {
3010 if (!BuildDefs::dump())
3011 return;
3012 Ostream &Str = Func->getContext()->getStrEmit();
3013 if (this->Locked) {
3014 Str << "\t"
3015 "lock";
3016 }
3017 Str << "\t"
3018 "xadd"
3019 << this->getWidthString(this->getSrc(0)->getType()) << "\t";
3020 this->getSrc(1)->emit(Func);
3021 Str << ", ";
3022 this->getSrc(0)->emit(Func);
3023 }
3024
3025 template <typename TraitsType>
emitIAS(const Cfg * Func)3026 void InstImpl<TraitsType>::InstX86Xadd::emitIAS(const Cfg *Func) const {
3027 assert(this->getSrcSize() == 2);
3028 Assembler *Asm = Func->getAssembler<Assembler>();
3029 Type Ty = this->getSrc(0)->getType();
3030 const auto Mem = llvm::cast<X86OperandMem>(this->getSrc(0));
3031 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
3032 auto *Target = InstX86Base::getTarget(Func);
3033 const Address Addr = Mem->toAsmAddress(Asm, Target);
3034 const auto *VarReg = llvm::cast<Variable>(this->getSrc(1));
3035 assert(VarReg->hasReg());
3036 const GPRRegister Reg = Traits::getEncodedGPR(VarReg->getRegNum());
3037 Asm->xadd(Ty, Addr, Reg, this->Locked);
3038 }
3039
3040 template <typename TraitsType>
dump(const Cfg * Func)3041 void InstImpl<TraitsType>::InstX86Xadd::dump(const Cfg *Func) const {
3042 if (!BuildDefs::dump())
3043 return;
3044 Ostream &Str = Func->getContext()->getStrDump();
3045 if (this->Locked) {
3046 Str << "lock ";
3047 }
3048 Type Ty = this->getSrc(0)->getType();
3049 Str << "xadd." << Ty << " ";
3050 this->dumpSources(Func);
3051 }
3052
3053 template <typename TraitsType>
emit(const Cfg * Func)3054 void InstImpl<TraitsType>::InstX86Xchg::emit(const Cfg *Func) const {
3055 if (!BuildDefs::dump())
3056 return;
3057 Ostream &Str = Func->getContext()->getStrEmit();
3058 Str << "\t"
3059 "xchg"
3060 << this->getWidthString(this->getSrc(0)->getType()) << "\t";
3061 this->getSrc(1)->emit(Func);
3062 Str << ", ";
3063 this->getSrc(0)->emit(Func);
3064 }
3065
3066 template <typename TraitsType>
emitIAS(const Cfg * Func)3067 void InstImpl<TraitsType>::InstX86Xchg::emitIAS(const Cfg *Func) const {
3068 assert(this->getSrcSize() == 2);
3069 Assembler *Asm = Func->getAssembler<Assembler>();
3070 Type Ty = this->getSrc(0)->getType();
3071 const auto *VarReg1 = llvm::cast<Variable>(this->getSrc(1));
3072 assert(VarReg1->hasReg());
3073 const GPRRegister Reg1 = Traits::getEncodedGPR(VarReg1->getRegNum());
3074
3075 if (const auto *VarReg0 = llvm::dyn_cast<Variable>(this->getSrc(0))) {
3076 assert(VarReg0->hasReg());
3077 const GPRRegister Reg0 = Traits::getEncodedGPR(VarReg0->getRegNum());
3078 Asm->xchg(Ty, Reg0, Reg1);
3079 return;
3080 }
3081
3082 const auto *Mem = llvm::cast<X86OperandMem>(this->getSrc(0));
3083 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
3084 auto *Target = InstX86Base::getTarget(Func);
3085 const Address Addr = Mem->toAsmAddress(Asm, Target);
3086 Asm->xchg(Ty, Addr, Reg1);
3087 }
3088
3089 template <typename TraitsType>
dump(const Cfg * Func)3090 void InstImpl<TraitsType>::InstX86Xchg::dump(const Cfg *Func) const {
3091 if (!BuildDefs::dump())
3092 return;
3093 Ostream &Str = Func->getContext()->getStrDump();
3094 Type Ty = this->getSrc(0)->getType();
3095 Str << "xchg." << Ty << " ";
3096 this->dumpSources(Func);
3097 }
3098
3099 template <typename TraitsType>
emit(const Cfg * Func)3100 void InstImpl<TraitsType>::InstX86IacaStart::emit(const Cfg *Func) const {
3101 if (!BuildDefs::dump())
3102 return;
3103 Ostream &Str = Func->getContext()->getStrEmit();
3104 Str << "\t# IACA_START\n"
3105 "\t.byte 0x0F, 0x0B\n"
3106 "\t"
3107 "movl\t$111, %ebx\n"
3108 "\t.byte 0x64, 0x67, 0x90";
3109 }
3110
3111 template <typename TraitsType>
emitIAS(const Cfg * Func)3112 void InstImpl<TraitsType>::InstX86IacaStart::emitIAS(const Cfg *Func) const {
3113 Assembler *Asm = Func->getAssembler<Assembler>();
3114 Asm->iaca_start();
3115 }
3116
3117 template <typename TraitsType>
dump(const Cfg * Func)3118 void InstImpl<TraitsType>::InstX86IacaStart::dump(const Cfg *Func) const {
3119 if (!BuildDefs::dump())
3120 return;
3121 Ostream &Str = Func->getContext()->getStrDump();
3122 Str << "IACA_START";
3123 }
3124
3125 template <typename TraitsType>
emit(const Cfg * Func)3126 void InstImpl<TraitsType>::InstX86IacaEnd::emit(const Cfg *Func) const {
3127 if (!BuildDefs::dump())
3128 return;
3129 Ostream &Str = Func->getContext()->getStrEmit();
3130 Str << "\t# IACA_END\n"
3131 "\t"
3132 "movl\t$222, %ebx\n"
3133 "\t.byte 0x64, 0x67, 0x90\n"
3134 "\t.byte 0x0F, 0x0B";
3135 }
3136
3137 template <typename TraitsType>
emitIAS(const Cfg * Func)3138 void InstImpl<TraitsType>::InstX86IacaEnd::emitIAS(const Cfg *Func) const {
3139 Assembler *Asm = Func->getAssembler<Assembler>();
3140 Asm->iaca_end();
3141 }
3142
3143 template <typename TraitsType>
dump(const Cfg * Func)3144 void InstImpl<TraitsType>::InstX86IacaEnd::dump(const Cfg *Func) const {
3145 if (!BuildDefs::dump())
3146 return;
3147 Ostream &Str = Func->getContext()->getStrDump();
3148 Str << "IACA_END";
3149 }
3150
3151 } // end of namespace X86NAMESPACE
3152
3153 } // end of namespace Ice
3154
3155 #endif // SUBZERO_SRC_ICEINSTX86BASEIMPL_H
3156