1 //===- subzero/src/IceInstX86BaseImpl.h - Generic X86 instructions -*- C++ -*=//
2 //
3 // The Subzero Code Generator
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief Implements the InstX86Base class and its descendants.
12 ///
13 //===----------------------------------------------------------------------===//
14
15 #ifndef SUBZERO_SRC_ICEINSTX86BASEIMPL_H
16 #define SUBZERO_SRC_ICEINSTX86BASEIMPL_H
17
18 #include "IceInstX86Base.h"
19
20 #include "IceAssemblerX86Base.h"
21 #include "IceCfg.h"
22 #include "IceCfgNode.h"
23 #include "IceDefs.h"
24 #include "IceInst.h"
25 #include "IceOperand.h"
26 #include "IceTargetLowering.h"
27 #include "IceTargetLoweringX86Base.h"
28
29 namespace Ice {
30
31 namespace X86NAMESPACE {
32
33 template <typename TraitsType>
getWidthString(Type Ty)34 const char *InstImpl<TraitsType>::InstX86Base::getWidthString(Type Ty) {
35 return Traits::TypeAttributes[Ty].WidthString;
36 }
37
38 template <typename TraitsType>
getFldString(Type Ty)39 const char *InstImpl<TraitsType>::InstX86Base::getFldString(Type Ty) {
40 return Traits::TypeAttributes[Ty].FldString;
41 }
42
43 template <typename TraitsType>
44 typename InstImpl<TraitsType>::Cond::BrCond
getOppositeCondition(BrCond Cond)45 InstImpl<TraitsType>::InstX86Base::getOppositeCondition(BrCond Cond) {
46 return Traits::InstBrAttributes[Cond].Opposite;
47 }
48
49 template <typename TraitsType>
InstX86FakeRMW(Cfg * Func,Operand * Data,Operand * Addr,InstArithmetic::OpKind Op,Variable * Beacon)50 InstImpl<TraitsType>::InstX86FakeRMW::InstX86FakeRMW(Cfg *Func, Operand *Data,
51 Operand *Addr,
52 InstArithmetic::OpKind Op,
53 Variable *Beacon)
54 : InstX86Base(Func, InstX86Base::FakeRMW, 3, nullptr), Op(Op) {
55 this->addSource(Data);
56 this->addSource(Addr);
57 this->addSource(Beacon);
58 }
59
60 template <typename TraitsType>
InstX86GetIP(Cfg * Func,Variable * Dest)61 InstImpl<TraitsType>::InstX86GetIP::InstX86GetIP(Cfg *Func, Variable *Dest)
62 : InstX86Base(Func, InstX86Base::GetIP, 0, Dest) {}
63
64 template <typename TraitsType>
InstX86Mul(Cfg * Func,Variable * Dest,Variable * Source1,Operand * Source2)65 InstImpl<TraitsType>::InstX86Mul::InstX86Mul(Cfg *Func, Variable *Dest,
66 Variable *Source1,
67 Operand *Source2)
68 : InstX86Base(Func, InstX86Base::Mul, 2, Dest) {
69 this->addSource(Source1);
70 this->addSource(Source2);
71 }
72
73 template <typename TraitsType>
InstX86Shld(Cfg * Func,Variable * Dest,Variable * Source1,Operand * Source2)74 InstImpl<TraitsType>::InstX86Shld::InstX86Shld(Cfg *Func, Variable *Dest,
75 Variable *Source1,
76 Operand *Source2)
77 : InstX86Base(Func, InstX86Base::Shld, 3, Dest) {
78 this->addSource(Dest);
79 this->addSource(Source1);
80 this->addSource(Source2);
81 }
82
83 template <typename TraitsType>
InstX86Shrd(Cfg * Func,Variable * Dest,Variable * Source1,Operand * Source2)84 InstImpl<TraitsType>::InstX86Shrd::InstX86Shrd(Cfg *Func, Variable *Dest,
85 Variable *Source1,
86 Operand *Source2)
87 : InstX86Base(Func, InstX86Base::Shrd, 3, Dest) {
88 this->addSource(Dest);
89 this->addSource(Source1);
90 this->addSource(Source2);
91 }
92
93 template <typename TraitsType>
InstX86Label(Cfg * Func,TargetLowering * Target)94 InstImpl<TraitsType>::InstX86Label::InstX86Label(Cfg *Func,
95 TargetLowering *Target)
96 : InstX86Base(Func, InstX86Base::Label, 0, nullptr),
97 LabelNumber(Target->makeNextLabelNumber()) {
98 if (BuildDefs::dump()) {
99 Name = GlobalString::createWithString(
100 Func->getContext(), ".L" + Func->getFunctionName() + "$local$__" +
101 std::to_string(LabelNumber));
102 } else {
103 Name = GlobalString::createWithoutString(Func->getContext());
104 }
105 }
106
107 template <typename TraitsType>
InstX86Br(Cfg * Func,const CfgNode * TargetTrue,const CfgNode * TargetFalse,const InstX86Label * Label,BrCond Condition,Mode Kind)108 InstImpl<TraitsType>::InstX86Br::InstX86Br(Cfg *Func, const CfgNode *TargetTrue,
109 const CfgNode *TargetFalse,
110 const InstX86Label *Label,
111 BrCond Condition, Mode Kind)
112 : InstX86Base(Func, InstX86Base::Br, 0, nullptr), Condition(Condition),
113 TargetTrue(TargetTrue), TargetFalse(TargetFalse), Label(Label),
114 Kind(Kind) {}
115
116 template <typename TraitsType>
optimizeBranch(const CfgNode * NextNode)117 bool InstImpl<TraitsType>::InstX86Br::optimizeBranch(const CfgNode *NextNode) {
118 // If there is no next block, then there can be no fallthrough to optimize.
119 if (NextNode == nullptr)
120 return false;
121 // Intra-block conditional branches can't be optimized.
122 if (Label)
123 return false;
124 // If there is no fallthrough node, such as a non-default case label for a
125 // switch instruction, then there is no opportunity to optimize.
126 if (getTargetFalse() == nullptr)
127 return false;
128
129 // Unconditional branch to the next node can be removed.
130 if (Condition == Cond::Br_None && getTargetFalse() == NextNode) {
131 assert(getTargetTrue() == nullptr);
132 this->setDeleted();
133 return true;
134 }
135 // If the fallthrough is to the next node, set fallthrough to nullptr to
136 // indicate.
137 if (getTargetFalse() == NextNode) {
138 TargetFalse = nullptr;
139 return true;
140 }
141 // If TargetTrue is the next node, and TargetFalse is not nullptr (which was
142 // already tested above), then invert the branch condition, swap the targets,
143 // and set new fallthrough to nullptr.
144 if (getTargetTrue() == NextNode) {
145 assert(Condition != Cond::Br_None);
146 Condition = this->getOppositeCondition(Condition);
147 TargetTrue = getTargetFalse();
148 TargetFalse = nullptr;
149 return true;
150 }
151 return false;
152 }
153
154 template <typename TraitsType>
repointEdges(CfgNode * OldNode,CfgNode * NewNode)155 bool InstImpl<TraitsType>::InstX86Br::repointEdges(CfgNode *OldNode,
156 CfgNode *NewNode) {
157 bool Found = false;
158 if (TargetFalse == OldNode) {
159 TargetFalse = NewNode;
160 Found = true;
161 }
162 if (TargetTrue == OldNode) {
163 TargetTrue = NewNode;
164 Found = true;
165 }
166 return Found;
167 }
168
169 template <typename TraitsType>
InstX86Jmp(Cfg * Func,Operand * Target)170 InstImpl<TraitsType>::InstX86Jmp::InstX86Jmp(Cfg *Func, Operand *Target)
171 : InstX86Base(Func, InstX86Base::Jmp, 1, nullptr) {
172 this->addSource(Target);
173 }
174
175 template <typename TraitsType>
InstX86Call(Cfg * Func,Variable * Dest,Operand * CallTarget)176 InstImpl<TraitsType>::InstX86Call::InstX86Call(Cfg *Func, Variable *Dest,
177 Operand *CallTarget)
178 : InstX86Base(Func, InstX86Base::Call, 1, Dest) {
179 this->HasSideEffects = true;
180 this->addSource(CallTarget);
181 }
182
183 template <typename TraitsType>
InstX86Movmsk(Cfg * Func,Variable * Dest,Operand * Source)184 InstImpl<TraitsType>::InstX86Movmsk::InstX86Movmsk(Cfg *Func, Variable *Dest,
185 Operand *Source)
186 : InstX86Base(Func, InstX86Base::Movmsk, 1, Dest) {
187 this->addSource(Source);
188 }
189
190 template <typename TraitsType>
InstX86Cmov(Cfg * Func,Variable * Dest,Operand * Source,BrCond Condition)191 InstImpl<TraitsType>::InstX86Cmov::InstX86Cmov(Cfg *Func, Variable *Dest,
192 Operand *Source,
193 BrCond Condition)
194 : InstX86Base(Func, InstX86Base::Cmov, 2, Dest), Condition(Condition) {
195 // The final result is either the original Dest, or Source, so mark both as
196 // sources.
197 this->addSource(Dest);
198 this->addSource(Source);
199 }
200
201 template <typename TraitsType>
InstX86Cmpps(Cfg * Func,Variable * Dest,Operand * Source,CmppsCond Condition)202 InstImpl<TraitsType>::InstX86Cmpps::InstX86Cmpps(Cfg *Func, Variable *Dest,
203 Operand *Source,
204 CmppsCond Condition)
205 : InstX86Base(Func, InstX86Base::Cmpps, 2, Dest), Condition(Condition) {
206 this->addSource(Dest);
207 this->addSource(Source);
208 }
209
210 template <typename TraitsType>
InstX86Cmpxchg(Cfg * Func,Operand * DestOrAddr,Variable * Eax,Variable * Desired,bool Locked)211 InstImpl<TraitsType>::InstX86Cmpxchg::InstX86Cmpxchg(Cfg *Func,
212 Operand *DestOrAddr,
213 Variable *Eax,
214 Variable *Desired,
215 bool Locked)
216 : InstImpl<TraitsType>::InstX86BaseLockable(
217 Func, InstX86Base::Cmpxchg, 3, llvm::dyn_cast<Variable>(DestOrAddr),
218 Locked) {
219 constexpr uint16_t Encoded_rAX = 0;
220 (void)Encoded_rAX;
221 assert(Traits::getEncodedGPR(Eax->getRegNum()) == Encoded_rAX);
222 this->addSource(DestOrAddr);
223 this->addSource(Eax);
224 this->addSource(Desired);
225 }
226
227 template <typename TraitsType>
InstX86Cmpxchg8b(Cfg * Func,X86OperandMem * Addr,Variable * Edx,Variable * Eax,Variable * Ecx,Variable * Ebx,bool Locked)228 InstImpl<TraitsType>::InstX86Cmpxchg8b::InstX86Cmpxchg8b(
229 Cfg *Func, X86OperandMem *Addr, Variable *Edx, Variable *Eax, Variable *Ecx,
230 Variable *Ebx, bool Locked)
231 : InstImpl<TraitsType>::InstX86BaseLockable(Func, InstX86Base::Cmpxchg, 5,
232 nullptr, Locked) {
233 assert(Edx->getRegNum() == RegisterSet::Reg_edx);
234 assert(Eax->getRegNum() == RegisterSet::Reg_eax);
235 assert(Ecx->getRegNum() == RegisterSet::Reg_ecx);
236 assert(Ebx->getRegNum() == RegisterSet::Reg_ebx);
237 this->addSource(Addr);
238 this->addSource(Edx);
239 this->addSource(Eax);
240 this->addSource(Ecx);
241 this->addSource(Ebx);
242 }
243
244 template <typename TraitsType>
InstX86Cvt(Cfg * Func,Variable * Dest,Operand * Source,CvtVariant Variant)245 InstImpl<TraitsType>::InstX86Cvt::InstX86Cvt(Cfg *Func, Variable *Dest,
246 Operand *Source,
247 CvtVariant Variant)
248 : InstX86Base(Func, InstX86Base::Cvt, 1, Dest), Variant(Variant) {
249 this->addSource(Source);
250 }
251
252 template <typename TraitsType>
InstX86Icmp(Cfg * Func,Operand * Src0,Operand * Src1)253 InstImpl<TraitsType>::InstX86Icmp::InstX86Icmp(Cfg *Func, Operand *Src0,
254 Operand *Src1)
255 : InstX86Base(Func, InstX86Base::Icmp, 2, nullptr) {
256 this->addSource(Src0);
257 this->addSource(Src1);
258 }
259
260 template <typename TraitsType>
InstX86Ucomiss(Cfg * Func,Operand * Src0,Operand * Src1)261 InstImpl<TraitsType>::InstX86Ucomiss::InstX86Ucomiss(Cfg *Func, Operand *Src0,
262 Operand *Src1)
263 : InstX86Base(Func, InstX86Base::Ucomiss, 2, nullptr) {
264 this->addSource(Src0);
265 this->addSource(Src1);
266 }
267
268 template <typename TraitsType>
InstX86UD2(Cfg * Func)269 InstImpl<TraitsType>::InstX86UD2::InstX86UD2(Cfg *Func)
270 : InstX86Base(Func, InstX86Base::UD2, 0, nullptr) {}
271
272 template <typename TraitsType>
InstX86Int3(Cfg * Func)273 InstImpl<TraitsType>::InstX86Int3::InstX86Int3(Cfg *Func)
274 : InstX86Base(Func, InstX86Base::Int3, 0, nullptr) {}
275
276 template <typename TraitsType>
InstX86Test(Cfg * Func,Operand * Src1,Operand * Src2)277 InstImpl<TraitsType>::InstX86Test::InstX86Test(Cfg *Func, Operand *Src1,
278 Operand *Src2)
279 : InstX86Base(Func, InstX86Base::Test, 2, nullptr) {
280 this->addSource(Src1);
281 this->addSource(Src2);
282 }
283
284 template <typename TraitsType>
InstX86Mfence(Cfg * Func)285 InstImpl<TraitsType>::InstX86Mfence::InstX86Mfence(Cfg *Func)
286 : InstX86Base(Func, InstX86Base::Mfence, 0, nullptr) {
287 this->HasSideEffects = true;
288 }
289
290 template <typename TraitsType>
InstX86Store(Cfg * Func,Operand * Value,X86Operand * Mem)291 InstImpl<TraitsType>::InstX86Store::InstX86Store(Cfg *Func, Operand *Value,
292 X86Operand *Mem)
293 : InstX86Base(Func, InstX86Base::Store, 2, nullptr) {
294 this->addSource(Value);
295 this->addSource(Mem);
296 }
297
298 template <typename TraitsType>
InstX86StoreP(Cfg * Func,Variable * Value,X86OperandMem * Mem)299 InstImpl<TraitsType>::InstX86StoreP::InstX86StoreP(Cfg *Func, Variable *Value,
300 X86OperandMem *Mem)
301 : InstX86Base(Func, InstX86Base::StoreP, 2, nullptr) {
302 this->addSource(Value);
303 this->addSource(Mem);
304 }
305
306 template <typename TraitsType>
InstX86StoreQ(Cfg * Func,Operand * Value,X86OperandMem * Mem)307 InstImpl<TraitsType>::InstX86StoreQ::InstX86StoreQ(Cfg *Func, Operand *Value,
308 X86OperandMem *Mem)
309 : InstX86Base(Func, InstX86Base::StoreQ, 2, nullptr) {
310 this->addSource(Value);
311 this->addSource(Mem);
312 }
313
314 template <typename TraitsType>
InstX86StoreD(Cfg * Func,Operand * Value,X86OperandMem * Mem)315 InstImpl<TraitsType>::InstX86StoreD::InstX86StoreD(Cfg *Func, Operand *Value,
316 X86OperandMem *Mem)
317 : InstX86Base(Func, InstX86Base::StoreD, 2, nullptr) {
318 this->addSource(Value);
319 this->addSource(Mem);
320 }
321
322 template <typename TraitsType>
InstX86Nop(Cfg * Func,NopVariant Variant)323 InstImpl<TraitsType>::InstX86Nop::InstX86Nop(Cfg *Func, NopVariant Variant)
324 : InstX86Base(Func, InstX86Base::Nop, 0, nullptr), Variant(Variant) {}
325
326 template <typename TraitsType>
InstX86Fld(Cfg * Func,Operand * Src)327 InstImpl<TraitsType>::InstX86Fld::InstX86Fld(Cfg *Func, Operand *Src)
328 : InstX86Base(Func, InstX86Base::Fld, 1, nullptr) {
329 this->addSource(Src);
330 }
331
332 template <typename TraitsType>
InstX86Fstp(Cfg * Func,Variable * Dest)333 InstImpl<TraitsType>::InstX86Fstp::InstX86Fstp(Cfg *Func, Variable *Dest)
334 : InstX86Base(Func, InstX86Base::Fstp, 0, Dest) {}
335
336 template <typename TraitsType>
InstX86Pop(Cfg * Func,Variable * Dest)337 InstImpl<TraitsType>::InstX86Pop::InstX86Pop(Cfg *Func, Variable *Dest)
338 : InstX86Base(Func, InstX86Base::Pop, 0, Dest) {
339 // A pop instruction affects the stack pointer and so it should not be
340 // allowed to be automatically dead-code eliminated. (The corresponding push
341 // instruction doesn't need this treatment because it has no dest variable
342 // and therefore won't be dead-code eliminated.) This is needed for
343 // late-stage liveness analysis (e.g. asm-verbose mode).
344 this->HasSideEffects = true;
345 }
346
347 template <typename TraitsType>
InstX86Push(Cfg * Func,Operand * Source)348 InstImpl<TraitsType>::InstX86Push::InstX86Push(Cfg *Func, Operand *Source)
349 : InstX86Base(Func, InstX86Base::Push, 1, nullptr) {
350 this->addSource(Source);
351 }
352
353 template <typename TraitsType>
InstX86Push(Cfg * Func,InstX86Label * L)354 InstImpl<TraitsType>::InstX86Push::InstX86Push(Cfg *Func, InstX86Label *L)
355 : InstX86Base(Func, InstX86Base::Push, 0, nullptr), Label(L) {}
356
357 template <typename TraitsType>
InstX86Ret(Cfg * Func,Variable * Source)358 InstImpl<TraitsType>::InstX86Ret::InstX86Ret(Cfg *Func, Variable *Source)
359 : InstX86Base(Func, InstX86Base::Ret, Source ? 1 : 0, nullptr) {
360 if (Source)
361 this->addSource(Source);
362 }
363
364 template <typename TraitsType>
InstX86Setcc(Cfg * Func,Variable * Dest,BrCond Cond)365 InstImpl<TraitsType>::InstX86Setcc::InstX86Setcc(Cfg *Func, Variable *Dest,
366 BrCond Cond)
367 : InstX86Base(Func, InstX86Base::Setcc, 0, Dest), Condition(Cond) {}
368
369 template <typename TraitsType>
InstX86Xadd(Cfg * Func,Operand * Dest,Variable * Source,bool Locked)370 InstImpl<TraitsType>::InstX86Xadd::InstX86Xadd(Cfg *Func, Operand *Dest,
371 Variable *Source, bool Locked)
372 : InstImpl<TraitsType>::InstX86BaseLockable(
373 Func, InstX86Base::Xadd, 2, llvm::dyn_cast<Variable>(Dest), Locked) {
374 this->addSource(Dest);
375 this->addSource(Source);
376 }
377
378 template <typename TraitsType>
InstX86Xchg(Cfg * Func,Operand * Dest,Variable * Source)379 InstImpl<TraitsType>::InstX86Xchg::InstX86Xchg(Cfg *Func, Operand *Dest,
380 Variable *Source)
381 : InstX86Base(Func, InstX86Base::Xchg, 2, llvm::dyn_cast<Variable>(Dest)) {
382 this->addSource(Dest);
383 this->addSource(Source);
384 }
385
386 template <typename TraitsType>
InstX86IacaStart(Cfg * Func)387 InstImpl<TraitsType>::InstX86IacaStart::InstX86IacaStart(Cfg *Func)
388 : InstX86Base(Func, InstX86Base::IacaStart, 0, nullptr) {
389 assert(getFlags().getAllowIacaMarks());
390 }
391
392 template <typename TraitsType>
InstX86IacaEnd(Cfg * Func)393 InstImpl<TraitsType>::InstX86IacaEnd::InstX86IacaEnd(Cfg *Func)
394 : InstX86Base(Func, InstX86Base::IacaEnd, 0, nullptr) {
395 assert(getFlags().getAllowIacaMarks());
396 }
397
398 // ======================== Dump routines ======================== //
399
400 template <typename TraitsType>
dump(const Cfg * Func)401 void InstImpl<TraitsType>::InstX86Base::dump(const Cfg *Func) const {
402 if (!BuildDefs::dump())
403 return;
404 Ostream &Str = Func->getContext()->getStrDump();
405 Str << "[" << Traits::TargetName << "] ";
406 Inst::dump(Func);
407 }
408
409 template <typename TraitsType>
dump(const Cfg * Func)410 void InstImpl<TraitsType>::InstX86FakeRMW::dump(const Cfg *Func) const {
411 if (!BuildDefs::dump())
412 return;
413 Ostream &Str = Func->getContext()->getStrDump();
414 Type Ty = getData()->getType();
415 Str << "rmw " << InstArithmetic::getOpName(getOp()) << " " << Ty << " *";
416 getAddr()->dump(Func);
417 Str << ", ";
418 getData()->dump(Func);
419 Str << ", beacon=";
420 getBeacon()->dump(Func);
421 }
422
423 template <typename TraitsType>
emit(const Cfg * Func)424 void InstImpl<TraitsType>::InstX86GetIP::emit(const Cfg *Func) const {
425 if (!BuildDefs::dump())
426 return;
427 const auto *Dest = this->getDest();
428 assert(Dest->hasReg());
429 Ostream &Str = Func->getContext()->getStrEmit();
430 Str << "\t"
431 "call"
432 "\t";
433 auto *Target = static_cast<TargetLowering *>(Func->getTarget());
434 Target->emitWithoutPrefix(Target->createGetIPForRegister(Dest));
435 }
436
437 template <typename TraitsType>
emitIAS(const Cfg * Func)438 void InstImpl<TraitsType>::InstX86GetIP::emitIAS(const Cfg *Func) const {
439 const auto *Dest = this->getDest();
440 Assembler *Asm = Func->getAssembler<Assembler>();
441 assert(Dest->hasReg());
442 Asm->call(static_cast<TargetLowering *>(Func->getTarget())
443 ->createGetIPForRegister(Dest));
444 }
445
446 template <typename TraitsType>
dump(const Cfg * Func)447 void InstImpl<TraitsType>::InstX86GetIP::dump(const Cfg *Func) const {
448 if (!BuildDefs::dump())
449 return;
450 Ostream &Str = Func->getContext()->getStrDump();
451 this->getDest()->dump(Func);
452 Str << " = call getIP";
453 }
454
455 template <typename TraitsType>
emit(const Cfg * Func)456 void InstImpl<TraitsType>::InstX86Label::emit(const Cfg *Func) const {
457 if (!BuildDefs::dump())
458 return;
459 Ostream &Str = Func->getContext()->getStrEmit();
460 Str << getLabelName() << ":";
461 }
462
463 template <typename TraitsType>
emitIAS(const Cfg * Func)464 void InstImpl<TraitsType>::InstX86Label::emitIAS(const Cfg *Func) const {
465 Assembler *Asm = Func->getAssembler<Assembler>();
466 Asm->bindLocalLabel(LabelNumber);
467 if (OffsetReloc != nullptr) {
468 Asm->bindRelocOffset(OffsetReloc);
469 }
470 }
471
472 template <typename TraitsType>
dump(const Cfg * Func)473 void InstImpl<TraitsType>::InstX86Label::dump(const Cfg *Func) const {
474 if (!BuildDefs::dump())
475 return;
476 Ostream &Str = Func->getContext()->getStrDump();
477 Str << getLabelName() << ":";
478 }
479
480 template <typename TraitsType>
emit(const Cfg * Func)481 void InstImpl<TraitsType>::InstX86Br::emit(const Cfg *Func) const {
482 if (!BuildDefs::dump())
483 return;
484 Ostream &Str = Func->getContext()->getStrEmit();
485 Str << "\t";
486
487 if (Condition == Cond::Br_None) {
488 Str << "jmp";
489 } else {
490 Str << Traits::InstBrAttributes[Condition].EmitString;
491 }
492
493 if (Label) {
494 Str << "\t" << Label->getLabelName();
495 } else {
496 if (Condition == Cond::Br_None) {
497 Str << "\t" << getTargetFalse()->getAsmName();
498 } else {
499 Str << "\t" << getTargetTrue()->getAsmName();
500 if (getTargetFalse()) {
501 Str << "\n\t"
502 "jmp\t" << getTargetFalse()->getAsmName();
503 }
504 }
505 }
506 }
507
508 template <typename TraitsType>
emitIAS(const Cfg * Func)509 void InstImpl<TraitsType>::InstX86Br::emitIAS(const Cfg *Func) const {
510 Assembler *Asm = Func->getAssembler<Assembler>();
511 if (Label) {
512 auto *L = Asm->getOrCreateLocalLabel(Label->getLabelNumber());
513 if (Condition == Cond::Br_None) {
514 Asm->jmp(L, isNear());
515 } else {
516 Asm->j(Condition, L, isNear());
517 }
518 } else {
519 if (Condition == Cond::Br_None) {
520 auto *L = Asm->getOrCreateCfgNodeLabel(getTargetFalse()->getIndex());
521 assert(!getTargetTrue());
522 Asm->jmp(L, isNear());
523 } else {
524 auto *L = Asm->getOrCreateCfgNodeLabel(getTargetTrue()->getIndex());
525 Asm->j(Condition, L, isNear());
526 if (getTargetFalse()) {
527 auto *L2 = Asm->getOrCreateCfgNodeLabel(getTargetFalse()->getIndex());
528 Asm->jmp(L2, isNear());
529 }
530 }
531 }
532 }
533
534 template <typename TraitsType>
dump(const Cfg * Func)535 void InstImpl<TraitsType>::InstX86Br::dump(const Cfg *Func) const {
536 if (!BuildDefs::dump())
537 return;
538 Ostream &Str = Func->getContext()->getStrDump();
539 Str << "br ";
540
541 if (Condition == Cond::Br_None) {
542 if (Label) {
543 Str << "label %" << Label->getLabelName();
544 } else {
545 Str << "label %" << getTargetFalse()->getName();
546 }
547 return;
548 }
549
550 Str << Traits::InstBrAttributes[Condition].DisplayString;
551 if (Label) {
552 Str << ", label %" << Label->getLabelName();
553 } else {
554 Str << ", label %" << getTargetTrue()->getName();
555 if (getTargetFalse()) {
556 Str << ", label %" << getTargetFalse()->getName();
557 }
558 }
559
560 Str << " // (" << (isNear() ? "near" : "far") << " jump)";
561 }
562
563 template <typename TraitsType>
emit(const Cfg * Func)564 void InstImpl<TraitsType>::InstX86Jmp::emit(const Cfg *Func) const {
565 if (!BuildDefs::dump())
566 return;
567 Ostream &Str = Func->getContext()->getStrEmit();
568 assert(this->getSrcSize() == 1);
569 const Operand *Src = this->getSrc(0);
570 if (Traits::Is64Bit) {
571 if (const auto *CR = llvm::dyn_cast<ConstantRelocatable>(Src)) {
572 Str << "\t"
573 "jmp"
574 "\t" << CR->getName();
575 return;
576 }
577 }
578 Str << "\t"
579 "jmp"
580 "\t*";
581 getJmpTarget()->emit(Func);
582 }
583
584 template <typename TraitsType>
emitIAS(const Cfg * Func)585 void InstImpl<TraitsType>::InstX86Jmp::emitIAS(const Cfg *Func) const {
586 // Note: Adapted (mostly copied) from
587 // InstImpl<TraitsType>::InstX86Call::emitIAS().
588 Assembler *Asm = Func->getAssembler<Assembler>();
589 Operand *Target = getJmpTarget();
590 if (const auto *Var = llvm::dyn_cast<Variable>(Target)) {
591 if (Var->hasReg()) {
592 Asm->jmp(Traits::getEncodedGPR(Var->getRegNum()));
593 } else {
594 // The jmp instruction with a memory operand should be possible to
595 // encode, but it isn't a valid sandboxed instruction, and there
596 // shouldn't be a register allocation issue to jump through a scratch
597 // register, so we don't really need to bother implementing it.
598 llvm::report_fatal_error("Assembler can't jmp to memory operand");
599 }
600 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Target)) {
601 (void)Mem;
602 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
603 llvm::report_fatal_error("Assembler can't jmp to memory operand");
604 } else if (const auto *CR = llvm::dyn_cast<ConstantRelocatable>(Target)) {
605 Asm->jmp(CR);
606 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Target)) {
607 // NaCl trampoline calls refer to an address within the sandbox directly.
608 // This is usually only needed for non-IRT builds and otherwise not very
609 // portable or stable. Usually this is only done for "calls" and not jumps.
610 Asm->jmp(AssemblerImmediate(Imm->getValue()));
611 } else {
612 llvm::report_fatal_error("Unexpected operand type");
613 }
614 }
615
616 template <typename TraitsType>
dump(const Cfg * Func)617 void InstImpl<TraitsType>::InstX86Jmp::dump(const Cfg *Func) const {
618 if (!BuildDefs::dump())
619 return;
620 Ostream &Str = Func->getContext()->getStrDump();
621 Str << "jmp ";
622 getJmpTarget()->dump(Func);
623 }
624
625 template <typename TraitsType>
emit(const Cfg * Func)626 void InstImpl<TraitsType>::InstX86Call::emit(const Cfg *Func) const {
627 if (!BuildDefs::dump())
628 return;
629 Ostream &Str = Func->getContext()->getStrEmit();
630 assert(this->getSrcSize() == 1);
631 Str << "\t"
632 "call\t";
633 Operand *CallTarget = getCallTarget();
634 auto *Target = InstX86Base::getTarget(Func);
635 if (const auto *CI = llvm::dyn_cast<ConstantInteger32>(CallTarget)) {
636 // Emit without a leading '$'.
637 Str << CI->getValue();
638 } else if (const auto DirectCallTarget =
639 llvm::dyn_cast<ConstantRelocatable>(CallTarget)) {
640 DirectCallTarget->emitWithoutPrefix(Target);
641 } else {
642 Str << "*";
643 CallTarget->emit(Func);
644 }
645 }
646
647 template <typename TraitsType>
emitIAS(const Cfg * Func)648 void InstImpl<TraitsType>::InstX86Call::emitIAS(const Cfg *Func) const {
649 Assembler *Asm = Func->getAssembler<Assembler>();
650 Operand *CallTarget = getCallTarget();
651 auto *Target = InstX86Base::getTarget(Func);
652 if (const auto *Var = llvm::dyn_cast<Variable>(CallTarget)) {
653 if (Var->hasReg()) {
654 Asm->call(Traits::getEncodedGPR(Var->getRegNum()));
655 } else {
656 Asm->call(Target->stackVarToAsmOperand(Var));
657 }
658 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(CallTarget)) {
659 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
660 Asm->call(Mem->toAsmAddress(Asm, Target));
661 } else if (const auto *CR = llvm::dyn_cast<ConstantRelocatable>(CallTarget)) {
662 Asm->call(CR);
663 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(CallTarget)) {
664 Asm->call(AssemblerImmediate(Imm->getValue()));
665 } else {
666 llvm_unreachable("Unexpected operand type");
667 }
668 }
669
670 template <typename TraitsType>
dump(const Cfg * Func)671 void InstImpl<TraitsType>::InstX86Call::dump(const Cfg *Func) const {
672 if (!BuildDefs::dump())
673 return;
674 Ostream &Str = Func->getContext()->getStrDump();
675 if (this->getDest()) {
676 this->dumpDest(Func);
677 Str << " = ";
678 }
679 Str << "call ";
680 getCallTarget()->dump(Func);
681 }
682
683 // The this->Opcode parameter needs to be char* and not std::string because of
684 // template issues.
685 template <typename TraitsType>
emitTwoAddress(const Cfg * Func,const char * Opcode,const char * Suffix)686 void InstImpl<TraitsType>::InstX86Base::emitTwoAddress(
687 const Cfg *Func, const char *Opcode, const char *Suffix) const {
688 if (!BuildDefs::dump())
689 return;
690 Ostream &Str = Func->getContext()->getStrEmit();
691 assert(getSrcSize() == 2);
692 Operand *Dest = getDest();
693 if (Dest == nullptr)
694 Dest = getSrc(0);
695 assert(Dest == getSrc(0));
696 Operand *Src1 = getSrc(1);
697 Str << "\t" << Opcode << Suffix
698 << InstX86Base::getWidthString(Dest->getType()) << "\t";
699 Src1->emit(Func);
700 Str << ", ";
701 Dest->emit(Func);
702 }
703
704 template <typename TraitsType>
emitIASOpTyGPR(const Cfg * Func,Type Ty,const Operand * Op,const GPREmitterOneOp & Emitter)705 void InstImpl<TraitsType>::emitIASOpTyGPR(const Cfg *Func, Type Ty,
706 const Operand *Op,
707 const GPREmitterOneOp &Emitter) {
708 auto *Target = InstX86Base::getTarget(Func);
709 Assembler *Asm = Func->getAssembler<Assembler>();
710 if (const auto *Var = llvm::dyn_cast<Variable>(Op)) {
711 if (Var->hasReg()) {
712 // We cheat a little and use GPRRegister even for byte operations.
713 GPRRegister VarReg = Traits::getEncodedGPR(Var->getRegNum());
714 (Asm->*(Emitter.Reg))(Ty, VarReg);
715 } else {
716 Address StackAddr(Target->stackVarToAsmOperand(Var));
717 (Asm->*(Emitter.Addr))(Ty, StackAddr);
718 }
719 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Op)) {
720 Mem->emitSegmentOverride(Asm);
721 (Asm->*(Emitter.Addr))(Ty, Mem->toAsmAddress(Asm, Target));
722 } else {
723 llvm_unreachable("Unexpected operand type");
724 }
725 }
726
727 template <typename TraitsType>
728 template <bool VarCanBeByte, bool SrcCanBeByte>
emitIASRegOpTyGPR(const Cfg * Func,bool IsLea,Type Ty,const Variable * Var,const Operand * Src,const GPREmitterRegOp & Emitter)729 void InstImpl<TraitsType>::emitIASRegOpTyGPR(const Cfg *Func, bool IsLea,
730 Type Ty, const Variable *Var,
731 const Operand *Src,
732 const GPREmitterRegOp &Emitter) {
733 auto *Target = InstX86Base::getTarget(Func);
734 Assembler *Asm = Func->getAssembler<Assembler>();
735 assert(Var->hasReg());
736 // We cheat a little and use GPRRegister even for byte operations.
737 GPRRegister VarReg = VarCanBeByte ? Traits::getEncodedGPR(Var->getRegNum())
738 : Traits::getEncodedGPR(Var->getRegNum());
739 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
740 if (SrcVar->hasReg()) {
741 GPRRegister SrcReg = SrcCanBeByte
742 ? Traits::getEncodedGPR(SrcVar->getRegNum())
743 : Traits::getEncodedGPR(SrcVar->getRegNum());
744 (Asm->*(Emitter.GPRGPR))(Ty, VarReg, SrcReg);
745 } else {
746 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar);
747 (Asm->*(Emitter.GPRAddr))(Ty, VarReg, SrcStackAddr);
748 }
749 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) {
750 Mem->emitSegmentOverride(Asm);
751 (Asm->*(Emitter.GPRAddr))(Ty, VarReg,
752 Mem->toAsmAddress(Asm, Target, IsLea));
753 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src)) {
754 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue()));
755 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger64>(Src)) {
756 assert(Traits::Is64Bit);
757 assert(Utils::IsInt(32, Imm->getValue()));
758 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue()));
759 } else if (const auto *Reloc = llvm::dyn_cast<ConstantRelocatable>(Src)) {
760 const auto FixupKind = (Reloc->getName().hasStdString() &&
761 Reloc->getName().toString() == GlobalOffsetTable)
762 ? Traits::FK_GotPC
763 : Traits::TargetLowering::getAbsFixup();
764 AssemblerFixup *Fixup = Asm->createFixup(FixupKind, Reloc);
765 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Fixup));
766 } else if (const auto *Split = llvm::dyn_cast<VariableSplit>(Src)) {
767 (Asm->*(Emitter.GPRAddr))(Ty, VarReg, Split->toAsmAddress(Func));
768 } else {
769 llvm_unreachable("Unexpected operand type");
770 }
771 }
772
773 template <typename TraitsType>
emitIASAddrOpTyGPR(const Cfg * Func,Type Ty,const Address & Addr,const Operand * Src,const GPREmitterAddrOp & Emitter)774 void InstImpl<TraitsType>::emitIASAddrOpTyGPR(const Cfg *Func, Type Ty,
775 const Address &Addr,
776 const Operand *Src,
777 const GPREmitterAddrOp &Emitter) {
778 Assembler *Asm = Func->getAssembler<Assembler>();
779 // Src can only be Reg or AssemblerImmediate.
780 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
781 assert(SrcVar->hasReg());
782 GPRRegister SrcReg = Traits::getEncodedGPR(SrcVar->getRegNum());
783 (Asm->*(Emitter.AddrGPR))(Ty, Addr, SrcReg);
784 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src)) {
785 (Asm->*(Emitter.AddrImm))(Ty, Addr, AssemblerImmediate(Imm->getValue()));
786 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger64>(Src)) {
787 assert(Traits::Is64Bit);
788 assert(Utils::IsInt(32, Imm->getValue()));
789 (Asm->*(Emitter.AddrImm))(Ty, Addr, AssemblerImmediate(Imm->getValue()));
790 } else if (const auto *Reloc = llvm::dyn_cast<ConstantRelocatable>(Src)) {
791 const auto FixupKind = (Reloc->getName().hasStdString() &&
792 Reloc->getName().toString() == GlobalOffsetTable)
793 ? Traits::FK_GotPC
794 : Traits::TargetLowering::getAbsFixup();
795 AssemblerFixup *Fixup = Asm->createFixup(FixupKind, Reloc);
796 (Asm->*(Emitter.AddrImm))(Ty, Addr, AssemblerImmediate(Fixup));
797 } else {
798 llvm_unreachable("Unexpected operand type");
799 }
800 }
801
802 template <typename TraitsType>
emitIASAsAddrOpTyGPR(const Cfg * Func,Type Ty,const Operand * Op0,const Operand * Op1,const GPREmitterAddrOp & Emitter)803 void InstImpl<TraitsType>::emitIASAsAddrOpTyGPR(
804 const Cfg *Func, Type Ty, const Operand *Op0, const Operand *Op1,
805 const GPREmitterAddrOp &Emitter) {
806 auto *Target = InstX86Base::getTarget(Func);
807 if (const auto *Op0Var = llvm::dyn_cast<Variable>(Op0)) {
808 assert(!Op0Var->hasReg());
809 Address StackAddr(Target->stackVarToAsmOperand(Op0Var));
810 emitIASAddrOpTyGPR(Func, Ty, StackAddr, Op1, Emitter);
811 } else if (const auto *Op0Mem = llvm::dyn_cast<X86OperandMem>(Op0)) {
812 Assembler *Asm = Func->getAssembler<Assembler>();
813 Op0Mem->emitSegmentOverride(Asm);
814 emitIASAddrOpTyGPR(Func, Ty, Op0Mem->toAsmAddress(Asm, Target), Op1,
815 Emitter);
816 } else if (const auto *Split = llvm::dyn_cast<VariableSplit>(Op0)) {
817 emitIASAddrOpTyGPR(Func, Ty, Split->toAsmAddress(Func), Op1, Emitter);
818 } else {
819 llvm_unreachable("Unexpected operand type");
820 }
821 }
822
823 template <typename TraitsType>
emitIASGPRShift(const Cfg * Func,Type Ty,const Variable * Var,const Operand * Src,const GPREmitterShiftOp & Emitter)824 void InstImpl<TraitsType>::emitIASGPRShift(const Cfg *Func, Type Ty,
825 const Variable *Var,
826 const Operand *Src,
827 const GPREmitterShiftOp &Emitter) {
828 Assembler *Asm = Func->getAssembler<Assembler>();
829 // Technically, the Dest Var can be mem as well, but we only use Reg. We can
830 // extend this to check Dest if we decide to use that form.
831 assert(Var->hasReg());
832 // We cheat a little and use GPRRegister even for byte operations.
833 GPRRegister VarReg = Traits::getEncodedGPR(Var->getRegNum());
834 // Src must be reg == ECX or an Imm8. This is asserted by the assembler.
835 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
836 assert(SrcVar->hasReg());
837 GPRRegister SrcReg = Traits::getEncodedGPR(SrcVar->getRegNum());
838 (Asm->*(Emitter.GPRGPR))(Ty, VarReg, SrcReg);
839 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src)) {
840 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue()));
841 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger64>(Src)) {
842 assert(Traits::Is64Bit);
843 assert(Utils::IsInt(32, Imm->getValue()));
844 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue()));
845 } else {
846 llvm_unreachable("Unexpected operand type");
847 }
848 }
849
850 template <typename TraitsType>
emitIASGPRShiftDouble(const Cfg * Func,const Variable * Dest,const Operand * Src1Op,const Operand * Src2Op,const GPREmitterShiftD & Emitter)851 void InstImpl<TraitsType>::emitIASGPRShiftDouble(
852 const Cfg *Func, const Variable *Dest, const Operand *Src1Op,
853 const Operand *Src2Op, const GPREmitterShiftD &Emitter) {
854 Assembler *Asm = Func->getAssembler<Assembler>();
855 // Dest can be reg or mem, but we only use the reg variant.
856 assert(Dest->hasReg());
857 GPRRegister DestReg = Traits::getEncodedGPR(Dest->getRegNum());
858 // SrcVar1 must be reg.
859 const auto *SrcVar1 = llvm::cast<Variable>(Src1Op);
860 assert(SrcVar1->hasReg());
861 GPRRegister SrcReg = Traits::getEncodedGPR(SrcVar1->getRegNum());
862 Type Ty = SrcVar1->getType();
863 // Src2 can be the implicit CL register or an immediate.
864 if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src2Op)) {
865 (Asm->*(Emitter.GPRGPRImm))(Ty, DestReg, SrcReg,
866 AssemblerImmediate(Imm->getValue()));
867 } else {
868 assert(llvm::cast<Variable>(Src2Op)->getRegNum() == RegisterSet::Reg_cl);
869 (Asm->*(Emitter.GPRGPR))(Ty, DestReg, SrcReg);
870 }
871 }
872
873 template <typename TraitsType>
emitIASXmmShift(const Cfg * Func,Type Ty,const Variable * Var,const Operand * Src,const XmmEmitterShiftOp & Emitter)874 void InstImpl<TraitsType>::emitIASXmmShift(const Cfg *Func, Type Ty,
875 const Variable *Var,
876 const Operand *Src,
877 const XmmEmitterShiftOp &Emitter) {
878 auto *Target = InstX86Base::getTarget(Func);
879 Assembler *Asm = Func->getAssembler<Assembler>();
880 assert(Var->hasReg());
881 XmmRegister VarReg = Traits::getEncodedXmm(Var->getRegNum());
882 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
883 if (SrcVar->hasReg()) {
884 XmmRegister SrcReg = Traits::getEncodedXmm(SrcVar->getRegNum());
885 (Asm->*(Emitter.XmmXmm))(Ty, VarReg, SrcReg);
886 } else {
887 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar);
888 (Asm->*(Emitter.XmmAddr))(Ty, VarReg, SrcStackAddr);
889 }
890 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) {
891 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
892 (Asm->*(Emitter.XmmAddr))(Ty, VarReg, Mem->toAsmAddress(Asm, Target));
893 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src)) {
894 (Asm->*(Emitter.XmmImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue()));
895 } else {
896 llvm_unreachable("Unexpected operand type");
897 }
898 }
899
900 template <typename TraitsType>
emitIASRegOpTyXMM(const Cfg * Func,Type Ty,const Variable * Var,const Operand * Src,const XmmEmitterRegOp & Emitter)901 void InstImpl<TraitsType>::emitIASRegOpTyXMM(const Cfg *Func, Type Ty,
902 const Variable *Var,
903 const Operand *Src,
904 const XmmEmitterRegOp &Emitter) {
905 auto *Target = InstX86Base::getTarget(Func);
906 Assembler *Asm = Func->getAssembler<Assembler>();
907 assert(Var->hasReg());
908 XmmRegister VarReg = Traits::getEncodedXmm(Var->getRegNum());
909 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
910 if (SrcVar->hasReg()) {
911 XmmRegister SrcReg = Traits::getEncodedXmm(SrcVar->getRegNum());
912 (Asm->*(Emitter.XmmXmm))(Ty, VarReg, SrcReg);
913 } else {
914 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar);
915 (Asm->*(Emitter.XmmAddr))(Ty, VarReg, SrcStackAddr);
916 }
917 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) {
918 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
919 (Asm->*(Emitter.XmmAddr))(Ty, VarReg, Mem->toAsmAddress(Asm, Target));
920 } else if (const auto *Imm = llvm::dyn_cast<Constant>(Src)) {
921 (Asm->*(Emitter.XmmAddr))(Ty, VarReg,
922 Traits::Address::ofConstPool(Asm, Imm));
923 } else {
924 llvm_unreachable("Unexpected operand type");
925 }
926 }
927
928 template <typename TraitsType>
929 template <typename DReg_t, typename SReg_t, DReg_t (*destEnc)(RegNumT),
930 SReg_t (*srcEnc)(RegNumT)>
emitIASCastRegOp(const Cfg * Func,Type DestTy,const Variable * Dest,Type SrcTy,const Operand * Src,const CastEmitterRegOp<DReg_t,SReg_t> & Emitter)931 void InstImpl<TraitsType>::emitIASCastRegOp(
932 const Cfg *Func, Type DestTy, const Variable *Dest, Type SrcTy,
933 const Operand *Src, const CastEmitterRegOp<DReg_t, SReg_t> &Emitter) {
934 auto *Target = InstX86Base::getTarget(Func);
935 Assembler *Asm = Func->getAssembler<Assembler>();
936 assert(Dest->hasReg());
937 DReg_t DestReg = destEnc(Dest->getRegNum());
938 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
939 if (SrcVar->hasReg()) {
940 SReg_t SrcReg = srcEnc(SrcVar->getRegNum());
941 (Asm->*(Emitter.RegReg))(DestTy, DestReg, SrcTy, SrcReg);
942 } else {
943 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar);
944 (Asm->*(Emitter.RegAddr))(DestTy, DestReg, SrcTy, SrcStackAddr);
945 }
946 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) {
947 Mem->emitSegmentOverride(Asm);
948 (Asm->*(Emitter.RegAddr))(DestTy, DestReg, SrcTy,
949 Mem->toAsmAddress(Asm, Target));
950 } else {
951 llvm_unreachable("Unexpected operand type");
952 }
953 }
954
955 template <typename TraitsType>
956 template <typename DReg_t, typename SReg_t, DReg_t (*destEnc)(RegNumT),
957 SReg_t (*srcEnc)(RegNumT)>
emitIASThreeOpImmOps(const Cfg * Func,Type DispatchTy,const Variable * Dest,const Operand * Src0,const Operand * Src1,const ThreeOpImmEmitter<DReg_t,SReg_t> Emitter)958 void InstImpl<TraitsType>::emitIASThreeOpImmOps(
959 const Cfg *Func, Type DispatchTy, const Variable *Dest, const Operand *Src0,
960 const Operand *Src1, const ThreeOpImmEmitter<DReg_t, SReg_t> Emitter) {
961 auto *Target = InstX86Base::getTarget(Func);
962 Assembler *Asm = Func->getAssembler<Assembler>();
963 // This only handles Dest being a register, and Src1 being an immediate.
964 assert(Dest->hasReg());
965 DReg_t DestReg = destEnc(Dest->getRegNum());
966 AssemblerImmediate Imm(llvm::cast<ConstantInteger32>(Src1)->getValue());
967 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src0)) {
968 if (SrcVar->hasReg()) {
969 SReg_t SrcReg = srcEnc(SrcVar->getRegNum());
970 (Asm->*(Emitter.RegRegImm))(DispatchTy, DestReg, SrcReg, Imm);
971 } else {
972 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar);
973 (Asm->*(Emitter.RegAddrImm))(DispatchTy, DestReg, SrcStackAddr, Imm);
974 }
975 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src0)) {
976 Mem->emitSegmentOverride(Asm);
977 (Asm->*(Emitter.RegAddrImm))(DispatchTy, DestReg,
978 Mem->toAsmAddress(Asm, Target), Imm);
979 } else {
980 llvm_unreachable("Unexpected operand type");
981 }
982 }
983
984 template <typename TraitsType>
emitIASMovlikeXMM(const Cfg * Func,const Variable * Dest,const Operand * Src,const XmmEmitterMovOps Emitter)985 void InstImpl<TraitsType>::emitIASMovlikeXMM(const Cfg *Func,
986 const Variable *Dest,
987 const Operand *Src,
988 const XmmEmitterMovOps Emitter) {
989 auto *Target = InstX86Base::getTarget(Func);
990 Assembler *Asm = Func->getAssembler<Assembler>();
991 if (Dest->hasReg()) {
992 XmmRegister DestReg = Traits::getEncodedXmm(Dest->getRegNum());
993 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
994 if (SrcVar->hasReg()) {
995 (Asm->*(Emitter.XmmXmm))(DestReg,
996 Traits::getEncodedXmm(SrcVar->getRegNum()));
997 } else {
998 Address StackAddr(Target->stackVarToAsmOperand(SrcVar));
999 (Asm->*(Emitter.XmmAddr))(DestReg, StackAddr);
1000 }
1001 } else if (const auto *SrcMem = llvm::dyn_cast<X86OperandMem>(Src)) {
1002 assert(SrcMem->getSegmentRegister() == X86OperandMem::DefaultSegment);
1003 (Asm->*(Emitter.XmmAddr))(DestReg, SrcMem->toAsmAddress(Asm, Target));
1004 } else {
1005 llvm_unreachable("Unexpected operand type");
1006 }
1007 } else {
1008 Address StackAddr(Target->stackVarToAsmOperand(Dest));
1009 // Src must be a register in this case.
1010 const auto *SrcVar = llvm::cast<Variable>(Src);
1011 assert(SrcVar->hasReg());
1012 (Asm->*(Emitter.AddrXmm))(StackAddr,
1013 Traits::getEncodedXmm(SrcVar->getRegNum()));
1014 }
1015 }
1016
1017 template <typename TraitsType>
dump(const Cfg * Func)1018 void InstImpl<TraitsType>::InstX86Movmsk::dump(const Cfg *Func) const {
1019 if (!BuildDefs::dump())
1020 return;
1021 Ostream &Str = Func->getContext()->getStrDump();
1022 this->dumpDest(Func);
1023 Str << " = movmsk." << this->getSrc(0)->getType() << " ";
1024 this->dumpSources(Func);
1025 }
1026
1027 template <typename TraitsType>
emit(const Cfg * Func)1028 void InstImpl<TraitsType>::InstX86Movmsk::emit(const Cfg *Func) const {
1029 if (!BuildDefs::dump())
1030 return;
1031 Ostream &Str = Func->getContext()->getStrEmit();
1032 assert(this->getSrcSize() == 1);
1033 Type SrcTy = this->getSrc(0)->getType();
1034 assert(isVectorType(SrcTy));
1035 switch (SrcTy) {
1036 case IceType_v16i8:
1037 Str << "\t"
1038 "pmovmskb"
1039 "\t";
1040 break;
1041 case IceType_v4i32:
1042 case IceType_v4f32:
1043 Str << "\t"
1044 "movmskps"
1045 "\t";
1046 break;
1047 default:
1048 llvm_unreachable("Unexpected operand type");
1049 }
1050 this->getSrc(0)->emit(Func);
1051 Str << ", ";
1052 this->getDest()->emit(Func);
1053 }
1054
1055 template <typename TraitsType>
emitIAS(const Cfg * Func)1056 void InstImpl<TraitsType>::InstX86Movmsk::emitIAS(const Cfg *Func) const {
1057 assert(this->getSrcSize() == 1);
1058 Assembler *Asm = Func->getAssembler<Assembler>();
1059 const Variable *Dest = this->getDest();
1060 const Variable *Src = llvm::cast<Variable>(this->getSrc(0));
1061 const Type DestTy = Dest->getType();
1062 (void)DestTy;
1063 const Type SrcTy = Src->getType();
1064 assert(isVectorType(SrcTy));
1065 assert(isScalarIntegerType(DestTy));
1066 if (Traits::Is64Bit) {
1067 assert(DestTy == IceType_i32 || DestTy == IceType_i64);
1068 } else {
1069 assert(typeWidthInBytes(DestTy) <= 4);
1070 }
1071 XmmRegister SrcReg = Traits::getEncodedXmm(Src->getRegNum());
1072 GPRRegister DestReg = Traits::getEncodedGPR(Dest->getRegNum());
1073 Asm->movmsk(SrcTy, DestReg, SrcReg);
1074 }
1075
1076 template <typename TraitsType>
emit(const Cfg * Func)1077 void InstImpl<TraitsType>::InstX86Sqrt::emit(const Cfg *Func) const {
1078 if (!BuildDefs::dump())
1079 return;
1080 Ostream &Str = Func->getContext()->getStrEmit();
1081 assert(this->getSrcSize() == 1);
1082 Type Ty = this->getSrc(0)->getType();
1083 assert(isScalarFloatingType(Ty));
1084 Str << "\t"
1085 "sqrt" << Traits::TypeAttributes[Ty].SpSdString << "\t";
1086 this->getSrc(0)->emit(Func);
1087 Str << ", ";
1088 this->getDest()->emit(Func);
1089 }
1090
1091 template <typename TraitsType>
emit(const Cfg * Func)1092 void InstImpl<TraitsType>::InstX86Div::emit(const Cfg *Func) const {
1093 if (!BuildDefs::dump())
1094 return;
1095 Ostream &Str = Func->getContext()->getStrEmit();
1096 assert(this->getSrcSize() == 3);
1097 Operand *Src1 = this->getSrc(1);
1098 Str << "\t" << this->Opcode << this->getWidthString(Src1->getType()) << "\t";
1099 Src1->emit(Func);
1100 }
1101
1102 template <typename TraitsType>
emitIAS(const Cfg * Func)1103 void InstImpl<TraitsType>::InstX86Div::emitIAS(const Cfg *Func) const {
1104 assert(this->getSrcSize() == 3);
1105 const Operand *Src = this->getSrc(1);
1106 Type Ty = Src->getType();
1107 static GPREmitterOneOp Emitter = {&Assembler::div, &Assembler::div};
1108 emitIASOpTyGPR(Func, Ty, Src, Emitter);
1109 }
1110
1111 template <typename TraitsType>
emit(const Cfg * Func)1112 void InstImpl<TraitsType>::InstX86Idiv::emit(const Cfg *Func) const {
1113 if (!BuildDefs::dump())
1114 return;
1115 Ostream &Str = Func->getContext()->getStrEmit();
1116 assert(this->getSrcSize() == 3);
1117 Operand *Src1 = this->getSrc(1);
1118 Str << "\t" << this->Opcode << this->getWidthString(Src1->getType()) << "\t";
1119 Src1->emit(Func);
1120 }
1121
1122 template <typename TraitsType>
emitIAS(const Cfg * Func)1123 void InstImpl<TraitsType>::InstX86Idiv::emitIAS(const Cfg *Func) const {
1124 assert(this->getSrcSize() == 3);
1125 const Operand *Src = this->getSrc(1);
1126 Type Ty = Src->getType();
1127 static const GPREmitterOneOp Emitter = {&Assembler::idiv, &Assembler::idiv};
1128 emitIASOpTyGPR(Func, Ty, Src, Emitter);
1129 }
1130
1131 // pblendvb and blendvps take xmm0 as a final implicit argument.
1132 template <typename TraitsType>
emitVariableBlendInst(const char * Opcode,const Inst * Instr,const Cfg * Func)1133 void InstImpl<TraitsType>::emitVariableBlendInst(const char *Opcode,
1134 const Inst *Instr,
1135 const Cfg *Func) {
1136 if (!BuildDefs::dump())
1137 return;
1138 Ostream &Str = Func->getContext()->getStrEmit();
1139 assert(Instr->getSrcSize() == 3);
1140 assert(llvm::cast<Variable>(Instr->getSrc(2))->getRegNum() ==
1141 RegisterSet::Reg_xmm0);
1142 Str << "\t" << Opcode << "\t";
1143 Instr->getSrc(1)->emit(Func);
1144 Str << ", ";
1145 Instr->getDest()->emit(Func);
1146 }
1147
1148 template <typename TraitsType>
emitIASVariableBlendInst(const Inst * Instr,const Cfg * Func,const XmmEmitterRegOp & Emitter)1149 void InstImpl<TraitsType>::emitIASVariableBlendInst(
1150 const Inst *Instr, const Cfg *Func, const XmmEmitterRegOp &Emitter) {
1151 assert(Instr->getSrcSize() == 3);
1152 assert(llvm::cast<Variable>(Instr->getSrc(2))->getRegNum() ==
1153 RegisterSet::Reg_xmm0);
1154 const Variable *Dest = Instr->getDest();
1155 const Operand *Src = Instr->getSrc(1);
1156 emitIASRegOpTyXMM(Func, Dest->getType(), Dest, Src, Emitter);
1157 }
1158
1159 template <typename TraitsType>
emit(const Cfg * Func)1160 void InstImpl<TraitsType>::InstX86Blendvps::emit(const Cfg *Func) const {
1161 if (!BuildDefs::dump())
1162 return;
1163 emitVariableBlendInst(this->Opcode, this, Func);
1164 }
1165
1166 template <typename TraitsType>
emitIAS(const Cfg * Func)1167 void InstImpl<TraitsType>::InstX86Blendvps::emitIAS(const Cfg *Func) const {
1168 static const XmmEmitterRegOp Emitter = {&Assembler::blendvps,
1169 &Assembler::blendvps};
1170 emitIASVariableBlendInst(this, Func, Emitter);
1171 }
1172
1173 template <typename TraitsType>
emit(const Cfg * Func)1174 void InstImpl<TraitsType>::InstX86Pblendvb::emit(const Cfg *Func) const {
1175 if (!BuildDefs::dump())
1176 return;
1177 emitVariableBlendInst(this->Opcode, this, Func);
1178 }
1179
1180 template <typename TraitsType>
emitIAS(const Cfg * Func)1181 void InstImpl<TraitsType>::InstX86Pblendvb::emitIAS(const Cfg *Func) const {
1182 static const XmmEmitterRegOp Emitter = {&Assembler::pblendvb,
1183 &Assembler::pblendvb};
1184 emitIASVariableBlendInst(this, Func, Emitter);
1185 }
1186
1187 template <typename TraitsType>
emit(const Cfg * Func)1188 void InstImpl<TraitsType>::InstX86Imul::emit(const Cfg *Func) const {
1189 if (!BuildDefs::dump())
1190 return;
1191 Ostream &Str = Func->getContext()->getStrEmit();
1192 assert(this->getSrcSize() == 2);
1193 Variable *Dest = this->getDest();
1194 if (isByteSizedArithType(Dest->getType())) {
1195 // The 8-bit version of imul only allows the form "imul r/m8".
1196 const auto *Src0Var = llvm::dyn_cast<Variable>(this->getSrc(0));
1197 (void)Src0Var;
1198 assert(Src0Var->getRegNum() == RegisterSet::Reg_al);
1199 Str << "\t"
1200 "imulb\t";
1201 this->getSrc(1)->emit(Func);
1202 } else if (llvm::isa<Constant>(this->getSrc(1))) {
1203 Str << "\t"
1204 "imul" << this->getWidthString(Dest->getType()) << "\t";
1205 this->getSrc(1)->emit(Func);
1206 Str << ", ";
1207 this->getSrc(0)->emit(Func);
1208 Str << ", ";
1209 Dest->emit(Func);
1210 } else {
1211 this->emitTwoAddress(Func, this->Opcode);
1212 }
1213 }
1214
1215 template <typename TraitsType>
emitIAS(const Cfg * Func)1216 void InstImpl<TraitsType>::InstX86Imul::emitIAS(const Cfg *Func) const {
1217 assert(this->getSrcSize() == 2);
1218 const Variable *Var = this->getDest();
1219 Type Ty = Var->getType();
1220 const Operand *Src = this->getSrc(1);
1221 if (isByteSizedArithType(Ty)) {
1222 // The 8-bit version of imul only allows the form "imul r/m8".
1223 const auto *Src0Var = llvm::dyn_cast<Variable>(this->getSrc(0));
1224 (void)Src0Var;
1225 assert(Src0Var->getRegNum() == RegisterSet::Reg_al);
1226 static const GPREmitterOneOp Emitter = {&Assembler::imul, &Assembler::imul};
1227 emitIASOpTyGPR(Func, Ty, this->getSrc(1), Emitter);
1228 } else {
1229 // The two-address version is used when multiplying by a non-constant
1230 // or doing an 8-bit multiply.
1231 assert(Var == this->getSrc(0));
1232 static const GPREmitterRegOp Emitter = {&Assembler::imul, &Assembler::imul,
1233 &Assembler::imul};
1234 constexpr bool NotLea = false;
1235 emitIASRegOpTyGPR(Func, NotLea, Ty, Var, Src, Emitter);
1236 }
1237 }
1238
1239 template <typename TraitsType>
emit(const Cfg * Func)1240 void InstImpl<TraitsType>::InstX86ImulImm::emit(const Cfg *Func) const {
1241 if (!BuildDefs::dump())
1242 return;
1243 Ostream &Str = Func->getContext()->getStrEmit();
1244 assert(this->getSrcSize() == 2);
1245 Variable *Dest = this->getDest();
1246 assert(Dest->getType() == IceType_i16 || Dest->getType() == IceType_i32);
1247 assert(llvm::isa<Constant>(this->getSrc(1)));
1248 Str << "\t"
1249 "imul" << this->getWidthString(Dest->getType()) << "\t";
1250 this->getSrc(1)->emit(Func);
1251 Str << ", ";
1252 this->getSrc(0)->emit(Func);
1253 Str << ", ";
1254 Dest->emit(Func);
1255 }
1256
1257 template <typename TraitsType>
emitIAS(const Cfg * Func)1258 void InstImpl<TraitsType>::InstX86ImulImm::emitIAS(const Cfg *Func) const {
1259 assert(this->getSrcSize() == 2);
1260 const Variable *Dest = this->getDest();
1261 Type Ty = Dest->getType();
1262 assert(llvm::isa<Constant>(this->getSrc(1)));
1263 static const ThreeOpImmEmitter<GPRRegister, GPRRegister> Emitter = {
1264 &Assembler::imul, &Assembler::imul};
1265 emitIASThreeOpImmOps<GPRRegister, GPRRegister, Traits::getEncodedGPR,
1266 Traits::getEncodedGPR>(Func, Ty, Dest, this->getSrc(0),
1267 this->getSrc(1), Emitter);
1268 }
1269
1270 template <typename TraitsType>
emitIAS(const Cfg * Func)1271 void InstImpl<TraitsType>::InstX86Insertps::emitIAS(const Cfg *Func) const {
1272 assert(this->getSrcSize() == 3);
1273 assert(InstX86Base::getTarget(Func)->getInstructionSet() >= Traits::SSE4_1);
1274 const Variable *Dest = this->getDest();
1275 assert(Dest == this->getSrc(0));
1276 Type Ty = Dest->getType();
1277 static const ThreeOpImmEmitter<XmmRegister, XmmRegister> Emitter = {
1278 &Assembler::insertps, &Assembler::insertps};
1279 emitIASThreeOpImmOps<XmmRegister, XmmRegister, Traits::getEncodedXmm,
1280 Traits::getEncodedXmm>(Func, Ty, Dest, this->getSrc(1),
1281 this->getSrc(2), Emitter);
1282 }
1283
1284 template <typename TraitsType>
emit(const Cfg * Func)1285 void InstImpl<TraitsType>::InstX86Cbwdq::emit(const Cfg *Func) const {
1286 if (!BuildDefs::dump())
1287 return;
1288 Ostream &Str = Func->getContext()->getStrEmit();
1289 assert(this->getSrcSize() == 1);
1290 Operand *Src0 = this->getSrc(0);
1291 const auto DestReg = this->getDest()->getRegNum();
1292 const auto SrcReg = llvm::cast<Variable>(Src0)->getRegNum();
1293 (void)DestReg;
1294 (void)SrcReg;
1295 switch (Src0->getType()) {
1296 default:
1297 llvm_unreachable("unexpected source type!");
1298 break;
1299 case IceType_i8:
1300 assert(SrcReg == RegisterSet::Reg_al);
1301 assert(DestReg == RegisterSet::Reg_ax || DestReg == RegisterSet::Reg_ah);
1302 Str << "\t"
1303 "cbtw";
1304 break;
1305 case IceType_i16:
1306 assert(SrcReg == RegisterSet::Reg_ax);
1307 assert(DestReg == RegisterSet::Reg_dx);
1308 Str << "\t"
1309 "cwtd";
1310 break;
1311 case IceType_i32:
1312 assert(SrcReg == RegisterSet::Reg_eax);
1313 assert(DestReg == RegisterSet::Reg_edx);
1314 Str << "\t"
1315 "cltd";
1316 break;
1317 case IceType_i64:
1318 assert(Traits::Is64Bit);
1319 assert(SrcReg == Traits::getRaxOrDie());
1320 assert(DestReg == Traits::getRdxOrDie());
1321 Str << "\t"
1322 "cqo";
1323 break;
1324 }
1325 }
1326
1327 template <typename TraitsType>
emitIAS(const Cfg * Func)1328 void InstImpl<TraitsType>::InstX86Cbwdq::emitIAS(const Cfg *Func) const {
1329 Assembler *Asm = Func->getAssembler<Assembler>();
1330 assert(this->getSrcSize() == 1);
1331 Operand *Src0 = this->getSrc(0);
1332 const auto DestReg = this->getDest()->getRegNum();
1333 const auto SrcReg = llvm::cast<Variable>(Src0)->getRegNum();
1334 (void)DestReg;
1335 (void)SrcReg;
1336 switch (Src0->getType()) {
1337 default:
1338 llvm_unreachable("unexpected source type!");
1339 break;
1340 case IceType_i8:
1341 assert(SrcReg == RegisterSet::Reg_al);
1342 assert(DestReg == RegisterSet::Reg_ax || DestReg == RegisterSet::Reg_ah);
1343 Asm->cbw();
1344 break;
1345 case IceType_i16:
1346 assert(SrcReg == RegisterSet::Reg_ax);
1347 assert(DestReg == RegisterSet::Reg_dx);
1348 Asm->cwd();
1349 break;
1350 case IceType_i32:
1351 assert(SrcReg == RegisterSet::Reg_eax);
1352 assert(DestReg == RegisterSet::Reg_edx);
1353 Asm->cdq();
1354 break;
1355 case IceType_i64:
1356 assert(Traits::Is64Bit);
1357 assert(SrcReg == Traits::getRaxOrDie());
1358 assert(DestReg == Traits::getRdxOrDie());
1359 Asm->cqo();
1360 break;
1361 }
1362 }
1363
1364 template <typename TraitsType>
emit(const Cfg * Func)1365 void InstImpl<TraitsType>::InstX86Mul::emit(const Cfg *Func) const {
1366 if (!BuildDefs::dump())
1367 return;
1368 Ostream &Str = Func->getContext()->getStrEmit();
1369 assert(this->getSrcSize() == 2);
1370 assert(llvm::isa<Variable>(this->getSrc(0)));
1371 assert(llvm::cast<Variable>(this->getSrc(0))->getRegNum() ==
1372 RegisterSet::Reg_eax);
1373 assert(this->getDest()->getRegNum() == RegisterSet::Reg_eax); // TODO:
1374 // allow
1375 // edx?
1376 Str << "\t"
1377 "mul" << this->getWidthString(this->getDest()->getType()) << "\t";
1378 this->getSrc(1)->emit(Func);
1379 }
1380
1381 template <typename TraitsType>
emitIAS(const Cfg * Func)1382 void InstImpl<TraitsType>::InstX86Mul::emitIAS(const Cfg *Func) const {
1383 assert(this->getSrcSize() == 2);
1384 assert(llvm::isa<Variable>(this->getSrc(0)));
1385 assert(llvm::cast<Variable>(this->getSrc(0))->getRegNum() ==
1386 RegisterSet::Reg_eax);
1387 assert(this->getDest()->getRegNum() == RegisterSet::Reg_eax); // TODO:
1388 // allow
1389 // edx?
1390 const Operand *Src = this->getSrc(1);
1391 Type Ty = Src->getType();
1392 static const GPREmitterOneOp Emitter = {&Assembler::mul, &Assembler::mul};
1393 emitIASOpTyGPR(Func, Ty, Src, Emitter);
1394 }
1395
1396 template <typename TraitsType>
dump(const Cfg * Func)1397 void InstImpl<TraitsType>::InstX86Mul::dump(const Cfg *Func) const {
1398 if (!BuildDefs::dump())
1399 return;
1400 Ostream &Str = Func->getContext()->getStrDump();
1401 this->dumpDest(Func);
1402 Str << " = mul." << this->getDest()->getType() << " ";
1403 this->dumpSources(Func);
1404 }
1405
1406 template <typename TraitsType>
emit(const Cfg * Func)1407 void InstImpl<TraitsType>::InstX86Shld::emit(const Cfg *Func) const {
1408 if (!BuildDefs::dump())
1409 return;
1410 Ostream &Str = Func->getContext()->getStrEmit();
1411 Variable *Dest = this->getDest();
1412 assert(this->getSrcSize() == 3);
1413 assert(Dest == this->getSrc(0));
1414 Str << "\t"
1415 "shld" << this->getWidthString(Dest->getType()) << "\t";
1416 this->getSrc(2)->emit(Func);
1417 Str << ", ";
1418 this->getSrc(1)->emit(Func);
1419 Str << ", ";
1420 Dest->emit(Func);
1421 }
1422
1423 template <typename TraitsType>
emitIAS(const Cfg * Func)1424 void InstImpl<TraitsType>::InstX86Shld::emitIAS(const Cfg *Func) const {
1425 assert(this->getSrcSize() == 3);
1426 assert(this->getDest() == this->getSrc(0));
1427 const Variable *Dest = this->getDest();
1428 const Operand *Src1 = this->getSrc(1);
1429 const Operand *Src2 = this->getSrc(2);
1430 static const GPREmitterShiftD Emitter = {&Assembler::shld, &Assembler::shld};
1431 emitIASGPRShiftDouble(Func, Dest, Src1, Src2, Emitter);
1432 }
1433
1434 template <typename TraitsType>
dump(const Cfg * Func)1435 void InstImpl<TraitsType>::InstX86Shld::dump(const Cfg *Func) const {
1436 if (!BuildDefs::dump())
1437 return;
1438 Ostream &Str = Func->getContext()->getStrDump();
1439 this->dumpDest(Func);
1440 Str << " = shld." << this->getDest()->getType() << " ";
1441 this->dumpSources(Func);
1442 }
1443
1444 template <typename TraitsType>
emit(const Cfg * Func)1445 void InstImpl<TraitsType>::InstX86Shrd::emit(const Cfg *Func) const {
1446 if (!BuildDefs::dump())
1447 return;
1448 Ostream &Str = Func->getContext()->getStrEmit();
1449 Variable *Dest = this->getDest();
1450 assert(this->getSrcSize() == 3);
1451 assert(Dest == this->getSrc(0));
1452 Str << "\t"
1453 "shrd" << this->getWidthString(Dest->getType()) << "\t";
1454 this->getSrc(2)->emit(Func);
1455 Str << ", ";
1456 this->getSrc(1)->emit(Func);
1457 Str << ", ";
1458 Dest->emit(Func);
1459 }
1460
1461 template <typename TraitsType>
emitIAS(const Cfg * Func)1462 void InstImpl<TraitsType>::InstX86Shrd::emitIAS(const Cfg *Func) const {
1463 assert(this->getSrcSize() == 3);
1464 assert(this->getDest() == this->getSrc(0));
1465 const Variable *Dest = this->getDest();
1466 const Operand *Src1 = this->getSrc(1);
1467 const Operand *Src2 = this->getSrc(2);
1468 static const GPREmitterShiftD Emitter = {&Assembler::shrd, &Assembler::shrd};
1469 emitIASGPRShiftDouble(Func, Dest, Src1, Src2, Emitter);
1470 }
1471
1472 template <typename TraitsType>
dump(const Cfg * Func)1473 void InstImpl<TraitsType>::InstX86Shrd::dump(const Cfg *Func) const {
1474 if (!BuildDefs::dump())
1475 return;
1476 Ostream &Str = Func->getContext()->getStrDump();
1477 this->dumpDest(Func);
1478 Str << " = shrd." << this->getDest()->getType() << " ";
1479 this->dumpSources(Func);
1480 }
1481
1482 template <typename TraitsType>
emit(const Cfg * Func)1483 void InstImpl<TraitsType>::InstX86Cmov::emit(const Cfg *Func) const {
1484 if (!BuildDefs::dump())
1485 return;
1486 Ostream &Str = Func->getContext()->getStrEmit();
1487 Variable *Dest = this->getDest();
1488 Str << "\t";
1489 assert(Condition != Cond::Br_None);
1490 assert(this->getDest()->hasReg());
1491 Str << "cmov" << Traits::InstBrAttributes[Condition].DisplayString
1492 << this->getWidthString(Dest->getType()) << "\t";
1493 this->getSrc(1)->emit(Func);
1494 Str << ", ";
1495 Dest->emit(Func);
1496 }
1497
1498 template <typename TraitsType>
emitIAS(const Cfg * Func)1499 void InstImpl<TraitsType>::InstX86Cmov::emitIAS(const Cfg *Func) const {
1500 assert(Condition != Cond::Br_None);
1501 assert(this->getDest()->hasReg());
1502 assert(this->getSrcSize() == 2);
1503 Operand *Src = this->getSrc(1);
1504 Type SrcTy = Src->getType();
1505 assert(SrcTy == IceType_i16 || SrcTy == IceType_i32 || (Traits::Is64Bit));
1506 Assembler *Asm = Func->getAssembler<Assembler>();
1507 auto *Target = InstX86Base::getTarget(Func);
1508 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
1509 if (SrcVar->hasReg()) {
1510 Asm->cmov(SrcTy, Condition,
1511 Traits::getEncodedGPR(this->getDest()->getRegNum()),
1512 Traits::getEncodedGPR(SrcVar->getRegNum()));
1513 } else {
1514 Asm->cmov(SrcTy, Condition,
1515 Traits::getEncodedGPR(this->getDest()->getRegNum()),
1516 Target->stackVarToAsmOperand(SrcVar));
1517 }
1518 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) {
1519 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
1520 Asm->cmov(SrcTy, Condition,
1521 Traits::getEncodedGPR(this->getDest()->getRegNum()),
1522 Mem->toAsmAddress(Asm, Target));
1523 } else {
1524 llvm_unreachable("Unexpected operand type");
1525 }
1526 }
1527
1528 template <typename TraitsType>
dump(const Cfg * Func)1529 void InstImpl<TraitsType>::InstX86Cmov::dump(const Cfg *Func) const {
1530 if (!BuildDefs::dump())
1531 return;
1532 Ostream &Str = Func->getContext()->getStrDump();
1533 Str << "cmov" << Traits::InstBrAttributes[Condition].DisplayString << ".";
1534 Str << this->getDest()->getType() << " ";
1535 this->dumpDest(Func);
1536 Str << ", ";
1537 this->dumpSources(Func);
1538 }
1539
1540 template <typename TraitsType>
emit(const Cfg * Func)1541 void InstImpl<TraitsType>::InstX86Cmpps::emit(const Cfg *Func) const {
1542 if (!BuildDefs::dump())
1543 return;
1544 Ostream &Str = Func->getContext()->getStrEmit();
1545 assert(this->getSrcSize() == 2);
1546 assert(Condition < Cond::Cmpps_Invalid);
1547 Type DestTy = this->Dest->getType();
1548 Str << "\t"
1549 "cmp" << Traits::InstCmppsAttributes[Condition].EmitString
1550 << Traits::TypeAttributes[DestTy].PdPsString << "\t";
1551 this->getSrc(1)->emit(Func);
1552 Str << ", ";
1553 this->getDest()->emit(Func);
1554 }
1555
1556 template <typename TraitsType>
emitIAS(const Cfg * Func)1557 void InstImpl<TraitsType>::InstX86Cmpps::emitIAS(const Cfg *Func) const {
1558 Assembler *Asm = Func->getAssembler<Assembler>();
1559 assert(this->getSrcSize() == 2);
1560 assert(Condition < Cond::Cmpps_Invalid);
1561 // Assuming there isn't any load folding for cmpps, and vector constants are
1562 // not allowed in PNaCl.
1563 assert(llvm::isa<Variable>(this->getSrc(1)));
1564 auto *Target = InstX86Base::getTarget(Func);
1565 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(1));
1566 if (SrcVar->hasReg()) {
1567 Asm->cmpps(this->getDest()->getType(),
1568 Traits::getEncodedXmm(this->getDest()->getRegNum()),
1569 Traits::getEncodedXmm(SrcVar->getRegNum()), Condition);
1570 } else {
1571 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar);
1572 Asm->cmpps(this->getDest()->getType(),
1573 Traits::getEncodedXmm(this->getDest()->getRegNum()),
1574 SrcStackAddr, Condition);
1575 }
1576 }
1577
1578 template <typename TraitsType>
dump(const Cfg * Func)1579 void InstImpl<TraitsType>::InstX86Cmpps::dump(const Cfg *Func) const {
1580 if (!BuildDefs::dump())
1581 return;
1582 Ostream &Str = Func->getContext()->getStrDump();
1583 assert(Condition < Cond::Cmpps_Invalid);
1584 this->dumpDest(Func);
1585 Str << " = cmp" << Traits::InstCmppsAttributes[Condition].EmitString << "ps"
1586 "\t";
1587 this->dumpSources(Func);
1588 }
1589
1590 template <typename TraitsType>
emit(const Cfg * Func)1591 void InstImpl<TraitsType>::InstX86Cmpxchg::emit(const Cfg *Func) const {
1592 if (!BuildDefs::dump())
1593 return;
1594 Ostream &Str = Func->getContext()->getStrEmit();
1595 assert(this->getSrcSize() == 3);
1596 if (this->Locked) {
1597 Str << "\t"
1598 "lock";
1599 }
1600 Str << "\t"
1601 "cmpxchg" << this->getWidthString(this->getSrc(0)->getType()) << "\t";
1602 this->getSrc(2)->emit(Func);
1603 Str << ", ";
1604 this->getSrc(0)->emit(Func);
1605 }
1606
1607 template <typename TraitsType>
emitIAS(const Cfg * Func)1608 void InstImpl<TraitsType>::InstX86Cmpxchg::emitIAS(const Cfg *Func) const {
1609 assert(this->getSrcSize() == 3);
1610 Assembler *Asm = Func->getAssembler<Assembler>();
1611 Type Ty = this->getSrc(0)->getType();
1612 auto *Target = InstX86Base::getTarget(Func);
1613 const auto Mem = llvm::cast<X86OperandMem>(this->getSrc(0));
1614 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
1615 const Address Addr = Mem->toAsmAddress(Asm, Target);
1616 const auto *VarReg = llvm::cast<Variable>(this->getSrc(2));
1617 assert(VarReg->hasReg());
1618 const GPRRegister Reg = Traits::getEncodedGPR(VarReg->getRegNum());
1619 Asm->cmpxchg(Ty, Addr, Reg, this->Locked);
1620 }
1621
1622 template <typename TraitsType>
dump(const Cfg * Func)1623 void InstImpl<TraitsType>::InstX86Cmpxchg::dump(const Cfg *Func) const {
1624 if (!BuildDefs::dump())
1625 return;
1626 Ostream &Str = Func->getContext()->getStrDump();
1627 if (this->Locked) {
1628 Str << "lock ";
1629 }
1630 Str << "cmpxchg." << this->getSrc(0)->getType() << " ";
1631 this->dumpSources(Func);
1632 }
1633
1634 template <typename TraitsType>
emit(const Cfg * Func)1635 void InstImpl<TraitsType>::InstX86Cmpxchg8b::emit(const Cfg *Func) const {
1636 if (!BuildDefs::dump())
1637 return;
1638 Ostream &Str = Func->getContext()->getStrEmit();
1639 assert(this->getSrcSize() == 5);
1640 if (this->Locked) {
1641 Str << "\t"
1642 "lock";
1643 }
1644 Str << "\t"
1645 "cmpxchg8b\t";
1646 this->getSrc(0)->emit(Func);
1647 }
1648
1649 template <typename TraitsType>
emitIAS(const Cfg * Func)1650 void InstImpl<TraitsType>::InstX86Cmpxchg8b::emitIAS(const Cfg *Func) const {
1651 assert(this->getSrcSize() == 5);
1652 Assembler *Asm = Func->getAssembler<Assembler>();
1653 const auto Mem = llvm::cast<X86OperandMem>(this->getSrc(0));
1654 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
1655 auto *Target = InstX86Base::getTarget(Func);
1656 const Address Addr = Mem->toAsmAddress(Asm, Target);
1657 Asm->cmpxchg8b(Addr, this->Locked);
1658 }
1659
1660 template <typename TraitsType>
dump(const Cfg * Func)1661 void InstImpl<TraitsType>::InstX86Cmpxchg8b::dump(const Cfg *Func) const {
1662 if (!BuildDefs::dump())
1663 return;
1664 Ostream &Str = Func->getContext()->getStrDump();
1665 if (this->Locked) {
1666 Str << "lock ";
1667 }
1668 Str << "cmpxchg8b ";
1669 this->dumpSources(Func);
1670 }
1671
1672 template <typename TraitsType>
emit(const Cfg * Func)1673 void InstImpl<TraitsType>::InstX86Cvt::emit(const Cfg *Func) const {
1674 if (!BuildDefs::dump())
1675 return;
1676 Ostream &Str = Func->getContext()->getStrEmit();
1677 assert(this->getSrcSize() == 1);
1678 Str << "\t"
1679 "cvt";
1680 if (isTruncating())
1681 Str << "t";
1682 Str << Traits::TypeAttributes[this->getSrc(0)->getType()].CvtString << "2"
1683 << Traits::TypeAttributes[this->getDest()->getType()].CvtString << "\t";
1684 this->getSrc(0)->emit(Func);
1685 Str << ", ";
1686 this->getDest()->emit(Func);
1687 }
1688
1689 template <typename TraitsType>
emitIAS(const Cfg * Func)1690 void InstImpl<TraitsType>::InstX86Cvt::emitIAS(const Cfg *Func) const {
1691 assert(this->getSrcSize() == 1);
1692 const Variable *Dest = this->getDest();
1693 const Operand *Src = this->getSrc(0);
1694 Type DestTy = Dest->getType();
1695 Type SrcTy = Src->getType();
1696 switch (Variant) {
1697 case Si2ss: {
1698 assert(isScalarIntegerType(SrcTy));
1699 if (!Traits::Is64Bit) {
1700 assert(typeWidthInBytes(SrcTy) <= 4);
1701 } else {
1702 assert(SrcTy == IceType_i32 || SrcTy == IceType_i64);
1703 }
1704 assert(isScalarFloatingType(DestTy));
1705 static const CastEmitterRegOp<XmmRegister, GPRRegister> Emitter = {
1706 &Assembler::cvtsi2ss, &Assembler::cvtsi2ss};
1707 emitIASCastRegOp<XmmRegister, GPRRegister, Traits::getEncodedXmm,
1708 Traits::getEncodedGPR>(Func, DestTy, Dest, SrcTy, Src,
1709 Emitter);
1710 return;
1711 }
1712 case Tss2si: {
1713 assert(isScalarFloatingType(SrcTy));
1714 assert(isScalarIntegerType(DestTy));
1715 if (Traits::Is64Bit) {
1716 assert(DestTy == IceType_i32 || DestTy == IceType_i64);
1717 } else {
1718 assert(typeWidthInBytes(DestTy) <= 4);
1719 }
1720 static const CastEmitterRegOp<GPRRegister, XmmRegister> Emitter = {
1721 &Assembler::cvttss2si, &Assembler::cvttss2si};
1722 emitIASCastRegOp<GPRRegister, XmmRegister, Traits::getEncodedGPR,
1723 Traits::getEncodedXmm>(Func, DestTy, Dest, SrcTy, Src,
1724 Emitter);
1725 return;
1726 }
1727 case Ss2si: {
1728 assert(isScalarFloatingType(SrcTy));
1729 assert(isScalarIntegerType(DestTy));
1730 if (Traits::Is64Bit) {
1731 assert(DestTy == IceType_i32 || DestTy == IceType_i64);
1732 } else {
1733 assert(typeWidthInBytes(DestTy) <= 4);
1734 }
1735 static const CastEmitterRegOp<GPRRegister, XmmRegister> Emitter = {
1736 &Assembler::cvtss2si, &Assembler::cvtss2si};
1737 emitIASCastRegOp<GPRRegister, XmmRegister, Traits::getEncodedGPR,
1738 Traits::getEncodedXmm>(Func, DestTy, Dest, SrcTy, Src,
1739 Emitter);
1740 return;
1741 }
1742 case Float2float: {
1743 assert(isScalarFloatingType(SrcTy));
1744 assert(isScalarFloatingType(DestTy));
1745 assert(DestTy != SrcTy);
1746 static const XmmEmitterRegOp Emitter = {&Assembler::cvtfloat2float,
1747 &Assembler::cvtfloat2float};
1748 emitIASRegOpTyXMM(Func, SrcTy, Dest, Src, Emitter);
1749 return;
1750 }
1751 case Dq2ps: {
1752 assert(isVectorIntegerType(SrcTy));
1753 assert(isVectorFloatingType(DestTy));
1754 static const XmmEmitterRegOp Emitter = {&Assembler::cvtdq2ps,
1755 &Assembler::cvtdq2ps};
1756 emitIASRegOpTyXMM(Func, DestTy, Dest, Src, Emitter);
1757 return;
1758 }
1759 case Tps2dq: {
1760 assert(isVectorFloatingType(SrcTy));
1761 assert(isVectorIntegerType(DestTy));
1762 static const XmmEmitterRegOp Emitter = {&Assembler::cvttps2dq,
1763 &Assembler::cvttps2dq};
1764 emitIASRegOpTyXMM(Func, DestTy, Dest, Src, Emitter);
1765 return;
1766 }
1767 case Ps2dq: {
1768 assert(isVectorFloatingType(SrcTy));
1769 assert(isVectorIntegerType(DestTy));
1770 static const XmmEmitterRegOp Emitter = {&Assembler::cvtps2dq,
1771 &Assembler::cvtps2dq};
1772 emitIASRegOpTyXMM(Func, DestTy, Dest, Src, Emitter);
1773 return;
1774 }
1775 }
1776 }
1777
1778 template <typename TraitsType>
dump(const Cfg * Func)1779 void InstImpl<TraitsType>::InstX86Cvt::dump(const Cfg *Func) const {
1780 if (!BuildDefs::dump())
1781 return;
1782 Ostream &Str = Func->getContext()->getStrDump();
1783 this->dumpDest(Func);
1784 Str << " = cvt";
1785 if (isTruncating())
1786 Str << "t";
1787 Str << Traits::TypeAttributes[this->getSrc(0)->getType()].CvtString << "2"
1788 << Traits::TypeAttributes[this->getDest()->getType()].CvtString << " ";
1789 this->dumpSources(Func);
1790 }
1791
1792 template <typename TraitsType>
emit(const Cfg * Func)1793 void InstImpl<TraitsType>::InstX86Round::emit(const Cfg *Func) const {
1794 if (!BuildDefs::dump())
1795 return;
1796 Ostream &Str = Func->getContext()->getStrEmit();
1797 assert(this->getSrcSize() == 3);
1798 Str << "\t" << this->Opcode
1799 << Traits::TypeAttributes[this->getDest()->getType()].SpSdString << "\t";
1800 this->getSrc(1)->emit(Func);
1801 Str << ", ";
1802 this->getSrc(0)->emit(Func);
1803 Str << ", ";
1804 this->getDest()->emit(Func);
1805 }
1806
1807 template <typename TraitsType>
emitIAS(const Cfg * Func)1808 void InstImpl<TraitsType>::InstX86Round::emitIAS(const Cfg *Func) const {
1809 assert(this->getSrcSize() == 2);
1810 assert(InstX86Base::getTarget(Func)->getInstructionSet() >= Traits::SSE4_1);
1811 const Variable *Dest = this->getDest();
1812 Type Ty = Dest->getType();
1813 static const ThreeOpImmEmitter<XmmRegister, XmmRegister> Emitter = {
1814 &Assembler::round, &Assembler::round};
1815 emitIASThreeOpImmOps<XmmRegister, XmmRegister, Traits::getEncodedXmm,
1816 Traits::getEncodedXmm>(Func, Ty, Dest, this->getSrc(0),
1817 this->getSrc(1), Emitter);
1818 }
1819
1820 template <typename TraitsType>
emit(const Cfg * Func)1821 void InstImpl<TraitsType>::InstX86Icmp::emit(const Cfg *Func) const {
1822 if (!BuildDefs::dump())
1823 return;
1824 Ostream &Str = Func->getContext()->getStrEmit();
1825 assert(this->getSrcSize() == 2);
1826 Str << "\t"
1827 "cmp" << this->getWidthString(this->getSrc(0)->getType()) << "\t";
1828 this->getSrc(1)->emit(Func);
1829 Str << ", ";
1830 this->getSrc(0)->emit(Func);
1831 }
1832
1833 template <typename TraitsType>
emitIAS(const Cfg * Func)1834 void InstImpl<TraitsType>::InstX86Icmp::emitIAS(const Cfg *Func) const {
1835 assert(this->getSrcSize() == 2);
1836 const Operand *Src0 = this->getSrc(0);
1837 const Operand *Src1 = this->getSrc(1);
1838 Type Ty = Src0->getType();
1839 static const GPREmitterRegOp RegEmitter = {&Assembler::cmp, &Assembler::cmp,
1840 &Assembler::cmp};
1841 static const GPREmitterAddrOp AddrEmitter = {&Assembler::cmp,
1842 &Assembler::cmp};
1843 if (const auto *SrcVar0 = llvm::dyn_cast<Variable>(Src0)) {
1844 if (SrcVar0->hasReg()) {
1845 constexpr bool NotLea = false;
1846 emitIASRegOpTyGPR(Func, NotLea, Ty, SrcVar0, Src1, RegEmitter);
1847 return;
1848 }
1849 }
1850 emitIASAsAddrOpTyGPR(Func, Ty, Src0, Src1, AddrEmitter);
1851 }
1852
1853 template <typename TraitsType>
dump(const Cfg * Func)1854 void InstImpl<TraitsType>::InstX86Icmp::dump(const Cfg *Func) const {
1855 if (!BuildDefs::dump())
1856 return;
1857 Ostream &Str = Func->getContext()->getStrDump();
1858 Str << "cmp." << this->getSrc(0)->getType() << " ";
1859 this->dumpSources(Func);
1860 }
1861
1862 template <typename TraitsType>
emit(const Cfg * Func)1863 void InstImpl<TraitsType>::InstX86Ucomiss::emit(const Cfg *Func) const {
1864 if (!BuildDefs::dump())
1865 return;
1866 Ostream &Str = Func->getContext()->getStrEmit();
1867 assert(this->getSrcSize() == 2);
1868 Str << "\t"
1869 "ucomi"
1870 << Traits::TypeAttributes[this->getSrc(0)->getType()].SdSsString << "\t";
1871 this->getSrc(1)->emit(Func);
1872 Str << ", ";
1873 this->getSrc(0)->emit(Func);
1874 }
1875
1876 template <typename TraitsType>
emitIAS(const Cfg * Func)1877 void InstImpl<TraitsType>::InstX86Ucomiss::emitIAS(const Cfg *Func) const {
1878 assert(this->getSrcSize() == 2);
1879 // Currently src0 is always a variable by convention, to avoid having two
1880 // memory operands.
1881 assert(llvm::isa<Variable>(this->getSrc(0)));
1882 const auto *Src0Var = llvm::cast<Variable>(this->getSrc(0));
1883 Type Ty = Src0Var->getType();
1884 static const XmmEmitterRegOp Emitter = {&Assembler::ucomiss,
1885 &Assembler::ucomiss};
1886 emitIASRegOpTyXMM(Func, Ty, Src0Var, this->getSrc(1), Emitter);
1887 }
1888
1889 template <typename TraitsType>
dump(const Cfg * Func)1890 void InstImpl<TraitsType>::InstX86Ucomiss::dump(const Cfg *Func) const {
1891 if (!BuildDefs::dump())
1892 return;
1893 Ostream &Str = Func->getContext()->getStrDump();
1894 Str << "ucomiss." << this->getSrc(0)->getType() << " ";
1895 this->dumpSources(Func);
1896 }
1897
1898 template <typename TraitsType>
emit(const Cfg * Func)1899 void InstImpl<TraitsType>::InstX86UD2::emit(const Cfg *Func) const {
1900 if (!BuildDefs::dump())
1901 return;
1902 Ostream &Str = Func->getContext()->getStrEmit();
1903 assert(this->getSrcSize() == 0);
1904 Str << "\t"
1905 "ud2";
1906 }
1907
1908 template <typename TraitsType>
emitIAS(const Cfg * Func)1909 void InstImpl<TraitsType>::InstX86UD2::emitIAS(const Cfg *Func) const {
1910 Assembler *Asm = Func->getAssembler<Assembler>();
1911 Asm->ud2();
1912 }
1913
1914 template <typename TraitsType>
dump(const Cfg * Func)1915 void InstImpl<TraitsType>::InstX86UD2::dump(const Cfg *Func) const {
1916 if (!BuildDefs::dump())
1917 return;
1918 Ostream &Str = Func->getContext()->getStrDump();
1919 Str << "ud2";
1920 }
1921
1922 template <typename TraitsType>
emit(const Cfg * Func)1923 void InstImpl<TraitsType>::InstX86Int3::emit(const Cfg *Func) const {
1924 if (!BuildDefs::dump())
1925 return;
1926 Ostream &Str = Func->getContext()->getStrEmit();
1927 assert(this->getSrcSize() == 0);
1928 Str << "\t"
1929 "int 3";
1930 }
1931
1932 template <typename TraitsType>
emitIAS(const Cfg * Func)1933 void InstImpl<TraitsType>::InstX86Int3::emitIAS(const Cfg *Func) const {
1934 Assembler *Asm = Func->getAssembler<Assembler>();
1935 Asm->int3();
1936 }
1937
1938 template <typename TraitsType>
dump(const Cfg * Func)1939 void InstImpl<TraitsType>::InstX86Int3::dump(const Cfg *Func) const {
1940 if (!BuildDefs::dump())
1941 return;
1942 Ostream &Str = Func->getContext()->getStrDump();
1943 Str << "int 3";
1944 }
1945
1946 template <typename TraitsType>
emit(const Cfg * Func)1947 void InstImpl<TraitsType>::InstX86Test::emit(const Cfg *Func) const {
1948 if (!BuildDefs::dump())
1949 return;
1950 Ostream &Str = Func->getContext()->getStrEmit();
1951 assert(this->getSrcSize() == 2);
1952 Str << "\t"
1953 "test" << this->getWidthString(this->getSrc(0)->getType()) << "\t";
1954 this->getSrc(1)->emit(Func);
1955 Str << ", ";
1956 this->getSrc(0)->emit(Func);
1957 }
1958
1959 template <typename TraitsType>
emitIAS(const Cfg * Func)1960 void InstImpl<TraitsType>::InstX86Test::emitIAS(const Cfg *Func) const {
1961 assert(this->getSrcSize() == 2);
1962 const Operand *Src0 = this->getSrc(0);
1963 const Operand *Src1 = this->getSrc(1);
1964 Type Ty = Src0->getType();
1965 // The Reg/Addr form of test is not encodeable.
1966 static const GPREmitterRegOp RegEmitter = {&Assembler::test, nullptr,
1967 &Assembler::test};
1968 static const GPREmitterAddrOp AddrEmitter = {&Assembler::test,
1969 &Assembler::test};
1970 if (const auto *SrcVar0 = llvm::dyn_cast<Variable>(Src0)) {
1971 if (SrcVar0->hasReg()) {
1972 constexpr bool NotLea = false;
1973 emitIASRegOpTyGPR(Func, NotLea, Ty, SrcVar0, Src1, RegEmitter);
1974 return;
1975 }
1976 }
1977 emitIASAsAddrOpTyGPR(Func, Ty, Src0, Src1, AddrEmitter);
1978 }
1979
1980 template <typename TraitsType>
dump(const Cfg * Func)1981 void InstImpl<TraitsType>::InstX86Test::dump(const Cfg *Func) const {
1982 if (!BuildDefs::dump())
1983 return;
1984 Ostream &Str = Func->getContext()->getStrDump();
1985 Str << "test." << this->getSrc(0)->getType() << " ";
1986 this->dumpSources(Func);
1987 }
1988
1989 template <typename TraitsType>
emit(const Cfg * Func)1990 void InstImpl<TraitsType>::InstX86Mfence::emit(const Cfg *Func) const {
1991 if (!BuildDefs::dump())
1992 return;
1993 Ostream &Str = Func->getContext()->getStrEmit();
1994 assert(this->getSrcSize() == 0);
1995 Str << "\t"
1996 "mfence";
1997 }
1998
1999 template <typename TraitsType>
emitIAS(const Cfg * Func)2000 void InstImpl<TraitsType>::InstX86Mfence::emitIAS(const Cfg *Func) const {
2001 Assembler *Asm = Func->getAssembler<Assembler>();
2002 Asm->mfence();
2003 }
2004
2005 template <typename TraitsType>
dump(const Cfg * Func)2006 void InstImpl<TraitsType>::InstX86Mfence::dump(const Cfg *Func) const {
2007 if (!BuildDefs::dump())
2008 return;
2009 Ostream &Str = Func->getContext()->getStrDump();
2010 Str << "mfence";
2011 }
2012
2013 template <typename TraitsType>
emit(const Cfg * Func)2014 void InstImpl<TraitsType>::InstX86Store::emit(const Cfg *Func) const {
2015 if (!BuildDefs::dump())
2016 return;
2017 Ostream &Str = Func->getContext()->getStrEmit();
2018 assert(this->getSrcSize() == 2);
2019 Type Ty = this->getSrc(0)->getType();
2020 Str << "\t"
2021 "mov" << this->getWidthString(Ty)
2022 << Traits::TypeAttributes[Ty].SdSsString << "\t";
2023 this->getSrc(0)->emit(Func);
2024 Str << ", ";
2025 this->getSrc(1)->emit(Func);
2026 }
2027
2028 template <typename TraitsType>
emitIAS(const Cfg * Func)2029 void InstImpl<TraitsType>::InstX86Store::emitIAS(const Cfg *Func) const {
2030 assert(this->getSrcSize() == 2);
2031 const Operand *Dest = this->getSrc(1);
2032 const Operand *Src = this->getSrc(0);
2033 Type DestTy = Dest->getType();
2034 if (isScalarFloatingType(DestTy)) {
2035 // Src must be a register, since Dest is a Mem operand of some kind.
2036 const auto *SrcVar = llvm::cast<Variable>(Src);
2037 assert(SrcVar->hasReg());
2038 XmmRegister SrcReg = Traits::getEncodedXmm(SrcVar->getRegNum());
2039 Assembler *Asm = Func->getAssembler<Assembler>();
2040 auto *Target = InstX86Base::getTarget(Func);
2041 if (const auto *DestVar = llvm::dyn_cast<Variable>(Dest)) {
2042 assert(!DestVar->hasReg());
2043 Address StackAddr(Target->stackVarToAsmOperand(DestVar));
2044 Asm->movss(DestTy, StackAddr, SrcReg);
2045 } else {
2046 const auto DestMem = llvm::cast<X86OperandMem>(Dest);
2047 assert(DestMem->getSegmentRegister() == X86OperandMem::DefaultSegment);
2048 Asm->movss(DestTy, DestMem->toAsmAddress(Asm, Target), SrcReg);
2049 }
2050 return;
2051 } else {
2052 assert(isScalarIntegerType(DestTy));
2053 static const GPREmitterAddrOp GPRAddrEmitter = {&Assembler::mov,
2054 &Assembler::mov};
2055 emitIASAsAddrOpTyGPR(Func, DestTy, Dest, Src, GPRAddrEmitter);
2056 }
2057 }
2058
2059 template <typename TraitsType>
dump(const Cfg * Func)2060 void InstImpl<TraitsType>::InstX86Store::dump(const Cfg *Func) const {
2061 if (!BuildDefs::dump())
2062 return;
2063 Ostream &Str = Func->getContext()->getStrDump();
2064 Str << "mov." << this->getSrc(0)->getType() << " ";
2065 this->getSrc(1)->dump(Func);
2066 Str << ", ";
2067 this->getSrc(0)->dump(Func);
2068 }
2069
2070 template <typename TraitsType>
emit(const Cfg * Func)2071 void InstImpl<TraitsType>::InstX86StoreP::emit(const Cfg *Func) const {
2072 if (!BuildDefs::dump())
2073 return;
2074 Ostream &Str = Func->getContext()->getStrEmit();
2075 assert(this->getSrcSize() == 2);
2076 assert(isVectorType(this->getSrc(1)->getType()));
2077 Str << "\t"
2078 "movups\t";
2079 this->getSrc(0)->emit(Func);
2080 Str << ", ";
2081 this->getSrc(1)->emit(Func);
2082 }
2083
2084 template <typename TraitsType>
emitIAS(const Cfg * Func)2085 void InstImpl<TraitsType>::InstX86StoreP::emitIAS(const Cfg *Func) const {
2086 Assembler *Asm = Func->getAssembler<Assembler>();
2087 assert(this->getSrcSize() == 2);
2088 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(0));
2089 const auto DestMem = llvm::cast<X86OperandMem>(this->getSrc(1));
2090 assert(DestMem->getSegmentRegister() == X86OperandMem::DefaultSegment);
2091 assert(SrcVar->hasReg());
2092 auto *Target = InstX86Base::getTarget(Func);
2093 Asm->movups(DestMem->toAsmAddress(Asm, Target),
2094 Traits::getEncodedXmm(SrcVar->getRegNum()));
2095 }
2096
2097 template <typename TraitsType>
dump(const Cfg * Func)2098 void InstImpl<TraitsType>::InstX86StoreP::dump(const Cfg *Func) const {
2099 if (!BuildDefs::dump())
2100 return;
2101 Ostream &Str = Func->getContext()->getStrDump();
2102 Str << "storep." << this->getSrc(0)->getType() << " ";
2103 this->getSrc(1)->dump(Func);
2104 Str << ", ";
2105 this->getSrc(0)->dump(Func);
2106 }
2107
2108 template <typename TraitsType>
emit(const Cfg * Func)2109 void InstImpl<TraitsType>::InstX86StoreQ::emit(const Cfg *Func) const {
2110 if (!BuildDefs::dump())
2111 return;
2112 Ostream &Str = Func->getContext()->getStrEmit();
2113 assert(this->getSrcSize() == 2);
2114 assert(this->getSrc(1)->getType() == IceType_i64 ||
2115 this->getSrc(1)->getType() == IceType_f64 ||
2116 isVectorType(this->getSrc(1)->getType()));
2117 Str << "\t"
2118 "movq\t";
2119 this->getSrc(0)->emit(Func);
2120 Str << ", ";
2121 this->getSrc(1)->emit(Func);
2122 }
2123
2124 template <typename TraitsType>
emitIAS(const Cfg * Func)2125 void InstImpl<TraitsType>::InstX86StoreQ::emitIAS(const Cfg *Func) const {
2126 Assembler *Asm = Func->getAssembler<Assembler>();
2127 assert(this->getSrcSize() == 2);
2128 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(0));
2129 const auto DestMem = llvm::cast<X86OperandMem>(this->getSrc(1));
2130 assert(DestMem->getSegmentRegister() == X86OperandMem::DefaultSegment);
2131 assert(SrcVar->hasReg());
2132 auto *Target = InstX86Base::getTarget(Func);
2133 Asm->movq(DestMem->toAsmAddress(Asm, Target),
2134 Traits::getEncodedXmm(SrcVar->getRegNum()));
2135 }
2136
2137 template <typename TraitsType>
dump(const Cfg * Func)2138 void InstImpl<TraitsType>::InstX86StoreQ::dump(const Cfg *Func) const {
2139 if (!BuildDefs::dump())
2140 return;
2141 Ostream &Str = Func->getContext()->getStrDump();
2142 Str << "storeq." << this->getSrc(0)->getType() << " ";
2143 this->getSrc(1)->dump(Func);
2144 Str << ", ";
2145 this->getSrc(0)->dump(Func);
2146 }
2147
2148 template <typename TraitsType>
emit(const Cfg * Func)2149 void InstImpl<TraitsType>::InstX86StoreD::emit(const Cfg *Func) const {
2150 if (!BuildDefs::dump())
2151 return;
2152 Ostream &Str = Func->getContext()->getStrEmit();
2153 assert(this->getSrcSize() == 2);
2154 assert(this->getSrc(1)->getType() == IceType_i64 ||
2155 this->getSrc(1)->getType() == IceType_f64 ||
2156 isVectorType(this->getSrc(1)->getType()));
2157 Str << "\t"
2158 "movd\t";
2159 this->getSrc(0)->emit(Func);
2160 Str << ", ";
2161 this->getSrc(1)->emit(Func);
2162 }
2163
2164 template <typename TraitsType>
emitIAS(const Cfg * Func)2165 void InstImpl<TraitsType>::InstX86StoreD::emitIAS(const Cfg *Func) const {
2166 Assembler *Asm = Func->getAssembler<Assembler>();
2167 assert(this->getSrcSize() == 2);
2168 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(0));
2169 const auto DestMem = llvm::cast<X86OperandMem>(this->getSrc(1));
2170 assert(DestMem->getSegmentRegister() == X86OperandMem::DefaultSegment);
2171 assert(SrcVar->hasReg());
2172 auto *Target = InstX86Base::getTarget(Func);
2173 Asm->movd(SrcVar->getType(), DestMem->toAsmAddress(Asm, Target),
2174 Traits::getEncodedXmm(SrcVar->getRegNum()));
2175 }
2176
2177 template <typename TraitsType>
dump(const Cfg * Func)2178 void InstImpl<TraitsType>::InstX86StoreD::dump(const Cfg *Func) const {
2179 if (!BuildDefs::dump())
2180 return;
2181 Ostream &Str = Func->getContext()->getStrDump();
2182 Str << "stored." << this->getSrc(0)->getType() << " ";
2183 this->getSrc(1)->dump(Func);
2184 Str << ", ";
2185 this->getSrc(0)->dump(Func);
2186 }
2187
2188 template <typename TraitsType>
emit(const Cfg * Func)2189 void InstImpl<TraitsType>::InstX86Lea::emit(const Cfg *Func) const {
2190 if (!BuildDefs::dump())
2191 return;
2192 if (auto *Add = this->deoptLeaToAddOrNull(Func)) {
2193 Add->emit(Func);
2194 return;
2195 }
2196
2197 Ostream &Str = Func->getContext()->getStrEmit();
2198 assert(this->getSrcSize() == 1);
2199 assert(this->getDest()->hasReg());
2200 Str << "\t"
2201 "lea" << this->getWidthString(this->getDest()->getType()) << "\t";
2202 Operand *Src0 = this->getSrc(0);
2203 if (const auto *Src0Var = llvm::dyn_cast<Variable>(Src0)) {
2204 Type Ty = Src0Var->getType();
2205 // lea on x86-32 doesn't accept mem128 operands, so cast VSrc0 to an
2206 // acceptable type.
2207 Src0Var->asType(Func, isVectorType(Ty) ? IceType_i32 : Ty, RegNumT())
2208 ->emit(Func);
2209 } else {
2210 Src0->emit(Func);
2211 }
2212 Str << ", ";
2213 this->getDest()->emit(Func);
2214 }
2215
2216 template <typename TraitsType>
emit(const Cfg * Func)2217 void InstImpl<TraitsType>::InstX86Mov::emit(const Cfg *Func) const {
2218 if (!BuildDefs::dump())
2219 return;
2220 Ostream &Str = Func->getContext()->getStrEmit();
2221 assert(this->getSrcSize() == 1);
2222 Operand *Src = this->getSrc(0);
2223 Type SrcTy = Src->getType();
2224 Type DestTy = this->getDest()->getType();
2225 if (Traits::Is64Bit && DestTy == IceType_i64 &&
2226 llvm::isa<ConstantInteger64>(Src) &&
2227 !Utils::IsInt(32, llvm::cast<ConstantInteger64>(Src)->getValue())) {
2228 Str << "\t"
2229 "movabs"
2230 "\t";
2231 } else {
2232 Str << "\t"
2233 "mov" << (!isScalarFloatingType(DestTy)
2234 ? this->getWidthString(DestTy)
2235 : Traits::TypeAttributes[DestTy].SdSsString) << "\t";
2236 }
2237 // For an integer truncation operation, src is wider than dest. In this case,
2238 // we use a mov instruction whose data width matches the narrower dest.
2239 // TODO: This assert disallows usages such as copying a floating
2240 // point value between a vector and a scalar (which movss is used for). Clean
2241 // this up.
2242 assert(InstX86Base::getTarget(Func)->typeWidthInBytesOnStack(DestTy) ==
2243 InstX86Base::getTarget(Func)->typeWidthInBytesOnStack(SrcTy));
2244 const Operand *NewSrc = Src;
2245 if (auto *SrcVar = llvm::dyn_cast<Variable>(Src)) {
2246 RegNumT NewRegNum;
2247 if (SrcVar->hasReg())
2248 NewRegNum = Traits::getGprForType(DestTy, SrcVar->getRegNum());
2249 if (SrcTy != DestTy)
2250 NewSrc = SrcVar->asType(Func, DestTy, NewRegNum);
2251 }
2252 NewSrc->emit(Func);
2253 Str << ", ";
2254 this->getDest()->emit(Func);
2255 }
2256
2257 template <typename TraitsType>
emitIAS(const Cfg * Func)2258 void InstImpl<TraitsType>::InstX86Mov::emitIAS(const Cfg *Func) const {
2259 assert(this->getSrcSize() == 1);
2260 const Variable *Dest = this->getDest();
2261 const Operand *Src = this->getSrc(0);
2262 Type DestTy = Dest->getType();
2263 Type SrcTy = Src->getType();
2264 // Mov can be used for GPRs or XMM registers. Also, the type does not
2265 // necessarily match (Mov can be used for bitcasts). However, when the type
2266 // does not match, one of the operands must be a register. Thus, the strategy
2267 // is to find out if Src or Dest are a register, then use that register's
2268 // type to decide on which emitter set to use. The emitter set will include
2269 // reg-reg movs, but that case should be unused when the types don't match.
2270 static const XmmEmitterRegOp XmmRegEmitter = {&Assembler::movss,
2271 &Assembler::movss};
2272 static const GPREmitterRegOp GPRRegEmitter = {
2273 &Assembler::mov, &Assembler::mov, &Assembler::mov};
2274 static const GPREmitterAddrOp GPRAddrEmitter = {&Assembler::mov,
2275 &Assembler::mov};
2276 // For an integer truncation operation, src is wider than dest. In this case,
2277 // we use a mov instruction whose data width matches the narrower dest.
2278 // TODO: This assert disallows usages such as copying a floating
2279 // point value between a vector and a scalar (which movss is used for). Clean
2280 // this up.
2281 auto *Target = InstX86Base::getTarget(Func);
2282 assert(Target->typeWidthInBytesOnStack(this->getDest()->getType()) ==
2283 Target->typeWidthInBytesOnStack(Src->getType()));
2284 if (Dest->hasReg()) {
2285 if (isScalarFloatingType(DestTy)) {
2286 emitIASRegOpTyXMM(Func, DestTy, Dest, Src, XmmRegEmitter);
2287 return;
2288 } else {
2289 assert(isScalarIntegerType(DestTy));
2290 // Widen DestTy for truncation (see above note). We should only do this
2291 // when both Src and Dest are integer types.
2292 if (Traits::Is64Bit && DestTy == IceType_i64) {
2293 if (const auto *C64 = llvm::dyn_cast<ConstantInteger64>(Src)) {
2294 Func->getAssembler<Assembler>()->movabs(
2295 Traits::getEncodedGPR(Dest->getRegNum()), C64->getValue());
2296 return;
2297 }
2298 }
2299 if (isScalarIntegerType(SrcTy)) {
2300 SrcTy = DestTy;
2301 }
2302 constexpr bool NotLea = false;
2303 emitIASRegOpTyGPR(Func, NotLea, DestTy, Dest, Src, GPRRegEmitter);
2304 return;
2305 }
2306 } else {
2307 // Dest must be Stack and Src *could* be a register. Use Src's type to
2308 // decide on the emitters.
2309 Address StackAddr(Target->stackVarToAsmOperand(Dest));
2310 if (isScalarFloatingType(SrcTy)) {
2311 // Src must be a register.
2312 const auto *SrcVar = llvm::cast<Variable>(Src);
2313 assert(SrcVar->hasReg());
2314 Assembler *Asm = Func->getAssembler<Assembler>();
2315 Asm->movss(SrcTy, StackAddr, Traits::getEncodedXmm(SrcVar->getRegNum()));
2316 return;
2317 } else if (isVectorType(SrcTy)) {
2318 // Src must be a register
2319 const auto *SrcVar = llvm::cast<Variable>(Src);
2320 assert(SrcVar->hasReg());
2321 Assembler *Asm = Func->getAssembler<Assembler>();
2322 Asm->movups(StackAddr, Traits::getEncodedXmm(SrcVar->getRegNum()));
2323 } else {
2324 // Src can be a register or immediate.
2325 assert(isScalarIntegerType(SrcTy));
2326 emitIASAddrOpTyGPR(Func, SrcTy, StackAddr, Src, GPRAddrEmitter);
2327 return;
2328 }
2329 return;
2330 }
2331 }
2332
2333 template <typename TraitsType>
emit(const Cfg * Func)2334 void InstImpl<TraitsType>::InstX86Movd::emit(const Cfg *Func) const {
2335 if (!BuildDefs::dump())
2336 return;
2337 assert(this->getSrcSize() == 1);
2338 Variable *Dest = this->getDest();
2339 Operand *Src = this->getSrc(0);
2340
2341 if (Dest->getType() == IceType_i64 || Src->getType() == IceType_i64) {
2342 assert(Dest->getType() == IceType_f64 || Src->getType() == IceType_f64);
2343 assert(Dest->getType() != Src->getType());
2344 Ostream &Str = Func->getContext()->getStrEmit();
2345 Str << "\t"
2346 "movq"
2347 "\t";
2348 Src->emit(Func);
2349 Str << ", ";
2350 Dest->emit(Func);
2351 return;
2352 }
2353
2354 InstX86BaseUnaryopXmm<InstX86Base::Movd>::emit(Func);
2355 }
2356
2357 template <typename TraitsType>
emitIAS(const Cfg * Func)2358 void InstImpl<TraitsType>::InstX86Movd::emitIAS(const Cfg *Func) const {
2359 Assembler *Asm = Func->getAssembler<Assembler>();
2360 assert(this->getSrcSize() == 1);
2361 const Variable *Dest = this->getDest();
2362 auto *Target = InstX86Base::getTarget(Func);
2363 // For insert/extract element (one of Src/Dest is an Xmm vector and the other
2364 // is an int type).
2365 if (const auto *SrcVar = llvm::dyn_cast<Variable>(this->getSrc(0))) {
2366 if (SrcVar->getType() == IceType_i32 ||
2367 (Traits::Is64Bit && SrcVar->getType() == IceType_i64)) {
2368 assert(isVectorType(Dest->getType()) ||
2369 (isScalarFloatingType(Dest->getType()) &&
2370 typeWidthInBytes(SrcVar->getType()) ==
2371 typeWidthInBytes(Dest->getType())));
2372 assert(Dest->hasReg());
2373 XmmRegister DestReg = Traits::getEncodedXmm(Dest->getRegNum());
2374 if (SrcVar->hasReg()) {
2375 Asm->movd(SrcVar->getType(), DestReg,
2376 Traits::getEncodedGPR(SrcVar->getRegNum()));
2377 } else {
2378 Address StackAddr(Target->stackVarToAsmOperand(SrcVar));
2379 Asm->movd(SrcVar->getType(), DestReg, StackAddr);
2380 }
2381 } else {
2382 assert(isVectorType(SrcVar->getType()) ||
2383 (isScalarFloatingType(SrcVar->getType()) &&
2384 typeWidthInBytes(SrcVar->getType()) ==
2385 typeWidthInBytes(Dest->getType())));
2386 assert(SrcVar->hasReg());
2387 assert(Dest->getType() == IceType_i32 ||
2388 (Traits::Is64Bit && Dest->getType() == IceType_i64));
2389 XmmRegister SrcReg = Traits::getEncodedXmm(SrcVar->getRegNum());
2390 if (Dest->hasReg()) {
2391 Asm->movd(Dest->getType(), Traits::getEncodedGPR(Dest->getRegNum()),
2392 SrcReg);
2393 } else {
2394 Address StackAddr(Target->stackVarToAsmOperand(Dest));
2395 Asm->movd(Dest->getType(), StackAddr, SrcReg);
2396 }
2397 }
2398 } else {
2399 assert(Dest->hasReg());
2400 XmmRegister DestReg = Traits::getEncodedXmm(Dest->getRegNum());
2401 auto *Mem = llvm::cast<X86OperandMem>(this->getSrc(0));
2402 Asm->movd(Mem->getType(), DestReg, Mem->toAsmAddress(Asm, Target));
2403 }
2404 }
2405
2406 template <typename TraitsType>
emit(const Cfg * Func)2407 void InstImpl<TraitsType>::InstX86Movp::emit(const Cfg *Func) const {
2408 if (!BuildDefs::dump())
2409 return;
2410 // TODO(wala,stichnot): movups works with all vector operands, but there
2411 // exist other instructions (movaps, movdqa, movdqu) that may perform better,
2412 // depending on the data type and alignment of the operands.
2413 Ostream &Str = Func->getContext()->getStrEmit();
2414 assert(this->getSrcSize() == 1);
2415 Str << "\t"
2416 "movups\t";
2417 this->getSrc(0)->emit(Func);
2418 Str << ", ";
2419 this->getDest()->emit(Func);
2420 }
2421
2422 template <typename TraitsType>
emitIAS(const Cfg * Func)2423 void InstImpl<TraitsType>::InstX86Movp::emitIAS(const Cfg *Func) const {
2424 assert(this->getSrcSize() == 1);
2425 assert(isVectorType(this->getDest()->getType()));
2426 const Variable *Dest = this->getDest();
2427 const Operand *Src = this->getSrc(0);
2428 static const XmmEmitterMovOps Emitter = {
2429 &Assembler::movups, &Assembler::movups, &Assembler::movups};
2430 emitIASMovlikeXMM(Func, Dest, Src, Emitter);
2431 }
2432
2433 template <typename TraitsType>
emit(const Cfg * Func)2434 void InstImpl<TraitsType>::InstX86Movq::emit(const Cfg *Func) const {
2435 if (!BuildDefs::dump())
2436 return;
2437 Ostream &Str = Func->getContext()->getStrEmit();
2438 assert(this->getSrcSize() == 1);
2439 assert(this->getDest()->getType() == IceType_i64 ||
2440 this->getDest()->getType() == IceType_f64);
2441 Str << "\t"
2442 "movq"
2443 "\t";
2444 this->getSrc(0)->emit(Func);
2445 Str << ", ";
2446 this->getDest()->emit(Func);
2447 }
2448
2449 template <typename TraitsType>
emitIAS(const Cfg * Func)2450 void InstImpl<TraitsType>::InstX86Movq::emitIAS(const Cfg *Func) const {
2451 assert(this->getSrcSize() == 1);
2452 assert(this->getDest()->getType() == IceType_i64 ||
2453 this->getDest()->getType() == IceType_f64 ||
2454 isVectorType(this->getDest()->getType()));
2455 const Variable *Dest = this->getDest();
2456 const Operand *Src = this->getSrc(0);
2457 static const XmmEmitterMovOps Emitter = {&Assembler::movq, &Assembler::movq,
2458 &Assembler::movq};
2459 emitIASMovlikeXMM(Func, Dest, Src, Emitter);
2460 }
2461
2462 template <typename TraitsType>
emitIAS(const Cfg * Func)2463 void InstImpl<TraitsType>::InstX86MovssRegs::emitIAS(const Cfg *Func) const {
2464 // This is Binop variant is only intended to be used for reg-reg moves where
2465 // part of the Dest register is untouched.
2466 assert(this->getSrcSize() == 2);
2467 const Variable *Dest = this->getDest();
2468 assert(Dest == this->getSrc(0));
2469 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(1));
2470 assert(Dest->hasReg() && SrcVar->hasReg());
2471 Assembler *Asm = Func->getAssembler<Assembler>();
2472 Asm->movss(IceType_f32, Traits::getEncodedXmm(Dest->getRegNum()),
2473 Traits::getEncodedXmm(SrcVar->getRegNum()));
2474 }
2475
2476 template <typename TraitsType>
emitIAS(const Cfg * Func)2477 void InstImpl<TraitsType>::InstX86Movsx::emitIAS(const Cfg *Func) const {
2478 assert(this->getSrcSize() == 1);
2479 const Variable *Dest = this->getDest();
2480 const Operand *Src = this->getSrc(0);
2481 // Dest must be a > 8-bit register, but Src can be 8-bit. In practice we just
2482 // use the full register for Dest to avoid having an OperandSizeOverride
2483 // prefix. It also allows us to only dispatch on SrcTy.
2484 Type SrcTy = Src->getType();
2485 assert(typeWidthInBytes(Dest->getType()) > 1);
2486 assert(typeWidthInBytes(Dest->getType()) > typeWidthInBytes(SrcTy));
2487 constexpr bool NotLea = false;
2488 emitIASRegOpTyGPR<false, true>(Func, NotLea, SrcTy, Dest, Src, this->Emitter);
2489 }
2490
2491 template <typename TraitsType>
mayBeElided(const Variable * Dest,const Operand * SrcOpnd)2492 bool InstImpl<TraitsType>::InstX86Movzx::mayBeElided(
2493 const Variable *Dest, const Operand *SrcOpnd) const {
2494 assert(Traits::Is64Bit);
2495 const auto *Src = llvm::dyn_cast<Variable>(SrcOpnd);
2496
2497 // Src is not a Variable, so it does not have a register. Movzx can't be
2498 // elided.
2499 if (Src == nullptr)
2500 return false;
2501
2502 // Movzx to/from memory can't be elided.
2503 if (!Src->hasReg() || !Dest->hasReg())
2504 return false;
2505
2506 // Reg/reg move with different source and dest can't be elided.
2507 if (Traits::getEncodedGPR(Src->getRegNum()) !=
2508 Traits::getEncodedGPR(Dest->getRegNum()))
2509 return false;
2510
2511 // A must-keep movzx 32- to 64-bit is sometimes needed in x86-64 sandboxing.
2512 return !MustKeep;
2513 }
2514
2515 template <typename TraitsType>
emit(const Cfg * Func)2516 void InstImpl<TraitsType>::InstX86Movzx::emit(const Cfg *Func) const {
2517 if (!BuildDefs::dump())
2518 return;
2519 if (Traits::Is64Bit) {
2520 // There's no movzx %eXX, %rXX. To zero extend 32- to 64-bits, we emit a
2521 // mov %eXX, %eXX. The processor will still do a movzx[bw]q.
2522 assert(this->getSrcSize() == 1);
2523 const Operand *Src = this->getSrc(0);
2524 const Variable *Dest = this->Dest;
2525 if (Src->getType() == IceType_i32 && Dest->getType() == IceType_i64) {
2526 Ostream &Str = Func->getContext()->getStrEmit();
2527 if (mayBeElided(Dest, Src)) {
2528 Str << "\t/* elided movzx */";
2529 } else {
2530 Str << "\t"
2531 "mov"
2532 "\t";
2533 Src->emit(Func);
2534 Str << ", ";
2535 Dest->asType(Func, IceType_i32,
2536 Traits::getGprForType(IceType_i32, Dest->getRegNum()))
2537 ->emit(Func);
2538 Str << " /* movzx */";
2539 }
2540 return;
2541 }
2542 }
2543 InstX86BaseUnaryopGPR<InstX86Base::Movzx>::emit(Func);
2544 }
2545
2546 template <typename TraitsType>
emitIAS(const Cfg * Func)2547 void InstImpl<TraitsType>::InstX86Movzx::emitIAS(const Cfg *Func) const {
2548 assert(this->getSrcSize() == 1);
2549 const Variable *Dest = this->getDest();
2550 const Operand *Src = this->getSrc(0);
2551 Type SrcTy = Src->getType();
2552 assert(typeWidthInBytes(Dest->getType()) > 1);
2553 assert(typeWidthInBytes(Dest->getType()) > typeWidthInBytes(SrcTy));
2554 if (Traits::Is64Bit) {
2555 if (Src->getType() == IceType_i32 && Dest->getType() == IceType_i64 &&
2556 mayBeElided(Dest, Src)) {
2557 return;
2558 }
2559 }
2560 constexpr bool NotLea = false;
2561 emitIASRegOpTyGPR<false, true>(Func, NotLea, SrcTy, Dest, Src, this->Emitter);
2562 }
2563
2564 template <typename TraitsType>
emit(const Cfg * Func)2565 void InstImpl<TraitsType>::InstX86Nop::emit(const Cfg *Func) const {
2566 if (!BuildDefs::dump())
2567 return;
2568 Ostream &Str = Func->getContext()->getStrEmit();
2569 // TODO: Emit the right code for each variant.
2570 Str << "\t"
2571 "nop\t/* variant = " << Variant << " */";
2572 }
2573
2574 template <typename TraitsType>
emitIAS(const Cfg * Func)2575 void InstImpl<TraitsType>::InstX86Nop::emitIAS(const Cfg *Func) const {
2576 Assembler *Asm = Func->getAssembler<Assembler>();
2577 // TODO: Emit the right code for the variant.
2578 Asm->nop();
2579 }
2580
2581 template <typename TraitsType>
dump(const Cfg * Func)2582 void InstImpl<TraitsType>::InstX86Nop::dump(const Cfg *Func) const {
2583 if (!BuildDefs::dump())
2584 return;
2585 Ostream &Str = Func->getContext()->getStrDump();
2586 Str << "nop (variant = " << Variant << ")";
2587 }
2588
2589 template <typename TraitsType>
emit(const Cfg * Func)2590 void InstImpl<TraitsType>::InstX86Fld::emit(const Cfg *Func) const {
2591 if (!BuildDefs::dump())
2592 return;
2593 Ostream &Str = Func->getContext()->getStrEmit();
2594 assert(this->getSrcSize() == 1);
2595 Type Ty = this->getSrc(0)->getType();
2596 const auto *Var = llvm::dyn_cast<Variable>(this->getSrc(0));
2597 if (Var && Var->hasReg()) {
2598 // This is a physical xmm register, so we need to spill it to a temporary
2599 // stack slot. Function prolog emission guarantees that there is sufficient
2600 // space to do this.
2601 Str << "\t"
2602 "mov" << Traits::TypeAttributes[Ty].SdSsString << "\t";
2603 Var->emit(Func);
2604 Str << ", (%esp)\n"
2605 "\t"
2606 "fld" << this->getFldString(Ty) << "\t"
2607 "(%esp)";
2608 return;
2609 }
2610 Str << "\t"
2611 "fld" << this->getFldString(Ty) << "\t";
2612 this->getSrc(0)->emit(Func);
2613 }
2614
2615 template <typename TraitsType>
emitIAS(const Cfg * Func)2616 void InstImpl<TraitsType>::InstX86Fld::emitIAS(const Cfg *Func) const {
2617 Assembler *Asm = Func->getAssembler<Assembler>();
2618 assert(this->getSrcSize() == 1);
2619 const Operand *Src = this->getSrc(0);
2620 auto *Target = InstX86Base::getTarget(Func);
2621 Type Ty = Src->getType();
2622 if (const auto *Var = llvm::dyn_cast<Variable>(Src)) {
2623 if (Var->hasReg()) {
2624 // This is a physical xmm register, so we need to spill it to a temporary
2625 // stack slot. Function prolog emission guarantees that there is
2626 // sufficient space to do this.
2627 Address StackSlot =
2628 Address(RegisterSet::Encoded_Reg_esp, 0, AssemblerFixup::NoFixup);
2629 Asm->movss(Ty, StackSlot, Traits::getEncodedXmm(Var->getRegNum()));
2630 Asm->fld(Ty, StackSlot);
2631 } else {
2632 Address StackAddr(Target->stackVarToAsmOperand(Var));
2633 Asm->fld(Ty, StackAddr);
2634 }
2635 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) {
2636 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
2637 Asm->fld(Ty, Mem->toAsmAddress(Asm, Target));
2638 } else if (const auto *Imm = llvm::dyn_cast<Constant>(Src)) {
2639 Asm->fld(Ty, Traits::Address::ofConstPool(Asm, Imm));
2640 } else {
2641 llvm_unreachable("Unexpected operand type");
2642 }
2643 }
2644
2645 template <typename TraitsType>
dump(const Cfg * Func)2646 void InstImpl<TraitsType>::InstX86Fld::dump(const Cfg *Func) const {
2647 if (!BuildDefs::dump())
2648 return;
2649 Ostream &Str = Func->getContext()->getStrDump();
2650 Str << "fld." << this->getSrc(0)->getType() << " ";
2651 this->dumpSources(Func);
2652 }
2653
2654 template <typename TraitsType>
emit(const Cfg * Func)2655 void InstImpl<TraitsType>::InstX86Fstp::emit(const Cfg *Func) const {
2656 if (!BuildDefs::dump())
2657 return;
2658 Ostream &Str = Func->getContext()->getStrEmit();
2659 assert(this->getSrcSize() == 0);
2660 // TODO(jvoung,stichnot): Utilize this by setting Dest to nullptr to
2661 // "partially" delete the fstp if the Dest is unused. Even if Dest is unused,
2662 // the fstp should be kept for the SideEffects of popping the stack.
2663 if (!this->getDest()) {
2664 Str << "\t"
2665 "fstp\t"
2666 "st(0)";
2667 return;
2668 }
2669 Type Ty = this->getDest()->getType();
2670 if (!this->getDest()->hasReg()) {
2671 Str << "\t"
2672 "fstp" << this->getFldString(Ty) << "\t";
2673 this->getDest()->emit(Func);
2674 return;
2675 }
2676 // Dest is a physical (xmm) register, so st(0) needs to go through memory.
2677 // Hack this by using caller-reserved memory at the top of stack, spilling
2678 // st(0) there, and loading it into the xmm register.
2679 Str << "\t"
2680 "fstp" << this->getFldString(Ty) << "\t"
2681 "(%esp)\n";
2682 Str << "\t"
2683 "mov" << Traits::TypeAttributes[Ty].SdSsString << "\t"
2684 "(%esp), ";
2685 this->getDest()->emit(Func);
2686 }
2687
2688 template <typename TraitsType>
emitIAS(const Cfg * Func)2689 void InstImpl<TraitsType>::InstX86Fstp::emitIAS(const Cfg *Func) const {
2690 Assembler *Asm = Func->getAssembler<Assembler>();
2691 assert(this->getSrcSize() == 0);
2692 const Variable *Dest = this->getDest();
2693 // TODO(jvoung,stichnot): Utilize this by setting Dest to nullptr to
2694 // "partially" delete the fstp if the Dest is unused. Even if Dest is unused,
2695 // the fstp should be kept for the SideEffects of popping the stack.
2696 if (!Dest) {
2697 Asm->fstp(RegisterSet::getEncodedSTReg(0));
2698 return;
2699 }
2700 auto *Target = InstX86Base::getTarget(Func);
2701 Type Ty = Dest->getType();
2702 if (!Dest->hasReg()) {
2703 Address StackAddr(Target->stackVarToAsmOperand(Dest));
2704 Asm->fstp(Ty, StackAddr);
2705 } else {
2706 // Dest is a physical (xmm) register, so st(0) needs to go through memory.
2707 // Hack this by using caller-reserved memory at the top of stack, spilling
2708 // st(0) there, and loading it into the xmm register.
2709 Address StackSlot =
2710 Address(RegisterSet::Encoded_Reg_esp, 0, AssemblerFixup::NoFixup);
2711 Asm->fstp(Ty, StackSlot);
2712 Asm->movss(Ty, Traits::getEncodedXmm(Dest->getRegNum()), StackSlot);
2713 }
2714 }
2715
2716 template <typename TraitsType>
dump(const Cfg * Func)2717 void InstImpl<TraitsType>::InstX86Fstp::dump(const Cfg *Func) const {
2718 if (!BuildDefs::dump())
2719 return;
2720 Ostream &Str = Func->getContext()->getStrDump();
2721 this->dumpDest(Func);
2722 Str << " = fstp." << this->getDest()->getType() << ", st(0)";
2723 }
2724
2725 template <typename TraitsType>
emit(const Cfg * Func)2726 void InstImpl<TraitsType>::InstX86Pextr::emit(const Cfg *Func) const {
2727 if (!BuildDefs::dump())
2728 return;
2729 Ostream &Str = Func->getContext()->getStrEmit();
2730 assert(this->getSrcSize() == 2);
2731 // pextrb and pextrd are SSE4.1 instructions.
2732 Str << "\t" << this->Opcode
2733 << Traits::TypeAttributes[this->getSrc(0)->getType()].IntegralString
2734 << "\t";
2735 this->getSrc(1)->emit(Func);
2736 Str << ", ";
2737 this->getSrc(0)->emit(Func);
2738 Str << ", ";
2739 Variable *Dest = this->getDest();
2740 // pextrw must take a register dest. There is an SSE4.1 version that takes a
2741 // memory dest, but we aren't using it. For uniformity, just restrict them
2742 // all to have a register dest for now.
2743 assert(Dest->hasReg());
2744 Dest->asType(Func, IceType_i32, Dest->getRegNum())->emit(Func);
2745 }
2746
2747 template <typename TraitsType>
emitIAS(const Cfg * Func)2748 void InstImpl<TraitsType>::InstX86Pextr::emitIAS(const Cfg *Func) const {
2749 assert(this->getSrcSize() == 2);
2750 // pextrb and pextrd are SSE4.1 instructions.
2751 const Variable *Dest = this->getDest();
2752 Type DispatchTy = Traits::getInVectorElementType(this->getSrc(0)->getType());
2753 // pextrw must take a register dest. There is an SSE4.1 version that takes a
2754 // memory dest, but we aren't using it. For uniformity, just restrict them
2755 // all to have a register dest for now.
2756 assert(Dest->hasReg());
2757 // pextrw's Src(0) must be a register (both SSE4.1 and SSE2).
2758 assert(llvm::cast<Variable>(this->getSrc(0))->hasReg());
2759 static const ThreeOpImmEmitter<GPRRegister, XmmRegister> Emitter = {
2760 &Assembler::pextr, nullptr};
2761 emitIASThreeOpImmOps<GPRRegister, XmmRegister, Traits::getEncodedGPR,
2762 Traits::getEncodedXmm>(
2763 Func, DispatchTy, Dest, this->getSrc(0), this->getSrc(1), Emitter);
2764 }
2765
2766 template <typename TraitsType>
emit(const Cfg * Func)2767 void InstImpl<TraitsType>::InstX86Pinsr::emit(const Cfg *Func) const {
2768 if (!BuildDefs::dump())
2769 return;
2770 Ostream &Str = Func->getContext()->getStrEmit();
2771 assert(this->getSrcSize() == 3);
2772 Str << "\t" << this->Opcode
2773 << Traits::TypeAttributes[this->getDest()->getType()].IntegralString
2774 << "\t";
2775 this->getSrc(2)->emit(Func);
2776 Str << ", ";
2777 Operand *Src1 = this->getSrc(1);
2778 if (const auto *Src1Var = llvm::dyn_cast<Variable>(Src1)) {
2779 // If src1 is a register, it should always be r32.
2780 if (Src1Var->hasReg()) {
2781 const auto NewRegNum = Traits::getBaseReg(Src1Var->getRegNum());
2782 const Variable *NewSrc = Src1Var->asType(Func, IceType_i32, NewRegNum);
2783 NewSrc->emit(Func);
2784 } else {
2785 Src1Var->emit(Func);
2786 }
2787 } else {
2788 Src1->emit(Func);
2789 }
2790 Str << ", ";
2791 this->getDest()->emit(Func);
2792 }
2793
2794 template <typename TraitsType>
emitIAS(const Cfg * Func)2795 void InstImpl<TraitsType>::InstX86Pinsr::emitIAS(const Cfg *Func) const {
2796 assert(this->getSrcSize() == 3);
2797 assert(this->getDest() == this->getSrc(0));
2798 // pinsrb and pinsrd are SSE4.1 instructions.
2799 const Operand *Src0 = this->getSrc(1);
2800 Type DispatchTy = Src0->getType();
2801 // If src1 is a register, it should always be r32 (this should fall out from
2802 // the encodings for ByteRegs overlapping the encodings for r32), but we have
2803 // to make sure the register allocator didn't choose an 8-bit high register
2804 // like "ah".
2805 if (BuildDefs::asserts()) {
2806 if (auto *Src0Var = llvm::dyn_cast<Variable>(Src0)) {
2807 if (Src0Var->hasReg()) {
2808 const auto RegNum = Src0Var->getRegNum();
2809 const auto BaseRegNum = Traits::getBaseReg(RegNum);
2810 (void)BaseRegNum;
2811 assert(Traits::getEncodedGPR(RegNum) ==
2812 Traits::getEncodedGPR(BaseRegNum));
2813 }
2814 }
2815 }
2816 static const ThreeOpImmEmitter<XmmRegister, GPRRegister> Emitter = {
2817 &Assembler::pinsr, &Assembler::pinsr};
2818 emitIASThreeOpImmOps<XmmRegister, GPRRegister, Traits::getEncodedXmm,
2819 Traits::getEncodedGPR>(Func, DispatchTy, this->getDest(),
2820 Src0, this->getSrc(2), Emitter);
2821 }
2822
2823 template <typename TraitsType>
emitIAS(const Cfg * Func)2824 void InstImpl<TraitsType>::InstX86Pshufd::emitIAS(const Cfg *Func) const {
2825 assert(this->getSrcSize() == 2);
2826 const Variable *Dest = this->getDest();
2827 Type Ty = Dest->getType();
2828 static const ThreeOpImmEmitter<XmmRegister, XmmRegister> Emitter = {
2829 &Assembler::pshufd, &Assembler::pshufd};
2830 emitIASThreeOpImmOps<XmmRegister, XmmRegister, Traits::getEncodedXmm,
2831 Traits::getEncodedXmm>(Func, Ty, Dest, this->getSrc(0),
2832 this->getSrc(1), Emitter);
2833 }
2834
2835 template <typename TraitsType>
emitIAS(const Cfg * Func)2836 void InstImpl<TraitsType>::InstX86Shufps::emitIAS(const Cfg *Func) const {
2837 assert(this->getSrcSize() == 3);
2838 const Variable *Dest = this->getDest();
2839 assert(Dest == this->getSrc(0));
2840 Type Ty = Dest->getType();
2841 static const ThreeOpImmEmitter<XmmRegister, XmmRegister> Emitter = {
2842 &Assembler::shufps, &Assembler::shufps};
2843 emitIASThreeOpImmOps<XmmRegister, XmmRegister, Traits::getEncodedXmm,
2844 Traits::getEncodedXmm>(Func, Ty, Dest, this->getSrc(1),
2845 this->getSrc(2), Emitter);
2846 }
2847
2848 template <typename TraitsType>
emit(const Cfg * Func)2849 void InstImpl<TraitsType>::InstX86Pop::emit(const Cfg *Func) const {
2850 if (!BuildDefs::dump())
2851 return;
2852 Ostream &Str = Func->getContext()->getStrEmit();
2853 assert(this->getSrcSize() == 0);
2854 Str << "\t"
2855 "pop\t";
2856 this->getDest()->emit(Func);
2857 }
2858
2859 template <typename TraitsType>
emitIAS(const Cfg * Func)2860 void InstImpl<TraitsType>::InstX86Pop::emitIAS(const Cfg *Func) const {
2861 assert(this->getSrcSize() == 0);
2862 Assembler *Asm = Func->getAssembler<Assembler>();
2863 if (this->getDest()->hasReg()) {
2864 Asm->popl(Traits::getEncodedGPR(this->getDest()->getRegNum()));
2865 } else {
2866 auto *Target = InstX86Base::getTarget(Func);
2867 Asm->popl(Target->stackVarToAsmOperand(this->getDest()));
2868 }
2869 }
2870
2871 template <typename TraitsType>
dump(const Cfg * Func)2872 void InstImpl<TraitsType>::InstX86Pop::dump(const Cfg *Func) const {
2873 if (!BuildDefs::dump())
2874 return;
2875 Ostream &Str = Func->getContext()->getStrDump();
2876 this->dumpDest(Func);
2877 Str << " = pop." << this->getDest()->getType() << " ";
2878 }
2879
2880 template <typename TraitsType>
emit(const Cfg * Func)2881 void InstImpl<TraitsType>::InstX86Push::emit(const Cfg *Func) const {
2882 if (!BuildDefs::dump())
2883 return;
2884 Ostream &Str = Func->getContext()->getStrEmit();
2885 Str << "\t"
2886 "push"
2887 "\t";
2888 assert(this->getSrcSize() == 1);
2889 const Operand *Src = this->getSrc(0);
2890 Src->emit(Func);
2891 }
2892
2893 template <typename TraitsType>
emitIAS(const Cfg * Func)2894 void InstImpl<TraitsType>::InstX86Push::emitIAS(const Cfg *Func) const {
2895 Assembler *Asm = Func->getAssembler<Assembler>();
2896
2897 assert(this->getSrcSize() == 1);
2898 const Operand *Src = this->getSrc(0);
2899
2900 if (const auto *Var = llvm::dyn_cast<Variable>(Src)) {
2901 Asm->pushl(Traits::getEncodedGPR(Var->getRegNum()));
2902 } else if (const auto *Const32 = llvm::dyn_cast<ConstantInteger32>(Src)) {
2903 Asm->pushl(AssemblerImmediate(Const32->getValue()));
2904 } else if (auto *CR = llvm::dyn_cast<ConstantRelocatable>(Src)) {
2905 Asm->pushl(CR);
2906 } else {
2907 llvm_unreachable("Unexpected operand type");
2908 }
2909 }
2910
2911 template <typename TraitsType>
dump(const Cfg * Func)2912 void InstImpl<TraitsType>::InstX86Push::dump(const Cfg *Func) const {
2913 if (!BuildDefs::dump())
2914 return;
2915 Ostream &Str = Func->getContext()->getStrDump();
2916 Str << "push." << this->getSrc(0)->getType() << " ";
2917 this->dumpSources(Func);
2918 }
2919
2920 template <typename TraitsType>
emit(const Cfg * Func)2921 void InstImpl<TraitsType>::InstX86Ret::emit(const Cfg *Func) const {
2922 if (!BuildDefs::dump())
2923 return;
2924 Ostream &Str = Func->getContext()->getStrEmit();
2925 Str << "\t"
2926 "ret";
2927 }
2928
2929 template <typename TraitsType>
emitIAS(const Cfg * Func)2930 void InstImpl<TraitsType>::InstX86Ret::emitIAS(const Cfg *Func) const {
2931 Assembler *Asm = Func->getAssembler<Assembler>();
2932 Asm->ret();
2933 }
2934
2935 template <typename TraitsType>
dump(const Cfg * Func)2936 void InstImpl<TraitsType>::InstX86Ret::dump(const Cfg *Func) const {
2937 if (!BuildDefs::dump())
2938 return;
2939 Ostream &Str = Func->getContext()->getStrDump();
2940 Type Ty =
2941 (this->getSrcSize() == 0 ? IceType_void : this->getSrc(0)->getType());
2942 Str << "ret." << Ty << " ";
2943 this->dumpSources(Func);
2944 }
2945
2946 template <typename TraitsType>
emit(const Cfg * Func)2947 void InstImpl<TraitsType>::InstX86Setcc::emit(const Cfg *Func) const {
2948 if (!BuildDefs::dump())
2949 return;
2950 Ostream &Str = Func->getContext()->getStrEmit();
2951 Str << "\t"
2952 "set" << Traits::InstBrAttributes[Condition].DisplayString << "\t";
2953 this->Dest->emit(Func);
2954 }
2955
2956 template <typename TraitsType>
emitIAS(const Cfg * Func)2957 void InstImpl<TraitsType>::InstX86Setcc::emitIAS(const Cfg *Func) const {
2958 assert(Condition != Cond::Br_None);
2959 assert(this->getDest()->getType() == IceType_i1);
2960 assert(this->getSrcSize() == 0);
2961 Assembler *Asm = Func->getAssembler<Assembler>();
2962 auto *Target = InstX86Base::getTarget(Func);
2963 if (this->getDest()->hasReg())
2964 Asm->setcc(Condition,
2965 Traits::getEncodedByteReg(this->getDest()->getRegNum()));
2966 else
2967 Asm->setcc(Condition, Target->stackVarToAsmOperand(this->getDest()));
2968 return;
2969 }
2970
2971 template <typename TraitsType>
dump(const Cfg * Func)2972 void InstImpl<TraitsType>::InstX86Setcc::dump(const Cfg *Func) const {
2973 if (!BuildDefs::dump())
2974 return;
2975 Ostream &Str = Func->getContext()->getStrDump();
2976 Str << "setcc." << Traits::InstBrAttributes[Condition].DisplayString << " ";
2977 this->dumpDest(Func);
2978 }
2979
2980 template <typename TraitsType>
emit(const Cfg * Func)2981 void InstImpl<TraitsType>::InstX86Xadd::emit(const Cfg *Func) const {
2982 if (!BuildDefs::dump())
2983 return;
2984 Ostream &Str = Func->getContext()->getStrEmit();
2985 if (this->Locked) {
2986 Str << "\t"
2987 "lock";
2988 }
2989 Str << "\t"
2990 "xadd" << this->getWidthString(this->getSrc(0)->getType()) << "\t";
2991 this->getSrc(1)->emit(Func);
2992 Str << ", ";
2993 this->getSrc(0)->emit(Func);
2994 }
2995
2996 template <typename TraitsType>
emitIAS(const Cfg * Func)2997 void InstImpl<TraitsType>::InstX86Xadd::emitIAS(const Cfg *Func) const {
2998 assert(this->getSrcSize() == 2);
2999 Assembler *Asm = Func->getAssembler<Assembler>();
3000 Type Ty = this->getSrc(0)->getType();
3001 const auto Mem = llvm::cast<X86OperandMem>(this->getSrc(0));
3002 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
3003 auto *Target = InstX86Base::getTarget(Func);
3004 const Address Addr = Mem->toAsmAddress(Asm, Target);
3005 const auto *VarReg = llvm::cast<Variable>(this->getSrc(1));
3006 assert(VarReg->hasReg());
3007 const GPRRegister Reg = Traits::getEncodedGPR(VarReg->getRegNum());
3008 Asm->xadd(Ty, Addr, Reg, this->Locked);
3009 }
3010
3011 template <typename TraitsType>
dump(const Cfg * Func)3012 void InstImpl<TraitsType>::InstX86Xadd::dump(const Cfg *Func) const {
3013 if (!BuildDefs::dump())
3014 return;
3015 Ostream &Str = Func->getContext()->getStrDump();
3016 if (this->Locked) {
3017 Str << "lock ";
3018 }
3019 Type Ty = this->getSrc(0)->getType();
3020 Str << "xadd." << Ty << " ";
3021 this->dumpSources(Func);
3022 }
3023
3024 template <typename TraitsType>
emit(const Cfg * Func)3025 void InstImpl<TraitsType>::InstX86Xchg::emit(const Cfg *Func) const {
3026 if (!BuildDefs::dump())
3027 return;
3028 Ostream &Str = Func->getContext()->getStrEmit();
3029 Str << "\t"
3030 "xchg" << this->getWidthString(this->getSrc(0)->getType()) << "\t";
3031 this->getSrc(1)->emit(Func);
3032 Str << ", ";
3033 this->getSrc(0)->emit(Func);
3034 }
3035
3036 template <typename TraitsType>
emitIAS(const Cfg * Func)3037 void InstImpl<TraitsType>::InstX86Xchg::emitIAS(const Cfg *Func) const {
3038 assert(this->getSrcSize() == 2);
3039 Assembler *Asm = Func->getAssembler<Assembler>();
3040 Type Ty = this->getSrc(0)->getType();
3041 const auto *VarReg1 = llvm::cast<Variable>(this->getSrc(1));
3042 assert(VarReg1->hasReg());
3043 const GPRRegister Reg1 = Traits::getEncodedGPR(VarReg1->getRegNum());
3044
3045 if (const auto *VarReg0 = llvm::dyn_cast<Variable>(this->getSrc(0))) {
3046 assert(VarReg0->hasReg());
3047 const GPRRegister Reg0 = Traits::getEncodedGPR(VarReg0->getRegNum());
3048 Asm->xchg(Ty, Reg0, Reg1);
3049 return;
3050 }
3051
3052 const auto *Mem = llvm::cast<X86OperandMem>(this->getSrc(0));
3053 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment);
3054 auto *Target = InstX86Base::getTarget(Func);
3055 const Address Addr = Mem->toAsmAddress(Asm, Target);
3056 Asm->xchg(Ty, Addr, Reg1);
3057 }
3058
3059 template <typename TraitsType>
dump(const Cfg * Func)3060 void InstImpl<TraitsType>::InstX86Xchg::dump(const Cfg *Func) const {
3061 if (!BuildDefs::dump())
3062 return;
3063 Ostream &Str = Func->getContext()->getStrDump();
3064 Type Ty = this->getSrc(0)->getType();
3065 Str << "xchg." << Ty << " ";
3066 this->dumpSources(Func);
3067 }
3068
3069 template <typename TraitsType>
emit(const Cfg * Func)3070 void InstImpl<TraitsType>::InstX86IacaStart::emit(const Cfg *Func) const {
3071 if (!BuildDefs::dump())
3072 return;
3073 Ostream &Str = Func->getContext()->getStrEmit();
3074 Str << "\t# IACA_START\n"
3075 "\t.byte 0x0F, 0x0B\n"
3076 "\t"
3077 "movl\t$111, %ebx\n"
3078 "\t.byte 0x64, 0x67, 0x90";
3079 }
3080
3081 template <typename TraitsType>
emitIAS(const Cfg * Func)3082 void InstImpl<TraitsType>::InstX86IacaStart::emitIAS(const Cfg *Func) const {
3083 Assembler *Asm = Func->getAssembler<Assembler>();
3084 Asm->iaca_start();
3085 }
3086
3087 template <typename TraitsType>
dump(const Cfg * Func)3088 void InstImpl<TraitsType>::InstX86IacaStart::dump(const Cfg *Func) const {
3089 if (!BuildDefs::dump())
3090 return;
3091 Ostream &Str = Func->getContext()->getStrDump();
3092 Str << "IACA_START";
3093 }
3094
3095 template <typename TraitsType>
emit(const Cfg * Func)3096 void InstImpl<TraitsType>::InstX86IacaEnd::emit(const Cfg *Func) const {
3097 if (!BuildDefs::dump())
3098 return;
3099 Ostream &Str = Func->getContext()->getStrEmit();
3100 Str << "\t# IACA_END\n"
3101 "\t"
3102 "movl\t$222, %ebx\n"
3103 "\t.byte 0x64, 0x67, 0x90\n"
3104 "\t.byte 0x0F, 0x0B";
3105 }
3106
3107 template <typename TraitsType>
emitIAS(const Cfg * Func)3108 void InstImpl<TraitsType>::InstX86IacaEnd::emitIAS(const Cfg *Func) const {
3109 Assembler *Asm = Func->getAssembler<Assembler>();
3110 Asm->iaca_end();
3111 }
3112
3113 template <typename TraitsType>
dump(const Cfg * Func)3114 void InstImpl<TraitsType>::InstX86IacaEnd::dump(const Cfg *Func) const {
3115 if (!BuildDefs::dump())
3116 return;
3117 Ostream &Str = Func->getContext()->getStrDump();
3118 Str << "IACA_END";
3119 }
3120
3121 } // end of namespace X86NAMESPACE
3122
3123 } // end of namespace Ice
3124
3125 #endif // SUBZERO_SRC_ICEINSTX86BASEIMPL_H
3126