1 //===- subzero/src/IceAssemblerX86BaseImpl.h - base x86 assembler -*- C++ -*-=//
2 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
3 // for details. All rights reserved. Use of this source code is governed by a
4 // BSD-style license that can be found in the LICENSE file.
5 //
6 // Modified by the Subzero authors.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // The Subzero Code Generator
11 //
12 // This file is distributed under the University of Illinois Open Source
13 // License. See LICENSE.TXT for details.
14 //
15 //===----------------------------------------------------------------------===//
16 //
17 /// \file
18 /// \brief Implements the AssemblerX86Base template class, which is the base
19 /// Assembler class for X86 assemblers.
20 //
21 //===----------------------------------------------------------------------===//
22
23 #include "IceAssemblerX86Base.h"
24
25 #include "IceCfg.h"
26 #include "IceCfgNode.h"
27 #include "IceOperand.h"
28
29 namespace Ice {
30 namespace X86NAMESPACE {
31
32 template <typename TraitsType>
~AssemblerX86Base()33 AssemblerX86Base<TraitsType>::~AssemblerX86Base() {
34 if (BuildDefs::asserts()) {
35 for (const Label *Label : CfgNodeLabels) {
36 Label->finalCheck();
37 }
38 for (const Label *Label : LocalLabels) {
39 Label->finalCheck();
40 }
41 }
42 }
43
44 template <typename TraitsType>
alignFunction()45 void AssemblerX86Base<TraitsType>::alignFunction() {
46 const SizeT Align = 1 << getBundleAlignLog2Bytes();
47 SizeT BytesNeeded = Utils::OffsetToAlignment(Buffer.getPosition(), Align);
48 constexpr SizeT HltSize = 1;
49 while (BytesNeeded > 0) {
50 hlt();
51 BytesNeeded -= HltSize;
52 }
53 }
54
55 template <typename TraitsType>
56 typename AssemblerX86Base<TraitsType>::Label *
getOrCreateLabel(SizeT Number,LabelVector & Labels)57 AssemblerX86Base<TraitsType>::getOrCreateLabel(SizeT Number,
58 LabelVector &Labels) {
59 Label *L = nullptr;
60 if (Number == Labels.size()) {
61 L = new (this->allocate<Label>()) Label();
62 Labels.push_back(L);
63 return L;
64 }
65 if (Number > Labels.size()) {
66 Utils::reserveAndResize(Labels, Number + 1);
67 }
68 L = Labels[Number];
69 if (!L) {
70 L = new (this->allocate<Label>()) Label();
71 Labels[Number] = L;
72 }
73 return L;
74 }
75
76 template <typename TraitsType>
getCfgNodeLabel(SizeT NodeNumber)77 Ice::Label *AssemblerX86Base<TraitsType>::getCfgNodeLabel(SizeT NodeNumber) {
78 assert(NodeNumber < CfgNodeLabels.size());
79 return CfgNodeLabels[NodeNumber];
80 }
81
82 template <typename TraitsType>
83 typename AssemblerX86Base<TraitsType>::Label *
getOrCreateCfgNodeLabel(SizeT NodeNumber)84 AssemblerX86Base<TraitsType>::getOrCreateCfgNodeLabel(SizeT NodeNumber) {
85 return getOrCreateLabel(NodeNumber, CfgNodeLabels);
86 }
87
88 template <typename TraitsType>
89 typename AssemblerX86Base<TraitsType>::Label *
getOrCreateLocalLabel(SizeT Number)90 AssemblerX86Base<TraitsType>::getOrCreateLocalLabel(SizeT Number) {
91 return getOrCreateLabel(Number, LocalLabels);
92 }
93
94 template <typename TraitsType>
bindCfgNodeLabel(const CfgNode * Node)95 void AssemblerX86Base<TraitsType>::bindCfgNodeLabel(const CfgNode *Node) {
96 assert(!getPreliminary());
97 Label *L = getOrCreateCfgNodeLabel(Node->getIndex());
98 this->bind(L);
99 }
100
101 template <typename TraitsType>
bindLocalLabel(SizeT Number)102 void AssemblerX86Base<TraitsType>::bindLocalLabel(SizeT Number) {
103 Label *L = getOrCreateLocalLabel(Number);
104 if (!getPreliminary())
105 this->bind(L);
106 }
107
108 template <typename TraitsType>
call(GPRRegister reg)109 void AssemblerX86Base<TraitsType>::call(GPRRegister reg) {
110 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
111 emitRexB(RexTypeIrrelevant, reg);
112 emitUint8(0xFF);
113 emitRegisterOperand(2, gprEncoding(reg));
114 }
115
116 template <typename TraitsType>
call(const Address & address)117 void AssemblerX86Base<TraitsType>::call(const Address &address) {
118 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
119 emitAddrSizeOverridePrefix();
120 emitRex(RexTypeIrrelevant, address, RexRegIrrelevant);
121 emitUint8(0xFF);
122 emitOperand(2, address);
123 }
124
125 template <typename TraitsType>
call(const ConstantRelocatable * label)126 void AssemblerX86Base<TraitsType>::call(const ConstantRelocatable *label) {
127 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
128 intptr_t call_start = Buffer.getPosition();
129 emitUint8(0xE8);
130 auto *Fixup = this->createFixup(Traits::FK_PcRel, label);
131 Fixup->set_addend(-4);
132 emitFixup(Fixup);
133 emitInt32(0);
134 assert((Buffer.getPosition() - call_start) == kCallExternalLabelSize);
135 (void)call_start;
136 }
137
138 template <typename TraitsType>
call(const Immediate & abs_address)139 void AssemblerX86Base<TraitsType>::call(const Immediate &abs_address) {
140 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
141 intptr_t call_start = Buffer.getPosition();
142 emitUint8(0xE8);
143 auto *Fixup = this->createFixup(Traits::FK_PcRel, AssemblerFixup::NullSymbol);
144 Fixup->set_addend(abs_address.value() - 4);
145 emitFixup(Fixup);
146 emitInt32(0);
147 assert((Buffer.getPosition() - call_start) == kCallExternalLabelSize);
148 (void)call_start;
149 }
150
151 template <typename TraitsType>
pushl(GPRRegister reg)152 void AssemblerX86Base<TraitsType>::pushl(GPRRegister reg) {
153 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
154 emitRexB(RexTypeIrrelevant, reg);
155 emitUint8(0x50 + gprEncoding(reg));
156 }
157
158 template <typename TraitsType>
pushl(const Immediate & Imm)159 void AssemblerX86Base<TraitsType>::pushl(const Immediate &Imm) {
160 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
161 emitUint8(0x68);
162 emitInt32(Imm.value());
163 }
164
165 template <typename TraitsType>
pushl(const ConstantRelocatable * Label)166 void AssemblerX86Base<TraitsType>::pushl(const ConstantRelocatable *Label) {
167 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
168 emitUint8(0x68);
169 emitFixup(this->createFixup(Traits::FK_Abs, Label));
170 // In x86-32, the emitted value is an addend to the relocation. Therefore, we
171 // must emit a 0 (because we're pushing an absolute relocation.)
172 // In x86-64, the emitted value does not matter (the addend lives in the
173 // relocation record as an extra field.)
174 emitInt32(0);
175 }
176
177 template <typename TraitsType>
popl(GPRRegister reg)178 void AssemblerX86Base<TraitsType>::popl(GPRRegister reg) {
179 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
180 // Any type that would not force a REX prefix to be emitted can be provided
181 // here.
182 emitRexB(RexTypeIrrelevant, reg);
183 emitUint8(0x58 + gprEncoding(reg));
184 }
185
186 template <typename TraitsType>
popl(const Address & address)187 void AssemblerX86Base<TraitsType>::popl(const Address &address) {
188 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
189 emitAddrSizeOverridePrefix();
190 emitRex(RexTypeIrrelevant, address, RexRegIrrelevant);
191 emitUint8(0x8F);
192 emitOperand(0, address);
193 }
194
195 template <typename TraitsType>
196 template <typename, typename>
pushal()197 void AssemblerX86Base<TraitsType>::pushal() {
198 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
199 emitUint8(0x60);
200 }
201
202 template <typename TraitsType>
203 template <typename, typename>
popal()204 void AssemblerX86Base<TraitsType>::popal() {
205 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
206 emitUint8(0x61);
207 }
208
209 template <typename TraitsType>
setcc(BrCond condition,ByteRegister dst)210 void AssemblerX86Base<TraitsType>::setcc(BrCond condition, ByteRegister dst) {
211 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
212 emitRexB(IceType_i8, dst);
213 emitUint8(0x0F);
214 emitUint8(0x90 + condition);
215 emitUint8(0xC0 + gprEncoding(dst));
216 }
217
218 template <typename TraitsType>
setcc(BrCond condition,const Address & address)219 void AssemblerX86Base<TraitsType>::setcc(BrCond condition,
220 const Address &address) {
221 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
222 emitAddrSizeOverridePrefix();
223 emitRex(RexTypeIrrelevant, address, RexRegIrrelevant);
224 emitUint8(0x0F);
225 emitUint8(0x90 + condition);
226 emitOperand(0, address);
227 }
228
229 template <typename TraitsType>
mov(Type Ty,GPRRegister dst,const Immediate & imm)230 void AssemblerX86Base<TraitsType>::mov(Type Ty, GPRRegister dst,
231 const Immediate &imm) {
232 assert(Ty != IceType_i64 && "i64 not supported yet.");
233 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
234 if (Ty == IceType_i16)
235 emitOperandSizeOverride();
236 emitRexB(Ty, dst);
237 if (isByteSizedType(Ty)) {
238 emitUint8(0xB0 + gprEncoding(dst));
239 emitUint8(imm.value() & 0xFF);
240 } else {
241 // TODO(jpp): When removing the assertion above ensure that in x86-64 we
242 // emit a 64-bit immediate.
243 emitUint8(0xB8 + gprEncoding(dst));
244 emitImmediate(Ty, imm);
245 }
246 }
247
248 template <typename TraitsType>
mov(Type Ty,GPRRegister dst,GPRRegister src)249 void AssemblerX86Base<TraitsType>::mov(Type Ty, GPRRegister dst,
250 GPRRegister src) {
251 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
252 if (Ty == IceType_i16)
253 emitOperandSizeOverride();
254 emitRexRB(Ty, src, dst);
255 if (isByteSizedType(Ty)) {
256 emitUint8(0x88);
257 } else {
258 emitUint8(0x89);
259 }
260 emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
261 }
262
263 template <typename TraitsType>
mov(Type Ty,GPRRegister dst,const Address & src)264 void AssemblerX86Base<TraitsType>::mov(Type Ty, GPRRegister dst,
265 const Address &src) {
266 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
267 if (Ty == IceType_i16)
268 emitOperandSizeOverride();
269 emitAddrSizeOverridePrefix();
270 emitRex(Ty, src, dst);
271 if (isByteSizedType(Ty)) {
272 emitUint8(0x8A);
273 } else {
274 emitUint8(0x8B);
275 }
276 emitOperand(gprEncoding(dst), src);
277 }
278
279 template <typename TraitsType>
mov(Type Ty,const Address & dst,GPRRegister src)280 void AssemblerX86Base<TraitsType>::mov(Type Ty, const Address &dst,
281 GPRRegister src) {
282 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
283 if (Ty == IceType_i16)
284 emitOperandSizeOverride();
285 emitAddrSizeOverridePrefix();
286 emitRex(Ty, dst, src);
287 if (isByteSizedType(Ty)) {
288 emitUint8(0x88);
289 } else {
290 emitUint8(0x89);
291 }
292 emitOperand(gprEncoding(src), dst);
293 }
294
295 template <typename TraitsType>
mov(Type Ty,const Address & dst,const Immediate & imm)296 void AssemblerX86Base<TraitsType>::mov(Type Ty, const Address &dst,
297 const Immediate &imm) {
298 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
299 if (Ty == IceType_i16)
300 emitOperandSizeOverride();
301 emitAddrSizeOverridePrefix();
302 emitRex(Ty, dst, RexRegIrrelevant);
303 if (isByteSizedType(Ty)) {
304 emitUint8(0xC6);
305 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
306 emitOperand(0, dst, OffsetFromNextInstruction);
307 emitUint8(imm.value() & 0xFF);
308 } else {
309 emitUint8(0xC7);
310 const uint8_t OffsetFromNextInstruction = Ty == IceType_i16 ? 2 : 4;
311 emitOperand(0, dst, OffsetFromNextInstruction);
312 emitImmediate(Ty, imm);
313 }
314 }
315
316 template <typename TraitsType>
317 template <typename T>
318 typename std::enable_if<T::Is64Bit, void>::type
movabs(const GPRRegister Dst,uint64_t Imm64)319 AssemblerX86Base<TraitsType>::movabs(const GPRRegister Dst, uint64_t Imm64) {
320 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
321 const bool NeedsRexW = (Imm64 & ~0xFFFFFFFFull) != 0;
322 const Type RexType = NeedsRexW ? RexTypeForceRexW : RexTypeIrrelevant;
323 emitRexB(RexType, Dst);
324 emitUint8(0xB8 | gprEncoding(Dst));
325 // When emitting Imm64, we don't have to mask out the upper 32 bits for
326 // emitInt32 will/should only emit a 32-bit constant. In reality, we are
327 // paranoid, so we go ahead an mask the upper bits out anyway.
328 emitInt32(Imm64 & 0xFFFFFFFF);
329 if (NeedsRexW)
330 emitInt32((Imm64 >> 32) & 0xFFFFFFFF);
331 }
332
333 template <typename TraitsType>
movzx(Type SrcTy,GPRRegister dst,GPRRegister src)334 void AssemblerX86Base<TraitsType>::movzx(Type SrcTy, GPRRegister dst,
335 GPRRegister src) {
336 if (Traits::Is64Bit && SrcTy == IceType_i32) {
337 // 32-bit mov clears the upper 32 bits, hence zero-extending the 32-bit
338 // operand to 64-bit.
339 mov(IceType_i32, dst, src);
340 return;
341 }
342
343 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
344 bool ByteSized = isByteSizedType(SrcTy);
345 assert(ByteSized || SrcTy == IceType_i16);
346 emitRexRB(RexTypeIrrelevant, dst, SrcTy, src);
347 emitUint8(0x0F);
348 emitUint8(ByteSized ? 0xB6 : 0xB7);
349 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
350 }
351
352 template <typename TraitsType>
movzx(Type SrcTy,GPRRegister dst,const Address & src)353 void AssemblerX86Base<TraitsType>::movzx(Type SrcTy, GPRRegister dst,
354 const Address &src) {
355 if (Traits::Is64Bit && SrcTy == IceType_i32) {
356 // 32-bit mov clears the upper 32 bits, hence zero-extending the 32-bit
357 // operand to 64-bit.
358 mov(IceType_i32, dst, src);
359 return;
360 }
361
362 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
363 bool ByteSized = isByteSizedType(SrcTy);
364 assert(ByteSized || SrcTy == IceType_i16);
365 emitAddrSizeOverridePrefix();
366 emitRex(SrcTy, src, RexTypeIrrelevant, dst);
367 emitUint8(0x0F);
368 emitUint8(ByteSized ? 0xB6 : 0xB7);
369 emitOperand(gprEncoding(dst), src);
370 }
371
372 template <typename TraitsType>
movsx(Type SrcTy,GPRRegister dst,GPRRegister src)373 void AssemblerX86Base<TraitsType>::movsx(Type SrcTy, GPRRegister dst,
374 GPRRegister src) {
375 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
376 bool ByteSized = isByteSizedType(SrcTy);
377 emitRexRB(RexTypeForceRexW, dst, SrcTy, src);
378 if (ByteSized || SrcTy == IceType_i16) {
379 emitUint8(0x0F);
380 emitUint8(ByteSized ? 0xBE : 0xBF);
381 } else {
382 assert(Traits::Is64Bit && SrcTy == IceType_i32);
383 emitUint8(0x63);
384 }
385 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
386 }
387
388 template <typename TraitsType>
movsx(Type SrcTy,GPRRegister dst,const Address & src)389 void AssemblerX86Base<TraitsType>::movsx(Type SrcTy, GPRRegister dst,
390 const Address &src) {
391 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
392 bool ByteSized = isByteSizedType(SrcTy);
393 emitAddrSizeOverridePrefix();
394 emitRex(SrcTy, src, RexTypeForceRexW, dst);
395 if (ByteSized || SrcTy == IceType_i16) {
396 emitUint8(0x0F);
397 emitUint8(ByteSized ? 0xBE : 0xBF);
398 } else {
399 assert(Traits::Is64Bit && SrcTy == IceType_i32);
400 emitUint8(0x63);
401 }
402 emitOperand(gprEncoding(dst), src);
403 }
404
405 template <typename TraitsType>
lea(Type Ty,GPRRegister dst,const Address & src)406 void AssemblerX86Base<TraitsType>::lea(Type Ty, GPRRegister dst,
407 const Address &src) {
408 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
409 assert(Ty == IceType_i16 || Ty == IceType_i32 ||
410 (Traits::Is64Bit && Ty == IceType_i64));
411 if (Ty == IceType_i16)
412 emitOperandSizeOverride();
413 emitAddrSizeOverridePrefix();
414 emitRex(Ty, src, dst);
415 emitUint8(0x8D);
416 emitOperand(gprEncoding(dst), src);
417 }
418
419 template <typename TraitsType>
cmov(Type Ty,BrCond cond,GPRRegister dst,GPRRegister src)420 void AssemblerX86Base<TraitsType>::cmov(Type Ty, BrCond cond, GPRRegister dst,
421 GPRRegister src) {
422 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
423 if (Ty == IceType_i16)
424 emitOperandSizeOverride();
425 else
426 assert(Ty == IceType_i32 || (Traits::Is64Bit && Ty == IceType_i64));
427 emitRexRB(Ty, dst, src);
428 emitUint8(0x0F);
429 emitUint8(0x40 + cond);
430 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
431 }
432
433 template <typename TraitsType>
cmov(Type Ty,BrCond cond,GPRRegister dst,const Address & src)434 void AssemblerX86Base<TraitsType>::cmov(Type Ty, BrCond cond, GPRRegister dst,
435 const Address &src) {
436 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
437 if (Ty == IceType_i16)
438 emitOperandSizeOverride();
439 else
440 assert(Ty == IceType_i32 || (Traits::Is64Bit && Ty == IceType_i64));
441 emitAddrSizeOverridePrefix();
442 emitRex(Ty, src, dst);
443 emitUint8(0x0F);
444 emitUint8(0x40 + cond);
445 emitOperand(gprEncoding(dst), src);
446 }
447
rep_movsb()448 template <typename TraitsType> void AssemblerX86Base<TraitsType>::rep_movsb() {
449 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
450 emitUint8(0xF3);
451 emitUint8(0xA4);
452 }
453
454 template <typename TraitsType>
movss(Type Ty,XmmRegister dst,const Address & src)455 void AssemblerX86Base<TraitsType>::movss(Type Ty, XmmRegister dst,
456 const Address &src) {
457 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
458 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
459 emitAddrSizeOverridePrefix();
460 emitRex(RexTypeIrrelevant, src, dst);
461 emitUint8(0x0F);
462 emitUint8(0x10);
463 emitOperand(gprEncoding(dst), src);
464 }
465
466 template <typename TraitsType>
movss(Type Ty,const Address & dst,XmmRegister src)467 void AssemblerX86Base<TraitsType>::movss(Type Ty, const Address &dst,
468 XmmRegister src) {
469 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
470 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
471 emitAddrSizeOverridePrefix();
472 emitRex(RexTypeIrrelevant, dst, src);
473 emitUint8(0x0F);
474 emitUint8(0x11);
475 emitOperand(gprEncoding(src), dst);
476 }
477
478 template <typename TraitsType>
movss(Type Ty,XmmRegister dst,XmmRegister src)479 void AssemblerX86Base<TraitsType>::movss(Type Ty, XmmRegister dst,
480 XmmRegister src) {
481 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
482 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
483 emitRexRB(RexTypeIrrelevant, src, dst);
484 emitUint8(0x0F);
485 emitUint8(0x11);
486 emitXmmRegisterOperand(src, dst);
487 }
488
489 template <typename TraitsType>
movd(Type SrcTy,XmmRegister dst,GPRRegister src)490 void AssemblerX86Base<TraitsType>::movd(Type SrcTy, XmmRegister dst,
491 GPRRegister src) {
492 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
493 emitUint8(0x66);
494 emitRexRB(SrcTy, dst, src);
495 emitUint8(0x0F);
496 emitUint8(0x6E);
497 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
498 }
499
500 template <typename TraitsType>
movd(Type SrcTy,XmmRegister dst,const Address & src)501 void AssemblerX86Base<TraitsType>::movd(Type SrcTy, XmmRegister dst,
502 const Address &src) {
503 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
504 emitUint8(0x66);
505 emitAddrSizeOverridePrefix();
506 emitRex(SrcTy, src, dst);
507 emitUint8(0x0F);
508 emitUint8(0x6E);
509 emitOperand(gprEncoding(dst), src);
510 }
511
512 template <typename TraitsType>
movd(Type DestTy,GPRRegister dst,XmmRegister src)513 void AssemblerX86Base<TraitsType>::movd(Type DestTy, GPRRegister dst,
514 XmmRegister src) {
515 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
516 emitUint8(0x66);
517 emitRexRB(DestTy, src, dst);
518 emitUint8(0x0F);
519 emitUint8(0x7E);
520 emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
521 }
522
523 template <typename TraitsType>
movd(Type DestTy,const Address & dst,XmmRegister src)524 void AssemblerX86Base<TraitsType>::movd(Type DestTy, const Address &dst,
525 XmmRegister src) {
526 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
527 emitUint8(0x66);
528 emitAddrSizeOverridePrefix();
529 emitRex(DestTy, dst, src);
530 emitUint8(0x0F);
531 emitUint8(0x7E);
532 emitOperand(gprEncoding(src), dst);
533 }
534
535 template <typename TraitsType>
movq(XmmRegister dst,XmmRegister src)536 void AssemblerX86Base<TraitsType>::movq(XmmRegister dst, XmmRegister src) {
537 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
538 emitUint8(0xF3);
539 emitRexRB(RexTypeIrrelevant, dst, src);
540 emitUint8(0x0F);
541 emitUint8(0x7E);
542 emitXmmRegisterOperand(dst, src);
543 }
544
545 template <typename TraitsType>
movq(const Address & dst,XmmRegister src)546 void AssemblerX86Base<TraitsType>::movq(const Address &dst, XmmRegister src) {
547 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
548 emitUint8(0x66);
549 emitAddrSizeOverridePrefix();
550 emitRex(RexTypeIrrelevant, dst, src);
551 emitUint8(0x0F);
552 emitUint8(0xD6);
553 emitOperand(gprEncoding(src), dst);
554 }
555
556 template <typename TraitsType>
movq(XmmRegister dst,const Address & src)557 void AssemblerX86Base<TraitsType>::movq(XmmRegister dst, const Address &src) {
558 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
559 emitUint8(0xF3);
560 emitAddrSizeOverridePrefix();
561 emitRex(RexTypeIrrelevant, src, dst);
562 emitUint8(0x0F);
563 emitUint8(0x7E);
564 emitOperand(gprEncoding(dst), src);
565 }
566
567 template <typename TraitsType>
addss(Type Ty,XmmRegister dst,XmmRegister src)568 void AssemblerX86Base<TraitsType>::addss(Type Ty, XmmRegister dst,
569 XmmRegister src) {
570 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
571 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
572 emitRexRB(RexTypeIrrelevant, dst, src);
573 emitUint8(0x0F);
574 emitUint8(0x58);
575 emitXmmRegisterOperand(dst, src);
576 }
577
578 template <typename TraitsType>
addss(Type Ty,XmmRegister dst,const Address & src)579 void AssemblerX86Base<TraitsType>::addss(Type Ty, XmmRegister dst,
580 const Address &src) {
581 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
582 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
583 emitAddrSizeOverridePrefix();
584 emitRex(RexTypeIrrelevant, src, dst);
585 emitUint8(0x0F);
586 emitUint8(0x58);
587 emitOperand(gprEncoding(dst), src);
588 }
589
590 template <typename TraitsType>
subss(Type Ty,XmmRegister dst,XmmRegister src)591 void AssemblerX86Base<TraitsType>::subss(Type Ty, XmmRegister dst,
592 XmmRegister src) {
593 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
594 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
595 emitRexRB(RexTypeIrrelevant, dst, src);
596 emitUint8(0x0F);
597 emitUint8(0x5C);
598 emitXmmRegisterOperand(dst, src);
599 }
600
601 template <typename TraitsType>
subss(Type Ty,XmmRegister dst,const Address & src)602 void AssemblerX86Base<TraitsType>::subss(Type Ty, XmmRegister dst,
603 const Address &src) {
604 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
605 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
606 emitAddrSizeOverridePrefix();
607 emitRex(RexTypeIrrelevant, src, dst);
608 emitUint8(0x0F);
609 emitUint8(0x5C);
610 emitOperand(gprEncoding(dst), src);
611 }
612
613 template <typename TraitsType>
mulss(Type Ty,XmmRegister dst,XmmRegister src)614 void AssemblerX86Base<TraitsType>::mulss(Type Ty, XmmRegister dst,
615 XmmRegister src) {
616 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
617 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
618 emitRexRB(RexTypeIrrelevant, dst, src);
619 emitUint8(0x0F);
620 emitUint8(0x59);
621 emitXmmRegisterOperand(dst, src);
622 }
623
624 template <typename TraitsType>
mulss(Type Ty,XmmRegister dst,const Address & src)625 void AssemblerX86Base<TraitsType>::mulss(Type Ty, XmmRegister dst,
626 const Address &src) {
627 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
628 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
629 emitAddrSizeOverridePrefix();
630 emitRex(RexTypeIrrelevant, src, dst);
631 emitUint8(0x0F);
632 emitUint8(0x59);
633 emitOperand(gprEncoding(dst), src);
634 }
635
636 template <typename TraitsType>
divss(Type Ty,XmmRegister dst,XmmRegister src)637 void AssemblerX86Base<TraitsType>::divss(Type Ty, XmmRegister dst,
638 XmmRegister src) {
639 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
640 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
641 emitRexRB(RexTypeIrrelevant, dst, src);
642 emitUint8(0x0F);
643 emitUint8(0x5E);
644 emitXmmRegisterOperand(dst, src);
645 }
646
647 template <typename TraitsType>
divss(Type Ty,XmmRegister dst,const Address & src)648 void AssemblerX86Base<TraitsType>::divss(Type Ty, XmmRegister dst,
649 const Address &src) {
650 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
651 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
652 emitAddrSizeOverridePrefix();
653 emitRex(RexTypeIrrelevant, src, dst);
654 emitUint8(0x0F);
655 emitUint8(0x5E);
656 emitOperand(gprEncoding(dst), src);
657 }
658
659 template <typename TraitsType>
660 template <typename T, typename>
fld(Type Ty,const typename T::Address & src)661 void AssemblerX86Base<TraitsType>::fld(Type Ty,
662 const typename T::Address &src) {
663 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
664 emitAddrSizeOverridePrefix();
665 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xD9 : 0xDD);
666 emitOperand(0, src);
667 }
668
669 template <typename TraitsType>
670 template <typename T, typename>
fstp(Type Ty,const typename T::Address & dst)671 void AssemblerX86Base<TraitsType>::fstp(Type Ty,
672 const typename T::Address &dst) {
673 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
674 emitAddrSizeOverridePrefix();
675 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xD9 : 0xDD);
676 emitOperand(3, dst);
677 }
678
679 template <typename TraitsType>
680 template <typename T, typename>
fstp(typename T::X87STRegister st)681 void AssemblerX86Base<TraitsType>::fstp(typename T::X87STRegister st) {
682 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
683 emitUint8(0xDD);
684 emitUint8(0xD8 + st);
685 }
686
687 template <typename TraitsType>
movaps(XmmRegister dst,XmmRegister src)688 void AssemblerX86Base<TraitsType>::movaps(XmmRegister dst, XmmRegister src) {
689 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
690 emitRexRB(RexTypeIrrelevant, dst, src);
691 emitUint8(0x0F);
692 emitUint8(0x28);
693 emitXmmRegisterOperand(dst, src);
694 }
695
696 template <typename TraitsType>
movups(XmmRegister dst,XmmRegister src)697 void AssemblerX86Base<TraitsType>::movups(XmmRegister dst, XmmRegister src) {
698 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
699 emitRexRB(RexTypeIrrelevant, dst, src);
700 emitUint8(0x0F);
701 emitUint8(0x10);
702 emitXmmRegisterOperand(dst, src);
703 }
704
705 template <typename TraitsType>
movups(XmmRegister dst,const Address & src)706 void AssemblerX86Base<TraitsType>::movups(XmmRegister dst, const Address &src) {
707 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
708 emitAddrSizeOverridePrefix();
709 emitRex(RexTypeIrrelevant, src, dst);
710 emitUint8(0x0F);
711 emitUint8(0x10);
712 emitOperand(gprEncoding(dst), src);
713 }
714
715 template <typename TraitsType>
movups(const Address & dst,XmmRegister src)716 void AssemblerX86Base<TraitsType>::movups(const Address &dst, XmmRegister src) {
717 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
718 emitAddrSizeOverridePrefix();
719 emitRex(RexTypeIrrelevant, dst, src);
720 emitUint8(0x0F);
721 emitUint8(0x11);
722 emitOperand(gprEncoding(src), dst);
723 }
724
725 template <typename TraitsType>
padd(Type Ty,XmmRegister dst,XmmRegister src)726 void AssemblerX86Base<TraitsType>::padd(Type Ty, XmmRegister dst,
727 XmmRegister src) {
728 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
729 emitUint8(0x66);
730 emitRexRB(RexTypeIrrelevant, dst, src);
731 emitUint8(0x0F);
732 if (isByteSizedArithType(Ty)) {
733 emitUint8(0xFC);
734 } else if (Ty == IceType_i16) {
735 emitUint8(0xFD);
736 } else {
737 emitUint8(0xFE);
738 }
739 emitXmmRegisterOperand(dst, src);
740 }
741
742 template <typename TraitsType>
padd(Type Ty,XmmRegister dst,const Address & src)743 void AssemblerX86Base<TraitsType>::padd(Type Ty, XmmRegister dst,
744 const Address &src) {
745 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
746 emitUint8(0x66);
747 emitAddrSizeOverridePrefix();
748 emitRex(RexTypeIrrelevant, src, dst);
749 emitUint8(0x0F);
750 if (isByteSizedArithType(Ty)) {
751 emitUint8(0xFC);
752 } else if (Ty == IceType_i16) {
753 emitUint8(0xFD);
754 } else {
755 emitUint8(0xFE);
756 }
757 emitOperand(gprEncoding(dst), src);
758 }
759
760 template <typename TraitsType>
padds(Type Ty,XmmRegister dst,XmmRegister src)761 void AssemblerX86Base<TraitsType>::padds(Type Ty, XmmRegister dst,
762 XmmRegister src) {
763 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
764 emitUint8(0x66);
765 emitRexRB(RexTypeIrrelevant, dst, src);
766 emitUint8(0x0F);
767 if (isByteSizedArithType(Ty)) {
768 emitUint8(0xEC);
769 } else if (Ty == IceType_i16) {
770 emitUint8(0xED);
771 } else {
772 assert(false && "Unexpected padds operand type");
773 }
774 emitXmmRegisterOperand(dst, src);
775 }
776
777 template <typename TraitsType>
padds(Type Ty,XmmRegister dst,const Address & src)778 void AssemblerX86Base<TraitsType>::padds(Type Ty, XmmRegister dst,
779 const Address &src) {
780 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
781 emitUint8(0x66);
782 emitAddrSizeOverridePrefix();
783 emitRex(RexTypeIrrelevant, src, dst);
784 emitUint8(0x0F);
785 if (isByteSizedArithType(Ty)) {
786 emitUint8(0xEC);
787 } else if (Ty == IceType_i16) {
788 emitUint8(0xED);
789 } else {
790 assert(false && "Unexpected padds operand type");
791 }
792 emitOperand(gprEncoding(dst), src);
793 }
794
795 template <typename TraitsType>
paddus(Type Ty,XmmRegister dst,XmmRegister src)796 void AssemblerX86Base<TraitsType>::paddus(Type Ty, XmmRegister dst,
797 XmmRegister src) {
798 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
799 emitUint8(0x66);
800 emitRexRB(RexTypeIrrelevant, dst, src);
801 emitUint8(0x0F);
802 if (isByteSizedArithType(Ty)) {
803 emitUint8(0xDC);
804 } else if (Ty == IceType_i16) {
805 emitUint8(0xDD);
806 } else {
807 assert(false && "Unexpected paddus operand type");
808 }
809 emitXmmRegisterOperand(dst, src);
810 }
811
812 template <typename TraitsType>
paddus(Type Ty,XmmRegister dst,const Address & src)813 void AssemblerX86Base<TraitsType>::paddus(Type Ty, XmmRegister dst,
814 const Address &src) {
815 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
816 emitUint8(0x66);
817 emitAddrSizeOverridePrefix();
818 emitRex(RexTypeIrrelevant, src, dst);
819 emitUint8(0x0F);
820 if (isByteSizedArithType(Ty)) {
821 emitUint8(0xDC);
822 } else if (Ty == IceType_i16) {
823 emitUint8(0xDD);
824 } else {
825 assert(false && "Unexpected paddus operand type");
826 }
827 emitOperand(gprEncoding(dst), src);
828 }
829
830 template <typename TraitsType>
pand(Type,XmmRegister dst,XmmRegister src)831 void AssemblerX86Base<TraitsType>::pand(Type /* Ty */, XmmRegister dst,
832 XmmRegister src) {
833 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
834 emitUint8(0x66);
835 emitRexRB(RexTypeIrrelevant, dst, src);
836 emitUint8(0x0F);
837 emitUint8(0xDB);
838 emitXmmRegisterOperand(dst, src);
839 }
840
841 template <typename TraitsType>
pand(Type,XmmRegister dst,const Address & src)842 void AssemblerX86Base<TraitsType>::pand(Type /* Ty */, XmmRegister dst,
843 const Address &src) {
844 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
845 emitUint8(0x66);
846 emitAddrSizeOverridePrefix();
847 emitRex(RexTypeIrrelevant, src, dst);
848 emitUint8(0x0F);
849 emitUint8(0xDB);
850 emitOperand(gprEncoding(dst), src);
851 }
852
853 template <typename TraitsType>
pandn(Type,XmmRegister dst,XmmRegister src)854 void AssemblerX86Base<TraitsType>::pandn(Type /* Ty */, XmmRegister dst,
855 XmmRegister src) {
856 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
857 emitUint8(0x66);
858 emitRexRB(RexTypeIrrelevant, dst, src);
859 emitUint8(0x0F);
860 emitUint8(0xDF);
861 emitXmmRegisterOperand(dst, src);
862 }
863
864 template <typename TraitsType>
pandn(Type,XmmRegister dst,const Address & src)865 void AssemblerX86Base<TraitsType>::pandn(Type /* Ty */, XmmRegister dst,
866 const Address &src) {
867 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
868 emitUint8(0x66);
869 emitAddrSizeOverridePrefix();
870 emitRex(RexTypeIrrelevant, src, dst);
871 emitUint8(0x0F);
872 emitUint8(0xDF);
873 emitOperand(gprEncoding(dst), src);
874 }
875
876 template <typename TraitsType>
pmull(Type Ty,XmmRegister dst,XmmRegister src)877 void AssemblerX86Base<TraitsType>::pmull(Type Ty, XmmRegister dst,
878 XmmRegister src) {
879 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
880 emitUint8(0x66);
881 emitRexRB(RexTypeIrrelevant, dst, src);
882 emitUint8(0x0F);
883 if (Ty == IceType_i16) {
884 emitUint8(0xD5);
885 } else {
886 assert(Ty == IceType_i32);
887 emitUint8(0x38);
888 emitUint8(0x40);
889 }
890 emitXmmRegisterOperand(dst, src);
891 }
892
893 template <typename TraitsType>
pmull(Type Ty,XmmRegister dst,const Address & src)894 void AssemblerX86Base<TraitsType>::pmull(Type Ty, XmmRegister dst,
895 const Address &src) {
896 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
897 emitUint8(0x66);
898 emitAddrSizeOverridePrefix();
899 emitRex(RexTypeIrrelevant, src, dst);
900 emitUint8(0x0F);
901 if (Ty == IceType_i16) {
902 emitUint8(0xD5);
903 } else {
904 assert(Ty == IceType_i32);
905 emitUint8(0x38);
906 emitUint8(0x40);
907 }
908 emitOperand(gprEncoding(dst), src);
909 }
910
911 template <typename TraitsType>
pmulhw(Type Ty,XmmRegister dst,XmmRegister src)912 void AssemblerX86Base<TraitsType>::pmulhw(Type Ty, XmmRegister dst,
913 XmmRegister src) {
914 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
915 emitUint8(0x66);
916 emitRexRB(RexTypeIrrelevant, dst, src);
917 emitUint8(0x0F);
918 assert(Ty == IceType_v8i16);
919 (void)Ty;
920 emitUint8(0xE5);
921 emitXmmRegisterOperand(dst, src);
922 }
923
924 template <typename TraitsType>
pmulhw(Type Ty,XmmRegister dst,const Address & src)925 void AssemblerX86Base<TraitsType>::pmulhw(Type Ty, XmmRegister dst,
926 const Address &src) {
927 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
928 emitUint8(0x66);
929 emitAddrSizeOverridePrefix();
930 emitRex(RexTypeIrrelevant, src, dst);
931 emitUint8(0x0F);
932 assert(Ty == IceType_v8i16);
933 (void)Ty;
934 emitUint8(0xE5);
935 emitOperand(gprEncoding(dst), src);
936 }
937
938 template <typename TraitsType>
pmulhuw(Type Ty,XmmRegister dst,XmmRegister src)939 void AssemblerX86Base<TraitsType>::pmulhuw(Type Ty, XmmRegister dst,
940 XmmRegister src) {
941 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
942 emitUint8(0x66);
943 emitRexRB(RexTypeIrrelevant, dst, src);
944 emitUint8(0x0F);
945 assert(Ty == IceType_v8i16);
946 (void)Ty;
947 emitUint8(0xE4);
948 emitXmmRegisterOperand(dst, src);
949 }
950
951 template <typename TraitsType>
pmulhuw(Type Ty,XmmRegister dst,const Address & src)952 void AssemblerX86Base<TraitsType>::pmulhuw(Type Ty, XmmRegister dst,
953 const Address &src) {
954 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
955 emitUint8(0x66);
956 emitAddrSizeOverridePrefix();
957 emitRex(RexTypeIrrelevant, src, dst);
958 emitUint8(0x0F);
959 assert(Ty == IceType_v8i16);
960 (void)Ty;
961 emitUint8(0xE4);
962 emitOperand(gprEncoding(dst), src);
963 }
964
965 template <typename TraitsType>
pmaddwd(Type Ty,XmmRegister dst,XmmRegister src)966 void AssemblerX86Base<TraitsType>::pmaddwd(Type Ty, XmmRegister dst,
967 XmmRegister src) {
968 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
969 emitUint8(0x66);
970 emitRexRB(RexTypeIrrelevant, dst, src);
971 emitUint8(0x0F);
972 assert(Ty == IceType_v8i16);
973 (void)Ty;
974 emitUint8(0xF5);
975 emitXmmRegisterOperand(dst, src);
976 }
977
978 template <typename TraitsType>
pmaddwd(Type Ty,XmmRegister dst,const Address & src)979 void AssemblerX86Base<TraitsType>::pmaddwd(Type Ty, XmmRegister dst,
980 const Address &src) {
981 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
982 emitUint8(0x66);
983 emitAddrSizeOverridePrefix();
984 emitRex(RexTypeIrrelevant, src, dst);
985 emitUint8(0x0F);
986 assert(Ty == IceType_v8i16);
987 (void)Ty;
988 emitUint8(0xF5);
989 emitOperand(gprEncoding(dst), src);
990 }
991
992 template <typename TraitsType>
pmuludq(Type,XmmRegister dst,XmmRegister src)993 void AssemblerX86Base<TraitsType>::pmuludq(Type /* Ty */, XmmRegister dst,
994 XmmRegister src) {
995 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
996 emitUint8(0x66);
997 emitRexRB(RexTypeIrrelevant, dst, src);
998 emitUint8(0x0F);
999 emitUint8(0xF4);
1000 emitXmmRegisterOperand(dst, src);
1001 }
1002
1003 template <typename TraitsType>
pmuludq(Type,XmmRegister dst,const Address & src)1004 void AssemblerX86Base<TraitsType>::pmuludq(Type /* Ty */, XmmRegister dst,
1005 const Address &src) {
1006 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1007 emitUint8(0x66);
1008 emitAddrSizeOverridePrefix();
1009 emitRex(RexTypeIrrelevant, src, dst);
1010 emitUint8(0x0F);
1011 emitUint8(0xF4);
1012 emitOperand(gprEncoding(dst), src);
1013 }
1014
1015 template <typename TraitsType>
por(Type,XmmRegister dst,XmmRegister src)1016 void AssemblerX86Base<TraitsType>::por(Type /* Ty */, XmmRegister dst,
1017 XmmRegister src) {
1018 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1019 emitUint8(0x66);
1020 emitRexRB(RexTypeIrrelevant, dst, src);
1021 emitUint8(0x0F);
1022 emitUint8(0xEB);
1023 emitXmmRegisterOperand(dst, src);
1024 }
1025
1026 template <typename TraitsType>
por(Type,XmmRegister dst,const Address & src)1027 void AssemblerX86Base<TraitsType>::por(Type /* Ty */, XmmRegister dst,
1028 const Address &src) {
1029 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1030 emitUint8(0x66);
1031 emitAddrSizeOverridePrefix();
1032 emitRex(RexTypeIrrelevant, src, dst);
1033 emitUint8(0x0F);
1034 emitUint8(0xEB);
1035 emitOperand(gprEncoding(dst), src);
1036 }
1037
1038 template <typename TraitsType>
psub(Type Ty,XmmRegister dst,XmmRegister src)1039 void AssemblerX86Base<TraitsType>::psub(Type Ty, XmmRegister dst,
1040 XmmRegister src) {
1041 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1042 emitUint8(0x66);
1043 emitRexRB(RexTypeIrrelevant, dst, src);
1044 emitUint8(0x0F);
1045 if (isByteSizedArithType(Ty)) {
1046 emitUint8(0xF8);
1047 } else if (Ty == IceType_i16) {
1048 emitUint8(0xF9);
1049 } else {
1050 emitUint8(0xFA);
1051 }
1052 emitXmmRegisterOperand(dst, src);
1053 }
1054
1055 template <typename TraitsType>
psub(Type Ty,XmmRegister dst,const Address & src)1056 void AssemblerX86Base<TraitsType>::psub(Type Ty, XmmRegister dst,
1057 const Address &src) {
1058 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1059 emitUint8(0x66);
1060 emitAddrSizeOverridePrefix();
1061 emitRex(RexTypeIrrelevant, src, dst);
1062 emitUint8(0x0F);
1063 if (isByteSizedArithType(Ty)) {
1064 emitUint8(0xF8);
1065 } else if (Ty == IceType_i16) {
1066 emitUint8(0xF9);
1067 } else {
1068 emitUint8(0xFA);
1069 }
1070 emitOperand(gprEncoding(dst), src);
1071 }
1072
1073 template <typename TraitsType>
psubs(Type Ty,XmmRegister dst,XmmRegister src)1074 void AssemblerX86Base<TraitsType>::psubs(Type Ty, XmmRegister dst,
1075 XmmRegister src) {
1076 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1077 emitUint8(0x66);
1078 emitRexRB(RexTypeIrrelevant, dst, src);
1079 emitUint8(0x0F);
1080 if (isByteSizedArithType(Ty)) {
1081 emitUint8(0xE8);
1082 } else if (Ty == IceType_i16) {
1083 emitUint8(0xE9);
1084 } else {
1085 assert(false && "Unexpected psubs operand type");
1086 }
1087 emitXmmRegisterOperand(dst, src);
1088 }
1089
1090 template <typename TraitsType>
psubs(Type Ty,XmmRegister dst,const Address & src)1091 void AssemblerX86Base<TraitsType>::psubs(Type Ty, XmmRegister dst,
1092 const Address &src) {
1093 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1094 emitUint8(0x66);
1095 emitAddrSizeOverridePrefix();
1096 emitRex(RexTypeIrrelevant, src, dst);
1097 emitUint8(0x0F);
1098 if (isByteSizedArithType(Ty)) {
1099 emitUint8(0xE8);
1100 } else if (Ty == IceType_i16) {
1101 emitUint8(0xE9);
1102 } else {
1103 assert(false && "Unexpected psubs operand type");
1104 }
1105 emitOperand(gprEncoding(dst), src);
1106 }
1107 template <typename TraitsType>
psubus(Type Ty,XmmRegister dst,XmmRegister src)1108 void AssemblerX86Base<TraitsType>::psubus(Type Ty, XmmRegister dst,
1109 XmmRegister src) {
1110 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1111 emitUint8(0x66);
1112 emitRexRB(RexTypeIrrelevant, dst, src);
1113 emitUint8(0x0F);
1114 if (isByteSizedArithType(Ty)) {
1115 emitUint8(0xD8);
1116 } else if (Ty == IceType_i16) {
1117 emitUint8(0xD9);
1118 } else {
1119 assert(false && "Unexpected psubus operand type");
1120 }
1121 emitXmmRegisterOperand(dst, src);
1122 }
1123
1124 template <typename TraitsType>
psubus(Type Ty,XmmRegister dst,const Address & src)1125 void AssemblerX86Base<TraitsType>::psubus(Type Ty, XmmRegister dst,
1126 const Address &src) {
1127 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1128 emitUint8(0x66);
1129 emitAddrSizeOverridePrefix();
1130 emitRex(RexTypeIrrelevant, src, dst);
1131 emitUint8(0x0F);
1132 if (isByteSizedArithType(Ty)) {
1133 emitUint8(0xD8);
1134 } else if (Ty == IceType_i16) {
1135 emitUint8(0xD9);
1136 } else {
1137 assert(false && "Unexpected psubus operand type");
1138 }
1139 emitOperand(gprEncoding(dst), src);
1140 }
1141
1142 template <typename TraitsType>
pxor(Type,XmmRegister dst,XmmRegister src)1143 void AssemblerX86Base<TraitsType>::pxor(Type /* Ty */, XmmRegister dst,
1144 XmmRegister src) {
1145 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1146 emitUint8(0x66);
1147 emitRexRB(RexTypeIrrelevant, dst, src);
1148 emitUint8(0x0F);
1149 emitUint8(0xEF);
1150 emitXmmRegisterOperand(dst, src);
1151 }
1152
1153 template <typename TraitsType>
pxor(Type,XmmRegister dst,const Address & src)1154 void AssemblerX86Base<TraitsType>::pxor(Type /* Ty */, XmmRegister dst,
1155 const Address &src) {
1156 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1157 emitUint8(0x66);
1158 emitAddrSizeOverridePrefix();
1159 emitRex(RexTypeIrrelevant, src, dst);
1160 emitUint8(0x0F);
1161 emitUint8(0xEF);
1162 emitOperand(gprEncoding(dst), src);
1163 }
1164
1165 template <typename TraitsType>
psll(Type Ty,XmmRegister dst,XmmRegister src)1166 void AssemblerX86Base<TraitsType>::psll(Type Ty, XmmRegister dst,
1167 XmmRegister src) {
1168 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1169 emitUint8(0x66);
1170 emitRexRB(RexTypeIrrelevant, dst, src);
1171 emitUint8(0x0F);
1172 if (Ty == IceType_i16) {
1173 emitUint8(0xF1);
1174 } else {
1175 assert(Ty == IceType_i32);
1176 emitUint8(0xF2);
1177 }
1178 emitXmmRegisterOperand(dst, src);
1179 }
1180
1181 template <typename TraitsType>
psll(Type Ty,XmmRegister dst,const Address & src)1182 void AssemblerX86Base<TraitsType>::psll(Type Ty, XmmRegister dst,
1183 const Address &src) {
1184 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1185 emitUint8(0x66);
1186 emitAddrSizeOverridePrefix();
1187 emitRex(RexTypeIrrelevant, src, dst);
1188 emitUint8(0x0F);
1189 if (Ty == IceType_i16) {
1190 emitUint8(0xF1);
1191 } else {
1192 assert(Ty == IceType_i32);
1193 emitUint8(0xF2);
1194 }
1195 emitOperand(gprEncoding(dst), src);
1196 }
1197
1198 template <typename TraitsType>
psll(Type Ty,XmmRegister dst,const Immediate & imm)1199 void AssemblerX86Base<TraitsType>::psll(Type Ty, XmmRegister dst,
1200 const Immediate &imm) {
1201 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1202 assert(imm.is_int8());
1203 emitUint8(0x66);
1204 emitRexB(RexTypeIrrelevant, dst);
1205 emitUint8(0x0F);
1206 if (Ty == IceType_i16) {
1207 emitUint8(0x71);
1208 } else {
1209 assert(Ty == IceType_i32);
1210 emitUint8(0x72);
1211 }
1212 emitRegisterOperand(6, gprEncoding(dst));
1213 emitUint8(imm.value() & 0xFF);
1214 }
1215
1216 template <typename TraitsType>
psra(Type Ty,XmmRegister dst,XmmRegister src)1217 void AssemblerX86Base<TraitsType>::psra(Type Ty, XmmRegister dst,
1218 XmmRegister src) {
1219 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1220 emitUint8(0x66);
1221 emitRexRB(RexTypeIrrelevant, dst, src);
1222 emitUint8(0x0F);
1223 if (Ty == IceType_i16) {
1224 emitUint8(0xE1);
1225 } else {
1226 assert(Ty == IceType_i32);
1227 emitUint8(0xE2);
1228 }
1229 emitXmmRegisterOperand(dst, src);
1230 }
1231
1232 template <typename TraitsType>
psra(Type Ty,XmmRegister dst,const Address & src)1233 void AssemblerX86Base<TraitsType>::psra(Type Ty, XmmRegister dst,
1234 const Address &src) {
1235 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1236 emitUint8(0x66);
1237 emitAddrSizeOverridePrefix();
1238 emitRex(RexTypeIrrelevant, src, dst);
1239 emitUint8(0x0F);
1240 if (Ty == IceType_i16) {
1241 emitUint8(0xE1);
1242 } else {
1243 assert(Ty == IceType_i32);
1244 emitUint8(0xE2);
1245 }
1246 emitOperand(gprEncoding(dst), src);
1247 }
1248
1249 template <typename TraitsType>
psra(Type Ty,XmmRegister dst,const Immediate & imm)1250 void AssemblerX86Base<TraitsType>::psra(Type Ty, XmmRegister dst,
1251 const Immediate &imm) {
1252 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1253 assert(imm.is_int8());
1254 emitUint8(0x66);
1255 emitRexB(RexTypeIrrelevant, dst);
1256 emitUint8(0x0F);
1257 if (Ty == IceType_i16) {
1258 emitUint8(0x71);
1259 } else {
1260 assert(Ty == IceType_i32);
1261 emitUint8(0x72);
1262 }
1263 emitRegisterOperand(4, gprEncoding(dst));
1264 emitUint8(imm.value() & 0xFF);
1265 }
1266
1267 template <typename TraitsType>
psrl(Type Ty,XmmRegister dst,XmmRegister src)1268 void AssemblerX86Base<TraitsType>::psrl(Type Ty, XmmRegister dst,
1269 XmmRegister src) {
1270 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1271 emitUint8(0x66);
1272 emitRexRB(RexTypeIrrelevant, dst, src);
1273 emitUint8(0x0F);
1274 if (Ty == IceType_i16) {
1275 emitUint8(0xD1);
1276 } else if (Ty == IceType_f64) {
1277 emitUint8(0xD3);
1278 } else {
1279 assert(Ty == IceType_i32 || Ty == IceType_f32 || Ty == IceType_v4f32);
1280 emitUint8(0xD2);
1281 }
1282 emitXmmRegisterOperand(dst, src);
1283 }
1284
1285 template <typename TraitsType>
psrl(Type Ty,XmmRegister dst,const Address & src)1286 void AssemblerX86Base<TraitsType>::psrl(Type Ty, XmmRegister dst,
1287 const Address &src) {
1288 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1289 emitUint8(0x66);
1290 emitAddrSizeOverridePrefix();
1291 emitRex(RexTypeIrrelevant, src, dst);
1292 emitUint8(0x0F);
1293 if (Ty == IceType_i16) {
1294 emitUint8(0xD1);
1295 } else if (Ty == IceType_f64) {
1296 emitUint8(0xD3);
1297 } else {
1298 assert(Ty == IceType_i32 || Ty == IceType_f32 || Ty == IceType_v4f32);
1299 emitUint8(0xD2);
1300 }
1301 emitOperand(gprEncoding(dst), src);
1302 }
1303
1304 template <typename TraitsType>
psrl(Type Ty,XmmRegister dst,const Immediate & imm)1305 void AssemblerX86Base<TraitsType>::psrl(Type Ty, XmmRegister dst,
1306 const Immediate &imm) {
1307 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1308 assert(imm.is_int8());
1309 emitUint8(0x66);
1310 emitRexB(RexTypeIrrelevant, dst);
1311 emitUint8(0x0F);
1312 if (Ty == IceType_i16) {
1313 emitUint8(0x71);
1314 } else if (Ty == IceType_f64) {
1315 emitUint8(0x73);
1316 } else {
1317 assert(Ty == IceType_i32 || Ty == IceType_f32 || Ty == IceType_v4f32);
1318 emitUint8(0x72);
1319 }
1320 emitRegisterOperand(2, gprEncoding(dst));
1321 emitUint8(imm.value() & 0xFF);
1322 }
1323
1324 // {add,sub,mul,div}ps are given a Ty parameter for consistency with
1325 // {add,sub,mul,div}ss. In the future, when the PNaCl ABI allows addpd, etc.,
1326 // we can use the Ty parameter to decide on adding a 0x66 prefix.
1327 template <typename TraitsType>
addps(Type,XmmRegister dst,XmmRegister src)1328 void AssemblerX86Base<TraitsType>::addps(Type /* Ty */, XmmRegister dst,
1329 XmmRegister src) {
1330 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1331 emitRexRB(RexTypeIrrelevant, dst, src);
1332 emitUint8(0x0F);
1333 emitUint8(0x58);
1334 emitXmmRegisterOperand(dst, src);
1335 }
1336
1337 template <typename TraitsType>
addps(Type,XmmRegister dst,const Address & src)1338 void AssemblerX86Base<TraitsType>::addps(Type /* Ty */, XmmRegister dst,
1339 const Address &src) {
1340 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1341 emitAddrSizeOverridePrefix();
1342 emitRex(RexTypeIrrelevant, src, dst);
1343 emitUint8(0x0F);
1344 emitUint8(0x58);
1345 emitOperand(gprEncoding(dst), src);
1346 }
1347
1348 template <typename TraitsType>
subps(Type,XmmRegister dst,XmmRegister src)1349 void AssemblerX86Base<TraitsType>::subps(Type /* Ty */, XmmRegister dst,
1350 XmmRegister src) {
1351 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1352 emitRexRB(RexTypeIrrelevant, dst, src);
1353 emitUint8(0x0F);
1354 emitUint8(0x5C);
1355 emitXmmRegisterOperand(dst, src);
1356 }
1357
1358 template <typename TraitsType>
subps(Type,XmmRegister dst,const Address & src)1359 void AssemblerX86Base<TraitsType>::subps(Type /* Ty */, XmmRegister dst,
1360 const Address &src) {
1361 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1362 emitAddrSizeOverridePrefix();
1363 emitRex(RexTypeIrrelevant, src, dst);
1364 emitUint8(0x0F);
1365 emitUint8(0x5C);
1366 emitOperand(gprEncoding(dst), src);
1367 }
1368
1369 template <typename TraitsType>
divps(Type,XmmRegister dst,XmmRegister src)1370 void AssemblerX86Base<TraitsType>::divps(Type /* Ty */, XmmRegister dst,
1371 XmmRegister src) {
1372 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1373 emitRexRB(RexTypeIrrelevant, dst, src);
1374 emitUint8(0x0F);
1375 emitUint8(0x5E);
1376 emitXmmRegisterOperand(dst, src);
1377 }
1378
1379 template <typename TraitsType>
divps(Type,XmmRegister dst,const Address & src)1380 void AssemblerX86Base<TraitsType>::divps(Type /* Ty */, XmmRegister dst,
1381 const Address &src) {
1382 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1383 emitAddrSizeOverridePrefix();
1384 emitRex(RexTypeIrrelevant, src, dst);
1385 emitUint8(0x0F);
1386 emitUint8(0x5E);
1387 emitOperand(gprEncoding(dst), src);
1388 }
1389
1390 template <typename TraitsType>
mulps(Type,XmmRegister dst,XmmRegister src)1391 void AssemblerX86Base<TraitsType>::mulps(Type /* Ty */, XmmRegister dst,
1392 XmmRegister src) {
1393 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1394 emitRexRB(RexTypeIrrelevant, dst, src);
1395 emitUint8(0x0F);
1396 emitUint8(0x59);
1397 emitXmmRegisterOperand(dst, src);
1398 }
1399
1400 template <typename TraitsType>
mulps(Type,XmmRegister dst,const Address & src)1401 void AssemblerX86Base<TraitsType>::mulps(Type /* Ty */, XmmRegister dst,
1402 const Address &src) {
1403 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1404 emitAddrSizeOverridePrefix();
1405 emitRex(RexTypeIrrelevant, src, dst);
1406 emitUint8(0x0F);
1407 emitUint8(0x59);
1408 emitOperand(gprEncoding(dst), src);
1409 }
1410
1411 template <typename TraitsType>
minps(Type Ty,XmmRegister dst,XmmRegister src)1412 void AssemblerX86Base<TraitsType>::minps(Type Ty, XmmRegister dst,
1413 XmmRegister src) {
1414 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1415 if (!isFloat32Asserting32Or64(Ty))
1416 emitUint8(0x66);
1417 emitRexRB(RexTypeIrrelevant, dst, src);
1418 emitUint8(0x0F);
1419 emitUint8(0x5D);
1420 emitXmmRegisterOperand(dst, src);
1421 }
1422
1423 template <typename TraitsType>
minps(Type Ty,XmmRegister dst,const Address & src)1424 void AssemblerX86Base<TraitsType>::minps(Type Ty, XmmRegister dst,
1425 const Address &src) {
1426 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1427 if (!isFloat32Asserting32Or64(Ty))
1428 emitUint8(0x66);
1429 emitAddrSizeOverridePrefix();
1430 emitRex(RexTypeIrrelevant, src, dst);
1431 emitUint8(0x0F);
1432 emitUint8(0x5D);
1433 emitOperand(gprEncoding(dst), src);
1434 }
1435
1436 template <typename TraitsType>
minss(Type Ty,XmmRegister dst,XmmRegister src)1437 void AssemblerX86Base<TraitsType>::minss(Type Ty, XmmRegister dst,
1438 XmmRegister src) {
1439 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1440 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1441 emitRexRB(RexTypeIrrelevant, dst, src);
1442 emitUint8(0x0F);
1443 emitUint8(0x5D);
1444 emitXmmRegisterOperand(dst, src);
1445 }
1446
1447 template <typename TraitsType>
minss(Type Ty,XmmRegister dst,const Address & src)1448 void AssemblerX86Base<TraitsType>::minss(Type Ty, XmmRegister dst,
1449 const Address &src) {
1450 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1451 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1452 emitAddrSizeOverridePrefix();
1453 emitRex(RexTypeIrrelevant, src, dst);
1454 emitUint8(0x0F);
1455 emitUint8(0x5D);
1456 emitOperand(gprEncoding(dst), src);
1457 }
1458
1459 template <typename TraitsType>
maxps(Type Ty,XmmRegister dst,XmmRegister src)1460 void AssemblerX86Base<TraitsType>::maxps(Type Ty, XmmRegister dst,
1461 XmmRegister src) {
1462 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1463 if (!isFloat32Asserting32Or64(Ty))
1464 emitUint8(0x66);
1465 emitRexRB(RexTypeIrrelevant, dst, src);
1466 emitUint8(0x0F);
1467 emitUint8(0x5F);
1468 emitXmmRegisterOperand(dst, src);
1469 }
1470
1471 template <typename TraitsType>
maxps(Type Ty,XmmRegister dst,const Address & src)1472 void AssemblerX86Base<TraitsType>::maxps(Type Ty, XmmRegister dst,
1473 const Address &src) {
1474 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1475 if (!isFloat32Asserting32Or64(Ty))
1476 emitUint8(0x66);
1477 emitAddrSizeOverridePrefix();
1478 emitRex(RexTypeIrrelevant, src, dst);
1479 emitUint8(0x0F);
1480 emitUint8(0x5F);
1481 emitOperand(gprEncoding(dst), src);
1482 }
1483
1484 template <typename TraitsType>
maxss(Type Ty,XmmRegister dst,XmmRegister src)1485 void AssemblerX86Base<TraitsType>::maxss(Type Ty, XmmRegister dst,
1486 XmmRegister src) {
1487 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1488 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1489 emitRexRB(RexTypeIrrelevant, dst, src);
1490 emitUint8(0x0F);
1491 emitUint8(0x5F);
1492 emitXmmRegisterOperand(dst, src);
1493 }
1494
1495 template <typename TraitsType>
maxss(Type Ty,XmmRegister dst,const Address & src)1496 void AssemblerX86Base<TraitsType>::maxss(Type Ty, XmmRegister dst,
1497 const Address &src) {
1498 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1499 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
1500 emitAddrSizeOverridePrefix();
1501 emitRex(RexTypeIrrelevant, src, dst);
1502 emitUint8(0x0F);
1503 emitUint8(0x5F);
1504 emitOperand(gprEncoding(dst), src);
1505 }
1506
1507 template <typename TraitsType>
andnps(Type Ty,XmmRegister dst,XmmRegister src)1508 void AssemblerX86Base<TraitsType>::andnps(Type Ty, XmmRegister dst,
1509 XmmRegister src) {
1510 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1511 if (!isFloat32Asserting32Or64(Ty))
1512 emitUint8(0x66);
1513 emitRexRB(RexTypeIrrelevant, dst, src);
1514 emitUint8(0x0F);
1515 emitUint8(0x55);
1516 emitXmmRegisterOperand(dst, src);
1517 }
1518
1519 template <typename TraitsType>
andnps(Type Ty,XmmRegister dst,const Address & src)1520 void AssemblerX86Base<TraitsType>::andnps(Type Ty, XmmRegister dst,
1521 const Address &src) {
1522 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1523 if (!isFloat32Asserting32Or64(Ty))
1524 emitUint8(0x66);
1525 emitAddrSizeOverridePrefix();
1526 emitRex(RexTypeIrrelevant, src, dst);
1527 emitUint8(0x0F);
1528 emitUint8(0x55);
1529 emitOperand(gprEncoding(dst), src);
1530 }
1531
1532 template <typename TraitsType>
andps(Type Ty,XmmRegister dst,XmmRegister src)1533 void AssemblerX86Base<TraitsType>::andps(Type Ty, XmmRegister dst,
1534 XmmRegister src) {
1535 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1536 if (!isFloat32Asserting32Or64(Ty))
1537 emitUint8(0x66);
1538 emitRexRB(RexTypeIrrelevant, dst, src);
1539 emitUint8(0x0F);
1540 emitUint8(0x54);
1541 emitXmmRegisterOperand(dst, src);
1542 }
1543
1544 template <typename TraitsType>
andps(Type Ty,XmmRegister dst,const Address & src)1545 void AssemblerX86Base<TraitsType>::andps(Type Ty, XmmRegister dst,
1546 const Address &src) {
1547 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1548 if (!isFloat32Asserting32Or64(Ty))
1549 emitUint8(0x66);
1550 emitAddrSizeOverridePrefix();
1551 emitRex(RexTypeIrrelevant, src, dst);
1552 emitUint8(0x0F);
1553 emitUint8(0x54);
1554 emitOperand(gprEncoding(dst), src);
1555 }
1556
1557 template <typename TraitsType>
orps(Type Ty,XmmRegister dst,XmmRegister src)1558 void AssemblerX86Base<TraitsType>::orps(Type Ty, XmmRegister dst,
1559 XmmRegister src) {
1560 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1561 if (!isFloat32Asserting32Or64(Ty))
1562 emitUint8(0x66);
1563 emitRexRB(RexTypeIrrelevant, dst, src);
1564 emitUint8(0x0F);
1565 emitUint8(0x56);
1566 emitXmmRegisterOperand(dst, src);
1567 }
1568
1569 template <typename TraitsType>
orps(Type Ty,XmmRegister dst,const Address & src)1570 void AssemblerX86Base<TraitsType>::orps(Type Ty, XmmRegister dst,
1571 const Address &src) {
1572 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1573 if (!isFloat32Asserting32Or64(Ty))
1574 emitUint8(0x66);
1575 emitAddrSizeOverridePrefix();
1576 emitRex(RexTypeIrrelevant, src, dst);
1577 emitUint8(0x0F);
1578 emitUint8(0x56);
1579 emitOperand(gprEncoding(dst), src);
1580 }
1581
1582 template <typename TraitsType>
blendvps(Type,XmmRegister dst,XmmRegister src)1583 void AssemblerX86Base<TraitsType>::blendvps(Type /* Ty */, XmmRegister dst,
1584 XmmRegister src) {
1585 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1586 emitUint8(0x66);
1587 emitRexRB(RexTypeIrrelevant, dst, src);
1588 emitUint8(0x0F);
1589 emitUint8(0x38);
1590 emitUint8(0x14);
1591 emitXmmRegisterOperand(dst, src);
1592 }
1593
1594 template <typename TraitsType>
blendvps(Type,XmmRegister dst,const Address & src)1595 void AssemblerX86Base<TraitsType>::blendvps(Type /* Ty */, XmmRegister dst,
1596 const Address &src) {
1597 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1598 emitUint8(0x66);
1599 emitAddrSizeOverridePrefix();
1600 emitRex(RexTypeIrrelevant, src, dst);
1601 emitUint8(0x0F);
1602 emitUint8(0x38);
1603 emitUint8(0x14);
1604 emitOperand(gprEncoding(dst), src);
1605 }
1606
1607 template <typename TraitsType>
pblendvb(Type,XmmRegister dst,XmmRegister src)1608 void AssemblerX86Base<TraitsType>::pblendvb(Type /* Ty */, XmmRegister dst,
1609 XmmRegister src) {
1610 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1611 emitUint8(0x66);
1612 emitRexRB(RexTypeIrrelevant, dst, src);
1613 emitUint8(0x0F);
1614 emitUint8(0x38);
1615 emitUint8(0x10);
1616 emitXmmRegisterOperand(dst, src);
1617 }
1618
1619 template <typename TraitsType>
pblendvb(Type,XmmRegister dst,const Address & src)1620 void AssemblerX86Base<TraitsType>::pblendvb(Type /* Ty */, XmmRegister dst,
1621 const Address &src) {
1622 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1623 emitUint8(0x66);
1624 emitAddrSizeOverridePrefix();
1625 emitRex(RexTypeIrrelevant, src, dst);
1626 emitUint8(0x0F);
1627 emitUint8(0x38);
1628 emitUint8(0x10);
1629 emitOperand(gprEncoding(dst), src);
1630 }
1631
1632 template <typename TraitsType>
cmpps(Type Ty,XmmRegister dst,XmmRegister src,CmppsCond CmpCondition)1633 void AssemblerX86Base<TraitsType>::cmpps(Type Ty, XmmRegister dst,
1634 XmmRegister src,
1635 CmppsCond CmpCondition) {
1636 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1637 if (Ty == IceType_f64)
1638 emitUint8(0x66);
1639 emitRexRB(RexTypeIrrelevant, dst, src);
1640 emitUint8(0x0F);
1641 emitUint8(0xC2);
1642 emitXmmRegisterOperand(dst, src);
1643 emitUint8(CmpCondition);
1644 }
1645
1646 template <typename TraitsType>
cmpps(Type Ty,XmmRegister dst,const Address & src,CmppsCond CmpCondition)1647 void AssemblerX86Base<TraitsType>::cmpps(Type Ty, XmmRegister dst,
1648 const Address &src,
1649 CmppsCond CmpCondition) {
1650 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1651 if (Ty == IceType_f64)
1652 emitUint8(0x66);
1653 emitAddrSizeOverridePrefix();
1654 emitRex(RexTypeIrrelevant, src, dst);
1655 emitUint8(0x0F);
1656 emitUint8(0xC2);
1657 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
1658 emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
1659 emitUint8(CmpCondition);
1660 }
1661
1662 template <typename TraitsType>
sqrtps(XmmRegister dst)1663 void AssemblerX86Base<TraitsType>::sqrtps(XmmRegister dst) {
1664 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1665 emitRexRB(RexTypeIrrelevant, dst, dst);
1666 emitUint8(0x0F);
1667 emitUint8(0x51);
1668 emitXmmRegisterOperand(dst, dst);
1669 }
1670
1671 template <typename TraitsType>
rsqrtps(XmmRegister dst)1672 void AssemblerX86Base<TraitsType>::rsqrtps(XmmRegister dst) {
1673 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1674 emitRexRB(RexTypeIrrelevant, dst, dst);
1675 emitUint8(0x0F);
1676 emitUint8(0x52);
1677 emitXmmRegisterOperand(dst, dst);
1678 }
1679
1680 template <typename TraitsType>
reciprocalps(XmmRegister dst)1681 void AssemblerX86Base<TraitsType>::reciprocalps(XmmRegister dst) {
1682 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1683 emitRexRB(RexTypeIrrelevant, dst, dst);
1684 emitUint8(0x0F);
1685 emitUint8(0x53);
1686 emitXmmRegisterOperand(dst, dst);
1687 }
1688
1689 template <typename TraitsType>
movhlps(XmmRegister dst,XmmRegister src)1690 void AssemblerX86Base<TraitsType>::movhlps(XmmRegister dst, XmmRegister src) {
1691 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1692 emitRexRB(RexTypeIrrelevant, dst, src);
1693 emitUint8(0x0F);
1694 emitUint8(0x12);
1695 emitXmmRegisterOperand(dst, src);
1696 }
1697
1698 template <typename TraitsType>
movlhps(XmmRegister dst,XmmRegister src)1699 void AssemblerX86Base<TraitsType>::movlhps(XmmRegister dst, XmmRegister src) {
1700 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1701 emitRexRB(RexTypeIrrelevant, dst, src);
1702 emitUint8(0x0F);
1703 emitUint8(0x16);
1704 emitXmmRegisterOperand(dst, src);
1705 }
1706
1707 template <typename TraitsType>
unpcklps(XmmRegister dst,XmmRegister src)1708 void AssemblerX86Base<TraitsType>::unpcklps(XmmRegister dst, XmmRegister src) {
1709 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1710 emitRexRB(RexTypeIrrelevant, dst, src);
1711 emitUint8(0x0F);
1712 emitUint8(0x14);
1713 emitXmmRegisterOperand(dst, src);
1714 }
1715
1716 template <typename TraitsType>
unpckhps(XmmRegister dst,XmmRegister src)1717 void AssemblerX86Base<TraitsType>::unpckhps(XmmRegister dst, XmmRegister src) {
1718 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1719 emitRexRB(RexTypeIrrelevant, dst, src);
1720 emitUint8(0x0F);
1721 emitUint8(0x15);
1722 emitXmmRegisterOperand(dst, src);
1723 }
1724
1725 template <typename TraitsType>
unpcklpd(XmmRegister dst,XmmRegister src)1726 void AssemblerX86Base<TraitsType>::unpcklpd(XmmRegister dst, XmmRegister src) {
1727 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1728 emitUint8(0x66);
1729 emitRexRB(RexTypeIrrelevant, dst, src);
1730 emitUint8(0x0F);
1731 emitUint8(0x14);
1732 emitXmmRegisterOperand(dst, src);
1733 }
1734
1735 template <typename TraitsType>
unpckhpd(XmmRegister dst,XmmRegister src)1736 void AssemblerX86Base<TraitsType>::unpckhpd(XmmRegister dst, XmmRegister src) {
1737 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1738 emitUint8(0x66);
1739 emitRexRB(RexTypeIrrelevant, dst, src);
1740 emitUint8(0x0F);
1741 emitUint8(0x15);
1742 emitXmmRegisterOperand(dst, src);
1743 }
1744
1745 template <typename TraitsType>
set1ps(XmmRegister dst,GPRRegister tmp1,const Immediate & imm)1746 void AssemblerX86Base<TraitsType>::set1ps(XmmRegister dst, GPRRegister tmp1,
1747 const Immediate &imm) {
1748 // Load 32-bit immediate value into tmp1.
1749 mov(IceType_i32, tmp1, imm);
1750 // Move value from tmp1 into dst.
1751 movd(IceType_i32, dst, tmp1);
1752 // Broadcast low lane into other three lanes.
1753 shufps(RexTypeIrrelevant, dst, dst, Immediate(0x0));
1754 }
1755
1756 template <typename TraitsType>
pshufb(Type,XmmRegister dst,XmmRegister src)1757 void AssemblerX86Base<TraitsType>::pshufb(Type /* Ty */, XmmRegister dst,
1758 XmmRegister src) {
1759 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1760 emitUint8(0x66);
1761 emitRexRB(RexTypeIrrelevant, dst, src);
1762 emitUint8(0x0F);
1763 emitUint8(0x38);
1764 emitUint8(0x00);
1765 emitXmmRegisterOperand(dst, src);
1766 }
1767
1768 template <typename TraitsType>
pshufb(Type,XmmRegister dst,const Address & src)1769 void AssemblerX86Base<TraitsType>::pshufb(Type /* Ty */, XmmRegister dst,
1770 const Address &src) {
1771 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1772 emitUint8(0x66);
1773 emitAddrSizeOverridePrefix();
1774 emitRex(RexTypeIrrelevant, src, dst);
1775 emitUint8(0x0F);
1776 emitUint8(0x38);
1777 emitUint8(0x00);
1778 emitOperand(gprEncoding(dst), src);
1779 }
1780
1781 template <typename TraitsType>
pshufd(Type,XmmRegister dst,XmmRegister src,const Immediate & imm)1782 void AssemblerX86Base<TraitsType>::pshufd(Type /* Ty */, XmmRegister dst,
1783 XmmRegister src,
1784 const Immediate &imm) {
1785 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1786 emitUint8(0x66);
1787 emitRexRB(RexTypeIrrelevant, dst, src);
1788 emitUint8(0x0F);
1789 emitUint8(0x70);
1790 emitXmmRegisterOperand(dst, src);
1791 assert(imm.is_uint8());
1792 emitUint8(imm.value());
1793 }
1794
1795 template <typename TraitsType>
pshufd(Type,XmmRegister dst,const Address & src,const Immediate & imm)1796 void AssemblerX86Base<TraitsType>::pshufd(Type /* Ty */, XmmRegister dst,
1797 const Address &src,
1798 const Immediate &imm) {
1799 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1800 emitUint8(0x66);
1801 emitAddrSizeOverridePrefix();
1802 emitRex(RexTypeIrrelevant, src, dst);
1803 emitUint8(0x0F);
1804 emitUint8(0x70);
1805 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
1806 emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
1807 assert(imm.is_uint8());
1808 emitUint8(imm.value());
1809 }
1810
1811 template <typename TraitsType>
punpckl(Type Ty,XmmRegister Dst,XmmRegister Src)1812 void AssemblerX86Base<TraitsType>::punpckl(Type Ty, XmmRegister Dst,
1813 XmmRegister Src) {
1814 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1815 emitUint8(0x66);
1816 emitRexRB(RexTypeIrrelevant, Dst, Src);
1817 emitUint8(0x0F);
1818 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1819 emitUint8(0x62);
1820 } else if (Ty == IceType_v8i16) {
1821 emitUint8(0x61);
1822 } else if (Ty == IceType_v16i8) {
1823 emitUint8(0x60);
1824 } else {
1825 assert(false && "Unexpected vector unpack operand type");
1826 }
1827 emitXmmRegisterOperand(Dst, Src);
1828 }
1829
1830 template <typename TraitsType>
punpckl(Type Ty,XmmRegister Dst,const Address & Src)1831 void AssemblerX86Base<TraitsType>::punpckl(Type Ty, XmmRegister Dst,
1832 const Address &Src) {
1833 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1834 emitUint8(0x66);
1835 emitAddrSizeOverridePrefix();
1836 emitRex(RexTypeIrrelevant, Src, Dst);
1837 emitUint8(0x0F);
1838 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1839 emitUint8(0x62);
1840 } else if (Ty == IceType_v8i16) {
1841 emitUint8(0x61);
1842 } else if (Ty == IceType_v16i8) {
1843 emitUint8(0x60);
1844 } else {
1845 assert(false && "Unexpected vector unpack operand type");
1846 }
1847 emitOperand(gprEncoding(Dst), Src);
1848 }
1849
1850 template <typename TraitsType>
punpckh(Type Ty,XmmRegister Dst,XmmRegister Src)1851 void AssemblerX86Base<TraitsType>::punpckh(Type Ty, XmmRegister Dst,
1852 XmmRegister Src) {
1853 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1854 emitUint8(0x66);
1855 emitRexRB(RexTypeIrrelevant, Dst, Src);
1856 emitUint8(0x0F);
1857 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1858 emitUint8(0x6A);
1859 } else if (Ty == IceType_v8i16) {
1860 emitUint8(0x69);
1861 } else if (Ty == IceType_v16i8) {
1862 emitUint8(0x68);
1863 } else {
1864 assert(false && "Unexpected vector unpack operand type");
1865 }
1866 emitXmmRegisterOperand(Dst, Src);
1867 }
1868
1869 template <typename TraitsType>
punpckh(Type Ty,XmmRegister Dst,const Address & Src)1870 void AssemblerX86Base<TraitsType>::punpckh(Type Ty, XmmRegister Dst,
1871 const Address &Src) {
1872 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1873 emitUint8(0x66);
1874 emitAddrSizeOverridePrefix();
1875 emitRex(RexTypeIrrelevant, Src, Dst);
1876 emitUint8(0x0F);
1877 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1878 emitUint8(0x6A);
1879 } else if (Ty == IceType_v8i16) {
1880 emitUint8(0x69);
1881 } else if (Ty == IceType_v16i8) {
1882 emitUint8(0x68);
1883 } else {
1884 assert(false && "Unexpected vector unpack operand type");
1885 }
1886 emitOperand(gprEncoding(Dst), Src);
1887 }
1888
1889 template <typename TraitsType>
packss(Type Ty,XmmRegister Dst,XmmRegister Src)1890 void AssemblerX86Base<TraitsType>::packss(Type Ty, XmmRegister Dst,
1891 XmmRegister Src) {
1892 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1893 emitUint8(0x66);
1894 emitRexRB(RexTypeIrrelevant, Dst, Src);
1895 emitUint8(0x0F);
1896 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1897 emitUint8(0x6B);
1898 } else if (Ty == IceType_v8i16) {
1899 emitUint8(0x63);
1900 } else {
1901 assert(false && "Unexpected vector pack operand type");
1902 }
1903 emitXmmRegisterOperand(Dst, Src);
1904 }
1905
1906 template <typename TraitsType>
packss(Type Ty,XmmRegister Dst,const Address & Src)1907 void AssemblerX86Base<TraitsType>::packss(Type Ty, XmmRegister Dst,
1908 const Address &Src) {
1909 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1910 emitUint8(0x66);
1911 emitAddrSizeOverridePrefix();
1912 emitRex(RexTypeIrrelevant, Src, Dst);
1913 emitUint8(0x0F);
1914 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1915 emitUint8(0x6B);
1916 } else if (Ty == IceType_v8i16) {
1917 emitUint8(0x63);
1918 } else {
1919 assert(false && "Unexpected vector pack operand type");
1920 }
1921 emitOperand(gprEncoding(Dst), Src);
1922 }
1923
1924 template <typename TraitsType>
packus(Type Ty,XmmRegister Dst,XmmRegister Src)1925 void AssemblerX86Base<TraitsType>::packus(Type Ty, XmmRegister Dst,
1926 XmmRegister Src) {
1927 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1928 emitUint8(0x66);
1929 emitRexRB(RexTypeIrrelevant, Dst, Src);
1930 emitUint8(0x0F);
1931 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1932 emitUint8(0x38);
1933 emitUint8(0x2B);
1934 } else if (Ty == IceType_v8i16) {
1935 emitUint8(0x67);
1936 } else {
1937 assert(false && "Unexpected vector pack operand type");
1938 }
1939 emitXmmRegisterOperand(Dst, Src);
1940 }
1941
1942 template <typename TraitsType>
packus(Type Ty,XmmRegister Dst,const Address & Src)1943 void AssemblerX86Base<TraitsType>::packus(Type Ty, XmmRegister Dst,
1944 const Address &Src) {
1945 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1946 emitUint8(0x66);
1947 emitAddrSizeOverridePrefix();
1948 emitRex(RexTypeIrrelevant, Src, Dst);
1949 emitUint8(0x0F);
1950 if (Ty == IceType_v4i32 || Ty == IceType_v4f32) {
1951 emitUint8(0x38);
1952 emitUint8(0x2B);
1953 } else if (Ty == IceType_v8i16) {
1954 emitUint8(0x67);
1955 } else {
1956 assert(false && "Unexpected vector pack operand type");
1957 }
1958 emitOperand(gprEncoding(Dst), Src);
1959 }
1960
1961 template <typename TraitsType>
shufps(Type,XmmRegister dst,XmmRegister src,const Immediate & imm)1962 void AssemblerX86Base<TraitsType>::shufps(Type /* Ty */, XmmRegister dst,
1963 XmmRegister src,
1964 const Immediate &imm) {
1965 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1966 emitRexRB(RexTypeIrrelevant, dst, src);
1967 emitUint8(0x0F);
1968 emitUint8(0xC6);
1969 emitXmmRegisterOperand(dst, src);
1970 assert(imm.is_uint8());
1971 emitUint8(imm.value());
1972 }
1973
1974 template <typename TraitsType>
shufps(Type,XmmRegister dst,const Address & src,const Immediate & imm)1975 void AssemblerX86Base<TraitsType>::shufps(Type /* Ty */, XmmRegister dst,
1976 const Address &src,
1977 const Immediate &imm) {
1978 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1979 emitAddrSizeOverridePrefix();
1980 emitRex(RexTypeIrrelevant, src, dst);
1981 emitUint8(0x0F);
1982 emitUint8(0xC6);
1983 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
1984 emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
1985 assert(imm.is_uint8());
1986 emitUint8(imm.value());
1987 }
1988
1989 template <typename TraitsType>
sqrtpd(XmmRegister dst)1990 void AssemblerX86Base<TraitsType>::sqrtpd(XmmRegister dst) {
1991 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
1992 emitUint8(0x66);
1993 emitRexRB(RexTypeIrrelevant, dst, dst);
1994 emitUint8(0x0F);
1995 emitUint8(0x51);
1996 emitXmmRegisterOperand(dst, dst);
1997 }
1998
1999 template <typename TraitsType>
cvtdq2ps(Type,XmmRegister dst,XmmRegister src)2000 void AssemblerX86Base<TraitsType>::cvtdq2ps(Type /* Ignore */, XmmRegister dst,
2001 XmmRegister src) {
2002 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2003 emitRexRB(RexTypeIrrelevant, dst, src);
2004 emitUint8(0x0F);
2005 emitUint8(0x5B);
2006 emitXmmRegisterOperand(dst, src);
2007 }
2008
2009 template <typename TraitsType>
cvtdq2ps(Type,XmmRegister dst,const Address & src)2010 void AssemblerX86Base<TraitsType>::cvtdq2ps(Type /* Ignore */, XmmRegister dst,
2011 const Address &src) {
2012 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2013 emitAddrSizeOverridePrefix();
2014 emitRex(RexTypeIrrelevant, src, dst);
2015 emitUint8(0x0F);
2016 emitUint8(0x5B);
2017 emitOperand(gprEncoding(dst), src);
2018 }
2019
2020 template <typename TraitsType>
cvttps2dq(Type,XmmRegister dst,XmmRegister src)2021 void AssemblerX86Base<TraitsType>::cvttps2dq(Type /* Ignore */, XmmRegister dst,
2022 XmmRegister src) {
2023 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2024 emitUint8(0xF3);
2025 emitRexRB(RexTypeIrrelevant, dst, src);
2026 emitUint8(0x0F);
2027 emitUint8(0x5B);
2028 emitXmmRegisterOperand(dst, src);
2029 }
2030
2031 template <typename TraitsType>
cvttps2dq(Type,XmmRegister dst,const Address & src)2032 void AssemblerX86Base<TraitsType>::cvttps2dq(Type /* Ignore */, XmmRegister dst,
2033 const Address &src) {
2034 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2035 emitUint8(0xF3);
2036 emitAddrSizeOverridePrefix();
2037 emitRex(RexTypeIrrelevant, src, dst);
2038 emitUint8(0x0F);
2039 emitUint8(0x5B);
2040 emitOperand(gprEncoding(dst), src);
2041 }
2042
2043 template <typename TraitsType>
cvtps2dq(Type,XmmRegister dst,XmmRegister src)2044 void AssemblerX86Base<TraitsType>::cvtps2dq(Type /* Ignore */, XmmRegister dst,
2045 XmmRegister src) {
2046 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2047 emitUint8(0x66);
2048 emitRexRB(RexTypeIrrelevant, dst, src);
2049 emitUint8(0x0F);
2050 emitUint8(0x5B);
2051 emitXmmRegisterOperand(dst, src);
2052 }
2053
2054 template <typename TraitsType>
cvtps2dq(Type,XmmRegister dst,const Address & src)2055 void AssemblerX86Base<TraitsType>::cvtps2dq(Type /* Ignore */, XmmRegister dst,
2056 const Address &src) {
2057 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2058 emitUint8(0x66);
2059 emitAddrSizeOverridePrefix();
2060 emitRex(RexTypeIrrelevant, src, dst);
2061 emitUint8(0x0F);
2062 emitUint8(0x5B);
2063 emitOperand(gprEncoding(dst), src);
2064 }
2065
2066 template <typename TraitsType>
cvtsi2ss(Type DestTy,XmmRegister dst,Type SrcTy,GPRRegister src)2067 void AssemblerX86Base<TraitsType>::cvtsi2ss(Type DestTy, XmmRegister dst,
2068 Type SrcTy, GPRRegister src) {
2069 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2070 emitUint8(isFloat32Asserting32Or64(DestTy) ? 0xF3 : 0xF2);
2071 emitRexRB(SrcTy, dst, src);
2072 emitUint8(0x0F);
2073 emitUint8(0x2A);
2074 emitXmmRegisterOperand(dst, src);
2075 }
2076
2077 template <typename TraitsType>
cvtsi2ss(Type DestTy,XmmRegister dst,Type SrcTy,const Address & src)2078 void AssemblerX86Base<TraitsType>::cvtsi2ss(Type DestTy, XmmRegister dst,
2079 Type SrcTy, const Address &src) {
2080 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2081 emitUint8(isFloat32Asserting32Or64(DestTy) ? 0xF3 : 0xF2);
2082 emitAddrSizeOverridePrefix();
2083 emitRex(SrcTy, src, dst);
2084 emitUint8(0x0F);
2085 emitUint8(0x2A);
2086 emitOperand(gprEncoding(dst), src);
2087 }
2088
2089 template <typename TraitsType>
cvtfloat2float(Type SrcTy,XmmRegister dst,XmmRegister src)2090 void AssemblerX86Base<TraitsType>::cvtfloat2float(Type SrcTy, XmmRegister dst,
2091 XmmRegister src) {
2092 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2093 // ss2sd or sd2ss
2094 emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
2095 emitRexRB(RexTypeIrrelevant, dst, src);
2096 emitUint8(0x0F);
2097 emitUint8(0x5A);
2098 emitXmmRegisterOperand(dst, src);
2099 }
2100
2101 template <typename TraitsType>
cvtfloat2float(Type SrcTy,XmmRegister dst,const Address & src)2102 void AssemblerX86Base<TraitsType>::cvtfloat2float(Type SrcTy, XmmRegister dst,
2103 const Address &src) {
2104 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2105 emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
2106 emitAddrSizeOverridePrefix();
2107 emitRex(RexTypeIrrelevant, src, dst);
2108 emitUint8(0x0F);
2109 emitUint8(0x5A);
2110 emitOperand(gprEncoding(dst), src);
2111 }
2112
2113 template <typename TraitsType>
cvttss2si(Type DestTy,GPRRegister dst,Type SrcTy,XmmRegister src)2114 void AssemblerX86Base<TraitsType>::cvttss2si(Type DestTy, GPRRegister dst,
2115 Type SrcTy, XmmRegister src) {
2116 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2117 emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
2118 emitRexRB(DestTy, dst, src);
2119 emitUint8(0x0F);
2120 emitUint8(0x2C);
2121 emitXmmRegisterOperand(dst, src);
2122 }
2123
2124 template <typename TraitsType>
cvttss2si(Type DestTy,GPRRegister dst,Type SrcTy,const Address & src)2125 void AssemblerX86Base<TraitsType>::cvttss2si(Type DestTy, GPRRegister dst,
2126 Type SrcTy, const Address &src) {
2127 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2128 emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
2129 emitAddrSizeOverridePrefix();
2130 emitRex(DestTy, src, dst);
2131 emitUint8(0x0F);
2132 emitUint8(0x2C);
2133 emitOperand(gprEncoding(dst), src);
2134 }
2135
2136 template <typename TraitsType>
cvtss2si(Type DestTy,GPRRegister dst,Type SrcTy,XmmRegister src)2137 void AssemblerX86Base<TraitsType>::cvtss2si(Type DestTy, GPRRegister dst,
2138 Type SrcTy, XmmRegister src) {
2139 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2140 emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
2141 emitRexRB(DestTy, dst, src);
2142 emitUint8(0x0F);
2143 emitUint8(0x2D);
2144 emitXmmRegisterOperand(dst, src);
2145 }
2146
2147 template <typename TraitsType>
cvtss2si(Type DestTy,GPRRegister dst,Type SrcTy,const Address & src)2148 void AssemblerX86Base<TraitsType>::cvtss2si(Type DestTy, GPRRegister dst,
2149 Type SrcTy, const Address &src) {
2150 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2151 emitUint8(isFloat32Asserting32Or64(SrcTy) ? 0xF3 : 0xF2);
2152 emitAddrSizeOverridePrefix();
2153 emitRex(DestTy, src, dst);
2154 emitUint8(0x0F);
2155 emitUint8(0x2D);
2156 emitOperand(gprEncoding(dst), src);
2157 }
2158
2159 template <typename TraitsType>
ucomiss(Type Ty,XmmRegister a,XmmRegister b)2160 void AssemblerX86Base<TraitsType>::ucomiss(Type Ty, XmmRegister a,
2161 XmmRegister b) {
2162 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2163 if (Ty == IceType_f64)
2164 emitUint8(0x66);
2165 emitRexRB(RexTypeIrrelevant, a, b);
2166 emitUint8(0x0F);
2167 emitUint8(0x2E);
2168 emitXmmRegisterOperand(a, b);
2169 }
2170
2171 template <typename TraitsType>
ucomiss(Type Ty,XmmRegister a,const Address & b)2172 void AssemblerX86Base<TraitsType>::ucomiss(Type Ty, XmmRegister a,
2173 const Address &b) {
2174 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2175 if (Ty == IceType_f64)
2176 emitUint8(0x66);
2177 emitAddrSizeOverridePrefix();
2178 emitRex(RexTypeIrrelevant, b, a);
2179 emitUint8(0x0F);
2180 emitUint8(0x2E);
2181 emitOperand(gprEncoding(a), b);
2182 }
2183
2184 template <typename TraitsType>
movmsk(Type Ty,GPRRegister dst,XmmRegister src)2185 void AssemblerX86Base<TraitsType>::movmsk(Type Ty, GPRRegister dst,
2186 XmmRegister src) {
2187 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2188 if (Ty == IceType_v16i8) {
2189 emitUint8(0x66);
2190 } else if (Ty == IceType_v4f32 || Ty == IceType_v4i32) {
2191 // No operand size prefix
2192 } else {
2193 assert(false && "Unexpected movmsk operand type");
2194 }
2195 emitRexRB(RexTypeIrrelevant, dst, src);
2196 emitUint8(0x0F);
2197 if (Ty == IceType_v16i8) {
2198 emitUint8(0xD7);
2199 } else if (Ty == IceType_v4f32 || Ty == IceType_v4i32) {
2200 emitUint8(0x50);
2201 } else {
2202 assert(false && "Unexpected movmsk operand type");
2203 }
2204 emitXmmRegisterOperand(dst, src);
2205 }
2206
2207 template <typename TraitsType>
sqrt(Type Ty,XmmRegister dst,const Address & src)2208 void AssemblerX86Base<TraitsType>::sqrt(Type Ty, XmmRegister dst,
2209 const Address &src) {
2210 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2211 if (isScalarFloatingType(Ty))
2212 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
2213 emitAddrSizeOverridePrefix();
2214 emitRex(RexTypeIrrelevant, src, dst);
2215 emitUint8(0x0F);
2216 emitUint8(0x51);
2217 emitOperand(gprEncoding(dst), src);
2218 }
2219
2220 template <typename TraitsType>
sqrt(Type Ty,XmmRegister dst,XmmRegister src)2221 void AssemblerX86Base<TraitsType>::sqrt(Type Ty, XmmRegister dst,
2222 XmmRegister src) {
2223 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2224 if (isScalarFloatingType(Ty))
2225 emitUint8(isFloat32Asserting32Or64(Ty) ? 0xF3 : 0xF2);
2226 emitRexRB(RexTypeIrrelevant, dst, src);
2227 emitUint8(0x0F);
2228 emitUint8(0x51);
2229 emitXmmRegisterOperand(dst, src);
2230 }
2231
2232 template <typename TraitsType>
xorps(Type Ty,XmmRegister dst,const Address & src)2233 void AssemblerX86Base<TraitsType>::xorps(Type Ty, XmmRegister dst,
2234 const Address &src) {
2235 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2236 if (!isFloat32Asserting32Or64(Ty))
2237 emitUint8(0x66);
2238 emitAddrSizeOverridePrefix();
2239 emitRex(RexTypeIrrelevant, src, dst);
2240 emitUint8(0x0F);
2241 emitUint8(0x57);
2242 emitOperand(gprEncoding(dst), src);
2243 }
2244
2245 template <typename TraitsType>
xorps(Type Ty,XmmRegister dst,XmmRegister src)2246 void AssemblerX86Base<TraitsType>::xorps(Type Ty, XmmRegister dst,
2247 XmmRegister src) {
2248 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2249 if (!isFloat32Asserting32Or64(Ty))
2250 emitUint8(0x66);
2251 emitRexRB(RexTypeIrrelevant, dst, src);
2252 emitUint8(0x0F);
2253 emitUint8(0x57);
2254 emitXmmRegisterOperand(dst, src);
2255 }
2256
2257 template <typename TraitsType>
insertps(Type Ty,XmmRegister dst,XmmRegister src,const Immediate & imm)2258 void AssemblerX86Base<TraitsType>::insertps(Type Ty, XmmRegister dst,
2259 XmmRegister src,
2260 const Immediate &imm) {
2261 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2262 assert(imm.is_uint8());
2263 assert(isVectorFloatingType(Ty));
2264 (void)Ty;
2265 emitUint8(0x66);
2266 emitRexRB(RexTypeIrrelevant, dst, src);
2267 emitUint8(0x0F);
2268 emitUint8(0x3A);
2269 emitUint8(0x21);
2270 emitXmmRegisterOperand(dst, src);
2271 emitUint8(imm.value());
2272 }
2273
2274 template <typename TraitsType>
insertps(Type Ty,XmmRegister dst,const Address & src,const Immediate & imm)2275 void AssemblerX86Base<TraitsType>::insertps(Type Ty, XmmRegister dst,
2276 const Address &src,
2277 const Immediate &imm) {
2278 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2279 assert(imm.is_uint8());
2280 assert(isVectorFloatingType(Ty));
2281 (void)Ty;
2282 emitUint8(0x66);
2283 emitAddrSizeOverridePrefix();
2284 emitRex(RexTypeIrrelevant, src, dst);
2285 emitUint8(0x0F);
2286 emitUint8(0x3A);
2287 emitUint8(0x21);
2288 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
2289 emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
2290 emitUint8(imm.value());
2291 }
2292
2293 template <typename TraitsType>
pinsr(Type Ty,XmmRegister dst,GPRRegister src,const Immediate & imm)2294 void AssemblerX86Base<TraitsType>::pinsr(Type Ty, XmmRegister dst,
2295 GPRRegister src,
2296 const Immediate &imm) {
2297 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2298 assert(imm.is_uint8());
2299 emitUint8(0x66);
2300 emitRexRB(Ty, dst, src);
2301 emitUint8(0x0F);
2302 if (Ty == IceType_i16) {
2303 emitUint8(0xC4);
2304 } else {
2305 emitUint8(0x3A);
2306 emitUint8(isByteSizedType(Ty) ? 0x20 : 0x22);
2307 }
2308 emitXmmRegisterOperand(dst, src);
2309 emitUint8(imm.value());
2310 }
2311
2312 template <typename TraitsType>
pinsr(Type Ty,XmmRegister dst,const Address & src,const Immediate & imm)2313 void AssemblerX86Base<TraitsType>::pinsr(Type Ty, XmmRegister dst,
2314 const Address &src,
2315 const Immediate &imm) {
2316 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2317 assert(imm.is_uint8());
2318 emitUint8(0x66);
2319 emitAddrSizeOverridePrefix();
2320 emitRex(RexTypeIrrelevant, src, dst);
2321 emitUint8(0x0F);
2322 if (Ty == IceType_i16) {
2323 emitUint8(0xC4);
2324 } else {
2325 emitUint8(0x3A);
2326 emitUint8(isByteSizedType(Ty) ? 0x20 : 0x22);
2327 }
2328 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
2329 emitOperand(gprEncoding(dst), src, OffsetFromNextInstruction);
2330 emitUint8(imm.value());
2331 }
2332
2333 template <typename TraitsType>
pextr(Type Ty,GPRRegister dst,XmmRegister src,const Immediate & imm)2334 void AssemblerX86Base<TraitsType>::pextr(Type Ty, GPRRegister dst,
2335 XmmRegister src,
2336 const Immediate &imm) {
2337 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2338 assert(imm.is_uint8());
2339 if (Ty == IceType_i16) {
2340 emitUint8(0x66);
2341 emitRexRB(Ty, dst, src);
2342 emitUint8(0x0F);
2343 emitUint8(0xC5);
2344 emitXmmRegisterOperand(dst, src);
2345 emitUint8(imm.value());
2346 } else {
2347 emitUint8(0x66);
2348 emitRexRB(Ty, src, dst);
2349 emitUint8(0x0F);
2350 emitUint8(0x3A);
2351 emitUint8(isByteSizedType(Ty) ? 0x14 : 0x16);
2352 // SSE 4.1 versions are "MRI" because dst can be mem, while pextrw (SSE2)
2353 // is RMI because dst must be reg.
2354 emitXmmRegisterOperand(src, dst);
2355 emitUint8(imm.value());
2356 }
2357 }
2358
2359 template <typename TraitsType>
pmovsxdq(XmmRegister dst,XmmRegister src)2360 void AssemblerX86Base<TraitsType>::pmovsxdq(XmmRegister dst, XmmRegister src) {
2361 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2362 emitUint8(0x66);
2363 emitRexRB(RexTypeIrrelevant, dst, src);
2364 emitUint8(0x0F);
2365 emitUint8(0x38);
2366 emitUint8(0x25);
2367 emitXmmRegisterOperand(dst, src);
2368 }
2369
2370 template <typename TraitsType>
pcmpeq(Type Ty,XmmRegister dst,XmmRegister src)2371 void AssemblerX86Base<TraitsType>::pcmpeq(Type Ty, XmmRegister dst,
2372 XmmRegister src) {
2373 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2374 emitUint8(0x66);
2375 emitRexRB(RexTypeIrrelevant, dst, src);
2376 emitUint8(0x0F);
2377 if (isByteSizedArithType(Ty)) {
2378 emitUint8(0x74);
2379 } else if (Ty == IceType_i16) {
2380 emitUint8(0x75);
2381 } else {
2382 emitUint8(0x76);
2383 }
2384 emitXmmRegisterOperand(dst, src);
2385 }
2386
2387 template <typename TraitsType>
pcmpeq(Type Ty,XmmRegister dst,const Address & src)2388 void AssemblerX86Base<TraitsType>::pcmpeq(Type Ty, XmmRegister dst,
2389 const Address &src) {
2390 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2391 emitUint8(0x66);
2392 emitAddrSizeOverridePrefix();
2393 emitRex(RexTypeIrrelevant, src, dst);
2394 emitUint8(0x0F);
2395 if (isByteSizedArithType(Ty)) {
2396 emitUint8(0x74);
2397 } else if (Ty == IceType_i16) {
2398 emitUint8(0x75);
2399 } else {
2400 emitUint8(0x76);
2401 }
2402 emitOperand(gprEncoding(dst), src);
2403 }
2404
2405 template <typename TraitsType>
pcmpgt(Type Ty,XmmRegister dst,XmmRegister src)2406 void AssemblerX86Base<TraitsType>::pcmpgt(Type Ty, XmmRegister dst,
2407 XmmRegister src) {
2408 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2409 emitUint8(0x66);
2410 emitRexRB(RexTypeIrrelevant, dst, src);
2411 emitUint8(0x0F);
2412 if (isByteSizedArithType(Ty)) {
2413 emitUint8(0x64);
2414 } else if (Ty == IceType_i16) {
2415 emitUint8(0x65);
2416 } else {
2417 emitUint8(0x66);
2418 }
2419 emitXmmRegisterOperand(dst, src);
2420 }
2421
2422 template <typename TraitsType>
pcmpgt(Type Ty,XmmRegister dst,const Address & src)2423 void AssemblerX86Base<TraitsType>::pcmpgt(Type Ty, XmmRegister dst,
2424 const Address &src) {
2425 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2426 emitUint8(0x66);
2427 emitAddrSizeOverridePrefix();
2428 emitRex(RexTypeIrrelevant, src, dst);
2429 emitUint8(0x0F);
2430 if (isByteSizedArithType(Ty)) {
2431 emitUint8(0x64);
2432 } else if (Ty == IceType_i16) {
2433 emitUint8(0x65);
2434 } else {
2435 emitUint8(0x66);
2436 }
2437 emitOperand(gprEncoding(dst), src);
2438 }
2439
2440 template <typename TraitsType>
round(Type Ty,XmmRegister dst,XmmRegister src,const Immediate & mode)2441 void AssemblerX86Base<TraitsType>::round(Type Ty, XmmRegister dst,
2442 XmmRegister src,
2443 const Immediate &mode) {
2444 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2445 emitUint8(0x66);
2446 emitRexRB(RexTypeIrrelevant, dst, src);
2447 emitUint8(0x0F);
2448 emitUint8(0x3A);
2449 switch (Ty) {
2450 case IceType_v4f32:
2451 emitUint8(0x08);
2452 break;
2453 case IceType_f32:
2454 emitUint8(0x0A);
2455 break;
2456 case IceType_f64:
2457 emitUint8(0x0B);
2458 break;
2459 default:
2460 assert(false && "Unsupported round operand type");
2461 }
2462 emitXmmRegisterOperand(dst, src);
2463 // Mask precision exeption.
2464 emitUint8(static_cast<uint8_t>(mode.value()) | 0x8);
2465 }
2466
2467 template <typename TraitsType>
round(Type Ty,XmmRegister dst,const Address & src,const Immediate & mode)2468 void AssemblerX86Base<TraitsType>::round(Type Ty, XmmRegister dst,
2469 const Address &src,
2470 const Immediate &mode) {
2471 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2472 emitUint8(0x66);
2473 emitAddrSizeOverridePrefix();
2474 emitRex(RexTypeIrrelevant, src, dst);
2475 emitUint8(0x0F);
2476 emitUint8(0x3A);
2477 switch (Ty) {
2478 case IceType_v4f32:
2479 emitUint8(0x08);
2480 break;
2481 case IceType_f32:
2482 emitUint8(0x0A);
2483 break;
2484 case IceType_f64:
2485 emitUint8(0x0B);
2486 break;
2487 default:
2488 assert(false && "Unsupported round operand type");
2489 }
2490 emitOperand(gprEncoding(dst), src);
2491 // Mask precision exeption.
2492 emitUint8(static_cast<uint8_t>(mode.value()) | 0x8);
2493 }
2494
2495 template <typename TraitsType>
2496 template <typename T, typename>
fnstcw(const typename T::Address & dst)2497 void AssemblerX86Base<TraitsType>::fnstcw(const typename T::Address &dst) {
2498 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2499 emitAddrSizeOverridePrefix();
2500 emitUint8(0xD9);
2501 emitOperand(7, dst);
2502 }
2503
2504 template <typename TraitsType>
2505 template <typename T, typename>
fldcw(const typename T::Address & src)2506 void AssemblerX86Base<TraitsType>::fldcw(const typename T::Address &src) {
2507 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2508 emitAddrSizeOverridePrefix();
2509 emitUint8(0xD9);
2510 emitOperand(5, src);
2511 }
2512
2513 template <typename TraitsType>
2514 template <typename T, typename>
fistpl(const typename T::Address & dst)2515 void AssemblerX86Base<TraitsType>::fistpl(const typename T::Address &dst) {
2516 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2517 emitAddrSizeOverridePrefix();
2518 emitUint8(0xDF);
2519 emitOperand(7, dst);
2520 }
2521
2522 template <typename TraitsType>
2523 template <typename T, typename>
fistps(const typename T::Address & dst)2524 void AssemblerX86Base<TraitsType>::fistps(const typename T::Address &dst) {
2525 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2526 emitAddrSizeOverridePrefix();
2527 emitUint8(0xDB);
2528 emitOperand(3, dst);
2529 }
2530
2531 template <typename TraitsType>
2532 template <typename T, typename>
fildl(const typename T::Address & src)2533 void AssemblerX86Base<TraitsType>::fildl(const typename T::Address &src) {
2534 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2535 emitAddrSizeOverridePrefix();
2536 emitUint8(0xDF);
2537 emitOperand(5, src);
2538 }
2539
2540 template <typename TraitsType>
2541 template <typename T, typename>
filds(const typename T::Address & src)2542 void AssemblerX86Base<TraitsType>::filds(const typename T::Address &src) {
2543 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2544 emitAddrSizeOverridePrefix();
2545 emitUint8(0xDB);
2546 emitOperand(0, src);
2547 }
2548
2549 template <typename TraitsType>
2550 template <typename, typename>
fincstp()2551 void AssemblerX86Base<TraitsType>::fincstp() {
2552 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2553 emitUint8(0xD9);
2554 emitUint8(0xF7);
2555 }
2556
2557 template <typename TraitsType>
2558 template <uint32_t Tag>
arith_int(Type Ty,GPRRegister reg,const Immediate & imm)2559 void AssemblerX86Base<TraitsType>::arith_int(Type Ty, GPRRegister reg,
2560 const Immediate &imm) {
2561 static_assert(Tag < 8, "Tag must be between 0..7");
2562 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2563 if (Ty == IceType_i16)
2564 emitOperandSizeOverride();
2565 emitRexB(Ty, reg);
2566 if (isByteSizedType(Ty)) {
2567 emitComplexI8(Tag, Operand(reg), imm);
2568 } else {
2569 emitComplex(Ty, Tag, Operand(reg), imm);
2570 }
2571 }
2572
2573 template <typename TraitsType>
2574 template <uint32_t Tag>
arith_int(Type Ty,GPRRegister reg0,GPRRegister reg1)2575 void AssemblerX86Base<TraitsType>::arith_int(Type Ty, GPRRegister reg0,
2576 GPRRegister reg1) {
2577 static_assert(Tag < 8, "Tag must be between 0..7");
2578 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2579 if (Ty == IceType_i16)
2580 emitOperandSizeOverride();
2581 emitRexRB(Ty, reg0, reg1);
2582 if (isByteSizedType(Ty))
2583 emitUint8(Tag * 8 + 2);
2584 else
2585 emitUint8(Tag * 8 + 3);
2586 emitRegisterOperand(gprEncoding(reg0), gprEncoding(reg1));
2587 }
2588
2589 template <typename TraitsType>
2590 template <uint32_t Tag>
arith_int(Type Ty,GPRRegister reg,const Address & address)2591 void AssemblerX86Base<TraitsType>::arith_int(Type Ty, GPRRegister reg,
2592 const Address &address) {
2593 static_assert(Tag < 8, "Tag must be between 0..7");
2594 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2595 if (Ty == IceType_i16)
2596 emitOperandSizeOverride();
2597 emitAddrSizeOverridePrefix();
2598 emitRex(Ty, address, reg);
2599 if (isByteSizedType(Ty))
2600 emitUint8(Tag * 8 + 2);
2601 else
2602 emitUint8(Tag * 8 + 3);
2603 emitOperand(gprEncoding(reg), address);
2604 }
2605
2606 template <typename TraitsType>
2607 template <uint32_t Tag>
arith_int(Type Ty,const Address & address,GPRRegister reg)2608 void AssemblerX86Base<TraitsType>::arith_int(Type Ty, const Address &address,
2609 GPRRegister reg) {
2610 static_assert(Tag < 8, "Tag must be between 0..7");
2611 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2612 if (Ty == IceType_i16)
2613 emitOperandSizeOverride();
2614 emitAddrSizeOverridePrefix();
2615 emitRex(Ty, address, reg);
2616 if (isByteSizedType(Ty))
2617 emitUint8(Tag * 8 + 0);
2618 else
2619 emitUint8(Tag * 8 + 1);
2620 emitOperand(gprEncoding(reg), address);
2621 }
2622
2623 template <typename TraitsType>
2624 template <uint32_t Tag>
arith_int(Type Ty,const Address & address,const Immediate & imm)2625 void AssemblerX86Base<TraitsType>::arith_int(Type Ty, const Address &address,
2626 const Immediate &imm) {
2627 static_assert(Tag < 8, "Tag must be between 0..7");
2628 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2629 if (Ty == IceType_i16)
2630 emitOperandSizeOverride();
2631 emitAddrSizeOverridePrefix();
2632 emitRex(Ty, address, RexRegIrrelevant);
2633 if (isByteSizedType(Ty)) {
2634 emitComplexI8(Tag, address, imm);
2635 } else {
2636 emitComplex(Ty, Tag, address, imm);
2637 }
2638 }
2639
2640 template <typename TraitsType>
cmp(Type Ty,GPRRegister reg,const Immediate & imm)2641 void AssemblerX86Base<TraitsType>::cmp(Type Ty, GPRRegister reg,
2642 const Immediate &imm) {
2643 arith_int<7>(Ty, reg, imm);
2644 }
2645
2646 template <typename TraitsType>
cmp(Type Ty,GPRRegister reg0,GPRRegister reg1)2647 void AssemblerX86Base<TraitsType>::cmp(Type Ty, GPRRegister reg0,
2648 GPRRegister reg1) {
2649 arith_int<7>(Ty, reg0, reg1);
2650 }
2651
2652 template <typename TraitsType>
cmp(Type Ty,GPRRegister reg,const Address & address)2653 void AssemblerX86Base<TraitsType>::cmp(Type Ty, GPRRegister reg,
2654 const Address &address) {
2655 arith_int<7>(Ty, reg, address);
2656 }
2657
2658 template <typename TraitsType>
cmp(Type Ty,const Address & address,GPRRegister reg)2659 void AssemblerX86Base<TraitsType>::cmp(Type Ty, const Address &address,
2660 GPRRegister reg) {
2661 arith_int<7>(Ty, address, reg);
2662 }
2663
2664 template <typename TraitsType>
cmp(Type Ty,const Address & address,const Immediate & imm)2665 void AssemblerX86Base<TraitsType>::cmp(Type Ty, const Address &address,
2666 const Immediate &imm) {
2667 arith_int<7>(Ty, address, imm);
2668 }
2669
2670 template <typename TraitsType>
test(Type Ty,GPRRegister reg1,GPRRegister reg2)2671 void AssemblerX86Base<TraitsType>::test(Type Ty, GPRRegister reg1,
2672 GPRRegister reg2) {
2673 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2674 if (Ty == IceType_i16)
2675 emitOperandSizeOverride();
2676 emitRexRB(Ty, reg1, reg2);
2677 if (isByteSizedType(Ty))
2678 emitUint8(0x84);
2679 else
2680 emitUint8(0x85);
2681 emitRegisterOperand(gprEncoding(reg1), gprEncoding(reg2));
2682 }
2683
2684 template <typename TraitsType>
test(Type Ty,const Address & addr,GPRRegister reg)2685 void AssemblerX86Base<TraitsType>::test(Type Ty, const Address &addr,
2686 GPRRegister reg) {
2687 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2688 if (Ty == IceType_i16)
2689 emitOperandSizeOverride();
2690 emitAddrSizeOverridePrefix();
2691 emitRex(Ty, addr, reg);
2692 if (isByteSizedType(Ty))
2693 emitUint8(0x84);
2694 else
2695 emitUint8(0x85);
2696 emitOperand(gprEncoding(reg), addr);
2697 }
2698
2699 template <typename TraitsType>
test(Type Ty,GPRRegister reg,const Immediate & immediate)2700 void AssemblerX86Base<TraitsType>::test(Type Ty, GPRRegister reg,
2701 const Immediate &immediate) {
2702 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2703 // For registers that have a byte variant (EAX, EBX, ECX, and EDX) we only
2704 // test the byte register to keep the encoding short. This is legal even if
2705 // the register had high bits set since this only sets flags registers based
2706 // on the "AND" of the two operands, and the immediate had zeros at those
2707 // high bits.
2708 if (immediate.is_uint8() && reg <= Traits::Last8BitGPR) {
2709 // Use zero-extended 8-bit immediate.
2710 emitRexB(Ty, reg);
2711 if (reg == Traits::Encoded_Reg_Accumulator) {
2712 emitUint8(0xA8);
2713 } else {
2714 emitUint8(0xF6);
2715 emitUint8(0xC0 + gprEncoding(reg));
2716 }
2717 emitUint8(immediate.value() & 0xFF);
2718 } else if (reg == Traits::Encoded_Reg_Accumulator) {
2719 // Use short form if the destination is EAX.
2720 if (Ty == IceType_i16)
2721 emitOperandSizeOverride();
2722 emitUint8(0xA9);
2723 emitImmediate(Ty, immediate);
2724 } else {
2725 if (Ty == IceType_i16)
2726 emitOperandSizeOverride();
2727 emitRexB(Ty, reg);
2728 emitUint8(0xF7);
2729 emitRegisterOperand(0, gprEncoding(reg));
2730 emitImmediate(Ty, immediate);
2731 }
2732 }
2733
2734 template <typename TraitsType>
test(Type Ty,const Address & addr,const Immediate & immediate)2735 void AssemblerX86Base<TraitsType>::test(Type Ty, const Address &addr,
2736 const Immediate &immediate) {
2737 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2738 // If the immediate is short, we only test the byte addr to keep the encoding
2739 // short.
2740 if (immediate.is_uint8()) {
2741 // Use zero-extended 8-bit immediate.
2742 emitAddrSizeOverridePrefix();
2743 emitRex(Ty, addr, RexRegIrrelevant);
2744 emitUint8(0xF6);
2745 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
2746 emitOperand(0, addr, OffsetFromNextInstruction);
2747 emitUint8(immediate.value() & 0xFF);
2748 } else {
2749 if (Ty == IceType_i16)
2750 emitOperandSizeOverride();
2751 emitAddrSizeOverridePrefix();
2752 emitRex(Ty, addr, RexRegIrrelevant);
2753 emitUint8(0xF7);
2754 const uint8_t OffsetFromNextInstruction = Ty == IceType_i16 ? 2 : 4;
2755 emitOperand(0, addr, OffsetFromNextInstruction);
2756 emitImmediate(Ty, immediate);
2757 }
2758 }
2759
2760 template <typename TraitsType>
And(Type Ty,GPRRegister dst,GPRRegister src)2761 void AssemblerX86Base<TraitsType>::And(Type Ty, GPRRegister dst,
2762 GPRRegister src) {
2763 arith_int<4>(Ty, dst, src);
2764 }
2765
2766 template <typename TraitsType>
And(Type Ty,GPRRegister dst,const Address & address)2767 void AssemblerX86Base<TraitsType>::And(Type Ty, GPRRegister dst,
2768 const Address &address) {
2769 arith_int<4>(Ty, dst, address);
2770 }
2771
2772 template <typename TraitsType>
And(Type Ty,GPRRegister dst,const Immediate & imm)2773 void AssemblerX86Base<TraitsType>::And(Type Ty, GPRRegister dst,
2774 const Immediate &imm) {
2775 arith_int<4>(Ty, dst, imm);
2776 }
2777
2778 template <typename TraitsType>
And(Type Ty,const Address & address,GPRRegister reg)2779 void AssemblerX86Base<TraitsType>::And(Type Ty, const Address &address,
2780 GPRRegister reg) {
2781 arith_int<4>(Ty, address, reg);
2782 }
2783
2784 template <typename TraitsType>
And(Type Ty,const Address & address,const Immediate & imm)2785 void AssemblerX86Base<TraitsType>::And(Type Ty, const Address &address,
2786 const Immediate &imm) {
2787 arith_int<4>(Ty, address, imm);
2788 }
2789
2790 template <typename TraitsType>
Or(Type Ty,GPRRegister dst,GPRRegister src)2791 void AssemblerX86Base<TraitsType>::Or(Type Ty, GPRRegister dst,
2792 GPRRegister src) {
2793 arith_int<1>(Ty, dst, src);
2794 }
2795
2796 template <typename TraitsType>
Or(Type Ty,GPRRegister dst,const Address & address)2797 void AssemblerX86Base<TraitsType>::Or(Type Ty, GPRRegister dst,
2798 const Address &address) {
2799 arith_int<1>(Ty, dst, address);
2800 }
2801
2802 template <typename TraitsType>
Or(Type Ty,GPRRegister dst,const Immediate & imm)2803 void AssemblerX86Base<TraitsType>::Or(Type Ty, GPRRegister dst,
2804 const Immediate &imm) {
2805 arith_int<1>(Ty, dst, imm);
2806 }
2807
2808 template <typename TraitsType>
Or(Type Ty,const Address & address,GPRRegister reg)2809 void AssemblerX86Base<TraitsType>::Or(Type Ty, const Address &address,
2810 GPRRegister reg) {
2811 arith_int<1>(Ty, address, reg);
2812 }
2813
2814 template <typename TraitsType>
Or(Type Ty,const Address & address,const Immediate & imm)2815 void AssemblerX86Base<TraitsType>::Or(Type Ty, const Address &address,
2816 const Immediate &imm) {
2817 arith_int<1>(Ty, address, imm);
2818 }
2819
2820 template <typename TraitsType>
Xor(Type Ty,GPRRegister dst,GPRRegister src)2821 void AssemblerX86Base<TraitsType>::Xor(Type Ty, GPRRegister dst,
2822 GPRRegister src) {
2823 arith_int<6>(Ty, dst, src);
2824 }
2825
2826 template <typename TraitsType>
Xor(Type Ty,GPRRegister dst,const Address & address)2827 void AssemblerX86Base<TraitsType>::Xor(Type Ty, GPRRegister dst,
2828 const Address &address) {
2829 arith_int<6>(Ty, dst, address);
2830 }
2831
2832 template <typename TraitsType>
Xor(Type Ty,GPRRegister dst,const Immediate & imm)2833 void AssemblerX86Base<TraitsType>::Xor(Type Ty, GPRRegister dst,
2834 const Immediate &imm) {
2835 arith_int<6>(Ty, dst, imm);
2836 }
2837
2838 template <typename TraitsType>
Xor(Type Ty,const Address & address,GPRRegister reg)2839 void AssemblerX86Base<TraitsType>::Xor(Type Ty, const Address &address,
2840 GPRRegister reg) {
2841 arith_int<6>(Ty, address, reg);
2842 }
2843
2844 template <typename TraitsType>
Xor(Type Ty,const Address & address,const Immediate & imm)2845 void AssemblerX86Base<TraitsType>::Xor(Type Ty, const Address &address,
2846 const Immediate &imm) {
2847 arith_int<6>(Ty, address, imm);
2848 }
2849
2850 template <typename TraitsType>
add(Type Ty,GPRRegister dst,GPRRegister src)2851 void AssemblerX86Base<TraitsType>::add(Type Ty, GPRRegister dst,
2852 GPRRegister src) {
2853 arith_int<0>(Ty, dst, src);
2854 }
2855
2856 template <typename TraitsType>
add(Type Ty,GPRRegister reg,const Address & address)2857 void AssemblerX86Base<TraitsType>::add(Type Ty, GPRRegister reg,
2858 const Address &address) {
2859 arith_int<0>(Ty, reg, address);
2860 }
2861
2862 template <typename TraitsType>
add(Type Ty,GPRRegister reg,const Immediate & imm)2863 void AssemblerX86Base<TraitsType>::add(Type Ty, GPRRegister reg,
2864 const Immediate &imm) {
2865 arith_int<0>(Ty, reg, imm);
2866 }
2867
2868 template <typename TraitsType>
add(Type Ty,const Address & address,GPRRegister reg)2869 void AssemblerX86Base<TraitsType>::add(Type Ty, const Address &address,
2870 GPRRegister reg) {
2871 arith_int<0>(Ty, address, reg);
2872 }
2873
2874 template <typename TraitsType>
add(Type Ty,const Address & address,const Immediate & imm)2875 void AssemblerX86Base<TraitsType>::add(Type Ty, const Address &address,
2876 const Immediate &imm) {
2877 arith_int<0>(Ty, address, imm);
2878 }
2879
2880 template <typename TraitsType>
adc(Type Ty,GPRRegister dst,GPRRegister src)2881 void AssemblerX86Base<TraitsType>::adc(Type Ty, GPRRegister dst,
2882 GPRRegister src) {
2883 arith_int<2>(Ty, dst, src);
2884 }
2885
2886 template <typename TraitsType>
adc(Type Ty,GPRRegister dst,const Address & address)2887 void AssemblerX86Base<TraitsType>::adc(Type Ty, GPRRegister dst,
2888 const Address &address) {
2889 arith_int<2>(Ty, dst, address);
2890 }
2891
2892 template <typename TraitsType>
adc(Type Ty,GPRRegister reg,const Immediate & imm)2893 void AssemblerX86Base<TraitsType>::adc(Type Ty, GPRRegister reg,
2894 const Immediate &imm) {
2895 arith_int<2>(Ty, reg, imm);
2896 }
2897
2898 template <typename TraitsType>
adc(Type Ty,const Address & address,GPRRegister reg)2899 void AssemblerX86Base<TraitsType>::adc(Type Ty, const Address &address,
2900 GPRRegister reg) {
2901 arith_int<2>(Ty, address, reg);
2902 }
2903
2904 template <typename TraitsType>
adc(Type Ty,const Address & address,const Immediate & imm)2905 void AssemblerX86Base<TraitsType>::adc(Type Ty, const Address &address,
2906 const Immediate &imm) {
2907 arith_int<2>(Ty, address, imm);
2908 }
2909
2910 template <typename TraitsType>
sub(Type Ty,GPRRegister dst,GPRRegister src)2911 void AssemblerX86Base<TraitsType>::sub(Type Ty, GPRRegister dst,
2912 GPRRegister src) {
2913 arith_int<5>(Ty, dst, src);
2914 }
2915
2916 template <typename TraitsType>
sub(Type Ty,GPRRegister reg,const Address & address)2917 void AssemblerX86Base<TraitsType>::sub(Type Ty, GPRRegister reg,
2918 const Address &address) {
2919 arith_int<5>(Ty, reg, address);
2920 }
2921
2922 template <typename TraitsType>
sub(Type Ty,GPRRegister reg,const Immediate & imm)2923 void AssemblerX86Base<TraitsType>::sub(Type Ty, GPRRegister reg,
2924 const Immediate &imm) {
2925 arith_int<5>(Ty, reg, imm);
2926 }
2927
2928 template <typename TraitsType>
sub(Type Ty,const Address & address,GPRRegister reg)2929 void AssemblerX86Base<TraitsType>::sub(Type Ty, const Address &address,
2930 GPRRegister reg) {
2931 arith_int<5>(Ty, address, reg);
2932 }
2933
2934 template <typename TraitsType>
sub(Type Ty,const Address & address,const Immediate & imm)2935 void AssemblerX86Base<TraitsType>::sub(Type Ty, const Address &address,
2936 const Immediate &imm) {
2937 arith_int<5>(Ty, address, imm);
2938 }
2939
2940 template <typename TraitsType>
sbb(Type Ty,GPRRegister dst,GPRRegister src)2941 void AssemblerX86Base<TraitsType>::sbb(Type Ty, GPRRegister dst,
2942 GPRRegister src) {
2943 arith_int<3>(Ty, dst, src);
2944 }
2945
2946 template <typename TraitsType>
sbb(Type Ty,GPRRegister dst,const Address & address)2947 void AssemblerX86Base<TraitsType>::sbb(Type Ty, GPRRegister dst,
2948 const Address &address) {
2949 arith_int<3>(Ty, dst, address);
2950 }
2951
2952 template <typename TraitsType>
sbb(Type Ty,GPRRegister reg,const Immediate & imm)2953 void AssemblerX86Base<TraitsType>::sbb(Type Ty, GPRRegister reg,
2954 const Immediate &imm) {
2955 arith_int<3>(Ty, reg, imm);
2956 }
2957
2958 template <typename TraitsType>
sbb(Type Ty,const Address & address,GPRRegister reg)2959 void AssemblerX86Base<TraitsType>::sbb(Type Ty, const Address &address,
2960 GPRRegister reg) {
2961 arith_int<3>(Ty, address, reg);
2962 }
2963
2964 template <typename TraitsType>
sbb(Type Ty,const Address & address,const Immediate & imm)2965 void AssemblerX86Base<TraitsType>::sbb(Type Ty, const Address &address,
2966 const Immediate &imm) {
2967 arith_int<3>(Ty, address, imm);
2968 }
2969
cbw()2970 template <typename TraitsType> void AssemblerX86Base<TraitsType>::cbw() {
2971 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2972 emitOperandSizeOverride();
2973 emitUint8(0x98);
2974 }
2975
cwd()2976 template <typename TraitsType> void AssemblerX86Base<TraitsType>::cwd() {
2977 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2978 emitOperandSizeOverride();
2979 emitUint8(0x99);
2980 }
2981
cdq()2982 template <typename TraitsType> void AssemblerX86Base<TraitsType>::cdq() {
2983 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2984 emitUint8(0x99);
2985 }
2986
2987 template <typename TraitsType>
2988 template <typename T>
2989 typename std::enable_if<T::Is64Bit, void>::type
cqo()2990 AssemblerX86Base<TraitsType>::cqo() {
2991 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2992 emitRexB(RexTypeForceRexW, RexRegIrrelevant);
2993 emitUint8(0x99);
2994 }
2995
2996 template <typename TraitsType>
div(Type Ty,GPRRegister reg)2997 void AssemblerX86Base<TraitsType>::div(Type Ty, GPRRegister reg) {
2998 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
2999 if (Ty == IceType_i16)
3000 emitOperandSizeOverride();
3001 emitRexB(Ty, reg);
3002 if (isByteSizedArithType(Ty))
3003 emitUint8(0xF6);
3004 else
3005 emitUint8(0xF7);
3006 emitRegisterOperand(6, gprEncoding(reg));
3007 }
3008
3009 template <typename TraitsType>
div(Type Ty,const Address & addr)3010 void AssemblerX86Base<TraitsType>::div(Type Ty, const Address &addr) {
3011 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3012 if (Ty == IceType_i16)
3013 emitOperandSizeOverride();
3014 emitAddrSizeOverridePrefix();
3015 emitRex(Ty, addr, RexRegIrrelevant);
3016 if (isByteSizedArithType(Ty))
3017 emitUint8(0xF6);
3018 else
3019 emitUint8(0xF7);
3020 emitOperand(6, addr);
3021 }
3022
3023 template <typename TraitsType>
idiv(Type Ty,GPRRegister reg)3024 void AssemblerX86Base<TraitsType>::idiv(Type Ty, GPRRegister reg) {
3025 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3026 if (Ty == IceType_i16)
3027 emitOperandSizeOverride();
3028 emitRexB(Ty, reg);
3029 if (isByteSizedArithType(Ty))
3030 emitUint8(0xF6);
3031 else
3032 emitUint8(0xF7);
3033 emitRegisterOperand(7, gprEncoding(reg));
3034 }
3035
3036 template <typename TraitsType>
idiv(Type Ty,const Address & addr)3037 void AssemblerX86Base<TraitsType>::idiv(Type Ty, const Address &addr) {
3038 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3039 if (Ty == IceType_i16)
3040 emitOperandSizeOverride();
3041 emitAddrSizeOverridePrefix();
3042 emitRex(Ty, addr, RexRegIrrelevant);
3043 if (isByteSizedArithType(Ty))
3044 emitUint8(0xF6);
3045 else
3046 emitUint8(0xF7);
3047 emitOperand(7, addr);
3048 }
3049
3050 template <typename TraitsType>
imul(Type Ty,GPRRegister dst,GPRRegister src)3051 void AssemblerX86Base<TraitsType>::imul(Type Ty, GPRRegister dst,
3052 GPRRegister src) {
3053 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3054 assert(Ty == IceType_i16 || Ty == IceType_i32 ||
3055 (Traits::Is64Bit && Ty == IceType_i64));
3056 if (Ty == IceType_i16)
3057 emitOperandSizeOverride();
3058 emitRexRB(Ty, dst, src);
3059 emitUint8(0x0F);
3060 emitUint8(0xAF);
3061 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
3062 }
3063
3064 template <typename TraitsType>
imul(Type Ty,GPRRegister reg,const Address & address)3065 void AssemblerX86Base<TraitsType>::imul(Type Ty, GPRRegister reg,
3066 const Address &address) {
3067 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3068 assert(Ty == IceType_i16 || Ty == IceType_i32 ||
3069 (Traits::Is64Bit && Ty == IceType_i64));
3070 if (Ty == IceType_i16)
3071 emitOperandSizeOverride();
3072 emitAddrSizeOverridePrefix();
3073 emitRex(Ty, address, reg);
3074 emitUint8(0x0F);
3075 emitUint8(0xAF);
3076 emitOperand(gprEncoding(reg), address);
3077 }
3078
3079 template <typename TraitsType>
imul(Type Ty,GPRRegister reg,const Immediate & imm)3080 void AssemblerX86Base<TraitsType>::imul(Type Ty, GPRRegister reg,
3081 const Immediate &imm) {
3082 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3083 assert(Ty == IceType_i16 || Ty == IceType_i32 || Ty == IceType_i64);
3084 if (Ty == IceType_i16)
3085 emitOperandSizeOverride();
3086 emitRexRB(Ty, reg, reg);
3087 if (imm.is_int8()) {
3088 emitUint8(0x6B);
3089 emitRegisterOperand(gprEncoding(reg), gprEncoding(reg));
3090 emitUint8(imm.value() & 0xFF);
3091 } else {
3092 emitUint8(0x69);
3093 emitRegisterOperand(gprEncoding(reg), gprEncoding(reg));
3094 emitImmediate(Ty, imm);
3095 }
3096 }
3097
3098 template <typename TraitsType>
imul(Type Ty,GPRRegister reg)3099 void AssemblerX86Base<TraitsType>::imul(Type Ty, GPRRegister reg) {
3100 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3101 if (Ty == IceType_i16)
3102 emitOperandSizeOverride();
3103 emitRexB(Ty, reg);
3104 if (isByteSizedArithType(Ty))
3105 emitUint8(0xF6);
3106 else
3107 emitUint8(0xF7);
3108 emitRegisterOperand(5, gprEncoding(reg));
3109 }
3110
3111 template <typename TraitsType>
imul(Type Ty,const Address & address)3112 void AssemblerX86Base<TraitsType>::imul(Type Ty, const Address &address) {
3113 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3114 if (Ty == IceType_i16)
3115 emitOperandSizeOverride();
3116 emitAddrSizeOverridePrefix();
3117 emitRex(Ty, address, RexRegIrrelevant);
3118 if (isByteSizedArithType(Ty))
3119 emitUint8(0xF6);
3120 else
3121 emitUint8(0xF7);
3122 emitOperand(5, address);
3123 }
3124
3125 template <typename TraitsType>
imul(Type Ty,GPRRegister dst,GPRRegister src,const Immediate & imm)3126 void AssemblerX86Base<TraitsType>::imul(Type Ty, GPRRegister dst,
3127 GPRRegister src, const Immediate &imm) {
3128 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3129 assert(Ty == IceType_i16 || Ty == IceType_i32);
3130 if (Ty == IceType_i16)
3131 emitOperandSizeOverride();
3132 emitRexRB(Ty, dst, src);
3133 if (imm.is_int8()) {
3134 emitUint8(0x6B);
3135 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
3136 emitUint8(imm.value() & 0xFF);
3137 } else {
3138 emitUint8(0x69);
3139 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
3140 emitImmediate(Ty, imm);
3141 }
3142 }
3143
3144 template <typename TraitsType>
imul(Type Ty,GPRRegister dst,const Address & address,const Immediate & imm)3145 void AssemblerX86Base<TraitsType>::imul(Type Ty, GPRRegister dst,
3146 const Address &address,
3147 const Immediate &imm) {
3148 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3149 assert(Ty == IceType_i16 || Ty == IceType_i32);
3150 if (Ty == IceType_i16)
3151 emitOperandSizeOverride();
3152 emitAddrSizeOverridePrefix();
3153 emitRex(Ty, address, dst);
3154 if (imm.is_int8()) {
3155 emitUint8(0x6B);
3156 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
3157 emitOperand(gprEncoding(dst), address, OffsetFromNextInstruction);
3158 emitUint8(imm.value() & 0xFF);
3159 } else {
3160 emitUint8(0x69);
3161 const uint8_t OffsetFromNextInstruction = Ty == IceType_i16 ? 2 : 4;
3162 emitOperand(gprEncoding(dst), address, OffsetFromNextInstruction);
3163 emitImmediate(Ty, imm);
3164 }
3165 }
3166
3167 template <typename TraitsType>
mul(Type Ty,GPRRegister reg)3168 void AssemblerX86Base<TraitsType>::mul(Type Ty, GPRRegister reg) {
3169 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3170 if (Ty == IceType_i16)
3171 emitOperandSizeOverride();
3172 emitRexB(Ty, reg);
3173 if (isByteSizedArithType(Ty))
3174 emitUint8(0xF6);
3175 else
3176 emitUint8(0xF7);
3177 emitRegisterOperand(4, gprEncoding(reg));
3178 }
3179
3180 template <typename TraitsType>
mul(Type Ty,const Address & address)3181 void AssemblerX86Base<TraitsType>::mul(Type Ty, const Address &address) {
3182 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3183 if (Ty == IceType_i16)
3184 emitOperandSizeOverride();
3185 emitAddrSizeOverridePrefix();
3186 emitRex(Ty, address, RexRegIrrelevant);
3187 if (isByteSizedArithType(Ty))
3188 emitUint8(0xF6);
3189 else
3190 emitUint8(0xF7);
3191 emitOperand(4, address);
3192 }
3193
3194 template <typename TraitsType>
3195 template <typename, typename>
incl(GPRRegister reg)3196 void AssemblerX86Base<TraitsType>::incl(GPRRegister reg) {
3197 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3198 emitUint8(0x40 + reg);
3199 }
3200
3201 template <typename TraitsType>
incl(const Address & address)3202 void AssemblerX86Base<TraitsType>::incl(const Address &address) {
3203 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3204 emitAddrSizeOverridePrefix();
3205 emitRex(IceType_i32, address, RexRegIrrelevant);
3206 emitUint8(0xFF);
3207 emitOperand(0, address);
3208 }
3209
3210 template <typename TraitsType>
3211 template <typename, typename>
decl(GPRRegister reg)3212 void AssemblerX86Base<TraitsType>::decl(GPRRegister reg) {
3213 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3214 emitUint8(0x48 + reg);
3215 }
3216
3217 template <typename TraitsType>
decl(const Address & address)3218 void AssemblerX86Base<TraitsType>::decl(const Address &address) {
3219 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3220 emitAddrSizeOverridePrefix();
3221 emitRex(IceType_i32, address, RexRegIrrelevant);
3222 emitUint8(0xFF);
3223 emitOperand(1, address);
3224 }
3225
3226 template <typename TraitsType>
rol(Type Ty,GPRRegister reg,const Immediate & imm)3227 void AssemblerX86Base<TraitsType>::rol(Type Ty, GPRRegister reg,
3228 const Immediate &imm) {
3229 emitGenericShift(0, Ty, reg, imm);
3230 }
3231
3232 template <typename TraitsType>
rol(Type Ty,GPRRegister operand,GPRRegister shifter)3233 void AssemblerX86Base<TraitsType>::rol(Type Ty, GPRRegister operand,
3234 GPRRegister shifter) {
3235 emitGenericShift(0, Ty, Operand(operand), shifter);
3236 }
3237
3238 template <typename TraitsType>
rol(Type Ty,const Address & operand,GPRRegister shifter)3239 void AssemblerX86Base<TraitsType>::rol(Type Ty, const Address &operand,
3240 GPRRegister shifter) {
3241 emitGenericShift(0, Ty, operand, shifter);
3242 }
3243
3244 template <typename TraitsType>
shl(Type Ty,GPRRegister reg,const Immediate & imm)3245 void AssemblerX86Base<TraitsType>::shl(Type Ty, GPRRegister reg,
3246 const Immediate &imm) {
3247 emitGenericShift(4, Ty, reg, imm);
3248 }
3249
3250 template <typename TraitsType>
shl(Type Ty,GPRRegister operand,GPRRegister shifter)3251 void AssemblerX86Base<TraitsType>::shl(Type Ty, GPRRegister operand,
3252 GPRRegister shifter) {
3253 emitGenericShift(4, Ty, Operand(operand), shifter);
3254 }
3255
3256 template <typename TraitsType>
shl(Type Ty,const Address & operand,GPRRegister shifter)3257 void AssemblerX86Base<TraitsType>::shl(Type Ty, const Address &operand,
3258 GPRRegister shifter) {
3259 emitGenericShift(4, Ty, operand, shifter);
3260 }
3261
3262 template <typename TraitsType>
shr(Type Ty,GPRRegister reg,const Immediate & imm)3263 void AssemblerX86Base<TraitsType>::shr(Type Ty, GPRRegister reg,
3264 const Immediate &imm) {
3265 emitGenericShift(5, Ty, reg, imm);
3266 }
3267
3268 template <typename TraitsType>
shr(Type Ty,GPRRegister operand,GPRRegister shifter)3269 void AssemblerX86Base<TraitsType>::shr(Type Ty, GPRRegister operand,
3270 GPRRegister shifter) {
3271 emitGenericShift(5, Ty, Operand(operand), shifter);
3272 }
3273
3274 template <typename TraitsType>
shr(Type Ty,const Address & operand,GPRRegister shifter)3275 void AssemblerX86Base<TraitsType>::shr(Type Ty, const Address &operand,
3276 GPRRegister shifter) {
3277 emitGenericShift(5, Ty, operand, shifter);
3278 }
3279
3280 template <typename TraitsType>
sar(Type Ty,GPRRegister reg,const Immediate & imm)3281 void AssemblerX86Base<TraitsType>::sar(Type Ty, GPRRegister reg,
3282 const Immediate &imm) {
3283 emitGenericShift(7, Ty, reg, imm);
3284 }
3285
3286 template <typename TraitsType>
sar(Type Ty,GPRRegister operand,GPRRegister shifter)3287 void AssemblerX86Base<TraitsType>::sar(Type Ty, GPRRegister operand,
3288 GPRRegister shifter) {
3289 emitGenericShift(7, Ty, Operand(operand), shifter);
3290 }
3291
3292 template <typename TraitsType>
sar(Type Ty,const Address & address,GPRRegister shifter)3293 void AssemblerX86Base<TraitsType>::sar(Type Ty, const Address &address,
3294 GPRRegister shifter) {
3295 emitGenericShift(7, Ty, address, shifter);
3296 }
3297
3298 template <typename TraitsType>
shld(Type Ty,GPRRegister dst,GPRRegister src)3299 void AssemblerX86Base<TraitsType>::shld(Type Ty, GPRRegister dst,
3300 GPRRegister src) {
3301 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3302 assert(Ty == IceType_i16 || Ty == IceType_i32);
3303 if (Ty == IceType_i16)
3304 emitOperandSizeOverride();
3305 emitRexRB(Ty, src, dst);
3306 emitUint8(0x0F);
3307 emitUint8(0xA5);
3308 emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
3309 }
3310
3311 template <typename TraitsType>
shld(Type Ty,GPRRegister dst,GPRRegister src,const Immediate & imm)3312 void AssemblerX86Base<TraitsType>::shld(Type Ty, GPRRegister dst,
3313 GPRRegister src, const Immediate &imm) {
3314 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3315 assert(Ty == IceType_i16 || Ty == IceType_i32);
3316 assert(imm.is_int8());
3317 if (Ty == IceType_i16)
3318 emitOperandSizeOverride();
3319 emitRexRB(Ty, src, dst);
3320 emitUint8(0x0F);
3321 emitUint8(0xA4);
3322 emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
3323 emitUint8(imm.value() & 0xFF);
3324 }
3325
3326 template <typename TraitsType>
shld(Type Ty,const Address & operand,GPRRegister src)3327 void AssemblerX86Base<TraitsType>::shld(Type Ty, const Address &operand,
3328 GPRRegister src) {
3329 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3330 assert(Ty == IceType_i16 || Ty == IceType_i32);
3331 if (Ty == IceType_i16)
3332 emitOperandSizeOverride();
3333 emitAddrSizeOverridePrefix();
3334 emitRex(Ty, operand, src);
3335 emitUint8(0x0F);
3336 emitUint8(0xA5);
3337 emitOperand(gprEncoding(src), operand);
3338 }
3339
3340 template <typename TraitsType>
shrd(Type Ty,GPRRegister dst,GPRRegister src)3341 void AssemblerX86Base<TraitsType>::shrd(Type Ty, GPRRegister dst,
3342 GPRRegister src) {
3343 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3344 assert(Ty == IceType_i16 || Ty == IceType_i32);
3345 if (Ty == IceType_i16)
3346 emitOperandSizeOverride();
3347 emitRexRB(Ty, src, dst);
3348 emitUint8(0x0F);
3349 emitUint8(0xAD);
3350 emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
3351 }
3352
3353 template <typename TraitsType>
shrd(Type Ty,GPRRegister dst,GPRRegister src,const Immediate & imm)3354 void AssemblerX86Base<TraitsType>::shrd(Type Ty, GPRRegister dst,
3355 GPRRegister src, const Immediate &imm) {
3356 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3357 assert(Ty == IceType_i16 || Ty == IceType_i32);
3358 assert(imm.is_int8());
3359 if (Ty == IceType_i16)
3360 emitOperandSizeOverride();
3361 emitRexRB(Ty, src, dst);
3362 emitUint8(0x0F);
3363 emitUint8(0xAC);
3364 emitRegisterOperand(gprEncoding(src), gprEncoding(dst));
3365 emitUint8(imm.value() & 0xFF);
3366 }
3367
3368 template <typename TraitsType>
shrd(Type Ty,const Address & dst,GPRRegister src)3369 void AssemblerX86Base<TraitsType>::shrd(Type Ty, const Address &dst,
3370 GPRRegister src) {
3371 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3372 assert(Ty == IceType_i16 || Ty == IceType_i32);
3373 if (Ty == IceType_i16)
3374 emitOperandSizeOverride();
3375 emitAddrSizeOverridePrefix();
3376 emitRex(Ty, dst, src);
3377 emitUint8(0x0F);
3378 emitUint8(0xAD);
3379 emitOperand(gprEncoding(src), dst);
3380 }
3381
3382 template <typename TraitsType>
neg(Type Ty,GPRRegister reg)3383 void AssemblerX86Base<TraitsType>::neg(Type Ty, GPRRegister reg) {
3384 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3385 if (Ty == IceType_i16)
3386 emitOperandSizeOverride();
3387 emitRexB(Ty, reg);
3388 if (isByteSizedArithType(Ty))
3389 emitUint8(0xF6);
3390 else
3391 emitUint8(0xF7);
3392 emitRegisterOperand(3, gprEncoding(reg));
3393 }
3394
3395 template <typename TraitsType>
neg(Type Ty,const Address & addr)3396 void AssemblerX86Base<TraitsType>::neg(Type Ty, const Address &addr) {
3397 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3398 if (Ty == IceType_i16)
3399 emitOperandSizeOverride();
3400 emitAddrSizeOverridePrefix();
3401 emitRex(Ty, addr, RexRegIrrelevant);
3402 if (isByteSizedArithType(Ty))
3403 emitUint8(0xF6);
3404 else
3405 emitUint8(0xF7);
3406 emitOperand(3, addr);
3407 }
3408
3409 template <typename TraitsType>
notl(GPRRegister reg)3410 void AssemblerX86Base<TraitsType>::notl(GPRRegister reg) {
3411 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3412 emitRexB(IceType_i32, reg);
3413 emitUint8(0xF7);
3414 emitUint8(0xD0 | gprEncoding(reg));
3415 }
3416
3417 template <typename TraitsType>
bswap(Type Ty,GPRRegister reg)3418 void AssemblerX86Base<TraitsType>::bswap(Type Ty, GPRRegister reg) {
3419 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3420 assert(Ty == IceType_i32 || (Traits::Is64Bit && Ty == IceType_i64));
3421 emitRexB(Ty, reg);
3422 emitUint8(0x0F);
3423 emitUint8(0xC8 | gprEncoding(reg));
3424 }
3425
3426 template <typename TraitsType>
bsf(Type Ty,GPRRegister dst,GPRRegister src)3427 void AssemblerX86Base<TraitsType>::bsf(Type Ty, GPRRegister dst,
3428 GPRRegister src) {
3429 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3430 assert(Ty == IceType_i16 || Ty == IceType_i32 ||
3431 (Traits::Is64Bit && Ty == IceType_i64));
3432 if (Ty == IceType_i16)
3433 emitOperandSizeOverride();
3434 emitRexRB(Ty, dst, src);
3435 emitUint8(0x0F);
3436 emitUint8(0xBC);
3437 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
3438 }
3439
3440 template <typename TraitsType>
bsf(Type Ty,GPRRegister dst,const Address & src)3441 void AssemblerX86Base<TraitsType>::bsf(Type Ty, GPRRegister dst,
3442 const Address &src) {
3443 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3444 assert(Ty == IceType_i16 || Ty == IceType_i32 ||
3445 (Traits::Is64Bit && Ty == IceType_i64));
3446 if (Ty == IceType_i16)
3447 emitOperandSizeOverride();
3448 emitAddrSizeOverridePrefix();
3449 emitRex(Ty, src, dst);
3450 emitUint8(0x0F);
3451 emitUint8(0xBC);
3452 emitOperand(gprEncoding(dst), src);
3453 }
3454
3455 template <typename TraitsType>
bsr(Type Ty,GPRRegister dst,GPRRegister src)3456 void AssemblerX86Base<TraitsType>::bsr(Type Ty, GPRRegister dst,
3457 GPRRegister src) {
3458 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3459 assert(Ty == IceType_i16 || Ty == IceType_i32 ||
3460 (Traits::Is64Bit && Ty == IceType_i64));
3461 if (Ty == IceType_i16)
3462 emitOperandSizeOverride();
3463 emitRexRB(Ty, dst, src);
3464 emitUint8(0x0F);
3465 emitUint8(0xBD);
3466 emitRegisterOperand(gprEncoding(dst), gprEncoding(src));
3467 }
3468
3469 template <typename TraitsType>
bsr(Type Ty,GPRRegister dst,const Address & src)3470 void AssemblerX86Base<TraitsType>::bsr(Type Ty, GPRRegister dst,
3471 const Address &src) {
3472 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3473 assert(Ty == IceType_i16 || Ty == IceType_i32 ||
3474 (Traits::Is64Bit && Ty == IceType_i64));
3475 if (Ty == IceType_i16)
3476 emitOperandSizeOverride();
3477 emitAddrSizeOverridePrefix();
3478 emitRex(Ty, src, dst);
3479 emitUint8(0x0F);
3480 emitUint8(0xBD);
3481 emitOperand(gprEncoding(dst), src);
3482 }
3483
3484 template <typename TraitsType>
bt(GPRRegister base,GPRRegister offset)3485 void AssemblerX86Base<TraitsType>::bt(GPRRegister base, GPRRegister offset) {
3486 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3487 emitRexRB(IceType_i32, offset, base);
3488 emitUint8(0x0F);
3489 emitUint8(0xA3);
3490 emitRegisterOperand(gprEncoding(offset), gprEncoding(base));
3491 }
3492
ret()3493 template <typename TraitsType> void AssemblerX86Base<TraitsType>::ret() {
3494 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3495 emitUint8(0xC3);
3496 }
3497
3498 template <typename TraitsType>
ret(const Immediate & imm)3499 void AssemblerX86Base<TraitsType>::ret(const Immediate &imm) {
3500 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3501 emitUint8(0xC2);
3502 assert(imm.is_uint16());
3503 emitUint8(imm.value() & 0xFF);
3504 emitUint8((imm.value() >> 8) & 0xFF);
3505 }
3506
3507 template <typename TraitsType>
nop(int size)3508 void AssemblerX86Base<TraitsType>::nop(int size) {
3509 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3510 // There are nops up to size 15, but for now just provide up to size 8.
3511 assert(0 < size && size <= MAX_NOP_SIZE);
3512 switch (size) {
3513 case 1:
3514 emitUint8(0x90);
3515 break;
3516 case 2:
3517 emitUint8(0x66);
3518 emitUint8(0x90);
3519 break;
3520 case 3:
3521 emitUint8(0x0F);
3522 emitUint8(0x1F);
3523 emitUint8(0x00);
3524 break;
3525 case 4:
3526 emitUint8(0x0F);
3527 emitUint8(0x1F);
3528 emitUint8(0x40);
3529 emitUint8(0x00);
3530 break;
3531 case 5:
3532 emitUint8(0x0F);
3533 emitUint8(0x1F);
3534 emitUint8(0x44);
3535 emitUint8(0x00);
3536 emitUint8(0x00);
3537 break;
3538 case 6:
3539 emitUint8(0x66);
3540 emitUint8(0x0F);
3541 emitUint8(0x1F);
3542 emitUint8(0x44);
3543 emitUint8(0x00);
3544 emitUint8(0x00);
3545 break;
3546 case 7:
3547 emitUint8(0x0F);
3548 emitUint8(0x1F);
3549 emitUint8(0x80);
3550 emitUint8(0x00);
3551 emitUint8(0x00);
3552 emitUint8(0x00);
3553 emitUint8(0x00);
3554 break;
3555 case 8:
3556 emitUint8(0x0F);
3557 emitUint8(0x1F);
3558 emitUint8(0x84);
3559 emitUint8(0x00);
3560 emitUint8(0x00);
3561 emitUint8(0x00);
3562 emitUint8(0x00);
3563 emitUint8(0x00);
3564 break;
3565 default:
3566 llvm_unreachable("Unimplemented");
3567 }
3568 }
3569
int3()3570 template <typename TraitsType> void AssemblerX86Base<TraitsType>::int3() {
3571 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3572 emitUint8(0xCC);
3573 }
3574
hlt()3575 template <typename TraitsType> void AssemblerX86Base<TraitsType>::hlt() {
3576 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3577 emitUint8(0xF4);
3578 }
3579
ud2()3580 template <typename TraitsType> void AssemblerX86Base<TraitsType>::ud2() {
3581 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3582 emitUint8(0x0F);
3583 emitUint8(0x0B);
3584 }
3585
3586 template <typename TraitsType>
j(BrCond condition,Label * label,bool near)3587 void AssemblerX86Base<TraitsType>::j(BrCond condition, Label *label,
3588 bool near) {
3589 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3590 if (label->isBound()) {
3591 static const int kShortSize = 2;
3592 static const int kLongSize = 6;
3593 intptr_t offset = label->getPosition() - Buffer.size();
3594 assert(offset <= 0);
3595 if (Utils::IsInt(8, offset - kShortSize)) {
3596 // TODO(stichnot): Here and in jmp(), we may need to be more
3597 // conservative about the backward branch distance if the branch
3598 // instruction is within a bundle_lock sequence, because the
3599 // distance may increase when padding is added. This isn't an issue for
3600 // branches outside a bundle_lock, because if padding is added, the retry
3601 // may change it to a long backward branch without affecting any of the
3602 // bookkeeping.
3603 emitUint8(0x70 + condition);
3604 emitUint8((offset - kShortSize) & 0xFF);
3605 } else {
3606 emitUint8(0x0F);
3607 emitUint8(0x80 + condition);
3608 emitInt32(offset - kLongSize);
3609 }
3610 } else if (near) {
3611 emitUint8(0x70 + condition);
3612 emitNearLabelLink(label);
3613 } else {
3614 emitUint8(0x0F);
3615 emitUint8(0x80 + condition);
3616 emitLabelLink(label);
3617 }
3618 }
3619
3620 template <typename TraitsType>
j(BrCond condition,const ConstantRelocatable * label)3621 void AssemblerX86Base<TraitsType>::j(BrCond condition,
3622 const ConstantRelocatable *label) {
3623 llvm::report_fatal_error("Untested - please verify and then reenable.");
3624 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3625 emitUint8(0x0F);
3626 emitUint8(0x80 + condition);
3627 auto *Fixup = this->createFixup(Traits::FK_PcRel, label);
3628 Fixup->set_addend(-4);
3629 emitFixup(Fixup);
3630 emitInt32(0);
3631 }
3632
3633 template <typename TraitsType>
jmp(GPRRegister reg)3634 void AssemblerX86Base<TraitsType>::jmp(GPRRegister reg) {
3635 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3636 emitRexB(RexTypeIrrelevant, reg);
3637 emitUint8(0xFF);
3638 emitRegisterOperand(4, gprEncoding(reg));
3639 }
3640
3641 template <typename TraitsType>
jmp(Label * label,bool near)3642 void AssemblerX86Base<TraitsType>::jmp(Label *label, bool near) {
3643 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3644 if (label->isBound()) {
3645 static const int kShortSize = 2;
3646 static const int kLongSize = 5;
3647 intptr_t offset = label->getPosition() - Buffer.size();
3648 assert(offset <= 0);
3649 if (Utils::IsInt(8, offset - kShortSize)) {
3650 emitUint8(0xEB);
3651 emitUint8((offset - kShortSize) & 0xFF);
3652 } else {
3653 emitUint8(0xE9);
3654 emitInt32(offset - kLongSize);
3655 }
3656 } else if (near) {
3657 emitUint8(0xEB);
3658 emitNearLabelLink(label);
3659 } else {
3660 emitUint8(0xE9);
3661 emitLabelLink(label);
3662 }
3663 }
3664
3665 template <typename TraitsType>
jmp(const ConstantRelocatable * label)3666 void AssemblerX86Base<TraitsType>::jmp(const ConstantRelocatable *label) {
3667 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3668 emitUint8(0xE9);
3669 auto *Fixup = this->createFixup(Traits::FK_PcRel, label);
3670 Fixup->set_addend(-4);
3671 emitFixup(Fixup);
3672 emitInt32(0);
3673 }
3674
3675 template <typename TraitsType>
jmp(const Immediate & abs_address)3676 void AssemblerX86Base<TraitsType>::jmp(const Immediate &abs_address) {
3677 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3678 emitUint8(0xE9);
3679 AssemblerFixup *Fixup =
3680 createFixup(Traits::FK_PcRel, AssemblerFixup::NullSymbol);
3681 Fixup->set_addend(abs_address.value() - 4);
3682 emitFixup(Fixup);
3683 emitInt32(0);
3684 }
3685
mfence()3686 template <typename TraitsType> void AssemblerX86Base<TraitsType>::mfence() {
3687 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3688 emitUint8(0x0F);
3689 emitUint8(0xAE);
3690 emitUint8(0xF0);
3691 }
3692
lock()3693 template <typename TraitsType> void AssemblerX86Base<TraitsType>::lock() {
3694 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3695 emitUint8(0xF0);
3696 }
3697
3698 template <typename TraitsType>
cmpxchg(Type Ty,const Address & address,GPRRegister reg,bool Locked)3699 void AssemblerX86Base<TraitsType>::cmpxchg(Type Ty, const Address &address,
3700 GPRRegister reg, bool Locked) {
3701 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3702 if (Ty == IceType_i16)
3703 emitOperandSizeOverride();
3704 if (Locked)
3705 emitUint8(0xF0);
3706 emitAddrSizeOverridePrefix();
3707 emitRex(Ty, address, reg);
3708 emitUint8(0x0F);
3709 if (isByteSizedArithType(Ty))
3710 emitUint8(0xB0);
3711 else
3712 emitUint8(0xB1);
3713 emitOperand(gprEncoding(reg), address);
3714 }
3715
3716 template <typename TraitsType>
cmpxchg8b(const Address & address,bool Locked)3717 void AssemblerX86Base<TraitsType>::cmpxchg8b(const Address &address,
3718 bool Locked) {
3719 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3720 if (Locked)
3721 emitUint8(0xF0);
3722 emitAddrSizeOverridePrefix();
3723 emitRex(IceType_i32, address, RexRegIrrelevant);
3724 emitUint8(0x0F);
3725 emitUint8(0xC7);
3726 emitOperand(1, address);
3727 }
3728
3729 template <typename TraitsType>
xadd(Type Ty,const Address & addr,GPRRegister reg,bool Locked)3730 void AssemblerX86Base<TraitsType>::xadd(Type Ty, const Address &addr,
3731 GPRRegister reg, bool Locked) {
3732 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3733 if (Ty == IceType_i16)
3734 emitOperandSizeOverride();
3735 if (Locked)
3736 emitUint8(0xF0);
3737 emitAddrSizeOverridePrefix();
3738 emitRex(Ty, addr, reg);
3739 emitUint8(0x0F);
3740 if (isByteSizedArithType(Ty))
3741 emitUint8(0xC0);
3742 else
3743 emitUint8(0xC1);
3744 emitOperand(gprEncoding(reg), addr);
3745 }
3746
3747 template <typename TraitsType>
xchg(Type Ty,GPRRegister reg0,GPRRegister reg1)3748 void AssemblerX86Base<TraitsType>::xchg(Type Ty, GPRRegister reg0,
3749 GPRRegister reg1) {
3750 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3751 if (Ty == IceType_i16)
3752 emitOperandSizeOverride();
3753 // Use short form if either register is EAX.
3754 if (reg0 == Traits::Encoded_Reg_Accumulator) {
3755 emitRexB(Ty, reg1);
3756 emitUint8(0x90 + gprEncoding(reg1));
3757 } else if (reg1 == Traits::Encoded_Reg_Accumulator) {
3758 emitRexB(Ty, reg0);
3759 emitUint8(0x90 + gprEncoding(reg0));
3760 } else {
3761 emitRexRB(Ty, reg0, reg1);
3762 if (isByteSizedArithType(Ty))
3763 emitUint8(0x86);
3764 else
3765 emitUint8(0x87);
3766 emitRegisterOperand(gprEncoding(reg0), gprEncoding(reg1));
3767 }
3768 }
3769
3770 template <typename TraitsType>
xchg(Type Ty,const Address & addr,GPRRegister reg)3771 void AssemblerX86Base<TraitsType>::xchg(Type Ty, const Address &addr,
3772 GPRRegister reg) {
3773 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3774 if (Ty == IceType_i16)
3775 emitOperandSizeOverride();
3776 emitAddrSizeOverridePrefix();
3777 emitRex(Ty, addr, reg);
3778 if (isByteSizedArithType(Ty))
3779 emitUint8(0x86);
3780 else
3781 emitUint8(0x87);
3782 emitOperand(gprEncoding(reg), addr);
3783 }
3784
iaca_start()3785 template <typename TraitsType> void AssemblerX86Base<TraitsType>::iaca_start() {
3786 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3787 emitUint8(0x0F);
3788 emitUint8(0x0B);
3789
3790 // mov $111, ebx
3791 constexpr GPRRegister dst = Traits::GPRRegister::Encoded_Reg_ebx;
3792 constexpr Type Ty = IceType_i32;
3793 emitRexB(Ty, dst);
3794 emitUint8(0xB8 + gprEncoding(dst));
3795 emitImmediate(Ty, Immediate(111));
3796
3797 emitUint8(0x64);
3798 emitUint8(0x67);
3799 emitUint8(0x90);
3800 }
3801
iaca_end()3802 template <typename TraitsType> void AssemblerX86Base<TraitsType>::iaca_end() {
3803 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3804
3805 // mov $222, ebx
3806 constexpr GPRRegister dst = Traits::GPRRegister::Encoded_Reg_ebx;
3807 constexpr Type Ty = IceType_i32;
3808 emitRexB(Ty, dst);
3809 emitUint8(0xB8 + gprEncoding(dst));
3810 emitImmediate(Ty, Immediate(222));
3811
3812 emitUint8(0x64);
3813 emitUint8(0x67);
3814 emitUint8(0x90);
3815
3816 emitUint8(0x0F);
3817 emitUint8(0x0B);
3818 }
3819
3820 template <typename TraitsType>
emitSegmentOverride(uint8_t prefix)3821 void AssemblerX86Base<TraitsType>::emitSegmentOverride(uint8_t prefix) {
3822 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3823 emitUint8(prefix);
3824 }
3825
3826 template <typename TraitsType>
align(intptr_t alignment,intptr_t offset)3827 void AssemblerX86Base<TraitsType>::align(intptr_t alignment, intptr_t offset) {
3828 assert(llvm::isPowerOf2_32(alignment));
3829 intptr_t pos = offset + Buffer.getPosition();
3830 intptr_t mod = pos & (alignment - 1);
3831 if (mod == 0) {
3832 return;
3833 }
3834 intptr_t bytes_needed = alignment - mod;
3835 while (bytes_needed > MAX_NOP_SIZE) {
3836 nop(MAX_NOP_SIZE);
3837 bytes_needed -= MAX_NOP_SIZE;
3838 }
3839 if (bytes_needed) {
3840 nop(bytes_needed);
3841 }
3842 assert(((offset + Buffer.getPosition()) & (alignment - 1)) == 0);
3843 }
3844
3845 template <typename TraitsType>
bind(Label * L)3846 void AssemblerX86Base<TraitsType>::bind(Label *L) {
3847 const intptr_t Bound = Buffer.size();
3848 assert(!L->isBound()); // Labels can only be bound once.
3849 while (L->isLinked()) {
3850 const intptr_t Position = L->getLinkPosition();
3851 const intptr_t Next = Buffer.load<int32_t>(Position);
3852 const intptr_t Offset = Bound - (Position + 4);
3853 Buffer.store<int32_t>(Position, Offset);
3854 L->Position = Next;
3855 }
3856 while (L->hasNear()) {
3857 intptr_t Position = L->getNearPosition();
3858 const intptr_t Offset = Bound - (Position + 1);
3859 assert(Utils::IsInt(8, Offset));
3860 Buffer.store<int8_t>(Position, Offset);
3861 }
3862 L->bindTo(Bound);
3863 }
3864
3865 template <typename TraitsType>
emitOperand(int rm,const Operand & operand,RelocOffsetT Addend)3866 void AssemblerX86Base<TraitsType>::emitOperand(int rm, const Operand &operand,
3867 RelocOffsetT Addend) {
3868 assert(rm >= 0 && rm < 8);
3869 const intptr_t length = operand.length_;
3870 assert(length > 0);
3871 intptr_t displacement_start = 1;
3872 // Emit the ModRM byte updated with the given RM value.
3873 assert((operand.encoding_[0] & 0x38) == 0);
3874 emitUint8(operand.encoding_[0] + (rm << 3));
3875 // Whenever the addressing mode is not register indirect, using esp == 0x4
3876 // as the register operation indicates an SIB byte follows.
3877 if (((operand.encoding_[0] & 0xc0) != 0xc0) &&
3878 ((operand.encoding_[0] & 0x07) == 0x04)) {
3879 emitUint8(operand.encoding_[1]);
3880 displacement_start = 2;
3881 }
3882
3883 AssemblerFixup *Fixup = operand.fixup();
3884 if (Fixup == nullptr) {
3885 for (intptr_t i = displacement_start; i < length; i++) {
3886 emitUint8(operand.encoding_[i]);
3887 }
3888 return;
3889 }
3890
3891 // Emit the fixup, and a dummy 4-byte immediate. Note that the Disp32 in
3892 // operand.encoding_[i, i+1, i+2, i+3] is part of the constant relocatable
3893 // used to create the fixup, so there's no need to add it to the addend.
3894 assert(length - displacement_start == 4);
3895 if (fixupIsPCRel(Fixup->kind())) {
3896 Fixup->set_addend(Fixup->get_addend() - Addend);
3897 } else {
3898 Fixup->set_addend(Fixup->get_addend());
3899 }
3900 emitFixup(Fixup);
3901 emitInt32(0);
3902 }
3903
3904 template <typename TraitsType>
emitImmediate(Type Ty,const Immediate & imm)3905 void AssemblerX86Base<TraitsType>::emitImmediate(Type Ty,
3906 const Immediate &imm) {
3907 auto *const Fixup = imm.fixup();
3908 if (Ty == IceType_i16) {
3909 assert(Fixup == nullptr);
3910 emitInt16(imm.value());
3911 return;
3912 }
3913
3914 if (Fixup == nullptr) {
3915 emitInt32(imm.value());
3916 return;
3917 }
3918
3919 Fixup->set_addend(Fixup->get_addend() + imm.value());
3920 emitFixup(Fixup);
3921 emitInt32(0);
3922 }
3923
3924 template <typename TraitsType>
emitComplexI8(int rm,const Operand & operand,const Immediate & immediate)3925 void AssemblerX86Base<TraitsType>::emitComplexI8(int rm, const Operand &operand,
3926 const Immediate &immediate) {
3927 assert(rm >= 0 && rm < 8);
3928 assert(immediate.is_int8());
3929 if (operand.IsRegister(Traits::Encoded_Reg_Accumulator)) {
3930 // Use short form if the destination is al.
3931 emitUint8(0x04 + (rm << 3));
3932 emitUint8(immediate.value() & 0xFF);
3933 } else {
3934 // Use sign-extended 8-bit immediate.
3935 emitUint8(0x80);
3936 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
3937 emitOperand(rm, operand, OffsetFromNextInstruction);
3938 emitUint8(immediate.value() & 0xFF);
3939 }
3940 }
3941
3942 template <typename TraitsType>
emitComplex(Type Ty,int rm,const Operand & operand,const Immediate & immediate)3943 void AssemblerX86Base<TraitsType>::emitComplex(Type Ty, int rm,
3944 const Operand &operand,
3945 const Immediate &immediate) {
3946 assert(rm >= 0 && rm < 8);
3947 if (immediate.is_int8()) {
3948 // Use sign-extended 8-bit immediate.
3949 emitUint8(0x83);
3950 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
3951 emitOperand(rm, operand, OffsetFromNextInstruction);
3952 emitUint8(immediate.value() & 0xFF);
3953 } else if (operand.IsRegister(Traits::Encoded_Reg_Accumulator)) {
3954 // Use short form if the destination is eax.
3955 emitUint8(0x05 + (rm << 3));
3956 emitImmediate(Ty, immediate);
3957 } else {
3958 emitUint8(0x81);
3959 const uint8_t OffsetFromNextInstruction = Ty == IceType_i16 ? 2 : 4;
3960 emitOperand(rm, operand, OffsetFromNextInstruction);
3961 emitImmediate(Ty, immediate);
3962 }
3963 }
3964
3965 template <typename TraitsType>
emitLabel(Label * label,intptr_t instruction_size)3966 void AssemblerX86Base<TraitsType>::emitLabel(Label *label,
3967 intptr_t instruction_size) {
3968 if (label->isBound()) {
3969 intptr_t offset = label->getPosition() - Buffer.size();
3970 assert(offset <= 0);
3971 emitInt32(offset - instruction_size);
3972 } else {
3973 emitLabelLink(label);
3974 }
3975 }
3976
3977 template <typename TraitsType>
emitLabelLink(Label * Label)3978 void AssemblerX86Base<TraitsType>::emitLabelLink(Label *Label) {
3979 assert(!Label->isBound());
3980 intptr_t Position = Buffer.size();
3981 emitInt32(Label->Position);
3982 Label->linkTo(*this, Position);
3983 }
3984
3985 template <typename TraitsType>
emitNearLabelLink(Label * Label)3986 void AssemblerX86Base<TraitsType>::emitNearLabelLink(Label *Label) {
3987 assert(!Label->isBound());
3988 intptr_t Position = Buffer.size();
3989 emitUint8(0);
3990 Label->nearLinkTo(*this, Position);
3991 }
3992
3993 template <typename TraitsType>
emitGenericShift(int rm,Type Ty,GPRRegister reg,const Immediate & imm)3994 void AssemblerX86Base<TraitsType>::emitGenericShift(int rm, Type Ty,
3995 GPRRegister reg,
3996 const Immediate &imm) {
3997 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
3998 // We don't assert that imm fits into 8 bits; instead, it gets masked below.
3999 // Note that we don't mask it further (e.g. to 5 bits) because we want the
4000 // same processor behavior regardless of whether it's an immediate (masked to
4001 // 8 bits) or in register cl (essentially ecx masked to 8 bits).
4002 if (Ty == IceType_i16)
4003 emitOperandSizeOverride();
4004 emitRexB(Ty, reg);
4005 if (imm.value() == 1) {
4006 emitUint8(isByteSizedArithType(Ty) ? 0xD0 : 0xD1);
4007 emitOperand(rm, Operand(reg));
4008 } else {
4009 emitUint8(isByteSizedArithType(Ty) ? 0xC0 : 0xC1);
4010 static constexpr RelocOffsetT OffsetFromNextInstruction = 1;
4011 emitOperand(rm, Operand(reg), OffsetFromNextInstruction);
4012 emitUint8(imm.value() & 0xFF);
4013 }
4014 }
4015
4016 template <typename TraitsType>
emitGenericShift(int rm,Type Ty,const Operand & operand,GPRRegister shifter)4017 void AssemblerX86Base<TraitsType>::emitGenericShift(int rm, Type Ty,
4018 const Operand &operand,
4019 GPRRegister shifter) {
4020 AssemblerBuffer::EnsureCapacity ensured(&Buffer);
4021 assert(shifter == Traits::Encoded_Reg_Counter);
4022 (void)shifter;
4023 if (Ty == IceType_i16)
4024 emitOperandSizeOverride();
4025 emitRexB(Ty, operand.rm());
4026 emitUint8(isByteSizedArithType(Ty) ? 0xD2 : 0xD3);
4027 emitOperand(rm, operand);
4028 }
4029
4030 } // end of namespace X86NAMESPACE
4031 } // end of namespace Ice
4032