1 // Copyright 2012 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/x64/assembler-x64.h"
6
7 #include <cstring>
8
9 #if V8_TARGET_ARCH_X64
10
11 #if V8_LIBC_MSVCRT
12 #include <intrin.h> // _xgetbv()
13 #endif
14 #if V8_OS_MACOSX
15 #include <sys/sysctl.h>
16 #endif
17
18 #include "src/assembler-inl.h"
19 #include "src/base/bits.h"
20 #include "src/base/cpu.h"
21 #include "src/macro-assembler.h"
22 #include "src/v8.h"
23
24 namespace v8 {
25 namespace internal {
26
27 // -----------------------------------------------------------------------------
28 // Implementation of CpuFeatures
29
30 namespace {
31
32 #if !V8_LIBC_MSVCRT
33
_xgetbv(unsigned int xcr)34 V8_INLINE uint64_t _xgetbv(unsigned int xcr) {
35 unsigned eax, edx;
36 // Check xgetbv; this uses a .byte sequence instead of the instruction
37 // directly because older assemblers do not include support for xgetbv and
38 // there is no easy way to conditionally compile based on the assembler
39 // used.
40 __asm__ volatile(".byte 0x0f, 0x01, 0xd0" : "=a"(eax), "=d"(edx) : "c"(xcr));
41 return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32);
42 }
43
44 #define _XCR_XFEATURE_ENABLED_MASK 0
45
46 #endif // !V8_LIBC_MSVCRT
47
48
OSHasAVXSupport()49 bool OSHasAVXSupport() {
50 #if V8_OS_MACOSX
51 // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being
52 // caused by ISRs, so we detect that here and disable AVX in that case.
53 char buffer[128];
54 size_t buffer_size = arraysize(buffer);
55 int ctl_name[] = {CTL_KERN, KERN_OSRELEASE};
56 if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) {
57 V8_Fatal(__FILE__, __LINE__, "V8 failed to get kernel version");
58 }
59 // The buffer now contains a string of the form XX.YY.ZZ, where
60 // XX is the major kernel version component.
61 char* period_pos = strchr(buffer, '.');
62 DCHECK_NOT_NULL(period_pos);
63 *period_pos = '\0';
64 long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT
65 if (kernel_version_major <= 13) return false;
66 #endif // V8_OS_MACOSX
67 // Check whether OS claims to support AVX.
68 uint64_t feature_mask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
69 return (feature_mask & 0x6) == 0x6;
70 }
71
72 } // namespace
73
74
ProbeImpl(bool cross_compile)75 void CpuFeatures::ProbeImpl(bool cross_compile) {
76 base::CPU cpu;
77 CHECK(cpu.has_sse2()); // SSE2 support is mandatory.
78 CHECK(cpu.has_cmov()); // CMOV support is mandatory.
79
80 // Only use statically determined features for cross compile (snapshot).
81 if (cross_compile) return;
82
83 if (cpu.has_sse41() && FLAG_enable_sse4_1) supported_ |= 1u << SSE4_1;
84 if (cpu.has_ssse3() && FLAG_enable_ssse3) supported_ |= 1u << SSSE3;
85 if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3;
86 // SAHF is not generally available in long mode.
87 if (cpu.has_sahf() && FLAG_enable_sahf) supported_ |= 1u << SAHF;
88 if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() &&
89 OSHasAVXSupport()) {
90 supported_ |= 1u << AVX;
91 }
92 if (cpu.has_fma3() && FLAG_enable_fma3 && cpu.has_osxsave() &&
93 OSHasAVXSupport()) {
94 supported_ |= 1u << FMA3;
95 }
96 if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1;
97 if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2;
98 if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT;
99 if (cpu.has_popcnt() && FLAG_enable_popcnt) supported_ |= 1u << POPCNT;
100 if (strcmp(FLAG_mcpu, "auto") == 0) {
101 if (cpu.is_atom()) supported_ |= 1u << ATOM;
102 } else if (strcmp(FLAG_mcpu, "atom") == 0) {
103 supported_ |= 1u << ATOM;
104 }
105 }
106
107
PrintTarget()108 void CpuFeatures::PrintTarget() { }
PrintFeatures()109 void CpuFeatures::PrintFeatures() {
110 printf(
111 "SSE3=%d SSSE3=%d SSE4_1=%d SAHF=%d AVX=%d FMA3=%d BMI1=%d BMI2=%d "
112 "LZCNT=%d "
113 "POPCNT=%d ATOM=%d\n",
114 CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSSE3),
115 CpuFeatures::IsSupported(SSE4_1), CpuFeatures::IsSupported(SAHF),
116 CpuFeatures::IsSupported(AVX), CpuFeatures::IsSupported(FMA3),
117 CpuFeatures::IsSupported(BMI1), CpuFeatures::IsSupported(BMI2),
118 CpuFeatures::IsSupported(LZCNT), CpuFeatures::IsSupported(POPCNT),
119 CpuFeatures::IsSupported(ATOM));
120 }
121
122 // -----------------------------------------------------------------------------
123 // Implementation of RelocInfo
124
wasm_memory_reference()125 Address RelocInfo::wasm_memory_reference() {
126 DCHECK(IsWasmMemoryReference(rmode_));
127 return Memory::Address_at(pc_);
128 }
129
wasm_global_reference()130 Address RelocInfo::wasm_global_reference() {
131 DCHECK(IsWasmGlobalReference(rmode_));
132 return Memory::Address_at(pc_);
133 }
134
wasm_memory_size_reference()135 uint32_t RelocInfo::wasm_memory_size_reference() {
136 DCHECK(IsWasmMemorySizeReference(rmode_));
137 return Memory::uint32_at(pc_);
138 }
139
wasm_function_table_size_reference()140 uint32_t RelocInfo::wasm_function_table_size_reference() {
141 DCHECK(IsWasmFunctionTableSizeReference(rmode_));
142 return Memory::uint32_at(pc_);
143 }
144
unchecked_update_wasm_memory_reference(Address address,ICacheFlushMode flush_mode)145 void RelocInfo::unchecked_update_wasm_memory_reference(
146 Address address, ICacheFlushMode flush_mode) {
147 Memory::Address_at(pc_) = address;
148 }
149
unchecked_update_wasm_size(uint32_t size,ICacheFlushMode flush_mode)150 void RelocInfo::unchecked_update_wasm_size(uint32_t size,
151 ICacheFlushMode flush_mode) {
152 Memory::uint32_at(pc_) = size;
153 }
154
155 // -----------------------------------------------------------------------------
156 // Implementation of Operand
157
Operand(Register base,int32_t disp)158 Operand::Operand(Register base, int32_t disp) : rex_(0) {
159 len_ = 1;
160 if (base.is(rsp) || base.is(r12)) {
161 // SIB byte is needed to encode (rsp + offset) or (r12 + offset).
162 set_sib(times_1, rsp, base);
163 }
164
165 if (disp == 0 && !base.is(rbp) && !base.is(r13)) {
166 set_modrm(0, base);
167 } else if (is_int8(disp)) {
168 set_modrm(1, base);
169 set_disp8(disp);
170 } else {
171 set_modrm(2, base);
172 set_disp32(disp);
173 }
174 }
175
176
Operand(Register base,Register index,ScaleFactor scale,int32_t disp)177 Operand::Operand(Register base,
178 Register index,
179 ScaleFactor scale,
180 int32_t disp) : rex_(0) {
181 DCHECK(!index.is(rsp));
182 len_ = 1;
183 set_sib(scale, index, base);
184 if (disp == 0 && !base.is(rbp) && !base.is(r13)) {
185 // This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits
186 // possibly set by set_sib.
187 set_modrm(0, rsp);
188 } else if (is_int8(disp)) {
189 set_modrm(1, rsp);
190 set_disp8(disp);
191 } else {
192 set_modrm(2, rsp);
193 set_disp32(disp);
194 }
195 }
196
197
Operand(Register index,ScaleFactor scale,int32_t disp)198 Operand::Operand(Register index,
199 ScaleFactor scale,
200 int32_t disp) : rex_(0) {
201 DCHECK(!index.is(rsp));
202 len_ = 1;
203 set_modrm(0, rsp);
204 set_sib(scale, index, rbp);
205 set_disp32(disp);
206 }
207
208
Operand(Label * label)209 Operand::Operand(Label* label) : rex_(0), len_(1) {
210 DCHECK_NOT_NULL(label);
211 set_modrm(0, rbp);
212 set_disp64(reinterpret_cast<intptr_t>(label));
213 }
214
215
Operand(const Operand & operand,int32_t offset)216 Operand::Operand(const Operand& operand, int32_t offset) {
217 DCHECK(operand.len_ >= 1);
218 // Operand encodes REX ModR/M [SIB] [Disp].
219 byte modrm = operand.buf_[0];
220 DCHECK(modrm < 0xC0); // Disallow mode 3 (register target).
221 bool has_sib = ((modrm & 0x07) == 0x04);
222 byte mode = modrm & 0xC0;
223 int disp_offset = has_sib ? 2 : 1;
224 int base_reg = (has_sib ? operand.buf_[1] : modrm) & 0x07;
225 // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit
226 // displacement.
227 bool is_baseless = (mode == 0) && (base_reg == 0x05); // No base or RIP base.
228 int32_t disp_value = 0;
229 if (mode == 0x80 || is_baseless) {
230 // Mode 2 or mode 0 with rbp/r13 as base: Word displacement.
231 disp_value = *bit_cast<const int32_t*>(&operand.buf_[disp_offset]);
232 } else if (mode == 0x40) {
233 // Mode 1: Byte displacement.
234 disp_value = static_cast<signed char>(operand.buf_[disp_offset]);
235 }
236
237 // Write new operand with same registers, but with modified displacement.
238 DCHECK(offset >= 0 ? disp_value + offset > disp_value
239 : disp_value + offset < disp_value); // No overflow.
240 disp_value += offset;
241 rex_ = operand.rex_;
242 if (!is_int8(disp_value) || is_baseless) {
243 // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13.
244 buf_[0] = (modrm & 0x3f) | (is_baseless ? 0x00 : 0x80);
245 len_ = disp_offset + 4;
246 Memory::int32_at(&buf_[disp_offset]) = disp_value;
247 } else if (disp_value != 0 || (base_reg == 0x05)) {
248 // Need 8 bits of displacement.
249 buf_[0] = (modrm & 0x3f) | 0x40; // Mode 1.
250 len_ = disp_offset + 1;
251 buf_[disp_offset] = static_cast<byte>(disp_value);
252 } else {
253 // Need no displacement.
254 buf_[0] = (modrm & 0x3f); // Mode 0.
255 len_ = disp_offset;
256 }
257 if (has_sib) {
258 buf_[1] = operand.buf_[1];
259 }
260 }
261
262
AddressUsesRegister(Register reg) const263 bool Operand::AddressUsesRegister(Register reg) const {
264 int code = reg.code();
265 DCHECK((buf_[0] & 0xC0) != 0xC0); // Always a memory operand.
266 // Start with only low three bits of base register. Initial decoding doesn't
267 // distinguish on the REX.B bit.
268 int base_code = buf_[0] & 0x07;
269 if (base_code == rsp.code()) {
270 // SIB byte present in buf_[1].
271 // Check the index register from the SIB byte + REX.X prefix.
272 int index_code = ((buf_[1] >> 3) & 0x07) | ((rex_ & 0x02) << 2);
273 // Index code (including REX.X) of 0x04 (rsp) means no index register.
274 if (index_code != rsp.code() && index_code == code) return true;
275 // Add REX.B to get the full base register code.
276 base_code = (buf_[1] & 0x07) | ((rex_ & 0x01) << 3);
277 // A base register of 0x05 (rbp) with mod = 0 means no base register.
278 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false;
279 return code == base_code;
280 } else {
281 // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means
282 // no base register.
283 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false;
284 base_code |= ((rex_ & 0x01) << 3);
285 return code == base_code;
286 }
287 }
288
289
290 // -----------------------------------------------------------------------------
291 // Implementation of Assembler.
292
Assembler(Isolate * isolate,void * buffer,int buffer_size)293 Assembler::Assembler(Isolate* isolate, void* buffer, int buffer_size)
294 : AssemblerBase(isolate, buffer, buffer_size), code_targets_(100) {
295 // Clear the buffer in debug mode unless it was provided by the
296 // caller in which case we can't be sure it's okay to overwrite
297 // existing code in it.
298 #ifdef DEBUG
299 if (own_buffer_) {
300 memset(buffer_, 0xCC, buffer_size_); // int3
301 }
302 #endif
303
304 reloc_info_writer.Reposition(buffer_ + buffer_size_, pc_);
305 }
306
307
GetCode(CodeDesc * desc)308 void Assembler::GetCode(CodeDesc* desc) {
309 // Finalize code (at this point overflow() may be true, but the gap ensures
310 // that we are still not overlapping instructions and relocation info).
311 DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap.
312 // Set up code descriptor.
313 desc->buffer = buffer_;
314 desc->buffer_size = buffer_size_;
315 desc->instr_size = pc_offset();
316 DCHECK(desc->instr_size > 0); // Zero-size code objects upset the system.
317 desc->reloc_size =
318 static_cast<int>((buffer_ + buffer_size_) - reloc_info_writer.pos());
319 desc->origin = this;
320 desc->constant_pool_size = 0;
321 desc->unwinding_info_size = 0;
322 desc->unwinding_info = nullptr;
323 }
324
325
Align(int m)326 void Assembler::Align(int m) {
327 DCHECK(base::bits::IsPowerOfTwo32(m));
328 int delta = (m - (pc_offset() & (m - 1))) & (m - 1);
329 Nop(delta);
330 }
331
332
CodeTargetAlign()333 void Assembler::CodeTargetAlign() {
334 Align(16); // Preferred alignment of jump targets on x64.
335 }
336
337
IsNop(Address addr)338 bool Assembler::IsNop(Address addr) {
339 Address a = addr;
340 while (*a == 0x66) a++;
341 if (*a == 0x90) return true;
342 if (a[0] == 0xf && a[1] == 0x1f) return true;
343 return false;
344 }
345
346
bind_to(Label * L,int pos)347 void Assembler::bind_to(Label* L, int pos) {
348 DCHECK(!L->is_bound()); // Label may only be bound once.
349 DCHECK(0 <= pos && pos <= pc_offset()); // Position must be valid.
350 if (L->is_linked()) {
351 int current = L->pos();
352 int next = long_at(current);
353 while (next != current) {
354 if (current >= 4 && long_at(current - 4) == 0) {
355 // Absolute address.
356 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos);
357 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64;
358 internal_reference_positions_.push_back(current - 4);
359 } else {
360 // Relative address, relative to point after address.
361 int imm32 = pos - (current + sizeof(int32_t));
362 long_at_put(current, imm32);
363 }
364 current = next;
365 next = long_at(next);
366 }
367 // Fix up last fixup on linked list.
368 if (current >= 4 && long_at(current - 4) == 0) {
369 // Absolute address.
370 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos);
371 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64;
372 internal_reference_positions_.push_back(current - 4);
373 } else {
374 // Relative address, relative to point after address.
375 int imm32 = pos - (current + sizeof(int32_t));
376 long_at_put(current, imm32);
377 }
378 }
379 while (L->is_near_linked()) {
380 int fixup_pos = L->near_link_pos();
381 int offset_to_next =
382 static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos)));
383 DCHECK(offset_to_next <= 0);
384 int disp = pos - (fixup_pos + sizeof(int8_t));
385 CHECK(is_int8(disp));
386 set_byte_at(fixup_pos, disp);
387 if (offset_to_next < 0) {
388 L->link_to(fixup_pos + offset_to_next, Label::kNear);
389 } else {
390 L->UnuseNear();
391 }
392 }
393 L->bind_to(pos);
394 }
395
396
bind(Label * L)397 void Assembler::bind(Label* L) {
398 bind_to(L, pc_offset());
399 }
400
401
GrowBuffer()402 void Assembler::GrowBuffer() {
403 DCHECK(buffer_overflow());
404 if (!own_buffer_) FATAL("external code buffer is too small");
405
406 // Compute new buffer size.
407 CodeDesc desc; // the new buffer
408 desc.buffer_size = 2 * buffer_size_;
409
410 // Some internal data structures overflow for very large buffers,
411 // they must ensure that kMaximalBufferSize is not too large.
412 if (desc.buffer_size > kMaximalBufferSize ||
413 static_cast<size_t>(desc.buffer_size) >
414 isolate()->heap()->MaxOldGenerationSize()) {
415 V8::FatalProcessOutOfMemory("Assembler::GrowBuffer");
416 }
417
418 // Set up new buffer.
419 desc.buffer = NewArray<byte>(desc.buffer_size);
420 desc.origin = this;
421 desc.instr_size = pc_offset();
422 desc.reloc_size =
423 static_cast<int>((buffer_ + buffer_size_) - (reloc_info_writer.pos()));
424
425 // Clear the buffer in debug mode. Use 'int3' instructions to make
426 // sure to get into problems if we ever run uninitialized code.
427 #ifdef DEBUG
428 memset(desc.buffer, 0xCC, desc.buffer_size);
429 #endif
430
431 // Copy the data.
432 intptr_t pc_delta = desc.buffer - buffer_;
433 intptr_t rc_delta = (desc.buffer + desc.buffer_size) -
434 (buffer_ + buffer_size_);
435 MemMove(desc.buffer, buffer_, desc.instr_size);
436 MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(),
437 desc.reloc_size);
438
439 // Switch buffers.
440 DeleteArray(buffer_);
441 buffer_ = desc.buffer;
442 buffer_size_ = desc.buffer_size;
443 pc_ += pc_delta;
444 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
445 reloc_info_writer.last_pc() + pc_delta);
446
447 // Relocate internal references.
448 for (auto pos : internal_reference_positions_) {
449 intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos);
450 *p += pc_delta;
451 }
452
453 DCHECK(!buffer_overflow());
454 }
455
456
emit_operand(int code,const Operand & adr)457 void Assembler::emit_operand(int code, const Operand& adr) {
458 DCHECK(is_uint3(code));
459 const unsigned length = adr.len_;
460 DCHECK(length > 0);
461
462 // Emit updated ModR/M byte containing the given register.
463 DCHECK((adr.buf_[0] & 0x38) == 0);
464 *pc_++ = adr.buf_[0] | code << 3;
465
466 // Recognize RIP relative addressing.
467 if (adr.buf_[0] == 5) {
468 DCHECK_EQ(9u, length);
469 Label* label = *bit_cast<Label* const*>(&adr.buf_[1]);
470 if (label->is_bound()) {
471 int offset = label->pos() - pc_offset() - sizeof(int32_t);
472 DCHECK_GE(0, offset);
473 emitl(offset);
474 } else if (label->is_linked()) {
475 emitl(label->pos());
476 label->link_to(pc_offset() - sizeof(int32_t));
477 } else {
478 DCHECK(label->is_unused());
479 int32_t current = pc_offset();
480 emitl(current);
481 label->link_to(current);
482 }
483 } else {
484 // Emit the rest of the encoded operand.
485 for (unsigned i = 1; i < length; i++) *pc_++ = adr.buf_[i];
486 }
487 }
488
489
490 // Assembler Instruction implementations.
491
arithmetic_op(byte opcode,Register reg,const Operand & op,int size)492 void Assembler::arithmetic_op(byte opcode,
493 Register reg,
494 const Operand& op,
495 int size) {
496 EnsureSpace ensure_space(this);
497 emit_rex(reg, op, size);
498 emit(opcode);
499 emit_operand(reg, op);
500 }
501
502
arithmetic_op(byte opcode,Register reg,Register rm_reg,int size)503 void Assembler::arithmetic_op(byte opcode,
504 Register reg,
505 Register rm_reg,
506 int size) {
507 EnsureSpace ensure_space(this);
508 DCHECK((opcode & 0xC6) == 2);
509 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
510 // Swap reg and rm_reg and change opcode operand order.
511 emit_rex(rm_reg, reg, size);
512 emit(opcode ^ 0x02);
513 emit_modrm(rm_reg, reg);
514 } else {
515 emit_rex(reg, rm_reg, size);
516 emit(opcode);
517 emit_modrm(reg, rm_reg);
518 }
519 }
520
521
arithmetic_op_16(byte opcode,Register reg,Register rm_reg)522 void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) {
523 EnsureSpace ensure_space(this);
524 DCHECK((opcode & 0xC6) == 2);
525 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
526 // Swap reg and rm_reg and change opcode operand order.
527 emit(0x66);
528 emit_optional_rex_32(rm_reg, reg);
529 emit(opcode ^ 0x02);
530 emit_modrm(rm_reg, reg);
531 } else {
532 emit(0x66);
533 emit_optional_rex_32(reg, rm_reg);
534 emit(opcode);
535 emit_modrm(reg, rm_reg);
536 }
537 }
538
539
arithmetic_op_16(byte opcode,Register reg,const Operand & rm_reg)540 void Assembler::arithmetic_op_16(byte opcode,
541 Register reg,
542 const Operand& rm_reg) {
543 EnsureSpace ensure_space(this);
544 emit(0x66);
545 emit_optional_rex_32(reg, rm_reg);
546 emit(opcode);
547 emit_operand(reg, rm_reg);
548 }
549
550
arithmetic_op_8(byte opcode,Register reg,const Operand & op)551 void Assembler::arithmetic_op_8(byte opcode, Register reg, const Operand& op) {
552 EnsureSpace ensure_space(this);
553 if (!reg.is_byte_register()) {
554 emit_rex_32(reg, op);
555 } else {
556 emit_optional_rex_32(reg, op);
557 }
558 emit(opcode);
559 emit_operand(reg, op);
560 }
561
562
arithmetic_op_8(byte opcode,Register reg,Register rm_reg)563 void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) {
564 EnsureSpace ensure_space(this);
565 DCHECK((opcode & 0xC6) == 2);
566 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
567 // Swap reg and rm_reg and change opcode operand order.
568 if (!rm_reg.is_byte_register() || !reg.is_byte_register()) {
569 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
570 emit_rex_32(rm_reg, reg);
571 }
572 emit(opcode ^ 0x02);
573 emit_modrm(rm_reg, reg);
574 } else {
575 if (!reg.is_byte_register() || !rm_reg.is_byte_register()) {
576 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
577 emit_rex_32(reg, rm_reg);
578 }
579 emit(opcode);
580 emit_modrm(reg, rm_reg);
581 }
582 }
583
584
immediate_arithmetic_op(byte subcode,Register dst,Immediate src,int size)585 void Assembler::immediate_arithmetic_op(byte subcode,
586 Register dst,
587 Immediate src,
588 int size) {
589 EnsureSpace ensure_space(this);
590 emit_rex(dst, size);
591 if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) {
592 emit(0x83);
593 emit_modrm(subcode, dst);
594 emit(src.value_);
595 } else if (dst.is(rax)) {
596 emit(0x05 | (subcode << 3));
597 emit(src);
598 } else {
599 emit(0x81);
600 emit_modrm(subcode, dst);
601 emit(src);
602 }
603 }
604
immediate_arithmetic_op(byte subcode,const Operand & dst,Immediate src,int size)605 void Assembler::immediate_arithmetic_op(byte subcode,
606 const Operand& dst,
607 Immediate src,
608 int size) {
609 EnsureSpace ensure_space(this);
610 emit_rex(dst, size);
611 if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) {
612 emit(0x83);
613 emit_operand(subcode, dst);
614 emit(src.value_);
615 } else {
616 emit(0x81);
617 emit_operand(subcode, dst);
618 emit(src);
619 }
620 }
621
622
immediate_arithmetic_op_16(byte subcode,Register dst,Immediate src)623 void Assembler::immediate_arithmetic_op_16(byte subcode,
624 Register dst,
625 Immediate src) {
626 EnsureSpace ensure_space(this);
627 emit(0x66); // Operand size override prefix.
628 emit_optional_rex_32(dst);
629 if (is_int8(src.value_)) {
630 emit(0x83);
631 emit_modrm(subcode, dst);
632 emit(src.value_);
633 } else if (dst.is(rax)) {
634 emit(0x05 | (subcode << 3));
635 emitw(src.value_);
636 } else {
637 emit(0x81);
638 emit_modrm(subcode, dst);
639 emitw(src.value_);
640 }
641 }
642
643
immediate_arithmetic_op_16(byte subcode,const Operand & dst,Immediate src)644 void Assembler::immediate_arithmetic_op_16(byte subcode,
645 const Operand& dst,
646 Immediate src) {
647 EnsureSpace ensure_space(this);
648 emit(0x66); // Operand size override prefix.
649 emit_optional_rex_32(dst);
650 if (is_int8(src.value_)) {
651 emit(0x83);
652 emit_operand(subcode, dst);
653 emit(src.value_);
654 } else {
655 emit(0x81);
656 emit_operand(subcode, dst);
657 emitw(src.value_);
658 }
659 }
660
661
immediate_arithmetic_op_8(byte subcode,const Operand & dst,Immediate src)662 void Assembler::immediate_arithmetic_op_8(byte subcode,
663 const Operand& dst,
664 Immediate src) {
665 EnsureSpace ensure_space(this);
666 emit_optional_rex_32(dst);
667 DCHECK(is_int8(src.value_) || is_uint8(src.value_));
668 emit(0x80);
669 emit_operand(subcode, dst);
670 emit(src.value_);
671 }
672
673
immediate_arithmetic_op_8(byte subcode,Register dst,Immediate src)674 void Assembler::immediate_arithmetic_op_8(byte subcode,
675 Register dst,
676 Immediate src) {
677 EnsureSpace ensure_space(this);
678 if (!dst.is_byte_register()) {
679 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
680 emit_rex_32(dst);
681 }
682 DCHECK(is_int8(src.value_) || is_uint8(src.value_));
683 emit(0x80);
684 emit_modrm(subcode, dst);
685 emit(src.value_);
686 }
687
688
shift(Register dst,Immediate shift_amount,int subcode,int size)689 void Assembler::shift(Register dst,
690 Immediate shift_amount,
691 int subcode,
692 int size) {
693 EnsureSpace ensure_space(this);
694 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
695 : is_uint5(shift_amount.value_));
696 if (shift_amount.value_ == 1) {
697 emit_rex(dst, size);
698 emit(0xD1);
699 emit_modrm(subcode, dst);
700 } else {
701 emit_rex(dst, size);
702 emit(0xC1);
703 emit_modrm(subcode, dst);
704 emit(shift_amount.value_);
705 }
706 }
707
708
shift(Operand dst,Immediate shift_amount,int subcode,int size)709 void Assembler::shift(Operand dst, Immediate shift_amount, int subcode,
710 int size) {
711 EnsureSpace ensure_space(this);
712 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
713 : is_uint5(shift_amount.value_));
714 if (shift_amount.value_ == 1) {
715 emit_rex(dst, size);
716 emit(0xD1);
717 emit_operand(subcode, dst);
718 } else {
719 emit_rex(dst, size);
720 emit(0xC1);
721 emit_operand(subcode, dst);
722 emit(shift_amount.value_);
723 }
724 }
725
726
shift(Register dst,int subcode,int size)727 void Assembler::shift(Register dst, int subcode, int size) {
728 EnsureSpace ensure_space(this);
729 emit_rex(dst, size);
730 emit(0xD3);
731 emit_modrm(subcode, dst);
732 }
733
734
shift(Operand dst,int subcode,int size)735 void Assembler::shift(Operand dst, int subcode, int size) {
736 EnsureSpace ensure_space(this);
737 emit_rex(dst, size);
738 emit(0xD3);
739 emit_operand(subcode, dst);
740 }
741
742
bt(const Operand & dst,Register src)743 void Assembler::bt(const Operand& dst, Register src) {
744 EnsureSpace ensure_space(this);
745 emit_rex_64(src, dst);
746 emit(0x0F);
747 emit(0xA3);
748 emit_operand(src, dst);
749 }
750
751
bts(const Operand & dst,Register src)752 void Assembler::bts(const Operand& dst, Register src) {
753 EnsureSpace ensure_space(this);
754 emit_rex_64(src, dst);
755 emit(0x0F);
756 emit(0xAB);
757 emit_operand(src, dst);
758 }
759
760
bsrl(Register dst,Register src)761 void Assembler::bsrl(Register dst, Register src) {
762 EnsureSpace ensure_space(this);
763 emit_optional_rex_32(dst, src);
764 emit(0x0F);
765 emit(0xBD);
766 emit_modrm(dst, src);
767 }
768
769
bsrl(Register dst,const Operand & src)770 void Assembler::bsrl(Register dst, const Operand& src) {
771 EnsureSpace ensure_space(this);
772 emit_optional_rex_32(dst, src);
773 emit(0x0F);
774 emit(0xBD);
775 emit_operand(dst, src);
776 }
777
778
bsrq(Register dst,Register src)779 void Assembler::bsrq(Register dst, Register src) {
780 EnsureSpace ensure_space(this);
781 emit_rex_64(dst, src);
782 emit(0x0F);
783 emit(0xBD);
784 emit_modrm(dst, src);
785 }
786
787
bsrq(Register dst,const Operand & src)788 void Assembler::bsrq(Register dst, const Operand& src) {
789 EnsureSpace ensure_space(this);
790 emit_rex_64(dst, src);
791 emit(0x0F);
792 emit(0xBD);
793 emit_operand(dst, src);
794 }
795
796
bsfl(Register dst,Register src)797 void Assembler::bsfl(Register dst, Register src) {
798 EnsureSpace ensure_space(this);
799 emit_optional_rex_32(dst, src);
800 emit(0x0F);
801 emit(0xBC);
802 emit_modrm(dst, src);
803 }
804
805
bsfl(Register dst,const Operand & src)806 void Assembler::bsfl(Register dst, const Operand& src) {
807 EnsureSpace ensure_space(this);
808 emit_optional_rex_32(dst, src);
809 emit(0x0F);
810 emit(0xBC);
811 emit_operand(dst, src);
812 }
813
814
bsfq(Register dst,Register src)815 void Assembler::bsfq(Register dst, Register src) {
816 EnsureSpace ensure_space(this);
817 emit_rex_64(dst, src);
818 emit(0x0F);
819 emit(0xBC);
820 emit_modrm(dst, src);
821 }
822
823
bsfq(Register dst,const Operand & src)824 void Assembler::bsfq(Register dst, const Operand& src) {
825 EnsureSpace ensure_space(this);
826 emit_rex_64(dst, src);
827 emit(0x0F);
828 emit(0xBC);
829 emit_operand(dst, src);
830 }
831
832
call(Label * L)833 void Assembler::call(Label* L) {
834 EnsureSpace ensure_space(this);
835 // 1110 1000 #32-bit disp.
836 emit(0xE8);
837 if (L->is_bound()) {
838 int offset = L->pos() - pc_offset() - sizeof(int32_t);
839 DCHECK(offset <= 0);
840 emitl(offset);
841 } else if (L->is_linked()) {
842 emitl(L->pos());
843 L->link_to(pc_offset() - sizeof(int32_t));
844 } else {
845 DCHECK(L->is_unused());
846 int32_t current = pc_offset();
847 emitl(current);
848 L->link_to(current);
849 }
850 }
851
852
call(Address entry,RelocInfo::Mode rmode)853 void Assembler::call(Address entry, RelocInfo::Mode rmode) {
854 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
855 EnsureSpace ensure_space(this);
856 // 1110 1000 #32-bit disp.
857 emit(0xE8);
858 emit_runtime_entry(entry, rmode);
859 }
860
861
call(Handle<Code> target,RelocInfo::Mode rmode,TypeFeedbackId ast_id)862 void Assembler::call(Handle<Code> target,
863 RelocInfo::Mode rmode,
864 TypeFeedbackId ast_id) {
865 EnsureSpace ensure_space(this);
866 // 1110 1000 #32-bit disp.
867 emit(0xE8);
868 emit_code_target(target, rmode, ast_id);
869 }
870
871
call(Register adr)872 void Assembler::call(Register adr) {
873 EnsureSpace ensure_space(this);
874 // Opcode: FF /2 r64.
875 emit_optional_rex_32(adr);
876 emit(0xFF);
877 emit_modrm(0x2, adr);
878 }
879
880
call(const Operand & op)881 void Assembler::call(const Operand& op) {
882 EnsureSpace ensure_space(this);
883 // Opcode: FF /2 m64.
884 emit_optional_rex_32(op);
885 emit(0xFF);
886 emit_operand(0x2, op);
887 }
888
889
890 // Calls directly to the given address using a relative offset.
891 // Should only ever be used in Code objects for calls within the
892 // same Code object. Should not be used when generating new code (use labels),
893 // but only when patching existing code.
call(Address target)894 void Assembler::call(Address target) {
895 EnsureSpace ensure_space(this);
896 // 1110 1000 #32-bit disp.
897 emit(0xE8);
898 Address source = pc_ + 4;
899 intptr_t displacement = target - source;
900 DCHECK(is_int32(displacement));
901 emitl(static_cast<int32_t>(displacement));
902 }
903
904
clc()905 void Assembler::clc() {
906 EnsureSpace ensure_space(this);
907 emit(0xF8);
908 }
909
910
cld()911 void Assembler::cld() {
912 EnsureSpace ensure_space(this);
913 emit(0xFC);
914 }
915
916
cdq()917 void Assembler::cdq() {
918 EnsureSpace ensure_space(this);
919 emit(0x99);
920 }
921
922
cmovq(Condition cc,Register dst,Register src)923 void Assembler::cmovq(Condition cc, Register dst, Register src) {
924 if (cc == always) {
925 movq(dst, src);
926 } else if (cc == never) {
927 return;
928 }
929 // No need to check CpuInfo for CMOV support, it's a required part of the
930 // 64-bit architecture.
931 DCHECK(cc >= 0); // Use mov for unconditional moves.
932 EnsureSpace ensure_space(this);
933 // Opcode: REX.W 0f 40 + cc /r.
934 emit_rex_64(dst, src);
935 emit(0x0f);
936 emit(0x40 + cc);
937 emit_modrm(dst, src);
938 }
939
940
cmovq(Condition cc,Register dst,const Operand & src)941 void Assembler::cmovq(Condition cc, Register dst, const Operand& src) {
942 if (cc == always) {
943 movq(dst, src);
944 } else if (cc == never) {
945 return;
946 }
947 DCHECK(cc >= 0);
948 EnsureSpace ensure_space(this);
949 // Opcode: REX.W 0f 40 + cc /r.
950 emit_rex_64(dst, src);
951 emit(0x0f);
952 emit(0x40 + cc);
953 emit_operand(dst, src);
954 }
955
956
cmovl(Condition cc,Register dst,Register src)957 void Assembler::cmovl(Condition cc, Register dst, Register src) {
958 if (cc == always) {
959 movl(dst, src);
960 } else if (cc == never) {
961 return;
962 }
963 DCHECK(cc >= 0);
964 EnsureSpace ensure_space(this);
965 // Opcode: 0f 40 + cc /r.
966 emit_optional_rex_32(dst, src);
967 emit(0x0f);
968 emit(0x40 + cc);
969 emit_modrm(dst, src);
970 }
971
972
cmovl(Condition cc,Register dst,const Operand & src)973 void Assembler::cmovl(Condition cc, Register dst, const Operand& src) {
974 if (cc == always) {
975 movl(dst, src);
976 } else if (cc == never) {
977 return;
978 }
979 DCHECK(cc >= 0);
980 EnsureSpace ensure_space(this);
981 // Opcode: 0f 40 + cc /r.
982 emit_optional_rex_32(dst, src);
983 emit(0x0f);
984 emit(0x40 + cc);
985 emit_operand(dst, src);
986 }
987
988
cmpb_al(Immediate imm8)989 void Assembler::cmpb_al(Immediate imm8) {
990 DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_));
991 EnsureSpace ensure_space(this);
992 emit(0x3c);
993 emit(imm8.value_);
994 }
995
lock()996 void Assembler::lock() {
997 EnsureSpace ensure_space(this);
998 emit(0xf0);
999 }
1000
cmpxchgb(const Operand & dst,Register src)1001 void Assembler::cmpxchgb(const Operand& dst, Register src) {
1002 EnsureSpace ensure_space(this);
1003 if (!src.is_byte_register()) {
1004 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1005 emit_rex_32(src, dst);
1006 } else {
1007 emit_optional_rex_32(src, dst);
1008 }
1009 emit(0x0f);
1010 emit(0xb0);
1011 emit_operand(src, dst);
1012 }
1013
cmpxchgw(const Operand & dst,Register src)1014 void Assembler::cmpxchgw(const Operand& dst, Register src) {
1015 EnsureSpace ensure_space(this);
1016 emit(0x66);
1017 emit_optional_rex_32(src, dst);
1018 emit(0x0f);
1019 emit(0xb1);
1020 emit_operand(src, dst);
1021 }
1022
emit_cmpxchg(const Operand & dst,Register src,int size)1023 void Assembler::emit_cmpxchg(const Operand& dst, Register src, int size) {
1024 EnsureSpace ensure_space(this);
1025 emit_rex(src, dst, size);
1026 emit(0x0f);
1027 emit(0xb1);
1028 emit_operand(src, dst);
1029 }
1030
cpuid()1031 void Assembler::cpuid() {
1032 EnsureSpace ensure_space(this);
1033 emit(0x0F);
1034 emit(0xA2);
1035 }
1036
1037
cqo()1038 void Assembler::cqo() {
1039 EnsureSpace ensure_space(this);
1040 emit_rex_64();
1041 emit(0x99);
1042 }
1043
1044
emit_dec(Register dst,int size)1045 void Assembler::emit_dec(Register dst, int size) {
1046 EnsureSpace ensure_space(this);
1047 emit_rex(dst, size);
1048 emit(0xFF);
1049 emit_modrm(0x1, dst);
1050 }
1051
1052
emit_dec(const Operand & dst,int size)1053 void Assembler::emit_dec(const Operand& dst, int size) {
1054 EnsureSpace ensure_space(this);
1055 emit_rex(dst, size);
1056 emit(0xFF);
1057 emit_operand(1, dst);
1058 }
1059
1060
decb(Register dst)1061 void Assembler::decb(Register dst) {
1062 EnsureSpace ensure_space(this);
1063 if (!dst.is_byte_register()) {
1064 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1065 emit_rex_32(dst);
1066 }
1067 emit(0xFE);
1068 emit_modrm(0x1, dst);
1069 }
1070
1071
decb(const Operand & dst)1072 void Assembler::decb(const Operand& dst) {
1073 EnsureSpace ensure_space(this);
1074 emit_optional_rex_32(dst);
1075 emit(0xFE);
1076 emit_operand(1, dst);
1077 }
1078
1079
enter(Immediate size)1080 void Assembler::enter(Immediate size) {
1081 EnsureSpace ensure_space(this);
1082 emit(0xC8);
1083 emitw(size.value_); // 16 bit operand, always.
1084 emit(0);
1085 }
1086
1087
hlt()1088 void Assembler::hlt() {
1089 EnsureSpace ensure_space(this);
1090 emit(0xF4);
1091 }
1092
1093
emit_idiv(Register src,int size)1094 void Assembler::emit_idiv(Register src, int size) {
1095 EnsureSpace ensure_space(this);
1096 emit_rex(src, size);
1097 emit(0xF7);
1098 emit_modrm(0x7, src);
1099 }
1100
1101
emit_div(Register src,int size)1102 void Assembler::emit_div(Register src, int size) {
1103 EnsureSpace ensure_space(this);
1104 emit_rex(src, size);
1105 emit(0xF7);
1106 emit_modrm(0x6, src);
1107 }
1108
1109
emit_imul(Register src,int size)1110 void Assembler::emit_imul(Register src, int size) {
1111 EnsureSpace ensure_space(this);
1112 emit_rex(src, size);
1113 emit(0xF7);
1114 emit_modrm(0x5, src);
1115 }
1116
1117
emit_imul(const Operand & src,int size)1118 void Assembler::emit_imul(const Operand& src, int size) {
1119 EnsureSpace ensure_space(this);
1120 emit_rex(src, size);
1121 emit(0xF7);
1122 emit_operand(0x5, src);
1123 }
1124
1125
emit_imul(Register dst,Register src,int size)1126 void Assembler::emit_imul(Register dst, Register src, int size) {
1127 EnsureSpace ensure_space(this);
1128 emit_rex(dst, src, size);
1129 emit(0x0F);
1130 emit(0xAF);
1131 emit_modrm(dst, src);
1132 }
1133
1134
emit_imul(Register dst,const Operand & src,int size)1135 void Assembler::emit_imul(Register dst, const Operand& src, int size) {
1136 EnsureSpace ensure_space(this);
1137 emit_rex(dst, src, size);
1138 emit(0x0F);
1139 emit(0xAF);
1140 emit_operand(dst, src);
1141 }
1142
1143
emit_imul(Register dst,Register src,Immediate imm,int size)1144 void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) {
1145 EnsureSpace ensure_space(this);
1146 emit_rex(dst, src, size);
1147 if (is_int8(imm.value_)) {
1148 emit(0x6B);
1149 emit_modrm(dst, src);
1150 emit(imm.value_);
1151 } else {
1152 emit(0x69);
1153 emit_modrm(dst, src);
1154 emitl(imm.value_);
1155 }
1156 }
1157
1158
emit_imul(Register dst,const Operand & src,Immediate imm,int size)1159 void Assembler::emit_imul(Register dst, const Operand& src, Immediate imm,
1160 int size) {
1161 EnsureSpace ensure_space(this);
1162 emit_rex(dst, src, size);
1163 if (is_int8(imm.value_)) {
1164 emit(0x6B);
1165 emit_operand(dst, src);
1166 emit(imm.value_);
1167 } else {
1168 emit(0x69);
1169 emit_operand(dst, src);
1170 emitl(imm.value_);
1171 }
1172 }
1173
1174
emit_inc(Register dst,int size)1175 void Assembler::emit_inc(Register dst, int size) {
1176 EnsureSpace ensure_space(this);
1177 emit_rex(dst, size);
1178 emit(0xFF);
1179 emit_modrm(0x0, dst);
1180 }
1181
1182
emit_inc(const Operand & dst,int size)1183 void Assembler::emit_inc(const Operand& dst, int size) {
1184 EnsureSpace ensure_space(this);
1185 emit_rex(dst, size);
1186 emit(0xFF);
1187 emit_operand(0, dst);
1188 }
1189
1190
int3()1191 void Assembler::int3() {
1192 EnsureSpace ensure_space(this);
1193 emit(0xCC);
1194 }
1195
1196
j(Condition cc,Label * L,Label::Distance distance)1197 void Assembler::j(Condition cc, Label* L, Label::Distance distance) {
1198 if (cc == always) {
1199 jmp(L);
1200 return;
1201 } else if (cc == never) {
1202 return;
1203 }
1204 EnsureSpace ensure_space(this);
1205 DCHECK(is_uint4(cc));
1206 if (L->is_bound()) {
1207 const int short_size = 2;
1208 const int long_size = 6;
1209 int offs = L->pos() - pc_offset();
1210 DCHECK(offs <= 0);
1211 // Determine whether we can use 1-byte offsets for backwards branches,
1212 // which have a max range of 128 bytes.
1213
1214 // We also need to check predictable_code_size() flag here, because on x64,
1215 // when the full code generator recompiles code for debugging, some places
1216 // need to be padded out to a certain size. The debugger is keeping track of
1217 // how often it did this so that it can adjust return addresses on the
1218 // stack, but if the size of jump instructions can also change, that's not
1219 // enough and the calculated offsets would be incorrect.
1220 if (is_int8(offs - short_size) && !predictable_code_size()) {
1221 // 0111 tttn #8-bit disp.
1222 emit(0x70 | cc);
1223 emit((offs - short_size) & 0xFF);
1224 } else {
1225 // 0000 1111 1000 tttn #32-bit disp.
1226 emit(0x0F);
1227 emit(0x80 | cc);
1228 emitl(offs - long_size);
1229 }
1230 } else if (distance == Label::kNear) {
1231 // 0111 tttn #8-bit disp
1232 emit(0x70 | cc);
1233 byte disp = 0x00;
1234 if (L->is_near_linked()) {
1235 int offset = L->near_link_pos() - pc_offset();
1236 DCHECK(is_int8(offset));
1237 disp = static_cast<byte>(offset & 0xFF);
1238 }
1239 L->link_to(pc_offset(), Label::kNear);
1240 emit(disp);
1241 } else if (L->is_linked()) {
1242 // 0000 1111 1000 tttn #32-bit disp.
1243 emit(0x0F);
1244 emit(0x80 | cc);
1245 emitl(L->pos());
1246 L->link_to(pc_offset() - sizeof(int32_t));
1247 } else {
1248 DCHECK(L->is_unused());
1249 emit(0x0F);
1250 emit(0x80 | cc);
1251 int32_t current = pc_offset();
1252 emitl(current);
1253 L->link_to(current);
1254 }
1255 }
1256
1257
j(Condition cc,Address entry,RelocInfo::Mode rmode)1258 void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) {
1259 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1260 EnsureSpace ensure_space(this);
1261 DCHECK(is_uint4(cc));
1262 emit(0x0F);
1263 emit(0x80 | cc);
1264 emit_runtime_entry(entry, rmode);
1265 }
1266
1267
j(Condition cc,Handle<Code> target,RelocInfo::Mode rmode)1268 void Assembler::j(Condition cc,
1269 Handle<Code> target,
1270 RelocInfo::Mode rmode) {
1271 EnsureSpace ensure_space(this);
1272 DCHECK(is_uint4(cc));
1273 // 0000 1111 1000 tttn #32-bit disp.
1274 emit(0x0F);
1275 emit(0x80 | cc);
1276 emit_code_target(target, rmode);
1277 }
1278
1279
jmp(Label * L,Label::Distance distance)1280 void Assembler::jmp(Label* L, Label::Distance distance) {
1281 EnsureSpace ensure_space(this);
1282 const int short_size = sizeof(int8_t);
1283 const int long_size = sizeof(int32_t);
1284 if (L->is_bound()) {
1285 int offs = L->pos() - pc_offset() - 1;
1286 DCHECK(offs <= 0);
1287 if (is_int8(offs - short_size) && !predictable_code_size()) {
1288 // 1110 1011 #8-bit disp.
1289 emit(0xEB);
1290 emit((offs - short_size) & 0xFF);
1291 } else {
1292 // 1110 1001 #32-bit disp.
1293 emit(0xE9);
1294 emitl(offs - long_size);
1295 }
1296 } else if (distance == Label::kNear) {
1297 emit(0xEB);
1298 byte disp = 0x00;
1299 if (L->is_near_linked()) {
1300 int offset = L->near_link_pos() - pc_offset();
1301 DCHECK(is_int8(offset));
1302 disp = static_cast<byte>(offset & 0xFF);
1303 }
1304 L->link_to(pc_offset(), Label::kNear);
1305 emit(disp);
1306 } else if (L->is_linked()) {
1307 // 1110 1001 #32-bit disp.
1308 emit(0xE9);
1309 emitl(L->pos());
1310 L->link_to(pc_offset() - long_size);
1311 } else {
1312 // 1110 1001 #32-bit disp.
1313 DCHECK(L->is_unused());
1314 emit(0xE9);
1315 int32_t current = pc_offset();
1316 emitl(current);
1317 L->link_to(current);
1318 }
1319 }
1320
1321
jmp(Handle<Code> target,RelocInfo::Mode rmode)1322 void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) {
1323 EnsureSpace ensure_space(this);
1324 // 1110 1001 #32-bit disp.
1325 emit(0xE9);
1326 emit_code_target(target, rmode);
1327 }
1328
1329
jmp(Address entry,RelocInfo::Mode rmode)1330 void Assembler::jmp(Address entry, RelocInfo::Mode rmode) {
1331 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1332 EnsureSpace ensure_space(this);
1333 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1334 emit(0xE9);
1335 emit_runtime_entry(entry, rmode);
1336 }
1337
1338
jmp(Register target)1339 void Assembler::jmp(Register target) {
1340 EnsureSpace ensure_space(this);
1341 // Opcode FF/4 r64.
1342 emit_optional_rex_32(target);
1343 emit(0xFF);
1344 emit_modrm(0x4, target);
1345 }
1346
1347
jmp(const Operand & src)1348 void Assembler::jmp(const Operand& src) {
1349 EnsureSpace ensure_space(this);
1350 // Opcode FF/4 m64.
1351 emit_optional_rex_32(src);
1352 emit(0xFF);
1353 emit_operand(0x4, src);
1354 }
1355
1356
emit_lea(Register dst,const Operand & src,int size)1357 void Assembler::emit_lea(Register dst, const Operand& src, int size) {
1358 EnsureSpace ensure_space(this);
1359 emit_rex(dst, src, size);
1360 emit(0x8D);
1361 emit_operand(dst, src);
1362 }
1363
1364
load_rax(void * value,RelocInfo::Mode mode)1365 void Assembler::load_rax(void* value, RelocInfo::Mode mode) {
1366 EnsureSpace ensure_space(this);
1367 if (kPointerSize == kInt64Size) {
1368 emit(0x48); // REX.W
1369 emit(0xA1);
1370 emitp(value, mode);
1371 } else {
1372 DCHECK(kPointerSize == kInt32Size);
1373 emit(0xA1);
1374 emitp(value, mode);
1375 // In 64-bit mode, need to zero extend the operand to 8 bytes.
1376 // See 2.2.1.4 in Intel64 and IA32 Architectures Software
1377 // Developer's Manual Volume 2.
1378 emitl(0);
1379 }
1380 }
1381
1382
load_rax(ExternalReference ref)1383 void Assembler::load_rax(ExternalReference ref) {
1384 load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
1385 }
1386
1387
leave()1388 void Assembler::leave() {
1389 EnsureSpace ensure_space(this);
1390 emit(0xC9);
1391 }
1392
1393
movb(Register dst,const Operand & src)1394 void Assembler::movb(Register dst, const Operand& src) {
1395 EnsureSpace ensure_space(this);
1396 if (!dst.is_byte_register()) {
1397 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1398 emit_rex_32(dst, src);
1399 } else {
1400 emit_optional_rex_32(dst, src);
1401 }
1402 emit(0x8A);
1403 emit_operand(dst, src);
1404 }
1405
1406
movb(Register dst,Immediate imm)1407 void Assembler::movb(Register dst, Immediate imm) {
1408 EnsureSpace ensure_space(this);
1409 if (!dst.is_byte_register()) {
1410 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1411 emit_rex_32(dst);
1412 }
1413 emit(0xB0 + dst.low_bits());
1414 emit(imm.value_);
1415 }
1416
1417
movb(const Operand & dst,Register src)1418 void Assembler::movb(const Operand& dst, Register src) {
1419 EnsureSpace ensure_space(this);
1420 if (!src.is_byte_register()) {
1421 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1422 emit_rex_32(src, dst);
1423 } else {
1424 emit_optional_rex_32(src, dst);
1425 }
1426 emit(0x88);
1427 emit_operand(src, dst);
1428 }
1429
1430
movb(const Operand & dst,Immediate imm)1431 void Assembler::movb(const Operand& dst, Immediate imm) {
1432 EnsureSpace ensure_space(this);
1433 emit_optional_rex_32(dst);
1434 emit(0xC6);
1435 emit_operand(0x0, dst);
1436 emit(static_cast<byte>(imm.value_));
1437 }
1438
1439
movw(Register dst,const Operand & src)1440 void Assembler::movw(Register dst, const Operand& src) {
1441 EnsureSpace ensure_space(this);
1442 emit(0x66);
1443 emit_optional_rex_32(dst, src);
1444 emit(0x8B);
1445 emit_operand(dst, src);
1446 }
1447
1448
movw(const Operand & dst,Register src)1449 void Assembler::movw(const Operand& dst, Register src) {
1450 EnsureSpace ensure_space(this);
1451 emit(0x66);
1452 emit_optional_rex_32(src, dst);
1453 emit(0x89);
1454 emit_operand(src, dst);
1455 }
1456
1457
movw(const Operand & dst,Immediate imm)1458 void Assembler::movw(const Operand& dst, Immediate imm) {
1459 EnsureSpace ensure_space(this);
1460 emit(0x66);
1461 emit_optional_rex_32(dst);
1462 emit(0xC7);
1463 emit_operand(0x0, dst);
1464 emit(static_cast<byte>(imm.value_ & 0xff));
1465 emit(static_cast<byte>(imm.value_ >> 8));
1466 }
1467
1468
emit_mov(Register dst,const Operand & src,int size)1469 void Assembler::emit_mov(Register dst, const Operand& src, int size) {
1470 EnsureSpace ensure_space(this);
1471 emit_rex(dst, src, size);
1472 emit(0x8B);
1473 emit_operand(dst, src);
1474 }
1475
1476
emit_mov(Register dst,Register src,int size)1477 void Assembler::emit_mov(Register dst, Register src, int size) {
1478 EnsureSpace ensure_space(this);
1479 if (src.low_bits() == 4) {
1480 emit_rex(src, dst, size);
1481 emit(0x89);
1482 emit_modrm(src, dst);
1483 } else {
1484 emit_rex(dst, src, size);
1485 emit(0x8B);
1486 emit_modrm(dst, src);
1487 }
1488 }
1489
1490
emit_mov(const Operand & dst,Register src,int size)1491 void Assembler::emit_mov(const Operand& dst, Register src, int size) {
1492 EnsureSpace ensure_space(this);
1493 emit_rex(src, dst, size);
1494 emit(0x89);
1495 emit_operand(src, dst);
1496 }
1497
1498
emit_mov(Register dst,Immediate value,int size)1499 void Assembler::emit_mov(Register dst, Immediate value, int size) {
1500 EnsureSpace ensure_space(this);
1501 emit_rex(dst, size);
1502 if (size == kInt64Size) {
1503 emit(0xC7);
1504 emit_modrm(0x0, dst);
1505 } else {
1506 DCHECK(size == kInt32Size);
1507 emit(0xB8 + dst.low_bits());
1508 }
1509 emit(value);
1510 }
1511
1512
emit_mov(const Operand & dst,Immediate value,int size)1513 void Assembler::emit_mov(const Operand& dst, Immediate value, int size) {
1514 EnsureSpace ensure_space(this);
1515 emit_rex(dst, size);
1516 emit(0xC7);
1517 emit_operand(0x0, dst);
1518 emit(value);
1519 }
1520
1521
movp(Register dst,void * value,RelocInfo::Mode rmode)1522 void Assembler::movp(Register dst, void* value, RelocInfo::Mode rmode) {
1523 EnsureSpace ensure_space(this);
1524 emit_rex(dst, kPointerSize);
1525 emit(0xB8 | dst.low_bits());
1526 emitp(value, rmode);
1527 }
1528
movq(Register dst,int64_t value,RelocInfo::Mode rmode)1529 void Assembler::movq(Register dst, int64_t value, RelocInfo::Mode rmode) {
1530 EnsureSpace ensure_space(this);
1531 emit_rex_64(dst);
1532 emit(0xB8 | dst.low_bits());
1533 if (!RelocInfo::IsNone(rmode)) {
1534 RecordRelocInfo(rmode, value);
1535 }
1536 emitq(value);
1537 }
1538
movq(Register dst,uint64_t value,RelocInfo::Mode rmode)1539 void Assembler::movq(Register dst, uint64_t value, RelocInfo::Mode rmode) {
1540 movq(dst, static_cast<int64_t>(value), rmode);
1541 }
1542
1543 // Loads the ip-relative location of the src label into the target location
1544 // (as a 32-bit offset sign extended to 64-bit).
movl(const Operand & dst,Label * src)1545 void Assembler::movl(const Operand& dst, Label* src) {
1546 EnsureSpace ensure_space(this);
1547 emit_optional_rex_32(dst);
1548 emit(0xC7);
1549 emit_operand(0, dst);
1550 if (src->is_bound()) {
1551 int offset = src->pos() - pc_offset() - sizeof(int32_t);
1552 DCHECK(offset <= 0);
1553 emitl(offset);
1554 } else if (src->is_linked()) {
1555 emitl(src->pos());
1556 src->link_to(pc_offset() - sizeof(int32_t));
1557 } else {
1558 DCHECK(src->is_unused());
1559 int32_t current = pc_offset();
1560 emitl(current);
1561 src->link_to(current);
1562 }
1563 }
1564
1565
movsxbl(Register dst,Register src)1566 void Assembler::movsxbl(Register dst, Register src) {
1567 EnsureSpace ensure_space(this);
1568 if (!src.is_byte_register()) {
1569 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1570 emit_rex_32(dst, src);
1571 } else {
1572 emit_optional_rex_32(dst, src);
1573 }
1574 emit(0x0F);
1575 emit(0xBE);
1576 emit_modrm(dst, src);
1577 }
1578
1579
movsxbl(Register dst,const Operand & src)1580 void Assembler::movsxbl(Register dst, const Operand& src) {
1581 EnsureSpace ensure_space(this);
1582 emit_optional_rex_32(dst, src);
1583 emit(0x0F);
1584 emit(0xBE);
1585 emit_operand(dst, src);
1586 }
1587
1588
movsxbq(Register dst,const Operand & src)1589 void Assembler::movsxbq(Register dst, const Operand& src) {
1590 EnsureSpace ensure_space(this);
1591 emit_rex_64(dst, src);
1592 emit(0x0F);
1593 emit(0xBE);
1594 emit_operand(dst, src);
1595 }
1596
movsxbq(Register dst,Register src)1597 void Assembler::movsxbq(Register dst, Register src) {
1598 EnsureSpace ensure_space(this);
1599 emit_rex_64(dst, src);
1600 emit(0x0F);
1601 emit(0xBE);
1602 emit_modrm(dst, src);
1603 }
1604
movsxwl(Register dst,Register src)1605 void Assembler::movsxwl(Register dst, Register src) {
1606 EnsureSpace ensure_space(this);
1607 emit_optional_rex_32(dst, src);
1608 emit(0x0F);
1609 emit(0xBF);
1610 emit_modrm(dst, src);
1611 }
1612
1613
movsxwl(Register dst,const Operand & src)1614 void Assembler::movsxwl(Register dst, const Operand& src) {
1615 EnsureSpace ensure_space(this);
1616 emit_optional_rex_32(dst, src);
1617 emit(0x0F);
1618 emit(0xBF);
1619 emit_operand(dst, src);
1620 }
1621
1622
movsxwq(Register dst,const Operand & src)1623 void Assembler::movsxwq(Register dst, const Operand& src) {
1624 EnsureSpace ensure_space(this);
1625 emit_rex_64(dst, src);
1626 emit(0x0F);
1627 emit(0xBF);
1628 emit_operand(dst, src);
1629 }
1630
movsxwq(Register dst,Register src)1631 void Assembler::movsxwq(Register dst, Register src) {
1632 EnsureSpace ensure_space(this);
1633 emit_rex_64(dst, src);
1634 emit(0x0F);
1635 emit(0xBF);
1636 emit_modrm(dst, src);
1637 }
1638
movsxlq(Register dst,Register src)1639 void Assembler::movsxlq(Register dst, Register src) {
1640 EnsureSpace ensure_space(this);
1641 emit_rex_64(dst, src);
1642 emit(0x63);
1643 emit_modrm(dst, src);
1644 }
1645
1646
movsxlq(Register dst,const Operand & src)1647 void Assembler::movsxlq(Register dst, const Operand& src) {
1648 EnsureSpace ensure_space(this);
1649 emit_rex_64(dst, src);
1650 emit(0x63);
1651 emit_operand(dst, src);
1652 }
1653
1654
emit_movzxb(Register dst,const Operand & src,int size)1655 void Assembler::emit_movzxb(Register dst, const Operand& src, int size) {
1656 EnsureSpace ensure_space(this);
1657 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1658 // there is no need to make this a 64 bit operation.
1659 emit_optional_rex_32(dst, src);
1660 emit(0x0F);
1661 emit(0xB6);
1662 emit_operand(dst, src);
1663 }
1664
1665
emit_movzxb(Register dst,Register src,int size)1666 void Assembler::emit_movzxb(Register dst, Register src, int size) {
1667 EnsureSpace ensure_space(this);
1668 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1669 // there is no need to make this a 64 bit operation.
1670 if (!src.is_byte_register()) {
1671 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1672 emit_rex_32(dst, src);
1673 } else {
1674 emit_optional_rex_32(dst, src);
1675 }
1676 emit(0x0F);
1677 emit(0xB6);
1678 emit_modrm(dst, src);
1679 }
1680
1681
emit_movzxw(Register dst,const Operand & src,int size)1682 void Assembler::emit_movzxw(Register dst, const Operand& src, int size) {
1683 EnsureSpace ensure_space(this);
1684 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1685 // there is no need to make this a 64 bit operation.
1686 emit_optional_rex_32(dst, src);
1687 emit(0x0F);
1688 emit(0xB7);
1689 emit_operand(dst, src);
1690 }
1691
1692
emit_movzxw(Register dst,Register src,int size)1693 void Assembler::emit_movzxw(Register dst, Register src, int size) {
1694 EnsureSpace ensure_space(this);
1695 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1696 // there is no need to make this a 64 bit operation.
1697 emit_optional_rex_32(dst, src);
1698 emit(0x0F);
1699 emit(0xB7);
1700 emit_modrm(dst, src);
1701 }
1702
1703
repmovsb()1704 void Assembler::repmovsb() {
1705 EnsureSpace ensure_space(this);
1706 emit(0xF3);
1707 emit(0xA4);
1708 }
1709
1710
repmovsw()1711 void Assembler::repmovsw() {
1712 EnsureSpace ensure_space(this);
1713 emit(0x66); // Operand size override.
1714 emit(0xF3);
1715 emit(0xA4);
1716 }
1717
1718
emit_repmovs(int size)1719 void Assembler::emit_repmovs(int size) {
1720 EnsureSpace ensure_space(this);
1721 emit(0xF3);
1722 emit_rex(size);
1723 emit(0xA5);
1724 }
1725
1726
mull(Register src)1727 void Assembler::mull(Register src) {
1728 EnsureSpace ensure_space(this);
1729 emit_optional_rex_32(src);
1730 emit(0xF7);
1731 emit_modrm(0x4, src);
1732 }
1733
1734
mull(const Operand & src)1735 void Assembler::mull(const Operand& src) {
1736 EnsureSpace ensure_space(this);
1737 emit_optional_rex_32(src);
1738 emit(0xF7);
1739 emit_operand(0x4, src);
1740 }
1741
1742
mulq(Register src)1743 void Assembler::mulq(Register src) {
1744 EnsureSpace ensure_space(this);
1745 emit_rex_64(src);
1746 emit(0xF7);
1747 emit_modrm(0x4, src);
1748 }
1749
1750
emit_neg(Register dst,int size)1751 void Assembler::emit_neg(Register dst, int size) {
1752 EnsureSpace ensure_space(this);
1753 emit_rex(dst, size);
1754 emit(0xF7);
1755 emit_modrm(0x3, dst);
1756 }
1757
1758
emit_neg(const Operand & dst,int size)1759 void Assembler::emit_neg(const Operand& dst, int size) {
1760 EnsureSpace ensure_space(this);
1761 emit_rex_64(dst);
1762 emit(0xF7);
1763 emit_operand(3, dst);
1764 }
1765
1766
nop()1767 void Assembler::nop() {
1768 EnsureSpace ensure_space(this);
1769 emit(0x90);
1770 }
1771
1772
emit_not(Register dst,int size)1773 void Assembler::emit_not(Register dst, int size) {
1774 EnsureSpace ensure_space(this);
1775 emit_rex(dst, size);
1776 emit(0xF7);
1777 emit_modrm(0x2, dst);
1778 }
1779
1780
emit_not(const Operand & dst,int size)1781 void Assembler::emit_not(const Operand& dst, int size) {
1782 EnsureSpace ensure_space(this);
1783 emit_rex(dst, size);
1784 emit(0xF7);
1785 emit_operand(2, dst);
1786 }
1787
1788
Nop(int n)1789 void Assembler::Nop(int n) {
1790 // The recommended muti-byte sequences of NOP instructions from the Intel 64
1791 // and IA-32 Architectures Software Developer's Manual.
1792 //
1793 // Length Assembly Byte Sequence
1794 // 2 bytes 66 NOP 66 90H
1795 // 3 bytes NOP DWORD ptr [EAX] 0F 1F 00H
1796 // 4 bytes NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H
1797 // 5 bytes NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H
1798 // 6 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H
1799 // 7 bytes NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H
1800 // 8 bytes NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H
1801 // 9 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 66 0F 1F 84 00 00 00 00
1802 // 00000000H] 00H
1803
1804 EnsureSpace ensure_space(this);
1805 while (n > 0) {
1806 switch (n) {
1807 case 2:
1808 emit(0x66);
1809 case 1:
1810 emit(0x90);
1811 return;
1812 case 3:
1813 emit(0x0f);
1814 emit(0x1f);
1815 emit(0x00);
1816 return;
1817 case 4:
1818 emit(0x0f);
1819 emit(0x1f);
1820 emit(0x40);
1821 emit(0x00);
1822 return;
1823 case 6:
1824 emit(0x66);
1825 case 5:
1826 emit(0x0f);
1827 emit(0x1f);
1828 emit(0x44);
1829 emit(0x00);
1830 emit(0x00);
1831 return;
1832 case 7:
1833 emit(0x0f);
1834 emit(0x1f);
1835 emit(0x80);
1836 emit(0x00);
1837 emit(0x00);
1838 emit(0x00);
1839 emit(0x00);
1840 return;
1841 default:
1842 case 11:
1843 emit(0x66);
1844 n--;
1845 case 10:
1846 emit(0x66);
1847 n--;
1848 case 9:
1849 emit(0x66);
1850 n--;
1851 case 8:
1852 emit(0x0f);
1853 emit(0x1f);
1854 emit(0x84);
1855 emit(0x00);
1856 emit(0x00);
1857 emit(0x00);
1858 emit(0x00);
1859 emit(0x00);
1860 n -= 8;
1861 }
1862 }
1863 }
1864
1865
popq(Register dst)1866 void Assembler::popq(Register dst) {
1867 EnsureSpace ensure_space(this);
1868 emit_optional_rex_32(dst);
1869 emit(0x58 | dst.low_bits());
1870 }
1871
1872
popq(const Operand & dst)1873 void Assembler::popq(const Operand& dst) {
1874 EnsureSpace ensure_space(this);
1875 emit_optional_rex_32(dst);
1876 emit(0x8F);
1877 emit_operand(0, dst);
1878 }
1879
1880
popfq()1881 void Assembler::popfq() {
1882 EnsureSpace ensure_space(this);
1883 emit(0x9D);
1884 }
1885
1886
pushq(Register src)1887 void Assembler::pushq(Register src) {
1888 EnsureSpace ensure_space(this);
1889 emit_optional_rex_32(src);
1890 emit(0x50 | src.low_bits());
1891 }
1892
1893
pushq(const Operand & src)1894 void Assembler::pushq(const Operand& src) {
1895 EnsureSpace ensure_space(this);
1896 emit_optional_rex_32(src);
1897 emit(0xFF);
1898 emit_operand(6, src);
1899 }
1900
1901
pushq(Immediate value)1902 void Assembler::pushq(Immediate value) {
1903 EnsureSpace ensure_space(this);
1904 if (is_int8(value.value_)) {
1905 emit(0x6A);
1906 emit(value.value_); // Emit low byte of value.
1907 } else {
1908 emit(0x68);
1909 emitl(value.value_);
1910 }
1911 }
1912
1913
pushq_imm32(int32_t imm32)1914 void Assembler::pushq_imm32(int32_t imm32) {
1915 EnsureSpace ensure_space(this);
1916 emit(0x68);
1917 emitl(imm32);
1918 }
1919
1920
pushfq()1921 void Assembler::pushfq() {
1922 EnsureSpace ensure_space(this);
1923 emit(0x9C);
1924 }
1925
1926
ret(int imm16)1927 void Assembler::ret(int imm16) {
1928 EnsureSpace ensure_space(this);
1929 DCHECK(is_uint16(imm16));
1930 if (imm16 == 0) {
1931 emit(0xC3);
1932 } else {
1933 emit(0xC2);
1934 emit(imm16 & 0xFF);
1935 emit((imm16 >> 8) & 0xFF);
1936 }
1937 }
1938
1939
ud2()1940 void Assembler::ud2() {
1941 EnsureSpace ensure_space(this);
1942 emit(0x0F);
1943 emit(0x0B);
1944 }
1945
1946
setcc(Condition cc,Register reg)1947 void Assembler::setcc(Condition cc, Register reg) {
1948 if (cc > last_condition) {
1949 movb(reg, Immediate(cc == always ? 1 : 0));
1950 return;
1951 }
1952 EnsureSpace ensure_space(this);
1953 DCHECK(is_uint4(cc));
1954 if (!reg.is_byte_register()) {
1955 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1956 emit_rex_32(reg);
1957 }
1958 emit(0x0F);
1959 emit(0x90 | cc);
1960 emit_modrm(0x0, reg);
1961 }
1962
1963
shld(Register dst,Register src)1964 void Assembler::shld(Register dst, Register src) {
1965 EnsureSpace ensure_space(this);
1966 emit_rex_64(src, dst);
1967 emit(0x0F);
1968 emit(0xA5);
1969 emit_modrm(src, dst);
1970 }
1971
1972
shrd(Register dst,Register src)1973 void Assembler::shrd(Register dst, Register src) {
1974 EnsureSpace ensure_space(this);
1975 emit_rex_64(src, dst);
1976 emit(0x0F);
1977 emit(0xAD);
1978 emit_modrm(src, dst);
1979 }
1980
xchgb(Register reg,const Operand & op)1981 void Assembler::xchgb(Register reg, const Operand& op) {
1982 EnsureSpace ensure_space(this);
1983 if (!reg.is_byte_register()) {
1984 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1985 emit_rex_32(reg, op);
1986 } else {
1987 emit_optional_rex_32(reg, op);
1988 }
1989 emit(0x86);
1990 emit_operand(reg, op);
1991 }
1992
xchgw(Register reg,const Operand & op)1993 void Assembler::xchgw(Register reg, const Operand& op) {
1994 EnsureSpace ensure_space(this);
1995 emit(0x66);
1996 emit_optional_rex_32(reg, op);
1997 emit(0x87);
1998 emit_operand(reg, op);
1999 }
2000
emit_xchg(Register dst,Register src,int size)2001 void Assembler::emit_xchg(Register dst, Register src, int size) {
2002 EnsureSpace ensure_space(this);
2003 if (src.is(rax) || dst.is(rax)) { // Single-byte encoding
2004 Register other = src.is(rax) ? dst : src;
2005 emit_rex(other, size);
2006 emit(0x90 | other.low_bits());
2007 } else if (dst.low_bits() == 4) {
2008 emit_rex(dst, src, size);
2009 emit(0x87);
2010 emit_modrm(dst, src);
2011 } else {
2012 emit_rex(src, dst, size);
2013 emit(0x87);
2014 emit_modrm(src, dst);
2015 }
2016 }
2017
2018
emit_xchg(Register dst,const Operand & src,int size)2019 void Assembler::emit_xchg(Register dst, const Operand& src, int size) {
2020 EnsureSpace ensure_space(this);
2021 emit_rex(dst, src, size);
2022 emit(0x87);
2023 emit_operand(dst, src);
2024 }
2025
2026
store_rax(void * dst,RelocInfo::Mode mode)2027 void Assembler::store_rax(void* dst, RelocInfo::Mode mode) {
2028 EnsureSpace ensure_space(this);
2029 if (kPointerSize == kInt64Size) {
2030 emit(0x48); // REX.W
2031 emit(0xA3);
2032 emitp(dst, mode);
2033 } else {
2034 DCHECK(kPointerSize == kInt32Size);
2035 emit(0xA3);
2036 emitp(dst, mode);
2037 // In 64-bit mode, need to zero extend the operand to 8 bytes.
2038 // See 2.2.1.4 in Intel64 and IA32 Architectures Software
2039 // Developer's Manual Volume 2.
2040 emitl(0);
2041 }
2042 }
2043
2044
store_rax(ExternalReference ref)2045 void Assembler::store_rax(ExternalReference ref) {
2046 store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
2047 }
2048
2049
testb(Register dst,Register src)2050 void Assembler::testb(Register dst, Register src) {
2051 EnsureSpace ensure_space(this);
2052 emit_test(dst, src, sizeof(int8_t));
2053 }
2054
testb(Register reg,Immediate mask)2055 void Assembler::testb(Register reg, Immediate mask) {
2056 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
2057 emit_test(reg, mask, sizeof(int8_t));
2058 }
2059
testb(const Operand & op,Immediate mask)2060 void Assembler::testb(const Operand& op, Immediate mask) {
2061 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
2062 emit_test(op, mask, sizeof(int8_t));
2063 }
2064
2065
testb(const Operand & op,Register reg)2066 void Assembler::testb(const Operand& op, Register reg) {
2067 emit_test(op, reg, sizeof(int8_t));
2068 }
2069
testw(Register dst,Register src)2070 void Assembler::testw(Register dst, Register src) {
2071 emit_test(dst, src, sizeof(uint16_t));
2072 }
2073
testw(Register reg,Immediate mask)2074 void Assembler::testw(Register reg, Immediate mask) {
2075 emit_test(reg, mask, sizeof(int16_t));
2076 }
2077
testw(const Operand & op,Immediate mask)2078 void Assembler::testw(const Operand& op, Immediate mask) {
2079 emit_test(op, mask, sizeof(int16_t));
2080 }
2081
testw(const Operand & op,Register reg)2082 void Assembler::testw(const Operand& op, Register reg) {
2083 emit_test(op, reg, sizeof(int16_t));
2084 }
2085
emit_test(Register dst,Register src,int size)2086 void Assembler::emit_test(Register dst, Register src, int size) {
2087 EnsureSpace ensure_space(this);
2088 if (src.low_bits() == 4) std::swap(dst, src);
2089 if (size == sizeof(int16_t)) {
2090 emit(0x66);
2091 size = sizeof(int32_t);
2092 }
2093 bool byte_operand = size == sizeof(int8_t);
2094 if (byte_operand) {
2095 size = sizeof(int32_t);
2096 if (!src.is_byte_register() || !dst.is_byte_register()) {
2097 emit_rex_32(dst, src);
2098 }
2099 } else {
2100 emit_rex(dst, src, size);
2101 }
2102 emit(byte_operand ? 0x84 : 0x85);
2103 emit_modrm(dst, src);
2104 }
2105
2106
emit_test(Register reg,Immediate mask,int size)2107 void Assembler::emit_test(Register reg, Immediate mask, int size) {
2108 if (is_uint8(mask.value_)) {
2109 size = sizeof(int8_t);
2110 } else if (is_uint16(mask.value_)) {
2111 size = sizeof(int16_t);
2112 }
2113 EnsureSpace ensure_space(this);
2114 bool half_word = size == sizeof(int16_t);
2115 if (half_word) {
2116 emit(0x66);
2117 size = sizeof(int32_t);
2118 }
2119 bool byte_operand = size == sizeof(int8_t);
2120 if (byte_operand) {
2121 size = sizeof(int32_t);
2122 if (!reg.is_byte_register()) emit_rex_32(reg);
2123 } else {
2124 emit_rex(reg, size);
2125 }
2126 if (reg.is(rax)) {
2127 emit(byte_operand ? 0xA8 : 0xA9);
2128 } else {
2129 emit(byte_operand ? 0xF6 : 0xF7);
2130 emit_modrm(0x0, reg);
2131 }
2132 if (byte_operand) {
2133 emit(mask.value_);
2134 } else if (half_word) {
2135 emitw(mask.value_);
2136 } else {
2137 emit(mask);
2138 }
2139 }
2140
emit_test(const Operand & op,Immediate mask,int size)2141 void Assembler::emit_test(const Operand& op, Immediate mask, int size) {
2142 if (is_uint8(mask.value_)) {
2143 size = sizeof(int8_t);
2144 } else if (is_uint16(mask.value_)) {
2145 size = sizeof(int16_t);
2146 }
2147 EnsureSpace ensure_space(this);
2148 bool half_word = size == sizeof(int16_t);
2149 if (half_word) {
2150 emit(0x66);
2151 size = sizeof(int32_t);
2152 }
2153 bool byte_operand = size == sizeof(int8_t);
2154 if (byte_operand) {
2155 size = sizeof(int32_t);
2156 }
2157 emit_rex(rax, op, size);
2158 emit(byte_operand ? 0xF6 : 0xF7);
2159 emit_operand(rax, op); // Operation code 0
2160 if (byte_operand) {
2161 emit(mask.value_);
2162 } else if (half_word) {
2163 emitw(mask.value_);
2164 } else {
2165 emit(mask);
2166 }
2167 }
2168
emit_test(const Operand & op,Register reg,int size)2169 void Assembler::emit_test(const Operand& op, Register reg, int size) {
2170 EnsureSpace ensure_space(this);
2171 if (size == sizeof(int16_t)) {
2172 emit(0x66);
2173 size = sizeof(int32_t);
2174 }
2175 bool byte_operand = size == sizeof(int8_t);
2176 if (byte_operand) {
2177 size = sizeof(int32_t);
2178 if (!reg.is_byte_register()) {
2179 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2180 emit_rex_32(reg, op);
2181 } else {
2182 emit_optional_rex_32(reg, op);
2183 }
2184 } else {
2185 emit_rex(reg, op, size);
2186 }
2187 emit(byte_operand ? 0x84 : 0x85);
2188 emit_operand(reg, op);
2189 }
2190
2191
2192 // FPU instructions.
2193
2194
fld(int i)2195 void Assembler::fld(int i) {
2196 EnsureSpace ensure_space(this);
2197 emit_farith(0xD9, 0xC0, i);
2198 }
2199
2200
fld1()2201 void Assembler::fld1() {
2202 EnsureSpace ensure_space(this);
2203 emit(0xD9);
2204 emit(0xE8);
2205 }
2206
2207
fldz()2208 void Assembler::fldz() {
2209 EnsureSpace ensure_space(this);
2210 emit(0xD9);
2211 emit(0xEE);
2212 }
2213
2214
fldpi()2215 void Assembler::fldpi() {
2216 EnsureSpace ensure_space(this);
2217 emit(0xD9);
2218 emit(0xEB);
2219 }
2220
2221
fldln2()2222 void Assembler::fldln2() {
2223 EnsureSpace ensure_space(this);
2224 emit(0xD9);
2225 emit(0xED);
2226 }
2227
2228
fld_s(const Operand & adr)2229 void Assembler::fld_s(const Operand& adr) {
2230 EnsureSpace ensure_space(this);
2231 emit_optional_rex_32(adr);
2232 emit(0xD9);
2233 emit_operand(0, adr);
2234 }
2235
2236
fld_d(const Operand & adr)2237 void Assembler::fld_d(const Operand& adr) {
2238 EnsureSpace ensure_space(this);
2239 emit_optional_rex_32(adr);
2240 emit(0xDD);
2241 emit_operand(0, adr);
2242 }
2243
2244
fstp_s(const Operand & adr)2245 void Assembler::fstp_s(const Operand& adr) {
2246 EnsureSpace ensure_space(this);
2247 emit_optional_rex_32(adr);
2248 emit(0xD9);
2249 emit_operand(3, adr);
2250 }
2251
2252
fstp_d(const Operand & adr)2253 void Assembler::fstp_d(const Operand& adr) {
2254 EnsureSpace ensure_space(this);
2255 emit_optional_rex_32(adr);
2256 emit(0xDD);
2257 emit_operand(3, adr);
2258 }
2259
2260
fstp(int index)2261 void Assembler::fstp(int index) {
2262 DCHECK(is_uint3(index));
2263 EnsureSpace ensure_space(this);
2264 emit_farith(0xDD, 0xD8, index);
2265 }
2266
2267
fild_s(const Operand & adr)2268 void Assembler::fild_s(const Operand& adr) {
2269 EnsureSpace ensure_space(this);
2270 emit_optional_rex_32(adr);
2271 emit(0xDB);
2272 emit_operand(0, adr);
2273 }
2274
2275
fild_d(const Operand & adr)2276 void Assembler::fild_d(const Operand& adr) {
2277 EnsureSpace ensure_space(this);
2278 emit_optional_rex_32(adr);
2279 emit(0xDF);
2280 emit_operand(5, adr);
2281 }
2282
2283
fistp_s(const Operand & adr)2284 void Assembler::fistp_s(const Operand& adr) {
2285 EnsureSpace ensure_space(this);
2286 emit_optional_rex_32(adr);
2287 emit(0xDB);
2288 emit_operand(3, adr);
2289 }
2290
2291
fisttp_s(const Operand & adr)2292 void Assembler::fisttp_s(const Operand& adr) {
2293 DCHECK(IsEnabled(SSE3));
2294 EnsureSpace ensure_space(this);
2295 emit_optional_rex_32(adr);
2296 emit(0xDB);
2297 emit_operand(1, adr);
2298 }
2299
2300
fisttp_d(const Operand & adr)2301 void Assembler::fisttp_d(const Operand& adr) {
2302 DCHECK(IsEnabled(SSE3));
2303 EnsureSpace ensure_space(this);
2304 emit_optional_rex_32(adr);
2305 emit(0xDD);
2306 emit_operand(1, adr);
2307 }
2308
2309
fist_s(const Operand & adr)2310 void Assembler::fist_s(const Operand& adr) {
2311 EnsureSpace ensure_space(this);
2312 emit_optional_rex_32(adr);
2313 emit(0xDB);
2314 emit_operand(2, adr);
2315 }
2316
2317
fistp_d(const Operand & adr)2318 void Assembler::fistp_d(const Operand& adr) {
2319 EnsureSpace ensure_space(this);
2320 emit_optional_rex_32(adr);
2321 emit(0xDF);
2322 emit_operand(7, adr);
2323 }
2324
2325
fabs()2326 void Assembler::fabs() {
2327 EnsureSpace ensure_space(this);
2328 emit(0xD9);
2329 emit(0xE1);
2330 }
2331
2332
fchs()2333 void Assembler::fchs() {
2334 EnsureSpace ensure_space(this);
2335 emit(0xD9);
2336 emit(0xE0);
2337 }
2338
2339
fcos()2340 void Assembler::fcos() {
2341 EnsureSpace ensure_space(this);
2342 emit(0xD9);
2343 emit(0xFF);
2344 }
2345
2346
fsin()2347 void Assembler::fsin() {
2348 EnsureSpace ensure_space(this);
2349 emit(0xD9);
2350 emit(0xFE);
2351 }
2352
2353
fptan()2354 void Assembler::fptan() {
2355 EnsureSpace ensure_space(this);
2356 emit(0xD9);
2357 emit(0xF2);
2358 }
2359
2360
fyl2x()2361 void Assembler::fyl2x() {
2362 EnsureSpace ensure_space(this);
2363 emit(0xD9);
2364 emit(0xF1);
2365 }
2366
2367
f2xm1()2368 void Assembler::f2xm1() {
2369 EnsureSpace ensure_space(this);
2370 emit(0xD9);
2371 emit(0xF0);
2372 }
2373
2374
fscale()2375 void Assembler::fscale() {
2376 EnsureSpace ensure_space(this);
2377 emit(0xD9);
2378 emit(0xFD);
2379 }
2380
2381
fninit()2382 void Assembler::fninit() {
2383 EnsureSpace ensure_space(this);
2384 emit(0xDB);
2385 emit(0xE3);
2386 }
2387
2388
fadd(int i)2389 void Assembler::fadd(int i) {
2390 EnsureSpace ensure_space(this);
2391 emit_farith(0xDC, 0xC0, i);
2392 }
2393
2394
fsub(int i)2395 void Assembler::fsub(int i) {
2396 EnsureSpace ensure_space(this);
2397 emit_farith(0xDC, 0xE8, i);
2398 }
2399
2400
fisub_s(const Operand & adr)2401 void Assembler::fisub_s(const Operand& adr) {
2402 EnsureSpace ensure_space(this);
2403 emit_optional_rex_32(adr);
2404 emit(0xDA);
2405 emit_operand(4, adr);
2406 }
2407
2408
fmul(int i)2409 void Assembler::fmul(int i) {
2410 EnsureSpace ensure_space(this);
2411 emit_farith(0xDC, 0xC8, i);
2412 }
2413
2414
fdiv(int i)2415 void Assembler::fdiv(int i) {
2416 EnsureSpace ensure_space(this);
2417 emit_farith(0xDC, 0xF8, i);
2418 }
2419
2420
faddp(int i)2421 void Assembler::faddp(int i) {
2422 EnsureSpace ensure_space(this);
2423 emit_farith(0xDE, 0xC0, i);
2424 }
2425
2426
fsubp(int i)2427 void Assembler::fsubp(int i) {
2428 EnsureSpace ensure_space(this);
2429 emit_farith(0xDE, 0xE8, i);
2430 }
2431
2432
fsubrp(int i)2433 void Assembler::fsubrp(int i) {
2434 EnsureSpace ensure_space(this);
2435 emit_farith(0xDE, 0xE0, i);
2436 }
2437
2438
fmulp(int i)2439 void Assembler::fmulp(int i) {
2440 EnsureSpace ensure_space(this);
2441 emit_farith(0xDE, 0xC8, i);
2442 }
2443
2444
fdivp(int i)2445 void Assembler::fdivp(int i) {
2446 EnsureSpace ensure_space(this);
2447 emit_farith(0xDE, 0xF8, i);
2448 }
2449
2450
fprem()2451 void Assembler::fprem() {
2452 EnsureSpace ensure_space(this);
2453 emit(0xD9);
2454 emit(0xF8);
2455 }
2456
2457
fprem1()2458 void Assembler::fprem1() {
2459 EnsureSpace ensure_space(this);
2460 emit(0xD9);
2461 emit(0xF5);
2462 }
2463
2464
fxch(int i)2465 void Assembler::fxch(int i) {
2466 EnsureSpace ensure_space(this);
2467 emit_farith(0xD9, 0xC8, i);
2468 }
2469
2470
fincstp()2471 void Assembler::fincstp() {
2472 EnsureSpace ensure_space(this);
2473 emit(0xD9);
2474 emit(0xF7);
2475 }
2476
2477
ffree(int i)2478 void Assembler::ffree(int i) {
2479 EnsureSpace ensure_space(this);
2480 emit_farith(0xDD, 0xC0, i);
2481 }
2482
2483
ftst()2484 void Assembler::ftst() {
2485 EnsureSpace ensure_space(this);
2486 emit(0xD9);
2487 emit(0xE4);
2488 }
2489
2490
fucomp(int i)2491 void Assembler::fucomp(int i) {
2492 EnsureSpace ensure_space(this);
2493 emit_farith(0xDD, 0xE8, i);
2494 }
2495
2496
fucompp()2497 void Assembler::fucompp() {
2498 EnsureSpace ensure_space(this);
2499 emit(0xDA);
2500 emit(0xE9);
2501 }
2502
2503
fucomi(int i)2504 void Assembler::fucomi(int i) {
2505 EnsureSpace ensure_space(this);
2506 emit(0xDB);
2507 emit(0xE8 + i);
2508 }
2509
2510
fucomip()2511 void Assembler::fucomip() {
2512 EnsureSpace ensure_space(this);
2513 emit(0xDF);
2514 emit(0xE9);
2515 }
2516
2517
fcompp()2518 void Assembler::fcompp() {
2519 EnsureSpace ensure_space(this);
2520 emit(0xDE);
2521 emit(0xD9);
2522 }
2523
2524
fnstsw_ax()2525 void Assembler::fnstsw_ax() {
2526 EnsureSpace ensure_space(this);
2527 emit(0xDF);
2528 emit(0xE0);
2529 }
2530
2531
fwait()2532 void Assembler::fwait() {
2533 EnsureSpace ensure_space(this);
2534 emit(0x9B);
2535 }
2536
2537
frndint()2538 void Assembler::frndint() {
2539 EnsureSpace ensure_space(this);
2540 emit(0xD9);
2541 emit(0xFC);
2542 }
2543
2544
fnclex()2545 void Assembler::fnclex() {
2546 EnsureSpace ensure_space(this);
2547 emit(0xDB);
2548 emit(0xE2);
2549 }
2550
2551
sahf()2552 void Assembler::sahf() {
2553 // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf
2554 // in 64-bit mode. Test CpuID.
2555 DCHECK(IsEnabled(SAHF));
2556 EnsureSpace ensure_space(this);
2557 emit(0x9E);
2558 }
2559
2560
emit_farith(int b1,int b2,int i)2561 void Assembler::emit_farith(int b1, int b2, int i) {
2562 DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode
2563 DCHECK(is_uint3(i)); // illegal stack offset
2564 emit(b1);
2565 emit(b2 + i);
2566 }
2567
2568
2569 // SSE operations.
2570
andps(XMMRegister dst,XMMRegister src)2571 void Assembler::andps(XMMRegister dst, XMMRegister src) {
2572 EnsureSpace ensure_space(this);
2573 emit_optional_rex_32(dst, src);
2574 emit(0x0F);
2575 emit(0x54);
2576 emit_sse_operand(dst, src);
2577 }
2578
2579
andps(XMMRegister dst,const Operand & src)2580 void Assembler::andps(XMMRegister dst, const Operand& src) {
2581 EnsureSpace ensure_space(this);
2582 emit_optional_rex_32(dst, src);
2583 emit(0x0F);
2584 emit(0x54);
2585 emit_sse_operand(dst, src);
2586 }
2587
2588
orps(XMMRegister dst,XMMRegister src)2589 void Assembler::orps(XMMRegister dst, XMMRegister src) {
2590 EnsureSpace ensure_space(this);
2591 emit_optional_rex_32(dst, src);
2592 emit(0x0F);
2593 emit(0x56);
2594 emit_sse_operand(dst, src);
2595 }
2596
2597
orps(XMMRegister dst,const Operand & src)2598 void Assembler::orps(XMMRegister dst, const Operand& src) {
2599 EnsureSpace ensure_space(this);
2600 emit_optional_rex_32(dst, src);
2601 emit(0x0F);
2602 emit(0x56);
2603 emit_sse_operand(dst, src);
2604 }
2605
2606
xorps(XMMRegister dst,XMMRegister src)2607 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
2608 DCHECK(!IsEnabled(AVX));
2609 EnsureSpace ensure_space(this);
2610 emit_optional_rex_32(dst, src);
2611 emit(0x0F);
2612 emit(0x57);
2613 emit_sse_operand(dst, src);
2614 }
2615
2616
xorps(XMMRegister dst,const Operand & src)2617 void Assembler::xorps(XMMRegister dst, const Operand& src) {
2618 DCHECK(!IsEnabled(AVX));
2619 EnsureSpace ensure_space(this);
2620 emit_optional_rex_32(dst, src);
2621 emit(0x0F);
2622 emit(0x57);
2623 emit_sse_operand(dst, src);
2624 }
2625
2626
addps(XMMRegister dst,XMMRegister src)2627 void Assembler::addps(XMMRegister dst, XMMRegister src) {
2628 EnsureSpace ensure_space(this);
2629 emit_optional_rex_32(dst, src);
2630 emit(0x0F);
2631 emit(0x58);
2632 emit_sse_operand(dst, src);
2633 }
2634
2635
addps(XMMRegister dst,const Operand & src)2636 void Assembler::addps(XMMRegister dst, const Operand& src) {
2637 EnsureSpace ensure_space(this);
2638 emit_optional_rex_32(dst, src);
2639 emit(0x0F);
2640 emit(0x58);
2641 emit_sse_operand(dst, src);
2642 }
2643
2644
subps(XMMRegister dst,XMMRegister src)2645 void Assembler::subps(XMMRegister dst, XMMRegister src) {
2646 EnsureSpace ensure_space(this);
2647 emit_optional_rex_32(dst, src);
2648 emit(0x0F);
2649 emit(0x5C);
2650 emit_sse_operand(dst, src);
2651 }
2652
2653
subps(XMMRegister dst,const Operand & src)2654 void Assembler::subps(XMMRegister dst, const Operand& src) {
2655 EnsureSpace ensure_space(this);
2656 emit_optional_rex_32(dst, src);
2657 emit(0x0F);
2658 emit(0x5C);
2659 emit_sse_operand(dst, src);
2660 }
2661
2662
mulps(XMMRegister dst,XMMRegister src)2663 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
2664 EnsureSpace ensure_space(this);
2665 emit_optional_rex_32(dst, src);
2666 emit(0x0F);
2667 emit(0x59);
2668 emit_sse_operand(dst, src);
2669 }
2670
2671
mulps(XMMRegister dst,const Operand & src)2672 void Assembler::mulps(XMMRegister dst, const Operand& src) {
2673 EnsureSpace ensure_space(this);
2674 emit_optional_rex_32(dst, src);
2675 emit(0x0F);
2676 emit(0x59);
2677 emit_sse_operand(dst, src);
2678 }
2679
2680
divps(XMMRegister dst,XMMRegister src)2681 void Assembler::divps(XMMRegister dst, XMMRegister src) {
2682 EnsureSpace ensure_space(this);
2683 emit_optional_rex_32(dst, src);
2684 emit(0x0F);
2685 emit(0x5E);
2686 emit_sse_operand(dst, src);
2687 }
2688
2689
divps(XMMRegister dst,const Operand & src)2690 void Assembler::divps(XMMRegister dst, const Operand& src) {
2691 EnsureSpace ensure_space(this);
2692 emit_optional_rex_32(dst, src);
2693 emit(0x0F);
2694 emit(0x5E);
2695 emit_sse_operand(dst, src);
2696 }
2697
2698
2699 // SSE 2 operations.
2700
movd(XMMRegister dst,Register src)2701 void Assembler::movd(XMMRegister dst, Register src) {
2702 DCHECK(!IsEnabled(AVX));
2703 EnsureSpace ensure_space(this);
2704 emit(0x66);
2705 emit_optional_rex_32(dst, src);
2706 emit(0x0F);
2707 emit(0x6E);
2708 emit_sse_operand(dst, src);
2709 }
2710
2711
movd(XMMRegister dst,const Operand & src)2712 void Assembler::movd(XMMRegister dst, const Operand& src) {
2713 DCHECK(!IsEnabled(AVX));
2714 EnsureSpace ensure_space(this);
2715 emit(0x66);
2716 emit_optional_rex_32(dst, src);
2717 emit(0x0F);
2718 emit(0x6E);
2719 emit_sse_operand(dst, src);
2720 }
2721
2722
movd(Register dst,XMMRegister src)2723 void Assembler::movd(Register dst, XMMRegister src) {
2724 DCHECK(!IsEnabled(AVX));
2725 EnsureSpace ensure_space(this);
2726 emit(0x66);
2727 emit_optional_rex_32(src, dst);
2728 emit(0x0F);
2729 emit(0x7E);
2730 emit_sse_operand(src, dst);
2731 }
2732
2733
movq(XMMRegister dst,Register src)2734 void Assembler::movq(XMMRegister dst, Register src) {
2735 DCHECK(!IsEnabled(AVX));
2736 EnsureSpace ensure_space(this);
2737 emit(0x66);
2738 emit_rex_64(dst, src);
2739 emit(0x0F);
2740 emit(0x6E);
2741 emit_sse_operand(dst, src);
2742 }
2743
2744
movq(Register dst,XMMRegister src)2745 void Assembler::movq(Register dst, XMMRegister src) {
2746 DCHECK(!IsEnabled(AVX));
2747 EnsureSpace ensure_space(this);
2748 emit(0x66);
2749 emit_rex_64(src, dst);
2750 emit(0x0F);
2751 emit(0x7E);
2752 emit_sse_operand(src, dst);
2753 }
2754
2755
movq(XMMRegister dst,XMMRegister src)2756 void Assembler::movq(XMMRegister dst, XMMRegister src) {
2757 DCHECK(!IsEnabled(AVX));
2758 EnsureSpace ensure_space(this);
2759 if (dst.low_bits() == 4) {
2760 // Avoid unnecessary SIB byte.
2761 emit(0xf3);
2762 emit_optional_rex_32(dst, src);
2763 emit(0x0F);
2764 emit(0x7e);
2765 emit_sse_operand(dst, src);
2766 } else {
2767 emit(0x66);
2768 emit_optional_rex_32(src, dst);
2769 emit(0x0F);
2770 emit(0xD6);
2771 emit_sse_operand(src, dst);
2772 }
2773 }
2774
2775
movdqa(const Operand & dst,XMMRegister src)2776 void Assembler::movdqa(const Operand& dst, XMMRegister src) {
2777 EnsureSpace ensure_space(this);
2778 emit(0x66);
2779 emit_rex_64(src, dst);
2780 emit(0x0F);
2781 emit(0x7F);
2782 emit_sse_operand(src, dst);
2783 }
2784
2785
movdqa(XMMRegister dst,const Operand & src)2786 void Assembler::movdqa(XMMRegister dst, const Operand& src) {
2787 EnsureSpace ensure_space(this);
2788 emit(0x66);
2789 emit_rex_64(dst, src);
2790 emit(0x0F);
2791 emit(0x6F);
2792 emit_sse_operand(dst, src);
2793 }
2794
2795
movdqu(const Operand & dst,XMMRegister src)2796 void Assembler::movdqu(const Operand& dst, XMMRegister src) {
2797 EnsureSpace ensure_space(this);
2798 emit(0xF3);
2799 emit_rex_64(src, dst);
2800 emit(0x0F);
2801 emit(0x7F);
2802 emit_sse_operand(src, dst);
2803 }
2804
2805
movdqu(XMMRegister dst,const Operand & src)2806 void Assembler::movdqu(XMMRegister dst, const Operand& src) {
2807 EnsureSpace ensure_space(this);
2808 emit(0xF3);
2809 emit_rex_64(dst, src);
2810 emit(0x0F);
2811 emit(0x6F);
2812 emit_sse_operand(dst, src);
2813 }
2814
2815
extractps(Register dst,XMMRegister src,byte imm8)2816 void Assembler::extractps(Register dst, XMMRegister src, byte imm8) {
2817 DCHECK(IsEnabled(SSE4_1));
2818 DCHECK(is_uint8(imm8));
2819 EnsureSpace ensure_space(this);
2820 emit(0x66);
2821 emit_optional_rex_32(src, dst);
2822 emit(0x0F);
2823 emit(0x3A);
2824 emit(0x17);
2825 emit_sse_operand(src, dst);
2826 emit(imm8);
2827 }
2828
pextrb(Register dst,XMMRegister src,int8_t imm8)2829 void Assembler::pextrb(Register dst, XMMRegister src, int8_t imm8) {
2830 DCHECK(IsEnabled(SSE4_1));
2831 DCHECK(is_uint8(imm8));
2832 EnsureSpace ensure_space(this);
2833 emit(0x66);
2834 emit_optional_rex_32(src, dst);
2835 emit(0x0F);
2836 emit(0x3A);
2837 emit(0x14);
2838 emit_sse_operand(src, dst);
2839 emit(imm8);
2840 }
2841
pextrb(const Operand & dst,XMMRegister src,int8_t imm8)2842 void Assembler::pextrb(const Operand& dst, XMMRegister src, int8_t imm8) {
2843 DCHECK(IsEnabled(SSE4_1));
2844 DCHECK(is_uint8(imm8));
2845 EnsureSpace ensure_space(this);
2846 emit(0x66);
2847 emit_optional_rex_32(src, dst);
2848 emit(0x0F);
2849 emit(0x3A);
2850 emit(0x14);
2851 emit_sse_operand(src, dst);
2852 emit(imm8);
2853 }
2854
pinsrw(XMMRegister dst,Register src,int8_t imm8)2855 void Assembler::pinsrw(XMMRegister dst, Register src, int8_t imm8) {
2856 DCHECK(is_uint8(imm8));
2857 EnsureSpace ensure_space(this);
2858 emit(0x66);
2859 emit_optional_rex_32(dst, src);
2860 emit(0x0F);
2861 emit(0xC4);
2862 emit_sse_operand(dst, src);
2863 emit(imm8);
2864 }
2865
pinsrw(XMMRegister dst,const Operand & src,int8_t imm8)2866 void Assembler::pinsrw(XMMRegister dst, const Operand& src, int8_t imm8) {
2867 DCHECK(is_uint8(imm8));
2868 EnsureSpace ensure_space(this);
2869 emit(0x66);
2870 emit_optional_rex_32(dst, src);
2871 emit(0x0F);
2872 emit(0xC4);
2873 emit_sse_operand(dst, src);
2874 emit(imm8);
2875 }
2876
pextrw(Register dst,XMMRegister src,int8_t imm8)2877 void Assembler::pextrw(Register dst, XMMRegister src, int8_t imm8) {
2878 DCHECK(is_uint8(imm8));
2879 EnsureSpace ensure_space(this);
2880 emit(0x66);
2881 emit_optional_rex_32(src, dst);
2882 emit(0x0F);
2883 emit(0xC5);
2884 emit_sse_operand(src, dst);
2885 emit(imm8);
2886 }
2887
pextrw(const Operand & dst,XMMRegister src,int8_t imm8)2888 void Assembler::pextrw(const Operand& dst, XMMRegister src, int8_t imm8) {
2889 DCHECK(IsEnabled(SSE4_1));
2890 DCHECK(is_uint8(imm8));
2891 EnsureSpace ensure_space(this);
2892 emit(0x66);
2893 emit_optional_rex_32(src, dst);
2894 emit(0x0F);
2895 emit(0x3A);
2896 emit(0x15);
2897 emit_sse_operand(src, dst);
2898 emit(imm8);
2899 }
2900
pextrd(Register dst,XMMRegister src,int8_t imm8)2901 void Assembler::pextrd(Register dst, XMMRegister src, int8_t imm8) {
2902 DCHECK(IsEnabled(SSE4_1));
2903 EnsureSpace ensure_space(this);
2904 emit(0x66);
2905 emit_optional_rex_32(src, dst);
2906 emit(0x0F);
2907 emit(0x3A);
2908 emit(0x16);
2909 emit_sse_operand(src, dst);
2910 emit(imm8);
2911 }
2912
pextrd(const Operand & dst,XMMRegister src,int8_t imm8)2913 void Assembler::pextrd(const Operand& dst, XMMRegister src, int8_t imm8) {
2914 DCHECK(IsEnabled(SSE4_1));
2915 EnsureSpace ensure_space(this);
2916 emit(0x66);
2917 emit_optional_rex_32(src, dst);
2918 emit(0x0F);
2919 emit(0x3A);
2920 emit(0x16);
2921 emit_sse_operand(src, dst);
2922 emit(imm8);
2923 }
2924
pinsrd(XMMRegister dst,Register src,int8_t imm8)2925 void Assembler::pinsrd(XMMRegister dst, Register src, int8_t imm8) {
2926 DCHECK(IsEnabled(SSE4_1));
2927 EnsureSpace ensure_space(this);
2928 emit(0x66);
2929 emit_optional_rex_32(dst, src);
2930 emit(0x0F);
2931 emit(0x3A);
2932 emit(0x22);
2933 emit_sse_operand(dst, src);
2934 emit(imm8);
2935 }
2936
2937
pinsrd(XMMRegister dst,const Operand & src,int8_t imm8)2938 void Assembler::pinsrd(XMMRegister dst, const Operand& src, int8_t imm8) {
2939 DCHECK(IsEnabled(SSE4_1));
2940 EnsureSpace ensure_space(this);
2941 emit(0x66);
2942 emit_optional_rex_32(dst, src);
2943 emit(0x0F);
2944 emit(0x3A);
2945 emit(0x22);
2946 emit_sse_operand(dst, src);
2947 emit(imm8);
2948 }
2949
pinsrb(XMMRegister dst,Register src,int8_t imm8)2950 void Assembler::pinsrb(XMMRegister dst, Register src, int8_t imm8) {
2951 DCHECK(IsEnabled(SSE4_1));
2952 EnsureSpace ensure_space(this);
2953 emit(0x66);
2954 emit_optional_rex_32(dst, src);
2955 emit(0x0F);
2956 emit(0x3A);
2957 emit(0x20);
2958 emit_sse_operand(dst, src);
2959 emit(imm8);
2960 }
2961
pinsrb(XMMRegister dst,const Operand & src,int8_t imm8)2962 void Assembler::pinsrb(XMMRegister dst, const Operand& src, int8_t imm8) {
2963 DCHECK(IsEnabled(SSE4_1));
2964 EnsureSpace ensure_space(this);
2965 emit(0x66);
2966 emit_optional_rex_32(dst, src);
2967 emit(0x0F);
2968 emit(0x3A);
2969 emit(0x20);
2970 emit_sse_operand(dst, src);
2971 emit(imm8);
2972 }
2973
insertps(XMMRegister dst,XMMRegister src,byte imm8)2974 void Assembler::insertps(XMMRegister dst, XMMRegister src, byte imm8) {
2975 DCHECK(CpuFeatures::IsSupported(SSE4_1));
2976 DCHECK(is_uint8(imm8));
2977 EnsureSpace ensure_space(this);
2978 emit(0x66);
2979 emit_optional_rex_32(dst, src);
2980 emit(0x0F);
2981 emit(0x3A);
2982 emit(0x21);
2983 emit_sse_operand(dst, src);
2984 emit(imm8);
2985 }
2986
movsd(const Operand & dst,XMMRegister src)2987 void Assembler::movsd(const Operand& dst, XMMRegister src) {
2988 DCHECK(!IsEnabled(AVX));
2989 EnsureSpace ensure_space(this);
2990 emit(0xF2); // double
2991 emit_optional_rex_32(src, dst);
2992 emit(0x0F);
2993 emit(0x11); // store
2994 emit_sse_operand(src, dst);
2995 }
2996
2997
movsd(XMMRegister dst,XMMRegister src)2998 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2999 DCHECK(!IsEnabled(AVX));
3000 EnsureSpace ensure_space(this);
3001 emit(0xF2); // double
3002 emit_optional_rex_32(dst, src);
3003 emit(0x0F);
3004 emit(0x10); // load
3005 emit_sse_operand(dst, src);
3006 }
3007
3008
movsd(XMMRegister dst,const Operand & src)3009 void Assembler::movsd(XMMRegister dst, const Operand& src) {
3010 DCHECK(!IsEnabled(AVX));
3011 EnsureSpace ensure_space(this);
3012 emit(0xF2); // double
3013 emit_optional_rex_32(dst, src);
3014 emit(0x0F);
3015 emit(0x10); // load
3016 emit_sse_operand(dst, src);
3017 }
3018
3019
movaps(XMMRegister dst,XMMRegister src)3020 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
3021 DCHECK(!IsEnabled(AVX));
3022 EnsureSpace ensure_space(this);
3023 if (src.low_bits() == 4) {
3024 // Try to avoid an unnecessary SIB byte.
3025 emit_optional_rex_32(src, dst);
3026 emit(0x0F);
3027 emit(0x29);
3028 emit_sse_operand(src, dst);
3029 } else {
3030 emit_optional_rex_32(dst, src);
3031 emit(0x0F);
3032 emit(0x28);
3033 emit_sse_operand(dst, src);
3034 }
3035 }
3036
3037
shufps(XMMRegister dst,XMMRegister src,byte imm8)3038 void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) {
3039 DCHECK(is_uint8(imm8));
3040 EnsureSpace ensure_space(this);
3041 emit_optional_rex_32(dst, src);
3042 emit(0x0F);
3043 emit(0xC6);
3044 emit_sse_operand(dst, src);
3045 emit(imm8);
3046 }
3047
3048
movapd(XMMRegister dst,XMMRegister src)3049 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
3050 DCHECK(!IsEnabled(AVX));
3051 EnsureSpace ensure_space(this);
3052 if (src.low_bits() == 4) {
3053 // Try to avoid an unnecessary SIB byte.
3054 emit(0x66);
3055 emit_optional_rex_32(src, dst);
3056 emit(0x0F);
3057 emit(0x29);
3058 emit_sse_operand(src, dst);
3059 } else {
3060 emit(0x66);
3061 emit_optional_rex_32(dst, src);
3062 emit(0x0F);
3063 emit(0x28);
3064 emit_sse_operand(dst, src);
3065 }
3066 }
3067
3068
movupd(XMMRegister dst,const Operand & src)3069 void Assembler::movupd(XMMRegister dst, const Operand& src) {
3070 EnsureSpace ensure_space(this);
3071 emit(0x66);
3072 emit_optional_rex_32(dst, src);
3073 emit(0x0F);
3074 emit(0x10);
3075 emit_sse_operand(dst, src);
3076 }
3077
movupd(const Operand & dst,XMMRegister src)3078 void Assembler::movupd(const Operand& dst, XMMRegister src) {
3079 EnsureSpace ensure_space(this);
3080 emit(0x66);
3081 emit_optional_rex_32(src, dst);
3082 emit(0x0F);
3083 emit(0x11);
3084 emit_sse_operand(src, dst);
3085 }
3086
addss(XMMRegister dst,XMMRegister src)3087 void Assembler::addss(XMMRegister dst, XMMRegister src) {
3088 EnsureSpace ensure_space(this);
3089 emit(0xF3);
3090 emit_optional_rex_32(dst, src);
3091 emit(0x0F);
3092 emit(0x58);
3093 emit_sse_operand(dst, src);
3094 }
3095
3096
addss(XMMRegister dst,const Operand & src)3097 void Assembler::addss(XMMRegister dst, const Operand& src) {
3098 EnsureSpace ensure_space(this);
3099 emit(0xF3);
3100 emit_optional_rex_32(dst, src);
3101 emit(0x0F);
3102 emit(0x58);
3103 emit_sse_operand(dst, src);
3104 }
3105
3106
subss(XMMRegister dst,XMMRegister src)3107 void Assembler::subss(XMMRegister dst, XMMRegister src) {
3108 EnsureSpace ensure_space(this);
3109 emit(0xF3);
3110 emit_optional_rex_32(dst, src);
3111 emit(0x0F);
3112 emit(0x5C);
3113 emit_sse_operand(dst, src);
3114 }
3115
3116
subss(XMMRegister dst,const Operand & src)3117 void Assembler::subss(XMMRegister dst, const Operand& src) {
3118 EnsureSpace ensure_space(this);
3119 emit(0xF3);
3120 emit_optional_rex_32(dst, src);
3121 emit(0x0F);
3122 emit(0x5C);
3123 emit_sse_operand(dst, src);
3124 }
3125
3126
mulss(XMMRegister dst,XMMRegister src)3127 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
3128 EnsureSpace ensure_space(this);
3129 emit(0xF3);
3130 emit_optional_rex_32(dst, src);
3131 emit(0x0F);
3132 emit(0x59);
3133 emit_sse_operand(dst, src);
3134 }
3135
3136
mulss(XMMRegister dst,const Operand & src)3137 void Assembler::mulss(XMMRegister dst, const Operand& src) {
3138 EnsureSpace ensure_space(this);
3139 emit(0xF3);
3140 emit_optional_rex_32(dst, src);
3141 emit(0x0F);
3142 emit(0x59);
3143 emit_sse_operand(dst, src);
3144 }
3145
3146
divss(XMMRegister dst,XMMRegister src)3147 void Assembler::divss(XMMRegister dst, XMMRegister src) {
3148 EnsureSpace ensure_space(this);
3149 emit(0xF3);
3150 emit_optional_rex_32(dst, src);
3151 emit(0x0F);
3152 emit(0x5E);
3153 emit_sse_operand(dst, src);
3154 }
3155
3156
divss(XMMRegister dst,const Operand & src)3157 void Assembler::divss(XMMRegister dst, const Operand& src) {
3158 EnsureSpace ensure_space(this);
3159 emit(0xF3);
3160 emit_optional_rex_32(dst, src);
3161 emit(0x0F);
3162 emit(0x5E);
3163 emit_sse_operand(dst, src);
3164 }
3165
3166
maxss(XMMRegister dst,XMMRegister src)3167 void Assembler::maxss(XMMRegister dst, XMMRegister src) {
3168 EnsureSpace ensure_space(this);
3169 emit(0xF3);
3170 emit_optional_rex_32(dst, src);
3171 emit(0x0F);
3172 emit(0x5F);
3173 emit_sse_operand(dst, src);
3174 }
3175
3176
maxss(XMMRegister dst,const Operand & src)3177 void Assembler::maxss(XMMRegister dst, const Operand& src) {
3178 EnsureSpace ensure_space(this);
3179 emit(0xF3);
3180 emit_optional_rex_32(dst, src);
3181 emit(0x0F);
3182 emit(0x5F);
3183 emit_sse_operand(dst, src);
3184 }
3185
3186
minss(XMMRegister dst,XMMRegister src)3187 void Assembler::minss(XMMRegister dst, XMMRegister src) {
3188 EnsureSpace ensure_space(this);
3189 emit(0xF3);
3190 emit_optional_rex_32(dst, src);
3191 emit(0x0F);
3192 emit(0x5D);
3193 emit_sse_operand(dst, src);
3194 }
3195
3196
minss(XMMRegister dst,const Operand & src)3197 void Assembler::minss(XMMRegister dst, const Operand& src) {
3198 EnsureSpace ensure_space(this);
3199 emit(0xF3);
3200 emit_optional_rex_32(dst, src);
3201 emit(0x0F);
3202 emit(0x5D);
3203 emit_sse_operand(dst, src);
3204 }
3205
3206
sqrtss(XMMRegister dst,XMMRegister src)3207 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
3208 EnsureSpace ensure_space(this);
3209 emit(0xF3);
3210 emit_optional_rex_32(dst, src);
3211 emit(0x0F);
3212 emit(0x51);
3213 emit_sse_operand(dst, src);
3214 }
3215
3216
sqrtss(XMMRegister dst,const Operand & src)3217 void Assembler::sqrtss(XMMRegister dst, const Operand& src) {
3218 EnsureSpace ensure_space(this);
3219 emit(0xF3);
3220 emit_optional_rex_32(dst, src);
3221 emit(0x0F);
3222 emit(0x51);
3223 emit_sse_operand(dst, src);
3224 }
3225
3226
ucomiss(XMMRegister dst,XMMRegister src)3227 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
3228 DCHECK(!IsEnabled(AVX));
3229 EnsureSpace ensure_space(this);
3230 emit_optional_rex_32(dst, src);
3231 emit(0x0f);
3232 emit(0x2e);
3233 emit_sse_operand(dst, src);
3234 }
3235
3236
ucomiss(XMMRegister dst,const Operand & src)3237 void Assembler::ucomiss(XMMRegister dst, const Operand& src) {
3238 DCHECK(!IsEnabled(AVX));
3239 EnsureSpace ensure_space(this);
3240 emit_optional_rex_32(dst, src);
3241 emit(0x0f);
3242 emit(0x2e);
3243 emit_sse_operand(dst, src);
3244 }
3245
3246
movss(XMMRegister dst,XMMRegister src)3247 void Assembler::movss(XMMRegister dst, XMMRegister src) {
3248 DCHECK(!IsEnabled(AVX));
3249 EnsureSpace ensure_space(this);
3250 emit(0xF3); // single
3251 emit_optional_rex_32(dst, src);
3252 emit(0x0F);
3253 emit(0x10); // load
3254 emit_sse_operand(dst, src);
3255 }
3256
3257
movss(XMMRegister dst,const Operand & src)3258 void Assembler::movss(XMMRegister dst, const Operand& src) {
3259 DCHECK(!IsEnabled(AVX));
3260 EnsureSpace ensure_space(this);
3261 emit(0xF3); // single
3262 emit_optional_rex_32(dst, src);
3263 emit(0x0F);
3264 emit(0x10); // load
3265 emit_sse_operand(dst, src);
3266 }
3267
3268
movss(const Operand & src,XMMRegister dst)3269 void Assembler::movss(const Operand& src, XMMRegister dst) {
3270 DCHECK(!IsEnabled(AVX));
3271 EnsureSpace ensure_space(this);
3272 emit(0xF3); // single
3273 emit_optional_rex_32(dst, src);
3274 emit(0x0F);
3275 emit(0x11); // store
3276 emit_sse_operand(dst, src);
3277 }
3278
3279
psllq(XMMRegister reg,byte imm8)3280 void Assembler::psllq(XMMRegister reg, byte imm8) {
3281 DCHECK(!IsEnabled(AVX));
3282 EnsureSpace ensure_space(this);
3283 emit(0x66);
3284 emit_optional_rex_32(reg);
3285 emit(0x0F);
3286 emit(0x73);
3287 emit_sse_operand(rsi, reg); // rsi == 6
3288 emit(imm8);
3289 }
3290
3291
psrlq(XMMRegister reg,byte imm8)3292 void Assembler::psrlq(XMMRegister reg, byte imm8) {
3293 DCHECK(!IsEnabled(AVX));
3294 EnsureSpace ensure_space(this);
3295 emit(0x66);
3296 emit_optional_rex_32(reg);
3297 emit(0x0F);
3298 emit(0x73);
3299 emit_sse_operand(rdx, reg); // rdx == 2
3300 emit(imm8);
3301 }
3302
psllw(XMMRegister reg,byte imm8)3303 void Assembler::psllw(XMMRegister reg, byte imm8) {
3304 EnsureSpace ensure_space(this);
3305 emit(0x66);
3306 emit_optional_rex_32(reg);
3307 emit(0x0F);
3308 emit(0x71);
3309 emit_sse_operand(rsi, reg); // rsi == 6
3310 emit(imm8);
3311 }
3312
pslld(XMMRegister reg,byte imm8)3313 void Assembler::pslld(XMMRegister reg, byte imm8) {
3314 EnsureSpace ensure_space(this);
3315 emit(0x66);
3316 emit_optional_rex_32(reg);
3317 emit(0x0F);
3318 emit(0x72);
3319 emit_sse_operand(rsi, reg); // rsi == 6
3320 emit(imm8);
3321 }
3322
psrlw(XMMRegister reg,byte imm8)3323 void Assembler::psrlw(XMMRegister reg, byte imm8) {
3324 EnsureSpace ensure_space(this);
3325 emit(0x66);
3326 emit_optional_rex_32(reg);
3327 emit(0x0F);
3328 emit(0x71);
3329 emit_sse_operand(rdx, reg); // rdx == 2
3330 emit(imm8);
3331 }
3332
psrld(XMMRegister reg,byte imm8)3333 void Assembler::psrld(XMMRegister reg, byte imm8) {
3334 EnsureSpace ensure_space(this);
3335 emit(0x66);
3336 emit_optional_rex_32(reg);
3337 emit(0x0F);
3338 emit(0x72);
3339 emit_sse_operand(rdx, reg); // rdx == 2
3340 emit(imm8);
3341 }
3342
psraw(XMMRegister reg,byte imm8)3343 void Assembler::psraw(XMMRegister reg, byte imm8) {
3344 EnsureSpace ensure_space(this);
3345 emit(0x66);
3346 emit_optional_rex_32(reg);
3347 emit(0x0F);
3348 emit(0x71);
3349 emit_sse_operand(rsp, reg); // rsp == 4
3350 emit(imm8);
3351 }
3352
psrad(XMMRegister reg,byte imm8)3353 void Assembler::psrad(XMMRegister reg, byte imm8) {
3354 EnsureSpace ensure_space(this);
3355 emit(0x66);
3356 emit_optional_rex_32(reg);
3357 emit(0x0F);
3358 emit(0x72);
3359 emit_sse_operand(rsp, reg); // rsp == 4
3360 emit(imm8);
3361 }
3362
cmpps(XMMRegister dst,XMMRegister src,int8_t cmp)3363 void Assembler::cmpps(XMMRegister dst, XMMRegister src, int8_t cmp) {
3364 EnsureSpace ensure_space(this);
3365 emit_optional_rex_32(dst, src);
3366 emit(0x0F);
3367 emit(0xC2);
3368 emit_sse_operand(dst, src);
3369 emit(cmp);
3370 }
3371
cmpps(XMMRegister dst,const Operand & src,int8_t cmp)3372 void Assembler::cmpps(XMMRegister dst, const Operand& src, int8_t cmp) {
3373 EnsureSpace ensure_space(this);
3374 emit_optional_rex_32(dst, src);
3375 emit(0x0F);
3376 emit(0xC2);
3377 emit_sse_operand(dst, src);
3378 emit(cmp);
3379 }
3380
cmppd(XMMRegister dst,XMMRegister src,int8_t cmp)3381 void Assembler::cmppd(XMMRegister dst, XMMRegister src, int8_t cmp) {
3382 EnsureSpace ensure_space(this);
3383 emit_optional_rex_32(dst, src);
3384 emit(0x66);
3385 emit(0x0F);
3386 emit(0xC2);
3387 emit_sse_operand(dst, src);
3388 emit(cmp);
3389 }
3390
cmppd(XMMRegister dst,const Operand & src,int8_t cmp)3391 void Assembler::cmppd(XMMRegister dst, const Operand& src, int8_t cmp) {
3392 EnsureSpace ensure_space(this);
3393 emit_optional_rex_32(dst, src);
3394 emit(0x66);
3395 emit(0x0F);
3396 emit(0xC2);
3397 emit_sse_operand(dst, src);
3398 emit(cmp);
3399 }
3400
cvttss2si(Register dst,const Operand & src)3401 void Assembler::cvttss2si(Register dst, const Operand& src) {
3402 DCHECK(!IsEnabled(AVX));
3403 EnsureSpace ensure_space(this);
3404 emit(0xF3);
3405 emit_optional_rex_32(dst, src);
3406 emit(0x0F);
3407 emit(0x2C);
3408 emit_operand(dst, src);
3409 }
3410
3411
cvttss2si(Register dst,XMMRegister src)3412 void Assembler::cvttss2si(Register dst, XMMRegister src) {
3413 DCHECK(!IsEnabled(AVX));
3414 EnsureSpace ensure_space(this);
3415 emit(0xF3);
3416 emit_optional_rex_32(dst, src);
3417 emit(0x0F);
3418 emit(0x2C);
3419 emit_sse_operand(dst, src);
3420 }
3421
3422
cvttsd2si(Register dst,const Operand & src)3423 void Assembler::cvttsd2si(Register dst, const Operand& src) {
3424 DCHECK(!IsEnabled(AVX));
3425 EnsureSpace ensure_space(this);
3426 emit(0xF2);
3427 emit_optional_rex_32(dst, src);
3428 emit(0x0F);
3429 emit(0x2C);
3430 emit_operand(dst, src);
3431 }
3432
3433
cvttsd2si(Register dst,XMMRegister src)3434 void Assembler::cvttsd2si(Register dst, XMMRegister src) {
3435 DCHECK(!IsEnabled(AVX));
3436 EnsureSpace ensure_space(this);
3437 emit(0xF2);
3438 emit_optional_rex_32(dst, src);
3439 emit(0x0F);
3440 emit(0x2C);
3441 emit_sse_operand(dst, src);
3442 }
3443
3444
cvttss2siq(Register dst,XMMRegister src)3445 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
3446 DCHECK(!IsEnabled(AVX));
3447 EnsureSpace ensure_space(this);
3448 emit(0xF3);
3449 emit_rex_64(dst, src);
3450 emit(0x0F);
3451 emit(0x2C);
3452 emit_sse_operand(dst, src);
3453 }
3454
3455
cvttss2siq(Register dst,const Operand & src)3456 void Assembler::cvttss2siq(Register dst, const Operand& src) {
3457 DCHECK(!IsEnabled(AVX));
3458 EnsureSpace ensure_space(this);
3459 emit(0xF3);
3460 emit_rex_64(dst, src);
3461 emit(0x0F);
3462 emit(0x2C);
3463 emit_sse_operand(dst, src);
3464 }
3465
3466
cvttsd2siq(Register dst,XMMRegister src)3467 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
3468 DCHECK(!IsEnabled(AVX));
3469 EnsureSpace ensure_space(this);
3470 emit(0xF2);
3471 emit_rex_64(dst, src);
3472 emit(0x0F);
3473 emit(0x2C);
3474 emit_sse_operand(dst, src);
3475 }
3476
3477
cvttsd2siq(Register dst,const Operand & src)3478 void Assembler::cvttsd2siq(Register dst, const Operand& src) {
3479 DCHECK(!IsEnabled(AVX));
3480 EnsureSpace ensure_space(this);
3481 emit(0xF2);
3482 emit_rex_64(dst, src);
3483 emit(0x0F);
3484 emit(0x2C);
3485 emit_sse_operand(dst, src);
3486 }
3487
3488
cvtlsi2sd(XMMRegister dst,const Operand & src)3489 void Assembler::cvtlsi2sd(XMMRegister dst, const Operand& src) {
3490 DCHECK(!IsEnabled(AVX));
3491 EnsureSpace ensure_space(this);
3492 emit(0xF2);
3493 emit_optional_rex_32(dst, src);
3494 emit(0x0F);
3495 emit(0x2A);
3496 emit_sse_operand(dst, src);
3497 }
3498
3499
cvtlsi2sd(XMMRegister dst,Register src)3500 void Assembler::cvtlsi2sd(XMMRegister dst, Register src) {
3501 DCHECK(!IsEnabled(AVX));
3502 EnsureSpace ensure_space(this);
3503 emit(0xF2);
3504 emit_optional_rex_32(dst, src);
3505 emit(0x0F);
3506 emit(0x2A);
3507 emit_sse_operand(dst, src);
3508 }
3509
3510
cvtlsi2ss(XMMRegister dst,const Operand & src)3511 void Assembler::cvtlsi2ss(XMMRegister dst, const Operand& src) {
3512 DCHECK(!IsEnabled(AVX));
3513 EnsureSpace ensure_space(this);
3514 emit(0xF3);
3515 emit_optional_rex_32(dst, src);
3516 emit(0x0F);
3517 emit(0x2A);
3518 emit_sse_operand(dst, src);
3519 }
3520
3521
cvtlsi2ss(XMMRegister dst,Register src)3522 void Assembler::cvtlsi2ss(XMMRegister dst, Register src) {
3523 EnsureSpace ensure_space(this);
3524 emit(0xF3);
3525 emit_optional_rex_32(dst, src);
3526 emit(0x0F);
3527 emit(0x2A);
3528 emit_sse_operand(dst, src);
3529 }
3530
3531
cvtqsi2ss(XMMRegister dst,const Operand & src)3532 void Assembler::cvtqsi2ss(XMMRegister dst, const Operand& src) {
3533 DCHECK(!IsEnabled(AVX));
3534 EnsureSpace ensure_space(this);
3535 emit(0xF3);
3536 emit_rex_64(dst, src);
3537 emit(0x0F);
3538 emit(0x2A);
3539 emit_sse_operand(dst, src);
3540 }
3541
3542
cvtqsi2ss(XMMRegister dst,Register src)3543 void Assembler::cvtqsi2ss(XMMRegister dst, Register src) {
3544 DCHECK(!IsEnabled(AVX));
3545 EnsureSpace ensure_space(this);
3546 emit(0xF3);
3547 emit_rex_64(dst, src);
3548 emit(0x0F);
3549 emit(0x2A);
3550 emit_sse_operand(dst, src);
3551 }
3552
3553
cvtqsi2sd(XMMRegister dst,const Operand & src)3554 void Assembler::cvtqsi2sd(XMMRegister dst, const Operand& src) {
3555 DCHECK(!IsEnabled(AVX));
3556 EnsureSpace ensure_space(this);
3557 emit(0xF2);
3558 emit_rex_64(dst, src);
3559 emit(0x0F);
3560 emit(0x2A);
3561 emit_sse_operand(dst, src);
3562 }
3563
3564
cvtqsi2sd(XMMRegister dst,Register src)3565 void Assembler::cvtqsi2sd(XMMRegister dst, Register src) {
3566 DCHECK(!IsEnabled(AVX));
3567 EnsureSpace ensure_space(this);
3568 emit(0xF2);
3569 emit_rex_64(dst, src);
3570 emit(0x0F);
3571 emit(0x2A);
3572 emit_sse_operand(dst, src);
3573 }
3574
3575
cvtss2sd(XMMRegister dst,XMMRegister src)3576 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
3577 DCHECK(!IsEnabled(AVX));
3578 EnsureSpace ensure_space(this);
3579 emit(0xF3);
3580 emit_optional_rex_32(dst, src);
3581 emit(0x0F);
3582 emit(0x5A);
3583 emit_sse_operand(dst, src);
3584 }
3585
3586
cvtss2sd(XMMRegister dst,const Operand & src)3587 void Assembler::cvtss2sd(XMMRegister dst, const Operand& src) {
3588 DCHECK(!IsEnabled(AVX));
3589 EnsureSpace ensure_space(this);
3590 emit(0xF3);
3591 emit_optional_rex_32(dst, src);
3592 emit(0x0F);
3593 emit(0x5A);
3594 emit_sse_operand(dst, src);
3595 }
3596
3597
cvtsd2ss(XMMRegister dst,XMMRegister src)3598 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
3599 DCHECK(!IsEnabled(AVX));
3600 EnsureSpace ensure_space(this);
3601 emit(0xF2);
3602 emit_optional_rex_32(dst, src);
3603 emit(0x0F);
3604 emit(0x5A);
3605 emit_sse_operand(dst, src);
3606 }
3607
3608
cvtsd2ss(XMMRegister dst,const Operand & src)3609 void Assembler::cvtsd2ss(XMMRegister dst, const Operand& src) {
3610 DCHECK(!IsEnabled(AVX));
3611 EnsureSpace ensure_space(this);
3612 emit(0xF2);
3613 emit_optional_rex_32(dst, src);
3614 emit(0x0F);
3615 emit(0x5A);
3616 emit_sse_operand(dst, src);
3617 }
3618
3619
cvtsd2si(Register dst,XMMRegister src)3620 void Assembler::cvtsd2si(Register dst, XMMRegister src) {
3621 DCHECK(!IsEnabled(AVX));
3622 EnsureSpace ensure_space(this);
3623 emit(0xF2);
3624 emit_optional_rex_32(dst, src);
3625 emit(0x0F);
3626 emit(0x2D);
3627 emit_sse_operand(dst, src);
3628 }
3629
3630
cvtsd2siq(Register dst,XMMRegister src)3631 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
3632 DCHECK(!IsEnabled(AVX));
3633 EnsureSpace ensure_space(this);
3634 emit(0xF2);
3635 emit_rex_64(dst, src);
3636 emit(0x0F);
3637 emit(0x2D);
3638 emit_sse_operand(dst, src);
3639 }
3640
3641
addsd(XMMRegister dst,XMMRegister src)3642 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
3643 EnsureSpace ensure_space(this);
3644 emit(0xF2);
3645 emit_optional_rex_32(dst, src);
3646 emit(0x0F);
3647 emit(0x58);
3648 emit_sse_operand(dst, src);
3649 }
3650
3651
addsd(XMMRegister dst,const Operand & src)3652 void Assembler::addsd(XMMRegister dst, const Operand& src) {
3653 EnsureSpace ensure_space(this);
3654 emit(0xF2);
3655 emit_optional_rex_32(dst, src);
3656 emit(0x0F);
3657 emit(0x58);
3658 emit_sse_operand(dst, src);
3659 }
3660
3661
mulsd(XMMRegister dst,XMMRegister src)3662 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
3663 EnsureSpace ensure_space(this);
3664 emit(0xF2);
3665 emit_optional_rex_32(dst, src);
3666 emit(0x0F);
3667 emit(0x59);
3668 emit_sse_operand(dst, src);
3669 }
3670
3671
mulsd(XMMRegister dst,const Operand & src)3672 void Assembler::mulsd(XMMRegister dst, const Operand& src) {
3673 EnsureSpace ensure_space(this);
3674 emit(0xF2);
3675 emit_optional_rex_32(dst, src);
3676 emit(0x0F);
3677 emit(0x59);
3678 emit_sse_operand(dst, src);
3679 }
3680
3681
subsd(XMMRegister dst,XMMRegister src)3682 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
3683 EnsureSpace ensure_space(this);
3684 emit(0xF2);
3685 emit_optional_rex_32(dst, src);
3686 emit(0x0F);
3687 emit(0x5C);
3688 emit_sse_operand(dst, src);
3689 }
3690
3691
subsd(XMMRegister dst,const Operand & src)3692 void Assembler::subsd(XMMRegister dst, const Operand& src) {
3693 EnsureSpace ensure_space(this);
3694 emit(0xF2);
3695 emit_optional_rex_32(dst, src);
3696 emit(0x0F);
3697 emit(0x5C);
3698 emit_sse_operand(dst, src);
3699 }
3700
3701
divsd(XMMRegister dst,XMMRegister src)3702 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
3703 EnsureSpace ensure_space(this);
3704 emit(0xF2);
3705 emit_optional_rex_32(dst, src);
3706 emit(0x0F);
3707 emit(0x5E);
3708 emit_sse_operand(dst, src);
3709 }
3710
3711
divsd(XMMRegister dst,const Operand & src)3712 void Assembler::divsd(XMMRegister dst, const Operand& src) {
3713 EnsureSpace ensure_space(this);
3714 emit(0xF2);
3715 emit_optional_rex_32(dst, src);
3716 emit(0x0F);
3717 emit(0x5E);
3718 emit_sse_operand(dst, src);
3719 }
3720
3721
maxsd(XMMRegister dst,XMMRegister src)3722 void Assembler::maxsd(XMMRegister dst, XMMRegister src) {
3723 EnsureSpace ensure_space(this);
3724 emit(0xF2);
3725 emit_optional_rex_32(dst, src);
3726 emit(0x0F);
3727 emit(0x5F);
3728 emit_sse_operand(dst, src);
3729 }
3730
3731
maxsd(XMMRegister dst,const Operand & src)3732 void Assembler::maxsd(XMMRegister dst, const Operand& src) {
3733 EnsureSpace ensure_space(this);
3734 emit(0xF2);
3735 emit_optional_rex_32(dst, src);
3736 emit(0x0F);
3737 emit(0x5F);
3738 emit_sse_operand(dst, src);
3739 }
3740
3741
minsd(XMMRegister dst,XMMRegister src)3742 void Assembler::minsd(XMMRegister dst, XMMRegister src) {
3743 EnsureSpace ensure_space(this);
3744 emit(0xF2);
3745 emit_optional_rex_32(dst, src);
3746 emit(0x0F);
3747 emit(0x5D);
3748 emit_sse_operand(dst, src);
3749 }
3750
3751
minsd(XMMRegister dst,const Operand & src)3752 void Assembler::minsd(XMMRegister dst, const Operand& src) {
3753 EnsureSpace ensure_space(this);
3754 emit(0xF2);
3755 emit_optional_rex_32(dst, src);
3756 emit(0x0F);
3757 emit(0x5D);
3758 emit_sse_operand(dst, src);
3759 }
3760
3761
andpd(XMMRegister dst,XMMRegister src)3762 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
3763 EnsureSpace ensure_space(this);
3764 emit(0x66);
3765 emit_optional_rex_32(dst, src);
3766 emit(0x0F);
3767 emit(0x54);
3768 emit_sse_operand(dst, src);
3769 }
3770
3771
andpd(XMMRegister dst,const Operand & src)3772 void Assembler::andpd(XMMRegister dst, const Operand& src) {
3773 EnsureSpace ensure_space(this);
3774 emit(0x66);
3775 emit_optional_rex_32(dst, src);
3776 emit(0x0F);
3777 emit(0x54);
3778 emit_sse_operand(dst, src);
3779 }
3780
3781
orpd(XMMRegister dst,XMMRegister src)3782 void Assembler::orpd(XMMRegister dst, XMMRegister src) {
3783 EnsureSpace ensure_space(this);
3784 emit(0x66);
3785 emit_optional_rex_32(dst, src);
3786 emit(0x0F);
3787 emit(0x56);
3788 emit_sse_operand(dst, src);
3789 }
3790
3791
orpd(XMMRegister dst,const Operand & src)3792 void Assembler::orpd(XMMRegister dst, const Operand& src) {
3793 EnsureSpace ensure_space(this);
3794 emit(0x66);
3795 emit_optional_rex_32(dst, src);
3796 emit(0x0F);
3797 emit(0x56);
3798 emit_sse_operand(dst, src);
3799 }
3800
3801
xorpd(XMMRegister dst,XMMRegister src)3802 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
3803 DCHECK(!IsEnabled(AVX));
3804 EnsureSpace ensure_space(this);
3805 emit(0x66);
3806 emit_optional_rex_32(dst, src);
3807 emit(0x0F);
3808 emit(0x57);
3809 emit_sse_operand(dst, src);
3810 }
3811
3812
xorpd(XMMRegister dst,const Operand & src)3813 void Assembler::xorpd(XMMRegister dst, const Operand& src) {
3814 DCHECK(!IsEnabled(AVX));
3815 EnsureSpace ensure_space(this);
3816 emit(0x66);
3817 emit_optional_rex_32(dst, src);
3818 emit(0x0F);
3819 emit(0x57);
3820 emit_sse_operand(dst, src);
3821 }
3822
3823
sqrtsd(XMMRegister dst,XMMRegister src)3824 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
3825 DCHECK(!IsEnabled(AVX));
3826 EnsureSpace ensure_space(this);
3827 emit(0xF2);
3828 emit_optional_rex_32(dst, src);
3829 emit(0x0F);
3830 emit(0x51);
3831 emit_sse_operand(dst, src);
3832 }
3833
3834
sqrtsd(XMMRegister dst,const Operand & src)3835 void Assembler::sqrtsd(XMMRegister dst, const Operand& src) {
3836 DCHECK(!IsEnabled(AVX));
3837 EnsureSpace ensure_space(this);
3838 emit(0xF2);
3839 emit_optional_rex_32(dst, src);
3840 emit(0x0F);
3841 emit(0x51);
3842 emit_sse_operand(dst, src);
3843 }
3844
3845
ucomisd(XMMRegister dst,XMMRegister src)3846 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
3847 DCHECK(!IsEnabled(AVX));
3848 EnsureSpace ensure_space(this);
3849 emit(0x66);
3850 emit_optional_rex_32(dst, src);
3851 emit(0x0f);
3852 emit(0x2e);
3853 emit_sse_operand(dst, src);
3854 }
3855
3856
ucomisd(XMMRegister dst,const Operand & src)3857 void Assembler::ucomisd(XMMRegister dst, const Operand& src) {
3858 DCHECK(!IsEnabled(AVX));
3859 EnsureSpace ensure_space(this);
3860 emit(0x66);
3861 emit_optional_rex_32(dst, src);
3862 emit(0x0f);
3863 emit(0x2e);
3864 emit_sse_operand(dst, src);
3865 }
3866
3867
cmpltsd(XMMRegister dst,XMMRegister src)3868 void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) {
3869 EnsureSpace ensure_space(this);
3870 emit(0xF2);
3871 emit_optional_rex_32(dst, src);
3872 emit(0x0F);
3873 emit(0xC2);
3874 emit_sse_operand(dst, src);
3875 emit(0x01); // LT == 1
3876 }
3877
3878
roundss(XMMRegister dst,XMMRegister src,RoundingMode mode)3879 void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3880 DCHECK(!IsEnabled(AVX));
3881 DCHECK(IsEnabled(SSE4_1));
3882 EnsureSpace ensure_space(this);
3883 emit(0x66);
3884 emit_optional_rex_32(dst, src);
3885 emit(0x0f);
3886 emit(0x3a);
3887 emit(0x0a);
3888 emit_sse_operand(dst, src);
3889 // Mask precision exception.
3890 emit(static_cast<byte>(mode) | 0x8);
3891 }
3892
3893
roundsd(XMMRegister dst,XMMRegister src,RoundingMode mode)3894 void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3895 DCHECK(!IsEnabled(AVX));
3896 DCHECK(IsEnabled(SSE4_1));
3897 EnsureSpace ensure_space(this);
3898 emit(0x66);
3899 emit_optional_rex_32(dst, src);
3900 emit(0x0f);
3901 emit(0x3a);
3902 emit(0x0b);
3903 emit_sse_operand(dst, src);
3904 // Mask precision exception.
3905 emit(static_cast<byte>(mode) | 0x8);
3906 }
3907
3908
movmskpd(Register dst,XMMRegister src)3909 void Assembler::movmskpd(Register dst, XMMRegister src) {
3910 EnsureSpace ensure_space(this);
3911 emit(0x66);
3912 emit_optional_rex_32(dst, src);
3913 emit(0x0f);
3914 emit(0x50);
3915 emit_sse_operand(dst, src);
3916 }
3917
3918
movmskps(Register dst,XMMRegister src)3919 void Assembler::movmskps(Register dst, XMMRegister src) {
3920 EnsureSpace ensure_space(this);
3921 emit_optional_rex_32(dst, src);
3922 emit(0x0f);
3923 emit(0x50);
3924 emit_sse_operand(dst, src);
3925 }
3926
3927
punpckldq(XMMRegister dst,XMMRegister src)3928 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
3929 EnsureSpace ensure_space(this);
3930 emit(0x66);
3931 emit_optional_rex_32(dst, src);
3932 emit(0x0F);
3933 emit(0x62);
3934 emit_sse_operand(dst, src);
3935 }
3936
punpckldq(XMMRegister dst,const Operand & src)3937 void Assembler::punpckldq(XMMRegister dst, const Operand& src) {
3938 EnsureSpace ensure_space(this);
3939 emit(0x66);
3940 emit_optional_rex_32(dst, src);
3941 emit(0x0F);
3942 emit(0x62);
3943 emit_sse_operand(dst, src);
3944 }
3945
punpckhdq(XMMRegister dst,XMMRegister src)3946 void Assembler::punpckhdq(XMMRegister dst, XMMRegister src) {
3947 EnsureSpace ensure_space(this);
3948 emit(0x66);
3949 emit_optional_rex_32(dst, src);
3950 emit(0x0F);
3951 emit(0x6A);
3952 emit_sse_operand(dst, src);
3953 }
3954
3955
3956 // AVX instructions
vfmasd(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)3957 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1,
3958 XMMRegister src2) {
3959 DCHECK(IsEnabled(FMA3));
3960 EnsureSpace ensure_space(this);
3961 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1);
3962 emit(op);
3963 emit_sse_operand(dst, src2);
3964 }
3965
3966
vfmasd(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)3967 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1,
3968 const Operand& src2) {
3969 DCHECK(IsEnabled(FMA3));
3970 EnsureSpace ensure_space(this);
3971 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1);
3972 emit(op);
3973 emit_sse_operand(dst, src2);
3974 }
3975
3976
vfmass(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)3977 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1,
3978 XMMRegister src2) {
3979 DCHECK(IsEnabled(FMA3));
3980 EnsureSpace ensure_space(this);
3981 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0);
3982 emit(op);
3983 emit_sse_operand(dst, src2);
3984 }
3985
3986
vfmass(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)3987 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1,
3988 const Operand& src2) {
3989 DCHECK(IsEnabled(FMA3));
3990 EnsureSpace ensure_space(this);
3991 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0);
3992 emit(op);
3993 emit_sse_operand(dst, src2);
3994 }
3995
3996
vmovd(XMMRegister dst,Register src)3997 void Assembler::vmovd(XMMRegister dst, Register src) {
3998 DCHECK(IsEnabled(AVX));
3999 EnsureSpace ensure_space(this);
4000 XMMRegister isrc = {src.code()};
4001 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW0);
4002 emit(0x6e);
4003 emit_sse_operand(dst, src);
4004 }
4005
4006
vmovd(XMMRegister dst,const Operand & src)4007 void Assembler::vmovd(XMMRegister dst, const Operand& src) {
4008 DCHECK(IsEnabled(AVX));
4009 EnsureSpace ensure_space(this);
4010 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW0);
4011 emit(0x6e);
4012 emit_sse_operand(dst, src);
4013 }
4014
4015
vmovd(Register dst,XMMRegister src)4016 void Assembler::vmovd(Register dst, XMMRegister src) {
4017 DCHECK(IsEnabled(AVX));
4018 EnsureSpace ensure_space(this);
4019 XMMRegister idst = {dst.code()};
4020 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW0);
4021 emit(0x7e);
4022 emit_sse_operand(src, dst);
4023 }
4024
4025
vmovq(XMMRegister dst,Register src)4026 void Assembler::vmovq(XMMRegister dst, Register src) {
4027 DCHECK(IsEnabled(AVX));
4028 EnsureSpace ensure_space(this);
4029 XMMRegister isrc = {src.code()};
4030 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW1);
4031 emit(0x6e);
4032 emit_sse_operand(dst, src);
4033 }
4034
4035
vmovq(XMMRegister dst,const Operand & src)4036 void Assembler::vmovq(XMMRegister dst, const Operand& src) {
4037 DCHECK(IsEnabled(AVX));
4038 EnsureSpace ensure_space(this);
4039 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW1);
4040 emit(0x6e);
4041 emit_sse_operand(dst, src);
4042 }
4043
4044
vmovq(Register dst,XMMRegister src)4045 void Assembler::vmovq(Register dst, XMMRegister src) {
4046 DCHECK(IsEnabled(AVX));
4047 EnsureSpace ensure_space(this);
4048 XMMRegister idst = {dst.code()};
4049 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW1);
4050 emit(0x7e);
4051 emit_sse_operand(src, dst);
4052 }
4053
vinstr(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2,SIMDPrefix pp,LeadingOpcode m,VexW w)4054 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1,
4055 XMMRegister src2, SIMDPrefix pp, LeadingOpcode m,
4056 VexW w) {
4057 DCHECK(IsEnabled(AVX));
4058 EnsureSpace ensure_space(this);
4059 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
4060 emit(op);
4061 emit_sse_operand(dst, src2);
4062 }
4063
vinstr(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2,SIMDPrefix pp,LeadingOpcode m,VexW w)4064 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1,
4065 const Operand& src2, SIMDPrefix pp, LeadingOpcode m,
4066 VexW w) {
4067 DCHECK(IsEnabled(AVX));
4068 EnsureSpace ensure_space(this);
4069 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
4070 emit(op);
4071 emit_sse_operand(dst, src2);
4072 }
4073
4074
vps(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)4075 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
4076 XMMRegister src2) {
4077 DCHECK(IsEnabled(AVX));
4078 EnsureSpace ensure_space(this);
4079 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
4080 emit(op);
4081 emit_sse_operand(dst, src2);
4082 }
4083
4084
vps(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)4085 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
4086 const Operand& src2) {
4087 DCHECK(IsEnabled(AVX));
4088 EnsureSpace ensure_space(this);
4089 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
4090 emit(op);
4091 emit_sse_operand(dst, src2);
4092 }
4093
4094
vpd(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)4095 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1,
4096 XMMRegister src2) {
4097 DCHECK(IsEnabled(AVX));
4098 EnsureSpace ensure_space(this);
4099 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
4100 emit(op);
4101 emit_sse_operand(dst, src2);
4102 }
4103
4104
vpd(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)4105 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1,
4106 const Operand& src2) {
4107 DCHECK(IsEnabled(AVX));
4108 EnsureSpace ensure_space(this);
4109 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
4110 emit(op);
4111 emit_sse_operand(dst, src2);
4112 }
4113
4114
vucomiss(XMMRegister dst,XMMRegister src)4115 void Assembler::vucomiss(XMMRegister dst, XMMRegister src) {
4116 DCHECK(IsEnabled(AVX));
4117 EnsureSpace ensure_space(this);
4118 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
4119 emit(0x2e);
4120 emit_sse_operand(dst, src);
4121 }
4122
4123
vucomiss(XMMRegister dst,const Operand & src)4124 void Assembler::vucomiss(XMMRegister dst, const Operand& src) {
4125 DCHECK(IsEnabled(AVX));
4126 EnsureSpace ensure_space(this);
4127 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
4128 emit(0x2e);
4129 emit_sse_operand(dst, src);
4130 }
4131
4132
vss(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)4133 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1,
4134 XMMRegister src2) {
4135 DCHECK(IsEnabled(AVX));
4136 EnsureSpace ensure_space(this);
4137 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
4138 emit(op);
4139 emit_sse_operand(dst, src2);
4140 }
4141
4142
vss(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)4143 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1,
4144 const Operand& src2) {
4145 DCHECK(IsEnabled(AVX));
4146 EnsureSpace ensure_space(this);
4147 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
4148 emit(op);
4149 emit_sse_operand(dst, src2);
4150 }
4151
4152
bmi1q(byte op,Register reg,Register vreg,Register rm)4153 void Assembler::bmi1q(byte op, Register reg, Register vreg, Register rm) {
4154 DCHECK(IsEnabled(BMI1));
4155 EnsureSpace ensure_space(this);
4156 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
4157 emit(op);
4158 emit_modrm(reg, rm);
4159 }
4160
4161
bmi1q(byte op,Register reg,Register vreg,const Operand & rm)4162 void Assembler::bmi1q(byte op, Register reg, Register vreg, const Operand& rm) {
4163 DCHECK(IsEnabled(BMI1));
4164 EnsureSpace ensure_space(this);
4165 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
4166 emit(op);
4167 emit_operand(reg, rm);
4168 }
4169
4170
bmi1l(byte op,Register reg,Register vreg,Register rm)4171 void Assembler::bmi1l(byte op, Register reg, Register vreg, Register rm) {
4172 DCHECK(IsEnabled(BMI1));
4173 EnsureSpace ensure_space(this);
4174 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
4175 emit(op);
4176 emit_modrm(reg, rm);
4177 }
4178
4179
bmi1l(byte op,Register reg,Register vreg,const Operand & rm)4180 void Assembler::bmi1l(byte op, Register reg, Register vreg, const Operand& rm) {
4181 DCHECK(IsEnabled(BMI1));
4182 EnsureSpace ensure_space(this);
4183 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
4184 emit(op);
4185 emit_operand(reg, rm);
4186 }
4187
4188
tzcntq(Register dst,Register src)4189 void Assembler::tzcntq(Register dst, Register src) {
4190 DCHECK(IsEnabled(BMI1));
4191 EnsureSpace ensure_space(this);
4192 emit(0xF3);
4193 emit_rex_64(dst, src);
4194 emit(0x0F);
4195 emit(0xBC);
4196 emit_modrm(dst, src);
4197 }
4198
4199
tzcntq(Register dst,const Operand & src)4200 void Assembler::tzcntq(Register dst, const Operand& src) {
4201 DCHECK(IsEnabled(BMI1));
4202 EnsureSpace ensure_space(this);
4203 emit(0xF3);
4204 emit_rex_64(dst, src);
4205 emit(0x0F);
4206 emit(0xBC);
4207 emit_operand(dst, src);
4208 }
4209
4210
tzcntl(Register dst,Register src)4211 void Assembler::tzcntl(Register dst, Register src) {
4212 DCHECK(IsEnabled(BMI1));
4213 EnsureSpace ensure_space(this);
4214 emit(0xF3);
4215 emit_optional_rex_32(dst, src);
4216 emit(0x0F);
4217 emit(0xBC);
4218 emit_modrm(dst, src);
4219 }
4220
4221
tzcntl(Register dst,const Operand & src)4222 void Assembler::tzcntl(Register dst, const Operand& src) {
4223 DCHECK(IsEnabled(BMI1));
4224 EnsureSpace ensure_space(this);
4225 emit(0xF3);
4226 emit_optional_rex_32(dst, src);
4227 emit(0x0F);
4228 emit(0xBC);
4229 emit_operand(dst, src);
4230 }
4231
4232
lzcntq(Register dst,Register src)4233 void Assembler::lzcntq(Register dst, Register src) {
4234 DCHECK(IsEnabled(LZCNT));
4235 EnsureSpace ensure_space(this);
4236 emit(0xF3);
4237 emit_rex_64(dst, src);
4238 emit(0x0F);
4239 emit(0xBD);
4240 emit_modrm(dst, src);
4241 }
4242
4243
lzcntq(Register dst,const Operand & src)4244 void Assembler::lzcntq(Register dst, const Operand& src) {
4245 DCHECK(IsEnabled(LZCNT));
4246 EnsureSpace ensure_space(this);
4247 emit(0xF3);
4248 emit_rex_64(dst, src);
4249 emit(0x0F);
4250 emit(0xBD);
4251 emit_operand(dst, src);
4252 }
4253
4254
lzcntl(Register dst,Register src)4255 void Assembler::lzcntl(Register dst, Register src) {
4256 DCHECK(IsEnabled(LZCNT));
4257 EnsureSpace ensure_space(this);
4258 emit(0xF3);
4259 emit_optional_rex_32(dst, src);
4260 emit(0x0F);
4261 emit(0xBD);
4262 emit_modrm(dst, src);
4263 }
4264
4265
lzcntl(Register dst,const Operand & src)4266 void Assembler::lzcntl(Register dst, const Operand& src) {
4267 DCHECK(IsEnabled(LZCNT));
4268 EnsureSpace ensure_space(this);
4269 emit(0xF3);
4270 emit_optional_rex_32(dst, src);
4271 emit(0x0F);
4272 emit(0xBD);
4273 emit_operand(dst, src);
4274 }
4275
4276
popcntq(Register dst,Register src)4277 void Assembler::popcntq(Register dst, Register src) {
4278 DCHECK(IsEnabled(POPCNT));
4279 EnsureSpace ensure_space(this);
4280 emit(0xF3);
4281 emit_rex_64(dst, src);
4282 emit(0x0F);
4283 emit(0xB8);
4284 emit_modrm(dst, src);
4285 }
4286
4287
popcntq(Register dst,const Operand & src)4288 void Assembler::popcntq(Register dst, const Operand& src) {
4289 DCHECK(IsEnabled(POPCNT));
4290 EnsureSpace ensure_space(this);
4291 emit(0xF3);
4292 emit_rex_64(dst, src);
4293 emit(0x0F);
4294 emit(0xB8);
4295 emit_operand(dst, src);
4296 }
4297
4298
popcntl(Register dst,Register src)4299 void Assembler::popcntl(Register dst, Register src) {
4300 DCHECK(IsEnabled(POPCNT));
4301 EnsureSpace ensure_space(this);
4302 emit(0xF3);
4303 emit_optional_rex_32(dst, src);
4304 emit(0x0F);
4305 emit(0xB8);
4306 emit_modrm(dst, src);
4307 }
4308
4309
popcntl(Register dst,const Operand & src)4310 void Assembler::popcntl(Register dst, const Operand& src) {
4311 DCHECK(IsEnabled(POPCNT));
4312 EnsureSpace ensure_space(this);
4313 emit(0xF3);
4314 emit_optional_rex_32(dst, src);
4315 emit(0x0F);
4316 emit(0xB8);
4317 emit_operand(dst, src);
4318 }
4319
4320
bmi2q(SIMDPrefix pp,byte op,Register reg,Register vreg,Register rm)4321 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
4322 Register rm) {
4323 DCHECK(IsEnabled(BMI2));
4324 EnsureSpace ensure_space(this);
4325 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
4326 emit(op);
4327 emit_modrm(reg, rm);
4328 }
4329
4330
bmi2q(SIMDPrefix pp,byte op,Register reg,Register vreg,const Operand & rm)4331 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
4332 const Operand& rm) {
4333 DCHECK(IsEnabled(BMI2));
4334 EnsureSpace ensure_space(this);
4335 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
4336 emit(op);
4337 emit_operand(reg, rm);
4338 }
4339
4340
bmi2l(SIMDPrefix pp,byte op,Register reg,Register vreg,Register rm)4341 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
4342 Register rm) {
4343 DCHECK(IsEnabled(BMI2));
4344 EnsureSpace ensure_space(this);
4345 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
4346 emit(op);
4347 emit_modrm(reg, rm);
4348 }
4349
4350
bmi2l(SIMDPrefix pp,byte op,Register reg,Register vreg,const Operand & rm)4351 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
4352 const Operand& rm) {
4353 DCHECK(IsEnabled(BMI2));
4354 EnsureSpace ensure_space(this);
4355 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
4356 emit(op);
4357 emit_operand(reg, rm);
4358 }
4359
4360
rorxq(Register dst,Register src,byte imm8)4361 void Assembler::rorxq(Register dst, Register src, byte imm8) {
4362 DCHECK(IsEnabled(BMI2));
4363 DCHECK(is_uint8(imm8));
4364 Register vreg = {0}; // VEX.vvvv unused
4365 EnsureSpace ensure_space(this);
4366 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
4367 emit(0xF0);
4368 emit_modrm(dst, src);
4369 emit(imm8);
4370 }
4371
4372
rorxq(Register dst,const Operand & src,byte imm8)4373 void Assembler::rorxq(Register dst, const Operand& src, byte imm8) {
4374 DCHECK(IsEnabled(BMI2));
4375 DCHECK(is_uint8(imm8));
4376 Register vreg = {0}; // VEX.vvvv unused
4377 EnsureSpace ensure_space(this);
4378 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
4379 emit(0xF0);
4380 emit_operand(dst, src);
4381 emit(imm8);
4382 }
4383
4384
rorxl(Register dst,Register src,byte imm8)4385 void Assembler::rorxl(Register dst, Register src, byte imm8) {
4386 DCHECK(IsEnabled(BMI2));
4387 DCHECK(is_uint8(imm8));
4388 Register vreg = {0}; // VEX.vvvv unused
4389 EnsureSpace ensure_space(this);
4390 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
4391 emit(0xF0);
4392 emit_modrm(dst, src);
4393 emit(imm8);
4394 }
4395
4396
rorxl(Register dst,const Operand & src,byte imm8)4397 void Assembler::rorxl(Register dst, const Operand& src, byte imm8) {
4398 DCHECK(IsEnabled(BMI2));
4399 DCHECK(is_uint8(imm8));
4400 Register vreg = {0}; // VEX.vvvv unused
4401 EnsureSpace ensure_space(this);
4402 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
4403 emit(0xF0);
4404 emit_operand(dst, src);
4405 emit(imm8);
4406 }
4407
minps(XMMRegister dst,XMMRegister src)4408 void Assembler::minps(XMMRegister dst, XMMRegister src) {
4409 EnsureSpace ensure_space(this);
4410 emit_optional_rex_32(dst, src);
4411 emit(0x0F);
4412 emit(0x5D);
4413 emit_sse_operand(dst, src);
4414 }
4415
minps(XMMRegister dst,const Operand & src)4416 void Assembler::minps(XMMRegister dst, const Operand& src) {
4417 EnsureSpace ensure_space(this);
4418 emit_optional_rex_32(dst, src);
4419 emit(0x0F);
4420 emit(0x5D);
4421 emit_sse_operand(dst, src);
4422 }
4423
maxps(XMMRegister dst,XMMRegister src)4424 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
4425 EnsureSpace ensure_space(this);
4426 emit_optional_rex_32(dst, src);
4427 emit(0x0F);
4428 emit(0x5F);
4429 emit_sse_operand(dst, src);
4430 }
4431
maxps(XMMRegister dst,const Operand & src)4432 void Assembler::maxps(XMMRegister dst, const Operand& src) {
4433 EnsureSpace ensure_space(this);
4434 emit_optional_rex_32(dst, src);
4435 emit(0x0F);
4436 emit(0x5F);
4437 emit_sse_operand(dst, src);
4438 }
4439
rcpps(XMMRegister dst,XMMRegister src)4440 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
4441 EnsureSpace ensure_space(this);
4442 emit_optional_rex_32(dst, src);
4443 emit(0x0F);
4444 emit(0x53);
4445 emit_sse_operand(dst, src);
4446 }
4447
rcpps(XMMRegister dst,const Operand & src)4448 void Assembler::rcpps(XMMRegister dst, const Operand& src) {
4449 EnsureSpace ensure_space(this);
4450 emit_optional_rex_32(dst, src);
4451 emit(0x0F);
4452 emit(0x53);
4453 emit_sse_operand(dst, src);
4454 }
4455
rsqrtps(XMMRegister dst,XMMRegister src)4456 void Assembler::rsqrtps(XMMRegister dst, XMMRegister src) {
4457 EnsureSpace ensure_space(this);
4458 emit_optional_rex_32(dst, src);
4459 emit(0x0F);
4460 emit(0x52);
4461 emit_sse_operand(dst, src);
4462 }
4463
rsqrtps(XMMRegister dst,const Operand & src)4464 void Assembler::rsqrtps(XMMRegister dst, const Operand& src) {
4465 EnsureSpace ensure_space(this);
4466 emit_optional_rex_32(dst, src);
4467 emit(0x0F);
4468 emit(0x52);
4469 emit_sse_operand(dst, src);
4470 }
4471
sqrtps(XMMRegister dst,XMMRegister src)4472 void Assembler::sqrtps(XMMRegister dst, XMMRegister src) {
4473 EnsureSpace ensure_space(this);
4474 emit_optional_rex_32(dst, src);
4475 emit(0x0F);
4476 emit(0x51);
4477 emit_sse_operand(dst, src);
4478 }
4479
sqrtps(XMMRegister dst,const Operand & src)4480 void Assembler::sqrtps(XMMRegister dst, const Operand& src) {
4481 EnsureSpace ensure_space(this);
4482 emit_optional_rex_32(dst, src);
4483 emit(0x0F);
4484 emit(0x51);
4485 emit_sse_operand(dst, src);
4486 }
4487
cvtdq2ps(XMMRegister dst,XMMRegister src)4488 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
4489 EnsureSpace ensure_space(this);
4490 emit_optional_rex_32(dst, src);
4491 emit(0x0F);
4492 emit(0x5B);
4493 emit_sse_operand(dst, src);
4494 }
4495
cvtdq2ps(XMMRegister dst,const Operand & src)4496 void Assembler::cvtdq2ps(XMMRegister dst, const Operand& src) {
4497 EnsureSpace ensure_space(this);
4498 emit_optional_rex_32(dst, src);
4499 emit(0x0F);
4500 emit(0x5B);
4501 emit_sse_operand(dst, src);
4502 }
4503
movups(XMMRegister dst,XMMRegister src)4504 void Assembler::movups(XMMRegister dst, XMMRegister src) {
4505 EnsureSpace ensure_space(this);
4506 if (src.low_bits() == 4) {
4507 // Try to avoid an unnecessary SIB byte.
4508 emit_optional_rex_32(src, dst);
4509 emit(0x0F);
4510 emit(0x11);
4511 emit_sse_operand(src, dst);
4512 } else {
4513 emit_optional_rex_32(dst, src);
4514 emit(0x0F);
4515 emit(0x10);
4516 emit_sse_operand(dst, src);
4517 }
4518 }
4519
movups(XMMRegister dst,const Operand & src)4520 void Assembler::movups(XMMRegister dst, const Operand& src) {
4521 EnsureSpace ensure_space(this);
4522 emit_optional_rex_32(dst, src);
4523 emit(0x0F);
4524 emit(0x10);
4525 emit_sse_operand(dst, src);
4526 }
4527
movups(const Operand & dst,XMMRegister src)4528 void Assembler::movups(const Operand& dst, XMMRegister src) {
4529 EnsureSpace ensure_space(this);
4530 emit_optional_rex_32(src, dst);
4531 emit(0x0F);
4532 emit(0x11);
4533 emit_sse_operand(src, dst);
4534 }
4535
sse2_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape,byte opcode)4536 void Assembler::sse2_instr(XMMRegister dst, XMMRegister src, byte prefix,
4537 byte escape, byte opcode) {
4538 EnsureSpace ensure_space(this);
4539 emit(prefix);
4540 emit_optional_rex_32(dst, src);
4541 emit(escape);
4542 emit(opcode);
4543 emit_sse_operand(dst, src);
4544 }
4545
sse2_instr(XMMRegister dst,const Operand & src,byte prefix,byte escape,byte opcode)4546 void Assembler::sse2_instr(XMMRegister dst, const Operand& src, byte prefix,
4547 byte escape, byte opcode) {
4548 EnsureSpace ensure_space(this);
4549 emit(prefix);
4550 emit_optional_rex_32(dst, src);
4551 emit(escape);
4552 emit(opcode);
4553 emit_sse_operand(dst, src);
4554 }
4555
ssse3_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode)4556 void Assembler::ssse3_instr(XMMRegister dst, XMMRegister src, byte prefix,
4557 byte escape1, byte escape2, byte opcode) {
4558 DCHECK(IsEnabled(SSSE3));
4559 EnsureSpace ensure_space(this);
4560 emit(prefix);
4561 emit_optional_rex_32(dst, src);
4562 emit(escape1);
4563 emit(escape2);
4564 emit(opcode);
4565 emit_sse_operand(dst, src);
4566 }
4567
ssse3_instr(XMMRegister dst,const Operand & src,byte prefix,byte escape1,byte escape2,byte opcode)4568 void Assembler::ssse3_instr(XMMRegister dst, const Operand& src, byte prefix,
4569 byte escape1, byte escape2, byte opcode) {
4570 DCHECK(IsEnabled(SSSE3));
4571 EnsureSpace ensure_space(this);
4572 emit(prefix);
4573 emit_optional_rex_32(dst, src);
4574 emit(escape1);
4575 emit(escape2);
4576 emit(opcode);
4577 emit_sse_operand(dst, src);
4578 }
4579
sse4_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode)4580 void Assembler::sse4_instr(XMMRegister dst, XMMRegister src, byte prefix,
4581 byte escape1, byte escape2, byte opcode) {
4582 DCHECK(IsEnabled(SSE4_1));
4583 EnsureSpace ensure_space(this);
4584 emit(prefix);
4585 emit_optional_rex_32(dst, src);
4586 emit(escape1);
4587 emit(escape2);
4588 emit(opcode);
4589 emit_sse_operand(dst, src);
4590 }
4591
sse4_instr(XMMRegister dst,const Operand & src,byte prefix,byte escape1,byte escape2,byte opcode)4592 void Assembler::sse4_instr(XMMRegister dst, const Operand& src, byte prefix,
4593 byte escape1, byte escape2, byte opcode) {
4594 DCHECK(IsEnabled(SSE4_1));
4595 EnsureSpace ensure_space(this);
4596 emit(prefix);
4597 emit_optional_rex_32(dst, src);
4598 emit(escape1);
4599 emit(escape2);
4600 emit(opcode);
4601 emit_sse_operand(dst, src);
4602 }
4603
lddqu(XMMRegister dst,const Operand & src)4604 void Assembler::lddqu(XMMRegister dst, const Operand& src) {
4605 DCHECK(IsEnabled(SSE3));
4606 EnsureSpace ensure_space(this);
4607 emit(0xF2);
4608 emit_optional_rex_32(dst, src);
4609 emit(0x0F);
4610 emit(0xF0);
4611 emit_sse_operand(dst, src);
4612 }
4613
psrldq(XMMRegister dst,uint8_t shift)4614 void Assembler::psrldq(XMMRegister dst, uint8_t shift) {
4615 EnsureSpace ensure_space(this);
4616 emit(0x66);
4617 emit_optional_rex_32(dst);
4618 emit(0x0F);
4619 emit(0x73);
4620 emit_sse_operand(dst);
4621 emit(shift);
4622 }
4623
pshufd(XMMRegister dst,XMMRegister src,uint8_t shuffle)4624 void Assembler::pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4625 EnsureSpace ensure_space(this);
4626 emit(0x66);
4627 emit_optional_rex_32(dst, src);
4628 emit(0x0F);
4629 emit(0x70);
4630 emit_sse_operand(dst, src);
4631 emit(shuffle);
4632 }
4633
pshufd(XMMRegister dst,const Operand & src,uint8_t shuffle)4634 void Assembler::pshufd(XMMRegister dst, const Operand& src, uint8_t shuffle) {
4635 EnsureSpace ensure_space(this);
4636 emit(0x66);
4637 emit_optional_rex_32(dst, src);
4638 emit(0x0F);
4639 emit(0x70);
4640 emit_sse_operand(dst, src);
4641 emit(shuffle);
4642 }
4643
emit_sse_operand(XMMRegister reg,const Operand & adr)4644 void Assembler::emit_sse_operand(XMMRegister reg, const Operand& adr) {
4645 Register ireg = { reg.code() };
4646 emit_operand(ireg, adr);
4647 }
4648
4649
emit_sse_operand(Register reg,const Operand & adr)4650 void Assembler::emit_sse_operand(Register reg, const Operand& adr) {
4651 Register ireg = {reg.code()};
4652 emit_operand(ireg, adr);
4653 }
4654
4655
emit_sse_operand(XMMRegister dst,XMMRegister src)4656 void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) {
4657 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4658 }
4659
4660
emit_sse_operand(XMMRegister dst,Register src)4661 void Assembler::emit_sse_operand(XMMRegister dst, Register src) {
4662 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4663 }
4664
4665
emit_sse_operand(Register dst,XMMRegister src)4666 void Assembler::emit_sse_operand(Register dst, XMMRegister src) {
4667 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4668 }
4669
emit_sse_operand(XMMRegister dst)4670 void Assembler::emit_sse_operand(XMMRegister dst) {
4671 emit(0xD8 | dst.low_bits());
4672 }
4673
RecordProtectedInstructionLanding(int pc_offset)4674 void Assembler::RecordProtectedInstructionLanding(int pc_offset) {
4675 EnsureSpace ensure_space(this);
4676 RelocInfo rinfo(isolate(), pc(),
4677 RelocInfo::WASM_PROTECTED_INSTRUCTION_LANDING, pc_offset,
4678 nullptr);
4679 reloc_info_writer.Write(&rinfo);
4680 }
4681
4682
db(uint8_t data)4683 void Assembler::db(uint8_t data) {
4684 EnsureSpace ensure_space(this);
4685 emit(data);
4686 }
4687
4688
dd(uint32_t data)4689 void Assembler::dd(uint32_t data) {
4690 EnsureSpace ensure_space(this);
4691 emitl(data);
4692 }
4693
4694
dq(uint64_t data)4695 void Assembler::dq(uint64_t data) {
4696 EnsureSpace ensure_space(this);
4697 emitq(data);
4698 }
4699
4700
dq(Label * label)4701 void Assembler::dq(Label* label) {
4702 EnsureSpace ensure_space(this);
4703 if (label->is_bound()) {
4704 internal_reference_positions_.push_back(pc_offset());
4705 emitp(buffer_ + label->pos(), RelocInfo::INTERNAL_REFERENCE);
4706 } else {
4707 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
4708 emitl(0); // Zero for the first 32bit marks it as 64bit absolute address.
4709 if (label->is_linked()) {
4710 emitl(label->pos());
4711 label->link_to(pc_offset() - sizeof(int32_t));
4712 } else {
4713 DCHECK(label->is_unused());
4714 int32_t current = pc_offset();
4715 emitl(current);
4716 label->link_to(current);
4717 }
4718 }
4719 }
4720
4721
4722 // Relocation information implementations.
4723
RecordRelocInfo(RelocInfo::Mode rmode,intptr_t data)4724 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
4725 DCHECK(!RelocInfo::IsNone(rmode));
4726 // Don't record external references unless the heap will be serialized.
4727 if (rmode == RelocInfo::EXTERNAL_REFERENCE &&
4728 !serializer_enabled() && !emit_debug_code()) {
4729 return;
4730 } else if (rmode == RelocInfo::CODE_AGE_SEQUENCE) {
4731 // Don't record psuedo relocation info for code age sequence mode.
4732 return;
4733 }
4734 RelocInfo rinfo(isolate(), pc_, rmode, data, NULL);
4735 reloc_info_writer.Write(&rinfo);
4736 }
4737
4738
4739 const int RelocInfo::kApplyMask = RelocInfo::kCodeTargetMask |
4740 1 << RelocInfo::RUNTIME_ENTRY |
4741 1 << RelocInfo::INTERNAL_REFERENCE |
4742 1 << RelocInfo::CODE_AGE_SEQUENCE;
4743
4744
IsCodedSpecially()4745 bool RelocInfo::IsCodedSpecially() {
4746 // The deserializer needs to know whether a pointer is specially coded. Being
4747 // specially coded on x64 means that it is a relative 32 bit address, as used
4748 // by branch instructions.
4749 return (1 << rmode_) & kApplyMask;
4750 }
4751
4752
IsInConstantPool()4753 bool RelocInfo::IsInConstantPool() {
4754 return false;
4755 }
4756
4757
4758 } // namespace internal
4759 } // namespace v8
4760
4761 #endif // V8_TARGET_ARCH_X64
4762