1 // Copyright 2012 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/x64/assembler-x64.h"
6
7 #include <cstring>
8
9 #if V8_TARGET_ARCH_X64
10
11 #if V8_LIBC_MSVCRT
12 #include <intrin.h> // _xgetbv()
13 #endif
14 #if V8_OS_MACOSX
15 #include <sys/sysctl.h>
16 #endif
17
18 #include "src/assembler-inl.h"
19 #include "src/base/bits.h"
20 #include "src/base/cpu.h"
21 #include "src/code-stubs.h"
22 #include "src/deoptimizer.h"
23 #include "src/macro-assembler.h"
24 #include "src/v8.h"
25
26 namespace v8 {
27 namespace internal {
28
29 // -----------------------------------------------------------------------------
30 // Implementation of CpuFeatures
31
32 namespace {
33
34 #if !V8_LIBC_MSVCRT
35
_xgetbv(unsigned int xcr)36 V8_INLINE uint64_t _xgetbv(unsigned int xcr) {
37 unsigned eax, edx;
38 // Check xgetbv; this uses a .byte sequence instead of the instruction
39 // directly because older assemblers do not include support for xgetbv and
40 // there is no easy way to conditionally compile based on the assembler
41 // used.
42 __asm__ volatile(".byte 0x0F, 0x01, 0xD0" : "=a"(eax), "=d"(edx) : "c"(xcr));
43 return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32);
44 }
45
46 #define _XCR_XFEATURE_ENABLED_MASK 0
47
48 #endif // !V8_LIBC_MSVCRT
49
50
OSHasAVXSupport()51 bool OSHasAVXSupport() {
52 #if V8_OS_MACOSX
53 // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being
54 // caused by ISRs, so we detect that here and disable AVX in that case.
55 char buffer[128];
56 size_t buffer_size = arraysize(buffer);
57 int ctl_name[] = {CTL_KERN, KERN_OSRELEASE};
58 if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) {
59 FATAL("V8 failed to get kernel version");
60 }
61 // The buffer now contains a string of the form XX.YY.ZZ, where
62 // XX is the major kernel version component.
63 char* period_pos = strchr(buffer, '.');
64 DCHECK_NOT_NULL(period_pos);
65 *period_pos = '\0';
66 long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT
67 if (kernel_version_major <= 13) return false;
68 #endif // V8_OS_MACOSX
69 // Check whether OS claims to support AVX.
70 uint64_t feature_mask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
71 return (feature_mask & 0x6) == 0x6;
72 }
73
74 } // namespace
75
76
ProbeImpl(bool cross_compile)77 void CpuFeatures::ProbeImpl(bool cross_compile) {
78 base::CPU cpu;
79 CHECK(cpu.has_sse2()); // SSE2 support is mandatory.
80 CHECK(cpu.has_cmov()); // CMOV support is mandatory.
81
82 // Only use statically determined features for cross compile (snapshot).
83 if (cross_compile) return;
84
85 if (cpu.has_sse41() && FLAG_enable_sse4_1) supported_ |= 1u << SSE4_1;
86 if (cpu.has_ssse3() && FLAG_enable_ssse3) supported_ |= 1u << SSSE3;
87 if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3;
88 // SAHF is not generally available in long mode.
89 if (cpu.has_sahf() && FLAG_enable_sahf) supported_ |= 1u << SAHF;
90 if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() &&
91 OSHasAVXSupport()) {
92 supported_ |= 1u << AVX;
93 }
94 if (cpu.has_fma3() && FLAG_enable_fma3 && cpu.has_osxsave() &&
95 OSHasAVXSupport()) {
96 supported_ |= 1u << FMA3;
97 }
98 if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1;
99 if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2;
100 if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT;
101 if (cpu.has_popcnt() && FLAG_enable_popcnt) supported_ |= 1u << POPCNT;
102 if (strcmp(FLAG_mcpu, "auto") == 0) {
103 if (cpu.is_atom()) supported_ |= 1u << ATOM;
104 } else if (strcmp(FLAG_mcpu, "atom") == 0) {
105 supported_ |= 1u << ATOM;
106 }
107 }
108
109
PrintTarget()110 void CpuFeatures::PrintTarget() { }
PrintFeatures()111 void CpuFeatures::PrintFeatures() {
112 printf(
113 "SSE3=%d SSSE3=%d SSE4_1=%d SAHF=%d AVX=%d FMA3=%d BMI1=%d BMI2=%d "
114 "LZCNT=%d "
115 "POPCNT=%d ATOM=%d\n",
116 CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSSE3),
117 CpuFeatures::IsSupported(SSE4_1), CpuFeatures::IsSupported(SAHF),
118 CpuFeatures::IsSupported(AVX), CpuFeatures::IsSupported(FMA3),
119 CpuFeatures::IsSupported(BMI1), CpuFeatures::IsSupported(BMI2),
120 CpuFeatures::IsSupported(LZCNT), CpuFeatures::IsSupported(POPCNT),
121 CpuFeatures::IsSupported(ATOM));
122 }
123
124 // -----------------------------------------------------------------------------
125 // Implementation of RelocInfo
126
set_js_to_wasm_address(Address address,ICacheFlushMode icache_flush_mode)127 void RelocInfo::set_js_to_wasm_address(Address address,
128 ICacheFlushMode icache_flush_mode) {
129 DCHECK_EQ(rmode_, JS_TO_WASM_CALL);
130 Memory<Address>(pc_) = address;
131 if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
132 Assembler::FlushICache(pc_, sizeof(Address));
133 }
134 }
135
js_to_wasm_address() const136 Address RelocInfo::js_to_wasm_address() const {
137 DCHECK_EQ(rmode_, JS_TO_WASM_CALL);
138 return Memory<Address>(pc_);
139 }
140
wasm_call_tag() const141 uint32_t RelocInfo::wasm_call_tag() const {
142 DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL);
143 return Memory<uint32_t>(pc_);
144 }
145
146 // -----------------------------------------------------------------------------
147 // Implementation of Operand
148
149 namespace {
150 class OperandBuilder {
151 public:
OperandBuilder(Register base,int32_t disp)152 OperandBuilder(Register base, int32_t disp) {
153 if (base == rsp || base == r12) {
154 // SIB byte is needed to encode (rsp + offset) or (r12 + offset).
155 set_sib(times_1, rsp, base);
156 }
157
158 if (disp == 0 && base != rbp && base != r13) {
159 set_modrm(0, base);
160 } else if (is_int8(disp)) {
161 set_modrm(1, base);
162 set_disp8(disp);
163 } else {
164 set_modrm(2, base);
165 set_disp32(disp);
166 }
167 }
168
OperandBuilder(Register base,Register index,ScaleFactor scale,int32_t disp)169 OperandBuilder(Register base, Register index, ScaleFactor scale,
170 int32_t disp) {
171 DCHECK(index != rsp);
172 set_sib(scale, index, base);
173 if (disp == 0 && base != rbp && base != r13) {
174 // This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits
175 // possibly set by set_sib.
176 set_modrm(0, rsp);
177 } else if (is_int8(disp)) {
178 set_modrm(1, rsp);
179 set_disp8(disp);
180 } else {
181 set_modrm(2, rsp);
182 set_disp32(disp);
183 }
184 }
185
OperandBuilder(Register index,ScaleFactor scale,int32_t disp)186 OperandBuilder(Register index, ScaleFactor scale, int32_t disp) {
187 DCHECK(index != rsp);
188 set_modrm(0, rsp);
189 set_sib(scale, index, rbp);
190 set_disp32(disp);
191 }
192
OperandBuilder(Label * label,int addend)193 OperandBuilder(Label* label, int addend) {
194 data_.addend = addend;
195 DCHECK_NOT_NULL(label);
196 DCHECK(addend == 0 || (is_int8(addend) && label->is_bound()));
197 set_modrm(0, rbp);
198 set_disp64(reinterpret_cast<intptr_t>(label));
199 }
200
OperandBuilder(Operand operand,int32_t offset)201 OperandBuilder(Operand operand, int32_t offset) {
202 DCHECK_GE(operand.data().len, 1);
203 // Operand encodes REX ModR/M [SIB] [Disp].
204 byte modrm = operand.data().buf[0];
205 DCHECK_LT(modrm, 0xC0); // Disallow mode 3 (register target).
206 bool has_sib = ((modrm & 0x07) == 0x04);
207 byte mode = modrm & 0xC0;
208 int disp_offset = has_sib ? 2 : 1;
209 int base_reg = (has_sib ? operand.data().buf[1] : modrm) & 0x07;
210 // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit
211 // displacement.
212 bool is_baseless =
213 (mode == 0) && (base_reg == 0x05); // No base or RIP base.
214 int32_t disp_value = 0;
215 if (mode == 0x80 || is_baseless) {
216 // Mode 2 or mode 0 with rbp/r13 as base: Word displacement.
217 disp_value = *bit_cast<const int32_t*>(&operand.data().buf[disp_offset]);
218 } else if (mode == 0x40) {
219 // Mode 1: Byte displacement.
220 disp_value = static_cast<signed char>(operand.data().buf[disp_offset]);
221 }
222
223 // Write new operand with same registers, but with modified displacement.
224 DCHECK(offset >= 0 ? disp_value + offset > disp_value
225 : disp_value + offset < disp_value); // No overflow.
226 disp_value += offset;
227 data_.rex = operand.data().rex;
228 if (!is_int8(disp_value) || is_baseless) {
229 // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13.
230 data_.buf[0] = (modrm & 0x3F) | (is_baseless ? 0x00 : 0x80);
231 data_.len = disp_offset + 4;
232 Memory<int32_t>(reinterpret_cast<Address>(&data_.buf[disp_offset])) =
233 disp_value;
234 } else if (disp_value != 0 || (base_reg == 0x05)) {
235 // Need 8 bits of displacement.
236 data_.buf[0] = (modrm & 0x3F) | 0x40; // Mode 1.
237 data_.len = disp_offset + 1;
238 data_.buf[disp_offset] = static_cast<byte>(disp_value);
239 } else {
240 // Need no displacement.
241 data_.buf[0] = (modrm & 0x3F); // Mode 0.
242 data_.len = disp_offset;
243 }
244 if (has_sib) {
245 data_.buf[1] = operand.data().buf[1];
246 }
247 }
248
set_modrm(int mod,Register rm_reg)249 void set_modrm(int mod, Register rm_reg) {
250 DCHECK(is_uint2(mod));
251 data_.buf[0] = mod << 6 | rm_reg.low_bits();
252 // Set REX.B to the high bit of rm.code().
253 data_.rex |= rm_reg.high_bit();
254 }
255
set_sib(ScaleFactor scale,Register index,Register base)256 void set_sib(ScaleFactor scale, Register index, Register base) {
257 DCHECK_EQ(data_.len, 1);
258 DCHECK(is_uint2(scale));
259 // Use SIB with no index register only for base rsp or r12. Otherwise we
260 // would skip the SIB byte entirely.
261 DCHECK(index != rsp || base == rsp || base == r12);
262 data_.buf[1] = (scale << 6) | (index.low_bits() << 3) | base.low_bits();
263 data_.rex |= index.high_bit() << 1 | base.high_bit();
264 data_.len = 2;
265 }
266
set_disp8(int disp)267 void set_disp8(int disp) {
268 DCHECK(is_int8(disp));
269 DCHECK(data_.len == 1 || data_.len == 2);
270 int8_t* p = reinterpret_cast<int8_t*>(&data_.buf[data_.len]);
271 *p = disp;
272 data_.len += sizeof(int8_t);
273 }
274
set_disp32(int disp)275 void set_disp32(int disp) {
276 DCHECK(data_.len == 1 || data_.len == 2);
277 int32_t* p = reinterpret_cast<int32_t*>(&data_.buf[data_.len]);
278 *p = disp;
279 data_.len += sizeof(int32_t);
280 }
281
set_disp64(int64_t disp)282 void set_disp64(int64_t disp) {
283 DCHECK_EQ(1, data_.len);
284 int64_t* p = reinterpret_cast<int64_t*>(&data_.buf[data_.len]);
285 *p = disp;
286 data_.len += sizeof(disp);
287 }
288
data() const289 const Operand::Data& data() const { return data_; }
290
291 private:
292 Operand::Data data_;
293 };
294 } // namespace
295
Operand(Register base,int32_t disp)296 Operand::Operand(Register base, int32_t disp)
297 : data_(OperandBuilder(base, disp).data()) {}
298
Operand(Register base,Register index,ScaleFactor scale,int32_t disp)299 Operand::Operand(Register base, Register index, ScaleFactor scale, int32_t disp)
300 : data_(OperandBuilder(base, index, scale, disp).data()) {}
301
Operand(Register index,ScaleFactor scale,int32_t disp)302 Operand::Operand(Register index, ScaleFactor scale, int32_t disp)
303 : data_(OperandBuilder(index, scale, disp).data()) {}
304
Operand(Label * label,int addend)305 Operand::Operand(Label* label, int addend)
306 : data_(OperandBuilder(label, addend).data()) {}
307
Operand(Operand operand,int32_t offset)308 Operand::Operand(Operand operand, int32_t offset)
309 : data_(OperandBuilder(operand, offset).data()) {}
310
AddressUsesRegister(Register reg) const311 bool Operand::AddressUsesRegister(Register reg) const {
312 int code = reg.code();
313 DCHECK_NE(data_.buf[0] & 0xC0, 0xC0); // Always a memory operand.
314 // Start with only low three bits of base register. Initial decoding
315 // doesn't distinguish on the REX.B bit.
316 int base_code = data_.buf[0] & 0x07;
317 if (base_code == rsp.code()) {
318 // SIB byte present in buf_[1].
319 // Check the index register from the SIB byte + REX.X prefix.
320 int index_code = ((data_.buf[1] >> 3) & 0x07) | ((data_.rex & 0x02) << 2);
321 // Index code (including REX.X) of 0x04 (rsp) means no index register.
322 if (index_code != rsp.code() && index_code == code) return true;
323 // Add REX.B to get the full base register code.
324 base_code = (data_.buf[1] & 0x07) | ((data_.rex & 0x01) << 3);
325 // A base register of 0x05 (rbp) with mod = 0 means no base register.
326 if (base_code == rbp.code() && ((data_.buf[0] & 0xC0) == 0)) return false;
327 return code == base_code;
328 } else {
329 // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means
330 // no base register.
331 if (base_code == rbp.code() && ((data_.buf[0] & 0xC0) == 0)) return false;
332 base_code |= ((data_.rex & 0x01) << 3);
333 return code == base_code;
334 }
335 }
336
AllocateAndInstallRequestedHeapObjects(Isolate * isolate)337 void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
338 for (auto& request : heap_object_requests_) {
339 Address pc = reinterpret_cast<Address>(buffer_) + request.offset();
340 switch (request.kind()) {
341 case HeapObjectRequest::kHeapNumber: {
342 Handle<HeapNumber> object =
343 isolate->factory()->NewHeapNumber(request.heap_number(), TENURED);
344 Memory<Handle<Object>>(pc) = object;
345 break;
346 }
347 case HeapObjectRequest::kCodeStub: {
348 request.code_stub()->set_isolate(isolate);
349 UpdateCodeTarget(Memory<int32_t>(pc), request.code_stub()->GetCode());
350 break;
351 }
352 }
353 }
354 }
355
356 // Partial Constant Pool.
AddSharedEntry(uint64_t data,int offset)357 bool ConstPool::AddSharedEntry(uint64_t data, int offset) {
358 auto existing = entries_.find(data);
359 if (existing == entries_.end()) {
360 entries_.insert(std::make_pair(data, offset + kMoveImm64Offset));
361 return false;
362 }
363
364 // Make sure this is called with strictly ascending offsets.
365 DCHECK_GT(offset + kMoveImm64Offset, existing->second);
366
367 entries_.insert(std::make_pair(data, offset + kMoveRipRelativeDispOffset));
368 return true;
369 }
370
TryRecordEntry(intptr_t data,RelocInfo::Mode mode)371 bool ConstPool::TryRecordEntry(intptr_t data, RelocInfo::Mode mode) {
372 if (!FLAG_partial_constant_pool) return false;
373 if (!RelocInfo::IsShareableRelocMode(mode)) return false;
374
375 // Currently, partial constant pool only handles the following kinds of
376 // RelocInfo.
377 if (mode != RelocInfo::NONE && mode != RelocInfo::EXTERNAL_REFERENCE &&
378 mode != RelocInfo::OFF_HEAP_TARGET)
379 return false;
380
381 uint64_t raw_data = static_cast<uint64_t>(data);
382 int offset = assm_->pc_offset();
383 return AddSharedEntry(raw_data, offset);
384 }
385
IsMoveRipRelative(byte * instr)386 bool ConstPool::IsMoveRipRelative(byte* instr) {
387 if ((*reinterpret_cast<uint32_t*>(instr) & kMoveRipRelativeMask) ==
388 kMoveRipRelativeInstr)
389 return true;
390 return false;
391 }
392
Clear()393 void ConstPool::Clear() { entries_.clear(); }
394
PatchEntries()395 void ConstPool::PatchEntries() {
396 for (EntryMap::iterator iter = entries_.begin(); iter != entries_.end();
397 iter = entries_.upper_bound(iter->first)) {
398 std::pair<EntryMap::iterator, EntryMap::iterator> range =
399 entries_.equal_range(iter->first);
400 int constant_entry_offset = 0;
401 for (EntryMap::iterator it = range.first; it != range.second; it++) {
402 if (it == range.first) {
403 constant_entry_offset = it->second;
404 continue;
405 }
406
407 DCHECK_GT(constant_entry_offset, 0);
408 DCHECK_LT(constant_entry_offset, it->second);
409 int32_t disp32 =
410 constant_entry_offset - (it->second + kRipRelativeDispSize);
411 byte* disp_addr = assm_->addr_at(it->second);
412
413 // Check if the instruction is actually a rip-relative move.
414 DCHECK(IsMoveRipRelative(disp_addr - kMoveRipRelativeDispOffset));
415 // The displacement of the rip-relative move should be 0 before patching.
416 DCHECK(*reinterpret_cast<uint32_t*>(disp_addr) == 0);
417 *reinterpret_cast<int32_t*>(disp_addr) = disp32;
418 }
419 }
420 Clear();
421 }
422
PatchConstPool()423 void Assembler::PatchConstPool() {
424 // There is nothing to do if there are no pending entries.
425 if (constpool_.IsEmpty()) {
426 return;
427 }
428 constpool_.PatchEntries();
429 }
430
UseConstPoolFor(RelocInfo::Mode rmode)431 bool Assembler::UseConstPoolFor(RelocInfo::Mode rmode) {
432 if (!FLAG_partial_constant_pool) return false;
433 return (rmode == RelocInfo::NONE || rmode == RelocInfo::EXTERNAL_REFERENCE ||
434 rmode == RelocInfo::OFF_HEAP_TARGET);
435 }
436
437 // -----------------------------------------------------------------------------
438 // Implementation of Assembler.
439
Assembler(const AssemblerOptions & options,void * buffer,int buffer_size)440 Assembler::Assembler(const AssemblerOptions& options, void* buffer,
441 int buffer_size)
442 : AssemblerBase(options, buffer, buffer_size), constpool_(this) {
443 // Clear the buffer in debug mode unless it was provided by the
444 // caller in which case we can't be sure it's okay to overwrite
445 // existing code in it.
446 #ifdef DEBUG
447 if (own_buffer_) ZapCode(reinterpret_cast<Address>(buffer_), buffer_size_);
448 #endif
449
450 ReserveCodeTargetSpace(100);
451 reloc_info_writer.Reposition(buffer_ + buffer_size_, pc_);
452 }
453
GetCode(Isolate * isolate,CodeDesc * desc)454 void Assembler::GetCode(Isolate* isolate, CodeDesc* desc) {
455 PatchConstPool();
456 DCHECK(constpool_.IsEmpty());
457
458 // At this point overflow() may be true, but the gap ensures
459 // that we are still not overlapping instructions and relocation info.
460 DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap.
461
462 AllocateAndInstallRequestedHeapObjects(isolate);
463
464 // Set up code descriptor.
465 desc->buffer = buffer_;
466 desc->buffer_size = buffer_size_;
467 desc->instr_size = pc_offset();
468 DCHECK_GT(desc->instr_size, 0); // Zero-size code objects upset the system.
469 desc->reloc_size =
470 static_cast<int>((buffer_ + buffer_size_) - reloc_info_writer.pos());
471 desc->origin = this;
472 desc->constant_pool_size = 0;
473 desc->unwinding_info_size = 0;
474 desc->unwinding_info = nullptr;
475
476 // Collection stage
477 auto jump_opt = jump_optimization_info();
478 if (jump_opt && jump_opt->is_collecting()) {
479 auto& bitmap = jump_opt->farjmp_bitmap();
480 int num = static_cast<int>(farjmp_positions_.size());
481 if (num && bitmap.empty()) {
482 bool can_opt = false;
483
484 bitmap.resize((num + 31) / 32, 0);
485 for (int i = 0; i < num; i++) {
486 int disp_pos = farjmp_positions_[i];
487 int disp = long_at(disp_pos);
488 if (is_int8(disp)) {
489 bitmap[i / 32] |= 1 << (i & 31);
490 can_opt = true;
491 }
492 }
493 if (can_opt) {
494 jump_opt->set_optimizable();
495 }
496 }
497 }
498 }
499
500
Align(int m)501 void Assembler::Align(int m) {
502 DCHECK(base::bits::IsPowerOfTwo(m));
503 int delta = (m - (pc_offset() & (m - 1))) & (m - 1);
504 Nop(delta);
505 }
506
507
CodeTargetAlign()508 void Assembler::CodeTargetAlign() {
509 Align(16); // Preferred alignment of jump targets on x64.
510 }
511
512
IsNop(Address addr)513 bool Assembler::IsNop(Address addr) {
514 byte* a = reinterpret_cast<byte*>(addr);
515 while (*a == 0x66) a++;
516 if (*a == 0x90) return true;
517 if (a[0] == 0xF && a[1] == 0x1F) return true;
518 return false;
519 }
520
521
bind_to(Label * L,int pos)522 void Assembler::bind_to(Label* L, int pos) {
523 DCHECK(!L->is_bound()); // Label may only be bound once.
524 DCHECK(0 <= pos && pos <= pc_offset()); // Position must be valid.
525 if (L->is_linked()) {
526 int current = L->pos();
527 int next = long_at(current);
528 while (next != current) {
529 if (current >= 4 && long_at(current - 4) == 0) {
530 // Absolute address.
531 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos);
532 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64;
533 internal_reference_positions_.push_back(current - 4);
534 } else {
535 // Relative address, relative to point after address.
536 int imm32 = pos - (current + sizeof(int32_t));
537 long_at_put(current, imm32);
538 }
539 current = next;
540 next = long_at(next);
541 }
542 // Fix up last fixup on linked list.
543 if (current >= 4 && long_at(current - 4) == 0) {
544 // Absolute address.
545 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos);
546 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64;
547 internal_reference_positions_.push_back(current - 4);
548 } else {
549 // Relative address, relative to point after address.
550 int imm32 = pos - (current + sizeof(int32_t));
551 long_at_put(current, imm32);
552 }
553 }
554 while (L->is_near_linked()) {
555 int fixup_pos = L->near_link_pos();
556 int offset_to_next =
557 static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos)));
558 DCHECK_LE(offset_to_next, 0);
559 int disp = pos - (fixup_pos + sizeof(int8_t));
560 CHECK(is_int8(disp));
561 set_byte_at(fixup_pos, disp);
562 if (offset_to_next < 0) {
563 L->link_to(fixup_pos + offset_to_next, Label::kNear);
564 } else {
565 L->UnuseNear();
566 }
567 }
568
569 // Optimization stage
570 auto jump_opt = jump_optimization_info();
571 if (jump_opt && jump_opt->is_optimizing()) {
572 auto it = label_farjmp_maps_.find(L);
573 if (it != label_farjmp_maps_.end()) {
574 auto& pos_vector = it->second;
575 for (auto fixup_pos : pos_vector) {
576 int disp = pos - (fixup_pos + sizeof(int8_t));
577 CHECK(is_int8(disp));
578 set_byte_at(fixup_pos, disp);
579 }
580 label_farjmp_maps_.erase(it);
581 }
582 }
583 L->bind_to(pos);
584 }
585
586
bind(Label * L)587 void Assembler::bind(Label* L) {
588 bind_to(L, pc_offset());
589 }
590
record_farjmp_position(Label * L,int pos)591 void Assembler::record_farjmp_position(Label* L, int pos) {
592 auto& pos_vector = label_farjmp_maps_[L];
593 pos_vector.push_back(pos);
594 }
595
is_optimizable_farjmp(int idx)596 bool Assembler::is_optimizable_farjmp(int idx) {
597 if (predictable_code_size()) return false;
598
599 auto jump_opt = jump_optimization_info();
600 CHECK(jump_opt->is_optimizing());
601
602 auto& bitmap = jump_opt->farjmp_bitmap();
603 CHECK(idx < static_cast<int>(bitmap.size() * 32));
604 return !!(bitmap[idx / 32] & (1 << (idx & 31)));
605 }
606
GrowBuffer()607 void Assembler::GrowBuffer() {
608 DCHECK(buffer_overflow());
609 if (!own_buffer_) FATAL("external code buffer is too small");
610
611 // Compute new buffer size.
612 CodeDesc desc; // the new buffer
613 desc.buffer_size = 2 * buffer_size_;
614
615 // Some internal data structures overflow for very large buffers,
616 // they must ensure that kMaximalBufferSize is not too large.
617 if (desc.buffer_size > kMaximalBufferSize) {
618 V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
619 }
620
621 // Set up new buffer.
622 desc.buffer = NewArray<byte>(desc.buffer_size);
623 desc.origin = this;
624 desc.instr_size = pc_offset();
625 desc.reloc_size =
626 static_cast<int>((buffer_ + buffer_size_) - (reloc_info_writer.pos()));
627
628 // Clear the buffer in debug mode. Use 'int3' instructions to make
629 // sure to get into problems if we ever run uninitialized code.
630 #ifdef DEBUG
631 ZapCode(reinterpret_cast<Address>(desc.buffer), desc.buffer_size);
632 #endif
633
634 // Copy the data.
635 intptr_t pc_delta = desc.buffer - buffer_;
636 intptr_t rc_delta = (desc.buffer + desc.buffer_size) -
637 (buffer_ + buffer_size_);
638 MemMove(desc.buffer, buffer_, desc.instr_size);
639 MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(),
640 desc.reloc_size);
641
642 // Switch buffers.
643 DeleteArray(buffer_);
644 buffer_ = desc.buffer;
645 buffer_size_ = desc.buffer_size;
646 pc_ += pc_delta;
647 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
648 reloc_info_writer.last_pc() + pc_delta);
649
650 // Relocate internal references.
651 for (auto pos : internal_reference_positions_) {
652 intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos);
653 *p += pc_delta;
654 }
655
656 DCHECK(!buffer_overflow());
657 }
658
emit_operand(int code,Operand adr)659 void Assembler::emit_operand(int code, Operand adr) {
660 DCHECK(is_uint3(code));
661 const unsigned length = adr.data().len;
662 DCHECK_GT(length, 0);
663
664 // Emit updated ModR/M byte containing the given register.
665 DCHECK_EQ(adr.data().buf[0] & 0x38, 0);
666 *pc_++ = adr.data().buf[0] | code << 3;
667
668 // Recognize RIP relative addressing.
669 if (adr.data().buf[0] == 5) {
670 DCHECK_EQ(9u, length);
671 Label* label = *bit_cast<Label* const*>(&adr.data().buf[1]);
672 if (label->is_bound()) {
673 int offset =
674 label->pos() - pc_offset() - sizeof(int32_t) + adr.data().addend;
675 DCHECK_GE(0, offset);
676 emitl(offset);
677 } else if (label->is_linked()) {
678 emitl(label->pos());
679 label->link_to(pc_offset() - sizeof(int32_t));
680 } else {
681 DCHECK(label->is_unused());
682 int32_t current = pc_offset();
683 emitl(current);
684 label->link_to(current);
685 }
686 } else {
687 // Emit the rest of the encoded operand.
688 for (unsigned i = 1; i < length; i++) *pc_++ = adr.data().buf[i];
689 }
690 }
691
692
693 // Assembler Instruction implementations.
694
arithmetic_op(byte opcode,Register reg,Operand op,int size)695 void Assembler::arithmetic_op(byte opcode, Register reg, Operand op, int size) {
696 EnsureSpace ensure_space(this);
697 emit_rex(reg, op, size);
698 emit(opcode);
699 emit_operand(reg, op);
700 }
701
702
arithmetic_op(byte opcode,Register reg,Register rm_reg,int size)703 void Assembler::arithmetic_op(byte opcode,
704 Register reg,
705 Register rm_reg,
706 int size) {
707 EnsureSpace ensure_space(this);
708 DCHECK_EQ(opcode & 0xC6, 2);
709 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
710 // Swap reg and rm_reg and change opcode operand order.
711 emit_rex(rm_reg, reg, size);
712 emit(opcode ^ 0x02);
713 emit_modrm(rm_reg, reg);
714 } else {
715 emit_rex(reg, rm_reg, size);
716 emit(opcode);
717 emit_modrm(reg, rm_reg);
718 }
719 }
720
721
arithmetic_op_16(byte opcode,Register reg,Register rm_reg)722 void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) {
723 EnsureSpace ensure_space(this);
724 DCHECK_EQ(opcode & 0xC6, 2);
725 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
726 // Swap reg and rm_reg and change opcode operand order.
727 emit(0x66);
728 emit_optional_rex_32(rm_reg, reg);
729 emit(opcode ^ 0x02);
730 emit_modrm(rm_reg, reg);
731 } else {
732 emit(0x66);
733 emit_optional_rex_32(reg, rm_reg);
734 emit(opcode);
735 emit_modrm(reg, rm_reg);
736 }
737 }
738
arithmetic_op_16(byte opcode,Register reg,Operand rm_reg)739 void Assembler::arithmetic_op_16(byte opcode, Register reg, Operand rm_reg) {
740 EnsureSpace ensure_space(this);
741 emit(0x66);
742 emit_optional_rex_32(reg, rm_reg);
743 emit(opcode);
744 emit_operand(reg, rm_reg);
745 }
746
arithmetic_op_8(byte opcode,Register reg,Operand op)747 void Assembler::arithmetic_op_8(byte opcode, Register reg, Operand op) {
748 EnsureSpace ensure_space(this);
749 if (!reg.is_byte_register()) {
750 emit_rex_32(reg, op);
751 } else {
752 emit_optional_rex_32(reg, op);
753 }
754 emit(opcode);
755 emit_operand(reg, op);
756 }
757
758
arithmetic_op_8(byte opcode,Register reg,Register rm_reg)759 void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) {
760 EnsureSpace ensure_space(this);
761 DCHECK_EQ(opcode & 0xC6, 2);
762 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
763 // Swap reg and rm_reg and change opcode operand order.
764 if (!rm_reg.is_byte_register() || !reg.is_byte_register()) {
765 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
766 emit_rex_32(rm_reg, reg);
767 }
768 emit(opcode ^ 0x02);
769 emit_modrm(rm_reg, reg);
770 } else {
771 if (!reg.is_byte_register() || !rm_reg.is_byte_register()) {
772 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
773 emit_rex_32(reg, rm_reg);
774 }
775 emit(opcode);
776 emit_modrm(reg, rm_reg);
777 }
778 }
779
780
immediate_arithmetic_op(byte subcode,Register dst,Immediate src,int size)781 void Assembler::immediate_arithmetic_op(byte subcode,
782 Register dst,
783 Immediate src,
784 int size) {
785 EnsureSpace ensure_space(this);
786 emit_rex(dst, size);
787 if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) {
788 emit(0x83);
789 emit_modrm(subcode, dst);
790 emit(src.value_);
791 } else if (dst == rax) {
792 emit(0x05 | (subcode << 3));
793 emit(src);
794 } else {
795 emit(0x81);
796 emit_modrm(subcode, dst);
797 emit(src);
798 }
799 }
800
immediate_arithmetic_op(byte subcode,Operand dst,Immediate src,int size)801 void Assembler::immediate_arithmetic_op(byte subcode, Operand dst,
802 Immediate src, int size) {
803 EnsureSpace ensure_space(this);
804 emit_rex(dst, size);
805 if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) {
806 emit(0x83);
807 emit_operand(subcode, dst);
808 emit(src.value_);
809 } else {
810 emit(0x81);
811 emit_operand(subcode, dst);
812 emit(src);
813 }
814 }
815
816
immediate_arithmetic_op_16(byte subcode,Register dst,Immediate src)817 void Assembler::immediate_arithmetic_op_16(byte subcode,
818 Register dst,
819 Immediate src) {
820 EnsureSpace ensure_space(this);
821 emit(0x66); // Operand size override prefix.
822 emit_optional_rex_32(dst);
823 if (is_int8(src.value_)) {
824 emit(0x83);
825 emit_modrm(subcode, dst);
826 emit(src.value_);
827 } else if (dst == rax) {
828 emit(0x05 | (subcode << 3));
829 emitw(src.value_);
830 } else {
831 emit(0x81);
832 emit_modrm(subcode, dst);
833 emitw(src.value_);
834 }
835 }
836
immediate_arithmetic_op_16(byte subcode,Operand dst,Immediate src)837 void Assembler::immediate_arithmetic_op_16(byte subcode, Operand dst,
838 Immediate src) {
839 EnsureSpace ensure_space(this);
840 emit(0x66); // Operand size override prefix.
841 emit_optional_rex_32(dst);
842 if (is_int8(src.value_)) {
843 emit(0x83);
844 emit_operand(subcode, dst);
845 emit(src.value_);
846 } else {
847 emit(0x81);
848 emit_operand(subcode, dst);
849 emitw(src.value_);
850 }
851 }
852
immediate_arithmetic_op_8(byte subcode,Operand dst,Immediate src)853 void Assembler::immediate_arithmetic_op_8(byte subcode, Operand dst,
854 Immediate src) {
855 EnsureSpace ensure_space(this);
856 emit_optional_rex_32(dst);
857 DCHECK(is_int8(src.value_) || is_uint8(src.value_));
858 emit(0x80);
859 emit_operand(subcode, dst);
860 emit(src.value_);
861 }
862
863
immediate_arithmetic_op_8(byte subcode,Register dst,Immediate src)864 void Assembler::immediate_arithmetic_op_8(byte subcode,
865 Register dst,
866 Immediate src) {
867 EnsureSpace ensure_space(this);
868 if (!dst.is_byte_register()) {
869 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
870 emit_rex_32(dst);
871 }
872 DCHECK(is_int8(src.value_) || is_uint8(src.value_));
873 emit(0x80);
874 emit_modrm(subcode, dst);
875 emit(src.value_);
876 }
877
878
shift(Register dst,Immediate shift_amount,int subcode,int size)879 void Assembler::shift(Register dst,
880 Immediate shift_amount,
881 int subcode,
882 int size) {
883 EnsureSpace ensure_space(this);
884 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
885 : is_uint5(shift_amount.value_));
886 if (shift_amount.value_ == 1) {
887 emit_rex(dst, size);
888 emit(0xD1);
889 emit_modrm(subcode, dst);
890 } else {
891 emit_rex(dst, size);
892 emit(0xC1);
893 emit_modrm(subcode, dst);
894 emit(shift_amount.value_);
895 }
896 }
897
898
shift(Operand dst,Immediate shift_amount,int subcode,int size)899 void Assembler::shift(Operand dst, Immediate shift_amount, int subcode,
900 int size) {
901 EnsureSpace ensure_space(this);
902 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
903 : is_uint5(shift_amount.value_));
904 if (shift_amount.value_ == 1) {
905 emit_rex(dst, size);
906 emit(0xD1);
907 emit_operand(subcode, dst);
908 } else {
909 emit_rex(dst, size);
910 emit(0xC1);
911 emit_operand(subcode, dst);
912 emit(shift_amount.value_);
913 }
914 }
915
916
shift(Register dst,int subcode,int size)917 void Assembler::shift(Register dst, int subcode, int size) {
918 EnsureSpace ensure_space(this);
919 emit_rex(dst, size);
920 emit(0xD3);
921 emit_modrm(subcode, dst);
922 }
923
924
shift(Operand dst,int subcode,int size)925 void Assembler::shift(Operand dst, int subcode, int size) {
926 EnsureSpace ensure_space(this);
927 emit_rex(dst, size);
928 emit(0xD3);
929 emit_operand(subcode, dst);
930 }
931
bswapl(Register dst)932 void Assembler::bswapl(Register dst) {
933 EnsureSpace ensure_space(this);
934 emit_rex_32(dst);
935 emit(0x0F);
936 emit(0xC8 + dst.low_bits());
937 }
938
bswapq(Register dst)939 void Assembler::bswapq(Register dst) {
940 EnsureSpace ensure_space(this);
941 emit_rex_64(dst);
942 emit(0x0F);
943 emit(0xC8 + dst.low_bits());
944 }
945
bt(Operand dst,Register src)946 void Assembler::bt(Operand dst, Register src) {
947 EnsureSpace ensure_space(this);
948 emit_rex_64(src, dst);
949 emit(0x0F);
950 emit(0xA3);
951 emit_operand(src, dst);
952 }
953
bts(Operand dst,Register src)954 void Assembler::bts(Operand dst, Register src) {
955 EnsureSpace ensure_space(this);
956 emit_rex_64(src, dst);
957 emit(0x0F);
958 emit(0xAB);
959 emit_operand(src, dst);
960 }
961
962
bsrl(Register dst,Register src)963 void Assembler::bsrl(Register dst, Register src) {
964 EnsureSpace ensure_space(this);
965 emit_optional_rex_32(dst, src);
966 emit(0x0F);
967 emit(0xBD);
968 emit_modrm(dst, src);
969 }
970
bsrl(Register dst,Operand src)971 void Assembler::bsrl(Register dst, Operand src) {
972 EnsureSpace ensure_space(this);
973 emit_optional_rex_32(dst, src);
974 emit(0x0F);
975 emit(0xBD);
976 emit_operand(dst, src);
977 }
978
979
bsrq(Register dst,Register src)980 void Assembler::bsrq(Register dst, Register src) {
981 EnsureSpace ensure_space(this);
982 emit_rex_64(dst, src);
983 emit(0x0F);
984 emit(0xBD);
985 emit_modrm(dst, src);
986 }
987
bsrq(Register dst,Operand src)988 void Assembler::bsrq(Register dst, Operand src) {
989 EnsureSpace ensure_space(this);
990 emit_rex_64(dst, src);
991 emit(0x0F);
992 emit(0xBD);
993 emit_operand(dst, src);
994 }
995
996
bsfl(Register dst,Register src)997 void Assembler::bsfl(Register dst, Register src) {
998 EnsureSpace ensure_space(this);
999 emit_optional_rex_32(dst, src);
1000 emit(0x0F);
1001 emit(0xBC);
1002 emit_modrm(dst, src);
1003 }
1004
bsfl(Register dst,Operand src)1005 void Assembler::bsfl(Register dst, Operand src) {
1006 EnsureSpace ensure_space(this);
1007 emit_optional_rex_32(dst, src);
1008 emit(0x0F);
1009 emit(0xBC);
1010 emit_operand(dst, src);
1011 }
1012
1013
bsfq(Register dst,Register src)1014 void Assembler::bsfq(Register dst, Register src) {
1015 EnsureSpace ensure_space(this);
1016 emit_rex_64(dst, src);
1017 emit(0x0F);
1018 emit(0xBC);
1019 emit_modrm(dst, src);
1020 }
1021
bsfq(Register dst,Operand src)1022 void Assembler::bsfq(Register dst, Operand src) {
1023 EnsureSpace ensure_space(this);
1024 emit_rex_64(dst, src);
1025 emit(0x0F);
1026 emit(0xBC);
1027 emit_operand(dst, src);
1028 }
1029
pshufw(XMMRegister dst,XMMRegister src,uint8_t shuffle)1030 void Assembler::pshufw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
1031 EnsureSpace ensure_space(this);
1032 emit_optional_rex_32(dst, src);
1033 emit(0x0F);
1034 emit(0x70);
1035 emit_sse_operand(dst, src);
1036 emit(shuffle);
1037 }
1038
pshufw(XMMRegister dst,Operand src,uint8_t shuffle)1039 void Assembler::pshufw(XMMRegister dst, Operand src, uint8_t shuffle) {
1040 EnsureSpace ensure_space(this);
1041 emit_optional_rex_32(dst, src);
1042 emit(0x0F);
1043 emit(0x70);
1044 emit_operand(dst.code(), src);
1045 emit(shuffle);
1046 }
1047
pblendw(XMMRegister dst,Operand src,uint8_t mask)1048 void Assembler::pblendw(XMMRegister dst, Operand src, uint8_t mask) {
1049 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0E);
1050 emit(mask);
1051 }
1052
pblendw(XMMRegister dst,XMMRegister src,uint8_t mask)1053 void Assembler::pblendw(XMMRegister dst, XMMRegister src, uint8_t mask) {
1054 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0E);
1055 emit(mask);
1056 }
1057
palignr(XMMRegister dst,Operand src,uint8_t mask)1058 void Assembler::palignr(XMMRegister dst, Operand src, uint8_t mask) {
1059 ssse3_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0F);
1060 emit(mask);
1061 }
1062
palignr(XMMRegister dst,XMMRegister src,uint8_t mask)1063 void Assembler::palignr(XMMRegister dst, XMMRegister src, uint8_t mask) {
1064 ssse3_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0F);
1065 emit(mask);
1066 }
1067
call(Label * L)1068 void Assembler::call(Label* L) {
1069 EnsureSpace ensure_space(this);
1070 // 1110 1000 #32-bit disp.
1071 emit(0xE8);
1072 if (L->is_bound()) {
1073 int offset = L->pos() - pc_offset() - sizeof(int32_t);
1074 DCHECK_LE(offset, 0);
1075 emitl(offset);
1076 } else if (L->is_linked()) {
1077 emitl(L->pos());
1078 L->link_to(pc_offset() - sizeof(int32_t));
1079 } else {
1080 DCHECK(L->is_unused());
1081 int32_t current = pc_offset();
1082 emitl(current);
1083 L->link_to(current);
1084 }
1085 }
1086
1087
call(Address entry,RelocInfo::Mode rmode)1088 void Assembler::call(Address entry, RelocInfo::Mode rmode) {
1089 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1090 EnsureSpace ensure_space(this);
1091 // 1110 1000 #32-bit disp.
1092 emit(0xE8);
1093 emit_runtime_entry(entry, rmode);
1094 }
1095
call(CodeStub * stub)1096 void Assembler::call(CodeStub* stub) {
1097 EnsureSpace ensure_space(this);
1098 // 1110 1000 #32-bit disp.
1099 emit(0xE8);
1100 RequestHeapObject(HeapObjectRequest(stub));
1101 RecordRelocInfo(RelocInfo::CODE_TARGET);
1102 int code_target_index = AddCodeTarget(Handle<Code>());
1103 emitl(code_target_index);
1104 }
1105
call(Handle<Code> target,RelocInfo::Mode rmode)1106 void Assembler::call(Handle<Code> target, RelocInfo::Mode rmode) {
1107 DCHECK(RelocInfo::IsCodeTarget(rmode));
1108 EnsureSpace ensure_space(this);
1109 // 1110 1000 #32-bit disp.
1110 emit(0xE8);
1111 RecordRelocInfo(rmode);
1112 int code_target_index = AddCodeTarget(target);
1113 emitl(code_target_index);
1114 }
1115
near_call(Address addr,RelocInfo::Mode rmode)1116 void Assembler::near_call(Address addr, RelocInfo::Mode rmode) {
1117 EnsureSpace ensure_space(this);
1118 emit(0xE8);
1119 intptr_t value = static_cast<intptr_t>(addr);
1120 DCHECK(is_int32(value));
1121 RecordRelocInfo(rmode);
1122 emitl(static_cast<int32_t>(value));
1123 }
1124
near_jmp(Address addr,RelocInfo::Mode rmode)1125 void Assembler::near_jmp(Address addr, RelocInfo::Mode rmode) {
1126 EnsureSpace ensure_space(this);
1127 emit(0xE9);
1128 intptr_t value = static_cast<intptr_t>(addr);
1129 DCHECK(is_int32(value));
1130 RecordRelocInfo(rmode);
1131 emitl(static_cast<int32_t>(value));
1132 }
1133
call(Register adr)1134 void Assembler::call(Register adr) {
1135 EnsureSpace ensure_space(this);
1136 // Opcode: FF /2 r64.
1137 emit_optional_rex_32(adr);
1138 emit(0xFF);
1139 emit_modrm(0x2, adr);
1140 }
1141
call(Operand op)1142 void Assembler::call(Operand op) {
1143 EnsureSpace ensure_space(this);
1144 // Opcode: FF /2 m64.
1145 emit_optional_rex_32(op);
1146 emit(0xFF);
1147 emit_operand(0x2, op);
1148 }
1149
1150
1151 // Calls directly to the given address using a relative offset.
1152 // Should only ever be used in Code objects for calls within the
1153 // same Code object. Should not be used when generating new code (use labels),
1154 // but only when patching existing code.
call(Address target)1155 void Assembler::call(Address target) {
1156 EnsureSpace ensure_space(this);
1157 // 1110 1000 #32-bit disp.
1158 emit(0xE8);
1159 Address source = reinterpret_cast<Address>(pc_) + 4;
1160 intptr_t displacement = target - source;
1161 DCHECK(is_int32(displacement));
1162 emitl(static_cast<int32_t>(displacement));
1163 }
1164
1165
clc()1166 void Assembler::clc() {
1167 EnsureSpace ensure_space(this);
1168 emit(0xF8);
1169 }
1170
1171
cld()1172 void Assembler::cld() {
1173 EnsureSpace ensure_space(this);
1174 emit(0xFC);
1175 }
1176
cdq()1177 void Assembler::cdq() {
1178 EnsureSpace ensure_space(this);
1179 emit(0x99);
1180 }
1181
1182
cmovq(Condition cc,Register dst,Register src)1183 void Assembler::cmovq(Condition cc, Register dst, Register src) {
1184 if (cc == always) {
1185 movq(dst, src);
1186 } else if (cc == never) {
1187 return;
1188 }
1189 // No need to check CpuInfo for CMOV support, it's a required part of the
1190 // 64-bit architecture.
1191 DCHECK_GE(cc, 0); // Use mov for unconditional moves.
1192 EnsureSpace ensure_space(this);
1193 // Opcode: REX.W 0f 40 + cc /r.
1194 emit_rex_64(dst, src);
1195 emit(0x0F);
1196 emit(0x40 + cc);
1197 emit_modrm(dst, src);
1198 }
1199
cmovq(Condition cc,Register dst,Operand src)1200 void Assembler::cmovq(Condition cc, Register dst, Operand src) {
1201 if (cc == always) {
1202 movq(dst, src);
1203 } else if (cc == never) {
1204 return;
1205 }
1206 DCHECK_GE(cc, 0);
1207 EnsureSpace ensure_space(this);
1208 // Opcode: REX.W 0f 40 + cc /r.
1209 emit_rex_64(dst, src);
1210 emit(0x0F);
1211 emit(0x40 + cc);
1212 emit_operand(dst, src);
1213 }
1214
1215
cmovl(Condition cc,Register dst,Register src)1216 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1217 if (cc == always) {
1218 movl(dst, src);
1219 } else if (cc == never) {
1220 return;
1221 }
1222 DCHECK_GE(cc, 0);
1223 EnsureSpace ensure_space(this);
1224 // Opcode: 0f 40 + cc /r.
1225 emit_optional_rex_32(dst, src);
1226 emit(0x0F);
1227 emit(0x40 + cc);
1228 emit_modrm(dst, src);
1229 }
1230
cmovl(Condition cc,Register dst,Operand src)1231 void Assembler::cmovl(Condition cc, Register dst, Operand src) {
1232 if (cc == always) {
1233 movl(dst, src);
1234 } else if (cc == never) {
1235 return;
1236 }
1237 DCHECK_GE(cc, 0);
1238 EnsureSpace ensure_space(this);
1239 // Opcode: 0f 40 + cc /r.
1240 emit_optional_rex_32(dst, src);
1241 emit(0x0F);
1242 emit(0x40 + cc);
1243 emit_operand(dst, src);
1244 }
1245
1246
cmpb_al(Immediate imm8)1247 void Assembler::cmpb_al(Immediate imm8) {
1248 DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_));
1249 EnsureSpace ensure_space(this);
1250 emit(0x3C);
1251 emit(imm8.value_);
1252 }
1253
lock()1254 void Assembler::lock() {
1255 EnsureSpace ensure_space(this);
1256 emit(0xF0);
1257 }
1258
cmpxchgb(Operand dst,Register src)1259 void Assembler::cmpxchgb(Operand dst, Register src) {
1260 EnsureSpace ensure_space(this);
1261 if (!src.is_byte_register()) {
1262 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1263 emit_rex_32(src, dst);
1264 } else {
1265 emit_optional_rex_32(src, dst);
1266 }
1267 emit(0x0F);
1268 emit(0xB0);
1269 emit_operand(src, dst);
1270 }
1271
cmpxchgw(Operand dst,Register src)1272 void Assembler::cmpxchgw(Operand dst, Register src) {
1273 EnsureSpace ensure_space(this);
1274 emit(0x66);
1275 emit_optional_rex_32(src, dst);
1276 emit(0x0F);
1277 emit(0xB1);
1278 emit_operand(src, dst);
1279 }
1280
emit_cmpxchg(Operand dst,Register src,int size)1281 void Assembler::emit_cmpxchg(Operand dst, Register src, int size) {
1282 EnsureSpace ensure_space(this);
1283 emit_rex(src, dst, size);
1284 emit(0x0F);
1285 emit(0xB1);
1286 emit_operand(src, dst);
1287 }
1288
lfence()1289 void Assembler::lfence() {
1290 EnsureSpace ensure_space(this);
1291 emit(0x0F);
1292 emit(0xAE);
1293 emit(0xE8);
1294 }
1295
cpuid()1296 void Assembler::cpuid() {
1297 EnsureSpace ensure_space(this);
1298 emit(0x0F);
1299 emit(0xA2);
1300 }
1301
1302
cqo()1303 void Assembler::cqo() {
1304 EnsureSpace ensure_space(this);
1305 emit_rex_64();
1306 emit(0x99);
1307 }
1308
1309
emit_dec(Register dst,int size)1310 void Assembler::emit_dec(Register dst, int size) {
1311 EnsureSpace ensure_space(this);
1312 emit_rex(dst, size);
1313 emit(0xFF);
1314 emit_modrm(0x1, dst);
1315 }
1316
emit_dec(Operand dst,int size)1317 void Assembler::emit_dec(Operand dst, int size) {
1318 EnsureSpace ensure_space(this);
1319 emit_rex(dst, size);
1320 emit(0xFF);
1321 emit_operand(1, dst);
1322 }
1323
1324
decb(Register dst)1325 void Assembler::decb(Register dst) {
1326 EnsureSpace ensure_space(this);
1327 if (!dst.is_byte_register()) {
1328 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1329 emit_rex_32(dst);
1330 }
1331 emit(0xFE);
1332 emit_modrm(0x1, dst);
1333 }
1334
decb(Operand dst)1335 void Assembler::decb(Operand dst) {
1336 EnsureSpace ensure_space(this);
1337 emit_optional_rex_32(dst);
1338 emit(0xFE);
1339 emit_operand(1, dst);
1340 }
1341
1342
enter(Immediate size)1343 void Assembler::enter(Immediate size) {
1344 EnsureSpace ensure_space(this);
1345 emit(0xC8);
1346 emitw(size.value_); // 16 bit operand, always.
1347 emit(0);
1348 }
1349
1350
hlt()1351 void Assembler::hlt() {
1352 EnsureSpace ensure_space(this);
1353 emit(0xF4);
1354 }
1355
1356
emit_idiv(Register src,int size)1357 void Assembler::emit_idiv(Register src, int size) {
1358 EnsureSpace ensure_space(this);
1359 emit_rex(src, size);
1360 emit(0xF7);
1361 emit_modrm(0x7, src);
1362 }
1363
1364
emit_div(Register src,int size)1365 void Assembler::emit_div(Register src, int size) {
1366 EnsureSpace ensure_space(this);
1367 emit_rex(src, size);
1368 emit(0xF7);
1369 emit_modrm(0x6, src);
1370 }
1371
1372
emit_imul(Register src,int size)1373 void Assembler::emit_imul(Register src, int size) {
1374 EnsureSpace ensure_space(this);
1375 emit_rex(src, size);
1376 emit(0xF7);
1377 emit_modrm(0x5, src);
1378 }
1379
emit_imul(Operand src,int size)1380 void Assembler::emit_imul(Operand src, int size) {
1381 EnsureSpace ensure_space(this);
1382 emit_rex(src, size);
1383 emit(0xF7);
1384 emit_operand(0x5, src);
1385 }
1386
1387
emit_imul(Register dst,Register src,int size)1388 void Assembler::emit_imul(Register dst, Register src, int size) {
1389 EnsureSpace ensure_space(this);
1390 emit_rex(dst, src, size);
1391 emit(0x0F);
1392 emit(0xAF);
1393 emit_modrm(dst, src);
1394 }
1395
emit_imul(Register dst,Operand src,int size)1396 void Assembler::emit_imul(Register dst, Operand src, int size) {
1397 EnsureSpace ensure_space(this);
1398 emit_rex(dst, src, size);
1399 emit(0x0F);
1400 emit(0xAF);
1401 emit_operand(dst, src);
1402 }
1403
1404
emit_imul(Register dst,Register src,Immediate imm,int size)1405 void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) {
1406 EnsureSpace ensure_space(this);
1407 emit_rex(dst, src, size);
1408 if (is_int8(imm.value_)) {
1409 emit(0x6B);
1410 emit_modrm(dst, src);
1411 emit(imm.value_);
1412 } else {
1413 emit(0x69);
1414 emit_modrm(dst, src);
1415 emitl(imm.value_);
1416 }
1417 }
1418
emit_imul(Register dst,Operand src,Immediate imm,int size)1419 void Assembler::emit_imul(Register dst, Operand src, Immediate imm, int size) {
1420 EnsureSpace ensure_space(this);
1421 emit_rex(dst, src, size);
1422 if (is_int8(imm.value_)) {
1423 emit(0x6B);
1424 emit_operand(dst, src);
1425 emit(imm.value_);
1426 } else {
1427 emit(0x69);
1428 emit_operand(dst, src);
1429 emitl(imm.value_);
1430 }
1431 }
1432
1433
emit_inc(Register dst,int size)1434 void Assembler::emit_inc(Register dst, int size) {
1435 EnsureSpace ensure_space(this);
1436 emit_rex(dst, size);
1437 emit(0xFF);
1438 emit_modrm(0x0, dst);
1439 }
1440
emit_inc(Operand dst,int size)1441 void Assembler::emit_inc(Operand dst, int size) {
1442 EnsureSpace ensure_space(this);
1443 emit_rex(dst, size);
1444 emit(0xFF);
1445 emit_operand(0, dst);
1446 }
1447
1448
int3()1449 void Assembler::int3() {
1450 EnsureSpace ensure_space(this);
1451 emit(0xCC);
1452 }
1453
1454
j(Condition cc,Label * L,Label::Distance distance)1455 void Assembler::j(Condition cc, Label* L, Label::Distance distance) {
1456 if (cc == always) {
1457 jmp(L);
1458 return;
1459 } else if (cc == never) {
1460 return;
1461 }
1462 EnsureSpace ensure_space(this);
1463 DCHECK(is_uint4(cc));
1464 if (L->is_bound()) {
1465 const int short_size = 2;
1466 const int long_size = 6;
1467 int offs = L->pos() - pc_offset();
1468 DCHECK_LE(offs, 0);
1469 // Determine whether we can use 1-byte offsets for backwards branches,
1470 // which have a max range of 128 bytes.
1471
1472 // We also need to check predictable_code_size() flag here, because on x64,
1473 // when the full code generator recompiles code for debugging, some places
1474 // need to be padded out to a certain size. The debugger is keeping track of
1475 // how often it did this so that it can adjust return addresses on the
1476 // stack, but if the size of jump instructions can also change, that's not
1477 // enough and the calculated offsets would be incorrect.
1478 if (is_int8(offs - short_size) && !predictable_code_size()) {
1479 // 0111 tttn #8-bit disp.
1480 emit(0x70 | cc);
1481 emit((offs - short_size) & 0xFF);
1482 } else {
1483 // 0000 1111 1000 tttn #32-bit disp.
1484 emit(0x0F);
1485 emit(0x80 | cc);
1486 emitl(offs - long_size);
1487 }
1488 } else if (distance == Label::kNear) {
1489 // 0111 tttn #8-bit disp
1490 emit(0x70 | cc);
1491 byte disp = 0x00;
1492 if (L->is_near_linked()) {
1493 int offset = L->near_link_pos() - pc_offset();
1494 DCHECK(is_int8(offset));
1495 disp = static_cast<byte>(offset & 0xFF);
1496 }
1497 L->link_to(pc_offset(), Label::kNear);
1498 emit(disp);
1499 } else {
1500 auto jump_opt = jump_optimization_info();
1501 if (V8_UNLIKELY(jump_opt)) {
1502 if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
1503 // 0111 tttn #8-bit disp
1504 emit(0x70 | cc);
1505 record_farjmp_position(L, pc_offset());
1506 emit(0);
1507 return;
1508 }
1509 if (jump_opt->is_collecting()) {
1510 farjmp_positions_.push_back(pc_offset() + 2);
1511 }
1512 }
1513 if (L->is_linked()) {
1514 // 0000 1111 1000 tttn #32-bit disp.
1515 emit(0x0F);
1516 emit(0x80 | cc);
1517 emitl(L->pos());
1518 L->link_to(pc_offset() - sizeof(int32_t));
1519 } else {
1520 DCHECK(L->is_unused());
1521 emit(0x0F);
1522 emit(0x80 | cc);
1523 int32_t current = pc_offset();
1524 emitl(current);
1525 L->link_to(current);
1526 }
1527 }
1528 }
1529
1530
j(Condition cc,Address entry,RelocInfo::Mode rmode)1531 void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) {
1532 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1533 EnsureSpace ensure_space(this);
1534 DCHECK(is_uint4(cc));
1535 emit(0x0F);
1536 emit(0x80 | cc);
1537 emit_runtime_entry(entry, rmode);
1538 }
1539
1540
j(Condition cc,Handle<Code> target,RelocInfo::Mode rmode)1541 void Assembler::j(Condition cc,
1542 Handle<Code> target,
1543 RelocInfo::Mode rmode) {
1544 if (cc == always) {
1545 jmp(target, rmode);
1546 return;
1547 } else if (cc == never) {
1548 return;
1549 }
1550 EnsureSpace ensure_space(this);
1551 DCHECK(is_uint4(cc));
1552 // 0000 1111 1000 tttn #32-bit disp.
1553 emit(0x0F);
1554 emit(0x80 | cc);
1555 DCHECK(RelocInfo::IsCodeTarget(rmode));
1556 RecordRelocInfo(rmode);
1557 int code_target_index = AddCodeTarget(target);
1558 emitl(code_target_index);
1559 }
1560
1561
jmp(Label * L,Label::Distance distance)1562 void Assembler::jmp(Label* L, Label::Distance distance) {
1563 EnsureSpace ensure_space(this);
1564 const int short_size = sizeof(int8_t);
1565 const int long_size = sizeof(int32_t);
1566 if (L->is_bound()) {
1567 int offs = L->pos() - pc_offset() - 1;
1568 DCHECK_LE(offs, 0);
1569 if (is_int8(offs - short_size) && !predictable_code_size()) {
1570 // 1110 1011 #8-bit disp.
1571 emit(0xEB);
1572 emit((offs - short_size) & 0xFF);
1573 } else {
1574 // 1110 1001 #32-bit disp.
1575 emit(0xE9);
1576 emitl(offs - long_size);
1577 }
1578 } else if (distance == Label::kNear) {
1579 emit(0xEB);
1580 byte disp = 0x00;
1581 if (L->is_near_linked()) {
1582 int offset = L->near_link_pos() - pc_offset();
1583 DCHECK(is_int8(offset));
1584 disp = static_cast<byte>(offset & 0xFF);
1585 }
1586 L->link_to(pc_offset(), Label::kNear);
1587 emit(disp);
1588 } else {
1589 auto jump_opt = jump_optimization_info();
1590 if (V8_UNLIKELY(jump_opt)) {
1591 if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
1592 emit(0xEB);
1593 record_farjmp_position(L, pc_offset());
1594 emit(0);
1595 return;
1596 }
1597 if (jump_opt->is_collecting()) {
1598 farjmp_positions_.push_back(pc_offset() + 1);
1599 }
1600 }
1601 if (L->is_linked()) {
1602 // 1110 1001 #32-bit disp.
1603 emit(0xE9);
1604 emitl(L->pos());
1605 L->link_to(pc_offset() - long_size);
1606 } else {
1607 // 1110 1001 #32-bit disp.
1608 DCHECK(L->is_unused());
1609 emit(0xE9);
1610 int32_t current = pc_offset();
1611 emitl(current);
1612 L->link_to(current);
1613 }
1614 }
1615 }
1616
1617
jmp(Handle<Code> target,RelocInfo::Mode rmode)1618 void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) {
1619 DCHECK(RelocInfo::IsCodeTarget(rmode));
1620 EnsureSpace ensure_space(this);
1621 // 1110 1001 #32-bit disp.
1622 emit(0xE9);
1623 RecordRelocInfo(rmode);
1624 int code_target_index = AddCodeTarget(target);
1625 emitl(code_target_index);
1626 }
1627
1628
jmp(Register target)1629 void Assembler::jmp(Register target) {
1630 EnsureSpace ensure_space(this);
1631 // Opcode FF/4 r64.
1632 emit_optional_rex_32(target);
1633 emit(0xFF);
1634 emit_modrm(0x4, target);
1635 }
1636
jmp(Operand src)1637 void Assembler::jmp(Operand src) {
1638 EnsureSpace ensure_space(this);
1639 // Opcode FF/4 m64.
1640 emit_optional_rex_32(src);
1641 emit(0xFF);
1642 emit_operand(0x4, src);
1643 }
1644
emit_lea(Register dst,Operand src,int size)1645 void Assembler::emit_lea(Register dst, Operand src, int size) {
1646 EnsureSpace ensure_space(this);
1647 emit_rex(dst, src, size);
1648 emit(0x8D);
1649 emit_operand(dst, src);
1650 }
1651
load_rax(Address value,RelocInfo::Mode mode)1652 void Assembler::load_rax(Address value, RelocInfo::Mode mode) {
1653 EnsureSpace ensure_space(this);
1654 if (kPointerSize == kInt64Size) {
1655 emit(0x48); // REX.W
1656 emit(0xA1);
1657 emitp(value, mode);
1658 } else {
1659 DCHECK_EQ(kPointerSize, kInt32Size);
1660 emit(0xA1);
1661 emitp(value, mode);
1662 // In 64-bit mode, need to zero extend the operand to 8 bytes.
1663 // See 2.2.1.4 in Intel64 and IA32 Architectures Software
1664 // Developer's Manual Volume 2.
1665 emitl(0);
1666 }
1667 }
1668
1669
load_rax(ExternalReference ref)1670 void Assembler::load_rax(ExternalReference ref) {
1671 load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
1672 }
1673
1674
leave()1675 void Assembler::leave() {
1676 EnsureSpace ensure_space(this);
1677 emit(0xC9);
1678 }
1679
movb(Register dst,Operand src)1680 void Assembler::movb(Register dst, Operand src) {
1681 EnsureSpace ensure_space(this);
1682 if (!dst.is_byte_register()) {
1683 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1684 emit_rex_32(dst, src);
1685 } else {
1686 emit_optional_rex_32(dst, src);
1687 }
1688 emit(0x8A);
1689 emit_operand(dst, src);
1690 }
1691
1692
movb(Register dst,Immediate imm)1693 void Assembler::movb(Register dst, Immediate imm) {
1694 EnsureSpace ensure_space(this);
1695 if (!dst.is_byte_register()) {
1696 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1697 emit_rex_32(dst);
1698 }
1699 emit(0xB0 + dst.low_bits());
1700 emit(imm.value_);
1701 }
1702
movb(Operand dst,Register src)1703 void Assembler::movb(Operand dst, Register src) {
1704 EnsureSpace ensure_space(this);
1705 if (!src.is_byte_register()) {
1706 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1707 emit_rex_32(src, dst);
1708 } else {
1709 emit_optional_rex_32(src, dst);
1710 }
1711 emit(0x88);
1712 emit_operand(src, dst);
1713 }
1714
movb(Operand dst,Immediate imm)1715 void Assembler::movb(Operand dst, Immediate imm) {
1716 EnsureSpace ensure_space(this);
1717 emit_optional_rex_32(dst);
1718 emit(0xC6);
1719 emit_operand(0x0, dst);
1720 emit(static_cast<byte>(imm.value_));
1721 }
1722
movw(Register dst,Operand src)1723 void Assembler::movw(Register dst, Operand src) {
1724 EnsureSpace ensure_space(this);
1725 emit(0x66);
1726 emit_optional_rex_32(dst, src);
1727 emit(0x8B);
1728 emit_operand(dst, src);
1729 }
1730
movw(Operand dst,Register src)1731 void Assembler::movw(Operand dst, Register src) {
1732 EnsureSpace ensure_space(this);
1733 emit(0x66);
1734 emit_optional_rex_32(src, dst);
1735 emit(0x89);
1736 emit_operand(src, dst);
1737 }
1738
movw(Operand dst,Immediate imm)1739 void Assembler::movw(Operand dst, Immediate imm) {
1740 EnsureSpace ensure_space(this);
1741 emit(0x66);
1742 emit_optional_rex_32(dst);
1743 emit(0xC7);
1744 emit_operand(0x0, dst);
1745 emit(static_cast<byte>(imm.value_ & 0xFF));
1746 emit(static_cast<byte>(imm.value_ >> 8));
1747 }
1748
emit_mov(Register dst,Operand src,int size)1749 void Assembler::emit_mov(Register dst, Operand src, int size) {
1750 EnsureSpace ensure_space(this);
1751 emit_rex(dst, src, size);
1752 emit(0x8B);
1753 emit_operand(dst, src);
1754 }
1755
1756
emit_mov(Register dst,Register src,int size)1757 void Assembler::emit_mov(Register dst, Register src, int size) {
1758 EnsureSpace ensure_space(this);
1759 if (src.low_bits() == 4) {
1760 emit_rex(src, dst, size);
1761 emit(0x89);
1762 emit_modrm(src, dst);
1763 } else {
1764 emit_rex(dst, src, size);
1765 emit(0x8B);
1766 emit_modrm(dst, src);
1767 }
1768 }
1769
emit_mov(Operand dst,Register src,int size)1770 void Assembler::emit_mov(Operand dst, Register src, int size) {
1771 EnsureSpace ensure_space(this);
1772 emit_rex(src, dst, size);
1773 emit(0x89);
1774 emit_operand(src, dst);
1775 }
1776
1777
emit_mov(Register dst,Immediate value,int size)1778 void Assembler::emit_mov(Register dst, Immediate value, int size) {
1779 EnsureSpace ensure_space(this);
1780 emit_rex(dst, size);
1781 if (size == kInt64Size) {
1782 emit(0xC7);
1783 emit_modrm(0x0, dst);
1784 } else {
1785 DCHECK_EQ(size, kInt32Size);
1786 emit(0xB8 + dst.low_bits());
1787 }
1788 emit(value);
1789 }
1790
emit_mov(Operand dst,Immediate value,int size)1791 void Assembler::emit_mov(Operand dst, Immediate value, int size) {
1792 EnsureSpace ensure_space(this);
1793 emit_rex(dst, size);
1794 emit(0xC7);
1795 emit_operand(0x0, dst);
1796 emit(value);
1797 }
1798
movp(Register dst,Address value,RelocInfo::Mode rmode)1799 void Assembler::movp(Register dst, Address value, RelocInfo::Mode rmode) {
1800 if (constpool_.TryRecordEntry(value, rmode)) {
1801 // Emit rip-relative move with offset = 0
1802 Label label;
1803 emit_mov(dst, Operand(&label, 0), kPointerSize);
1804 bind(&label);
1805 } else {
1806 EnsureSpace ensure_space(this);
1807 emit_rex(dst, kPointerSize);
1808 emit(0xB8 | dst.low_bits());
1809 emitp(value, rmode);
1810 }
1811 }
1812
movp_heap_number(Register dst,double value)1813 void Assembler::movp_heap_number(Register dst, double value) {
1814 EnsureSpace ensure_space(this);
1815 emit_rex(dst, kPointerSize);
1816 emit(0xB8 | dst.low_bits());
1817 RequestHeapObject(HeapObjectRequest(value));
1818 emitp(0, RelocInfo::EMBEDDED_OBJECT);
1819 }
1820
movq(Register dst,int64_t value,RelocInfo::Mode rmode)1821 void Assembler::movq(Register dst, int64_t value, RelocInfo::Mode rmode) {
1822 if (constpool_.TryRecordEntry(value, rmode)) {
1823 // Emit rip-relative move with offset = 0
1824 Label label;
1825 emit_mov(dst, Operand(&label, 0), kPointerSize);
1826 bind(&label);
1827 } else {
1828 EnsureSpace ensure_space(this);
1829 emit_rex_64(dst);
1830 emit(0xB8 | dst.low_bits());
1831 if (!RelocInfo::IsNone(rmode)) {
1832 RecordRelocInfo(rmode, value);
1833 }
1834 emitq(value);
1835 }
1836 }
1837
movq(Register dst,uint64_t value,RelocInfo::Mode rmode)1838 void Assembler::movq(Register dst, uint64_t value, RelocInfo::Mode rmode) {
1839 movq(dst, static_cast<int64_t>(value), rmode);
1840 }
1841
1842 // Loads the ip-relative location of the src label into the target location
1843 // (as a 32-bit offset sign extended to 64-bit).
movl(Operand dst,Label * src)1844 void Assembler::movl(Operand dst, Label* src) {
1845 EnsureSpace ensure_space(this);
1846 emit_optional_rex_32(dst);
1847 emit(0xC7);
1848 emit_operand(0, dst);
1849 if (src->is_bound()) {
1850 int offset = src->pos() - pc_offset() - sizeof(int32_t);
1851 DCHECK_LE(offset, 0);
1852 emitl(offset);
1853 } else if (src->is_linked()) {
1854 emitl(src->pos());
1855 src->link_to(pc_offset() - sizeof(int32_t));
1856 } else {
1857 DCHECK(src->is_unused());
1858 int32_t current = pc_offset();
1859 emitl(current);
1860 src->link_to(current);
1861 }
1862 }
1863
1864
movsxbl(Register dst,Register src)1865 void Assembler::movsxbl(Register dst, Register src) {
1866 EnsureSpace ensure_space(this);
1867 if (!src.is_byte_register()) {
1868 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1869 emit_rex_32(dst, src);
1870 } else {
1871 emit_optional_rex_32(dst, src);
1872 }
1873 emit(0x0F);
1874 emit(0xBE);
1875 emit_modrm(dst, src);
1876 }
1877
movsxbl(Register dst,Operand src)1878 void Assembler::movsxbl(Register dst, Operand src) {
1879 EnsureSpace ensure_space(this);
1880 emit_optional_rex_32(dst, src);
1881 emit(0x0F);
1882 emit(0xBE);
1883 emit_operand(dst, src);
1884 }
1885
movsxbq(Register dst,Operand src)1886 void Assembler::movsxbq(Register dst, Operand src) {
1887 EnsureSpace ensure_space(this);
1888 emit_rex_64(dst, src);
1889 emit(0x0F);
1890 emit(0xBE);
1891 emit_operand(dst, src);
1892 }
1893
movsxbq(Register dst,Register src)1894 void Assembler::movsxbq(Register dst, Register src) {
1895 EnsureSpace ensure_space(this);
1896 emit_rex_64(dst, src);
1897 emit(0x0F);
1898 emit(0xBE);
1899 emit_modrm(dst, src);
1900 }
1901
movsxwl(Register dst,Register src)1902 void Assembler::movsxwl(Register dst, Register src) {
1903 EnsureSpace ensure_space(this);
1904 emit_optional_rex_32(dst, src);
1905 emit(0x0F);
1906 emit(0xBF);
1907 emit_modrm(dst, src);
1908 }
1909
movsxwl(Register dst,Operand src)1910 void Assembler::movsxwl(Register dst, Operand src) {
1911 EnsureSpace ensure_space(this);
1912 emit_optional_rex_32(dst, src);
1913 emit(0x0F);
1914 emit(0xBF);
1915 emit_operand(dst, src);
1916 }
1917
movsxwq(Register dst,Operand src)1918 void Assembler::movsxwq(Register dst, Operand src) {
1919 EnsureSpace ensure_space(this);
1920 emit_rex_64(dst, src);
1921 emit(0x0F);
1922 emit(0xBF);
1923 emit_operand(dst, src);
1924 }
1925
movsxwq(Register dst,Register src)1926 void Assembler::movsxwq(Register dst, Register src) {
1927 EnsureSpace ensure_space(this);
1928 emit_rex_64(dst, src);
1929 emit(0x0F);
1930 emit(0xBF);
1931 emit_modrm(dst, src);
1932 }
1933
movsxlq(Register dst,Register src)1934 void Assembler::movsxlq(Register dst, Register src) {
1935 EnsureSpace ensure_space(this);
1936 emit_rex_64(dst, src);
1937 emit(0x63);
1938 emit_modrm(dst, src);
1939 }
1940
movsxlq(Register dst,Operand src)1941 void Assembler::movsxlq(Register dst, Operand src) {
1942 EnsureSpace ensure_space(this);
1943 emit_rex_64(dst, src);
1944 emit(0x63);
1945 emit_operand(dst, src);
1946 }
1947
emit_movzxb(Register dst,Operand src,int size)1948 void Assembler::emit_movzxb(Register dst, Operand src, int size) {
1949 EnsureSpace ensure_space(this);
1950 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1951 // there is no need to make this a 64 bit operation.
1952 emit_optional_rex_32(dst, src);
1953 emit(0x0F);
1954 emit(0xB6);
1955 emit_operand(dst, src);
1956 }
1957
1958
emit_movzxb(Register dst,Register src,int size)1959 void Assembler::emit_movzxb(Register dst, Register src, int size) {
1960 EnsureSpace ensure_space(this);
1961 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1962 // there is no need to make this a 64 bit operation.
1963 if (!src.is_byte_register()) {
1964 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1965 emit_rex_32(dst, src);
1966 } else {
1967 emit_optional_rex_32(dst, src);
1968 }
1969 emit(0x0F);
1970 emit(0xB6);
1971 emit_modrm(dst, src);
1972 }
1973
emit_movzxw(Register dst,Operand src,int size)1974 void Assembler::emit_movzxw(Register dst, Operand src, int size) {
1975 EnsureSpace ensure_space(this);
1976 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1977 // there is no need to make this a 64 bit operation.
1978 emit_optional_rex_32(dst, src);
1979 emit(0x0F);
1980 emit(0xB7);
1981 emit_operand(dst, src);
1982 }
1983
1984
emit_movzxw(Register dst,Register src,int size)1985 void Assembler::emit_movzxw(Register dst, Register src, int size) {
1986 EnsureSpace ensure_space(this);
1987 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1988 // there is no need to make this a 64 bit operation.
1989 emit_optional_rex_32(dst, src);
1990 emit(0x0F);
1991 emit(0xB7);
1992 emit_modrm(dst, src);
1993 }
1994
1995
repmovsb()1996 void Assembler::repmovsb() {
1997 EnsureSpace ensure_space(this);
1998 emit(0xF3);
1999 emit(0xA4);
2000 }
2001
2002
repmovsw()2003 void Assembler::repmovsw() {
2004 EnsureSpace ensure_space(this);
2005 emit(0x66); // Operand size override.
2006 emit(0xF3);
2007 emit(0xA4);
2008 }
2009
2010
emit_repmovs(int size)2011 void Assembler::emit_repmovs(int size) {
2012 EnsureSpace ensure_space(this);
2013 emit(0xF3);
2014 emit_rex(size);
2015 emit(0xA5);
2016 }
2017
2018
mull(Register src)2019 void Assembler::mull(Register src) {
2020 EnsureSpace ensure_space(this);
2021 emit_optional_rex_32(src);
2022 emit(0xF7);
2023 emit_modrm(0x4, src);
2024 }
2025
mull(Operand src)2026 void Assembler::mull(Operand src) {
2027 EnsureSpace ensure_space(this);
2028 emit_optional_rex_32(src);
2029 emit(0xF7);
2030 emit_operand(0x4, src);
2031 }
2032
2033
mulq(Register src)2034 void Assembler::mulq(Register src) {
2035 EnsureSpace ensure_space(this);
2036 emit_rex_64(src);
2037 emit(0xF7);
2038 emit_modrm(0x4, src);
2039 }
2040
2041
emit_neg(Register dst,int size)2042 void Assembler::emit_neg(Register dst, int size) {
2043 EnsureSpace ensure_space(this);
2044 emit_rex(dst, size);
2045 emit(0xF7);
2046 emit_modrm(0x3, dst);
2047 }
2048
emit_neg(Operand dst,int size)2049 void Assembler::emit_neg(Operand dst, int size) {
2050 EnsureSpace ensure_space(this);
2051 emit_rex_64(dst);
2052 emit(0xF7);
2053 emit_operand(3, dst);
2054 }
2055
2056
nop()2057 void Assembler::nop() {
2058 EnsureSpace ensure_space(this);
2059 emit(0x90);
2060 }
2061
2062
emit_not(Register dst,int size)2063 void Assembler::emit_not(Register dst, int size) {
2064 EnsureSpace ensure_space(this);
2065 emit_rex(dst, size);
2066 emit(0xF7);
2067 emit_modrm(0x2, dst);
2068 }
2069
emit_not(Operand dst,int size)2070 void Assembler::emit_not(Operand dst, int size) {
2071 EnsureSpace ensure_space(this);
2072 emit_rex(dst, size);
2073 emit(0xF7);
2074 emit_operand(2, dst);
2075 }
2076
2077
Nop(int n)2078 void Assembler::Nop(int n) {
2079 // The recommended muti-byte sequences of NOP instructions from the Intel 64
2080 // and IA-32 Architectures Software Developer's Manual.
2081 //
2082 // Length Assembly Byte Sequence
2083 // 2 bytes 66 NOP 66 90H
2084 // 3 bytes NOP DWORD ptr [EAX] 0F 1F 00H
2085 // 4 bytes NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H
2086 // 5 bytes NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H
2087 // 6 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H
2088 // 7 bytes NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H
2089 // 8 bytes NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H
2090 // 9 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 66 0F 1F 84 00 00 00 00
2091 // 00000000H] 00H
2092
2093 EnsureSpace ensure_space(this);
2094 while (n > 0) {
2095 switch (n) {
2096 case 2:
2097 emit(0x66);
2098 V8_FALLTHROUGH;
2099 case 1:
2100 emit(0x90);
2101 return;
2102 case 3:
2103 emit(0x0F);
2104 emit(0x1F);
2105 emit(0x00);
2106 return;
2107 case 4:
2108 emit(0x0F);
2109 emit(0x1F);
2110 emit(0x40);
2111 emit(0x00);
2112 return;
2113 case 6:
2114 emit(0x66);
2115 V8_FALLTHROUGH;
2116 case 5:
2117 emit(0x0F);
2118 emit(0x1F);
2119 emit(0x44);
2120 emit(0x00);
2121 emit(0x00);
2122 return;
2123 case 7:
2124 emit(0x0F);
2125 emit(0x1F);
2126 emit(0x80);
2127 emit(0x00);
2128 emit(0x00);
2129 emit(0x00);
2130 emit(0x00);
2131 return;
2132 default:
2133 case 11:
2134 emit(0x66);
2135 n--;
2136 V8_FALLTHROUGH;
2137 case 10:
2138 emit(0x66);
2139 n--;
2140 V8_FALLTHROUGH;
2141 case 9:
2142 emit(0x66);
2143 n--;
2144 V8_FALLTHROUGH;
2145 case 8:
2146 emit(0x0F);
2147 emit(0x1F);
2148 emit(0x84);
2149 emit(0x00);
2150 emit(0x00);
2151 emit(0x00);
2152 emit(0x00);
2153 emit(0x00);
2154 n -= 8;
2155 }
2156 }
2157 }
2158
2159
popq(Register dst)2160 void Assembler::popq(Register dst) {
2161 EnsureSpace ensure_space(this);
2162 emit_optional_rex_32(dst);
2163 emit(0x58 | dst.low_bits());
2164 }
2165
popq(Operand dst)2166 void Assembler::popq(Operand dst) {
2167 EnsureSpace ensure_space(this);
2168 emit_optional_rex_32(dst);
2169 emit(0x8F);
2170 emit_operand(0, dst);
2171 }
2172
2173
popfq()2174 void Assembler::popfq() {
2175 EnsureSpace ensure_space(this);
2176 emit(0x9D);
2177 }
2178
2179
pushq(Register src)2180 void Assembler::pushq(Register src) {
2181 EnsureSpace ensure_space(this);
2182 emit_optional_rex_32(src);
2183 emit(0x50 | src.low_bits());
2184 }
2185
pushq(Operand src)2186 void Assembler::pushq(Operand src) {
2187 EnsureSpace ensure_space(this);
2188 emit_optional_rex_32(src);
2189 emit(0xFF);
2190 emit_operand(6, src);
2191 }
2192
2193
pushq(Immediate value)2194 void Assembler::pushq(Immediate value) {
2195 EnsureSpace ensure_space(this);
2196 if (is_int8(value.value_)) {
2197 emit(0x6A);
2198 emit(value.value_); // Emit low byte of value.
2199 } else {
2200 emit(0x68);
2201 emitl(value.value_);
2202 }
2203 }
2204
2205
pushq_imm32(int32_t imm32)2206 void Assembler::pushq_imm32(int32_t imm32) {
2207 EnsureSpace ensure_space(this);
2208 emit(0x68);
2209 emitl(imm32);
2210 }
2211
2212
pushfq()2213 void Assembler::pushfq() {
2214 EnsureSpace ensure_space(this);
2215 emit(0x9C);
2216 }
2217
2218
ret(int imm16)2219 void Assembler::ret(int imm16) {
2220 EnsureSpace ensure_space(this);
2221 DCHECK(is_uint16(imm16));
2222 if (imm16 == 0) {
2223 emit(0xC3);
2224 } else {
2225 emit(0xC2);
2226 emit(imm16 & 0xFF);
2227 emit((imm16 >> 8) & 0xFF);
2228 }
2229 }
2230
2231
ud2()2232 void Assembler::ud2() {
2233 EnsureSpace ensure_space(this);
2234 emit(0x0F);
2235 emit(0x0B);
2236 }
2237
2238
setcc(Condition cc,Register reg)2239 void Assembler::setcc(Condition cc, Register reg) {
2240 if (cc > last_condition) {
2241 movb(reg, Immediate(cc == always ? 1 : 0));
2242 return;
2243 }
2244 EnsureSpace ensure_space(this);
2245 DCHECK(is_uint4(cc));
2246 if (!reg.is_byte_register()) {
2247 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2248 emit_rex_32(reg);
2249 }
2250 emit(0x0F);
2251 emit(0x90 | cc);
2252 emit_modrm(0x0, reg);
2253 }
2254
2255
shld(Register dst,Register src)2256 void Assembler::shld(Register dst, Register src) {
2257 EnsureSpace ensure_space(this);
2258 emit_rex_64(src, dst);
2259 emit(0x0F);
2260 emit(0xA5);
2261 emit_modrm(src, dst);
2262 }
2263
2264
shrd(Register dst,Register src)2265 void Assembler::shrd(Register dst, Register src) {
2266 EnsureSpace ensure_space(this);
2267 emit_rex_64(src, dst);
2268 emit(0x0F);
2269 emit(0xAD);
2270 emit_modrm(src, dst);
2271 }
2272
xchgb(Register reg,Operand op)2273 void Assembler::xchgb(Register reg, Operand op) {
2274 EnsureSpace ensure_space(this);
2275 if (!reg.is_byte_register()) {
2276 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2277 emit_rex_32(reg, op);
2278 } else {
2279 emit_optional_rex_32(reg, op);
2280 }
2281 emit(0x86);
2282 emit_operand(reg, op);
2283 }
2284
xchgw(Register reg,Operand op)2285 void Assembler::xchgw(Register reg, Operand op) {
2286 EnsureSpace ensure_space(this);
2287 emit(0x66);
2288 emit_optional_rex_32(reg, op);
2289 emit(0x87);
2290 emit_operand(reg, op);
2291 }
2292
emit_xchg(Register dst,Register src,int size)2293 void Assembler::emit_xchg(Register dst, Register src, int size) {
2294 EnsureSpace ensure_space(this);
2295 if (src == rax || dst == rax) { // Single-byte encoding
2296 Register other = src == rax ? dst : src;
2297 emit_rex(other, size);
2298 emit(0x90 | other.low_bits());
2299 } else if (dst.low_bits() == 4) {
2300 emit_rex(dst, src, size);
2301 emit(0x87);
2302 emit_modrm(dst, src);
2303 } else {
2304 emit_rex(src, dst, size);
2305 emit(0x87);
2306 emit_modrm(src, dst);
2307 }
2308 }
2309
emit_xchg(Register dst,Operand src,int size)2310 void Assembler::emit_xchg(Register dst, Operand src, int size) {
2311 EnsureSpace ensure_space(this);
2312 emit_rex(dst, src, size);
2313 emit(0x87);
2314 emit_operand(dst, src);
2315 }
2316
store_rax(Address dst,RelocInfo::Mode mode)2317 void Assembler::store_rax(Address dst, RelocInfo::Mode mode) {
2318 EnsureSpace ensure_space(this);
2319 if (kPointerSize == kInt64Size) {
2320 emit(0x48); // REX.W
2321 emit(0xA3);
2322 emitp(dst, mode);
2323 } else {
2324 DCHECK_EQ(kPointerSize, kInt32Size);
2325 emit(0xA3);
2326 emitp(dst, mode);
2327 // In 64-bit mode, need to zero extend the operand to 8 bytes.
2328 // See 2.2.1.4 in Intel64 and IA32 Architectures Software
2329 // Developer's Manual Volume 2.
2330 emitl(0);
2331 }
2332 }
2333
2334
store_rax(ExternalReference ref)2335 void Assembler::store_rax(ExternalReference ref) {
2336 store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
2337 }
2338
sub_sp_32(uint32_t imm)2339 void Assembler::sub_sp_32(uint32_t imm) {
2340 emit_rex_64();
2341 emit(0x81); // using a literal 32-bit immediate.
2342 emit_modrm(0x5, rsp);
2343 emitl(imm);
2344 }
2345
testb(Register dst,Register src)2346 void Assembler::testb(Register dst, Register src) {
2347 EnsureSpace ensure_space(this);
2348 emit_test(dst, src, sizeof(int8_t));
2349 }
2350
testb(Register reg,Immediate mask)2351 void Assembler::testb(Register reg, Immediate mask) {
2352 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
2353 emit_test(reg, mask, sizeof(int8_t));
2354 }
2355
testb(Operand op,Immediate mask)2356 void Assembler::testb(Operand op, Immediate mask) {
2357 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
2358 emit_test(op, mask, sizeof(int8_t));
2359 }
2360
testb(Operand op,Register reg)2361 void Assembler::testb(Operand op, Register reg) {
2362 emit_test(op, reg, sizeof(int8_t));
2363 }
2364
testw(Register dst,Register src)2365 void Assembler::testw(Register dst, Register src) {
2366 emit_test(dst, src, sizeof(uint16_t));
2367 }
2368
testw(Register reg,Immediate mask)2369 void Assembler::testw(Register reg, Immediate mask) {
2370 emit_test(reg, mask, sizeof(int16_t));
2371 }
2372
testw(Operand op,Immediate mask)2373 void Assembler::testw(Operand op, Immediate mask) {
2374 emit_test(op, mask, sizeof(int16_t));
2375 }
2376
testw(Operand op,Register reg)2377 void Assembler::testw(Operand op, Register reg) {
2378 emit_test(op, reg, sizeof(int16_t));
2379 }
2380
emit_test(Register dst,Register src,int size)2381 void Assembler::emit_test(Register dst, Register src, int size) {
2382 EnsureSpace ensure_space(this);
2383 if (src.low_bits() == 4) std::swap(dst, src);
2384 if (size == sizeof(int16_t)) {
2385 emit(0x66);
2386 size = sizeof(int32_t);
2387 }
2388 bool byte_operand = size == sizeof(int8_t);
2389 if (byte_operand) {
2390 size = sizeof(int32_t);
2391 if (!src.is_byte_register() || !dst.is_byte_register()) {
2392 emit_rex_32(dst, src);
2393 }
2394 } else {
2395 emit_rex(dst, src, size);
2396 }
2397 emit(byte_operand ? 0x84 : 0x85);
2398 emit_modrm(dst, src);
2399 }
2400
2401
emit_test(Register reg,Immediate mask,int size)2402 void Assembler::emit_test(Register reg, Immediate mask, int size) {
2403 if (is_uint8(mask.value_)) {
2404 size = sizeof(int8_t);
2405 } else if (is_uint16(mask.value_)) {
2406 size = sizeof(int16_t);
2407 }
2408 EnsureSpace ensure_space(this);
2409 bool half_word = size == sizeof(int16_t);
2410 if (half_word) {
2411 emit(0x66);
2412 size = sizeof(int32_t);
2413 }
2414 bool byte_operand = size == sizeof(int8_t);
2415 if (byte_operand) {
2416 size = sizeof(int32_t);
2417 if (!reg.is_byte_register()) emit_rex_32(reg);
2418 } else {
2419 emit_rex(reg, size);
2420 }
2421 if (reg == rax) {
2422 emit(byte_operand ? 0xA8 : 0xA9);
2423 } else {
2424 emit(byte_operand ? 0xF6 : 0xF7);
2425 emit_modrm(0x0, reg);
2426 }
2427 if (byte_operand) {
2428 emit(mask.value_);
2429 } else if (half_word) {
2430 emitw(mask.value_);
2431 } else {
2432 emit(mask);
2433 }
2434 }
2435
emit_test(Operand op,Immediate mask,int size)2436 void Assembler::emit_test(Operand op, Immediate mask, int size) {
2437 if (is_uint8(mask.value_)) {
2438 size = sizeof(int8_t);
2439 } else if (is_uint16(mask.value_)) {
2440 size = sizeof(int16_t);
2441 }
2442 EnsureSpace ensure_space(this);
2443 bool half_word = size == sizeof(int16_t);
2444 if (half_word) {
2445 emit(0x66);
2446 size = sizeof(int32_t);
2447 }
2448 bool byte_operand = size == sizeof(int8_t);
2449 if (byte_operand) {
2450 size = sizeof(int32_t);
2451 }
2452 emit_rex(rax, op, size);
2453 emit(byte_operand ? 0xF6 : 0xF7);
2454 emit_operand(rax, op); // Operation code 0
2455 if (byte_operand) {
2456 emit(mask.value_);
2457 } else if (half_word) {
2458 emitw(mask.value_);
2459 } else {
2460 emit(mask);
2461 }
2462 }
2463
emit_test(Operand op,Register reg,int size)2464 void Assembler::emit_test(Operand op, Register reg, int size) {
2465 EnsureSpace ensure_space(this);
2466 if (size == sizeof(int16_t)) {
2467 emit(0x66);
2468 size = sizeof(int32_t);
2469 }
2470 bool byte_operand = size == sizeof(int8_t);
2471 if (byte_operand) {
2472 size = sizeof(int32_t);
2473 if (!reg.is_byte_register()) {
2474 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2475 emit_rex_32(reg, op);
2476 } else {
2477 emit_optional_rex_32(reg, op);
2478 }
2479 } else {
2480 emit_rex(reg, op, size);
2481 }
2482 emit(byte_operand ? 0x84 : 0x85);
2483 emit_operand(reg, op);
2484 }
2485
2486
2487 // FPU instructions.
2488
2489
fld(int i)2490 void Assembler::fld(int i) {
2491 EnsureSpace ensure_space(this);
2492 emit_farith(0xD9, 0xC0, i);
2493 }
2494
2495
fld1()2496 void Assembler::fld1() {
2497 EnsureSpace ensure_space(this);
2498 emit(0xD9);
2499 emit(0xE8);
2500 }
2501
2502
fldz()2503 void Assembler::fldz() {
2504 EnsureSpace ensure_space(this);
2505 emit(0xD9);
2506 emit(0xEE);
2507 }
2508
2509
fldpi()2510 void Assembler::fldpi() {
2511 EnsureSpace ensure_space(this);
2512 emit(0xD9);
2513 emit(0xEB);
2514 }
2515
2516
fldln2()2517 void Assembler::fldln2() {
2518 EnsureSpace ensure_space(this);
2519 emit(0xD9);
2520 emit(0xED);
2521 }
2522
fld_s(Operand adr)2523 void Assembler::fld_s(Operand adr) {
2524 EnsureSpace ensure_space(this);
2525 emit_optional_rex_32(adr);
2526 emit(0xD9);
2527 emit_operand(0, adr);
2528 }
2529
fld_d(Operand adr)2530 void Assembler::fld_d(Operand adr) {
2531 EnsureSpace ensure_space(this);
2532 emit_optional_rex_32(adr);
2533 emit(0xDD);
2534 emit_operand(0, adr);
2535 }
2536
fstp_s(Operand adr)2537 void Assembler::fstp_s(Operand adr) {
2538 EnsureSpace ensure_space(this);
2539 emit_optional_rex_32(adr);
2540 emit(0xD9);
2541 emit_operand(3, adr);
2542 }
2543
fstp_d(Operand adr)2544 void Assembler::fstp_d(Operand adr) {
2545 EnsureSpace ensure_space(this);
2546 emit_optional_rex_32(adr);
2547 emit(0xDD);
2548 emit_operand(3, adr);
2549 }
2550
2551
fstp(int index)2552 void Assembler::fstp(int index) {
2553 DCHECK(is_uint3(index));
2554 EnsureSpace ensure_space(this);
2555 emit_farith(0xDD, 0xD8, index);
2556 }
2557
fild_s(Operand adr)2558 void Assembler::fild_s(Operand adr) {
2559 EnsureSpace ensure_space(this);
2560 emit_optional_rex_32(adr);
2561 emit(0xDB);
2562 emit_operand(0, adr);
2563 }
2564
fild_d(Operand adr)2565 void Assembler::fild_d(Operand adr) {
2566 EnsureSpace ensure_space(this);
2567 emit_optional_rex_32(adr);
2568 emit(0xDF);
2569 emit_operand(5, adr);
2570 }
2571
fistp_s(Operand adr)2572 void Assembler::fistp_s(Operand adr) {
2573 EnsureSpace ensure_space(this);
2574 emit_optional_rex_32(adr);
2575 emit(0xDB);
2576 emit_operand(3, adr);
2577 }
2578
fisttp_s(Operand adr)2579 void Assembler::fisttp_s(Operand adr) {
2580 DCHECK(IsEnabled(SSE3));
2581 EnsureSpace ensure_space(this);
2582 emit_optional_rex_32(adr);
2583 emit(0xDB);
2584 emit_operand(1, adr);
2585 }
2586
fisttp_d(Operand adr)2587 void Assembler::fisttp_d(Operand adr) {
2588 DCHECK(IsEnabled(SSE3));
2589 EnsureSpace ensure_space(this);
2590 emit_optional_rex_32(adr);
2591 emit(0xDD);
2592 emit_operand(1, adr);
2593 }
2594
fist_s(Operand adr)2595 void Assembler::fist_s(Operand adr) {
2596 EnsureSpace ensure_space(this);
2597 emit_optional_rex_32(adr);
2598 emit(0xDB);
2599 emit_operand(2, adr);
2600 }
2601
fistp_d(Operand adr)2602 void Assembler::fistp_d(Operand adr) {
2603 EnsureSpace ensure_space(this);
2604 emit_optional_rex_32(adr);
2605 emit(0xDF);
2606 emit_operand(7, adr);
2607 }
2608
2609
fabs()2610 void Assembler::fabs() {
2611 EnsureSpace ensure_space(this);
2612 emit(0xD9);
2613 emit(0xE1);
2614 }
2615
2616
fchs()2617 void Assembler::fchs() {
2618 EnsureSpace ensure_space(this);
2619 emit(0xD9);
2620 emit(0xE0);
2621 }
2622
2623
fcos()2624 void Assembler::fcos() {
2625 EnsureSpace ensure_space(this);
2626 emit(0xD9);
2627 emit(0xFF);
2628 }
2629
2630
fsin()2631 void Assembler::fsin() {
2632 EnsureSpace ensure_space(this);
2633 emit(0xD9);
2634 emit(0xFE);
2635 }
2636
2637
fptan()2638 void Assembler::fptan() {
2639 EnsureSpace ensure_space(this);
2640 emit(0xD9);
2641 emit(0xF2);
2642 }
2643
2644
fyl2x()2645 void Assembler::fyl2x() {
2646 EnsureSpace ensure_space(this);
2647 emit(0xD9);
2648 emit(0xF1);
2649 }
2650
2651
f2xm1()2652 void Assembler::f2xm1() {
2653 EnsureSpace ensure_space(this);
2654 emit(0xD9);
2655 emit(0xF0);
2656 }
2657
2658
fscale()2659 void Assembler::fscale() {
2660 EnsureSpace ensure_space(this);
2661 emit(0xD9);
2662 emit(0xFD);
2663 }
2664
2665
fninit()2666 void Assembler::fninit() {
2667 EnsureSpace ensure_space(this);
2668 emit(0xDB);
2669 emit(0xE3);
2670 }
2671
2672
fadd(int i)2673 void Assembler::fadd(int i) {
2674 EnsureSpace ensure_space(this);
2675 emit_farith(0xDC, 0xC0, i);
2676 }
2677
2678
fsub(int i)2679 void Assembler::fsub(int i) {
2680 EnsureSpace ensure_space(this);
2681 emit_farith(0xDC, 0xE8, i);
2682 }
2683
fisub_s(Operand adr)2684 void Assembler::fisub_s(Operand adr) {
2685 EnsureSpace ensure_space(this);
2686 emit_optional_rex_32(adr);
2687 emit(0xDA);
2688 emit_operand(4, adr);
2689 }
2690
2691
fmul(int i)2692 void Assembler::fmul(int i) {
2693 EnsureSpace ensure_space(this);
2694 emit_farith(0xDC, 0xC8, i);
2695 }
2696
2697
fdiv(int i)2698 void Assembler::fdiv(int i) {
2699 EnsureSpace ensure_space(this);
2700 emit_farith(0xDC, 0xF8, i);
2701 }
2702
2703
faddp(int i)2704 void Assembler::faddp(int i) {
2705 EnsureSpace ensure_space(this);
2706 emit_farith(0xDE, 0xC0, i);
2707 }
2708
2709
fsubp(int i)2710 void Assembler::fsubp(int i) {
2711 EnsureSpace ensure_space(this);
2712 emit_farith(0xDE, 0xE8, i);
2713 }
2714
2715
fsubrp(int i)2716 void Assembler::fsubrp(int i) {
2717 EnsureSpace ensure_space(this);
2718 emit_farith(0xDE, 0xE0, i);
2719 }
2720
2721
fmulp(int i)2722 void Assembler::fmulp(int i) {
2723 EnsureSpace ensure_space(this);
2724 emit_farith(0xDE, 0xC8, i);
2725 }
2726
2727
fdivp(int i)2728 void Assembler::fdivp(int i) {
2729 EnsureSpace ensure_space(this);
2730 emit_farith(0xDE, 0xF8, i);
2731 }
2732
2733
fprem()2734 void Assembler::fprem() {
2735 EnsureSpace ensure_space(this);
2736 emit(0xD9);
2737 emit(0xF8);
2738 }
2739
2740
fprem1()2741 void Assembler::fprem1() {
2742 EnsureSpace ensure_space(this);
2743 emit(0xD9);
2744 emit(0xF5);
2745 }
2746
2747
fxch(int i)2748 void Assembler::fxch(int i) {
2749 EnsureSpace ensure_space(this);
2750 emit_farith(0xD9, 0xC8, i);
2751 }
2752
2753
fincstp()2754 void Assembler::fincstp() {
2755 EnsureSpace ensure_space(this);
2756 emit(0xD9);
2757 emit(0xF7);
2758 }
2759
2760
ffree(int i)2761 void Assembler::ffree(int i) {
2762 EnsureSpace ensure_space(this);
2763 emit_farith(0xDD, 0xC0, i);
2764 }
2765
2766
ftst()2767 void Assembler::ftst() {
2768 EnsureSpace ensure_space(this);
2769 emit(0xD9);
2770 emit(0xE4);
2771 }
2772
2773
fucomp(int i)2774 void Assembler::fucomp(int i) {
2775 EnsureSpace ensure_space(this);
2776 emit_farith(0xDD, 0xE8, i);
2777 }
2778
2779
fucompp()2780 void Assembler::fucompp() {
2781 EnsureSpace ensure_space(this);
2782 emit(0xDA);
2783 emit(0xE9);
2784 }
2785
2786
fucomi(int i)2787 void Assembler::fucomi(int i) {
2788 EnsureSpace ensure_space(this);
2789 emit(0xDB);
2790 emit(0xE8 + i);
2791 }
2792
2793
fucomip()2794 void Assembler::fucomip() {
2795 EnsureSpace ensure_space(this);
2796 emit(0xDF);
2797 emit(0xE9);
2798 }
2799
2800
fcompp()2801 void Assembler::fcompp() {
2802 EnsureSpace ensure_space(this);
2803 emit(0xDE);
2804 emit(0xD9);
2805 }
2806
2807
fnstsw_ax()2808 void Assembler::fnstsw_ax() {
2809 EnsureSpace ensure_space(this);
2810 emit(0xDF);
2811 emit(0xE0);
2812 }
2813
2814
fwait()2815 void Assembler::fwait() {
2816 EnsureSpace ensure_space(this);
2817 emit(0x9B);
2818 }
2819
2820
frndint()2821 void Assembler::frndint() {
2822 EnsureSpace ensure_space(this);
2823 emit(0xD9);
2824 emit(0xFC);
2825 }
2826
2827
fnclex()2828 void Assembler::fnclex() {
2829 EnsureSpace ensure_space(this);
2830 emit(0xDB);
2831 emit(0xE2);
2832 }
2833
2834
sahf()2835 void Assembler::sahf() {
2836 // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf
2837 // in 64-bit mode. Test CpuID.
2838 DCHECK(IsEnabled(SAHF));
2839 EnsureSpace ensure_space(this);
2840 emit(0x9E);
2841 }
2842
2843
emit_farith(int b1,int b2,int i)2844 void Assembler::emit_farith(int b1, int b2, int i) {
2845 DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode
2846 DCHECK(is_uint3(i)); // illegal stack offset
2847 emit(b1);
2848 emit(b2 + i);
2849 }
2850
2851
2852 // SSE operations.
2853
andps(XMMRegister dst,XMMRegister src)2854 void Assembler::andps(XMMRegister dst, XMMRegister src) {
2855 EnsureSpace ensure_space(this);
2856 emit_optional_rex_32(dst, src);
2857 emit(0x0F);
2858 emit(0x54);
2859 emit_sse_operand(dst, src);
2860 }
2861
andps(XMMRegister dst,Operand src)2862 void Assembler::andps(XMMRegister dst, Operand src) {
2863 EnsureSpace ensure_space(this);
2864 emit_optional_rex_32(dst, src);
2865 emit(0x0F);
2866 emit(0x54);
2867 emit_sse_operand(dst, src);
2868 }
2869
2870
orps(XMMRegister dst,XMMRegister src)2871 void Assembler::orps(XMMRegister dst, XMMRegister src) {
2872 EnsureSpace ensure_space(this);
2873 emit_optional_rex_32(dst, src);
2874 emit(0x0F);
2875 emit(0x56);
2876 emit_sse_operand(dst, src);
2877 }
2878
orps(XMMRegister dst,Operand src)2879 void Assembler::orps(XMMRegister dst, Operand src) {
2880 EnsureSpace ensure_space(this);
2881 emit_optional_rex_32(dst, src);
2882 emit(0x0F);
2883 emit(0x56);
2884 emit_sse_operand(dst, src);
2885 }
2886
2887
xorps(XMMRegister dst,XMMRegister src)2888 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
2889 DCHECK(!IsEnabled(AVX));
2890 EnsureSpace ensure_space(this);
2891 emit_optional_rex_32(dst, src);
2892 emit(0x0F);
2893 emit(0x57);
2894 emit_sse_operand(dst, src);
2895 }
2896
xorps(XMMRegister dst,Operand src)2897 void Assembler::xorps(XMMRegister dst, Operand src) {
2898 DCHECK(!IsEnabled(AVX));
2899 EnsureSpace ensure_space(this);
2900 emit_optional_rex_32(dst, src);
2901 emit(0x0F);
2902 emit(0x57);
2903 emit_sse_operand(dst, src);
2904 }
2905
2906
addps(XMMRegister dst,XMMRegister src)2907 void Assembler::addps(XMMRegister dst, XMMRegister src) {
2908 EnsureSpace ensure_space(this);
2909 emit_optional_rex_32(dst, src);
2910 emit(0x0F);
2911 emit(0x58);
2912 emit_sse_operand(dst, src);
2913 }
2914
addps(XMMRegister dst,Operand src)2915 void Assembler::addps(XMMRegister dst, Operand src) {
2916 EnsureSpace ensure_space(this);
2917 emit_optional_rex_32(dst, src);
2918 emit(0x0F);
2919 emit(0x58);
2920 emit_sse_operand(dst, src);
2921 }
2922
2923
subps(XMMRegister dst,XMMRegister src)2924 void Assembler::subps(XMMRegister dst, XMMRegister src) {
2925 EnsureSpace ensure_space(this);
2926 emit_optional_rex_32(dst, src);
2927 emit(0x0F);
2928 emit(0x5C);
2929 emit_sse_operand(dst, src);
2930 }
2931
subps(XMMRegister dst,Operand src)2932 void Assembler::subps(XMMRegister dst, Operand src) {
2933 EnsureSpace ensure_space(this);
2934 emit_optional_rex_32(dst, src);
2935 emit(0x0F);
2936 emit(0x5C);
2937 emit_sse_operand(dst, src);
2938 }
2939
2940
mulps(XMMRegister dst,XMMRegister src)2941 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
2942 EnsureSpace ensure_space(this);
2943 emit_optional_rex_32(dst, src);
2944 emit(0x0F);
2945 emit(0x59);
2946 emit_sse_operand(dst, src);
2947 }
2948
mulps(XMMRegister dst,Operand src)2949 void Assembler::mulps(XMMRegister dst, Operand src) {
2950 EnsureSpace ensure_space(this);
2951 emit_optional_rex_32(dst, src);
2952 emit(0x0F);
2953 emit(0x59);
2954 emit_sse_operand(dst, src);
2955 }
2956
2957
divps(XMMRegister dst,XMMRegister src)2958 void Assembler::divps(XMMRegister dst, XMMRegister src) {
2959 EnsureSpace ensure_space(this);
2960 emit_optional_rex_32(dst, src);
2961 emit(0x0F);
2962 emit(0x5E);
2963 emit_sse_operand(dst, src);
2964 }
2965
divps(XMMRegister dst,Operand src)2966 void Assembler::divps(XMMRegister dst, Operand src) {
2967 EnsureSpace ensure_space(this);
2968 emit_optional_rex_32(dst, src);
2969 emit(0x0F);
2970 emit(0x5E);
2971 emit_sse_operand(dst, src);
2972 }
2973
2974
2975 // SSE 2 operations.
2976
movd(XMMRegister dst,Register src)2977 void Assembler::movd(XMMRegister dst, Register src) {
2978 DCHECK(!IsEnabled(AVX));
2979 EnsureSpace ensure_space(this);
2980 emit(0x66);
2981 emit_optional_rex_32(dst, src);
2982 emit(0x0F);
2983 emit(0x6E);
2984 emit_sse_operand(dst, src);
2985 }
2986
movd(XMMRegister dst,Operand src)2987 void Assembler::movd(XMMRegister dst, Operand src) {
2988 DCHECK(!IsEnabled(AVX));
2989 EnsureSpace ensure_space(this);
2990 emit(0x66);
2991 emit_optional_rex_32(dst, src);
2992 emit(0x0F);
2993 emit(0x6E);
2994 emit_sse_operand(dst, src);
2995 }
2996
2997
movd(Register dst,XMMRegister src)2998 void Assembler::movd(Register dst, XMMRegister src) {
2999 DCHECK(!IsEnabled(AVX));
3000 EnsureSpace ensure_space(this);
3001 emit(0x66);
3002 emit_optional_rex_32(src, dst);
3003 emit(0x0F);
3004 emit(0x7E);
3005 emit_sse_operand(src, dst);
3006 }
3007
3008
movq(XMMRegister dst,Register src)3009 void Assembler::movq(XMMRegister dst, Register src) {
3010 DCHECK(!IsEnabled(AVX));
3011 EnsureSpace ensure_space(this);
3012 emit(0x66);
3013 emit_rex_64(dst, src);
3014 emit(0x0F);
3015 emit(0x6E);
3016 emit_sse_operand(dst, src);
3017 }
3018
3019
movq(Register dst,XMMRegister src)3020 void Assembler::movq(Register dst, XMMRegister src) {
3021 DCHECK(!IsEnabled(AVX));
3022 EnsureSpace ensure_space(this);
3023 emit(0x66);
3024 emit_rex_64(src, dst);
3025 emit(0x0F);
3026 emit(0x7E);
3027 emit_sse_operand(src, dst);
3028 }
3029
3030
movq(XMMRegister dst,XMMRegister src)3031 void Assembler::movq(XMMRegister dst, XMMRegister src) {
3032 DCHECK(!IsEnabled(AVX));
3033 EnsureSpace ensure_space(this);
3034 if (dst.low_bits() == 4) {
3035 // Avoid unnecessary SIB byte.
3036 emit(0xF3);
3037 emit_optional_rex_32(dst, src);
3038 emit(0x0F);
3039 emit(0x7E);
3040 emit_sse_operand(dst, src);
3041 } else {
3042 emit(0x66);
3043 emit_optional_rex_32(src, dst);
3044 emit(0x0F);
3045 emit(0xD6);
3046 emit_sse_operand(src, dst);
3047 }
3048 }
3049
movdqa(Operand dst,XMMRegister src)3050 void Assembler::movdqa(Operand dst, XMMRegister src) {
3051 EnsureSpace ensure_space(this);
3052 emit(0x66);
3053 emit_rex_64(src, dst);
3054 emit(0x0F);
3055 emit(0x7F);
3056 emit_sse_operand(src, dst);
3057 }
3058
movdqa(XMMRegister dst,Operand src)3059 void Assembler::movdqa(XMMRegister dst, Operand src) {
3060 EnsureSpace ensure_space(this);
3061 emit(0x66);
3062 emit_rex_64(dst, src);
3063 emit(0x0F);
3064 emit(0x6F);
3065 emit_sse_operand(dst, src);
3066 }
3067
movdqu(Operand dst,XMMRegister src)3068 void Assembler::movdqu(Operand dst, XMMRegister src) {
3069 EnsureSpace ensure_space(this);
3070 emit(0xF3);
3071 emit_rex_64(src, dst);
3072 emit(0x0F);
3073 emit(0x7F);
3074 emit_sse_operand(src, dst);
3075 }
3076
movdqu(XMMRegister dst,Operand src)3077 void Assembler::movdqu(XMMRegister dst, Operand src) {
3078 EnsureSpace ensure_space(this);
3079 emit(0xF3);
3080 emit_rex_64(dst, src);
3081 emit(0x0F);
3082 emit(0x6F);
3083 emit_sse_operand(dst, src);
3084 }
3085
3086
extractps(Register dst,XMMRegister src,byte imm8)3087 void Assembler::extractps(Register dst, XMMRegister src, byte imm8) {
3088 DCHECK(IsEnabled(SSE4_1));
3089 DCHECK(is_uint8(imm8));
3090 EnsureSpace ensure_space(this);
3091 emit(0x66);
3092 emit_optional_rex_32(src, dst);
3093 emit(0x0F);
3094 emit(0x3A);
3095 emit(0x17);
3096 emit_sse_operand(src, dst);
3097 emit(imm8);
3098 }
3099
pextrb(Register dst,XMMRegister src,int8_t imm8)3100 void Assembler::pextrb(Register dst, XMMRegister src, int8_t imm8) {
3101 DCHECK(IsEnabled(SSE4_1));
3102 DCHECK(is_uint8(imm8));
3103 EnsureSpace ensure_space(this);
3104 emit(0x66);
3105 emit_optional_rex_32(src, dst);
3106 emit(0x0F);
3107 emit(0x3A);
3108 emit(0x14);
3109 emit_sse_operand(src, dst);
3110 emit(imm8);
3111 }
3112
pextrb(Operand dst,XMMRegister src,int8_t imm8)3113 void Assembler::pextrb(Operand dst, XMMRegister src, int8_t imm8) {
3114 DCHECK(IsEnabled(SSE4_1));
3115 DCHECK(is_uint8(imm8));
3116 EnsureSpace ensure_space(this);
3117 emit(0x66);
3118 emit_optional_rex_32(src, dst);
3119 emit(0x0F);
3120 emit(0x3A);
3121 emit(0x14);
3122 emit_sse_operand(src, dst);
3123 emit(imm8);
3124 }
3125
pinsrw(XMMRegister dst,Register src,int8_t imm8)3126 void Assembler::pinsrw(XMMRegister dst, Register src, int8_t imm8) {
3127 DCHECK(is_uint8(imm8));
3128 EnsureSpace ensure_space(this);
3129 emit(0x66);
3130 emit_optional_rex_32(dst, src);
3131 emit(0x0F);
3132 emit(0xC4);
3133 emit_sse_operand(dst, src);
3134 emit(imm8);
3135 }
3136
pinsrw(XMMRegister dst,Operand src,int8_t imm8)3137 void Assembler::pinsrw(XMMRegister dst, Operand src, int8_t imm8) {
3138 DCHECK(is_uint8(imm8));
3139 EnsureSpace ensure_space(this);
3140 emit(0x66);
3141 emit_optional_rex_32(dst, src);
3142 emit(0x0F);
3143 emit(0xC4);
3144 emit_sse_operand(dst, src);
3145 emit(imm8);
3146 }
3147
pextrw(Register dst,XMMRegister src,int8_t imm8)3148 void Assembler::pextrw(Register dst, XMMRegister src, int8_t imm8) {
3149 DCHECK(IsEnabled(SSE4_1));
3150 DCHECK(is_uint8(imm8));
3151 EnsureSpace ensure_space(this);
3152 emit(0x66);
3153 emit_optional_rex_32(src, dst);
3154 emit(0x0F);
3155 emit(0x3A);
3156 emit(0x15);
3157 emit_sse_operand(src, dst);
3158 emit(imm8);
3159 }
3160
pextrw(Operand dst,XMMRegister src,int8_t imm8)3161 void Assembler::pextrw(Operand dst, XMMRegister src, int8_t imm8) {
3162 DCHECK(IsEnabled(SSE4_1));
3163 DCHECK(is_uint8(imm8));
3164 EnsureSpace ensure_space(this);
3165 emit(0x66);
3166 emit_optional_rex_32(src, dst);
3167 emit(0x0F);
3168 emit(0x3A);
3169 emit(0x15);
3170 emit_sse_operand(src, dst);
3171 emit(imm8);
3172 }
3173
pextrd(Register dst,XMMRegister src,int8_t imm8)3174 void Assembler::pextrd(Register dst, XMMRegister src, int8_t imm8) {
3175 DCHECK(IsEnabled(SSE4_1));
3176 EnsureSpace ensure_space(this);
3177 emit(0x66);
3178 emit_optional_rex_32(src, dst);
3179 emit(0x0F);
3180 emit(0x3A);
3181 emit(0x16);
3182 emit_sse_operand(src, dst);
3183 emit(imm8);
3184 }
3185
pextrd(Operand dst,XMMRegister src,int8_t imm8)3186 void Assembler::pextrd(Operand dst, XMMRegister src, int8_t imm8) {
3187 DCHECK(IsEnabled(SSE4_1));
3188 EnsureSpace ensure_space(this);
3189 emit(0x66);
3190 emit_optional_rex_32(src, dst);
3191 emit(0x0F);
3192 emit(0x3A);
3193 emit(0x16);
3194 emit_sse_operand(src, dst);
3195 emit(imm8);
3196 }
3197
pinsrd(XMMRegister dst,Register src,int8_t imm8)3198 void Assembler::pinsrd(XMMRegister dst, Register src, int8_t imm8) {
3199 DCHECK(IsEnabled(SSE4_1));
3200 EnsureSpace ensure_space(this);
3201 emit(0x66);
3202 emit_optional_rex_32(dst, src);
3203 emit(0x0F);
3204 emit(0x3A);
3205 emit(0x22);
3206 emit_sse_operand(dst, src);
3207 emit(imm8);
3208 }
3209
pinsrd(XMMRegister dst,Operand src,int8_t imm8)3210 void Assembler::pinsrd(XMMRegister dst, Operand src, int8_t imm8) {
3211 DCHECK(IsEnabled(SSE4_1));
3212 EnsureSpace ensure_space(this);
3213 emit(0x66);
3214 emit_optional_rex_32(dst, src);
3215 emit(0x0F);
3216 emit(0x3A);
3217 emit(0x22);
3218 emit_sse_operand(dst, src);
3219 emit(imm8);
3220 }
3221
pinsrb(XMMRegister dst,Register src,int8_t imm8)3222 void Assembler::pinsrb(XMMRegister dst, Register src, int8_t imm8) {
3223 DCHECK(IsEnabled(SSE4_1));
3224 EnsureSpace ensure_space(this);
3225 emit(0x66);
3226 emit_optional_rex_32(dst, src);
3227 emit(0x0F);
3228 emit(0x3A);
3229 emit(0x20);
3230 emit_sse_operand(dst, src);
3231 emit(imm8);
3232 }
3233
pinsrb(XMMRegister dst,Operand src,int8_t imm8)3234 void Assembler::pinsrb(XMMRegister dst, Operand src, int8_t imm8) {
3235 DCHECK(IsEnabled(SSE4_1));
3236 EnsureSpace ensure_space(this);
3237 emit(0x66);
3238 emit_optional_rex_32(dst, src);
3239 emit(0x0F);
3240 emit(0x3A);
3241 emit(0x20);
3242 emit_sse_operand(dst, src);
3243 emit(imm8);
3244 }
3245
insertps(XMMRegister dst,XMMRegister src,byte imm8)3246 void Assembler::insertps(XMMRegister dst, XMMRegister src, byte imm8) {
3247 DCHECK(CpuFeatures::IsSupported(SSE4_1));
3248 DCHECK(is_uint8(imm8));
3249 EnsureSpace ensure_space(this);
3250 emit(0x66);
3251 emit_optional_rex_32(dst, src);
3252 emit(0x0F);
3253 emit(0x3A);
3254 emit(0x21);
3255 emit_sse_operand(dst, src);
3256 emit(imm8);
3257 }
3258
movsd(Operand dst,XMMRegister src)3259 void Assembler::movsd(Operand dst, XMMRegister src) {
3260 DCHECK(!IsEnabled(AVX));
3261 EnsureSpace ensure_space(this);
3262 emit(0xF2); // double
3263 emit_optional_rex_32(src, dst);
3264 emit(0x0F);
3265 emit(0x11); // store
3266 emit_sse_operand(src, dst);
3267 }
3268
3269
movsd(XMMRegister dst,XMMRegister src)3270 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
3271 DCHECK(!IsEnabled(AVX));
3272 EnsureSpace ensure_space(this);
3273 emit(0xF2); // double
3274 emit_optional_rex_32(dst, src);
3275 emit(0x0F);
3276 emit(0x10); // load
3277 emit_sse_operand(dst, src);
3278 }
3279
movsd(XMMRegister dst,Operand src)3280 void Assembler::movsd(XMMRegister dst, Operand src) {
3281 DCHECK(!IsEnabled(AVX));
3282 EnsureSpace ensure_space(this);
3283 emit(0xF2); // double
3284 emit_optional_rex_32(dst, src);
3285 emit(0x0F);
3286 emit(0x10); // load
3287 emit_sse_operand(dst, src);
3288 }
3289
3290
movaps(XMMRegister dst,XMMRegister src)3291 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
3292 DCHECK(!IsEnabled(AVX));
3293 EnsureSpace ensure_space(this);
3294 if (src.low_bits() == 4) {
3295 // Try to avoid an unnecessary SIB byte.
3296 emit_optional_rex_32(src, dst);
3297 emit(0x0F);
3298 emit(0x29);
3299 emit_sse_operand(src, dst);
3300 } else {
3301 emit_optional_rex_32(dst, src);
3302 emit(0x0F);
3303 emit(0x28);
3304 emit_sse_operand(dst, src);
3305 }
3306 }
3307
3308
shufps(XMMRegister dst,XMMRegister src,byte imm8)3309 void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) {
3310 DCHECK(is_uint8(imm8));
3311 EnsureSpace ensure_space(this);
3312 emit_optional_rex_32(dst, src);
3313 emit(0x0F);
3314 emit(0xC6);
3315 emit_sse_operand(dst, src);
3316 emit(imm8);
3317 }
3318
3319
movapd(XMMRegister dst,XMMRegister src)3320 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
3321 DCHECK(!IsEnabled(AVX));
3322 EnsureSpace ensure_space(this);
3323 if (src.low_bits() == 4) {
3324 // Try to avoid an unnecessary SIB byte.
3325 emit(0x66);
3326 emit_optional_rex_32(src, dst);
3327 emit(0x0F);
3328 emit(0x29);
3329 emit_sse_operand(src, dst);
3330 } else {
3331 emit(0x66);
3332 emit_optional_rex_32(dst, src);
3333 emit(0x0F);
3334 emit(0x28);
3335 emit_sse_operand(dst, src);
3336 }
3337 }
3338
movupd(XMMRegister dst,Operand src)3339 void Assembler::movupd(XMMRegister dst, Operand src) {
3340 EnsureSpace ensure_space(this);
3341 emit(0x66);
3342 emit_optional_rex_32(dst, src);
3343 emit(0x0F);
3344 emit(0x10);
3345 emit_sse_operand(dst, src);
3346 }
3347
movupd(Operand dst,XMMRegister src)3348 void Assembler::movupd(Operand dst, XMMRegister src) {
3349 EnsureSpace ensure_space(this);
3350 emit(0x66);
3351 emit_optional_rex_32(src, dst);
3352 emit(0x0F);
3353 emit(0x11);
3354 emit_sse_operand(src, dst);
3355 }
3356
addss(XMMRegister dst,XMMRegister src)3357 void Assembler::addss(XMMRegister dst, XMMRegister src) {
3358 EnsureSpace ensure_space(this);
3359 emit(0xF3);
3360 emit_optional_rex_32(dst, src);
3361 emit(0x0F);
3362 emit(0x58);
3363 emit_sse_operand(dst, src);
3364 }
3365
addss(XMMRegister dst,Operand src)3366 void Assembler::addss(XMMRegister dst, Operand src) {
3367 EnsureSpace ensure_space(this);
3368 emit(0xF3);
3369 emit_optional_rex_32(dst, src);
3370 emit(0x0F);
3371 emit(0x58);
3372 emit_sse_operand(dst, src);
3373 }
3374
3375
subss(XMMRegister dst,XMMRegister src)3376 void Assembler::subss(XMMRegister dst, XMMRegister src) {
3377 EnsureSpace ensure_space(this);
3378 emit(0xF3);
3379 emit_optional_rex_32(dst, src);
3380 emit(0x0F);
3381 emit(0x5C);
3382 emit_sse_operand(dst, src);
3383 }
3384
subss(XMMRegister dst,Operand src)3385 void Assembler::subss(XMMRegister dst, Operand src) {
3386 EnsureSpace ensure_space(this);
3387 emit(0xF3);
3388 emit_optional_rex_32(dst, src);
3389 emit(0x0F);
3390 emit(0x5C);
3391 emit_sse_operand(dst, src);
3392 }
3393
3394
mulss(XMMRegister dst,XMMRegister src)3395 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
3396 EnsureSpace ensure_space(this);
3397 emit(0xF3);
3398 emit_optional_rex_32(dst, src);
3399 emit(0x0F);
3400 emit(0x59);
3401 emit_sse_operand(dst, src);
3402 }
3403
mulss(XMMRegister dst,Operand src)3404 void Assembler::mulss(XMMRegister dst, Operand src) {
3405 EnsureSpace ensure_space(this);
3406 emit(0xF3);
3407 emit_optional_rex_32(dst, src);
3408 emit(0x0F);
3409 emit(0x59);
3410 emit_sse_operand(dst, src);
3411 }
3412
3413
divss(XMMRegister dst,XMMRegister src)3414 void Assembler::divss(XMMRegister dst, XMMRegister src) {
3415 EnsureSpace ensure_space(this);
3416 emit(0xF3);
3417 emit_optional_rex_32(dst, src);
3418 emit(0x0F);
3419 emit(0x5E);
3420 emit_sse_operand(dst, src);
3421 }
3422
divss(XMMRegister dst,Operand src)3423 void Assembler::divss(XMMRegister dst, Operand src) {
3424 EnsureSpace ensure_space(this);
3425 emit(0xF3);
3426 emit_optional_rex_32(dst, src);
3427 emit(0x0F);
3428 emit(0x5E);
3429 emit_sse_operand(dst, src);
3430 }
3431
3432
maxss(XMMRegister dst,XMMRegister src)3433 void Assembler::maxss(XMMRegister dst, XMMRegister src) {
3434 EnsureSpace ensure_space(this);
3435 emit(0xF3);
3436 emit_optional_rex_32(dst, src);
3437 emit(0x0F);
3438 emit(0x5F);
3439 emit_sse_operand(dst, src);
3440 }
3441
maxss(XMMRegister dst,Operand src)3442 void Assembler::maxss(XMMRegister dst, Operand src) {
3443 EnsureSpace ensure_space(this);
3444 emit(0xF3);
3445 emit_optional_rex_32(dst, src);
3446 emit(0x0F);
3447 emit(0x5F);
3448 emit_sse_operand(dst, src);
3449 }
3450
3451
minss(XMMRegister dst,XMMRegister src)3452 void Assembler::minss(XMMRegister dst, XMMRegister src) {
3453 EnsureSpace ensure_space(this);
3454 emit(0xF3);
3455 emit_optional_rex_32(dst, src);
3456 emit(0x0F);
3457 emit(0x5D);
3458 emit_sse_operand(dst, src);
3459 }
3460
minss(XMMRegister dst,Operand src)3461 void Assembler::minss(XMMRegister dst, Operand src) {
3462 EnsureSpace ensure_space(this);
3463 emit(0xF3);
3464 emit_optional_rex_32(dst, src);
3465 emit(0x0F);
3466 emit(0x5D);
3467 emit_sse_operand(dst, src);
3468 }
3469
3470
sqrtss(XMMRegister dst,XMMRegister src)3471 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
3472 EnsureSpace ensure_space(this);
3473 emit(0xF3);
3474 emit_optional_rex_32(dst, src);
3475 emit(0x0F);
3476 emit(0x51);
3477 emit_sse_operand(dst, src);
3478 }
3479
sqrtss(XMMRegister dst,Operand src)3480 void Assembler::sqrtss(XMMRegister dst, Operand src) {
3481 EnsureSpace ensure_space(this);
3482 emit(0xF3);
3483 emit_optional_rex_32(dst, src);
3484 emit(0x0F);
3485 emit(0x51);
3486 emit_sse_operand(dst, src);
3487 }
3488
3489
ucomiss(XMMRegister dst,XMMRegister src)3490 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
3491 DCHECK(!IsEnabled(AVX));
3492 EnsureSpace ensure_space(this);
3493 emit_optional_rex_32(dst, src);
3494 emit(0x0F);
3495 emit(0x2E);
3496 emit_sse_operand(dst, src);
3497 }
3498
ucomiss(XMMRegister dst,Operand src)3499 void Assembler::ucomiss(XMMRegister dst, Operand src) {
3500 DCHECK(!IsEnabled(AVX));
3501 EnsureSpace ensure_space(this);
3502 emit_optional_rex_32(dst, src);
3503 emit(0x0F);
3504 emit(0x2E);
3505 emit_sse_operand(dst, src);
3506 }
3507
3508
movss(XMMRegister dst,XMMRegister src)3509 void Assembler::movss(XMMRegister dst, XMMRegister src) {
3510 DCHECK(!IsEnabled(AVX));
3511 EnsureSpace ensure_space(this);
3512 emit(0xF3); // single
3513 emit_optional_rex_32(dst, src);
3514 emit(0x0F);
3515 emit(0x10); // load
3516 emit_sse_operand(dst, src);
3517 }
3518
movss(XMMRegister dst,Operand src)3519 void Assembler::movss(XMMRegister dst, Operand src) {
3520 DCHECK(!IsEnabled(AVX));
3521 EnsureSpace ensure_space(this);
3522 emit(0xF3); // single
3523 emit_optional_rex_32(dst, src);
3524 emit(0x0F);
3525 emit(0x10); // load
3526 emit_sse_operand(dst, src);
3527 }
3528
movss(Operand src,XMMRegister dst)3529 void Assembler::movss(Operand src, XMMRegister dst) {
3530 DCHECK(!IsEnabled(AVX));
3531 EnsureSpace ensure_space(this);
3532 emit(0xF3); // single
3533 emit_optional_rex_32(dst, src);
3534 emit(0x0F);
3535 emit(0x11); // store
3536 emit_sse_operand(dst, src);
3537 }
3538
3539
psllq(XMMRegister reg,byte imm8)3540 void Assembler::psllq(XMMRegister reg, byte imm8) {
3541 DCHECK(!IsEnabled(AVX));
3542 EnsureSpace ensure_space(this);
3543 emit(0x66);
3544 emit_optional_rex_32(reg);
3545 emit(0x0F);
3546 emit(0x73);
3547 emit_sse_operand(rsi, reg); // rsi == 6
3548 emit(imm8);
3549 }
3550
3551
psrlq(XMMRegister reg,byte imm8)3552 void Assembler::psrlq(XMMRegister reg, byte imm8) {
3553 DCHECK(!IsEnabled(AVX));
3554 EnsureSpace ensure_space(this);
3555 emit(0x66);
3556 emit_optional_rex_32(reg);
3557 emit(0x0F);
3558 emit(0x73);
3559 emit_sse_operand(rdx, reg); // rdx == 2
3560 emit(imm8);
3561 }
3562
psllw(XMMRegister reg,byte imm8)3563 void Assembler::psllw(XMMRegister reg, byte imm8) {
3564 EnsureSpace ensure_space(this);
3565 emit(0x66);
3566 emit_optional_rex_32(reg);
3567 emit(0x0F);
3568 emit(0x71);
3569 emit_sse_operand(rsi, reg); // rsi == 6
3570 emit(imm8);
3571 }
3572
pslld(XMMRegister reg,byte imm8)3573 void Assembler::pslld(XMMRegister reg, byte imm8) {
3574 EnsureSpace ensure_space(this);
3575 emit(0x66);
3576 emit_optional_rex_32(reg);
3577 emit(0x0F);
3578 emit(0x72);
3579 emit_sse_operand(rsi, reg); // rsi == 6
3580 emit(imm8);
3581 }
3582
psrlw(XMMRegister reg,byte imm8)3583 void Assembler::psrlw(XMMRegister reg, byte imm8) {
3584 EnsureSpace ensure_space(this);
3585 emit(0x66);
3586 emit_optional_rex_32(reg);
3587 emit(0x0F);
3588 emit(0x71);
3589 emit_sse_operand(rdx, reg); // rdx == 2
3590 emit(imm8);
3591 }
3592
psrld(XMMRegister reg,byte imm8)3593 void Assembler::psrld(XMMRegister reg, byte imm8) {
3594 EnsureSpace ensure_space(this);
3595 emit(0x66);
3596 emit_optional_rex_32(reg);
3597 emit(0x0F);
3598 emit(0x72);
3599 emit_sse_operand(rdx, reg); // rdx == 2
3600 emit(imm8);
3601 }
3602
psraw(XMMRegister reg,byte imm8)3603 void Assembler::psraw(XMMRegister reg, byte imm8) {
3604 EnsureSpace ensure_space(this);
3605 emit(0x66);
3606 emit_optional_rex_32(reg);
3607 emit(0x0F);
3608 emit(0x71);
3609 emit_sse_operand(rsp, reg); // rsp == 4
3610 emit(imm8);
3611 }
3612
psrad(XMMRegister reg,byte imm8)3613 void Assembler::psrad(XMMRegister reg, byte imm8) {
3614 EnsureSpace ensure_space(this);
3615 emit(0x66);
3616 emit_optional_rex_32(reg);
3617 emit(0x0F);
3618 emit(0x72);
3619 emit_sse_operand(rsp, reg); // rsp == 4
3620 emit(imm8);
3621 }
3622
cmpps(XMMRegister dst,XMMRegister src,int8_t cmp)3623 void Assembler::cmpps(XMMRegister dst, XMMRegister src, int8_t cmp) {
3624 EnsureSpace ensure_space(this);
3625 emit_optional_rex_32(dst, src);
3626 emit(0x0F);
3627 emit(0xC2);
3628 emit_sse_operand(dst, src);
3629 emit(cmp);
3630 }
3631
cmpps(XMMRegister dst,Operand src,int8_t cmp)3632 void Assembler::cmpps(XMMRegister dst, Operand src, int8_t cmp) {
3633 EnsureSpace ensure_space(this);
3634 emit_optional_rex_32(dst, src);
3635 emit(0x0F);
3636 emit(0xC2);
3637 emit_sse_operand(dst, src);
3638 emit(cmp);
3639 }
3640
cmppd(XMMRegister dst,XMMRegister src,int8_t cmp)3641 void Assembler::cmppd(XMMRegister dst, XMMRegister src, int8_t cmp) {
3642 EnsureSpace ensure_space(this);
3643 emit_optional_rex_32(dst, src);
3644 emit(0x66);
3645 emit(0x0F);
3646 emit(0xC2);
3647 emit_sse_operand(dst, src);
3648 emit(cmp);
3649 }
3650
cmppd(XMMRegister dst,Operand src,int8_t cmp)3651 void Assembler::cmppd(XMMRegister dst, Operand src, int8_t cmp) {
3652 EnsureSpace ensure_space(this);
3653 emit_optional_rex_32(dst, src);
3654 emit(0x66);
3655 emit(0x0F);
3656 emit(0xC2);
3657 emit_sse_operand(dst, src);
3658 emit(cmp);
3659 }
3660
cvttss2si(Register dst,Operand src)3661 void Assembler::cvttss2si(Register dst, Operand src) {
3662 DCHECK(!IsEnabled(AVX));
3663 EnsureSpace ensure_space(this);
3664 emit(0xF3);
3665 emit_optional_rex_32(dst, src);
3666 emit(0x0F);
3667 emit(0x2C);
3668 emit_operand(dst, src);
3669 }
3670
3671
cvttss2si(Register dst,XMMRegister src)3672 void Assembler::cvttss2si(Register dst, XMMRegister src) {
3673 DCHECK(!IsEnabled(AVX));
3674 EnsureSpace ensure_space(this);
3675 emit(0xF3);
3676 emit_optional_rex_32(dst, src);
3677 emit(0x0F);
3678 emit(0x2C);
3679 emit_sse_operand(dst, src);
3680 }
3681
cvttsd2si(Register dst,Operand src)3682 void Assembler::cvttsd2si(Register dst, Operand src) {
3683 DCHECK(!IsEnabled(AVX));
3684 EnsureSpace ensure_space(this);
3685 emit(0xF2);
3686 emit_optional_rex_32(dst, src);
3687 emit(0x0F);
3688 emit(0x2C);
3689 emit_operand(dst, src);
3690 }
3691
3692
cvttsd2si(Register dst,XMMRegister src)3693 void Assembler::cvttsd2si(Register dst, XMMRegister src) {
3694 DCHECK(!IsEnabled(AVX));
3695 EnsureSpace ensure_space(this);
3696 emit(0xF2);
3697 emit_optional_rex_32(dst, src);
3698 emit(0x0F);
3699 emit(0x2C);
3700 emit_sse_operand(dst, src);
3701 }
3702
3703
cvttss2siq(Register dst,XMMRegister src)3704 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
3705 DCHECK(!IsEnabled(AVX));
3706 EnsureSpace ensure_space(this);
3707 emit(0xF3);
3708 emit_rex_64(dst, src);
3709 emit(0x0F);
3710 emit(0x2C);
3711 emit_sse_operand(dst, src);
3712 }
3713
cvttss2siq(Register dst,Operand src)3714 void Assembler::cvttss2siq(Register dst, Operand src) {
3715 DCHECK(!IsEnabled(AVX));
3716 EnsureSpace ensure_space(this);
3717 emit(0xF3);
3718 emit_rex_64(dst, src);
3719 emit(0x0F);
3720 emit(0x2C);
3721 emit_sse_operand(dst, src);
3722 }
3723
3724
cvttsd2siq(Register dst,XMMRegister src)3725 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
3726 DCHECK(!IsEnabled(AVX));
3727 EnsureSpace ensure_space(this);
3728 emit(0xF2);
3729 emit_rex_64(dst, src);
3730 emit(0x0F);
3731 emit(0x2C);
3732 emit_sse_operand(dst, src);
3733 }
3734
cvttsd2siq(Register dst,Operand src)3735 void Assembler::cvttsd2siq(Register dst, Operand src) {
3736 DCHECK(!IsEnabled(AVX));
3737 EnsureSpace ensure_space(this);
3738 emit(0xF2);
3739 emit_rex_64(dst, src);
3740 emit(0x0F);
3741 emit(0x2C);
3742 emit_sse_operand(dst, src);
3743 }
3744
cvttps2dq(XMMRegister dst,Operand src)3745 void Assembler::cvttps2dq(XMMRegister dst, Operand src) {
3746 EnsureSpace ensure_space(this);
3747 emit(0xF3);
3748 emit_rex_64(dst, src);
3749 emit(0x0F);
3750 emit(0x5B);
3751 emit_sse_operand(dst, src);
3752 }
3753
cvttps2dq(XMMRegister dst,XMMRegister src)3754 void Assembler::cvttps2dq(XMMRegister dst, XMMRegister src) {
3755 EnsureSpace ensure_space(this);
3756 emit(0xF3);
3757 emit_rex_64(dst, src);
3758 emit(0x0F);
3759 emit(0x5B);
3760 emit_sse_operand(dst, src);
3761 }
3762
cvtlsi2sd(XMMRegister dst,Operand src)3763 void Assembler::cvtlsi2sd(XMMRegister dst, Operand src) {
3764 DCHECK(!IsEnabled(AVX));
3765 EnsureSpace ensure_space(this);
3766 emit(0xF2);
3767 emit_optional_rex_32(dst, src);
3768 emit(0x0F);
3769 emit(0x2A);
3770 emit_sse_operand(dst, src);
3771 }
3772
3773
cvtlsi2sd(XMMRegister dst,Register src)3774 void Assembler::cvtlsi2sd(XMMRegister dst, Register src) {
3775 DCHECK(!IsEnabled(AVX));
3776 EnsureSpace ensure_space(this);
3777 emit(0xF2);
3778 emit_optional_rex_32(dst, src);
3779 emit(0x0F);
3780 emit(0x2A);
3781 emit_sse_operand(dst, src);
3782 }
3783
cvtlsi2ss(XMMRegister dst,Operand src)3784 void Assembler::cvtlsi2ss(XMMRegister dst, Operand src) {
3785 DCHECK(!IsEnabled(AVX));
3786 EnsureSpace ensure_space(this);
3787 emit(0xF3);
3788 emit_optional_rex_32(dst, src);
3789 emit(0x0F);
3790 emit(0x2A);
3791 emit_sse_operand(dst, src);
3792 }
3793
3794
cvtlsi2ss(XMMRegister dst,Register src)3795 void Assembler::cvtlsi2ss(XMMRegister dst, Register src) {
3796 EnsureSpace ensure_space(this);
3797 emit(0xF3);
3798 emit_optional_rex_32(dst, src);
3799 emit(0x0F);
3800 emit(0x2A);
3801 emit_sse_operand(dst, src);
3802 }
3803
cvtqsi2ss(XMMRegister dst,Operand src)3804 void Assembler::cvtqsi2ss(XMMRegister dst, Operand src) {
3805 DCHECK(!IsEnabled(AVX));
3806 EnsureSpace ensure_space(this);
3807 emit(0xF3);
3808 emit_rex_64(dst, src);
3809 emit(0x0F);
3810 emit(0x2A);
3811 emit_sse_operand(dst, src);
3812 }
3813
3814
cvtqsi2ss(XMMRegister dst,Register src)3815 void Assembler::cvtqsi2ss(XMMRegister dst, Register src) {
3816 DCHECK(!IsEnabled(AVX));
3817 EnsureSpace ensure_space(this);
3818 emit(0xF3);
3819 emit_rex_64(dst, src);
3820 emit(0x0F);
3821 emit(0x2A);
3822 emit_sse_operand(dst, src);
3823 }
3824
cvtqsi2sd(XMMRegister dst,Operand src)3825 void Assembler::cvtqsi2sd(XMMRegister dst, Operand src) {
3826 DCHECK(!IsEnabled(AVX));
3827 EnsureSpace ensure_space(this);
3828 emit(0xF2);
3829 emit_rex_64(dst, src);
3830 emit(0x0F);
3831 emit(0x2A);
3832 emit_sse_operand(dst, src);
3833 }
3834
3835
cvtqsi2sd(XMMRegister dst,Register src)3836 void Assembler::cvtqsi2sd(XMMRegister dst, Register src) {
3837 DCHECK(!IsEnabled(AVX));
3838 EnsureSpace ensure_space(this);
3839 emit(0xF2);
3840 emit_rex_64(dst, src);
3841 emit(0x0F);
3842 emit(0x2A);
3843 emit_sse_operand(dst, src);
3844 }
3845
3846
cvtss2sd(XMMRegister dst,XMMRegister src)3847 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
3848 DCHECK(!IsEnabled(AVX));
3849 EnsureSpace ensure_space(this);
3850 emit(0xF3);
3851 emit_optional_rex_32(dst, src);
3852 emit(0x0F);
3853 emit(0x5A);
3854 emit_sse_operand(dst, src);
3855 }
3856
cvtss2sd(XMMRegister dst,Operand src)3857 void Assembler::cvtss2sd(XMMRegister dst, Operand src) {
3858 DCHECK(!IsEnabled(AVX));
3859 EnsureSpace ensure_space(this);
3860 emit(0xF3);
3861 emit_optional_rex_32(dst, src);
3862 emit(0x0F);
3863 emit(0x5A);
3864 emit_sse_operand(dst, src);
3865 }
3866
3867
cvtsd2ss(XMMRegister dst,XMMRegister src)3868 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
3869 DCHECK(!IsEnabled(AVX));
3870 EnsureSpace ensure_space(this);
3871 emit(0xF2);
3872 emit_optional_rex_32(dst, src);
3873 emit(0x0F);
3874 emit(0x5A);
3875 emit_sse_operand(dst, src);
3876 }
3877
cvtsd2ss(XMMRegister dst,Operand src)3878 void Assembler::cvtsd2ss(XMMRegister dst, Operand src) {
3879 DCHECK(!IsEnabled(AVX));
3880 EnsureSpace ensure_space(this);
3881 emit(0xF2);
3882 emit_optional_rex_32(dst, src);
3883 emit(0x0F);
3884 emit(0x5A);
3885 emit_sse_operand(dst, src);
3886 }
3887
3888
cvtsd2si(Register dst,XMMRegister src)3889 void Assembler::cvtsd2si(Register dst, XMMRegister src) {
3890 DCHECK(!IsEnabled(AVX));
3891 EnsureSpace ensure_space(this);
3892 emit(0xF2);
3893 emit_optional_rex_32(dst, src);
3894 emit(0x0F);
3895 emit(0x2D);
3896 emit_sse_operand(dst, src);
3897 }
3898
3899
cvtsd2siq(Register dst,XMMRegister src)3900 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
3901 DCHECK(!IsEnabled(AVX));
3902 EnsureSpace ensure_space(this);
3903 emit(0xF2);
3904 emit_rex_64(dst, src);
3905 emit(0x0F);
3906 emit(0x2D);
3907 emit_sse_operand(dst, src);
3908 }
3909
3910
addsd(XMMRegister dst,XMMRegister src)3911 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
3912 EnsureSpace ensure_space(this);
3913 emit(0xF2);
3914 emit_optional_rex_32(dst, src);
3915 emit(0x0F);
3916 emit(0x58);
3917 emit_sse_operand(dst, src);
3918 }
3919
addsd(XMMRegister dst,Operand src)3920 void Assembler::addsd(XMMRegister dst, Operand src) {
3921 EnsureSpace ensure_space(this);
3922 emit(0xF2);
3923 emit_optional_rex_32(dst, src);
3924 emit(0x0F);
3925 emit(0x58);
3926 emit_sse_operand(dst, src);
3927 }
3928
3929
mulsd(XMMRegister dst,XMMRegister src)3930 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
3931 EnsureSpace ensure_space(this);
3932 emit(0xF2);
3933 emit_optional_rex_32(dst, src);
3934 emit(0x0F);
3935 emit(0x59);
3936 emit_sse_operand(dst, src);
3937 }
3938
mulsd(XMMRegister dst,Operand src)3939 void Assembler::mulsd(XMMRegister dst, Operand src) {
3940 EnsureSpace ensure_space(this);
3941 emit(0xF2);
3942 emit_optional_rex_32(dst, src);
3943 emit(0x0F);
3944 emit(0x59);
3945 emit_sse_operand(dst, src);
3946 }
3947
3948
subsd(XMMRegister dst,XMMRegister src)3949 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
3950 EnsureSpace ensure_space(this);
3951 emit(0xF2);
3952 emit_optional_rex_32(dst, src);
3953 emit(0x0F);
3954 emit(0x5C);
3955 emit_sse_operand(dst, src);
3956 }
3957
subsd(XMMRegister dst,Operand src)3958 void Assembler::subsd(XMMRegister dst, Operand src) {
3959 EnsureSpace ensure_space(this);
3960 emit(0xF2);
3961 emit_optional_rex_32(dst, src);
3962 emit(0x0F);
3963 emit(0x5C);
3964 emit_sse_operand(dst, src);
3965 }
3966
3967
divsd(XMMRegister dst,XMMRegister src)3968 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
3969 EnsureSpace ensure_space(this);
3970 emit(0xF2);
3971 emit_optional_rex_32(dst, src);
3972 emit(0x0F);
3973 emit(0x5E);
3974 emit_sse_operand(dst, src);
3975 }
3976
divsd(XMMRegister dst,Operand src)3977 void Assembler::divsd(XMMRegister dst, Operand src) {
3978 EnsureSpace ensure_space(this);
3979 emit(0xF2);
3980 emit_optional_rex_32(dst, src);
3981 emit(0x0F);
3982 emit(0x5E);
3983 emit_sse_operand(dst, src);
3984 }
3985
3986
maxsd(XMMRegister dst,XMMRegister src)3987 void Assembler::maxsd(XMMRegister dst, XMMRegister src) {
3988 EnsureSpace ensure_space(this);
3989 emit(0xF2);
3990 emit_optional_rex_32(dst, src);
3991 emit(0x0F);
3992 emit(0x5F);
3993 emit_sse_operand(dst, src);
3994 }
3995
maxsd(XMMRegister dst,Operand src)3996 void Assembler::maxsd(XMMRegister dst, Operand src) {
3997 EnsureSpace ensure_space(this);
3998 emit(0xF2);
3999 emit_optional_rex_32(dst, src);
4000 emit(0x0F);
4001 emit(0x5F);
4002 emit_sse_operand(dst, src);
4003 }
4004
4005
minsd(XMMRegister dst,XMMRegister src)4006 void Assembler::minsd(XMMRegister dst, XMMRegister src) {
4007 EnsureSpace ensure_space(this);
4008 emit(0xF2);
4009 emit_optional_rex_32(dst, src);
4010 emit(0x0F);
4011 emit(0x5D);
4012 emit_sse_operand(dst, src);
4013 }
4014
minsd(XMMRegister dst,Operand src)4015 void Assembler::minsd(XMMRegister dst, Operand src) {
4016 EnsureSpace ensure_space(this);
4017 emit(0xF2);
4018 emit_optional_rex_32(dst, src);
4019 emit(0x0F);
4020 emit(0x5D);
4021 emit_sse_operand(dst, src);
4022 }
4023
4024
andpd(XMMRegister dst,XMMRegister src)4025 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
4026 EnsureSpace ensure_space(this);
4027 emit(0x66);
4028 emit_optional_rex_32(dst, src);
4029 emit(0x0F);
4030 emit(0x54);
4031 emit_sse_operand(dst, src);
4032 }
4033
andpd(XMMRegister dst,Operand src)4034 void Assembler::andpd(XMMRegister dst, Operand src) {
4035 EnsureSpace ensure_space(this);
4036 emit(0x66);
4037 emit_optional_rex_32(dst, src);
4038 emit(0x0F);
4039 emit(0x54);
4040 emit_sse_operand(dst, src);
4041 }
4042
4043
orpd(XMMRegister dst,XMMRegister src)4044 void Assembler::orpd(XMMRegister dst, XMMRegister src) {
4045 EnsureSpace ensure_space(this);
4046 emit(0x66);
4047 emit_optional_rex_32(dst, src);
4048 emit(0x0F);
4049 emit(0x56);
4050 emit_sse_operand(dst, src);
4051 }
4052
orpd(XMMRegister dst,Operand src)4053 void Assembler::orpd(XMMRegister dst, Operand src) {
4054 EnsureSpace ensure_space(this);
4055 emit(0x66);
4056 emit_optional_rex_32(dst, src);
4057 emit(0x0F);
4058 emit(0x56);
4059 emit_sse_operand(dst, src);
4060 }
4061
4062
xorpd(XMMRegister dst,XMMRegister src)4063 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
4064 DCHECK(!IsEnabled(AVX));
4065 EnsureSpace ensure_space(this);
4066 emit(0x66);
4067 emit_optional_rex_32(dst, src);
4068 emit(0x0F);
4069 emit(0x57);
4070 emit_sse_operand(dst, src);
4071 }
4072
xorpd(XMMRegister dst,Operand src)4073 void Assembler::xorpd(XMMRegister dst, Operand src) {
4074 DCHECK(!IsEnabled(AVX));
4075 EnsureSpace ensure_space(this);
4076 emit(0x66);
4077 emit_optional_rex_32(dst, src);
4078 emit(0x0F);
4079 emit(0x57);
4080 emit_sse_operand(dst, src);
4081 }
4082
4083
sqrtsd(XMMRegister dst,XMMRegister src)4084 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
4085 DCHECK(!IsEnabled(AVX));
4086 EnsureSpace ensure_space(this);
4087 emit(0xF2);
4088 emit_optional_rex_32(dst, src);
4089 emit(0x0F);
4090 emit(0x51);
4091 emit_sse_operand(dst, src);
4092 }
4093
sqrtsd(XMMRegister dst,Operand src)4094 void Assembler::sqrtsd(XMMRegister dst, Operand src) {
4095 DCHECK(!IsEnabled(AVX));
4096 EnsureSpace ensure_space(this);
4097 emit(0xF2);
4098 emit_optional_rex_32(dst, src);
4099 emit(0x0F);
4100 emit(0x51);
4101 emit_sse_operand(dst, src);
4102 }
4103
haddps(XMMRegister dst,XMMRegister src)4104 void Assembler::haddps(XMMRegister dst, XMMRegister src) {
4105 DCHECK(IsEnabled(SSE3));
4106 EnsureSpace ensure_space(this);
4107 emit(0xF2);
4108 emit_optional_rex_32(dst, src);
4109 emit(0x0F);
4110 emit(0x7C);
4111 emit_sse_operand(dst, src);
4112 }
4113
haddps(XMMRegister dst,Operand src)4114 void Assembler::haddps(XMMRegister dst, Operand src) {
4115 DCHECK(IsEnabled(SSE3));
4116 EnsureSpace ensure_space(this);
4117 emit(0xF2);
4118 emit_optional_rex_32(dst, src);
4119 emit(0x0F);
4120 emit(0x7C);
4121 emit_sse_operand(dst, src);
4122 }
4123
ucomisd(XMMRegister dst,XMMRegister src)4124 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
4125 DCHECK(!IsEnabled(AVX));
4126 EnsureSpace ensure_space(this);
4127 emit(0x66);
4128 emit_optional_rex_32(dst, src);
4129 emit(0x0F);
4130 emit(0x2E);
4131 emit_sse_operand(dst, src);
4132 }
4133
ucomisd(XMMRegister dst,Operand src)4134 void Assembler::ucomisd(XMMRegister dst, Operand src) {
4135 DCHECK(!IsEnabled(AVX));
4136 EnsureSpace ensure_space(this);
4137 emit(0x66);
4138 emit_optional_rex_32(dst, src);
4139 emit(0x0F);
4140 emit(0x2E);
4141 emit_sse_operand(dst, src);
4142 }
4143
4144
cmpltsd(XMMRegister dst,XMMRegister src)4145 void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) {
4146 EnsureSpace ensure_space(this);
4147 emit(0xF2);
4148 emit_optional_rex_32(dst, src);
4149 emit(0x0F);
4150 emit(0xC2);
4151 emit_sse_operand(dst, src);
4152 emit(0x01); // LT == 1
4153 }
4154
4155
roundss(XMMRegister dst,XMMRegister src,RoundingMode mode)4156 void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) {
4157 DCHECK(!IsEnabled(AVX));
4158 DCHECK(IsEnabled(SSE4_1));
4159 EnsureSpace ensure_space(this);
4160 emit(0x66);
4161 emit_optional_rex_32(dst, src);
4162 emit(0x0F);
4163 emit(0x3A);
4164 emit(0x0A);
4165 emit_sse_operand(dst, src);
4166 // Mask precision exception.
4167 emit(static_cast<byte>(mode) | 0x8);
4168 }
4169
4170
roundsd(XMMRegister dst,XMMRegister src,RoundingMode mode)4171 void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
4172 DCHECK(!IsEnabled(AVX));
4173 DCHECK(IsEnabled(SSE4_1));
4174 EnsureSpace ensure_space(this);
4175 emit(0x66);
4176 emit_optional_rex_32(dst, src);
4177 emit(0x0F);
4178 emit(0x3A);
4179 emit(0x0B);
4180 emit_sse_operand(dst, src);
4181 // Mask precision exception.
4182 emit(static_cast<byte>(mode) | 0x8);
4183 }
4184
4185
movmskpd(Register dst,XMMRegister src)4186 void Assembler::movmskpd(Register dst, XMMRegister src) {
4187 EnsureSpace ensure_space(this);
4188 emit(0x66);
4189 emit_optional_rex_32(dst, src);
4190 emit(0x0F);
4191 emit(0x50);
4192 emit_sse_operand(dst, src);
4193 }
4194
4195
movmskps(Register dst,XMMRegister src)4196 void Assembler::movmskps(Register dst, XMMRegister src) {
4197 EnsureSpace ensure_space(this);
4198 emit_optional_rex_32(dst, src);
4199 emit(0x0F);
4200 emit(0x50);
4201 emit_sse_operand(dst, src);
4202 }
4203
4204
4205 // AVX instructions
vfmasd(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)4206 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1,
4207 XMMRegister src2) {
4208 DCHECK(IsEnabled(FMA3));
4209 EnsureSpace ensure_space(this);
4210 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1);
4211 emit(op);
4212 emit_sse_operand(dst, src2);
4213 }
4214
vfmasd(byte op,XMMRegister dst,XMMRegister src1,Operand src2)4215 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1,
4216 Operand src2) {
4217 DCHECK(IsEnabled(FMA3));
4218 EnsureSpace ensure_space(this);
4219 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1);
4220 emit(op);
4221 emit_sse_operand(dst, src2);
4222 }
4223
4224
vfmass(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)4225 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1,
4226 XMMRegister src2) {
4227 DCHECK(IsEnabled(FMA3));
4228 EnsureSpace ensure_space(this);
4229 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0);
4230 emit(op);
4231 emit_sse_operand(dst, src2);
4232 }
4233
vfmass(byte op,XMMRegister dst,XMMRegister src1,Operand src2)4234 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1,
4235 Operand src2) {
4236 DCHECK(IsEnabled(FMA3));
4237 EnsureSpace ensure_space(this);
4238 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0);
4239 emit(op);
4240 emit_sse_operand(dst, src2);
4241 }
4242
4243
vmovd(XMMRegister dst,Register src)4244 void Assembler::vmovd(XMMRegister dst, Register src) {
4245 DCHECK(IsEnabled(AVX));
4246 EnsureSpace ensure_space(this);
4247 XMMRegister isrc = XMMRegister::from_code(src.code());
4248 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW0);
4249 emit(0x6E);
4250 emit_sse_operand(dst, src);
4251 }
4252
vmovd(XMMRegister dst,Operand src)4253 void Assembler::vmovd(XMMRegister dst, Operand src) {
4254 DCHECK(IsEnabled(AVX));
4255 EnsureSpace ensure_space(this);
4256 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW0);
4257 emit(0x6E);
4258 emit_sse_operand(dst, src);
4259 }
4260
4261
vmovd(Register dst,XMMRegister src)4262 void Assembler::vmovd(Register dst, XMMRegister src) {
4263 DCHECK(IsEnabled(AVX));
4264 EnsureSpace ensure_space(this);
4265 XMMRegister idst = XMMRegister::from_code(dst.code());
4266 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW0);
4267 emit(0x7E);
4268 emit_sse_operand(src, dst);
4269 }
4270
4271
vmovq(XMMRegister dst,Register src)4272 void Assembler::vmovq(XMMRegister dst, Register src) {
4273 DCHECK(IsEnabled(AVX));
4274 EnsureSpace ensure_space(this);
4275 XMMRegister isrc = XMMRegister::from_code(src.code());
4276 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW1);
4277 emit(0x6E);
4278 emit_sse_operand(dst, src);
4279 }
4280
vmovq(XMMRegister dst,Operand src)4281 void Assembler::vmovq(XMMRegister dst, Operand src) {
4282 DCHECK(IsEnabled(AVX));
4283 EnsureSpace ensure_space(this);
4284 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW1);
4285 emit(0x6E);
4286 emit_sse_operand(dst, src);
4287 }
4288
4289
vmovq(Register dst,XMMRegister src)4290 void Assembler::vmovq(Register dst, XMMRegister src) {
4291 DCHECK(IsEnabled(AVX));
4292 EnsureSpace ensure_space(this);
4293 XMMRegister idst = XMMRegister::from_code(dst.code());
4294 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW1);
4295 emit(0x7E);
4296 emit_sse_operand(src, dst);
4297 }
4298
vinstr(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2,SIMDPrefix pp,LeadingOpcode m,VexW w)4299 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1,
4300 XMMRegister src2, SIMDPrefix pp, LeadingOpcode m,
4301 VexW w) {
4302 DCHECK(IsEnabled(AVX));
4303 EnsureSpace ensure_space(this);
4304 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
4305 emit(op);
4306 emit_sse_operand(dst, src2);
4307 }
4308
vinstr(byte op,XMMRegister dst,XMMRegister src1,Operand src2,SIMDPrefix pp,LeadingOpcode m,VexW w)4309 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1, Operand src2,
4310 SIMDPrefix pp, LeadingOpcode m, VexW w) {
4311 DCHECK(IsEnabled(AVX));
4312 EnsureSpace ensure_space(this);
4313 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
4314 emit(op);
4315 emit_sse_operand(dst, src2);
4316 }
4317
4318
vps(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)4319 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
4320 XMMRegister src2) {
4321 DCHECK(IsEnabled(AVX));
4322 EnsureSpace ensure_space(this);
4323 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
4324 emit(op);
4325 emit_sse_operand(dst, src2);
4326 }
4327
vps(byte op,XMMRegister dst,XMMRegister src1,Operand src2)4328 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
4329 DCHECK(IsEnabled(AVX));
4330 EnsureSpace ensure_space(this);
4331 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
4332 emit(op);
4333 emit_sse_operand(dst, src2);
4334 }
4335
4336
vpd(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)4337 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1,
4338 XMMRegister src2) {
4339 DCHECK(IsEnabled(AVX));
4340 EnsureSpace ensure_space(this);
4341 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
4342 emit(op);
4343 emit_sse_operand(dst, src2);
4344 }
4345
vpd(byte op,XMMRegister dst,XMMRegister src1,Operand src2)4346 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
4347 DCHECK(IsEnabled(AVX));
4348 EnsureSpace ensure_space(this);
4349 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
4350 emit(op);
4351 emit_sse_operand(dst, src2);
4352 }
4353
4354
vucomiss(XMMRegister dst,XMMRegister src)4355 void Assembler::vucomiss(XMMRegister dst, XMMRegister src) {
4356 DCHECK(IsEnabled(AVX));
4357 EnsureSpace ensure_space(this);
4358 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
4359 emit(0x2E);
4360 emit_sse_operand(dst, src);
4361 }
4362
vucomiss(XMMRegister dst,Operand src)4363 void Assembler::vucomiss(XMMRegister dst, Operand src) {
4364 DCHECK(IsEnabled(AVX));
4365 EnsureSpace ensure_space(this);
4366 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
4367 emit(0x2E);
4368 emit_sse_operand(dst, src);
4369 }
4370
4371
vss(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)4372 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1,
4373 XMMRegister src2) {
4374 DCHECK(IsEnabled(AVX));
4375 EnsureSpace ensure_space(this);
4376 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
4377 emit(op);
4378 emit_sse_operand(dst, src2);
4379 }
4380
vss(byte op,XMMRegister dst,XMMRegister src1,Operand src2)4381 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
4382 DCHECK(IsEnabled(AVX));
4383 EnsureSpace ensure_space(this);
4384 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
4385 emit(op);
4386 emit_sse_operand(dst, src2);
4387 }
4388
4389
bmi1q(byte op,Register reg,Register vreg,Register rm)4390 void Assembler::bmi1q(byte op, Register reg, Register vreg, Register rm) {
4391 DCHECK(IsEnabled(BMI1));
4392 EnsureSpace ensure_space(this);
4393 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
4394 emit(op);
4395 emit_modrm(reg, rm);
4396 }
4397
bmi1q(byte op,Register reg,Register vreg,Operand rm)4398 void Assembler::bmi1q(byte op, Register reg, Register vreg, Operand rm) {
4399 DCHECK(IsEnabled(BMI1));
4400 EnsureSpace ensure_space(this);
4401 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
4402 emit(op);
4403 emit_operand(reg, rm);
4404 }
4405
4406
bmi1l(byte op,Register reg,Register vreg,Register rm)4407 void Assembler::bmi1l(byte op, Register reg, Register vreg, Register rm) {
4408 DCHECK(IsEnabled(BMI1));
4409 EnsureSpace ensure_space(this);
4410 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
4411 emit(op);
4412 emit_modrm(reg, rm);
4413 }
4414
bmi1l(byte op,Register reg,Register vreg,Operand rm)4415 void Assembler::bmi1l(byte op, Register reg, Register vreg, Operand rm) {
4416 DCHECK(IsEnabled(BMI1));
4417 EnsureSpace ensure_space(this);
4418 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
4419 emit(op);
4420 emit_operand(reg, rm);
4421 }
4422
4423
tzcntq(Register dst,Register src)4424 void Assembler::tzcntq(Register dst, Register src) {
4425 DCHECK(IsEnabled(BMI1));
4426 EnsureSpace ensure_space(this);
4427 emit(0xF3);
4428 emit_rex_64(dst, src);
4429 emit(0x0F);
4430 emit(0xBC);
4431 emit_modrm(dst, src);
4432 }
4433
tzcntq(Register dst,Operand src)4434 void Assembler::tzcntq(Register dst, Operand src) {
4435 DCHECK(IsEnabled(BMI1));
4436 EnsureSpace ensure_space(this);
4437 emit(0xF3);
4438 emit_rex_64(dst, src);
4439 emit(0x0F);
4440 emit(0xBC);
4441 emit_operand(dst, src);
4442 }
4443
4444
tzcntl(Register dst,Register src)4445 void Assembler::tzcntl(Register dst, Register src) {
4446 DCHECK(IsEnabled(BMI1));
4447 EnsureSpace ensure_space(this);
4448 emit(0xF3);
4449 emit_optional_rex_32(dst, src);
4450 emit(0x0F);
4451 emit(0xBC);
4452 emit_modrm(dst, src);
4453 }
4454
tzcntl(Register dst,Operand src)4455 void Assembler::tzcntl(Register dst, Operand src) {
4456 DCHECK(IsEnabled(BMI1));
4457 EnsureSpace ensure_space(this);
4458 emit(0xF3);
4459 emit_optional_rex_32(dst, src);
4460 emit(0x0F);
4461 emit(0xBC);
4462 emit_operand(dst, src);
4463 }
4464
4465
lzcntq(Register dst,Register src)4466 void Assembler::lzcntq(Register dst, Register src) {
4467 DCHECK(IsEnabled(LZCNT));
4468 EnsureSpace ensure_space(this);
4469 emit(0xF3);
4470 emit_rex_64(dst, src);
4471 emit(0x0F);
4472 emit(0xBD);
4473 emit_modrm(dst, src);
4474 }
4475
lzcntq(Register dst,Operand src)4476 void Assembler::lzcntq(Register dst, Operand src) {
4477 DCHECK(IsEnabled(LZCNT));
4478 EnsureSpace ensure_space(this);
4479 emit(0xF3);
4480 emit_rex_64(dst, src);
4481 emit(0x0F);
4482 emit(0xBD);
4483 emit_operand(dst, src);
4484 }
4485
4486
lzcntl(Register dst,Register src)4487 void Assembler::lzcntl(Register dst, Register src) {
4488 DCHECK(IsEnabled(LZCNT));
4489 EnsureSpace ensure_space(this);
4490 emit(0xF3);
4491 emit_optional_rex_32(dst, src);
4492 emit(0x0F);
4493 emit(0xBD);
4494 emit_modrm(dst, src);
4495 }
4496
lzcntl(Register dst,Operand src)4497 void Assembler::lzcntl(Register dst, Operand src) {
4498 DCHECK(IsEnabled(LZCNT));
4499 EnsureSpace ensure_space(this);
4500 emit(0xF3);
4501 emit_optional_rex_32(dst, src);
4502 emit(0x0F);
4503 emit(0xBD);
4504 emit_operand(dst, src);
4505 }
4506
4507
popcntq(Register dst,Register src)4508 void Assembler::popcntq(Register dst, Register src) {
4509 DCHECK(IsEnabled(POPCNT));
4510 EnsureSpace ensure_space(this);
4511 emit(0xF3);
4512 emit_rex_64(dst, src);
4513 emit(0x0F);
4514 emit(0xB8);
4515 emit_modrm(dst, src);
4516 }
4517
popcntq(Register dst,Operand src)4518 void Assembler::popcntq(Register dst, Operand src) {
4519 DCHECK(IsEnabled(POPCNT));
4520 EnsureSpace ensure_space(this);
4521 emit(0xF3);
4522 emit_rex_64(dst, src);
4523 emit(0x0F);
4524 emit(0xB8);
4525 emit_operand(dst, src);
4526 }
4527
4528
popcntl(Register dst,Register src)4529 void Assembler::popcntl(Register dst, Register src) {
4530 DCHECK(IsEnabled(POPCNT));
4531 EnsureSpace ensure_space(this);
4532 emit(0xF3);
4533 emit_optional_rex_32(dst, src);
4534 emit(0x0F);
4535 emit(0xB8);
4536 emit_modrm(dst, src);
4537 }
4538
popcntl(Register dst,Operand src)4539 void Assembler::popcntl(Register dst, Operand src) {
4540 DCHECK(IsEnabled(POPCNT));
4541 EnsureSpace ensure_space(this);
4542 emit(0xF3);
4543 emit_optional_rex_32(dst, src);
4544 emit(0x0F);
4545 emit(0xB8);
4546 emit_operand(dst, src);
4547 }
4548
4549
bmi2q(SIMDPrefix pp,byte op,Register reg,Register vreg,Register rm)4550 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
4551 Register rm) {
4552 DCHECK(IsEnabled(BMI2));
4553 EnsureSpace ensure_space(this);
4554 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
4555 emit(op);
4556 emit_modrm(reg, rm);
4557 }
4558
bmi2q(SIMDPrefix pp,byte op,Register reg,Register vreg,Operand rm)4559 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
4560 Operand rm) {
4561 DCHECK(IsEnabled(BMI2));
4562 EnsureSpace ensure_space(this);
4563 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
4564 emit(op);
4565 emit_operand(reg, rm);
4566 }
4567
4568
bmi2l(SIMDPrefix pp,byte op,Register reg,Register vreg,Register rm)4569 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
4570 Register rm) {
4571 DCHECK(IsEnabled(BMI2));
4572 EnsureSpace ensure_space(this);
4573 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
4574 emit(op);
4575 emit_modrm(reg, rm);
4576 }
4577
bmi2l(SIMDPrefix pp,byte op,Register reg,Register vreg,Operand rm)4578 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
4579 Operand rm) {
4580 DCHECK(IsEnabled(BMI2));
4581 EnsureSpace ensure_space(this);
4582 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
4583 emit(op);
4584 emit_operand(reg, rm);
4585 }
4586
4587
rorxq(Register dst,Register src,byte imm8)4588 void Assembler::rorxq(Register dst, Register src, byte imm8) {
4589 DCHECK(IsEnabled(BMI2));
4590 DCHECK(is_uint8(imm8));
4591 Register vreg = Register::from_code<0>(); // VEX.vvvv unused
4592 EnsureSpace ensure_space(this);
4593 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
4594 emit(0xF0);
4595 emit_modrm(dst, src);
4596 emit(imm8);
4597 }
4598
rorxq(Register dst,Operand src,byte imm8)4599 void Assembler::rorxq(Register dst, Operand src, byte imm8) {
4600 DCHECK(IsEnabled(BMI2));
4601 DCHECK(is_uint8(imm8));
4602 Register vreg = Register::from_code<0>(); // VEX.vvvv unused
4603 EnsureSpace ensure_space(this);
4604 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
4605 emit(0xF0);
4606 emit_operand(dst, src);
4607 emit(imm8);
4608 }
4609
4610
rorxl(Register dst,Register src,byte imm8)4611 void Assembler::rorxl(Register dst, Register src, byte imm8) {
4612 DCHECK(IsEnabled(BMI2));
4613 DCHECK(is_uint8(imm8));
4614 Register vreg = Register::from_code<0>(); // VEX.vvvv unused
4615 EnsureSpace ensure_space(this);
4616 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
4617 emit(0xF0);
4618 emit_modrm(dst, src);
4619 emit(imm8);
4620 }
4621
rorxl(Register dst,Operand src,byte imm8)4622 void Assembler::rorxl(Register dst, Operand src, byte imm8) {
4623 DCHECK(IsEnabled(BMI2));
4624 DCHECK(is_uint8(imm8));
4625 Register vreg = Register::from_code<0>(); // VEX.vvvv unused
4626 EnsureSpace ensure_space(this);
4627 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
4628 emit(0xF0);
4629 emit_operand(dst, src);
4630 emit(imm8);
4631 }
4632
pause()4633 void Assembler::pause() {
4634 emit(0xF3);
4635 emit(0x90);
4636 }
4637
minps(XMMRegister dst,XMMRegister src)4638 void Assembler::minps(XMMRegister dst, XMMRegister src) {
4639 EnsureSpace ensure_space(this);
4640 emit_optional_rex_32(dst, src);
4641 emit(0x0F);
4642 emit(0x5D);
4643 emit_sse_operand(dst, src);
4644 }
4645
minps(XMMRegister dst,Operand src)4646 void Assembler::minps(XMMRegister dst, Operand src) {
4647 EnsureSpace ensure_space(this);
4648 emit_optional_rex_32(dst, src);
4649 emit(0x0F);
4650 emit(0x5D);
4651 emit_sse_operand(dst, src);
4652 }
4653
maxps(XMMRegister dst,XMMRegister src)4654 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
4655 EnsureSpace ensure_space(this);
4656 emit_optional_rex_32(dst, src);
4657 emit(0x0F);
4658 emit(0x5F);
4659 emit_sse_operand(dst, src);
4660 }
4661
maxps(XMMRegister dst,Operand src)4662 void Assembler::maxps(XMMRegister dst, Operand src) {
4663 EnsureSpace ensure_space(this);
4664 emit_optional_rex_32(dst, src);
4665 emit(0x0F);
4666 emit(0x5F);
4667 emit_sse_operand(dst, src);
4668 }
4669
rcpps(XMMRegister dst,XMMRegister src)4670 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
4671 EnsureSpace ensure_space(this);
4672 emit_optional_rex_32(dst, src);
4673 emit(0x0F);
4674 emit(0x53);
4675 emit_sse_operand(dst, src);
4676 }
4677
rcpps(XMMRegister dst,Operand src)4678 void Assembler::rcpps(XMMRegister dst, Operand src) {
4679 EnsureSpace ensure_space(this);
4680 emit_optional_rex_32(dst, src);
4681 emit(0x0F);
4682 emit(0x53);
4683 emit_sse_operand(dst, src);
4684 }
4685
rsqrtps(XMMRegister dst,XMMRegister src)4686 void Assembler::rsqrtps(XMMRegister dst, XMMRegister src) {
4687 EnsureSpace ensure_space(this);
4688 emit_optional_rex_32(dst, src);
4689 emit(0x0F);
4690 emit(0x52);
4691 emit_sse_operand(dst, src);
4692 }
4693
rsqrtps(XMMRegister dst,Operand src)4694 void Assembler::rsqrtps(XMMRegister dst, Operand src) {
4695 EnsureSpace ensure_space(this);
4696 emit_optional_rex_32(dst, src);
4697 emit(0x0F);
4698 emit(0x52);
4699 emit_sse_operand(dst, src);
4700 }
4701
sqrtps(XMMRegister dst,XMMRegister src)4702 void Assembler::sqrtps(XMMRegister dst, XMMRegister src) {
4703 EnsureSpace ensure_space(this);
4704 emit_optional_rex_32(dst, src);
4705 emit(0x0F);
4706 emit(0x51);
4707 emit_sse_operand(dst, src);
4708 }
4709
sqrtps(XMMRegister dst,Operand src)4710 void Assembler::sqrtps(XMMRegister dst, Operand src) {
4711 EnsureSpace ensure_space(this);
4712 emit_optional_rex_32(dst, src);
4713 emit(0x0F);
4714 emit(0x51);
4715 emit_sse_operand(dst, src);
4716 }
4717
cvtdq2ps(XMMRegister dst,XMMRegister src)4718 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
4719 EnsureSpace ensure_space(this);
4720 emit_optional_rex_32(dst, src);
4721 emit(0x0F);
4722 emit(0x5B);
4723 emit_sse_operand(dst, src);
4724 }
4725
cvtdq2ps(XMMRegister dst,Operand src)4726 void Assembler::cvtdq2ps(XMMRegister dst, Operand src) {
4727 EnsureSpace ensure_space(this);
4728 emit_optional_rex_32(dst, src);
4729 emit(0x0F);
4730 emit(0x5B);
4731 emit_sse_operand(dst, src);
4732 }
4733
movups(XMMRegister dst,XMMRegister src)4734 void Assembler::movups(XMMRegister dst, XMMRegister src) {
4735 EnsureSpace ensure_space(this);
4736 if (src.low_bits() == 4) {
4737 // Try to avoid an unnecessary SIB byte.
4738 emit_optional_rex_32(src, dst);
4739 emit(0x0F);
4740 emit(0x11);
4741 emit_sse_operand(src, dst);
4742 } else {
4743 emit_optional_rex_32(dst, src);
4744 emit(0x0F);
4745 emit(0x10);
4746 emit_sse_operand(dst, src);
4747 }
4748 }
4749
movups(XMMRegister dst,Operand src)4750 void Assembler::movups(XMMRegister dst, Operand src) {
4751 EnsureSpace ensure_space(this);
4752 emit_optional_rex_32(dst, src);
4753 emit(0x0F);
4754 emit(0x10);
4755 emit_sse_operand(dst, src);
4756 }
4757
movups(Operand dst,XMMRegister src)4758 void Assembler::movups(Operand dst, XMMRegister src) {
4759 EnsureSpace ensure_space(this);
4760 emit_optional_rex_32(src, dst);
4761 emit(0x0F);
4762 emit(0x11);
4763 emit_sse_operand(src, dst);
4764 }
4765
sse2_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape,byte opcode)4766 void Assembler::sse2_instr(XMMRegister dst, XMMRegister src, byte prefix,
4767 byte escape, byte opcode) {
4768 EnsureSpace ensure_space(this);
4769 emit(prefix);
4770 emit_optional_rex_32(dst, src);
4771 emit(escape);
4772 emit(opcode);
4773 emit_sse_operand(dst, src);
4774 }
4775
sse2_instr(XMMRegister dst,Operand src,byte prefix,byte escape,byte opcode)4776 void Assembler::sse2_instr(XMMRegister dst, Operand src, byte prefix,
4777 byte escape, byte opcode) {
4778 EnsureSpace ensure_space(this);
4779 emit(prefix);
4780 emit_optional_rex_32(dst, src);
4781 emit(escape);
4782 emit(opcode);
4783 emit_sse_operand(dst, src);
4784 }
4785
ssse3_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode)4786 void Assembler::ssse3_instr(XMMRegister dst, XMMRegister src, byte prefix,
4787 byte escape1, byte escape2, byte opcode) {
4788 DCHECK(IsEnabled(SSSE3));
4789 EnsureSpace ensure_space(this);
4790 emit(prefix);
4791 emit_optional_rex_32(dst, src);
4792 emit(escape1);
4793 emit(escape2);
4794 emit(opcode);
4795 emit_sse_operand(dst, src);
4796 }
4797
ssse3_instr(XMMRegister dst,Operand src,byte prefix,byte escape1,byte escape2,byte opcode)4798 void Assembler::ssse3_instr(XMMRegister dst, Operand src, byte prefix,
4799 byte escape1, byte escape2, byte opcode) {
4800 DCHECK(IsEnabled(SSSE3));
4801 EnsureSpace ensure_space(this);
4802 emit(prefix);
4803 emit_optional_rex_32(dst, src);
4804 emit(escape1);
4805 emit(escape2);
4806 emit(opcode);
4807 emit_sse_operand(dst, src);
4808 }
4809
sse4_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode)4810 void Assembler::sse4_instr(XMMRegister dst, XMMRegister src, byte prefix,
4811 byte escape1, byte escape2, byte opcode) {
4812 DCHECK(IsEnabled(SSE4_1));
4813 EnsureSpace ensure_space(this);
4814 emit(prefix);
4815 emit_optional_rex_32(dst, src);
4816 emit(escape1);
4817 emit(escape2);
4818 emit(opcode);
4819 emit_sse_operand(dst, src);
4820 }
4821
sse4_instr(XMMRegister dst,Operand src,byte prefix,byte escape1,byte escape2,byte opcode)4822 void Assembler::sse4_instr(XMMRegister dst, Operand src, byte prefix,
4823 byte escape1, byte escape2, byte opcode) {
4824 DCHECK(IsEnabled(SSE4_1));
4825 EnsureSpace ensure_space(this);
4826 emit(prefix);
4827 emit_optional_rex_32(dst, src);
4828 emit(escape1);
4829 emit(escape2);
4830 emit(opcode);
4831 emit_sse_operand(dst, src);
4832 }
4833
lddqu(XMMRegister dst,Operand src)4834 void Assembler::lddqu(XMMRegister dst, Operand src) {
4835 DCHECK(IsEnabled(SSE3));
4836 EnsureSpace ensure_space(this);
4837 emit(0xF2);
4838 emit_optional_rex_32(dst, src);
4839 emit(0x0F);
4840 emit(0xF0);
4841 emit_sse_operand(dst, src);
4842 }
4843
psrldq(XMMRegister dst,uint8_t shift)4844 void Assembler::psrldq(XMMRegister dst, uint8_t shift) {
4845 EnsureSpace ensure_space(this);
4846 emit(0x66);
4847 emit_optional_rex_32(dst);
4848 emit(0x0F);
4849 emit(0x73);
4850 emit_sse_operand(dst);
4851 emit(shift);
4852 }
4853
pshufhw(XMMRegister dst,XMMRegister src,uint8_t shuffle)4854 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4855 EnsureSpace ensure_space(this);
4856 emit(0xF3);
4857 emit_optional_rex_32(dst, src);
4858 emit(0x0F);
4859 emit(0x70);
4860 emit_sse_operand(dst, src);
4861 emit(shuffle);
4862 }
4863
pshuflw(XMMRegister dst,XMMRegister src,uint8_t shuffle)4864 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4865 EnsureSpace ensure_space(this);
4866 emit(0xF2);
4867 emit_optional_rex_32(dst, src);
4868 emit(0x0F);
4869 emit(0x70);
4870 emit_sse_operand(dst, src);
4871 emit(shuffle);
4872 }
4873
pshufd(XMMRegister dst,XMMRegister src,uint8_t shuffle)4874 void Assembler::pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4875 EnsureSpace ensure_space(this);
4876 emit(0x66);
4877 emit_optional_rex_32(dst, src);
4878 emit(0x0F);
4879 emit(0x70);
4880 emit_sse_operand(dst, src);
4881 emit(shuffle);
4882 }
4883
pshufd(XMMRegister dst,Operand src,uint8_t shuffle)4884 void Assembler::pshufd(XMMRegister dst, Operand src, uint8_t shuffle) {
4885 EnsureSpace ensure_space(this);
4886 emit(0x66);
4887 emit_optional_rex_32(dst, src);
4888 emit(0x0F);
4889 emit(0x70);
4890 emit_sse_operand(dst, src);
4891 emit(shuffle);
4892 }
4893
emit_sse_operand(XMMRegister reg,Operand adr)4894 void Assembler::emit_sse_operand(XMMRegister reg, Operand adr) {
4895 Register ireg = Register::from_code(reg.code());
4896 emit_operand(ireg, adr);
4897 }
4898
emit_sse_operand(Register reg,Operand adr)4899 void Assembler::emit_sse_operand(Register reg, Operand adr) {
4900 emit_operand(reg, adr);
4901 }
4902
4903
emit_sse_operand(XMMRegister dst,XMMRegister src)4904 void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) {
4905 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4906 }
4907
4908
emit_sse_operand(XMMRegister dst,Register src)4909 void Assembler::emit_sse_operand(XMMRegister dst, Register src) {
4910 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4911 }
4912
4913
emit_sse_operand(Register dst,XMMRegister src)4914 void Assembler::emit_sse_operand(Register dst, XMMRegister src) {
4915 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4916 }
4917
emit_sse_operand(XMMRegister dst)4918 void Assembler::emit_sse_operand(XMMRegister dst) {
4919 emit(0xD8 | dst.low_bits());
4920 }
4921
db(uint8_t data)4922 void Assembler::db(uint8_t data) {
4923 EnsureSpace ensure_space(this);
4924 emit(data);
4925 }
4926
4927
dd(uint32_t data)4928 void Assembler::dd(uint32_t data) {
4929 EnsureSpace ensure_space(this);
4930 emitl(data);
4931 }
4932
4933
dq(uint64_t data)4934 void Assembler::dq(uint64_t data) {
4935 EnsureSpace ensure_space(this);
4936 emitq(data);
4937 }
4938
4939
dq(Label * label)4940 void Assembler::dq(Label* label) {
4941 EnsureSpace ensure_space(this);
4942 if (label->is_bound()) {
4943 internal_reference_positions_.push_back(pc_offset());
4944 emitp(reinterpret_cast<Address>(buffer_) + label->pos(),
4945 RelocInfo::INTERNAL_REFERENCE);
4946 } else {
4947 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
4948 emitl(0); // Zero for the first 32bit marks it as 64bit absolute address.
4949 if (label->is_linked()) {
4950 emitl(label->pos());
4951 label->link_to(pc_offset() - sizeof(int32_t));
4952 } else {
4953 DCHECK(label->is_unused());
4954 int32_t current = pc_offset();
4955 emitl(current);
4956 label->link_to(current);
4957 }
4958 }
4959 }
4960
4961
4962 // Relocation information implementations.
4963
RecordRelocInfo(RelocInfo::Mode rmode,intptr_t data)4964 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
4965 DCHECK(!RelocInfo::IsNone(rmode));
4966 if (options().disable_reloc_info_for_patching) return;
4967 if (RelocInfo::IsOnlyForSerializer(rmode) &&
4968 !options().record_reloc_info_for_serialization && !emit_debug_code()) {
4969 return;
4970 }
4971 RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, nullptr);
4972 reloc_info_writer.Write(&rinfo);
4973 }
4974
4975 const int RelocInfo::kApplyMask =
4976 RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
4977 RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY) |
4978 RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
4979 RelocInfo::ModeMask(RelocInfo::WASM_CALL);
4980
IsCodedSpecially()4981 bool RelocInfo::IsCodedSpecially() {
4982 // The deserializer needs to know whether a pointer is specially coded. Being
4983 // specially coded on x64 means that it is a relative 32 bit address, as used
4984 // by branch instructions.
4985 return (1 << rmode_) & kApplyMask;
4986 }
4987
4988
IsInConstantPool()4989 bool RelocInfo::IsInConstantPool() {
4990 return false;
4991 }
4992
GetDeoptimizationId(Isolate * isolate,DeoptimizeKind kind)4993 int RelocInfo::GetDeoptimizationId(Isolate* isolate, DeoptimizeKind kind) {
4994 DCHECK(IsRuntimeEntry(rmode_));
4995 return Deoptimizer::GetDeoptimizationId(isolate, target_address(), kind);
4996 }
4997
4998 } // namespace internal
4999 } // namespace v8
5000
5001 #endif // V8_TARGET_ARCH_X64
5002