1 // Copyright 2012 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/codegen/x64/assembler-x64.h"
6
7 #include <cstring>
8
9 #if V8_TARGET_ARCH_X64
10
11 #if V8_LIBC_MSVCRT
12 #include <intrin.h> // _xgetbv()
13 #endif
14 #if V8_OS_DARWIN
15 #include <sys/sysctl.h>
16 #endif
17
18 #include "src/base/bits.h"
19 #include "src/base/cpu.h"
20 #include "src/base/platform/wrappers.h"
21 #include "src/codegen/assembler-inl.h"
22 #include "src/codegen/macro-assembler.h"
23 #include "src/codegen/string-constants.h"
24 #include "src/deoptimizer/deoptimizer.h"
25 #include "src/init/v8.h"
26
27 namespace v8 {
28 namespace internal {
29
30 // -----------------------------------------------------------------------------
31 // Implementation of CpuFeatures
32
33 namespace {
34
35 #if V8_HOST_ARCH_IA32 || V8_HOST_ARCH_X64
36
xgetbv(unsigned int xcr)37 V8_INLINE uint64_t xgetbv(unsigned int xcr) {
38 #if V8_LIBC_MSVCRT
39 return _xgetbv(xcr);
40 #else
41 unsigned eax, edx;
42 // Check xgetbv; this uses a .byte sequence instead of the instruction
43 // directly because older assemblers do not include support for xgetbv and
44 // there is no easy way to conditionally compile based on the assembler
45 // used.
46 __asm__ volatile(".byte 0x0F, 0x01, 0xD0" : "=a"(eax), "=d"(edx) : "c"(xcr));
47 return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32);
48 #endif
49 }
50
OSHasAVXSupport()51 bool OSHasAVXSupport() {
52 #if V8_OS_DARWIN
53 // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being
54 // caused by ISRs, so we detect that here and disable AVX in that case.
55 char buffer[128];
56 size_t buffer_size = arraysize(buffer);
57 int ctl_name[] = {CTL_KERN, KERN_OSRELEASE};
58 if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) {
59 FATAL("V8 failed to get kernel version");
60 }
61 // The buffer now contains a string of the form XX.YY.ZZ, where
62 // XX is the major kernel version component.
63 char* period_pos = strchr(buffer, '.');
64 DCHECK_NOT_NULL(period_pos);
65 *period_pos = '\0';
66 long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT
67 if (kernel_version_major <= 13) return false;
68 #endif // V8_OS_DARWIN
69 // Check whether OS claims to support AVX.
70 uint64_t feature_mask = xgetbv(0); // XCR_XFEATURE_ENABLED_MASK
71 return (feature_mask & 0x6) == 0x6;
72 }
73
74 #endif // V8_HOST_ARCH_IA32 || V8_HOST_ARCH_X64
75
76 } // namespace
77
SupportsWasmSimd128()78 bool CpuFeatures::SupportsWasmSimd128() {
79 #if V8_ENABLE_WEBASSEMBLY
80 if (IsSupported(SSE4_1)) return true;
81 if (FLAG_wasm_simd_ssse3_codegen && IsSupported(SSSE3)) return true;
82 #endif // V8_ENABLE_WEBASSEMBLY
83 return false;
84 }
85
ProbeImpl(bool cross_compile)86 void CpuFeatures::ProbeImpl(bool cross_compile) {
87 // Only use statically determined features for cross compile (snapshot).
88 if (cross_compile) return;
89
90 #if V8_HOST_ARCH_IA32 || V8_HOST_ARCH_X64
91 base::CPU cpu;
92 CHECK(cpu.has_sse2()); // SSE2 support is mandatory.
93 CHECK(cpu.has_cmov()); // CMOV support is mandatory.
94
95 if (cpu.has_sse42()) SetSupported(SSE4_2);
96 if (cpu.has_sse41()) SetSupported(SSE4_1);
97 if (cpu.has_ssse3()) SetSupported(SSSE3);
98 if (cpu.has_sse3()) SetSupported(SSE3);
99 if (cpu.has_avx() && cpu.has_osxsave() && OSHasAVXSupport()) {
100 SetSupported(AVX);
101 if (cpu.has_avx2()) SetSupported(AVX2);
102 if (cpu.has_fma3()) SetSupported(FMA3);
103 }
104
105 // SAHF is not generally available in long mode.
106 if (cpu.has_sahf() && FLAG_enable_sahf) SetSupported(SAHF);
107 if (cpu.has_bmi1() && FLAG_enable_bmi1) SetSupported(BMI1);
108 if (cpu.has_bmi2() && FLAG_enable_bmi2) SetSupported(BMI2);
109 if (cpu.has_lzcnt() && FLAG_enable_lzcnt) SetSupported(LZCNT);
110 if (cpu.has_popcnt() && FLAG_enable_popcnt) SetSupported(POPCNT);
111 if (strcmp(FLAG_mcpu, "auto") == 0) {
112 if (cpu.is_atom()) SetSupported(INTEL_ATOM);
113 } else if (strcmp(FLAG_mcpu, "atom") == 0) {
114 SetSupported(INTEL_ATOM);
115 }
116
117 // Ensure that supported cpu features make sense. E.g. it is wrong to support
118 // AVX but not SSE4_2, if we have --enable-avx and --no-enable-sse4-2, the
119 // code above would set AVX to supported, and SSE4_2 to unsupported, then the
120 // checks below will set AVX to unsupported.
121 if (!FLAG_enable_sse3) SetUnsupported(SSE3);
122 if (!FLAG_enable_ssse3 || !IsSupported(SSE3)) SetUnsupported(SSSE3);
123 if (!FLAG_enable_sse4_1 || !IsSupported(SSSE3)) SetUnsupported(SSE4_1);
124 if (!FLAG_enable_sse4_2 || !IsSupported(SSE4_1)) SetUnsupported(SSE4_2);
125 if (!FLAG_enable_avx || !IsSupported(SSE4_2)) SetUnsupported(AVX);
126 if (!FLAG_enable_avx2 || !IsSupported(AVX)) SetUnsupported(AVX2);
127 if (!FLAG_enable_fma3 || !IsSupported(AVX)) SetUnsupported(FMA3);
128
129 // Set a static value on whether Simd is supported.
130 // This variable is only used for certain archs to query SupportWasmSimd128()
131 // at runtime in builtins using an extern ref. Other callers should use
132 // CpuFeatures::SupportWasmSimd128().
133 CpuFeatures::supports_wasm_simd_128_ = CpuFeatures::SupportsWasmSimd128();
134
135 if (cpu.has_cetss()) SetSupported(CETSS);
136 // The static variable is used for codegen of certain CETSS instructions.
137 CpuFeatures::supports_cetss_ = IsSupported(CETSS);
138 #endif // V8_HOST_ARCH_IA32 || V8_HOST_ARCH_X64
139 }
140
PrintTarget()141 void CpuFeatures::PrintTarget() {}
PrintFeatures()142 void CpuFeatures::PrintFeatures() {
143 printf(
144 "SSE3=%d SSSE3=%d SSE4_1=%d SSE4_2=%d SAHF=%d AVX=%d AVX2=%d FMA3=%d "
145 "BMI1=%d "
146 "BMI2=%d "
147 "LZCNT=%d "
148 "POPCNT=%d ATOM=%d\n",
149 CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSSE3),
150 CpuFeatures::IsSupported(SSE4_1), CpuFeatures::IsSupported(SSE4_2),
151 CpuFeatures::IsSupported(SAHF), CpuFeatures::IsSupported(AVX),
152 CpuFeatures::IsSupported(AVX2), CpuFeatures::IsSupported(FMA3),
153 CpuFeatures::IsSupported(BMI1), CpuFeatures::IsSupported(BMI2),
154 CpuFeatures::IsSupported(LZCNT), CpuFeatures::IsSupported(POPCNT),
155 CpuFeatures::IsSupported(INTEL_ATOM));
156 }
157
158 // -----------------------------------------------------------------------------
159 // Implementation of RelocInfo
160
wasm_call_tag() const161 uint32_t RelocInfo::wasm_call_tag() const {
162 DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL);
163 return ReadUnalignedValue<uint32_t>(pc_);
164 }
165
166 // -----------------------------------------------------------------------------
167 // Implementation of Operand
168
Operand(Operand operand,int32_t offset)169 Operand::Operand(Operand operand, int32_t offset) {
170 DCHECK_GE(operand.data().len, 1);
171 // Operand encodes REX ModR/M [SIB] [Disp].
172 byte modrm = operand.data().buf[0];
173 DCHECK_LT(modrm, 0xC0); // Disallow mode 3 (register target).
174 bool has_sib = ((modrm & 0x07) == 0x04);
175 byte mode = modrm & 0xC0;
176 int disp_offset = has_sib ? 2 : 1;
177 int base_reg = (has_sib ? operand.data().buf[1] : modrm) & 0x07;
178 // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit
179 // displacement.
180 bool is_baseless = (mode == 0) && (base_reg == 0x05); // No base or RIP base.
181 int32_t disp_value = 0;
182 if (mode == 0x80 || is_baseless) {
183 // Mode 2 or mode 0 with rbp/r13 as base: Word displacement.
184 disp_value = ReadUnalignedValue<int32_t>(
185 reinterpret_cast<Address>(&operand.data().buf[disp_offset]));
186 } else if (mode == 0x40) {
187 // Mode 1: Byte displacement.
188 disp_value = static_cast<signed char>(operand.data().buf[disp_offset]);
189 }
190
191 // Write new operand with same registers, but with modified displacement.
192 DCHECK(offset >= 0 ? disp_value + offset > disp_value
193 : disp_value + offset < disp_value); // No overflow.
194 disp_value += offset;
195 data_.rex = operand.data().rex;
196 if (!is_int8(disp_value) || is_baseless) {
197 // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13.
198 data_.buf[0] = (modrm & 0x3F) | (is_baseless ? 0x00 : 0x80);
199 data_.len = disp_offset + 4;
200 WriteUnalignedValue(reinterpret_cast<Address>(&data_.buf[disp_offset]),
201 disp_value);
202 } else if (disp_value != 0 || (base_reg == 0x05)) {
203 // Need 8 bits of displacement.
204 data_.buf[0] = (modrm & 0x3F) | 0x40; // Mode 1.
205 data_.len = disp_offset + 1;
206 data_.buf[disp_offset] = static_cast<byte>(disp_value);
207 } else {
208 // Need no displacement.
209 data_.buf[0] = (modrm & 0x3F); // Mode 0.
210 data_.len = disp_offset;
211 }
212 if (has_sib) {
213 data_.buf[1] = operand.data().buf[1];
214 }
215 }
216
AddressUsesRegister(Register reg) const217 bool Operand::AddressUsesRegister(Register reg) const {
218 int code = reg.code();
219 DCHECK_NE(data_.buf[0] & 0xC0, 0xC0); // Always a memory operand.
220 // Start with only low three bits of base register. Initial decoding
221 // doesn't distinguish on the REX.B bit.
222 int base_code = data_.buf[0] & 0x07;
223 if (base_code == rsp.code()) {
224 // SIB byte present in buf_[1].
225 // Check the index register from the SIB byte + REX.X prefix.
226 int index_code = ((data_.buf[1] >> 3) & 0x07) | ((data_.rex & 0x02) << 2);
227 // Index code (including REX.X) of 0x04 (rsp) means no index register.
228 if (index_code != rsp.code() && index_code == code) return true;
229 // Add REX.B to get the full base register code.
230 base_code = (data_.buf[1] & 0x07) | ((data_.rex & 0x01) << 3);
231 // A base register of 0x05 (rbp) with mod = 0 means no base register.
232 if (base_code == rbp.code() && ((data_.buf[0] & 0xC0) == 0)) return false;
233 return code == base_code;
234 } else {
235 // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means
236 // no base register.
237 if (base_code == rbp.code() && ((data_.buf[0] & 0xC0) == 0)) return false;
238 base_code |= ((data_.rex & 0x01) << 3);
239 return code == base_code;
240 }
241 }
242
AllocateAndInstallRequestedHeapObjects(Isolate * isolate)243 void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
244 DCHECK_IMPLIES(isolate == nullptr, heap_object_requests_.empty());
245 for (auto& request : heap_object_requests_) {
246 Address pc = reinterpret_cast<Address>(buffer_start_) + request.offset();
247 switch (request.kind()) {
248 case HeapObjectRequest::kHeapNumber: {
249 Handle<HeapNumber> object =
250 isolate->factory()->NewHeapNumber<AllocationType::kOld>(
251 request.heap_number());
252 WriteUnalignedValue(pc, object);
253 break;
254 }
255 case HeapObjectRequest::kStringConstant: {
256 const StringConstantBase* str = request.string();
257 CHECK_NOT_NULL(str);
258 Handle<String> allocated = str->AllocateStringConstant(isolate);
259 WriteUnalignedValue(pc, allocated);
260 break;
261 }
262 }
263 }
264 }
265
266 // Partial Constant Pool.
AddSharedEntry(uint64_t data,int offset)267 bool ConstPool::AddSharedEntry(uint64_t data, int offset) {
268 auto existing = entries_.find(data);
269 if (existing == entries_.end()) {
270 entries_.insert(std::make_pair(data, offset + kMoveImm64Offset));
271 return false;
272 }
273
274 // Make sure this is called with strictly ascending offsets.
275 DCHECK_GT(offset + kMoveImm64Offset, existing->second);
276
277 entries_.insert(std::make_pair(data, offset + kMoveRipRelativeDispOffset));
278 return true;
279 }
280
TryRecordEntry(intptr_t data,RelocInfo::Mode mode)281 bool ConstPool::TryRecordEntry(intptr_t data, RelocInfo::Mode mode) {
282 if (!FLAG_partial_constant_pool) return false;
283 DCHECK_WITH_MSG(
284 FLAG_text_is_readable,
285 "The partial constant pool requires a readable .text section");
286 if (!RelocInfo::IsShareableRelocMode(mode)) return false;
287
288 // Currently, partial constant pool only handles the following kinds of
289 // RelocInfo.
290 if (mode != RelocInfo::NO_INFO && mode != RelocInfo::EXTERNAL_REFERENCE &&
291 mode != RelocInfo::OFF_HEAP_TARGET)
292 return false;
293
294 uint64_t raw_data = static_cast<uint64_t>(data);
295 int offset = assm_->pc_offset();
296 return AddSharedEntry(raw_data, offset);
297 }
298
IsMoveRipRelative(Address instr)299 bool ConstPool::IsMoveRipRelative(Address instr) {
300 return (ReadUnalignedValue<uint32_t>(instr) & kMoveRipRelativeMask) ==
301 kMoveRipRelativeInstr;
302 }
303
Clear()304 void ConstPool::Clear() { entries_.clear(); }
305
PatchEntries()306 void ConstPool::PatchEntries() {
307 for (EntryMap::iterator iter = entries_.begin(); iter != entries_.end();
308 iter = entries_.upper_bound(iter->first)) {
309 std::pair<EntryMap::iterator, EntryMap::iterator> range =
310 entries_.equal_range(iter->first);
311 int constant_entry_offset = 0;
312 for (EntryMap::iterator it = range.first; it != range.second; it++) {
313 if (it == range.first) {
314 constant_entry_offset = it->second;
315 continue;
316 }
317
318 DCHECK_GT(constant_entry_offset, 0);
319 DCHECK_LT(constant_entry_offset, it->second);
320 int32_t disp32 =
321 constant_entry_offset - (it->second + kRipRelativeDispSize);
322 Address disp_addr = assm_->addr_at(it->second);
323
324 // Check if the instruction is actually a rip-relative move.
325 DCHECK(IsMoveRipRelative(disp_addr - kMoveRipRelativeDispOffset));
326 // The displacement of the rip-relative move should be 0 before patching.
327 DCHECK(ReadUnalignedValue<uint32_t>(disp_addr) == 0);
328 WriteUnalignedValue(disp_addr, disp32);
329 }
330 }
331 Clear();
332 }
333
PatchConstPool()334 void Assembler::PatchConstPool() {
335 // There is nothing to do if there are no pending entries.
336 if (constpool_.IsEmpty()) {
337 return;
338 }
339 constpool_.PatchEntries();
340 }
341
UseConstPoolFor(RelocInfo::Mode rmode)342 bool Assembler::UseConstPoolFor(RelocInfo::Mode rmode) {
343 if (!FLAG_partial_constant_pool) return false;
344 return (rmode == RelocInfo::NO_INFO ||
345 rmode == RelocInfo::EXTERNAL_REFERENCE ||
346 rmode == RelocInfo::OFF_HEAP_TARGET);
347 }
348
349 // -----------------------------------------------------------------------------
350 // Implementation of Assembler.
351
Assembler(const AssemblerOptions & options,std::unique_ptr<AssemblerBuffer> buffer)352 Assembler::Assembler(const AssemblerOptions& options,
353 std::unique_ptr<AssemblerBuffer> buffer)
354 : AssemblerBase(options, std::move(buffer)), constpool_(this) {
355 reloc_info_writer.Reposition(buffer_start_ + buffer_->size(), pc_);
356 if (CpuFeatures::IsSupported(SSE4_2)) {
357 EnableCpuFeature(SSE4_1);
358 }
359 if (CpuFeatures::IsSupported(SSE4_1)) {
360 EnableCpuFeature(SSSE3);
361 }
362 if (CpuFeatures::IsSupported(SSSE3)) {
363 EnableCpuFeature(SSE3);
364 }
365
366 #if defined(V8_OS_WIN_X64)
367 if (options.collect_win64_unwind_info) {
368 xdata_encoder_ = std::make_unique<win64_unwindinfo::XdataEncoder>(*this);
369 }
370 #endif
371 }
372
GetCode(Isolate * isolate,CodeDesc * desc,SafepointTableBuilder * safepoint_table_builder,int handler_table_offset)373 void Assembler::GetCode(Isolate* isolate, CodeDesc* desc,
374 SafepointTableBuilder* safepoint_table_builder,
375 int handler_table_offset) {
376 // As a crutch to avoid having to add manual Align calls wherever we use a
377 // raw workflow to create Code objects (mostly in tests), add another Align
378 // call here. It does no harm - the end of the Code object is aligned to the
379 // (larger) kCodeAlignment anyways.
380 // TODO(jgruber): Consider moving responsibility for proper alignment to
381 // metadata table builders (safepoint, handler, constant pool, code
382 // comments).
383 DataAlign(Code::kMetadataAlignment);
384
385 PatchConstPool();
386 DCHECK(constpool_.IsEmpty());
387
388 const int code_comments_size = WriteCodeComments();
389
390 // At this point overflow() may be true, but the gap ensures
391 // that we are still not overlapping instructions and relocation info.
392 DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap.
393
394 AllocateAndInstallRequestedHeapObjects(isolate);
395
396 // Set up code descriptor.
397 // TODO(jgruber): Reconsider how these offsets and sizes are maintained up to
398 // this point to make CodeDesc initialization less fiddly.
399
400 static constexpr int kConstantPoolSize = 0;
401 const int instruction_size = pc_offset();
402 const int code_comments_offset = instruction_size - code_comments_size;
403 const int constant_pool_offset = code_comments_offset - kConstantPoolSize;
404 const int handler_table_offset2 = (handler_table_offset == kNoHandlerTable)
405 ? constant_pool_offset
406 : handler_table_offset;
407 const int safepoint_table_offset =
408 (safepoint_table_builder == kNoSafepointTable)
409 ? handler_table_offset2
410 : safepoint_table_builder->safepoint_table_offset();
411 const int reloc_info_offset =
412 static_cast<int>(reloc_info_writer.pos() - buffer_->start());
413 CodeDesc::Initialize(desc, this, safepoint_table_offset,
414 handler_table_offset2, constant_pool_offset,
415 code_comments_offset, reloc_info_offset);
416 }
417
FinalizeJumpOptimizationInfo()418 void Assembler::FinalizeJumpOptimizationInfo() {
419 // Collection stage
420 auto jump_opt = jump_optimization_info();
421 if (jump_opt && jump_opt->is_collecting()) {
422 auto& bitmap = jump_opt->farjmp_bitmap();
423 int num = static_cast<int>(farjmp_positions_.size());
424 if (num && bitmap.empty()) {
425 bool can_opt = false;
426
427 bitmap.resize((num + 31) / 32, 0);
428 for (int i = 0; i < num; i++) {
429 int disp_pos = farjmp_positions_[i];
430 int disp = long_at(disp_pos);
431 if (is_int8(disp)) {
432 bitmap[i / 32] |= 1 << (i & 31);
433 can_opt = true;
434 }
435 }
436 if (can_opt) {
437 jump_opt->set_optimizable();
438 }
439 }
440 }
441 }
442
443 #if defined(V8_OS_WIN_X64)
GetUnwindInfo() const444 win64_unwindinfo::BuiltinUnwindInfo Assembler::GetUnwindInfo() const {
445 DCHECK(options().collect_win64_unwind_info);
446 DCHECK_NOT_NULL(xdata_encoder_);
447 return xdata_encoder_->unwinding_info();
448 }
449 #endif
450
Align(int m)451 void Assembler::Align(int m) {
452 DCHECK(base::bits::IsPowerOfTwo(m));
453 int delta = (m - (pc_offset() & (m - 1))) & (m - 1);
454 Nop(delta);
455 }
456
CodeTargetAlign()457 void Assembler::CodeTargetAlign() {
458 Align(16); // Preferred alignment of jump targets on x64.
459 }
460
LoopHeaderAlign()461 void Assembler::LoopHeaderAlign() {
462 Align(64); // Preferred alignment of loop header on x64.
463 }
464
IsNop(Address addr)465 bool Assembler::IsNop(Address addr) {
466 byte* a = reinterpret_cast<byte*>(addr);
467 while (*a == 0x66) a++;
468 if (*a == 0x90) return true;
469 if (a[0] == 0xF && a[1] == 0x1F) return true;
470 return false;
471 }
472
bind_to(Label * L,int pos)473 void Assembler::bind_to(Label* L, int pos) {
474 DCHECK(!L->is_bound()); // Label may only be bound once.
475 DCHECK(0 <= pos && pos <= pc_offset()); // Position must be valid.
476 if (L->is_linked()) {
477 int current = L->pos();
478 int next = long_at(current);
479 while (next != current) {
480 if (current >= 4 && long_at(current - 4) == 0) {
481 // Absolute address.
482 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_start_ + pos);
483 WriteUnalignedValue(addr_at(current - 4), imm64);
484 internal_reference_positions_.push_back(current - 4);
485 } else {
486 // Relative address, relative to point after address.
487 int imm32 = pos - (current + sizeof(int32_t));
488 long_at_put(current, imm32);
489 }
490 current = next;
491 next = long_at(next);
492 }
493 // Fix up last fixup on linked list.
494 if (current >= 4 && long_at(current - 4) == 0) {
495 // Absolute address.
496 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_start_ + pos);
497 WriteUnalignedValue(addr_at(current - 4), imm64);
498 internal_reference_positions_.push_back(current - 4);
499 } else {
500 // Relative address, relative to point after address.
501 int imm32 = pos - (current + sizeof(int32_t));
502 long_at_put(current, imm32);
503 }
504 }
505 while (L->is_near_linked()) {
506 int fixup_pos = L->near_link_pos();
507 int offset_to_next =
508 static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos)));
509 DCHECK_LE(offset_to_next, 0);
510 int disp = pos - (fixup_pos + sizeof(int8_t));
511 CHECK(is_int8(disp));
512 set_byte_at(fixup_pos, disp);
513 if (offset_to_next < 0) {
514 L->link_to(fixup_pos + offset_to_next, Label::kNear);
515 } else {
516 L->UnuseNear();
517 }
518 }
519
520 // Optimization stage
521 auto jump_opt = jump_optimization_info();
522 if (jump_opt && jump_opt->is_optimizing()) {
523 auto it = label_farjmp_maps_.find(L);
524 if (it != label_farjmp_maps_.end()) {
525 auto& pos_vector = it->second;
526 for (auto fixup_pos : pos_vector) {
527 int disp = pos - (fixup_pos + sizeof(int8_t));
528 CHECK(is_int8(disp));
529 set_byte_at(fixup_pos, disp);
530 }
531 label_farjmp_maps_.erase(it);
532 }
533 }
534 L->bind_to(pos);
535 }
536
bind(Label * L)537 void Assembler::bind(Label* L) { bind_to(L, pc_offset()); }
538
record_farjmp_position(Label * L,int pos)539 void Assembler::record_farjmp_position(Label* L, int pos) {
540 auto& pos_vector = label_farjmp_maps_[L];
541 pos_vector.push_back(pos);
542 }
543
is_optimizable_farjmp(int idx)544 bool Assembler::is_optimizable_farjmp(int idx) {
545 if (predictable_code_size()) return false;
546
547 auto jump_opt = jump_optimization_info();
548 CHECK(jump_opt->is_optimizing());
549
550 auto& bitmap = jump_opt->farjmp_bitmap();
551 CHECK(idx < static_cast<int>(bitmap.size() * 32));
552 return !!(bitmap[idx / 32] & (1 << (idx & 31)));
553 }
554
GrowBuffer()555 void Assembler::GrowBuffer() {
556 DCHECK(buffer_overflow());
557
558 // Compute new buffer size.
559 DCHECK_EQ(buffer_start_, buffer_->start());
560 int old_size = buffer_->size();
561 int new_size = 2 * old_size;
562
563 // Some internal data structures overflow for very large buffers,
564 // they must ensure that kMaximalBufferSize is not too large.
565 if (new_size > kMaximalBufferSize) {
566 V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
567 }
568
569 // Set up new buffer.
570 std::unique_ptr<AssemblerBuffer> new_buffer = buffer_->Grow(new_size);
571 DCHECK_EQ(new_size, new_buffer->size());
572 byte* new_start = new_buffer->start();
573
574 // Copy the data.
575 intptr_t pc_delta = new_start - buffer_start_;
576 intptr_t rc_delta = (new_start + new_size) - (buffer_start_ + old_size);
577 size_t reloc_size = (buffer_start_ + old_size) - reloc_info_writer.pos();
578 MemMove(new_start, buffer_start_, pc_offset());
579 MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(),
580 reloc_size);
581
582 // Switch buffers.
583 buffer_ = std::move(new_buffer);
584 buffer_start_ = new_start;
585 pc_ += pc_delta;
586 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
587 reloc_info_writer.last_pc() + pc_delta);
588
589 // Relocate internal references.
590 for (auto pos : internal_reference_positions_) {
591 Address p = reinterpret_cast<Address>(buffer_start_ + pos);
592 WriteUnalignedValue(p, ReadUnalignedValue<intptr_t>(p) + pc_delta);
593 }
594
595 DCHECK(!buffer_overflow());
596 }
597
emit_operand(int code,Operand adr)598 void Assembler::emit_operand(int code, Operand adr) {
599 DCHECK(is_uint3(code));
600 const unsigned length = adr.data().len;
601 DCHECK_GT(length, 0);
602
603 // Emit updated ModR/M byte containing the given register.
604 DCHECK_EQ(adr.data().buf[0] & 0x38, 0);
605 *pc_++ = adr.data().buf[0] | code << 3;
606
607 // Recognize RIP relative addressing.
608 if (adr.data().buf[0] == 5) {
609 DCHECK_EQ(9u, length);
610 Label* label = ReadUnalignedValue<Label*>(
611 reinterpret_cast<Address>(&adr.data().buf[1]));
612 if (label->is_bound()) {
613 int offset =
614 label->pos() - pc_offset() - sizeof(int32_t) + adr.data().addend;
615 DCHECK_GE(0, offset);
616 emitl(offset);
617 } else if (label->is_linked()) {
618 emitl(label->pos());
619 label->link_to(pc_offset() - sizeof(int32_t));
620 } else {
621 DCHECK(label->is_unused());
622 int32_t current = pc_offset();
623 emitl(current);
624 label->link_to(current);
625 }
626 } else {
627 // Emit the rest of the encoded operand.
628 for (unsigned i = 1; i < length; i++) *pc_++ = adr.data().buf[i];
629 }
630 }
631
632 // Assembler Instruction implementations.
633
arithmetic_op(byte opcode,Register reg,Operand op,int size)634 void Assembler::arithmetic_op(byte opcode, Register reg, Operand op, int size) {
635 EnsureSpace ensure_space(this);
636 emit_rex(reg, op, size);
637 emit(opcode);
638 emit_operand(reg, op);
639 }
640
arithmetic_op(byte opcode,Register reg,Register rm_reg,int size)641 void Assembler::arithmetic_op(byte opcode, Register reg, Register rm_reg,
642 int size) {
643 EnsureSpace ensure_space(this);
644 DCHECK_EQ(opcode & 0xC6, 2);
645 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
646 // Swap reg and rm_reg and change opcode operand order.
647 emit_rex(rm_reg, reg, size);
648 emit(opcode ^ 0x02);
649 emit_modrm(rm_reg, reg);
650 } else {
651 emit_rex(reg, rm_reg, size);
652 emit(opcode);
653 emit_modrm(reg, rm_reg);
654 }
655 }
656
arithmetic_op_16(byte opcode,Register reg,Register rm_reg)657 void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) {
658 EnsureSpace ensure_space(this);
659 DCHECK_EQ(opcode & 0xC6, 2);
660 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
661 // Swap reg and rm_reg and change opcode operand order.
662 emit(0x66);
663 emit_optional_rex_32(rm_reg, reg);
664 emit(opcode ^ 0x02);
665 emit_modrm(rm_reg, reg);
666 } else {
667 emit(0x66);
668 emit_optional_rex_32(reg, rm_reg);
669 emit(opcode);
670 emit_modrm(reg, rm_reg);
671 }
672 }
673
arithmetic_op_16(byte opcode,Register reg,Operand rm_reg)674 void Assembler::arithmetic_op_16(byte opcode, Register reg, Operand rm_reg) {
675 EnsureSpace ensure_space(this);
676 emit(0x66);
677 emit_optional_rex_32(reg, rm_reg);
678 emit(opcode);
679 emit_operand(reg, rm_reg);
680 }
681
arithmetic_op_8(byte opcode,Register reg,Operand op)682 void Assembler::arithmetic_op_8(byte opcode, Register reg, Operand op) {
683 EnsureSpace ensure_space(this);
684 if (!reg.is_byte_register()) {
685 emit_rex_32(reg, op);
686 } else {
687 emit_optional_rex_32(reg, op);
688 }
689 emit(opcode);
690 emit_operand(reg, op);
691 }
692
arithmetic_op_8(byte opcode,Register reg,Register rm_reg)693 void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) {
694 EnsureSpace ensure_space(this);
695 DCHECK_EQ(opcode & 0xC6, 2);
696 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
697 // Swap reg and rm_reg and change opcode operand order.
698 if (!rm_reg.is_byte_register() || !reg.is_byte_register()) {
699 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
700 emit_rex_32(rm_reg, reg);
701 }
702 emit(opcode ^ 0x02);
703 emit_modrm(rm_reg, reg);
704 } else {
705 if (!reg.is_byte_register() || !rm_reg.is_byte_register()) {
706 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
707 emit_rex_32(reg, rm_reg);
708 }
709 emit(opcode);
710 emit_modrm(reg, rm_reg);
711 }
712 }
713
immediate_arithmetic_op(byte subcode,Register dst,Immediate src,int size)714 void Assembler::immediate_arithmetic_op(byte subcode, Register dst,
715 Immediate src, int size) {
716 EnsureSpace ensure_space(this);
717 emit_rex(dst, size);
718 if (is_int8(src.value_) && RelocInfo::IsNoInfo(src.rmode_)) {
719 emit(0x83);
720 emit_modrm(subcode, dst);
721 emit(src.value_);
722 } else if (dst == rax) {
723 emit(0x05 | (subcode << 3));
724 emit(src);
725 } else {
726 emit(0x81);
727 emit_modrm(subcode, dst);
728 emit(src);
729 }
730 }
731
immediate_arithmetic_op(byte subcode,Operand dst,Immediate src,int size)732 void Assembler::immediate_arithmetic_op(byte subcode, Operand dst,
733 Immediate src, int size) {
734 EnsureSpace ensure_space(this);
735 emit_rex(dst, size);
736 if (is_int8(src.value_) && RelocInfo::IsNoInfo(src.rmode_)) {
737 emit(0x83);
738 emit_operand(subcode, dst);
739 emit(src.value_);
740 } else {
741 emit(0x81);
742 emit_operand(subcode, dst);
743 emit(src);
744 }
745 }
746
immediate_arithmetic_op_16(byte subcode,Register dst,Immediate src)747 void Assembler::immediate_arithmetic_op_16(byte subcode, Register dst,
748 Immediate src) {
749 EnsureSpace ensure_space(this);
750 emit(0x66); // Operand size override prefix.
751 emit_optional_rex_32(dst);
752 if (is_int8(src.value_)) {
753 emit(0x83);
754 emit_modrm(subcode, dst);
755 emit(src.value_);
756 } else if (dst == rax) {
757 emit(0x05 | (subcode << 3));
758 emitw(src.value_);
759 } else {
760 emit(0x81);
761 emit_modrm(subcode, dst);
762 emitw(src.value_);
763 }
764 }
765
immediate_arithmetic_op_16(byte subcode,Operand dst,Immediate src)766 void Assembler::immediate_arithmetic_op_16(byte subcode, Operand dst,
767 Immediate src) {
768 EnsureSpace ensure_space(this);
769 emit(0x66); // Operand size override prefix.
770 emit_optional_rex_32(dst);
771 if (is_int8(src.value_)) {
772 emit(0x83);
773 emit_operand(subcode, dst);
774 emit(src.value_);
775 } else {
776 emit(0x81);
777 emit_operand(subcode, dst);
778 emitw(src.value_);
779 }
780 }
781
immediate_arithmetic_op_8(byte subcode,Operand dst,Immediate src)782 void Assembler::immediate_arithmetic_op_8(byte subcode, Operand dst,
783 Immediate src) {
784 EnsureSpace ensure_space(this);
785 emit_optional_rex_32(dst);
786 DCHECK(is_int8(src.value_) || is_uint8(src.value_));
787 emit(0x80);
788 emit_operand(subcode, dst);
789 emit(src.value_);
790 }
791
immediate_arithmetic_op_8(byte subcode,Register dst,Immediate src)792 void Assembler::immediate_arithmetic_op_8(byte subcode, Register dst,
793 Immediate src) {
794 EnsureSpace ensure_space(this);
795 if (!dst.is_byte_register()) {
796 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
797 emit_rex_32(dst);
798 }
799 DCHECK(is_int8(src.value_) || is_uint8(src.value_));
800 emit(0x80);
801 emit_modrm(subcode, dst);
802 emit(src.value_);
803 }
804
shift(Register dst,Immediate shift_amount,int subcode,int size)805 void Assembler::shift(Register dst, Immediate shift_amount, int subcode,
806 int size) {
807 EnsureSpace ensure_space(this);
808 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
809 : is_uint5(shift_amount.value_));
810 if (shift_amount.value_ == 1) {
811 emit_rex(dst, size);
812 emit(0xD1);
813 emit_modrm(subcode, dst);
814 } else {
815 emit_rex(dst, size);
816 emit(0xC1);
817 emit_modrm(subcode, dst);
818 emit(shift_amount.value_);
819 }
820 }
821
shift(Operand dst,Immediate shift_amount,int subcode,int size)822 void Assembler::shift(Operand dst, Immediate shift_amount, int subcode,
823 int size) {
824 EnsureSpace ensure_space(this);
825 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
826 : is_uint5(shift_amount.value_));
827 if (shift_amount.value_ == 1) {
828 emit_rex(dst, size);
829 emit(0xD1);
830 emit_operand(subcode, dst);
831 } else {
832 emit_rex(dst, size);
833 emit(0xC1);
834 emit_operand(subcode, dst);
835 emit(shift_amount.value_);
836 }
837 }
838
shift(Register dst,int subcode,int size)839 void Assembler::shift(Register dst, int subcode, int size) {
840 EnsureSpace ensure_space(this);
841 emit_rex(dst, size);
842 emit(0xD3);
843 emit_modrm(subcode, dst);
844 }
845
shift(Operand dst,int subcode,int size)846 void Assembler::shift(Operand dst, int subcode, int size) {
847 EnsureSpace ensure_space(this);
848 emit_rex(dst, size);
849 emit(0xD3);
850 emit_operand(subcode, dst);
851 }
852
bswapl(Register dst)853 void Assembler::bswapl(Register dst) {
854 EnsureSpace ensure_space(this);
855 emit_rex_32(dst);
856 emit(0x0F);
857 emit(0xC8 + dst.low_bits());
858 }
859
bswapq(Register dst)860 void Assembler::bswapq(Register dst) {
861 EnsureSpace ensure_space(this);
862 emit_rex_64(dst);
863 emit(0x0F);
864 emit(0xC8 + dst.low_bits());
865 }
866
btq(Operand dst,Register src)867 void Assembler::btq(Operand dst, Register src) {
868 EnsureSpace ensure_space(this);
869 emit_rex_64(src, dst);
870 emit(0x0F);
871 emit(0xA3);
872 emit_operand(src, dst);
873 }
874
btsq(Operand dst,Register src)875 void Assembler::btsq(Operand dst, Register src) {
876 EnsureSpace ensure_space(this);
877 emit_rex_64(src, dst);
878 emit(0x0F);
879 emit(0xAB);
880 emit_operand(src, dst);
881 }
882
btsq(Register dst,Immediate imm8)883 void Assembler::btsq(Register dst, Immediate imm8) {
884 EnsureSpace ensure_space(this);
885 emit_rex_64(dst);
886 emit(0x0F);
887 emit(0xBA);
888 emit_modrm(0x5, dst);
889 emit(imm8.value_);
890 }
891
btrq(Register dst,Immediate imm8)892 void Assembler::btrq(Register dst, Immediate imm8) {
893 EnsureSpace ensure_space(this);
894 emit_rex_64(dst);
895 emit(0x0F);
896 emit(0xBA);
897 emit_modrm(0x6, dst);
898 emit(imm8.value_);
899 }
900
bsrl(Register dst,Register src)901 void Assembler::bsrl(Register dst, Register src) {
902 EnsureSpace ensure_space(this);
903 emit_optional_rex_32(dst, src);
904 emit(0x0F);
905 emit(0xBD);
906 emit_modrm(dst, src);
907 }
908
bsrl(Register dst,Operand src)909 void Assembler::bsrl(Register dst, Operand src) {
910 EnsureSpace ensure_space(this);
911 emit_optional_rex_32(dst, src);
912 emit(0x0F);
913 emit(0xBD);
914 emit_operand(dst, src);
915 }
916
bsrq(Register dst,Register src)917 void Assembler::bsrq(Register dst, Register src) {
918 EnsureSpace ensure_space(this);
919 emit_rex_64(dst, src);
920 emit(0x0F);
921 emit(0xBD);
922 emit_modrm(dst, src);
923 }
924
bsrq(Register dst,Operand src)925 void Assembler::bsrq(Register dst, Operand src) {
926 EnsureSpace ensure_space(this);
927 emit_rex_64(dst, src);
928 emit(0x0F);
929 emit(0xBD);
930 emit_operand(dst, src);
931 }
932
bsfl(Register dst,Register src)933 void Assembler::bsfl(Register dst, Register src) {
934 EnsureSpace ensure_space(this);
935 emit_optional_rex_32(dst, src);
936 emit(0x0F);
937 emit(0xBC);
938 emit_modrm(dst, src);
939 }
940
bsfl(Register dst,Operand src)941 void Assembler::bsfl(Register dst, Operand src) {
942 EnsureSpace ensure_space(this);
943 emit_optional_rex_32(dst, src);
944 emit(0x0F);
945 emit(0xBC);
946 emit_operand(dst, src);
947 }
948
bsfq(Register dst,Register src)949 void Assembler::bsfq(Register dst, Register src) {
950 EnsureSpace ensure_space(this);
951 emit_rex_64(dst, src);
952 emit(0x0F);
953 emit(0xBC);
954 emit_modrm(dst, src);
955 }
956
bsfq(Register dst,Operand src)957 void Assembler::bsfq(Register dst, Operand src) {
958 EnsureSpace ensure_space(this);
959 emit_rex_64(dst, src);
960 emit(0x0F);
961 emit(0xBC);
962 emit_operand(dst, src);
963 }
964
pblendw(XMMRegister dst,Operand src,uint8_t mask)965 void Assembler::pblendw(XMMRegister dst, Operand src, uint8_t mask) {
966 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0E);
967 emit(mask);
968 }
969
pblendw(XMMRegister dst,XMMRegister src,uint8_t mask)970 void Assembler::pblendw(XMMRegister dst, XMMRegister src, uint8_t mask) {
971 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0E);
972 emit(mask);
973 }
974
palignr(XMMRegister dst,Operand src,uint8_t mask)975 void Assembler::palignr(XMMRegister dst, Operand src, uint8_t mask) {
976 ssse3_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0F);
977 emit(mask);
978 }
979
palignr(XMMRegister dst,XMMRegister src,uint8_t mask)980 void Assembler::palignr(XMMRegister dst, XMMRegister src, uint8_t mask) {
981 ssse3_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0F);
982 emit(mask);
983 }
984
call(Label * L)985 void Assembler::call(Label* L) {
986 EnsureSpace ensure_space(this);
987 // 1110 1000 #32-bit disp.
988 emit(0xE8);
989 if (L->is_bound()) {
990 int offset = L->pos() - pc_offset() - sizeof(int32_t);
991 DCHECK_LE(offset, 0);
992 emitl(offset);
993 } else if (L->is_linked()) {
994 emitl(L->pos());
995 L->link_to(pc_offset() - sizeof(int32_t));
996 } else {
997 DCHECK(L->is_unused());
998 int32_t current = pc_offset();
999 emitl(current);
1000 L->link_to(current);
1001 }
1002 }
1003
call(Address entry,RelocInfo::Mode rmode)1004 void Assembler::call(Address entry, RelocInfo::Mode rmode) {
1005 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1006 EnsureSpace ensure_space(this);
1007 // 1110 1000 #32-bit disp.
1008 emit(0xE8);
1009 emit_runtime_entry(entry, rmode);
1010 }
1011
call(Handle<CodeT> target,RelocInfo::Mode rmode)1012 void Assembler::call(Handle<CodeT> target, RelocInfo::Mode rmode) {
1013 DCHECK(RelocInfo::IsCodeTarget(rmode));
1014 DCHECK(FromCodeT(*target).IsExecutable());
1015 EnsureSpace ensure_space(this);
1016 // 1110 1000 #32-bit disp.
1017 emit(0xE8);
1018 RecordRelocInfo(rmode);
1019 int code_target_index = AddCodeTarget(target);
1020 emitl(code_target_index);
1021 }
1022
near_call(intptr_t disp,RelocInfo::Mode rmode)1023 void Assembler::near_call(intptr_t disp, RelocInfo::Mode rmode) {
1024 EnsureSpace ensure_space(this);
1025 emit(0xE8);
1026 DCHECK(is_int32(disp));
1027 RecordRelocInfo(rmode);
1028 emitl(static_cast<int32_t>(disp));
1029 }
1030
near_jmp(intptr_t disp,RelocInfo::Mode rmode)1031 void Assembler::near_jmp(intptr_t disp, RelocInfo::Mode rmode) {
1032 EnsureSpace ensure_space(this);
1033 emit(0xE9);
1034 DCHECK(is_int32(disp));
1035 if (!RelocInfo::IsNoInfo(rmode)) RecordRelocInfo(rmode);
1036 emitl(static_cast<int32_t>(disp));
1037 }
1038
call(Register adr)1039 void Assembler::call(Register adr) {
1040 EnsureSpace ensure_space(this);
1041 // Opcode: FF /2 r64.
1042 emit_optional_rex_32(adr);
1043 emit(0xFF);
1044 emit_modrm(0x2, adr);
1045 }
1046
call(Operand op)1047 void Assembler::call(Operand op) {
1048 EnsureSpace ensure_space(this);
1049 // Opcode: FF /2 m64.
1050 emit_optional_rex_32(op);
1051 emit(0xFF);
1052 emit_operand(0x2, op);
1053 }
1054
clc()1055 void Assembler::clc() {
1056 EnsureSpace ensure_space(this);
1057 emit(0xF8);
1058 }
1059
cld()1060 void Assembler::cld() {
1061 EnsureSpace ensure_space(this);
1062 emit(0xFC);
1063 }
1064
cdq()1065 void Assembler::cdq() {
1066 EnsureSpace ensure_space(this);
1067 emit(0x99);
1068 }
1069
cmovq(Condition cc,Register dst,Register src)1070 void Assembler::cmovq(Condition cc, Register dst, Register src) {
1071 if (cc == always) {
1072 movq(dst, src);
1073 } else if (cc == never) {
1074 return;
1075 }
1076 // No need to check CpuInfo for CMOV support, it's a required part of the
1077 // 64-bit architecture.
1078 DCHECK_GE(cc, 0); // Use mov for unconditional moves.
1079 EnsureSpace ensure_space(this);
1080 // Opcode: REX.W 0f 40 + cc /r.
1081 emit_rex_64(dst, src);
1082 emit(0x0F);
1083 emit(0x40 + cc);
1084 emit_modrm(dst, src);
1085 }
1086
cmovq(Condition cc,Register dst,Operand src)1087 void Assembler::cmovq(Condition cc, Register dst, Operand src) {
1088 if (cc == always) {
1089 movq(dst, src);
1090 } else if (cc == never) {
1091 return;
1092 }
1093 DCHECK_GE(cc, 0);
1094 EnsureSpace ensure_space(this);
1095 // Opcode: REX.W 0f 40 + cc /r.
1096 emit_rex_64(dst, src);
1097 emit(0x0F);
1098 emit(0x40 + cc);
1099 emit_operand(dst, src);
1100 }
1101
cmovl(Condition cc,Register dst,Register src)1102 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1103 if (cc == always) {
1104 movl(dst, src);
1105 } else if (cc == never) {
1106 return;
1107 }
1108 DCHECK_GE(cc, 0);
1109 EnsureSpace ensure_space(this);
1110 // Opcode: 0f 40 + cc /r.
1111 emit_optional_rex_32(dst, src);
1112 emit(0x0F);
1113 emit(0x40 + cc);
1114 emit_modrm(dst, src);
1115 }
1116
cmovl(Condition cc,Register dst,Operand src)1117 void Assembler::cmovl(Condition cc, Register dst, Operand src) {
1118 if (cc == always) {
1119 movl(dst, src);
1120 } else if (cc == never) {
1121 return;
1122 }
1123 DCHECK_GE(cc, 0);
1124 EnsureSpace ensure_space(this);
1125 // Opcode: 0f 40 + cc /r.
1126 emit_optional_rex_32(dst, src);
1127 emit(0x0F);
1128 emit(0x40 + cc);
1129 emit_operand(dst, src);
1130 }
1131
cmpb_al(Immediate imm8)1132 void Assembler::cmpb_al(Immediate imm8) {
1133 DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_));
1134 EnsureSpace ensure_space(this);
1135 emit(0x3C);
1136 emit(imm8.value_);
1137 }
1138
lock()1139 void Assembler::lock() {
1140 EnsureSpace ensure_space(this);
1141 emit(0xF0);
1142 }
1143
xaddb(Operand dst,Register src)1144 void Assembler::xaddb(Operand dst, Register src) {
1145 EnsureSpace ensure_space(this);
1146 emit_optional_rex_8(src, dst);
1147 emit(0x0F);
1148 emit(0xC0);
1149 emit_operand(src, dst);
1150 }
1151
xaddw(Operand dst,Register src)1152 void Assembler::xaddw(Operand dst, Register src) {
1153 EnsureSpace ensure_space(this);
1154 emit(0x66);
1155 emit_optional_rex_32(src, dst);
1156 emit(0x0F);
1157 emit(0xC1);
1158 emit_operand(src, dst);
1159 }
1160
xaddl(Operand dst,Register src)1161 void Assembler::xaddl(Operand dst, Register src) {
1162 EnsureSpace ensure_space(this);
1163 emit_optional_rex_32(src, dst);
1164 emit(0x0F);
1165 emit(0xC1);
1166 emit_operand(src, dst);
1167 }
1168
xaddq(Operand dst,Register src)1169 void Assembler::xaddq(Operand dst, Register src) {
1170 EnsureSpace ensure_space(this);
1171 emit_rex(src, dst, kInt64Size);
1172 emit(0x0F);
1173 emit(0xC1);
1174 emit_operand(src, dst);
1175 }
1176
cmpxchgb(Operand dst,Register src)1177 void Assembler::cmpxchgb(Operand dst, Register src) {
1178 EnsureSpace ensure_space(this);
1179 if (!src.is_byte_register()) {
1180 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1181 emit_rex_32(src, dst);
1182 } else {
1183 emit_optional_rex_32(src, dst);
1184 }
1185 emit(0x0F);
1186 emit(0xB0);
1187 emit_operand(src, dst);
1188 }
1189
cmpxchgw(Operand dst,Register src)1190 void Assembler::cmpxchgw(Operand dst, Register src) {
1191 EnsureSpace ensure_space(this);
1192 emit(0x66);
1193 emit_optional_rex_32(src, dst);
1194 emit(0x0F);
1195 emit(0xB1);
1196 emit_operand(src, dst);
1197 }
1198
emit_cmpxchg(Operand dst,Register src,int size)1199 void Assembler::emit_cmpxchg(Operand dst, Register src, int size) {
1200 EnsureSpace ensure_space(this);
1201 emit_rex(src, dst, size);
1202 emit(0x0F);
1203 emit(0xB1);
1204 emit_operand(src, dst);
1205 }
1206
mfence()1207 void Assembler::mfence() {
1208 EnsureSpace ensure_space(this);
1209 emit(0x0F);
1210 emit(0xAE);
1211 emit(0xF0);
1212 }
1213
lfence()1214 void Assembler::lfence() {
1215 EnsureSpace ensure_space(this);
1216 emit(0x0F);
1217 emit(0xAE);
1218 emit(0xE8);
1219 }
1220
cpuid()1221 void Assembler::cpuid() {
1222 EnsureSpace ensure_space(this);
1223 emit(0x0F);
1224 emit(0xA2);
1225 }
1226
cqo()1227 void Assembler::cqo() {
1228 EnsureSpace ensure_space(this);
1229 emit_rex_64();
1230 emit(0x99);
1231 }
1232
emit_dec(Register dst,int size)1233 void Assembler::emit_dec(Register dst, int size) {
1234 EnsureSpace ensure_space(this);
1235 emit_rex(dst, size);
1236 emit(0xFF);
1237 emit_modrm(0x1, dst);
1238 }
1239
emit_dec(Operand dst,int size)1240 void Assembler::emit_dec(Operand dst, int size) {
1241 EnsureSpace ensure_space(this);
1242 emit_rex(dst, size);
1243 emit(0xFF);
1244 emit_operand(1, dst);
1245 }
1246
decb(Register dst)1247 void Assembler::decb(Register dst) {
1248 EnsureSpace ensure_space(this);
1249 if (!dst.is_byte_register()) {
1250 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1251 emit_rex_32(dst);
1252 }
1253 emit(0xFE);
1254 emit_modrm(0x1, dst);
1255 }
1256
decb(Operand dst)1257 void Assembler::decb(Operand dst) {
1258 EnsureSpace ensure_space(this);
1259 emit_optional_rex_32(dst);
1260 emit(0xFE);
1261 emit_operand(1, dst);
1262 }
1263
hlt()1264 void Assembler::hlt() {
1265 EnsureSpace ensure_space(this);
1266 emit(0xF4);
1267 }
1268
emit_idiv(Register src,int size)1269 void Assembler::emit_idiv(Register src, int size) {
1270 EnsureSpace ensure_space(this);
1271 emit_rex(src, size);
1272 emit(0xF7);
1273 emit_modrm(0x7, src);
1274 }
1275
emit_div(Register src,int size)1276 void Assembler::emit_div(Register src, int size) {
1277 EnsureSpace ensure_space(this);
1278 emit_rex(src, size);
1279 emit(0xF7);
1280 emit_modrm(0x6, src);
1281 }
1282
emit_imul(Register src,int size)1283 void Assembler::emit_imul(Register src, int size) {
1284 EnsureSpace ensure_space(this);
1285 emit_rex(src, size);
1286 emit(0xF7);
1287 emit_modrm(0x5, src);
1288 }
1289
emit_imul(Operand src,int size)1290 void Assembler::emit_imul(Operand src, int size) {
1291 EnsureSpace ensure_space(this);
1292 emit_rex(src, size);
1293 emit(0xF7);
1294 emit_operand(0x5, src);
1295 }
1296
emit_imul(Register dst,Register src,int size)1297 void Assembler::emit_imul(Register dst, Register src, int size) {
1298 EnsureSpace ensure_space(this);
1299 emit_rex(dst, src, size);
1300 emit(0x0F);
1301 emit(0xAF);
1302 emit_modrm(dst, src);
1303 }
1304
emit_imul(Register dst,Operand src,int size)1305 void Assembler::emit_imul(Register dst, Operand src, int size) {
1306 EnsureSpace ensure_space(this);
1307 emit_rex(dst, src, size);
1308 emit(0x0F);
1309 emit(0xAF);
1310 emit_operand(dst, src);
1311 }
1312
emit_imul(Register dst,Register src,Immediate imm,int size)1313 void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) {
1314 EnsureSpace ensure_space(this);
1315 emit_rex(dst, src, size);
1316 if (is_int8(imm.value_)) {
1317 emit(0x6B);
1318 emit_modrm(dst, src);
1319 emit(imm.value_);
1320 } else {
1321 emit(0x69);
1322 emit_modrm(dst, src);
1323 emitl(imm.value_);
1324 }
1325 }
1326
emit_imul(Register dst,Operand src,Immediate imm,int size)1327 void Assembler::emit_imul(Register dst, Operand src, Immediate imm, int size) {
1328 EnsureSpace ensure_space(this);
1329 emit_rex(dst, src, size);
1330 if (is_int8(imm.value_)) {
1331 emit(0x6B);
1332 emit_operand(dst, src);
1333 emit(imm.value_);
1334 } else {
1335 emit(0x69);
1336 emit_operand(dst, src);
1337 emitl(imm.value_);
1338 }
1339 }
1340
emit_inc(Register dst,int size)1341 void Assembler::emit_inc(Register dst, int size) {
1342 EnsureSpace ensure_space(this);
1343 emit_rex(dst, size);
1344 emit(0xFF);
1345 emit_modrm(0x0, dst);
1346 }
1347
emit_inc(Operand dst,int size)1348 void Assembler::emit_inc(Operand dst, int size) {
1349 EnsureSpace ensure_space(this);
1350 emit_rex(dst, size);
1351 emit(0xFF);
1352 emit_operand(0, dst);
1353 }
1354
int3()1355 void Assembler::int3() {
1356 EnsureSpace ensure_space(this);
1357 emit(0xCC);
1358 }
1359
j(Condition cc,Label * L,Label::Distance distance)1360 void Assembler::j(Condition cc, Label* L, Label::Distance distance) {
1361 if (cc == always) {
1362 jmp(L, distance);
1363 return;
1364 } else if (cc == never) {
1365 return;
1366 }
1367 EnsureSpace ensure_space(this);
1368 DCHECK(is_uint4(cc));
1369 if (L->is_bound()) {
1370 const int short_size = 2;
1371 const int long_size = 6;
1372 int offs = L->pos() - pc_offset();
1373 DCHECK_LE(offs, 0);
1374 // Determine whether we can use 1-byte offsets for backwards branches,
1375 // which have a max range of 128 bytes.
1376
1377 // We also need to check predictable_code_size() flag here, because on x64,
1378 // when the full code generator recompiles code for debugging, some places
1379 // need to be padded out to a certain size. The debugger is keeping track of
1380 // how often it did this so that it can adjust return addresses on the
1381 // stack, but if the size of jump instructions can also change, that's not
1382 // enough and the calculated offsets would be incorrect.
1383 if (is_int8(offs - short_size) && !predictable_code_size()) {
1384 // 0111 tttn #8-bit disp.
1385 emit(0x70 | cc);
1386 emit((offs - short_size) & 0xFF);
1387 } else {
1388 // 0000 1111 1000 tttn #32-bit disp.
1389 emit(0x0F);
1390 emit(0x80 | cc);
1391 emitl(offs - long_size);
1392 }
1393 } else if (distance == Label::kNear) {
1394 // 0111 tttn #8-bit disp
1395 emit(0x70 | cc);
1396 byte disp = 0x00;
1397 if (L->is_near_linked()) {
1398 int offset = L->near_link_pos() - pc_offset();
1399 DCHECK(is_int8(offset));
1400 disp = static_cast<byte>(offset & 0xFF);
1401 }
1402 L->link_to(pc_offset(), Label::kNear);
1403 emit(disp);
1404 } else {
1405 auto jump_opt = jump_optimization_info();
1406 if (V8_UNLIKELY(jump_opt)) {
1407 if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
1408 // 0111 tttn #8-bit disp
1409 emit(0x70 | cc);
1410 record_farjmp_position(L, pc_offset());
1411 emit(0);
1412 return;
1413 }
1414 if (jump_opt->is_collecting()) {
1415 farjmp_positions_.push_back(pc_offset() + 2);
1416 }
1417 }
1418 if (L->is_linked()) {
1419 // 0000 1111 1000 tttn #32-bit disp.
1420 emit(0x0F);
1421 emit(0x80 | cc);
1422 emitl(L->pos());
1423 L->link_to(pc_offset() - sizeof(int32_t));
1424 } else {
1425 DCHECK(L->is_unused());
1426 emit(0x0F);
1427 emit(0x80 | cc);
1428 int32_t current = pc_offset();
1429 emitl(current);
1430 L->link_to(current);
1431 }
1432 }
1433 }
1434
j(Condition cc,Address entry,RelocInfo::Mode rmode)1435 void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) {
1436 DCHECK(RelocInfo::IsWasmStubCall(rmode));
1437 EnsureSpace ensure_space(this);
1438 DCHECK(is_uint4(cc));
1439 emit(0x0F);
1440 emit(0x80 | cc);
1441 RecordRelocInfo(rmode);
1442 emitl(static_cast<int32_t>(entry));
1443 }
1444
j(Condition cc,Handle<CodeT> target,RelocInfo::Mode rmode)1445 void Assembler::j(Condition cc, Handle<CodeT> target, RelocInfo::Mode rmode) {
1446 if (cc == always) {
1447 jmp(target, rmode);
1448 return;
1449 } else if (cc == never) {
1450 return;
1451 }
1452 EnsureSpace ensure_space(this);
1453 DCHECK(is_uint4(cc));
1454 // 0000 1111 1000 tttn #32-bit disp.
1455 emit(0x0F);
1456 emit(0x80 | cc);
1457 DCHECK(RelocInfo::IsCodeTarget(rmode));
1458 RecordRelocInfo(rmode);
1459 int code_target_index = AddCodeTarget(target);
1460 emitl(code_target_index);
1461 }
1462
jmp(Address entry,RelocInfo::Mode rmode)1463 void Assembler::jmp(Address entry, RelocInfo::Mode rmode) {
1464 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1465 EnsureSpace ensure_space(this);
1466 // 1110 1001 #32-bit disp.
1467 emit(0xE9);
1468 emit_runtime_entry(entry, rmode);
1469 }
1470
jmp_rel(int32_t offset)1471 void Assembler::jmp_rel(int32_t offset) {
1472 EnsureSpace ensure_space(this);
1473 // The offset is encoded relative to the next instruction.
1474 constexpr int32_t kShortJmpDisplacement = 1 + sizeof(int8_t);
1475 constexpr int32_t kNearJmpDisplacement = 1 + sizeof(int32_t);
1476 DCHECK_LE(std::numeric_limits<int32_t>::min() + kNearJmpDisplacement, offset);
1477 if (is_int8(offset - kShortJmpDisplacement) && !predictable_code_size()) {
1478 // 0xEB #8-bit disp.
1479 emit(0xEB);
1480 emit(offset - kShortJmpDisplacement);
1481 } else {
1482 // 0xE9 #32-bit disp.
1483 emit(0xE9);
1484 emitl(offset - kNearJmpDisplacement);
1485 }
1486 }
1487
jmp(Label * L,Label::Distance distance)1488 void Assembler::jmp(Label* L, Label::Distance distance) {
1489 const int long_size = sizeof(int32_t);
1490
1491 if (L->is_bound()) {
1492 int offset = L->pos() - pc_offset();
1493 DCHECK_LE(offset, 0); // backward jump.
1494 jmp_rel(offset);
1495 return;
1496 }
1497
1498 EnsureSpace ensure_space(this);
1499 if (distance == Label::kNear) {
1500 emit(0xEB);
1501 byte disp = 0x00;
1502 if (L->is_near_linked()) {
1503 int offset = L->near_link_pos() - pc_offset();
1504 DCHECK(is_int8(offset));
1505 disp = static_cast<byte>(offset & 0xFF);
1506 }
1507 L->link_to(pc_offset(), Label::kNear);
1508 emit(disp);
1509 } else {
1510 auto jump_opt = jump_optimization_info();
1511 if (V8_UNLIKELY(jump_opt)) {
1512 if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
1513 emit(0xEB);
1514 record_farjmp_position(L, pc_offset());
1515 emit(0);
1516 return;
1517 }
1518 if (jump_opt->is_collecting()) {
1519 farjmp_positions_.push_back(pc_offset() + 1);
1520 }
1521 }
1522 if (L->is_linked()) {
1523 // 1110 1001 #32-bit disp.
1524 emit(0xE9);
1525 emitl(L->pos());
1526 L->link_to(pc_offset() - long_size);
1527 } else {
1528 // 1110 1001 #32-bit disp.
1529 DCHECK(L->is_unused());
1530 emit(0xE9);
1531 int32_t current = pc_offset();
1532 emitl(current);
1533 L->link_to(current);
1534 }
1535 }
1536 }
1537
jmp(Handle<CodeT> target,RelocInfo::Mode rmode)1538 void Assembler::jmp(Handle<CodeT> target, RelocInfo::Mode rmode) {
1539 DCHECK(RelocInfo::IsCodeTarget(rmode));
1540 EnsureSpace ensure_space(this);
1541 // 1110 1001 #32-bit disp.
1542 emit(0xE9);
1543 RecordRelocInfo(rmode);
1544 int code_target_index = AddCodeTarget(target);
1545 emitl(code_target_index);
1546 }
1547
jmp(Register target)1548 void Assembler::jmp(Register target) {
1549 EnsureSpace ensure_space(this);
1550 // Opcode FF/4 r64.
1551 emit_optional_rex_32(target);
1552 emit(0xFF);
1553 emit_modrm(0x4, target);
1554 }
1555
jmp(Operand src)1556 void Assembler::jmp(Operand src) {
1557 EnsureSpace ensure_space(this);
1558 // Opcode FF/4 m64.
1559 emit_optional_rex_32(src);
1560 emit(0xFF);
1561 emit_operand(0x4, src);
1562 }
1563
emit_lea(Register dst,Operand src,int size)1564 void Assembler::emit_lea(Register dst, Operand src, int size) {
1565 EnsureSpace ensure_space(this);
1566 emit_rex(dst, src, size);
1567 emit(0x8D);
1568 emit_operand(dst, src);
1569 }
1570
load_rax(Address value,RelocInfo::Mode mode)1571 void Assembler::load_rax(Address value, RelocInfo::Mode mode) {
1572 EnsureSpace ensure_space(this);
1573 emit(0x48); // REX.W
1574 emit(0xA1);
1575 emit(Immediate64(value, mode));
1576 }
1577
load_rax(ExternalReference ref)1578 void Assembler::load_rax(ExternalReference ref) {
1579 load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
1580 }
1581
leave()1582 void Assembler::leave() {
1583 EnsureSpace ensure_space(this);
1584 emit(0xC9);
1585 }
1586
movb(Register dst,Operand src)1587 void Assembler::movb(Register dst, Operand src) {
1588 EnsureSpace ensure_space(this);
1589 if (!dst.is_byte_register()) {
1590 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1591 emit_rex_32(dst, src);
1592 } else {
1593 emit_optional_rex_32(dst, src);
1594 }
1595 emit(0x8A);
1596 emit_operand(dst, src);
1597 }
1598
movb(Register dst,Immediate imm)1599 void Assembler::movb(Register dst, Immediate imm) {
1600 EnsureSpace ensure_space(this);
1601 if (!dst.is_byte_register()) {
1602 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1603 emit_rex_32(dst);
1604 }
1605 emit(0xB0 + dst.low_bits());
1606 emit(imm.value_);
1607 }
1608
movb(Operand dst,Register src)1609 void Assembler::movb(Operand dst, Register src) {
1610 EnsureSpace ensure_space(this);
1611 if (!src.is_byte_register()) {
1612 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1613 emit_rex_32(src, dst);
1614 } else {
1615 emit_optional_rex_32(src, dst);
1616 }
1617 emit(0x88);
1618 emit_operand(src, dst);
1619 }
1620
movb(Operand dst,Immediate imm)1621 void Assembler::movb(Operand dst, Immediate imm) {
1622 EnsureSpace ensure_space(this);
1623 emit_optional_rex_32(dst);
1624 emit(0xC6);
1625 emit_operand(0x0, dst);
1626 emit(static_cast<byte>(imm.value_));
1627 }
1628
movw(Register dst,Operand src)1629 void Assembler::movw(Register dst, Operand src) {
1630 EnsureSpace ensure_space(this);
1631 emit(0x66);
1632 emit_optional_rex_32(dst, src);
1633 emit(0x8B);
1634 emit_operand(dst, src);
1635 }
1636
movw(Operand dst,Register src)1637 void Assembler::movw(Operand dst, Register src) {
1638 EnsureSpace ensure_space(this);
1639 emit(0x66);
1640 emit_optional_rex_32(src, dst);
1641 emit(0x89);
1642 emit_operand(src, dst);
1643 }
1644
movw(Operand dst,Immediate imm)1645 void Assembler::movw(Operand dst, Immediate imm) {
1646 EnsureSpace ensure_space(this);
1647 emit(0x66);
1648 emit_optional_rex_32(dst);
1649 emit(0xC7);
1650 emit_operand(0x0, dst);
1651 emit(static_cast<byte>(imm.value_ & 0xFF));
1652 emit(static_cast<byte>(imm.value_ >> 8));
1653 }
1654
emit_mov(Register dst,Operand src,int size)1655 void Assembler::emit_mov(Register dst, Operand src, int size) {
1656 EnsureSpace ensure_space(this);
1657 emit_rex(dst, src, size);
1658 emit(0x8B);
1659 emit_operand(dst, src);
1660 }
1661
emit_mov(Register dst,Register src,int size)1662 void Assembler::emit_mov(Register dst, Register src, int size) {
1663 EnsureSpace ensure_space(this);
1664 if (src.low_bits() == 4) {
1665 emit_rex(src, dst, size);
1666 emit(0x89);
1667 emit_modrm(src, dst);
1668 } else {
1669 emit_rex(dst, src, size);
1670 emit(0x8B);
1671 emit_modrm(dst, src);
1672 }
1673
1674 #if defined(V8_OS_WIN_X64)
1675 if (xdata_encoder_ && dst == rbp && src == rsp) {
1676 xdata_encoder_->onMovRbpRsp();
1677 }
1678 #endif
1679 }
1680
emit_mov(Operand dst,Register src,int size)1681 void Assembler::emit_mov(Operand dst, Register src, int size) {
1682 EnsureSpace ensure_space(this);
1683 emit_rex(src, dst, size);
1684 emit(0x89);
1685 emit_operand(src, dst);
1686 }
1687
emit_mov(Register dst,Immediate value,int size)1688 void Assembler::emit_mov(Register dst, Immediate value, int size) {
1689 EnsureSpace ensure_space(this);
1690 emit_rex(dst, size);
1691 if (size == kInt64Size) {
1692 emit(0xC7);
1693 emit_modrm(0x0, dst);
1694 } else {
1695 DCHECK_EQ(size, kInt32Size);
1696 emit(0xB8 + dst.low_bits());
1697 }
1698 emit(value);
1699 }
1700
emit_mov(Operand dst,Immediate value,int size)1701 void Assembler::emit_mov(Operand dst, Immediate value, int size) {
1702 EnsureSpace ensure_space(this);
1703 emit_rex(dst, size);
1704 emit(0xC7);
1705 emit_operand(0x0, dst);
1706 emit(value);
1707 }
1708
emit_mov(Register dst,Immediate64 value,int size)1709 void Assembler::emit_mov(Register dst, Immediate64 value, int size) {
1710 DCHECK_EQ(size, kInt64Size);
1711 if (constpool_.TryRecordEntry(value.value_, value.rmode_)) {
1712 // Emit rip-relative move with offset = 0
1713 Label label;
1714 emit_mov(dst, Operand(&label, 0), size);
1715 bind(&label);
1716 } else {
1717 EnsureSpace ensure_space(this);
1718 emit_rex(dst, size);
1719 emit(0xB8 | dst.low_bits());
1720 emit(value);
1721 }
1722 }
1723
movq_imm64(Register dst,int64_t value)1724 void Assembler::movq_imm64(Register dst, int64_t value) {
1725 EnsureSpace ensure_space(this);
1726 emit_rex(dst, kInt64Size);
1727 emit(0xB8 | dst.low_bits());
1728 emitq(static_cast<uint64_t>(value));
1729 }
1730
movq_heap_number(Register dst,double value)1731 void Assembler::movq_heap_number(Register dst, double value) {
1732 EnsureSpace ensure_space(this);
1733 emit_rex(dst, kInt64Size);
1734 emit(0xB8 | dst.low_bits());
1735 RequestHeapObject(HeapObjectRequest(value));
1736 emit(Immediate64(kNullAddress, RelocInfo::FULL_EMBEDDED_OBJECT));
1737 }
1738
movq_string(Register dst,const StringConstantBase * str)1739 void Assembler::movq_string(Register dst, const StringConstantBase* str) {
1740 EnsureSpace ensure_space(this);
1741 emit_rex(dst, kInt64Size);
1742 emit(0xB8 | dst.low_bits());
1743 RequestHeapObject(HeapObjectRequest(str));
1744 emit(Immediate64(kNullAddress, RelocInfo::FULL_EMBEDDED_OBJECT));
1745 }
1746
1747 // Loads the ip-relative location of the src label into the target location
1748 // (as a 32-bit offset sign extended to 64-bit).
movl(Operand dst,Label * src)1749 void Assembler::movl(Operand dst, Label* src) {
1750 EnsureSpace ensure_space(this);
1751 emit_optional_rex_32(dst);
1752 emit(0xC7);
1753 emit_operand(0, dst);
1754 if (src->is_bound()) {
1755 int offset = src->pos() - pc_offset() - sizeof(int32_t);
1756 DCHECK_LE(offset, 0);
1757 emitl(offset);
1758 } else if (src->is_linked()) {
1759 emitl(src->pos());
1760 src->link_to(pc_offset() - sizeof(int32_t));
1761 } else {
1762 DCHECK(src->is_unused());
1763 int32_t current = pc_offset();
1764 emitl(current);
1765 src->link_to(current);
1766 }
1767 }
1768
movsxbl(Register dst,Register src)1769 void Assembler::movsxbl(Register dst, Register src) {
1770 EnsureSpace ensure_space(this);
1771 if (!src.is_byte_register()) {
1772 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1773 emit_rex_32(dst, src);
1774 } else {
1775 emit_optional_rex_32(dst, src);
1776 }
1777 emit(0x0F);
1778 emit(0xBE);
1779 emit_modrm(dst, src);
1780 }
1781
movsxbl(Register dst,Operand src)1782 void Assembler::movsxbl(Register dst, Operand src) {
1783 EnsureSpace ensure_space(this);
1784 emit_optional_rex_32(dst, src);
1785 emit(0x0F);
1786 emit(0xBE);
1787 emit_operand(dst, src);
1788 }
1789
movsxbq(Register dst,Operand src)1790 void Assembler::movsxbq(Register dst, Operand src) {
1791 EnsureSpace ensure_space(this);
1792 emit_rex_64(dst, src);
1793 emit(0x0F);
1794 emit(0xBE);
1795 emit_operand(dst, src);
1796 }
1797
movsxbq(Register dst,Register src)1798 void Assembler::movsxbq(Register dst, Register src) {
1799 EnsureSpace ensure_space(this);
1800 emit_rex_64(dst, src);
1801 emit(0x0F);
1802 emit(0xBE);
1803 emit_modrm(dst, src);
1804 }
1805
movsxwl(Register dst,Register src)1806 void Assembler::movsxwl(Register dst, Register src) {
1807 EnsureSpace ensure_space(this);
1808 emit_optional_rex_32(dst, src);
1809 emit(0x0F);
1810 emit(0xBF);
1811 emit_modrm(dst, src);
1812 }
1813
movsxwl(Register dst,Operand src)1814 void Assembler::movsxwl(Register dst, Operand src) {
1815 EnsureSpace ensure_space(this);
1816 emit_optional_rex_32(dst, src);
1817 emit(0x0F);
1818 emit(0xBF);
1819 emit_operand(dst, src);
1820 }
1821
movsxwq(Register dst,Operand src)1822 void Assembler::movsxwq(Register dst, Operand src) {
1823 EnsureSpace ensure_space(this);
1824 emit_rex_64(dst, src);
1825 emit(0x0F);
1826 emit(0xBF);
1827 emit_operand(dst, src);
1828 }
1829
movsxwq(Register dst,Register src)1830 void Assembler::movsxwq(Register dst, Register src) {
1831 EnsureSpace ensure_space(this);
1832 emit_rex_64(dst, src);
1833 emit(0x0F);
1834 emit(0xBF);
1835 emit_modrm(dst, src);
1836 }
1837
movsxlq(Register dst,Register src)1838 void Assembler::movsxlq(Register dst, Register src) {
1839 EnsureSpace ensure_space(this);
1840 emit_rex_64(dst, src);
1841 emit(0x63);
1842 emit_modrm(dst, src);
1843 }
1844
movsxlq(Register dst,Operand src)1845 void Assembler::movsxlq(Register dst, Operand src) {
1846 EnsureSpace ensure_space(this);
1847 emit_rex_64(dst, src);
1848 emit(0x63);
1849 emit_operand(dst, src);
1850 }
1851
emit_movzxb(Register dst,Operand src,int size)1852 void Assembler::emit_movzxb(Register dst, Operand src, int size) {
1853 EnsureSpace ensure_space(this);
1854 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1855 // there is no need to make this a 64 bit operation.
1856 emit_optional_rex_32(dst, src);
1857 emit(0x0F);
1858 emit(0xB6);
1859 emit_operand(dst, src);
1860 }
1861
emit_movzxb(Register dst,Register src,int size)1862 void Assembler::emit_movzxb(Register dst, Register src, int size) {
1863 EnsureSpace ensure_space(this);
1864 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1865 // there is no need to make this a 64 bit operation.
1866 if (!src.is_byte_register()) {
1867 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1868 emit_rex_32(dst, src);
1869 } else {
1870 emit_optional_rex_32(dst, src);
1871 }
1872 emit(0x0F);
1873 emit(0xB6);
1874 emit_modrm(dst, src);
1875 }
1876
emit_movzxw(Register dst,Operand src,int size)1877 void Assembler::emit_movzxw(Register dst, Operand src, int size) {
1878 EnsureSpace ensure_space(this);
1879 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1880 // there is no need to make this a 64 bit operation.
1881 emit_optional_rex_32(dst, src);
1882 emit(0x0F);
1883 emit(0xB7);
1884 emit_operand(dst, src);
1885 }
1886
emit_movzxw(Register dst,Register src,int size)1887 void Assembler::emit_movzxw(Register dst, Register src, int size) {
1888 EnsureSpace ensure_space(this);
1889 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1890 // there is no need to make this a 64 bit operation.
1891 emit_optional_rex_32(dst, src);
1892 emit(0x0F);
1893 emit(0xB7);
1894 emit_modrm(dst, src);
1895 }
1896
repmovsb()1897 void Assembler::repmovsb() {
1898 EnsureSpace ensure_space(this);
1899 emit(0xF3);
1900 emit(0xA4);
1901 }
1902
repmovsw()1903 void Assembler::repmovsw() {
1904 EnsureSpace ensure_space(this);
1905 emit(0x66); // Operand size override.
1906 emit(0xF3);
1907 emit(0xA4);
1908 }
1909
emit_repmovs(int size)1910 void Assembler::emit_repmovs(int size) {
1911 EnsureSpace ensure_space(this);
1912 emit(0xF3);
1913 emit_rex(size);
1914 emit(0xA5);
1915 }
1916
repstosl()1917 void Assembler::repstosl() {
1918 EnsureSpace ensure_space(this);
1919 emit(0xF3);
1920 emit(0xAB);
1921 }
1922
repstosq()1923 void Assembler::repstosq() {
1924 EnsureSpace ensure_space(this);
1925 emit(0xF3);
1926 emit_rex_64();
1927 emit(0xAB);
1928 }
1929
mull(Register src)1930 void Assembler::mull(Register src) {
1931 EnsureSpace ensure_space(this);
1932 emit_optional_rex_32(src);
1933 emit(0xF7);
1934 emit_modrm(0x4, src);
1935 }
1936
mull(Operand src)1937 void Assembler::mull(Operand src) {
1938 EnsureSpace ensure_space(this);
1939 emit_optional_rex_32(src);
1940 emit(0xF7);
1941 emit_operand(0x4, src);
1942 }
1943
mulq(Register src)1944 void Assembler::mulq(Register src) {
1945 EnsureSpace ensure_space(this);
1946 emit_rex_64(src);
1947 emit(0xF7);
1948 emit_modrm(0x4, src);
1949 }
1950
negb(Register reg)1951 void Assembler::negb(Register reg) {
1952 EnsureSpace ensure_space(this);
1953 emit_optional_rex_8(reg);
1954 emit(0xF6);
1955 emit_modrm(0x3, reg);
1956 }
1957
negw(Register reg)1958 void Assembler::negw(Register reg) {
1959 EnsureSpace ensure_space(this);
1960 emit(0x66);
1961 emit_optional_rex_32(reg);
1962 emit(0xF7);
1963 emit_modrm(0x3, reg);
1964 }
1965
negl(Register reg)1966 void Assembler::negl(Register reg) {
1967 EnsureSpace ensure_space(this);
1968 emit_optional_rex_32(reg);
1969 emit(0xF7);
1970 emit_modrm(0x3, reg);
1971 }
1972
negq(Register reg)1973 void Assembler::negq(Register reg) {
1974 EnsureSpace ensure_space(this);
1975 emit_rex_64(reg);
1976 emit(0xF7);
1977 emit_modrm(0x3, reg);
1978 }
1979
negb(Operand op)1980 void Assembler::negb(Operand op) {
1981 EnsureSpace ensure_space(this);
1982 emit_optional_rex_32(op);
1983 emit(0xF6);
1984 emit_operand(0x3, op);
1985 }
1986
negw(Operand op)1987 void Assembler::negw(Operand op) {
1988 EnsureSpace ensure_space(this);
1989 emit(0x66);
1990 emit_optional_rex_32(op);
1991 emit(0xF7);
1992 emit_operand(0x3, op);
1993 }
1994
negl(Operand op)1995 void Assembler::negl(Operand op) {
1996 EnsureSpace ensure_space(this);
1997 emit_optional_rex_32(op);
1998 emit(0xF7);
1999 emit_operand(0x3, op);
2000 }
2001
negq(Operand op)2002 void Assembler::negq(Operand op) {
2003 EnsureSpace ensure_space(this);
2004 emit_rex_64(op);
2005 emit(0xF7);
2006 emit_operand(0x3, op);
2007 }
2008
nop()2009 void Assembler::nop() {
2010 EnsureSpace ensure_space(this);
2011 emit(0x90);
2012 }
2013
emit_not(Register dst,int size)2014 void Assembler::emit_not(Register dst, int size) {
2015 EnsureSpace ensure_space(this);
2016 emit_rex(dst, size);
2017 emit(0xF7);
2018 emit_modrm(0x2, dst);
2019 }
2020
emit_not(Operand dst,int size)2021 void Assembler::emit_not(Operand dst, int size) {
2022 EnsureSpace ensure_space(this);
2023 emit_rex(dst, size);
2024 emit(0xF7);
2025 emit_operand(2, dst);
2026 }
2027
Nop(int n)2028 void Assembler::Nop(int n) {
2029 DCHECK_LE(0, n);
2030 // The recommended muti-byte sequences of NOP instructions from the Intel 64
2031 // and IA-32 Architectures Software Developer's Manual.
2032 //
2033 // Len Assembly Byte Sequence
2034 // 2 66 NOP 66 90H
2035 // 3 NOP DWORD ptr [EAX] 0F 1F 00H
2036 // 4 NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H
2037 // 5 NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H
2038 // 6 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H
2039 // 7 NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H
2040 // 8 NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H
2041 // 9 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] 66 0F 1F 84 00 00 00 00 00H
2042
2043 constexpr const char* kNopSequences =
2044 "\x66\x90" // length 1 (@1) / 2 (@0)
2045 "\x0F\x1F\x00" // length 3 (@2)
2046 "\x0F\x1F\x40\x00" // length 4 (@5)
2047 "\x66\x0F\x1F\x44\x00\x00" // length 5 (@10) / 6 (@9)
2048 "\x0F\x1F\x80\x00\x00\x00\x00" // length 7 (@15)
2049 "\x66\x0F\x1F\x84\x00\x00\x00\x00\x00"; // length 8 (@23) / 9 (@22)
2050 constexpr int8_t kNopOffsets[10] = {0, 1, 0, 2, 5, 10, 9, 15, 23, 22};
2051
2052 do {
2053 EnsureSpace ensure_space(this);
2054 int nop_bytes = std::min(n, 9);
2055 const char* sequence = kNopSequences + kNopOffsets[nop_bytes];
2056 memcpy(pc_, sequence, nop_bytes);
2057 pc_ += nop_bytes;
2058 n -= nop_bytes;
2059 } while (n);
2060 }
2061
popq(Register dst)2062 void Assembler::popq(Register dst) {
2063 EnsureSpace ensure_space(this);
2064 emit_optional_rex_32(dst);
2065 emit(0x58 | dst.low_bits());
2066 }
2067
popq(Operand dst)2068 void Assembler::popq(Operand dst) {
2069 EnsureSpace ensure_space(this);
2070 emit_optional_rex_32(dst);
2071 emit(0x8F);
2072 emit_operand(0, dst);
2073 }
2074
popfq()2075 void Assembler::popfq() {
2076 EnsureSpace ensure_space(this);
2077 emit(0x9D);
2078 }
2079
pushq(Register src)2080 void Assembler::pushq(Register src) {
2081 EnsureSpace ensure_space(this);
2082 emit_optional_rex_32(src);
2083 emit(0x50 | src.low_bits());
2084
2085 #if defined(V8_OS_WIN_X64)
2086 if (xdata_encoder_ && src == rbp) {
2087 xdata_encoder_->onPushRbp();
2088 }
2089 #endif
2090 }
2091
pushq(Operand src)2092 void Assembler::pushq(Operand src) {
2093 EnsureSpace ensure_space(this);
2094 emit_optional_rex_32(src);
2095 emit(0xFF);
2096 emit_operand(6, src);
2097 }
2098
pushq(Immediate value)2099 void Assembler::pushq(Immediate value) {
2100 EnsureSpace ensure_space(this);
2101 if (is_int8(value.value_)) {
2102 emit(0x6A);
2103 emit(value.value_); // Emit low byte of value.
2104 } else {
2105 emit(0x68);
2106 emitl(value.value_);
2107 }
2108 }
2109
pushq_imm32(int32_t imm32)2110 void Assembler::pushq_imm32(int32_t imm32) {
2111 EnsureSpace ensure_space(this);
2112 emit(0x68);
2113 emitl(imm32);
2114 }
2115
pushfq()2116 void Assembler::pushfq() {
2117 EnsureSpace ensure_space(this);
2118 emit(0x9C);
2119 }
2120
incsspq(Register number_of_words)2121 void Assembler::incsspq(Register number_of_words) {
2122 EnsureSpace ensure_space(this);
2123 emit(0xF3);
2124 emit_rex_64(number_of_words);
2125 emit(0x0F);
2126 emit(0xAE);
2127 emit(0xE8 | number_of_words.low_bits());
2128 }
2129
ret(int imm16)2130 void Assembler::ret(int imm16) {
2131 EnsureSpace ensure_space(this);
2132 DCHECK(is_uint16(imm16));
2133 if (imm16 == 0) {
2134 emit(0xC3);
2135 } else {
2136 emit(0xC2);
2137 emit(imm16 & 0xFF);
2138 emit((imm16 >> 8) & 0xFF);
2139 }
2140 }
2141
ud2()2142 void Assembler::ud2() {
2143 EnsureSpace ensure_space(this);
2144 emit(0x0F);
2145 emit(0x0B);
2146 }
2147
setcc(Condition cc,Register reg)2148 void Assembler::setcc(Condition cc, Register reg) {
2149 if (cc > last_condition) {
2150 movb(reg, Immediate(cc == always ? 1 : 0));
2151 return;
2152 }
2153 EnsureSpace ensure_space(this);
2154 DCHECK(is_uint4(cc));
2155 if (!reg.is_byte_register()) {
2156 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2157 emit_rex_32(reg);
2158 }
2159 emit(0x0F);
2160 emit(0x90 | cc);
2161 emit_modrm(0x0, reg);
2162 }
2163
shld(Register dst,Register src)2164 void Assembler::shld(Register dst, Register src) {
2165 EnsureSpace ensure_space(this);
2166 emit_rex_64(src, dst);
2167 emit(0x0F);
2168 emit(0xA5);
2169 emit_modrm(src, dst);
2170 }
2171
shrd(Register dst,Register src)2172 void Assembler::shrd(Register dst, Register src) {
2173 EnsureSpace ensure_space(this);
2174 emit_rex_64(src, dst);
2175 emit(0x0F);
2176 emit(0xAD);
2177 emit_modrm(src, dst);
2178 }
2179
xchgb(Register reg,Operand op)2180 void Assembler::xchgb(Register reg, Operand op) {
2181 EnsureSpace ensure_space(this);
2182 if (!reg.is_byte_register()) {
2183 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2184 emit_rex_32(reg, op);
2185 } else {
2186 emit_optional_rex_32(reg, op);
2187 }
2188 emit(0x86);
2189 emit_operand(reg, op);
2190 }
2191
xchgw(Register reg,Operand op)2192 void Assembler::xchgw(Register reg, Operand op) {
2193 EnsureSpace ensure_space(this);
2194 emit(0x66);
2195 emit_optional_rex_32(reg, op);
2196 emit(0x87);
2197 emit_operand(reg, op);
2198 }
2199
emit_xchg(Register dst,Register src,int size)2200 void Assembler::emit_xchg(Register dst, Register src, int size) {
2201 EnsureSpace ensure_space(this);
2202 if (src == rax || dst == rax) { // Single-byte encoding
2203 Register other = src == rax ? dst : src;
2204 emit_rex(other, size);
2205 emit(0x90 | other.low_bits());
2206 } else if (dst.low_bits() == 4) {
2207 emit_rex(dst, src, size);
2208 emit(0x87);
2209 emit_modrm(dst, src);
2210 } else {
2211 emit_rex(src, dst, size);
2212 emit(0x87);
2213 emit_modrm(src, dst);
2214 }
2215 }
2216
emit_xchg(Register dst,Operand src,int size)2217 void Assembler::emit_xchg(Register dst, Operand src, int size) {
2218 EnsureSpace ensure_space(this);
2219 emit_rex(dst, src, size);
2220 emit(0x87);
2221 emit_operand(dst, src);
2222 }
2223
store_rax(Address dst,RelocInfo::Mode mode)2224 void Assembler::store_rax(Address dst, RelocInfo::Mode mode) {
2225 EnsureSpace ensure_space(this);
2226 emit(0x48); // REX.W
2227 emit(0xA3);
2228 emit(Immediate64(dst, mode));
2229 }
2230
store_rax(ExternalReference ref)2231 void Assembler::store_rax(ExternalReference ref) {
2232 store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
2233 }
2234
sub_sp_32(uint32_t imm)2235 void Assembler::sub_sp_32(uint32_t imm) {
2236 emit_rex_64();
2237 emit(0x81); // using a literal 32-bit immediate.
2238 emit_modrm(0x5, rsp);
2239 emitl(imm);
2240 }
2241
testb(Register dst,Register src)2242 void Assembler::testb(Register dst, Register src) {
2243 EnsureSpace ensure_space(this);
2244 emit_test(dst, src, sizeof(int8_t));
2245 }
2246
testb(Register reg,Immediate mask)2247 void Assembler::testb(Register reg, Immediate mask) {
2248 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
2249 emit_test(reg, mask, sizeof(int8_t));
2250 }
2251
testb(Operand op,Immediate mask)2252 void Assembler::testb(Operand op, Immediate mask) {
2253 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
2254 emit_test(op, mask, sizeof(int8_t));
2255 }
2256
testb(Operand op,Register reg)2257 void Assembler::testb(Operand op, Register reg) {
2258 emit_test(op, reg, sizeof(int8_t));
2259 }
2260
testw(Register dst,Register src)2261 void Assembler::testw(Register dst, Register src) {
2262 emit_test(dst, src, sizeof(uint16_t));
2263 }
2264
testw(Register reg,Immediate mask)2265 void Assembler::testw(Register reg, Immediate mask) {
2266 emit_test(reg, mask, sizeof(int16_t));
2267 }
2268
testw(Operand op,Immediate mask)2269 void Assembler::testw(Operand op, Immediate mask) {
2270 emit_test(op, mask, sizeof(int16_t));
2271 }
2272
testw(Operand op,Register reg)2273 void Assembler::testw(Operand op, Register reg) {
2274 emit_test(op, reg, sizeof(int16_t));
2275 }
2276
emit_test(Register dst,Register src,int size)2277 void Assembler::emit_test(Register dst, Register src, int size) {
2278 EnsureSpace ensure_space(this);
2279 if (src.low_bits() == 4) std::swap(dst, src);
2280 if (size == sizeof(int16_t)) {
2281 emit(0x66);
2282 size = sizeof(int32_t);
2283 }
2284 bool byte_operand = size == sizeof(int8_t);
2285 if (byte_operand) {
2286 size = sizeof(int32_t);
2287 if (!src.is_byte_register() || !dst.is_byte_register()) {
2288 emit_rex_32(dst, src);
2289 }
2290 } else {
2291 emit_rex(dst, src, size);
2292 }
2293 emit(byte_operand ? 0x84 : 0x85);
2294 emit_modrm(dst, src);
2295 }
2296
emit_test(Register reg,Immediate mask,int size)2297 void Assembler::emit_test(Register reg, Immediate mask, int size) {
2298 if (is_uint8(mask.value_)) {
2299 size = sizeof(int8_t);
2300 } else if (is_uint16(mask.value_)) {
2301 size = sizeof(int16_t);
2302 }
2303 EnsureSpace ensure_space(this);
2304 bool half_word = size == sizeof(int16_t);
2305 if (half_word) {
2306 emit(0x66);
2307 size = sizeof(int32_t);
2308 }
2309 bool byte_operand = size == sizeof(int8_t);
2310 if (byte_operand) {
2311 size = sizeof(int32_t);
2312 if (!reg.is_byte_register()) emit_rex_32(reg);
2313 } else {
2314 emit_rex(reg, size);
2315 }
2316 if (reg == rax) {
2317 emit(byte_operand ? 0xA8 : 0xA9);
2318 } else {
2319 emit(byte_operand ? 0xF6 : 0xF7);
2320 emit_modrm(0x0, reg);
2321 }
2322 if (byte_operand) {
2323 emit(mask.value_);
2324 } else if (half_word) {
2325 emitw(mask.value_);
2326 } else {
2327 emit(mask);
2328 }
2329 }
2330
emit_test(Operand op,Immediate mask,int size)2331 void Assembler::emit_test(Operand op, Immediate mask, int size) {
2332 if (is_uint8(mask.value_)) {
2333 size = sizeof(int8_t);
2334 } else if (is_uint16(mask.value_)) {
2335 size = sizeof(int16_t);
2336 }
2337 EnsureSpace ensure_space(this);
2338 bool half_word = size == sizeof(int16_t);
2339 if (half_word) {
2340 emit(0x66);
2341 size = sizeof(int32_t);
2342 }
2343 bool byte_operand = size == sizeof(int8_t);
2344 if (byte_operand) {
2345 size = sizeof(int32_t);
2346 }
2347 emit_rex(rax, op, size);
2348 emit(byte_operand ? 0xF6 : 0xF7);
2349 emit_operand(rax, op); // Operation code 0
2350 if (byte_operand) {
2351 emit(mask.value_);
2352 } else if (half_word) {
2353 emitw(mask.value_);
2354 } else {
2355 emit(mask);
2356 }
2357 }
2358
emit_test(Operand op,Register reg,int size)2359 void Assembler::emit_test(Operand op, Register reg, int size) {
2360 EnsureSpace ensure_space(this);
2361 if (size == sizeof(int16_t)) {
2362 emit(0x66);
2363 size = sizeof(int32_t);
2364 }
2365 bool byte_operand = size == sizeof(int8_t);
2366 if (byte_operand) {
2367 size = sizeof(int32_t);
2368 if (!reg.is_byte_register()) {
2369 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2370 emit_rex_32(reg, op);
2371 } else {
2372 emit_optional_rex_32(reg, op);
2373 }
2374 } else {
2375 emit_rex(reg, op, size);
2376 }
2377 emit(byte_operand ? 0x84 : 0x85);
2378 emit_operand(reg, op);
2379 }
2380
2381 // FPU instructions.
2382
fld(int i)2383 void Assembler::fld(int i) {
2384 EnsureSpace ensure_space(this);
2385 emit_farith(0xD9, 0xC0, i);
2386 }
2387
fld1()2388 void Assembler::fld1() {
2389 EnsureSpace ensure_space(this);
2390 emit(0xD9);
2391 emit(0xE8);
2392 }
2393
fldz()2394 void Assembler::fldz() {
2395 EnsureSpace ensure_space(this);
2396 emit(0xD9);
2397 emit(0xEE);
2398 }
2399
fldpi()2400 void Assembler::fldpi() {
2401 EnsureSpace ensure_space(this);
2402 emit(0xD9);
2403 emit(0xEB);
2404 }
2405
fldln2()2406 void Assembler::fldln2() {
2407 EnsureSpace ensure_space(this);
2408 emit(0xD9);
2409 emit(0xED);
2410 }
2411
fld_s(Operand adr)2412 void Assembler::fld_s(Operand adr) {
2413 EnsureSpace ensure_space(this);
2414 emit_optional_rex_32(adr);
2415 emit(0xD9);
2416 emit_operand(0, adr);
2417 }
2418
fld_d(Operand adr)2419 void Assembler::fld_d(Operand adr) {
2420 EnsureSpace ensure_space(this);
2421 emit_optional_rex_32(adr);
2422 emit(0xDD);
2423 emit_operand(0, adr);
2424 }
2425
fstp_s(Operand adr)2426 void Assembler::fstp_s(Operand adr) {
2427 EnsureSpace ensure_space(this);
2428 emit_optional_rex_32(adr);
2429 emit(0xD9);
2430 emit_operand(3, adr);
2431 }
2432
fstp_d(Operand adr)2433 void Assembler::fstp_d(Operand adr) {
2434 EnsureSpace ensure_space(this);
2435 emit_optional_rex_32(adr);
2436 emit(0xDD);
2437 emit_operand(3, adr);
2438 }
2439
fstp(int index)2440 void Assembler::fstp(int index) {
2441 DCHECK(is_uint3(index));
2442 EnsureSpace ensure_space(this);
2443 emit_farith(0xDD, 0xD8, index);
2444 }
2445
fild_s(Operand adr)2446 void Assembler::fild_s(Operand adr) {
2447 EnsureSpace ensure_space(this);
2448 emit_optional_rex_32(adr);
2449 emit(0xDB);
2450 emit_operand(0, adr);
2451 }
2452
fild_d(Operand adr)2453 void Assembler::fild_d(Operand adr) {
2454 EnsureSpace ensure_space(this);
2455 emit_optional_rex_32(adr);
2456 emit(0xDF);
2457 emit_operand(5, adr);
2458 }
2459
fistp_s(Operand adr)2460 void Assembler::fistp_s(Operand adr) {
2461 EnsureSpace ensure_space(this);
2462 emit_optional_rex_32(adr);
2463 emit(0xDB);
2464 emit_operand(3, adr);
2465 }
2466
fisttp_s(Operand adr)2467 void Assembler::fisttp_s(Operand adr) {
2468 DCHECK(IsEnabled(SSE3));
2469 EnsureSpace ensure_space(this);
2470 emit_optional_rex_32(adr);
2471 emit(0xDB);
2472 emit_operand(1, adr);
2473 }
2474
fisttp_d(Operand adr)2475 void Assembler::fisttp_d(Operand adr) {
2476 DCHECK(IsEnabled(SSE3));
2477 EnsureSpace ensure_space(this);
2478 emit_optional_rex_32(adr);
2479 emit(0xDD);
2480 emit_operand(1, adr);
2481 }
2482
fist_s(Operand adr)2483 void Assembler::fist_s(Operand adr) {
2484 EnsureSpace ensure_space(this);
2485 emit_optional_rex_32(adr);
2486 emit(0xDB);
2487 emit_operand(2, adr);
2488 }
2489
fistp_d(Operand adr)2490 void Assembler::fistp_d(Operand adr) {
2491 EnsureSpace ensure_space(this);
2492 emit_optional_rex_32(adr);
2493 emit(0xDF);
2494 emit_operand(7, adr);
2495 }
2496
fabs()2497 void Assembler::fabs() {
2498 EnsureSpace ensure_space(this);
2499 emit(0xD9);
2500 emit(0xE1);
2501 }
2502
fchs()2503 void Assembler::fchs() {
2504 EnsureSpace ensure_space(this);
2505 emit(0xD9);
2506 emit(0xE0);
2507 }
2508
fcos()2509 void Assembler::fcos() {
2510 EnsureSpace ensure_space(this);
2511 emit(0xD9);
2512 emit(0xFF);
2513 }
2514
fsin()2515 void Assembler::fsin() {
2516 EnsureSpace ensure_space(this);
2517 emit(0xD9);
2518 emit(0xFE);
2519 }
2520
fptan()2521 void Assembler::fptan() {
2522 EnsureSpace ensure_space(this);
2523 emit(0xD9);
2524 emit(0xF2);
2525 }
2526
fyl2x()2527 void Assembler::fyl2x() {
2528 EnsureSpace ensure_space(this);
2529 emit(0xD9);
2530 emit(0xF1);
2531 }
2532
f2xm1()2533 void Assembler::f2xm1() {
2534 EnsureSpace ensure_space(this);
2535 emit(0xD9);
2536 emit(0xF0);
2537 }
2538
fscale()2539 void Assembler::fscale() {
2540 EnsureSpace ensure_space(this);
2541 emit(0xD9);
2542 emit(0xFD);
2543 }
2544
fninit()2545 void Assembler::fninit() {
2546 EnsureSpace ensure_space(this);
2547 emit(0xDB);
2548 emit(0xE3);
2549 }
2550
fadd(int i)2551 void Assembler::fadd(int i) {
2552 EnsureSpace ensure_space(this);
2553 emit_farith(0xDC, 0xC0, i);
2554 }
2555
fsub(int i)2556 void Assembler::fsub(int i) {
2557 EnsureSpace ensure_space(this);
2558 emit_farith(0xDC, 0xE8, i);
2559 }
2560
fisub_s(Operand adr)2561 void Assembler::fisub_s(Operand adr) {
2562 EnsureSpace ensure_space(this);
2563 emit_optional_rex_32(adr);
2564 emit(0xDA);
2565 emit_operand(4, adr);
2566 }
2567
fmul(int i)2568 void Assembler::fmul(int i) {
2569 EnsureSpace ensure_space(this);
2570 emit_farith(0xDC, 0xC8, i);
2571 }
2572
fdiv(int i)2573 void Assembler::fdiv(int i) {
2574 EnsureSpace ensure_space(this);
2575 emit_farith(0xDC, 0xF8, i);
2576 }
2577
faddp(int i)2578 void Assembler::faddp(int i) {
2579 EnsureSpace ensure_space(this);
2580 emit_farith(0xDE, 0xC0, i);
2581 }
2582
fsubp(int i)2583 void Assembler::fsubp(int i) {
2584 EnsureSpace ensure_space(this);
2585 emit_farith(0xDE, 0xE8, i);
2586 }
2587
fsubrp(int i)2588 void Assembler::fsubrp(int i) {
2589 EnsureSpace ensure_space(this);
2590 emit_farith(0xDE, 0xE0, i);
2591 }
2592
fmulp(int i)2593 void Assembler::fmulp(int i) {
2594 EnsureSpace ensure_space(this);
2595 emit_farith(0xDE, 0xC8, i);
2596 }
2597
fdivp(int i)2598 void Assembler::fdivp(int i) {
2599 EnsureSpace ensure_space(this);
2600 emit_farith(0xDE, 0xF8, i);
2601 }
2602
fprem()2603 void Assembler::fprem() {
2604 EnsureSpace ensure_space(this);
2605 emit(0xD9);
2606 emit(0xF8);
2607 }
2608
fprem1()2609 void Assembler::fprem1() {
2610 EnsureSpace ensure_space(this);
2611 emit(0xD9);
2612 emit(0xF5);
2613 }
2614
fxch(int i)2615 void Assembler::fxch(int i) {
2616 EnsureSpace ensure_space(this);
2617 emit_farith(0xD9, 0xC8, i);
2618 }
2619
fincstp()2620 void Assembler::fincstp() {
2621 EnsureSpace ensure_space(this);
2622 emit(0xD9);
2623 emit(0xF7);
2624 }
2625
ffree(int i)2626 void Assembler::ffree(int i) {
2627 EnsureSpace ensure_space(this);
2628 emit_farith(0xDD, 0xC0, i);
2629 }
2630
ftst()2631 void Assembler::ftst() {
2632 EnsureSpace ensure_space(this);
2633 emit(0xD9);
2634 emit(0xE4);
2635 }
2636
fucomp(int i)2637 void Assembler::fucomp(int i) {
2638 EnsureSpace ensure_space(this);
2639 emit_farith(0xDD, 0xE8, i);
2640 }
2641
fucompp()2642 void Assembler::fucompp() {
2643 EnsureSpace ensure_space(this);
2644 emit(0xDA);
2645 emit(0xE9);
2646 }
2647
fucomi(int i)2648 void Assembler::fucomi(int i) {
2649 EnsureSpace ensure_space(this);
2650 emit(0xDB);
2651 emit(0xE8 + i);
2652 }
2653
fucomip()2654 void Assembler::fucomip() {
2655 EnsureSpace ensure_space(this);
2656 emit(0xDF);
2657 emit(0xE9);
2658 }
2659
fcompp()2660 void Assembler::fcompp() {
2661 EnsureSpace ensure_space(this);
2662 emit(0xDE);
2663 emit(0xD9);
2664 }
2665
fnstsw_ax()2666 void Assembler::fnstsw_ax() {
2667 EnsureSpace ensure_space(this);
2668 emit(0xDF);
2669 emit(0xE0);
2670 }
2671
fwait()2672 void Assembler::fwait() {
2673 EnsureSpace ensure_space(this);
2674 emit(0x9B);
2675 }
2676
frndint()2677 void Assembler::frndint() {
2678 EnsureSpace ensure_space(this);
2679 emit(0xD9);
2680 emit(0xFC);
2681 }
2682
fnclex()2683 void Assembler::fnclex() {
2684 EnsureSpace ensure_space(this);
2685 emit(0xDB);
2686 emit(0xE2);
2687 }
2688
sahf()2689 void Assembler::sahf() {
2690 // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf
2691 // in 64-bit mode. Test CpuID.
2692 DCHECK(IsEnabled(SAHF));
2693 EnsureSpace ensure_space(this);
2694 emit(0x9E);
2695 }
2696
emit_farith(int b1,int b2,int i)2697 void Assembler::emit_farith(int b1, int b2, int i) {
2698 DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode
2699 DCHECK(is_uint3(i)); // illegal stack offset
2700 emit(b1);
2701 emit(b2 + i);
2702 }
2703
2704 // SSE 2 operations.
2705
movd(XMMRegister dst,Register src)2706 void Assembler::movd(XMMRegister dst, Register src) {
2707 DCHECK(!IsEnabled(AVX));
2708 EnsureSpace ensure_space(this);
2709 emit(0x66);
2710 emit_optional_rex_32(dst, src);
2711 emit(0x0F);
2712 emit(0x6E);
2713 emit_sse_operand(dst, src);
2714 }
2715
movd(XMMRegister dst,Operand src)2716 void Assembler::movd(XMMRegister dst, Operand src) {
2717 DCHECK(!IsEnabled(AVX));
2718 EnsureSpace ensure_space(this);
2719 emit(0x66);
2720 emit_optional_rex_32(dst, src);
2721 emit(0x0F);
2722 emit(0x6E);
2723 emit_sse_operand(dst, src);
2724 }
2725
movd(Register dst,XMMRegister src)2726 void Assembler::movd(Register dst, XMMRegister src) {
2727 DCHECK(!IsEnabled(AVX));
2728 EnsureSpace ensure_space(this);
2729 emit(0x66);
2730 emit_optional_rex_32(src, dst);
2731 emit(0x0F);
2732 emit(0x7E);
2733 emit_sse_operand(src, dst);
2734 }
2735
movq(XMMRegister dst,Register src)2736 void Assembler::movq(XMMRegister dst, Register src) {
2737 // Mixing AVX and non-AVX is expensive, catch those cases
2738 DCHECK(!IsEnabled(AVX));
2739 EnsureSpace ensure_space(this);
2740 emit(0x66);
2741 emit_rex_64(dst, src);
2742 emit(0x0F);
2743 emit(0x6E);
2744 emit_sse_operand(dst, src);
2745 }
2746
movq(XMMRegister dst,Operand src)2747 void Assembler::movq(XMMRegister dst, Operand src) {
2748 // Mixing AVX and non-AVX is expensive, catch those cases
2749 DCHECK(!IsEnabled(AVX));
2750 EnsureSpace ensure_space(this);
2751 emit(0x66);
2752 emit_rex_64(dst, src);
2753 emit(0x0F);
2754 emit(0x6E);
2755 emit_sse_operand(dst, src);
2756 }
2757
movq(Register dst,XMMRegister src)2758 void Assembler::movq(Register dst, XMMRegister src) {
2759 // Mixing AVX and non-AVX is expensive, catch those cases
2760 DCHECK(!IsEnabled(AVX));
2761 EnsureSpace ensure_space(this);
2762 emit(0x66);
2763 emit_rex_64(src, dst);
2764 emit(0x0F);
2765 emit(0x7E);
2766 emit_sse_operand(src, dst);
2767 }
2768
movq(XMMRegister dst,XMMRegister src)2769 void Assembler::movq(XMMRegister dst, XMMRegister src) {
2770 // Mixing AVX and non-AVX is expensive, catch those cases
2771 DCHECK(!IsEnabled(AVX));
2772 EnsureSpace ensure_space(this);
2773 if (dst.low_bits() == 4) {
2774 // Avoid unnecessary SIB byte.
2775 emit(0xF3);
2776 emit_optional_rex_32(dst, src);
2777 emit(0x0F);
2778 emit(0x7E);
2779 emit_sse_operand(dst, src);
2780 } else {
2781 emit(0x66);
2782 emit_optional_rex_32(src, dst);
2783 emit(0x0F);
2784 emit(0xD6);
2785 emit_sse_operand(src, dst);
2786 }
2787 }
2788
movdqa(Operand dst,XMMRegister src)2789 void Assembler::movdqa(Operand dst, XMMRegister src) {
2790 EnsureSpace ensure_space(this);
2791 emit(0x66);
2792 emit_rex_64(src, dst);
2793 emit(0x0F);
2794 emit(0x7F);
2795 emit_sse_operand(src, dst);
2796 }
2797
movdqa(XMMRegister dst,Operand src)2798 void Assembler::movdqa(XMMRegister dst, Operand src) {
2799 EnsureSpace ensure_space(this);
2800 emit(0x66);
2801 emit_rex_64(dst, src);
2802 emit(0x0F);
2803 emit(0x6F);
2804 emit_sse_operand(dst, src);
2805 }
2806
movdqa(XMMRegister dst,XMMRegister src)2807 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2808 EnsureSpace ensure_space(this);
2809 emit(0x66);
2810 emit_rex_64(src, dst);
2811 emit(0x0F);
2812 emit(0x7F);
2813 emit_sse_operand(src, dst);
2814 }
2815
movdqu(Operand dst,XMMRegister src)2816 void Assembler::movdqu(Operand dst, XMMRegister src) {
2817 EnsureSpace ensure_space(this);
2818 emit(0xF3);
2819 emit_rex_64(src, dst);
2820 emit(0x0F);
2821 emit(0x7F);
2822 emit_sse_operand(src, dst);
2823 }
2824
movdqu(XMMRegister dst,Operand src)2825 void Assembler::movdqu(XMMRegister dst, Operand src) {
2826 EnsureSpace ensure_space(this);
2827 emit(0xF3);
2828 emit_rex_64(dst, src);
2829 emit(0x0F);
2830 emit(0x6F);
2831 emit_sse_operand(dst, src);
2832 }
2833
movdqu(XMMRegister dst,XMMRegister src)2834 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2835 EnsureSpace ensure_space(this);
2836 emit(0xF3);
2837 emit_rex_64(dst, src);
2838 emit(0x0F);
2839 emit(0x6F);
2840 emit_sse_operand(dst, src);
2841 }
2842
pinsrw(XMMRegister dst,Register src,uint8_t imm8)2843 void Assembler::pinsrw(XMMRegister dst, Register src, uint8_t imm8) {
2844 EnsureSpace ensure_space(this);
2845 emit(0x66);
2846 emit_optional_rex_32(dst, src);
2847 emit(0x0F);
2848 emit(0xC4);
2849 emit_sse_operand(dst, src);
2850 emit(imm8);
2851 }
2852
pinsrw(XMMRegister dst,Operand src,uint8_t imm8)2853 void Assembler::pinsrw(XMMRegister dst, Operand src, uint8_t imm8) {
2854 EnsureSpace ensure_space(this);
2855 emit(0x66);
2856 emit_optional_rex_32(dst, src);
2857 emit(0x0F);
2858 emit(0xC4);
2859 emit_sse_operand(dst, src);
2860 emit(imm8);
2861 }
2862
pextrq(Register dst,XMMRegister src,int8_t imm8)2863 void Assembler::pextrq(Register dst, XMMRegister src, int8_t imm8) {
2864 DCHECK(IsEnabled(SSE4_1));
2865 EnsureSpace ensure_space(this);
2866 emit(0x66);
2867 emit_rex_64(src, dst);
2868 emit(0x0F);
2869 emit(0x3A);
2870 emit(0x16);
2871 emit_sse_operand(src, dst);
2872 emit(imm8);
2873 }
2874
pinsrq(XMMRegister dst,Register src,uint8_t imm8)2875 void Assembler::pinsrq(XMMRegister dst, Register src, uint8_t imm8) {
2876 DCHECK(IsEnabled(SSE4_1));
2877 EnsureSpace ensure_space(this);
2878 emit(0x66);
2879 emit_rex_64(dst, src);
2880 emit(0x0F);
2881 emit(0x3A);
2882 emit(0x22);
2883 emit_sse_operand(dst, src);
2884 emit(imm8);
2885 }
2886
pinsrq(XMMRegister dst,Operand src,uint8_t imm8)2887 void Assembler::pinsrq(XMMRegister dst, Operand src, uint8_t imm8) {
2888 DCHECK(IsEnabled(SSE4_1));
2889 EnsureSpace ensure_space(this);
2890 emit(0x66);
2891 emit_rex_64(dst, src);
2892 emit(0x0F);
2893 emit(0x3A);
2894 emit(0x22);
2895 emit_sse_operand(dst, src);
2896 emit(imm8);
2897 }
2898
pinsrd(XMMRegister dst,Register src,uint8_t imm8)2899 void Assembler::pinsrd(XMMRegister dst, Register src, uint8_t imm8) {
2900 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x22, imm8);
2901 }
2902
pinsrd(XMMRegister dst,Operand src,uint8_t imm8)2903 void Assembler::pinsrd(XMMRegister dst, Operand src, uint8_t imm8) {
2904 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x22);
2905 emit(imm8);
2906 }
2907
pinsrb(XMMRegister dst,Register src,uint8_t imm8)2908 void Assembler::pinsrb(XMMRegister dst, Register src, uint8_t imm8) {
2909 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x20, imm8);
2910 }
2911
pinsrb(XMMRegister dst,Operand src,uint8_t imm8)2912 void Assembler::pinsrb(XMMRegister dst, Operand src, uint8_t imm8) {
2913 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x20);
2914 emit(imm8);
2915 }
2916
insertps(XMMRegister dst,XMMRegister src,byte imm8)2917 void Assembler::insertps(XMMRegister dst, XMMRegister src, byte imm8) {
2918 DCHECK(is_uint8(imm8));
2919 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x21);
2920 emit(imm8);
2921 }
2922
insertps(XMMRegister dst,Operand src,byte imm8)2923 void Assembler::insertps(XMMRegister dst, Operand src, byte imm8) {
2924 DCHECK(is_uint8(imm8));
2925 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x21);
2926 emit(imm8);
2927 }
2928
movsd(Operand dst,XMMRegister src)2929 void Assembler::movsd(Operand dst, XMMRegister src) {
2930 DCHECK(!IsEnabled(AVX));
2931 EnsureSpace ensure_space(this);
2932 emit(0xF2); // double
2933 emit_optional_rex_32(src, dst);
2934 emit(0x0F);
2935 emit(0x11); // store
2936 emit_sse_operand(src, dst);
2937 }
2938
movsd(XMMRegister dst,XMMRegister src)2939 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2940 DCHECK(!IsEnabled(AVX));
2941 EnsureSpace ensure_space(this);
2942 emit(0xF2); // double
2943 emit_optional_rex_32(dst, src);
2944 emit(0x0F);
2945 emit(0x10); // load
2946 emit_sse_operand(dst, src);
2947 }
2948
movsd(XMMRegister dst,Operand src)2949 void Assembler::movsd(XMMRegister dst, Operand src) {
2950 DCHECK(!IsEnabled(AVX));
2951 EnsureSpace ensure_space(this);
2952 emit(0xF2); // double
2953 emit_optional_rex_32(dst, src);
2954 emit(0x0F);
2955 emit(0x10); // load
2956 emit_sse_operand(dst, src);
2957 }
2958
movaps(XMMRegister dst,XMMRegister src)2959 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2960 DCHECK(!IsEnabled(AVX));
2961 EnsureSpace ensure_space(this);
2962 if (src.low_bits() == 4) {
2963 // Try to avoid an unnecessary SIB byte.
2964 emit_optional_rex_32(src, dst);
2965 emit(0x0F);
2966 emit(0x29);
2967 emit_sse_operand(src, dst);
2968 } else {
2969 emit_optional_rex_32(dst, src);
2970 emit(0x0F);
2971 emit(0x28);
2972 emit_sse_operand(dst, src);
2973 }
2974 }
2975
movaps(XMMRegister dst,Operand src)2976 void Assembler::movaps(XMMRegister dst, Operand src) {
2977 DCHECK(!IsEnabled(AVX));
2978 EnsureSpace ensure_space(this);
2979 emit_optional_rex_32(dst, src);
2980 emit(0x0F);
2981 emit(0x28);
2982 emit_sse_operand(dst, src);
2983 }
2984
shufps(XMMRegister dst,XMMRegister src,byte imm8)2985 void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) {
2986 DCHECK(is_uint8(imm8));
2987 EnsureSpace ensure_space(this);
2988 emit_optional_rex_32(dst, src);
2989 emit(0x0F);
2990 emit(0xC6);
2991 emit_sse_operand(dst, src);
2992 emit(imm8);
2993 }
2994
movapd(XMMRegister dst,XMMRegister src)2995 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2996 DCHECK(!IsEnabled(AVX));
2997 EnsureSpace ensure_space(this);
2998 if (src.low_bits() == 4) {
2999 // Try to avoid an unnecessary SIB byte.
3000 emit(0x66);
3001 emit_optional_rex_32(src, dst);
3002 emit(0x0F);
3003 emit(0x29);
3004 emit_sse_operand(src, dst);
3005 } else {
3006 emit(0x66);
3007 emit_optional_rex_32(dst, src);
3008 emit(0x0F);
3009 emit(0x28);
3010 emit_sse_operand(dst, src);
3011 }
3012 }
3013
movupd(XMMRegister dst,Operand src)3014 void Assembler::movupd(XMMRegister dst, Operand src) {
3015 EnsureSpace ensure_space(this);
3016 emit(0x66);
3017 emit_optional_rex_32(dst, src);
3018 emit(0x0F);
3019 emit(0x10);
3020 emit_sse_operand(dst, src);
3021 }
3022
movupd(Operand dst,XMMRegister src)3023 void Assembler::movupd(Operand dst, XMMRegister src) {
3024 EnsureSpace ensure_space(this);
3025 emit(0x66);
3026 emit_optional_rex_32(src, dst);
3027 emit(0x0F);
3028 emit(0x11);
3029 emit_sse_operand(src, dst);
3030 }
3031
ucomiss(XMMRegister dst,XMMRegister src)3032 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
3033 DCHECK(!IsEnabled(AVX));
3034 EnsureSpace ensure_space(this);
3035 emit_optional_rex_32(dst, src);
3036 emit(0x0F);
3037 emit(0x2E);
3038 emit_sse_operand(dst, src);
3039 }
3040
ucomiss(XMMRegister dst,Operand src)3041 void Assembler::ucomiss(XMMRegister dst, Operand src) {
3042 DCHECK(!IsEnabled(AVX));
3043 EnsureSpace ensure_space(this);
3044 emit_optional_rex_32(dst, src);
3045 emit(0x0F);
3046 emit(0x2E);
3047 emit_sse_operand(dst, src);
3048 }
3049
movss(XMMRegister dst,XMMRegister src)3050 void Assembler::movss(XMMRegister dst, XMMRegister src) {
3051 DCHECK(!IsEnabled(AVX));
3052 EnsureSpace ensure_space(this);
3053 emit(0xF3); // single
3054 emit_optional_rex_32(dst, src);
3055 emit(0x0F);
3056 emit(0x10); // load
3057 emit_sse_operand(dst, src);
3058 }
3059
movss(XMMRegister dst,Operand src)3060 void Assembler::movss(XMMRegister dst, Operand src) {
3061 DCHECK(!IsEnabled(AVX));
3062 EnsureSpace ensure_space(this);
3063 emit(0xF3); // single
3064 emit_optional_rex_32(dst, src);
3065 emit(0x0F);
3066 emit(0x10); // load
3067 emit_sse_operand(dst, src);
3068 }
3069
movss(Operand src,XMMRegister dst)3070 void Assembler::movss(Operand src, XMMRegister dst) {
3071 DCHECK(!IsEnabled(AVX));
3072 EnsureSpace ensure_space(this);
3073 emit(0xF3); // single
3074 emit_optional_rex_32(dst, src);
3075 emit(0x0F);
3076 emit(0x11); // store
3077 emit_sse_operand(dst, src);
3078 }
3079
movlps(XMMRegister dst,Operand src)3080 void Assembler::movlps(XMMRegister dst, Operand src) {
3081 DCHECK(!IsEnabled(AVX));
3082 EnsureSpace ensure_space(this);
3083 emit_optional_rex_32(dst, src);
3084 emit(0x0F);
3085 emit(0x12);
3086 emit_sse_operand(dst, src);
3087 }
3088
movlps(Operand src,XMMRegister dst)3089 void Assembler::movlps(Operand src, XMMRegister dst) {
3090 DCHECK(!IsEnabled(AVX));
3091 EnsureSpace ensure_space(this);
3092 emit_optional_rex_32(dst, src);
3093 emit(0x0F);
3094 emit(0x13);
3095 emit_sse_operand(dst, src);
3096 }
3097
movhps(XMMRegister dst,Operand src)3098 void Assembler::movhps(XMMRegister dst, Operand src) {
3099 DCHECK(!IsEnabled(AVX));
3100 EnsureSpace ensure_space(this);
3101 emit_optional_rex_32(dst, src);
3102 emit(0x0F);
3103 emit(0x16);
3104 emit_sse_operand(dst, src);
3105 }
3106
movhps(Operand src,XMMRegister dst)3107 void Assembler::movhps(Operand src, XMMRegister dst) {
3108 DCHECK(!IsEnabled(AVX));
3109 EnsureSpace ensure_space(this);
3110 emit_optional_rex_32(dst, src);
3111 emit(0x0F);
3112 emit(0x17);
3113 emit_sse_operand(dst, src);
3114 }
3115
cmpps(XMMRegister dst,XMMRegister src,int8_t cmp)3116 void Assembler::cmpps(XMMRegister dst, XMMRegister src, int8_t cmp) {
3117 EnsureSpace ensure_space(this);
3118 emit_optional_rex_32(dst, src);
3119 emit(0x0F);
3120 emit(0xC2);
3121 emit_sse_operand(dst, src);
3122 emit(cmp);
3123 }
3124
cmpps(XMMRegister dst,Operand src,int8_t cmp)3125 void Assembler::cmpps(XMMRegister dst, Operand src, int8_t cmp) {
3126 EnsureSpace ensure_space(this);
3127 emit_optional_rex_32(dst, src);
3128 emit(0x0F);
3129 emit(0xC2);
3130 emit_sse_operand(dst, src);
3131 emit(cmp);
3132 }
3133
cmppd(XMMRegister dst,XMMRegister src,int8_t cmp)3134 void Assembler::cmppd(XMMRegister dst, XMMRegister src, int8_t cmp) {
3135 EnsureSpace ensure_space(this);
3136 emit(0x66);
3137 emit_optional_rex_32(dst, src);
3138 emit(0x0F);
3139 emit(0xC2);
3140 emit_sse_operand(dst, src);
3141 emit(cmp);
3142 }
3143
cmppd(XMMRegister dst,Operand src,int8_t cmp)3144 void Assembler::cmppd(XMMRegister dst, Operand src, int8_t cmp) {
3145 EnsureSpace ensure_space(this);
3146 emit(0x66);
3147 emit_optional_rex_32(dst, src);
3148 emit(0x0F);
3149 emit(0xC2);
3150 emit_sse_operand(dst, src);
3151 emit(cmp);
3152 }
3153
cvtdq2pd(XMMRegister dst,XMMRegister src)3154 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
3155 sse2_instr(dst, src, 0xF3, 0x0F, 0xE6);
3156 }
3157
cvttss2si(Register dst,Operand src)3158 void Assembler::cvttss2si(Register dst, Operand src) {
3159 DCHECK(!IsEnabled(AVX));
3160 EnsureSpace ensure_space(this);
3161 emit(0xF3);
3162 emit_optional_rex_32(dst, src);
3163 emit(0x0F);
3164 emit(0x2C);
3165 emit_operand(dst, src);
3166 }
3167
cvttss2si(Register dst,XMMRegister src)3168 void Assembler::cvttss2si(Register dst, XMMRegister src) {
3169 DCHECK(!IsEnabled(AVX));
3170 EnsureSpace ensure_space(this);
3171 emit(0xF3);
3172 emit_optional_rex_32(dst, src);
3173 emit(0x0F);
3174 emit(0x2C);
3175 emit_sse_operand(dst, src);
3176 }
3177
cvttsd2si(Register dst,Operand src)3178 void Assembler::cvttsd2si(Register dst, Operand src) {
3179 DCHECK(!IsEnabled(AVX));
3180 EnsureSpace ensure_space(this);
3181 emit(0xF2);
3182 emit_optional_rex_32(dst, src);
3183 emit(0x0F);
3184 emit(0x2C);
3185 emit_operand(dst, src);
3186 }
3187
cvttsd2si(Register dst,XMMRegister src)3188 void Assembler::cvttsd2si(Register dst, XMMRegister src) {
3189 DCHECK(!IsEnabled(AVX));
3190 EnsureSpace ensure_space(this);
3191 emit(0xF2);
3192 emit_optional_rex_32(dst, src);
3193 emit(0x0F);
3194 emit(0x2C);
3195 emit_sse_operand(dst, src);
3196 }
3197
cvttss2siq(Register dst,XMMRegister src)3198 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
3199 DCHECK(!IsEnabled(AVX));
3200 EnsureSpace ensure_space(this);
3201 emit(0xF3);
3202 emit_rex_64(dst, src);
3203 emit(0x0F);
3204 emit(0x2C);
3205 emit_sse_operand(dst, src);
3206 }
3207
cvttss2siq(Register dst,Operand src)3208 void Assembler::cvttss2siq(Register dst, Operand src) {
3209 DCHECK(!IsEnabled(AVX));
3210 EnsureSpace ensure_space(this);
3211 emit(0xF3);
3212 emit_rex_64(dst, src);
3213 emit(0x0F);
3214 emit(0x2C);
3215 emit_sse_operand(dst, src);
3216 }
3217
cvttsd2siq(Register dst,XMMRegister src)3218 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
3219 DCHECK(!IsEnabled(AVX));
3220 EnsureSpace ensure_space(this);
3221 emit(0xF2);
3222 emit_rex_64(dst, src);
3223 emit(0x0F);
3224 emit(0x2C);
3225 emit_sse_operand(dst, src);
3226 }
3227
cvttsd2siq(Register dst,Operand src)3228 void Assembler::cvttsd2siq(Register dst, Operand src) {
3229 DCHECK(!IsEnabled(AVX));
3230 EnsureSpace ensure_space(this);
3231 emit(0xF2);
3232 emit_rex_64(dst, src);
3233 emit(0x0F);
3234 emit(0x2C);
3235 emit_sse_operand(dst, src);
3236 }
3237
cvttps2dq(XMMRegister dst,Operand src)3238 void Assembler::cvttps2dq(XMMRegister dst, Operand src) {
3239 EnsureSpace ensure_space(this);
3240 emit(0xF3);
3241 emit_rex_64(dst, src);
3242 emit(0x0F);
3243 emit(0x5B);
3244 emit_sse_operand(dst, src);
3245 }
3246
cvttps2dq(XMMRegister dst,XMMRegister src)3247 void Assembler::cvttps2dq(XMMRegister dst, XMMRegister src) {
3248 EnsureSpace ensure_space(this);
3249 emit(0xF3);
3250 emit_rex_64(dst, src);
3251 emit(0x0F);
3252 emit(0x5B);
3253 emit_sse_operand(dst, src);
3254 }
3255
cvtlsi2sd(XMMRegister dst,Operand src)3256 void Assembler::cvtlsi2sd(XMMRegister dst, Operand src) {
3257 DCHECK(!IsEnabled(AVX));
3258 EnsureSpace ensure_space(this);
3259 emit(0xF2);
3260 emit_optional_rex_32(dst, src);
3261 emit(0x0F);
3262 emit(0x2A);
3263 emit_sse_operand(dst, src);
3264 }
3265
cvtlsi2sd(XMMRegister dst,Register src)3266 void Assembler::cvtlsi2sd(XMMRegister dst, Register src) {
3267 DCHECK(!IsEnabled(AVX));
3268 EnsureSpace ensure_space(this);
3269 emit(0xF2);
3270 emit_optional_rex_32(dst, src);
3271 emit(0x0F);
3272 emit(0x2A);
3273 emit_sse_operand(dst, src);
3274 }
3275
cvtlsi2ss(XMMRegister dst,Operand src)3276 void Assembler::cvtlsi2ss(XMMRegister dst, Operand src) {
3277 DCHECK(!IsEnabled(AVX));
3278 EnsureSpace ensure_space(this);
3279 emit(0xF3);
3280 emit_optional_rex_32(dst, src);
3281 emit(0x0F);
3282 emit(0x2A);
3283 emit_sse_operand(dst, src);
3284 }
3285
cvtlsi2ss(XMMRegister dst,Register src)3286 void Assembler::cvtlsi2ss(XMMRegister dst, Register src) {
3287 EnsureSpace ensure_space(this);
3288 emit(0xF3);
3289 emit_optional_rex_32(dst, src);
3290 emit(0x0F);
3291 emit(0x2A);
3292 emit_sse_operand(dst, src);
3293 }
3294
cvtqsi2ss(XMMRegister dst,Operand src)3295 void Assembler::cvtqsi2ss(XMMRegister dst, Operand src) {
3296 DCHECK(!IsEnabled(AVX));
3297 EnsureSpace ensure_space(this);
3298 emit(0xF3);
3299 emit_rex_64(dst, src);
3300 emit(0x0F);
3301 emit(0x2A);
3302 emit_sse_operand(dst, src);
3303 }
3304
cvtqsi2ss(XMMRegister dst,Register src)3305 void Assembler::cvtqsi2ss(XMMRegister dst, Register src) {
3306 DCHECK(!IsEnabled(AVX));
3307 EnsureSpace ensure_space(this);
3308 emit(0xF3);
3309 emit_rex_64(dst, src);
3310 emit(0x0F);
3311 emit(0x2A);
3312 emit_sse_operand(dst, src);
3313 }
3314
cvtqsi2sd(XMMRegister dst,Operand src)3315 void Assembler::cvtqsi2sd(XMMRegister dst, Operand src) {
3316 DCHECK(!IsEnabled(AVX));
3317 EnsureSpace ensure_space(this);
3318 emit(0xF2);
3319 emit_rex_64(dst, src);
3320 emit(0x0F);
3321 emit(0x2A);
3322 emit_sse_operand(dst, src);
3323 }
3324
cvtqsi2sd(XMMRegister dst,Register src)3325 void Assembler::cvtqsi2sd(XMMRegister dst, Register src) {
3326 DCHECK(!IsEnabled(AVX));
3327 EnsureSpace ensure_space(this);
3328 emit(0xF2);
3329 emit_rex_64(dst, src);
3330 emit(0x0F);
3331 emit(0x2A);
3332 emit_sse_operand(dst, src);
3333 }
3334
cvtsd2si(Register dst,XMMRegister src)3335 void Assembler::cvtsd2si(Register dst, XMMRegister src) {
3336 DCHECK(!IsEnabled(AVX));
3337 EnsureSpace ensure_space(this);
3338 emit(0xF2);
3339 emit_optional_rex_32(dst, src);
3340 emit(0x0F);
3341 emit(0x2D);
3342 emit_sse_operand(dst, src);
3343 }
3344
cvtsd2siq(Register dst,XMMRegister src)3345 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
3346 DCHECK(!IsEnabled(AVX));
3347 EnsureSpace ensure_space(this);
3348 emit(0xF2);
3349 emit_rex_64(dst, src);
3350 emit(0x0F);
3351 emit(0x2D);
3352 emit_sse_operand(dst, src);
3353 }
3354
haddps(XMMRegister dst,XMMRegister src)3355 void Assembler::haddps(XMMRegister dst, XMMRegister src) {
3356 DCHECK(IsEnabled(SSE3));
3357 EnsureSpace ensure_space(this);
3358 emit(0xF2);
3359 emit_optional_rex_32(dst, src);
3360 emit(0x0F);
3361 emit(0x7C);
3362 emit_sse_operand(dst, src);
3363 }
3364
haddps(XMMRegister dst,Operand src)3365 void Assembler::haddps(XMMRegister dst, Operand src) {
3366 DCHECK(IsEnabled(SSE3));
3367 EnsureSpace ensure_space(this);
3368 emit(0xF2);
3369 emit_optional_rex_32(dst, src);
3370 emit(0x0F);
3371 emit(0x7C);
3372 emit_sse_operand(dst, src);
3373 }
3374
cmpeqss(XMMRegister dst,XMMRegister src)3375 void Assembler::cmpeqss(XMMRegister dst, XMMRegister src) {
3376 DCHECK(!IsEnabled(AVX));
3377 EnsureSpace ensure_space(this);
3378 emit(0xF3);
3379 emit_optional_rex_32(dst, src);
3380 emit(0x0F);
3381 emit(0xC2);
3382 emit_sse_operand(dst, src);
3383 emit(0x00); // EQ == 0
3384 }
3385
cmpeqsd(XMMRegister dst,XMMRegister src)3386 void Assembler::cmpeqsd(XMMRegister dst, XMMRegister src) {
3387 DCHECK(!IsEnabled(AVX));
3388 EnsureSpace ensure_space(this);
3389 emit(0xF2);
3390 emit_optional_rex_32(dst, src);
3391 emit(0x0F);
3392 emit(0xC2);
3393 emit_sse_operand(dst, src);
3394 emit(0x00); // EQ == 0
3395 }
3396
cmpltsd(XMMRegister dst,XMMRegister src)3397 void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) {
3398 EnsureSpace ensure_space(this);
3399 emit(0xF2);
3400 emit_optional_rex_32(dst, src);
3401 emit(0x0F);
3402 emit(0xC2);
3403 emit_sse_operand(dst, src);
3404 emit(0x01); // LT == 1
3405 }
3406
roundss(XMMRegister dst,XMMRegister src,RoundingMode mode)3407 void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3408 DCHECK(!IsEnabled(AVX));
3409 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0A);
3410 // Mask precision exception.
3411 emit(static_cast<byte>(mode) | 0x8);
3412 }
3413
roundss(XMMRegister dst,Operand src,RoundingMode mode)3414 void Assembler::roundss(XMMRegister dst, Operand src, RoundingMode mode) {
3415 DCHECK(!IsEnabled(AVX));
3416 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0A);
3417 // Mask precision exception.
3418 emit(static_cast<byte>(mode) | 0x8);
3419 }
3420
roundsd(XMMRegister dst,XMMRegister src,RoundingMode mode)3421 void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3422 DCHECK(!IsEnabled(AVX));
3423 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0B);
3424 // Mask precision exception.
3425 emit(static_cast<byte>(mode) | 0x8);
3426 }
3427
roundsd(XMMRegister dst,Operand src,RoundingMode mode)3428 void Assembler::roundsd(XMMRegister dst, Operand src, RoundingMode mode) {
3429 DCHECK(!IsEnabled(AVX));
3430 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0B);
3431 // Mask precision exception.
3432 emit(static_cast<byte>(mode) | 0x8);
3433 }
3434
roundps(XMMRegister dst,XMMRegister src,RoundingMode mode)3435 void Assembler::roundps(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3436 DCHECK(!IsEnabled(AVX));
3437 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x08);
3438 // Mask precision exception.
3439 emit(static_cast<byte>(mode) | 0x8);
3440 }
3441
roundpd(XMMRegister dst,XMMRegister src,RoundingMode mode)3442 void Assembler::roundpd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3443 DCHECK(!IsEnabled(AVX));
3444 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x09);
3445 // Mask precision exception.
3446 emit(static_cast<byte>(mode) | 0x8);
3447 }
3448
movmskpd(Register dst,XMMRegister src)3449 void Assembler::movmskpd(Register dst, XMMRegister src) {
3450 EnsureSpace ensure_space(this);
3451 emit(0x66);
3452 emit_optional_rex_32(dst, src);
3453 emit(0x0F);
3454 emit(0x50);
3455 emit_sse_operand(dst, src);
3456 }
3457
movmskps(Register dst,XMMRegister src)3458 void Assembler::movmskps(Register dst, XMMRegister src) {
3459 EnsureSpace ensure_space(this);
3460 emit_optional_rex_32(dst, src);
3461 emit(0x0F);
3462 emit(0x50);
3463 emit_sse_operand(dst, src);
3464 }
3465
pmovmskb(Register dst,XMMRegister src)3466 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3467 EnsureSpace ensure_space(this);
3468 emit(0x66);
3469 emit_optional_rex_32(dst, src);
3470 emit(0x0F);
3471 emit(0xD7);
3472 emit_sse_operand(dst, src);
3473 }
3474
3475 // AVX instructions
3476 #define VMOV_DUP(SIMDRegister, length) \
3477 void Assembler::vmovddup(SIMDRegister dst, SIMDRegister src) { \
3478 DCHECK(IsEnabled(AVX)); \
3479 EnsureSpace ensure_space(this); \
3480 emit_vex_prefix(dst, xmm0, src, k##length, kF2, k0F, kWIG); \
3481 emit(0x12); \
3482 emit_sse_operand(dst, src); \
3483 } \
3484 \
3485 void Assembler::vmovddup(SIMDRegister dst, Operand src) { \
3486 DCHECK(IsEnabled(AVX)); \
3487 EnsureSpace ensure_space(this); \
3488 emit_vex_prefix(dst, xmm0, src, k##length, kF2, k0F, kWIG); \
3489 emit(0x12); \
3490 emit_sse_operand(dst, src); \
3491 } \
3492 \
3493 void Assembler::vmovshdup(SIMDRegister dst, SIMDRegister src) { \
3494 DCHECK(IsEnabled(AVX)); \
3495 EnsureSpace ensure_space(this); \
3496 emit_vex_prefix(dst, xmm0, src, k##length, kF3, k0F, kWIG); \
3497 emit(0x16); \
3498 emit_sse_operand(dst, src); \
3499 }
VMOV_DUP(XMMRegister,L128)3500 VMOV_DUP(XMMRegister, L128)
3501 VMOV_DUP(YMMRegister, L256)
3502 #undef VMOV_DUP
3503
3504 #define BROADCASTSS(SIMDRegister, length) \
3505 void Assembler::vbroadcastss(SIMDRegister dst, Operand src) { \
3506 DCHECK(IsEnabled(AVX)); \
3507 EnsureSpace ensure_space(this); \
3508 emit_vex_prefix(dst, xmm0, src, k##length, k66, k0F38, kW0); \
3509 emit(0x18); \
3510 emit_sse_operand(dst, src); \
3511 } \
3512 void Assembler::vbroadcastss(SIMDRegister dst, XMMRegister src) { \
3513 DCHECK(IsEnabled(AVX2)); \
3514 EnsureSpace ensure_space(this); \
3515 emit_vex_prefix(dst, xmm0, src, k##length, k66, k0F38, kW0); \
3516 emit(0x18); \
3517 emit_sse_operand(dst, src); \
3518 }
3519 BROADCASTSS(XMMRegister, L128)
3520 BROADCASTSS(YMMRegister, L256)
3521 #undef BROADCASTSS
3522
3523 void Assembler::fma_instr(byte op, XMMRegister dst, XMMRegister src1,
3524 XMMRegister src2, VectorLength l, SIMDPrefix pp,
3525 LeadingOpcode m, VexW w) {
3526 DCHECK(IsEnabled(FMA3));
3527 EnsureSpace ensure_space(this);
3528 emit_vex_prefix(dst, src1, src2, l, pp, m, w);
3529 emit(op);
3530 emit_sse_operand(dst, src2);
3531 }
3532
fma_instr(byte op,XMMRegister dst,XMMRegister src1,Operand src2,VectorLength l,SIMDPrefix pp,LeadingOpcode m,VexW w)3533 void Assembler::fma_instr(byte op, XMMRegister dst, XMMRegister src1,
3534 Operand src2, VectorLength l, SIMDPrefix pp,
3535 LeadingOpcode m, VexW w) {
3536 DCHECK(IsEnabled(FMA3));
3537 EnsureSpace ensure_space(this);
3538 emit_vex_prefix(dst, src1, src2, l, pp, m, w);
3539 emit(op);
3540 emit_sse_operand(dst, src2);
3541 }
3542
vmovd(XMMRegister dst,Register src)3543 void Assembler::vmovd(XMMRegister dst, Register src) {
3544 DCHECK(IsEnabled(AVX));
3545 EnsureSpace ensure_space(this);
3546 XMMRegister isrc = XMMRegister::from_code(src.code());
3547 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW0);
3548 emit(0x6E);
3549 emit_sse_operand(dst, src);
3550 }
3551
vmovd(XMMRegister dst,Operand src)3552 void Assembler::vmovd(XMMRegister dst, Operand src) {
3553 DCHECK(IsEnabled(AVX));
3554 EnsureSpace ensure_space(this);
3555 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW0);
3556 emit(0x6E);
3557 emit_sse_operand(dst, src);
3558 }
3559
vmovd(Register dst,XMMRegister src)3560 void Assembler::vmovd(Register dst, XMMRegister src) {
3561 DCHECK(IsEnabled(AVX));
3562 EnsureSpace ensure_space(this);
3563 XMMRegister idst = XMMRegister::from_code(dst.code());
3564 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW0);
3565 emit(0x7E);
3566 emit_sse_operand(src, dst);
3567 }
3568
vmovq(XMMRegister dst,Register src)3569 void Assembler::vmovq(XMMRegister dst, Register src) {
3570 DCHECK(IsEnabled(AVX));
3571 EnsureSpace ensure_space(this);
3572 XMMRegister isrc = XMMRegister::from_code(src.code());
3573 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW1);
3574 emit(0x6E);
3575 emit_sse_operand(dst, src);
3576 }
3577
vmovq(XMMRegister dst,Operand src)3578 void Assembler::vmovq(XMMRegister dst, Operand src) {
3579 DCHECK(IsEnabled(AVX));
3580 EnsureSpace ensure_space(this);
3581 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW1);
3582 emit(0x6E);
3583 emit_sse_operand(dst, src);
3584 }
3585
vmovq(Register dst,XMMRegister src)3586 void Assembler::vmovq(Register dst, XMMRegister src) {
3587 DCHECK(IsEnabled(AVX));
3588 EnsureSpace ensure_space(this);
3589 XMMRegister idst = XMMRegister::from_code(dst.code());
3590 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW1);
3591 emit(0x7E);
3592 emit_sse_operand(src, dst);
3593 }
3594
vmovdqa(XMMRegister dst,Operand src)3595 void Assembler::vmovdqa(XMMRegister dst, Operand src) {
3596 DCHECK(IsEnabled(AVX));
3597 EnsureSpace ensure_space(this);
3598 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kWIG);
3599 emit(0x6F);
3600 emit_sse_operand(dst, src);
3601 }
3602
vmovdqa(XMMRegister dst,XMMRegister src)3603 void Assembler::vmovdqa(XMMRegister dst, XMMRegister src) {
3604 DCHECK(IsEnabled(AVX));
3605 EnsureSpace ensure_space(this);
3606 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kWIG);
3607 emit(0x6F);
3608 emit_sse_operand(dst, src);
3609 }
3610
vmovdqa(YMMRegister dst,Operand src)3611 void Assembler::vmovdqa(YMMRegister dst, Operand src) {
3612 DCHECK(IsEnabled(AVX));
3613 EnsureSpace ensure_space(this);
3614 emit_vex_prefix(dst, ymm0, src, kL256, k66, k0F, kWIG);
3615 emit(0x6F);
3616 emit_sse_operand(dst, src);
3617 }
3618
vmovdqa(YMMRegister dst,YMMRegister src)3619 void Assembler::vmovdqa(YMMRegister dst, YMMRegister src) {
3620 DCHECK(IsEnabled(AVX));
3621 EnsureSpace ensure_space(this);
3622 emit_vex_prefix(dst, ymm0, src, kL256, k66, k0F, kWIG);
3623 emit(0x6F);
3624 emit_sse_operand(dst, src);
3625 }
3626
vmovdqu(XMMRegister dst,Operand src)3627 void Assembler::vmovdqu(XMMRegister dst, Operand src) {
3628 DCHECK(IsEnabled(AVX));
3629 EnsureSpace ensure_space(this);
3630 emit_vex_prefix(dst, xmm0, src, kL128, kF3, k0F, kWIG);
3631 emit(0x6F);
3632 emit_sse_operand(dst, src);
3633 }
3634
vmovdqu(Operand dst,XMMRegister src)3635 void Assembler::vmovdqu(Operand dst, XMMRegister src) {
3636 DCHECK(IsEnabled(AVX));
3637 EnsureSpace ensure_space(this);
3638 emit_vex_prefix(src, xmm0, dst, kL128, kF3, k0F, kWIG);
3639 emit(0x7F);
3640 emit_sse_operand(src, dst);
3641 }
3642
vmovdqu(XMMRegister dst,XMMRegister src)3643 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
3644 DCHECK(IsEnabled(AVX));
3645 EnsureSpace ensure_space(this);
3646 emit_vex_prefix(src, xmm0, dst, kL128, kF3, k0F, kWIG);
3647 emit(0x7F);
3648 emit_sse_operand(src, dst);
3649 }
3650
vmovdqu(YMMRegister dst,Operand src)3651 void Assembler::vmovdqu(YMMRegister dst, Operand src) {
3652 DCHECK(IsEnabled(AVX));
3653 EnsureSpace ensure_space(this);
3654 emit_vex_prefix(dst, ymm0, src, kL256, kF3, k0F, kWIG);
3655 emit(0x6F);
3656 emit_sse_operand(dst, src);
3657 }
3658
vmovdqu(Operand dst,YMMRegister src)3659 void Assembler::vmovdqu(Operand dst, YMMRegister src) {
3660 DCHECK(IsEnabled(AVX));
3661 EnsureSpace ensure_space(this);
3662 emit_vex_prefix(src, ymm0, dst, kL256, kF3, k0F, kWIG);
3663 emit(0x7F);
3664 emit_sse_operand(src, dst);
3665 }
3666
vmovdqu(YMMRegister dst,YMMRegister src)3667 void Assembler::vmovdqu(YMMRegister dst, YMMRegister src) {
3668 DCHECK(IsEnabled(AVX));
3669 EnsureSpace ensure_space(this);
3670 emit_vex_prefix(src, ymm0, dst, kL256, kF3, k0F, kWIG);
3671 emit(0x7F);
3672 emit_sse_operand(src, dst);
3673 }
3674
vmovlps(XMMRegister dst,XMMRegister src1,Operand src2)3675 void Assembler::vmovlps(XMMRegister dst, XMMRegister src1, Operand src2) {
3676 DCHECK(IsEnabled(AVX));
3677 EnsureSpace ensure_space(this);
3678 emit_vex_prefix(dst, src1, src2, kL128, kNoPrefix, k0F, kWIG);
3679 emit(0x12);
3680 emit_sse_operand(dst, src2);
3681 }
3682
vmovlps(Operand dst,XMMRegister src)3683 void Assembler::vmovlps(Operand dst, XMMRegister src) {
3684 DCHECK(IsEnabled(AVX));
3685 EnsureSpace ensure_space(this);
3686 emit_vex_prefix(src, xmm0, dst, kL128, kNoPrefix, k0F, kWIG);
3687 emit(0x13);
3688 emit_sse_operand(src, dst);
3689 }
3690
vmovhps(XMMRegister dst,XMMRegister src1,Operand src2)3691 void Assembler::vmovhps(XMMRegister dst, XMMRegister src1, Operand src2) {
3692 DCHECK(IsEnabled(AVX));
3693 EnsureSpace ensure_space(this);
3694 emit_vex_prefix(dst, src1, src2, kL128, kNoPrefix, k0F, kWIG);
3695 emit(0x16);
3696 emit_sse_operand(dst, src2);
3697 }
3698
vmovhps(Operand dst,XMMRegister src)3699 void Assembler::vmovhps(Operand dst, XMMRegister src) {
3700 DCHECK(IsEnabled(AVX));
3701 EnsureSpace ensure_space(this);
3702 emit_vex_prefix(src, xmm0, dst, kL128, kNoPrefix, k0F, kWIG);
3703 emit(0x17);
3704 emit_sse_operand(src, dst);
3705 }
3706
vinstr(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2,SIMDPrefix pp,LeadingOpcode m,VexW w,CpuFeature feature)3707 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1,
3708 XMMRegister src2, SIMDPrefix pp, LeadingOpcode m, VexW w,
3709 CpuFeature feature) {
3710 DCHECK(IsEnabled(feature));
3711 DCHECK(feature == AVX || feature == AVX2);
3712 EnsureSpace ensure_space(this);
3713 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
3714 emit(op);
3715 emit_sse_operand(dst, src2);
3716 }
3717
vinstr(byte op,XMMRegister dst,XMMRegister src1,Operand src2,SIMDPrefix pp,LeadingOpcode m,VexW w,CpuFeature feature)3718 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1, Operand src2,
3719 SIMDPrefix pp, LeadingOpcode m, VexW w,
3720 CpuFeature feature) {
3721 DCHECK(IsEnabled(feature));
3722 DCHECK(feature == AVX || feature == AVX2);
3723 EnsureSpace ensure_space(this);
3724 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
3725 emit(op);
3726 emit_sse_operand(dst, src2);
3727 }
3728
3729 template <typename Reg1, typename Reg2, typename Op>
vinstr(byte op,Reg1 dst,Reg2 src1,Op src2,SIMDPrefix pp,LeadingOpcode m,VexW w,CpuFeature feature)3730 void Assembler::vinstr(byte op, Reg1 dst, Reg2 src1, Op src2, SIMDPrefix pp,
3731 LeadingOpcode m, VexW w, CpuFeature feature) {
3732 DCHECK(IsEnabled(feature));
3733 DCHECK(feature == AVX || feature == AVX2);
3734 EnsureSpace ensure_space(this);
3735 emit_vex_prefix(dst, src1, src2, kL256, pp, m, w);
3736 emit(op);
3737 emit_sse_operand(dst, src2);
3738 }
3739
3740 template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void Assembler::vinstr(
3741 byte op, YMMRegister dst, YMMRegister src1, YMMRegister src2, SIMDPrefix pp,
3742 LeadingOpcode m, VexW w, CpuFeature feature);
3743 template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void Assembler::vinstr(
3744 byte op, YMMRegister dst, XMMRegister src1, XMMRegister src2, SIMDPrefix pp,
3745 LeadingOpcode m, VexW w, CpuFeature feature);
3746 template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void Assembler::vinstr(
3747 byte op, YMMRegister dst, YMMRegister src1, Operand src2, SIMDPrefix pp,
3748 LeadingOpcode m, VexW w, CpuFeature feature);
3749 template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void Assembler::vinstr(
3750 byte op, YMMRegister dst, YMMRegister src1, XMMRegister src2, SIMDPrefix pp,
3751 LeadingOpcode m, VexW w, CpuFeature feature);
3752 template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void Assembler::vinstr(
3753 byte op, YMMRegister dst, XMMRegister src1, Operand src2, SIMDPrefix pp,
3754 LeadingOpcode m, VexW w, CpuFeature feature);
3755
vps(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)3756 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
3757 XMMRegister src2) {
3758 DCHECK(IsEnabled(AVX));
3759 EnsureSpace ensure_space(this);
3760 emit_vex_prefix(dst, src1, src2, kL128, kNoPrefix, k0F, kWIG);
3761 emit(op);
3762 emit_sse_operand(dst, src2);
3763 }
3764
vps(byte op,YMMRegister dst,YMMRegister src1,YMMRegister src2)3765 void Assembler::vps(byte op, YMMRegister dst, YMMRegister src1,
3766 YMMRegister src2) {
3767 DCHECK(IsEnabled(AVX));
3768 EnsureSpace ensure_space(this);
3769 emit_vex_prefix(dst, src1, src2, kL256, kNoPrefix, k0F, kWIG);
3770 emit(op);
3771 emit_sse_operand(dst, src2);
3772 }
3773
vps(byte op,XMMRegister dst,XMMRegister src1,Operand src2)3774 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
3775 DCHECK(IsEnabled(AVX));
3776 EnsureSpace ensure_space(this);
3777 emit_vex_prefix(dst, src1, src2, kL128, kNoPrefix, k0F, kWIG);
3778 emit(op);
3779 emit_sse_operand(dst, src2);
3780 }
3781
vps(byte op,YMMRegister dst,YMMRegister src1,Operand src2)3782 void Assembler::vps(byte op, YMMRegister dst, YMMRegister src1, Operand src2) {
3783 DCHECK(IsEnabled(AVX));
3784 EnsureSpace ensure_space(this);
3785 emit_vex_prefix(dst, src1, src2, kL256, kNoPrefix, k0F, kWIG);
3786 emit(op);
3787 emit_sse_operand(dst, src2);
3788 }
3789
vps(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2,byte imm8)3790 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
3791 XMMRegister src2, byte imm8) {
3792 DCHECK(IsEnabled(AVX));
3793 EnsureSpace ensure_space(this);
3794 emit_vex_prefix(dst, src1, src2, kL128, kNoPrefix, k0F, kWIG);
3795 emit(op);
3796 emit_sse_operand(dst, src2);
3797 emit(imm8);
3798 }
3799
vps(byte op,YMMRegister dst,YMMRegister src1,YMMRegister src2,byte imm8)3800 void Assembler::vps(byte op, YMMRegister dst, YMMRegister src1,
3801 YMMRegister src2, byte imm8) {
3802 DCHECK(IsEnabled(AVX));
3803 EnsureSpace ensure_space(this);
3804 emit_vex_prefix(dst, src1, src2, kL256, kNoPrefix, k0F, kWIG);
3805 emit(op);
3806 emit_sse_operand(dst, src2);
3807 emit(imm8);
3808 }
3809
3810 #define VPD(SIMDRegister, length) \
3811 void Assembler::vpd(byte op, SIMDRegister dst, SIMDRegister src1, \
3812 SIMDRegister src2) { \
3813 DCHECK(IsEnabled(AVX)); \
3814 EnsureSpace ensure_space(this); \
3815 emit_vex_prefix(dst, src1, src2, k##length, k66, k0F, kWIG); \
3816 emit(op); \
3817 emit_sse_operand(dst, src2); \
3818 } \
3819 \
3820 void Assembler::vpd(byte op, SIMDRegister dst, SIMDRegister src1, \
3821 Operand src2) { \
3822 DCHECK(IsEnabled(AVX)); \
3823 EnsureSpace ensure_space(this); \
3824 emit_vex_prefix(dst, src1, src2, k##length, k66, k0F, kWIG); \
3825 emit(op); \
3826 emit_sse_operand(dst, src2); \
3827 }
VPD(XMMRegister,L128)3828 VPD(XMMRegister, L128)
3829 VPD(YMMRegister, L256)
3830 #undef VPD
3831
3832 void Assembler::vucomiss(XMMRegister dst, XMMRegister src) {
3833 DCHECK(IsEnabled(AVX));
3834 EnsureSpace ensure_space(this);
3835 emit_vex_prefix(dst, xmm0, src, kLIG, kNoPrefix, k0F, kWIG);
3836 emit(0x2E);
3837 emit_sse_operand(dst, src);
3838 }
3839
vucomiss(XMMRegister dst,Operand src)3840 void Assembler::vucomiss(XMMRegister dst, Operand src) {
3841 DCHECK(IsEnabled(AVX));
3842 EnsureSpace ensure_space(this);
3843 emit_vex_prefix(dst, xmm0, src, kLIG, kNoPrefix, k0F, kWIG);
3844 emit(0x2E);
3845 emit_sse_operand(dst, src);
3846 }
3847
vpmovmskb(Register dst,XMMRegister src)3848 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3849 XMMRegister idst = XMMRegister::from_code(dst.code());
3850 DCHECK(IsEnabled(AVX));
3851 EnsureSpace ensure_space(this);
3852 emit_vex_prefix(idst, xmm0, src, kL128, k66, k0F, kWIG);
3853 emit(0xD7);
3854 emit_sse_operand(idst, src);
3855 }
3856
vss(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)3857 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1,
3858 XMMRegister src2) {
3859 DCHECK(IsEnabled(AVX));
3860 EnsureSpace ensure_space(this);
3861 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
3862 emit(op);
3863 emit_sse_operand(dst, src2);
3864 }
3865
vss(byte op,XMMRegister dst,XMMRegister src1,Operand src2)3866 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
3867 DCHECK(IsEnabled(AVX));
3868 EnsureSpace ensure_space(this);
3869 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
3870 emit(op);
3871 emit_sse_operand(dst, src2);
3872 }
3873
bmi1q(byte op,Register reg,Register vreg,Register rm)3874 void Assembler::bmi1q(byte op, Register reg, Register vreg, Register rm) {
3875 DCHECK(IsEnabled(BMI1));
3876 EnsureSpace ensure_space(this);
3877 emit_vex_prefix(reg, vreg, rm, kLZ, kNoPrefix, k0F38, kW1);
3878 emit(op);
3879 emit_modrm(reg, rm);
3880 }
3881
bmi1q(byte op,Register reg,Register vreg,Operand rm)3882 void Assembler::bmi1q(byte op, Register reg, Register vreg, Operand rm) {
3883 DCHECK(IsEnabled(BMI1));
3884 EnsureSpace ensure_space(this);
3885 emit_vex_prefix(reg, vreg, rm, kLZ, kNoPrefix, k0F38, kW1);
3886 emit(op);
3887 emit_operand(reg, rm);
3888 }
3889
bmi1l(byte op,Register reg,Register vreg,Register rm)3890 void Assembler::bmi1l(byte op, Register reg, Register vreg, Register rm) {
3891 DCHECK(IsEnabled(BMI1));
3892 EnsureSpace ensure_space(this);
3893 emit_vex_prefix(reg, vreg, rm, kLZ, kNoPrefix, k0F38, kW0);
3894 emit(op);
3895 emit_modrm(reg, rm);
3896 }
3897
bmi1l(byte op,Register reg,Register vreg,Operand rm)3898 void Assembler::bmi1l(byte op, Register reg, Register vreg, Operand rm) {
3899 DCHECK(IsEnabled(BMI1));
3900 EnsureSpace ensure_space(this);
3901 emit_vex_prefix(reg, vreg, rm, kLZ, kNoPrefix, k0F38, kW0);
3902 emit(op);
3903 emit_operand(reg, rm);
3904 }
3905
tzcntq(Register dst,Register src)3906 void Assembler::tzcntq(Register dst, Register src) {
3907 DCHECK(IsEnabled(BMI1));
3908 EnsureSpace ensure_space(this);
3909 emit(0xF3);
3910 emit_rex_64(dst, src);
3911 emit(0x0F);
3912 emit(0xBC);
3913 emit_modrm(dst, src);
3914 }
3915
tzcntq(Register dst,Operand src)3916 void Assembler::tzcntq(Register dst, Operand src) {
3917 DCHECK(IsEnabled(BMI1));
3918 EnsureSpace ensure_space(this);
3919 emit(0xF3);
3920 emit_rex_64(dst, src);
3921 emit(0x0F);
3922 emit(0xBC);
3923 emit_operand(dst, src);
3924 }
3925
tzcntl(Register dst,Register src)3926 void Assembler::tzcntl(Register dst, Register src) {
3927 DCHECK(IsEnabled(BMI1));
3928 EnsureSpace ensure_space(this);
3929 emit(0xF3);
3930 emit_optional_rex_32(dst, src);
3931 emit(0x0F);
3932 emit(0xBC);
3933 emit_modrm(dst, src);
3934 }
3935
tzcntl(Register dst,Operand src)3936 void Assembler::tzcntl(Register dst, Operand src) {
3937 DCHECK(IsEnabled(BMI1));
3938 EnsureSpace ensure_space(this);
3939 emit(0xF3);
3940 emit_optional_rex_32(dst, src);
3941 emit(0x0F);
3942 emit(0xBC);
3943 emit_operand(dst, src);
3944 }
3945
lzcntq(Register dst,Register src)3946 void Assembler::lzcntq(Register dst, Register src) {
3947 DCHECK(IsEnabled(LZCNT));
3948 EnsureSpace ensure_space(this);
3949 emit(0xF3);
3950 emit_rex_64(dst, src);
3951 emit(0x0F);
3952 emit(0xBD);
3953 emit_modrm(dst, src);
3954 }
3955
lzcntq(Register dst,Operand src)3956 void Assembler::lzcntq(Register dst, Operand src) {
3957 DCHECK(IsEnabled(LZCNT));
3958 EnsureSpace ensure_space(this);
3959 emit(0xF3);
3960 emit_rex_64(dst, src);
3961 emit(0x0F);
3962 emit(0xBD);
3963 emit_operand(dst, src);
3964 }
3965
lzcntl(Register dst,Register src)3966 void Assembler::lzcntl(Register dst, Register src) {
3967 DCHECK(IsEnabled(LZCNT));
3968 EnsureSpace ensure_space(this);
3969 emit(0xF3);
3970 emit_optional_rex_32(dst, src);
3971 emit(0x0F);
3972 emit(0xBD);
3973 emit_modrm(dst, src);
3974 }
3975
lzcntl(Register dst,Operand src)3976 void Assembler::lzcntl(Register dst, Operand src) {
3977 DCHECK(IsEnabled(LZCNT));
3978 EnsureSpace ensure_space(this);
3979 emit(0xF3);
3980 emit_optional_rex_32(dst, src);
3981 emit(0x0F);
3982 emit(0xBD);
3983 emit_operand(dst, src);
3984 }
3985
popcntq(Register dst,Register src)3986 void Assembler::popcntq(Register dst, Register src) {
3987 DCHECK(IsEnabled(POPCNT));
3988 EnsureSpace ensure_space(this);
3989 emit(0xF3);
3990 emit_rex_64(dst, src);
3991 emit(0x0F);
3992 emit(0xB8);
3993 emit_modrm(dst, src);
3994 }
3995
popcntq(Register dst,Operand src)3996 void Assembler::popcntq(Register dst, Operand src) {
3997 DCHECK(IsEnabled(POPCNT));
3998 EnsureSpace ensure_space(this);
3999 emit(0xF3);
4000 emit_rex_64(dst, src);
4001 emit(0x0F);
4002 emit(0xB8);
4003 emit_operand(dst, src);
4004 }
4005
popcntl(Register dst,Register src)4006 void Assembler::popcntl(Register dst, Register src) {
4007 DCHECK(IsEnabled(POPCNT));
4008 EnsureSpace ensure_space(this);
4009 emit(0xF3);
4010 emit_optional_rex_32(dst, src);
4011 emit(0x0F);
4012 emit(0xB8);
4013 emit_modrm(dst, src);
4014 }
4015
popcntl(Register dst,Operand src)4016 void Assembler::popcntl(Register dst, Operand src) {
4017 DCHECK(IsEnabled(POPCNT));
4018 EnsureSpace ensure_space(this);
4019 emit(0xF3);
4020 emit_optional_rex_32(dst, src);
4021 emit(0x0F);
4022 emit(0xB8);
4023 emit_operand(dst, src);
4024 }
4025
bmi2q(SIMDPrefix pp,byte op,Register reg,Register vreg,Register rm)4026 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
4027 Register rm) {
4028 DCHECK(IsEnabled(BMI2));
4029 EnsureSpace ensure_space(this);
4030 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
4031 emit(op);
4032 emit_modrm(reg, rm);
4033 }
4034
bmi2q(SIMDPrefix pp,byte op,Register reg,Register vreg,Operand rm)4035 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
4036 Operand rm) {
4037 DCHECK(IsEnabled(BMI2));
4038 EnsureSpace ensure_space(this);
4039 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
4040 emit(op);
4041 emit_operand(reg, rm);
4042 }
4043
bmi2l(SIMDPrefix pp,byte op,Register reg,Register vreg,Register rm)4044 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
4045 Register rm) {
4046 DCHECK(IsEnabled(BMI2));
4047 EnsureSpace ensure_space(this);
4048 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
4049 emit(op);
4050 emit_modrm(reg, rm);
4051 }
4052
bmi2l(SIMDPrefix pp,byte op,Register reg,Register vreg,Operand rm)4053 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
4054 Operand rm) {
4055 DCHECK(IsEnabled(BMI2));
4056 EnsureSpace ensure_space(this);
4057 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
4058 emit(op);
4059 emit_operand(reg, rm);
4060 }
4061
rorxq(Register dst,Register src,byte imm8)4062 void Assembler::rorxq(Register dst, Register src, byte imm8) {
4063 DCHECK(IsEnabled(BMI2));
4064 DCHECK(is_uint8(imm8));
4065 Register vreg = Register::from_code(0); // VEX.vvvv unused
4066 EnsureSpace ensure_space(this);
4067 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
4068 emit(0xF0);
4069 emit_modrm(dst, src);
4070 emit(imm8);
4071 }
4072
rorxq(Register dst,Operand src,byte imm8)4073 void Assembler::rorxq(Register dst, Operand src, byte imm8) {
4074 DCHECK(IsEnabled(BMI2));
4075 DCHECK(is_uint8(imm8));
4076 Register vreg = Register::from_code(0); // VEX.vvvv unused
4077 EnsureSpace ensure_space(this);
4078 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
4079 emit(0xF0);
4080 emit_operand(dst, src);
4081 emit(imm8);
4082 }
4083
rorxl(Register dst,Register src,byte imm8)4084 void Assembler::rorxl(Register dst, Register src, byte imm8) {
4085 DCHECK(IsEnabled(BMI2));
4086 DCHECK(is_uint8(imm8));
4087 Register vreg = Register::from_code(0); // VEX.vvvv unused
4088 EnsureSpace ensure_space(this);
4089 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
4090 emit(0xF0);
4091 emit_modrm(dst, src);
4092 emit(imm8);
4093 }
4094
rorxl(Register dst,Operand src,byte imm8)4095 void Assembler::rorxl(Register dst, Operand src, byte imm8) {
4096 DCHECK(IsEnabled(BMI2));
4097 DCHECK(is_uint8(imm8));
4098 Register vreg = Register::from_code(0); // VEX.vvvv unused
4099 EnsureSpace ensure_space(this);
4100 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
4101 emit(0xF0);
4102 emit_operand(dst, src);
4103 emit(imm8);
4104 }
4105
pause()4106 void Assembler::pause() {
4107 emit(0xF3);
4108 emit(0x90);
4109 }
4110
movups(XMMRegister dst,XMMRegister src)4111 void Assembler::movups(XMMRegister dst, XMMRegister src) {
4112 EnsureSpace ensure_space(this);
4113 if (src.low_bits() == 4) {
4114 // Try to avoid an unnecessary SIB byte.
4115 emit_optional_rex_32(src, dst);
4116 emit(0x0F);
4117 emit(0x11);
4118 emit_sse_operand(src, dst);
4119 } else {
4120 emit_optional_rex_32(dst, src);
4121 emit(0x0F);
4122 emit(0x10);
4123 emit_sse_operand(dst, src);
4124 }
4125 }
4126
movups(XMMRegister dst,Operand src)4127 void Assembler::movups(XMMRegister dst, Operand src) {
4128 EnsureSpace ensure_space(this);
4129 emit_optional_rex_32(dst, src);
4130 emit(0x0F);
4131 emit(0x10);
4132 emit_sse_operand(dst, src);
4133 }
4134
movups(Operand dst,XMMRegister src)4135 void Assembler::movups(Operand dst, XMMRegister src) {
4136 EnsureSpace ensure_space(this);
4137 emit_optional_rex_32(src, dst);
4138 emit(0x0F);
4139 emit(0x11);
4140 emit_sse_operand(src, dst);
4141 }
4142
sse_instr(XMMRegister dst,XMMRegister src,byte escape,byte opcode)4143 void Assembler::sse_instr(XMMRegister dst, XMMRegister src, byte escape,
4144 byte opcode) {
4145 EnsureSpace ensure_space(this);
4146 emit_optional_rex_32(dst, src);
4147 emit(escape);
4148 emit(opcode);
4149 emit_sse_operand(dst, src);
4150 }
4151
sse_instr(XMMRegister dst,Operand src,byte escape,byte opcode)4152 void Assembler::sse_instr(XMMRegister dst, Operand src, byte escape,
4153 byte opcode) {
4154 EnsureSpace ensure_space(this);
4155 emit_optional_rex_32(dst, src);
4156 emit(escape);
4157 emit(opcode);
4158 emit_sse_operand(dst, src);
4159 }
4160
sse2_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape,byte opcode)4161 void Assembler::sse2_instr(XMMRegister dst, XMMRegister src, byte prefix,
4162 byte escape, byte opcode) {
4163 EnsureSpace ensure_space(this);
4164 emit(prefix);
4165 emit_optional_rex_32(dst, src);
4166 emit(escape);
4167 emit(opcode);
4168 emit_sse_operand(dst, src);
4169 }
4170
sse2_instr(XMMRegister dst,Operand src,byte prefix,byte escape,byte opcode)4171 void Assembler::sse2_instr(XMMRegister dst, Operand src, byte prefix,
4172 byte escape, byte opcode) {
4173 EnsureSpace ensure_space(this);
4174 emit(prefix);
4175 emit_optional_rex_32(dst, src);
4176 emit(escape);
4177 emit(opcode);
4178 emit_sse_operand(dst, src);
4179 }
4180
ssse3_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode)4181 void Assembler::ssse3_instr(XMMRegister dst, XMMRegister src, byte prefix,
4182 byte escape1, byte escape2, byte opcode) {
4183 DCHECK(IsEnabled(SSSE3));
4184 EnsureSpace ensure_space(this);
4185 emit(prefix);
4186 emit_optional_rex_32(dst, src);
4187 emit(escape1);
4188 emit(escape2);
4189 emit(opcode);
4190 emit_sse_operand(dst, src);
4191 }
4192
ssse3_instr(XMMRegister dst,Operand src,byte prefix,byte escape1,byte escape2,byte opcode)4193 void Assembler::ssse3_instr(XMMRegister dst, Operand src, byte prefix,
4194 byte escape1, byte escape2, byte opcode) {
4195 DCHECK(IsEnabled(SSSE3));
4196 EnsureSpace ensure_space(this);
4197 emit(prefix);
4198 emit_optional_rex_32(dst, src);
4199 emit(escape1);
4200 emit(escape2);
4201 emit(opcode);
4202 emit_sse_operand(dst, src);
4203 }
4204
sse4_instr(XMMRegister dst,Register src,byte prefix,byte escape1,byte escape2,byte opcode,int8_t imm8)4205 void Assembler::sse4_instr(XMMRegister dst, Register src, byte prefix,
4206 byte escape1, byte escape2, byte opcode,
4207 int8_t imm8) {
4208 DCHECK(is_uint8(imm8));
4209 DCHECK(IsEnabled(SSE4_1));
4210 EnsureSpace ensure_space(this);
4211 emit(prefix);
4212 emit_optional_rex_32(dst, src);
4213 emit(escape1);
4214 emit(escape2);
4215 emit(opcode);
4216 emit_sse_operand(dst, src);
4217 emit(imm8);
4218 }
4219
sse4_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode)4220 void Assembler::sse4_instr(XMMRegister dst, XMMRegister src, byte prefix,
4221 byte escape1, byte escape2, byte opcode) {
4222 DCHECK(IsEnabled(SSE4_1));
4223 EnsureSpace ensure_space(this);
4224 emit(prefix);
4225 emit_optional_rex_32(dst, src);
4226 emit(escape1);
4227 emit(escape2);
4228 emit(opcode);
4229 emit_sse_operand(dst, src);
4230 }
4231
sse4_instr(XMMRegister dst,Operand src,byte prefix,byte escape1,byte escape2,byte opcode)4232 void Assembler::sse4_instr(XMMRegister dst, Operand src, byte prefix,
4233 byte escape1, byte escape2, byte opcode) {
4234 DCHECK(IsEnabled(SSE4_1));
4235 EnsureSpace ensure_space(this);
4236 emit(prefix);
4237 emit_optional_rex_32(dst, src);
4238 emit(escape1);
4239 emit(escape2);
4240 emit(opcode);
4241 emit_sse_operand(dst, src);
4242 }
4243
sse4_instr(Register dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode,int8_t imm8)4244 void Assembler::sse4_instr(Register dst, XMMRegister src, byte prefix,
4245 byte escape1, byte escape2, byte opcode,
4246 int8_t imm8) {
4247 DCHECK(is_uint8(imm8));
4248 DCHECK(IsEnabled(SSE4_1));
4249 EnsureSpace ensure_space(this);
4250 emit(prefix);
4251 emit_optional_rex_32(src, dst);
4252 emit(escape1);
4253 emit(escape2);
4254 emit(opcode);
4255 emit_sse_operand(src, dst);
4256 emit(imm8);
4257 }
4258
sse4_instr(Operand dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode,int8_t imm8)4259 void Assembler::sse4_instr(Operand dst, XMMRegister src, byte prefix,
4260 byte escape1, byte escape2, byte opcode,
4261 int8_t imm8) {
4262 DCHECK(is_uint8(imm8));
4263 DCHECK(IsEnabled(SSE4_1));
4264 EnsureSpace ensure_space(this);
4265 emit(prefix);
4266 emit_optional_rex_32(src, dst);
4267 emit(escape1);
4268 emit(escape2);
4269 emit(opcode);
4270 emit_sse_operand(src, dst);
4271 emit(imm8);
4272 }
4273
sse4_2_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode)4274 void Assembler::sse4_2_instr(XMMRegister dst, XMMRegister src, byte prefix,
4275 byte escape1, byte escape2, byte opcode) {
4276 DCHECK(IsEnabled(SSE4_2));
4277 EnsureSpace ensure_space(this);
4278 emit(prefix);
4279 emit_optional_rex_32(dst, src);
4280 emit(escape1);
4281 emit(escape2);
4282 emit(opcode);
4283 emit_sse_operand(dst, src);
4284 }
4285
sse4_2_instr(XMMRegister dst,Operand src,byte prefix,byte escape1,byte escape2,byte opcode)4286 void Assembler::sse4_2_instr(XMMRegister dst, Operand src, byte prefix,
4287 byte escape1, byte escape2, byte opcode) {
4288 DCHECK(IsEnabled(SSE4_2));
4289 EnsureSpace ensure_space(this);
4290 emit(prefix);
4291 emit_optional_rex_32(dst, src);
4292 emit(escape1);
4293 emit(escape2);
4294 emit(opcode);
4295 emit_sse_operand(dst, src);
4296 }
4297
lddqu(XMMRegister dst,Operand src)4298 void Assembler::lddqu(XMMRegister dst, Operand src) {
4299 DCHECK(IsEnabled(SSE3));
4300 EnsureSpace ensure_space(this);
4301 emit(0xF2);
4302 emit_optional_rex_32(dst, src);
4303 emit(0x0F);
4304 emit(0xF0);
4305 emit_sse_operand(dst, src);
4306 }
4307
movddup(XMMRegister dst,XMMRegister src)4308 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
4309 DCHECK(IsEnabled(SSE3));
4310 EnsureSpace ensure_space(this);
4311 emit(0xF2);
4312 emit_optional_rex_32(dst, src);
4313 emit(0x0F);
4314 emit(0x12);
4315 emit_sse_operand(dst, src);
4316 }
4317
movddup(XMMRegister dst,Operand src)4318 void Assembler::movddup(XMMRegister dst, Operand src) {
4319 DCHECK(IsEnabled(SSE3));
4320 EnsureSpace ensure_space(this);
4321 emit(0xF2);
4322 emit_optional_rex_32(dst, src);
4323 emit(0x0F);
4324 emit(0x12);
4325 emit_sse_operand(dst, src);
4326 }
4327
movshdup(XMMRegister dst,XMMRegister src)4328 void Assembler::movshdup(XMMRegister dst, XMMRegister src) {
4329 DCHECK(IsEnabled(SSE3));
4330 EnsureSpace ensure_space(this);
4331 emit(0xF3);
4332 emit_optional_rex_32(dst, src);
4333 emit(0x0F);
4334 emit(0x16);
4335 emit_sse_operand(dst, src);
4336 }
4337
psrldq(XMMRegister dst,uint8_t shift)4338 void Assembler::psrldq(XMMRegister dst, uint8_t shift) {
4339 EnsureSpace ensure_space(this);
4340 emit(0x66);
4341 emit_optional_rex_32(dst);
4342 emit(0x0F);
4343 emit(0x73);
4344 emit_sse_operand(dst);
4345 emit(shift);
4346 }
4347
pshufhw(XMMRegister dst,XMMRegister src,uint8_t shuffle)4348 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4349 EnsureSpace ensure_space(this);
4350 emit(0xF3);
4351 emit_optional_rex_32(dst, src);
4352 emit(0x0F);
4353 emit(0x70);
4354 emit_sse_operand(dst, src);
4355 emit(shuffle);
4356 }
4357
pshufhw(XMMRegister dst,Operand src,uint8_t shuffle)4358 void Assembler::pshufhw(XMMRegister dst, Operand src, uint8_t shuffle) {
4359 EnsureSpace ensure_space(this);
4360 emit(0xF3);
4361 emit_optional_rex_32(dst, src);
4362 emit(0x0F);
4363 emit(0x70);
4364 emit_sse_operand(dst, src);
4365 emit(shuffle);
4366 }
4367
pshuflw(XMMRegister dst,XMMRegister src,uint8_t shuffle)4368 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4369 EnsureSpace ensure_space(this);
4370 emit(0xF2);
4371 emit_optional_rex_32(dst, src);
4372 emit(0x0F);
4373 emit(0x70);
4374 emit_sse_operand(dst, src);
4375 emit(shuffle);
4376 }
4377
pshuflw(XMMRegister dst,Operand src,uint8_t shuffle)4378 void Assembler::pshuflw(XMMRegister dst, Operand src, uint8_t shuffle) {
4379 EnsureSpace ensure_space(this);
4380 emit(0xF2);
4381 emit_optional_rex_32(dst, src);
4382 emit(0x0F);
4383 emit(0x70);
4384 emit_sse_operand(dst, src);
4385 emit(shuffle);
4386 }
4387
pshufd(XMMRegister dst,XMMRegister src,uint8_t shuffle)4388 void Assembler::pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4389 EnsureSpace ensure_space(this);
4390 emit(0x66);
4391 emit_optional_rex_32(dst, src);
4392 emit(0x0F);
4393 emit(0x70);
4394 emit_sse_operand(dst, src);
4395 emit(shuffle);
4396 }
4397
pshufd(XMMRegister dst,Operand src,uint8_t shuffle)4398 void Assembler::pshufd(XMMRegister dst, Operand src, uint8_t shuffle) {
4399 EnsureSpace ensure_space(this);
4400 emit(0x66);
4401 emit_optional_rex_32(dst, src);
4402 emit(0x0F);
4403 emit(0x70);
4404 emit_sse_operand(dst, src);
4405 emit(shuffle);
4406 }
4407
emit_sse_operand(XMMRegister reg,Operand adr)4408 void Assembler::emit_sse_operand(XMMRegister reg, Operand adr) {
4409 Register ireg = Register::from_code(reg.code());
4410 emit_operand(ireg, adr);
4411 }
4412
emit_sse_operand(Register reg,Operand adr)4413 void Assembler::emit_sse_operand(Register reg, Operand adr) {
4414 emit_operand(reg, adr);
4415 }
4416
emit_sse_operand(XMMRegister dst,XMMRegister src)4417 void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) {
4418 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4419 }
4420
emit_sse_operand(XMMRegister dst,Register src)4421 void Assembler::emit_sse_operand(XMMRegister dst, Register src) {
4422 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4423 }
4424
emit_sse_operand(Register dst,XMMRegister src)4425 void Assembler::emit_sse_operand(Register dst, XMMRegister src) {
4426 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4427 }
4428
emit_sse_operand(XMMRegister dst)4429 void Assembler::emit_sse_operand(XMMRegister dst) {
4430 emit(0xD8 | dst.low_bits());
4431 }
4432
db(uint8_t data)4433 void Assembler::db(uint8_t data) {
4434 EnsureSpace ensure_space(this);
4435 emit(data);
4436 }
4437
dd(uint32_t data,RelocInfo::Mode rmode)4438 void Assembler::dd(uint32_t data, RelocInfo::Mode rmode) {
4439 EnsureSpace ensure_space(this);
4440 if (!RelocInfo::IsNoInfo(rmode)) {
4441 DCHECK(RelocInfo::IsDataEmbeddedObject(rmode) ||
4442 RelocInfo::IsLiteralConstant(rmode));
4443 RecordRelocInfo(rmode);
4444 }
4445 emitl(data);
4446 }
4447
dq(uint64_t data,RelocInfo::Mode rmode)4448 void Assembler::dq(uint64_t data, RelocInfo::Mode rmode) {
4449 EnsureSpace ensure_space(this);
4450 if (!RelocInfo::IsNoInfo(rmode)) {
4451 DCHECK(RelocInfo::IsDataEmbeddedObject(rmode) ||
4452 RelocInfo::IsLiteralConstant(rmode));
4453 RecordRelocInfo(rmode);
4454 }
4455 emitq(data);
4456 }
4457
dq(Label * label)4458 void Assembler::dq(Label* label) {
4459 EnsureSpace ensure_space(this);
4460 if (label->is_bound()) {
4461 internal_reference_positions_.push_back(pc_offset());
4462 emit(Immediate64(reinterpret_cast<Address>(buffer_start_) + label->pos(),
4463 RelocInfo::INTERNAL_REFERENCE));
4464 } else {
4465 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
4466 emitl(0); // Zero for the first 32bit marks it as 64bit absolute address.
4467 if (label->is_linked()) {
4468 emitl(label->pos());
4469 label->link_to(pc_offset() - sizeof(int32_t));
4470 } else {
4471 DCHECK(label->is_unused());
4472 int32_t current = pc_offset();
4473 emitl(current);
4474 label->link_to(current);
4475 }
4476 }
4477 }
4478
4479 // Relocation information implementations.
4480
RecordRelocInfo(RelocInfo::Mode rmode,intptr_t data)4481 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
4482 if (!ShouldRecordRelocInfo(rmode)) return;
4483 RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code());
4484 reloc_info_writer.Write(&rinfo);
4485 }
4486
4487 const int RelocInfo::kApplyMask =
4488 RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
4489 RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY) |
4490 RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
4491 RelocInfo::ModeMask(RelocInfo::WASM_CALL);
4492
IsCodedSpecially()4493 bool RelocInfo::IsCodedSpecially() {
4494 // The deserializer needs to know whether a pointer is specially coded. Being
4495 // specially coded on x64 means that it is a relative 32 bit address, as used
4496 // by branch instructions.
4497 return (1 << rmode_) & kApplyMask;
4498 }
4499
IsInConstantPool()4500 bool RelocInfo::IsInConstantPool() { return false; }
4501
4502 } // namespace internal
4503 } // namespace v8
4504
4505 #endif // V8_TARGET_ARCH_X64
4506