// Copyright 2013 the V8 project authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "src/compiler/code-generator.h" #include #include "src/compiler/code-generator-impl.h" #include "src/compiler/gap-resolver.h" #include "src/compiler/node-matchers.h" #include "src/compiler/osr.h" #include "src/heap/heap-inl.h" #include "src/optimized-compilation-info.h" #include "src/wasm/wasm-code-manager.h" #include "src/wasm/wasm-objects.h" #include "src/x64/assembler-x64.h" #include "src/x64/macro-assembler-x64.h" namespace v8 { namespace internal { namespace compiler { #define __ tasm()-> // Adds X64 specific methods for decoding operands. class X64OperandConverter : public InstructionOperandConverter { public: X64OperandConverter(CodeGenerator* gen, Instruction* instr) : InstructionOperandConverter(gen, instr) {} Immediate InputImmediate(size_t index) { return ToImmediate(instr_->InputAt(index)); } Operand InputOperand(size_t index, int extra = 0) { return ToOperand(instr_->InputAt(index), extra); } Operand OutputOperand() { return ToOperand(instr_->Output()); } Immediate ToImmediate(InstructionOperand* operand) { Constant constant = ToConstant(operand); if (constant.type() == Constant::kFloat64) { DCHECK_EQ(0, constant.ToFloat64().AsUint64()); return Immediate(0); } if (RelocInfo::IsWasmReference(constant.rmode())) { return Immediate(constant.ToInt32(), constant.rmode()); } return Immediate(constant.ToInt32()); } Operand ToOperand(InstructionOperand* op, int extra = 0) { DCHECK(op->IsStackSlot() || op->IsFPStackSlot()); return SlotToOperand(AllocatedOperand::cast(op)->index(), extra); } Operand SlotToOperand(int slot_index, int extra = 0) { FrameOffset offset = frame_access_state()->GetFrameOffset(slot_index); return Operand(offset.from_stack_pointer() ? rsp : rbp, offset.offset() + extra); } static size_t NextOffset(size_t* offset) { size_t i = *offset; (*offset)++; return i; } static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) { STATIC_ASSERT(0 == static_cast(times_1)); STATIC_ASSERT(1 == static_cast(times_2)); STATIC_ASSERT(2 == static_cast(times_4)); STATIC_ASSERT(3 == static_cast(times_8)); int scale = static_cast(mode - one); DCHECK(scale >= 0 && scale < 4); return static_cast(scale); } Operand MemoryOperand(size_t* offset) { AddressingMode mode = AddressingModeField::decode(instr_->opcode()); switch (mode) { case kMode_MR: { Register base = InputRegister(NextOffset(offset)); int32_t disp = 0; return Operand(base, disp); } case kMode_MRI: { Register base = InputRegister(NextOffset(offset)); int32_t disp = InputInt32(NextOffset(offset)); return Operand(base, disp); } case kMode_MR1: case kMode_MR2: case kMode_MR4: case kMode_MR8: { Register base = InputRegister(NextOffset(offset)); Register index = InputRegister(NextOffset(offset)); ScaleFactor scale = ScaleFor(kMode_MR1, mode); int32_t disp = 0; return Operand(base, index, scale, disp); } case kMode_MR1I: case kMode_MR2I: case kMode_MR4I: case kMode_MR8I: { Register base = InputRegister(NextOffset(offset)); Register index = InputRegister(NextOffset(offset)); ScaleFactor scale = ScaleFor(kMode_MR1I, mode); int32_t disp = InputInt32(NextOffset(offset)); return Operand(base, index, scale, disp); } case kMode_M1: { Register base = InputRegister(NextOffset(offset)); int32_t disp = 0; return Operand(base, disp); } case kMode_M2: UNREACHABLE(); // Should use kModeMR with more compact encoding instead return Operand(no_reg, 0); case kMode_M4: case kMode_M8: { Register index = InputRegister(NextOffset(offset)); ScaleFactor scale = ScaleFor(kMode_M1, mode); int32_t disp = 0; return Operand(index, scale, disp); } case kMode_M1I: case kMode_M2I: case kMode_M4I: case kMode_M8I: { Register index = InputRegister(NextOffset(offset)); ScaleFactor scale = ScaleFor(kMode_M1I, mode); int32_t disp = InputInt32(NextOffset(offset)); return Operand(index, scale, disp); } case kMode_Root: { Register base = kRootRegister; int32_t disp = InputInt32(NextOffset(offset)); return Operand(base, disp); } case kMode_None: UNREACHABLE(); } UNREACHABLE(); } Operand MemoryOperand(size_t first_input = 0) { return MemoryOperand(&first_input); } }; namespace { bool HasImmediateInput(Instruction* instr, size_t index) { return instr->InputAt(index)->IsImmediate(); } class OutOfLineLoadFloat32NaN final : public OutOfLineCode { public: OutOfLineLoadFloat32NaN(CodeGenerator* gen, XMMRegister result) : OutOfLineCode(gen), result_(result) {} void Generate() final { __ Xorps(result_, result_); __ Divss(result_, result_); } private: XMMRegister const result_; }; class OutOfLineLoadFloat64NaN final : public OutOfLineCode { public: OutOfLineLoadFloat64NaN(CodeGenerator* gen, XMMRegister result) : OutOfLineCode(gen), result_(result) {} void Generate() final { __ Xorpd(result_, result_); __ Divsd(result_, result_); } private: XMMRegister const result_; }; class OutOfLineTruncateDoubleToI final : public OutOfLineCode { public: OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result, XMMRegister input, StubCallMode stub_mode, UnwindingInfoWriter* unwinding_info_writer) : OutOfLineCode(gen), result_(result), input_(input), stub_mode_(stub_mode), unwinding_info_writer_(unwinding_info_writer), isolate_(gen->isolate()), zone_(gen->zone()) {} void Generate() final { __ subp(rsp, Immediate(kDoubleSize)); unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(), kDoubleSize); __ Movsd(MemOperand(rsp, 0), input_); if (stub_mode_ == StubCallMode::kCallWasmRuntimeStub) { // A direct call to a wasm runtime stub defined in this module. // Just encode the stub index. This will be patched at relocation. __ near_call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL); } else { __ Call(BUILTIN_CODE(isolate_, DoubleToI), RelocInfo::CODE_TARGET); } __ movl(result_, MemOperand(rsp, 0)); __ addp(rsp, Immediate(kDoubleSize)); unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(), -kDoubleSize); } private: Register const result_; XMMRegister const input_; StubCallMode stub_mode_; UnwindingInfoWriter* const unwinding_info_writer_; Isolate* isolate_; Zone* zone_; }; class OutOfLineRecordWrite final : public OutOfLineCode { public: OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand operand, Register value, Register scratch0, Register scratch1, RecordWriteMode mode) : OutOfLineCode(gen), object_(object), operand_(operand), value_(value), scratch0_(scratch0), scratch1_(scratch1), mode_(mode), zone_(gen->zone()) {} void Generate() final { if (mode_ > RecordWriteMode::kValueIsPointer) { __ JumpIfSmi(value_, exit()); } __ CheckPageFlag(value_, scratch0_, MemoryChunk::kPointersToHereAreInterestingMask, zero, exit()); __ leap(scratch1_, operand_); RememberedSetAction const remembered_set_action = mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET : OMIT_REMEMBERED_SET; SaveFPRegsMode const save_fp_mode = frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs; __ CallRecordWriteStub(object_, scratch1_, remembered_set_action, save_fp_mode); } private: Register const object_; Operand const operand_; Register const value_; Register const scratch0_; Register const scratch1_; RecordWriteMode const mode_; Zone* zone_; }; class WasmOutOfLineTrap : public OutOfLineCode { public: WasmOutOfLineTrap(CodeGenerator* gen, Instruction* instr) : OutOfLineCode(gen), gen_(gen), instr_(instr) {} void Generate() override { X64OperandConverter i(gen_, instr_); TrapId trap_id = static_cast(i.InputInt32(instr_->InputCount() - 1)); GenerateWithTrapId(trap_id); } protected: CodeGenerator* gen_; void GenerateWithTrapId(TrapId trap_id) { GenerateCallToTrap(trap_id); } private: void GenerateCallToTrap(TrapId trap_id) { if (!gen_->wasm_runtime_exception_support()) { // We cannot test calls to the runtime in cctest/test-run-wasm. // Therefore we emit a call to C here instead of a call to the runtime. __ PrepareCallCFunction(0); __ CallCFunction(ExternalReference::wasm_call_trap_callback_for_testing(), 0); __ LeaveFrame(StackFrame::WASM_COMPILED); auto call_descriptor = gen_->linkage()->GetIncomingDescriptor(); size_t pop_size = call_descriptor->StackParameterCount() * kPointerSize; // Use rcx as a scratch register, we return anyways immediately. __ Ret(static_cast(pop_size), rcx); } else { gen_->AssembleSourcePosition(instr_); // A direct call to a wasm runtime stub defined in this module. // Just encode the stub index. This will be patched at relocation. __ near_call(static_cast
(trap_id), RelocInfo::WASM_STUB_CALL); ReferenceMap* reference_map = new (gen_->zone()) ReferenceMap(gen_->zone()); gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0, Safepoint::kNoLazyDeopt); __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap); } } Instruction* instr_; }; class WasmProtectedInstructionTrap final : public WasmOutOfLineTrap { public: WasmProtectedInstructionTrap(CodeGenerator* gen, int pc, Instruction* instr) : WasmOutOfLineTrap(gen, instr), pc_(pc) {} void Generate() final { gen_->AddProtectedInstructionLanding(pc_, __ pc_offset()); GenerateWithTrapId(TrapId::kTrapMemOutOfBounds); } private: int pc_; }; void EmitOOLTrapIfNeeded(Zone* zone, CodeGenerator* codegen, InstructionCode opcode, Instruction* instr, X64OperandConverter& i, int pc) { const MemoryAccessMode access_mode = static_cast(MiscField::decode(opcode)); if (access_mode == kMemoryAccessProtected) { new (zone) WasmProtectedInstructionTrap(codegen, pc, instr); } } void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen, InstructionCode opcode, Instruction* instr, X64OperandConverter& i) { const MemoryAccessMode access_mode = static_cast(MiscField::decode(opcode)); if (access_mode == kMemoryAccessPoisoned) { Register value = i.OutputRegister(); codegen->tasm()->andq(value, kSpeculationPoisonRegister); } } } // namespace #define ASSEMBLE_UNOP(asm_instr) \ do { \ if (instr->Output()->IsRegister()) { \ __ asm_instr(i.OutputRegister()); \ } else { \ __ asm_instr(i.OutputOperand()); \ } \ } while (0) #define ASSEMBLE_BINOP(asm_instr) \ do { \ if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \ size_t index = 1; \ Operand right = i.MemoryOperand(&index); \ __ asm_instr(i.InputRegister(0), right); \ } else { \ if (HasImmediateInput(instr, 1)) { \ if (instr->InputAt(0)->IsRegister()) { \ __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \ } else { \ __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \ } \ } else { \ if (instr->InputAt(1)->IsRegister()) { \ __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \ } else { \ __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \ } \ } \ } \ } while (0) #define ASSEMBLE_COMPARE(asm_instr) \ do { \ if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \ size_t index = 0; \ Operand left = i.MemoryOperand(&index); \ if (HasImmediateInput(instr, index)) { \ __ asm_instr(left, i.InputImmediate(index)); \ } else { \ __ asm_instr(left, i.InputRegister(index)); \ } \ } else { \ if (HasImmediateInput(instr, 1)) { \ if (instr->InputAt(0)->IsRegister()) { \ __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \ } else { \ __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \ } \ } else { \ if (instr->InputAt(1)->IsRegister()) { \ __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \ } else { \ __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \ } \ } \ } \ } while (0) #define ASSEMBLE_MULT(asm_instr) \ do { \ if (HasImmediateInput(instr, 1)) { \ if (instr->InputAt(0)->IsRegister()) { \ __ asm_instr(i.OutputRegister(), i.InputRegister(0), \ i.InputImmediate(1)); \ } else { \ __ asm_instr(i.OutputRegister(), i.InputOperand(0), \ i.InputImmediate(1)); \ } \ } else { \ if (instr->InputAt(1)->IsRegister()) { \ __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \ } else { \ __ asm_instr(i.OutputRegister(), i.InputOperand(1)); \ } \ } \ } while (0) #define ASSEMBLE_SHIFT(asm_instr, width) \ do { \ if (HasImmediateInput(instr, 1)) { \ if (instr->Output()->IsRegister()) { \ __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \ } else { \ __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1))); \ } \ } else { \ if (instr->Output()->IsRegister()) { \ __ asm_instr##_cl(i.OutputRegister()); \ } else { \ __ asm_instr##_cl(i.OutputOperand()); \ } \ } \ } while (0) #define ASSEMBLE_MOVX(asm_instr) \ do { \ if (instr->addressing_mode() != kMode_None) { \ __ asm_instr(i.OutputRegister(), i.MemoryOperand()); \ } else if (instr->InputAt(0)->IsRegister()) { \ __ asm_instr(i.OutputRegister(), i.InputRegister(0)); \ } else { \ __ asm_instr(i.OutputRegister(), i.InputOperand(0)); \ } \ } while (0) #define ASSEMBLE_SSE_BINOP(asm_instr) \ do { \ if (instr->InputAt(1)->IsFPRegister()) { \ __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \ } else { \ __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1)); \ } \ } while (0) #define ASSEMBLE_SSE_UNOP(asm_instr) \ do { \ if (instr->InputAt(0)->IsFPRegister()) { \ __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); \ } else { \ __ asm_instr(i.OutputDoubleRegister(), i.InputOperand(0)); \ } \ } while (0) #define ASSEMBLE_AVX_BINOP(asm_instr) \ do { \ CpuFeatureScope avx_scope(tasm(), AVX); \ if (instr->InputAt(1)->IsFPRegister()) { \ __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \ i.InputDoubleRegister(1)); \ } else { \ __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \ i.InputOperand(1)); \ } \ } while (0) #define ASSEMBLE_IEEE754_BINOP(name) \ do { \ __ PrepareCallCFunction(2); \ __ CallCFunction(ExternalReference::ieee754_##name##_function(), 2); \ } while (false) #define ASSEMBLE_IEEE754_UNOP(name) \ do { \ __ PrepareCallCFunction(1); \ __ CallCFunction(ExternalReference::ieee754_##name##_function(), 1); \ } while (false) #define ASSEMBLE_ATOMIC_BINOP(bin_inst, mov_inst, cmpxchg_inst) \ do { \ Label binop; \ __ bind(&binop); \ __ mov_inst(rax, i.MemoryOperand(1)); \ __ movl(i.TempRegister(0), rax); \ __ bin_inst(i.TempRegister(0), i.InputRegister(0)); \ __ lock(); \ __ cmpxchg_inst(i.MemoryOperand(1), i.TempRegister(0)); \ __ j(not_equal, &binop); \ } while (false) #define ASSEMBLE_ATOMIC64_BINOP(bin_inst, mov_inst, cmpxchg_inst) \ do { \ Label binop; \ __ bind(&binop); \ __ mov_inst(rax, i.MemoryOperand(1)); \ __ movq(i.TempRegister(0), rax); \ __ bin_inst(i.TempRegister(0), i.InputRegister(0)); \ __ lock(); \ __ cmpxchg_inst(i.MemoryOperand(1), i.TempRegister(0)); \ __ j(not_equal, &binop); \ } while (false) void CodeGenerator::AssembleDeconstructFrame() { unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset()); __ movq(rsp, rbp); __ popq(rbp); } void CodeGenerator::AssemblePrepareTailCall() { if (frame_access_state()->has_frame()) { __ movq(rbp, MemOperand(rbp, 0)); } frame_access_state()->SetFrameAccessToSP(); } void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg, Register scratch1, Register scratch2, Register scratch3) { DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3)); Label done; // Check if current frame is an arguments adaptor frame. __ cmpp(Operand(rbp, CommonFrameConstants::kContextOrFrameTypeOffset), Immediate(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR))); __ j(not_equal, &done, Label::kNear); // Load arguments count from current arguments adaptor frame (note, it // does not include receiver). Register caller_args_count_reg = scratch1; __ SmiUntag(caller_args_count_reg, Operand(rbp, ArgumentsAdaptorFrameConstants::kLengthOffset)); ParameterCount callee_args_count(args_reg); __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2, scratch3); __ bind(&done); } namespace { void AdjustStackPointerForTailCall(Assembler* assembler, FrameAccessState* state, int new_slot_above_sp, bool allow_shrinkage = true) { int current_sp_offset = state->GetSPToFPSlotCount() + StandardFrameConstants::kFixedSlotCountAboveFp; int stack_slot_delta = new_slot_above_sp - current_sp_offset; if (stack_slot_delta > 0) { assembler->subq(rsp, Immediate(stack_slot_delta * kPointerSize)); state->IncreaseSPDelta(stack_slot_delta); } else if (allow_shrinkage && stack_slot_delta < 0) { assembler->addq(rsp, Immediate(-stack_slot_delta * kPointerSize)); state->IncreaseSPDelta(stack_slot_delta); } } } // namespace void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr, int first_unused_stack_slot) { CodeGenerator::PushTypeFlags flags(kImmediatePush | kScalarPush); ZoneVector pushes(zone()); GetPushCompatibleMoves(instr, flags, &pushes); if (!pushes.empty() && (LocationOperand::cast(pushes.back()->destination()).index() + 1 == first_unused_stack_slot)) { X64OperandConverter g(this, instr); for (auto move : pushes) { LocationOperand destination_location( LocationOperand::cast(move->destination())); InstructionOperand source(move->source()); AdjustStackPointerForTailCall(tasm(), frame_access_state(), destination_location.index()); if (source.IsStackSlot()) { LocationOperand source_location(LocationOperand::cast(source)); __ Push(g.SlotToOperand(source_location.index())); } else if (source.IsRegister()) { LocationOperand source_location(LocationOperand::cast(source)); __ Push(source_location.GetRegister()); } else if (source.IsImmediate()) { __ Push(Immediate(ImmediateOperand::cast(source).inline_value())); } else { // Pushes of non-scalar data types is not supported. UNIMPLEMENTED(); } frame_access_state()->IncreaseSPDelta(1); move->Eliminate(); } } AdjustStackPointerForTailCall(tasm(), frame_access_state(), first_unused_stack_slot, false); } void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr, int first_unused_stack_slot) { AdjustStackPointerForTailCall(tasm(), frame_access_state(), first_unused_stack_slot); } // Check that {kJavaScriptCallCodeStartRegister} is correct. void CodeGenerator::AssembleCodeStartRegisterCheck() { __ ComputeCodeStartAddress(rbx); __ cmpq(rbx, kJavaScriptCallCodeStartRegister); __ Assert(equal, AbortReason::kWrongFunctionCodeStart); } // Check if the code object is marked for deoptimization. If it is, then it // jumps to the CompileLazyDeoptimizedCode builtin. In order to do this we need // to: // 1. read from memory the word that contains that bit, which can be found in // the flags in the referenced {CodeDataContainer} object; // 2. test kMarkedForDeoptimizationBit in those flags; and // 3. if it is not zero then it jumps to the builtin. void CodeGenerator::BailoutIfDeoptimized() { int offset = Code::kCodeDataContainerOffset - Code::kHeaderSize; __ movp(rbx, Operand(kJavaScriptCallCodeStartRegister, offset)); __ testl(FieldOperand(rbx, CodeDataContainer::kKindSpecificFlagsOffset), Immediate(1 << Code::kMarkedForDeoptimizationBit)); // Ensure we're not serializing (otherwise we'd need to use an indirection to // access the builtin below). DCHECK(!isolate()->ShouldLoadConstantsFromRootList()); Handle code = isolate()->builtins()->builtin_handle( Builtins::kCompileLazyDeoptimizedCode); __ j(not_zero, code, RelocInfo::CODE_TARGET); } void CodeGenerator::GenerateSpeculationPoisonFromCodeStartRegister() { // Set a mask which has all bits set in the normal case, but has all // bits cleared if we are speculatively executing the wrong PC. __ ComputeCodeStartAddress(rbx); __ xorq(kSpeculationPoisonRegister, kSpeculationPoisonRegister); __ cmpp(kJavaScriptCallCodeStartRegister, rbx); __ movp(rbx, Immediate(-1)); __ cmovq(equal, kSpeculationPoisonRegister, rbx); } void CodeGenerator::AssembleRegisterArgumentPoisoning() { __ andq(kJSFunctionRegister, kSpeculationPoisonRegister); __ andq(kContextRegister, kSpeculationPoisonRegister); __ andq(rsp, kSpeculationPoisonRegister); } // Assembles an instruction after register allocation, producing machine code. CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( Instruction* instr) { X64OperandConverter i(this, instr); InstructionCode opcode = instr->opcode(); ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode); switch (arch_opcode) { case kArchCallCodeObject: { if (HasImmediateInput(instr, 0)) { Handle code = i.InputCode(0); __ Call(code, RelocInfo::CODE_TARGET); } else { Register reg = i.InputRegister(0); DCHECK_IMPLIES( HasCallDescriptorFlag(instr, CallDescriptor::kFixedTargetRegister), reg == kJavaScriptCallCodeStartRegister); __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag)); if (HasCallDescriptorFlag(instr, CallDescriptor::kRetpoline)) { __ RetpolineCall(reg); } else { __ call(reg); } } RecordCallPosition(instr); frame_access_state()->ClearSPDelta(); break; } case kArchCallWasmFunction: { if (HasImmediateInput(instr, 0)) { Constant constant = i.ToConstant(instr->InputAt(0)); Address wasm_code = static_cast
(constant.ToInt64()); if (DetermineStubCallMode() == StubCallMode::kCallWasmRuntimeStub) { __ near_call(wasm_code, constant.rmode()); } else { if (HasCallDescriptorFlag(instr, CallDescriptor::kRetpoline)) { __ RetpolineCall(wasm_code, constant.rmode()); } else { __ Call(wasm_code, constant.rmode()); } } } else { Register reg = i.InputRegister(0); if (HasCallDescriptorFlag(instr, CallDescriptor::kRetpoline)) { __ RetpolineCall(reg); } else { __ call(reg); } } RecordCallPosition(instr); frame_access_state()->ClearSPDelta(); break; } case kArchTailCallCodeObjectFromJSFunction: case kArchTailCallCodeObject: { if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) { AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister, i.TempRegister(0), i.TempRegister(1), i.TempRegister(2)); } if (HasImmediateInput(instr, 0)) { Handle code = i.InputCode(0); __ Jump(code, RelocInfo::CODE_TARGET); } else { Register reg = i.InputRegister(0); DCHECK_IMPLIES( HasCallDescriptorFlag(instr, CallDescriptor::kFixedTargetRegister), reg == kJavaScriptCallCodeStartRegister); __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag)); if (HasCallDescriptorFlag(instr, CallDescriptor::kRetpoline)) { __ RetpolineJump(reg); } else { __ jmp(reg); } } unwinding_info_writer_.MarkBlockWillExit(); frame_access_state()->ClearSPDelta(); frame_access_state()->SetFrameAccessToDefault(); break; } case kArchTailCallWasm: { if (HasImmediateInput(instr, 0)) { Constant constant = i.ToConstant(instr->InputAt(0)); Address wasm_code = static_cast
(constant.ToInt64()); if (DetermineStubCallMode() == StubCallMode::kCallWasmRuntimeStub) { __ near_jmp(wasm_code, constant.rmode()); } else { __ Move(kScratchRegister, wasm_code, constant.rmode()); __ jmp(kScratchRegister); } } else { Register reg = i.InputRegister(0); if (HasCallDescriptorFlag(instr, CallDescriptor::kRetpoline)) { __ RetpolineJump(reg); } else { __ jmp(reg); } } unwinding_info_writer_.MarkBlockWillExit(); frame_access_state()->ClearSPDelta(); frame_access_state()->SetFrameAccessToDefault(); break; } case kArchTailCallAddress: { CHECK(!HasImmediateInput(instr, 0)); Register reg = i.InputRegister(0); DCHECK_IMPLIES( HasCallDescriptorFlag(instr, CallDescriptor::kFixedTargetRegister), reg == kJavaScriptCallCodeStartRegister); if (HasCallDescriptorFlag(instr, CallDescriptor::kRetpoline)) { __ RetpolineJump(reg); } else { __ jmp(reg); } unwinding_info_writer_.MarkBlockWillExit(); frame_access_state()->ClearSPDelta(); frame_access_state()->SetFrameAccessToDefault(); break; } case kArchCallJSFunction: { Register func = i.InputRegister(0); if (FLAG_debug_code) { // Check the function's context matches the context argument. __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset)); __ Assert(equal, AbortReason::kWrongFunctionContext); } static_assert(kJavaScriptCallCodeStartRegister == rcx, "ABI mismatch"); __ movp(rcx, FieldOperand(func, JSFunction::kCodeOffset)); __ addp(rcx, Immediate(Code::kHeaderSize - kHeapObjectTag)); __ call(rcx); frame_access_state()->ClearSPDelta(); RecordCallPosition(instr); break; } case kArchPrepareCallCFunction: { // Frame alignment requires using FP-relative frame addressing. frame_access_state()->SetFrameAccessToFP(); int const num_parameters = MiscField::decode(instr->opcode()); __ PrepareCallCFunction(num_parameters); break; } case kArchSaveCallerRegisters: { fp_mode_ = static_cast(MiscField::decode(instr->opcode())); DCHECK(fp_mode_ == kDontSaveFPRegs || fp_mode_ == kSaveFPRegs); // kReturnRegister0 should have been saved before entering the stub. int bytes = __ PushCallerSaved(fp_mode_, kReturnRegister0); DCHECK_EQ(0, bytes % kPointerSize); DCHECK_EQ(0, frame_access_state()->sp_delta()); frame_access_state()->IncreaseSPDelta(bytes / kPointerSize); DCHECK(!caller_registers_saved_); caller_registers_saved_ = true; break; } case kArchRestoreCallerRegisters: { DCHECK(fp_mode_ == static_cast(MiscField::decode(instr->opcode()))); DCHECK(fp_mode_ == kDontSaveFPRegs || fp_mode_ == kSaveFPRegs); // Don't overwrite the returned value. int bytes = __ PopCallerSaved(fp_mode_, kReturnRegister0); frame_access_state()->IncreaseSPDelta(-(bytes / kPointerSize)); DCHECK_EQ(0, frame_access_state()->sp_delta()); DCHECK(caller_registers_saved_); caller_registers_saved_ = false; break; } case kArchPrepareTailCall: AssemblePrepareTailCall(); break; case kArchCallCFunction: { int const num_parameters = MiscField::decode(instr->opcode()); if (HasImmediateInput(instr, 0)) { ExternalReference ref = i.InputExternalReference(0); __ CallCFunction(ref, num_parameters); } else { Register func = i.InputRegister(0); __ CallCFunction(func, num_parameters); } frame_access_state()->SetFrameAccessToDefault(); // Ideally, we should decrement SP delta to match the change of stack // pointer in CallCFunction. However, for certain architectures (e.g. // ARM), there may be more strict alignment requirement, causing old SP // to be saved on the stack. In those cases, we can not calculate the SP // delta statically. frame_access_state()->ClearSPDelta(); if (caller_registers_saved_) { // Need to re-sync SP delta introduced in kArchSaveCallerRegisters. // Here, we assume the sequence to be: // kArchSaveCallerRegisters; // kArchCallCFunction; // kArchRestoreCallerRegisters; int bytes = __ RequiredStackSizeForCallerSaved(fp_mode_, kReturnRegister0); frame_access_state()->IncreaseSPDelta(bytes / kPointerSize); } // TODO(tebbi): Do we need an lfence here? break; } case kArchJmp: AssembleArchJump(i.InputRpo(0)); break; case kArchBinarySearchSwitch: AssembleArchBinarySearchSwitch(instr); break; case kArchLookupSwitch: AssembleArchLookupSwitch(instr); break; case kArchTableSwitch: AssembleArchTableSwitch(instr); break; case kArchComment: __ RecordComment(reinterpret_cast(i.InputInt64(0))); break; case kArchDebugAbort: DCHECK(i.InputRegister(0) == rdx); if (!frame_access_state()->has_frame()) { // We don't actually want to generate a pile of code for this, so just // claim there is a stack frame, without generating one. FrameScope scope(tasm(), StackFrame::NONE); __ Call(isolate()->builtins()->builtin_handle(Builtins::kAbortJS), RelocInfo::CODE_TARGET); } else { __ Call(isolate()->builtins()->builtin_handle(Builtins::kAbortJS), RelocInfo::CODE_TARGET); } __ int3(); unwinding_info_writer_.MarkBlockWillExit(); break; case kArchDebugBreak: __ int3(); break; case kArchThrowTerminator: unwinding_info_writer_.MarkBlockWillExit(); break; case kArchNop: // don't emit code for nops. break; case kArchDeoptimize: { int deopt_state_id = BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore()); CodeGenResult result = AssembleDeoptimizerCall(deopt_state_id, current_source_position_); if (result != kSuccess) return result; unwinding_info_writer_.MarkBlockWillExit(); break; } case kArchRet: AssembleReturn(instr->InputAt(0)); break; case kArchStackPointer: __ movq(i.OutputRegister(), rsp); break; case kArchFramePointer: __ movq(i.OutputRegister(), rbp); break; case kArchParentFramePointer: if (frame_access_state()->has_frame()) { __ movq(i.OutputRegister(), Operand(rbp, 0)); } else { __ movq(i.OutputRegister(), rbp); } break; case kArchTruncateDoubleToI: { auto result = i.OutputRegister(); auto input = i.InputDoubleRegister(0); auto ool = new (zone()) OutOfLineTruncateDoubleToI( this, result, input, DetermineStubCallMode(), &unwinding_info_writer_); // We use Cvttsd2siq instead of Cvttsd2si due to performance reasons. The // use of Cvttsd2siq requires the movl below to avoid sign extension. __ Cvttsd2siq(result, input); __ cmpq(result, Immediate(1)); __ j(overflow, ool->entry()); __ bind(ool->exit()); __ movl(result, result); break; } case kArchStoreWithWriteBarrier: { RecordWriteMode mode = static_cast(MiscField::decode(instr->opcode())); Register object = i.InputRegister(0); size_t index = 0; Operand operand = i.MemoryOperand(&index); Register value = i.InputRegister(index); Register scratch0 = i.TempRegister(0); Register scratch1 = i.TempRegister(1); auto ool = new (zone()) OutOfLineRecordWrite(this, object, operand, value, scratch0, scratch1, mode); __ movp(operand, value); __ CheckPageFlag(object, scratch0, MemoryChunk::kPointersFromHereAreInterestingMask, not_zero, ool->entry()); __ bind(ool->exit()); break; } case kArchWordPoisonOnSpeculation: DCHECK_EQ(i.OutputRegister(), i.InputRegister(0)); __ andq(i.InputRegister(0), kSpeculationPoisonRegister); break; case kLFence: __ lfence(); break; case kArchStackSlot: { FrameOffset offset = frame_access_state()->GetFrameOffset(i.InputInt32(0)); Register base = offset.from_stack_pointer() ? rsp : rbp; __ leaq(i.OutputRegister(), Operand(base, offset.offset())); break; } case kIeee754Float64Acos: ASSEMBLE_IEEE754_UNOP(acos); break; case kIeee754Float64Acosh: ASSEMBLE_IEEE754_UNOP(acosh); break; case kIeee754Float64Asin: ASSEMBLE_IEEE754_UNOP(asin); break; case kIeee754Float64Asinh: ASSEMBLE_IEEE754_UNOP(asinh); break; case kIeee754Float64Atan: ASSEMBLE_IEEE754_UNOP(atan); break; case kIeee754Float64Atanh: ASSEMBLE_IEEE754_UNOP(atanh); break; case kIeee754Float64Atan2: ASSEMBLE_IEEE754_BINOP(atan2); break; case kIeee754Float64Cbrt: ASSEMBLE_IEEE754_UNOP(cbrt); break; case kIeee754Float64Cos: ASSEMBLE_IEEE754_UNOP(cos); break; case kIeee754Float64Cosh: ASSEMBLE_IEEE754_UNOP(cosh); break; case kIeee754Float64Exp: ASSEMBLE_IEEE754_UNOP(exp); break; case kIeee754Float64Expm1: ASSEMBLE_IEEE754_UNOP(expm1); break; case kIeee754Float64Log: ASSEMBLE_IEEE754_UNOP(log); break; case kIeee754Float64Log1p: ASSEMBLE_IEEE754_UNOP(log1p); break; case kIeee754Float64Log2: ASSEMBLE_IEEE754_UNOP(log2); break; case kIeee754Float64Log10: ASSEMBLE_IEEE754_UNOP(log10); break; case kIeee754Float64Pow: { // TODO(bmeurer): Improve integration of the stub. __ Movsd(xmm2, xmm0); __ Call(BUILTIN_CODE(isolate(), MathPowInternal), RelocInfo::CODE_TARGET); __ Movsd(xmm0, xmm3); break; } case kIeee754Float64Sin: ASSEMBLE_IEEE754_UNOP(sin); break; case kIeee754Float64Sinh: ASSEMBLE_IEEE754_UNOP(sinh); break; case kIeee754Float64Tan: ASSEMBLE_IEEE754_UNOP(tan); break; case kIeee754Float64Tanh: ASSEMBLE_IEEE754_UNOP(tanh); break; case kX64Add32: ASSEMBLE_BINOP(addl); break; case kX64Add: ASSEMBLE_BINOP(addq); break; case kX64Sub32: ASSEMBLE_BINOP(subl); break; case kX64Sub: ASSEMBLE_BINOP(subq); break; case kX64And32: ASSEMBLE_BINOP(andl); break; case kX64And: ASSEMBLE_BINOP(andq); break; case kX64Cmp8: ASSEMBLE_COMPARE(cmpb); break; case kX64Cmp16: ASSEMBLE_COMPARE(cmpw); break; case kX64Cmp32: ASSEMBLE_COMPARE(cmpl); break; case kX64Cmp: ASSEMBLE_COMPARE(cmpq); break; case kX64Test8: ASSEMBLE_COMPARE(testb); break; case kX64Test16: ASSEMBLE_COMPARE(testw); break; case kX64Test32: ASSEMBLE_COMPARE(testl); break; case kX64Test: ASSEMBLE_COMPARE(testq); break; case kX64Imul32: ASSEMBLE_MULT(imull); break; case kX64Imul: ASSEMBLE_MULT(imulq); break; case kX64ImulHigh32: if (instr->InputAt(1)->IsRegister()) { __ imull(i.InputRegister(1)); } else { __ imull(i.InputOperand(1)); } break; case kX64UmulHigh32: if (instr->InputAt(1)->IsRegister()) { __ mull(i.InputRegister(1)); } else { __ mull(i.InputOperand(1)); } break; case kX64Idiv32: __ cdq(); __ idivl(i.InputRegister(1)); break; case kX64Idiv: __ cqo(); __ idivq(i.InputRegister(1)); break; case kX64Udiv32: __ xorl(rdx, rdx); __ divl(i.InputRegister(1)); break; case kX64Udiv: __ xorq(rdx, rdx); __ divq(i.InputRegister(1)); break; case kX64Not: ASSEMBLE_UNOP(notq); break; case kX64Not32: ASSEMBLE_UNOP(notl); break; case kX64Neg: ASSEMBLE_UNOP(negq); break; case kX64Neg32: ASSEMBLE_UNOP(negl); break; case kX64Or32: ASSEMBLE_BINOP(orl); break; case kX64Or: ASSEMBLE_BINOP(orq); break; case kX64Xor32: ASSEMBLE_BINOP(xorl); break; case kX64Xor: ASSEMBLE_BINOP(xorq); break; case kX64Shl32: ASSEMBLE_SHIFT(shll, 5); break; case kX64Shl: ASSEMBLE_SHIFT(shlq, 6); break; case kX64Shr32: ASSEMBLE_SHIFT(shrl, 5); break; case kX64Shr: ASSEMBLE_SHIFT(shrq, 6); break; case kX64Sar32: ASSEMBLE_SHIFT(sarl, 5); break; case kX64Sar: ASSEMBLE_SHIFT(sarq, 6); break; case kX64Ror32: ASSEMBLE_SHIFT(rorl, 5); break; case kX64Ror: ASSEMBLE_SHIFT(rorq, 6); break; case kX64Lzcnt: if (instr->InputAt(0)->IsRegister()) { __ Lzcntq(i.OutputRegister(), i.InputRegister(0)); } else { __ Lzcntq(i.OutputRegister(), i.InputOperand(0)); } break; case kX64Lzcnt32: if (instr->InputAt(0)->IsRegister()) { __ Lzcntl(i.OutputRegister(), i.InputRegister(0)); } else { __ Lzcntl(i.OutputRegister(), i.InputOperand(0)); } break; case kX64Tzcnt: if (instr->InputAt(0)->IsRegister()) { __ Tzcntq(i.OutputRegister(), i.InputRegister(0)); } else { __ Tzcntq(i.OutputRegister(), i.InputOperand(0)); } break; case kX64Tzcnt32: if (instr->InputAt(0)->IsRegister()) { __ Tzcntl(i.OutputRegister(), i.InputRegister(0)); } else { __ Tzcntl(i.OutputRegister(), i.InputOperand(0)); } break; case kX64Popcnt: if (instr->InputAt(0)->IsRegister()) { __ Popcntq(i.OutputRegister(), i.InputRegister(0)); } else { __ Popcntq(i.OutputRegister(), i.InputOperand(0)); } break; case kX64Popcnt32: if (instr->InputAt(0)->IsRegister()) { __ Popcntl(i.OutputRegister(), i.InputRegister(0)); } else { __ Popcntl(i.OutputRegister(), i.InputOperand(0)); } break; case kX64Bswap: __ bswapq(i.OutputRegister()); break; case kX64Bswap32: __ bswapl(i.OutputRegister()); break; case kSSEFloat32Cmp: ASSEMBLE_SSE_BINOP(Ucomiss); break; case kSSEFloat32Add: ASSEMBLE_SSE_BINOP(addss); break; case kSSEFloat32Sub: ASSEMBLE_SSE_BINOP(subss); break; case kSSEFloat32Mul: ASSEMBLE_SSE_BINOP(mulss); break; case kSSEFloat32Div: ASSEMBLE_SSE_BINOP(divss); // Don't delete this mov. It may improve performance on some CPUs, // when there is a (v)mulss depending on the result. __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister()); break; case kSSEFloat32Abs: { // TODO(bmeurer): Use RIP relative 128-bit constants. __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); __ psrlq(kScratchDoubleReg, 33); __ andps(i.OutputDoubleRegister(), kScratchDoubleReg); break; } case kSSEFloat32Neg: { // TODO(bmeurer): Use RIP relative 128-bit constants. __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); __ psllq(kScratchDoubleReg, 31); __ xorps(i.OutputDoubleRegister(), kScratchDoubleReg); break; } case kSSEFloat32Sqrt: ASSEMBLE_SSE_UNOP(sqrtss); break; case kSSEFloat32ToFloat64: ASSEMBLE_SSE_UNOP(Cvtss2sd); break; case kSSEFloat32Round: { CpuFeatureScope sse_scope(tasm(), SSE4_1); RoundingMode const mode = static_cast(MiscField::decode(instr->opcode())); __ Roundss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode); break; } case kSSEFloat32ToInt32: if (instr->InputAt(0)->IsFPRegister()) { __ Cvttss2si(i.OutputRegister(), i.InputDoubleRegister(0)); } else { __ Cvttss2si(i.OutputRegister(), i.InputOperand(0)); } break; case kSSEFloat32ToUint32: { if (instr->InputAt(0)->IsFPRegister()) { __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0)); } else { __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0)); } break; } case kSSEFloat64Cmp: ASSEMBLE_SSE_BINOP(Ucomisd); break; case kSSEFloat64Add: ASSEMBLE_SSE_BINOP(addsd); break; case kSSEFloat64Sub: ASSEMBLE_SSE_BINOP(subsd); break; case kSSEFloat64Mul: ASSEMBLE_SSE_BINOP(mulsd); break; case kSSEFloat64Div: ASSEMBLE_SSE_BINOP(divsd); // Don't delete this mov. It may improve performance on some CPUs, // when there is a (v)mulsd depending on the result. __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister()); break; case kSSEFloat64Mod: { __ subq(rsp, Immediate(kDoubleSize)); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), kDoubleSize); // Move values to st(0) and st(1). __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(1)); __ fld_d(Operand(rsp, 0)); __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0)); __ fld_d(Operand(rsp, 0)); // Loop while fprem isn't done. Label mod_loop; __ bind(&mod_loop); // This instructions traps on all kinds inputs, but we are assuming the // floating point control word is set to ignore them all. __ fprem(); // The following 2 instruction implicitly use rax. __ fnstsw_ax(); if (CpuFeatures::IsSupported(SAHF)) { CpuFeatureScope sahf_scope(tasm(), SAHF); __ sahf(); } else { __ shrl(rax, Immediate(8)); __ andl(rax, Immediate(0xFF)); __ pushq(rax); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), kPointerSize); __ popfq(); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), -kPointerSize); } __ j(parity_even, &mod_loop); // Move output to stack and clean up. __ fstp(1); __ fstp_d(Operand(rsp, 0)); __ Movsd(i.OutputDoubleRegister(), Operand(rsp, 0)); __ addq(rsp, Immediate(kDoubleSize)); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), -kDoubleSize); break; } case kSSEFloat32Max: { Label compare_nan, compare_swap, done_compare; if (instr->InputAt(1)->IsFPRegister()) { __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); } else { __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1)); } auto ool = new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister()); __ j(parity_even, ool->entry()); __ j(above, &done_compare, Label::kNear); __ j(below, &compare_swap, Label::kNear); __ Movmskps(kScratchRegister, i.InputDoubleRegister(0)); __ testl(kScratchRegister, Immediate(1)); __ j(zero, &done_compare, Label::kNear); __ bind(&compare_swap); if (instr->InputAt(1)->IsFPRegister()) { __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); } else { __ Movss(i.InputDoubleRegister(0), i.InputOperand(1)); } __ bind(&done_compare); __ bind(ool->exit()); break; } case kSSEFloat32Min: { Label compare_swap, done_compare; if (instr->InputAt(1)->IsFPRegister()) { __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); } else { __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1)); } auto ool = new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister()); __ j(parity_even, ool->entry()); __ j(below, &done_compare, Label::kNear); __ j(above, &compare_swap, Label::kNear); if (instr->InputAt(1)->IsFPRegister()) { __ Movmskps(kScratchRegister, i.InputDoubleRegister(1)); } else { __ Movss(kScratchDoubleReg, i.InputOperand(1)); __ Movmskps(kScratchRegister, kScratchDoubleReg); } __ testl(kScratchRegister, Immediate(1)); __ j(zero, &done_compare, Label::kNear); __ bind(&compare_swap); if (instr->InputAt(1)->IsFPRegister()) { __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); } else { __ Movss(i.InputDoubleRegister(0), i.InputOperand(1)); } __ bind(&done_compare); __ bind(ool->exit()); break; } case kSSEFloat64Max: { Label compare_nan, compare_swap, done_compare; if (instr->InputAt(1)->IsFPRegister()) { __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); } else { __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1)); } auto ool = new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister()); __ j(parity_even, ool->entry()); __ j(above, &done_compare, Label::kNear); __ j(below, &compare_swap, Label::kNear); __ Movmskpd(kScratchRegister, i.InputDoubleRegister(0)); __ testl(kScratchRegister, Immediate(1)); __ j(zero, &done_compare, Label::kNear); __ bind(&compare_swap); if (instr->InputAt(1)->IsFPRegister()) { __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); } else { __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1)); } __ bind(&done_compare); __ bind(ool->exit()); break; } case kSSEFloat64Min: { Label compare_swap, done_compare; if (instr->InputAt(1)->IsFPRegister()) { __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); } else { __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1)); } auto ool = new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister()); __ j(parity_even, ool->entry()); __ j(below, &done_compare, Label::kNear); __ j(above, &compare_swap, Label::kNear); if (instr->InputAt(1)->IsFPRegister()) { __ Movmskpd(kScratchRegister, i.InputDoubleRegister(1)); } else { __ Movsd(kScratchDoubleReg, i.InputOperand(1)); __ Movmskpd(kScratchRegister, kScratchDoubleReg); } __ testl(kScratchRegister, Immediate(1)); __ j(zero, &done_compare, Label::kNear); __ bind(&compare_swap); if (instr->InputAt(1)->IsFPRegister()) { __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); } else { __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1)); } __ bind(&done_compare); __ bind(ool->exit()); break; } case kSSEFloat64Abs: { // TODO(bmeurer): Use RIP relative 128-bit constants. __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); __ psrlq(kScratchDoubleReg, 1); __ andpd(i.OutputDoubleRegister(), kScratchDoubleReg); break; } case kSSEFloat64Neg: { // TODO(bmeurer): Use RIP relative 128-bit constants. __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); __ psllq(kScratchDoubleReg, 63); __ xorpd(i.OutputDoubleRegister(), kScratchDoubleReg); break; } case kSSEFloat64Sqrt: ASSEMBLE_SSE_UNOP(Sqrtsd); break; case kSSEFloat64Round: { CpuFeatureScope sse_scope(tasm(), SSE4_1); RoundingMode const mode = static_cast(MiscField::decode(instr->opcode())); __ Roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode); break; } case kSSEFloat64ToFloat32: ASSEMBLE_SSE_UNOP(Cvtsd2ss); break; case kSSEFloat64ToInt32: if (instr->InputAt(0)->IsFPRegister()) { __ Cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0)); } else { __ Cvttsd2si(i.OutputRegister(), i.InputOperand(0)); } break; case kSSEFloat64ToUint32: { if (instr->InputAt(0)->IsFPRegister()) { __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0)); } else { __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0)); } if (MiscField::decode(instr->opcode())) { __ AssertZeroExtended(i.OutputRegister()); } break; } case kSSEFloat32ToInt64: if (instr->InputAt(0)->IsFPRegister()) { __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0)); } else { __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0)); } if (instr->OutputCount() > 1) { __ Set(i.OutputRegister(1), 1); Label done; Label fail; __ Move(kScratchDoubleReg, static_cast(INT64_MIN)); if (instr->InputAt(0)->IsFPRegister()) { __ Ucomiss(kScratchDoubleReg, i.InputDoubleRegister(0)); } else { __ Ucomiss(kScratchDoubleReg, i.InputOperand(0)); } // If the input is NaN, then the conversion fails. __ j(parity_even, &fail); // If the input is INT64_MIN, then the conversion succeeds. __ j(equal, &done); __ cmpq(i.OutputRegister(0), Immediate(1)); // If the conversion results in INT64_MIN, but the input was not // INT64_MIN, then the conversion fails. __ j(no_overflow, &done); __ bind(&fail); __ Set(i.OutputRegister(1), 0); __ bind(&done); } break; case kSSEFloat64ToInt64: if (instr->InputAt(0)->IsFPRegister()) { __ Cvttsd2siq(i.OutputRegister(0), i.InputDoubleRegister(0)); } else { __ Cvttsd2siq(i.OutputRegister(0), i.InputOperand(0)); } if (instr->OutputCount() > 1) { __ Set(i.OutputRegister(1), 1); Label done; Label fail; __ Move(kScratchDoubleReg, static_cast(INT64_MIN)); if (instr->InputAt(0)->IsFPRegister()) { __ Ucomisd(kScratchDoubleReg, i.InputDoubleRegister(0)); } else { __ Ucomisd(kScratchDoubleReg, i.InputOperand(0)); } // If the input is NaN, then the conversion fails. __ j(parity_even, &fail); // If the input is INT64_MIN, then the conversion succeeds. __ j(equal, &done); __ cmpq(i.OutputRegister(0), Immediate(1)); // If the conversion results in INT64_MIN, but the input was not // INT64_MIN, then the conversion fails. __ j(no_overflow, &done); __ bind(&fail); __ Set(i.OutputRegister(1), 0); __ bind(&done); } break; case kSSEFloat32ToUint64: { Label fail; if (instr->OutputCount() > 1) __ Set(i.OutputRegister(1), 0); if (instr->InputAt(0)->IsFPRegister()) { __ Cvttss2uiq(i.OutputRegister(), i.InputDoubleRegister(0), &fail); } else { __ Cvttss2uiq(i.OutputRegister(), i.InputOperand(0), &fail); } if (instr->OutputCount() > 1) __ Set(i.OutputRegister(1), 1); __ bind(&fail); break; } case kSSEFloat64ToUint64: { Label fail; if (instr->OutputCount() > 1) __ Set(i.OutputRegister(1), 0); if (instr->InputAt(0)->IsFPRegister()) { __ Cvttsd2uiq(i.OutputRegister(), i.InputDoubleRegister(0), &fail); } else { __ Cvttsd2uiq(i.OutputRegister(), i.InputOperand(0), &fail); } if (instr->OutputCount() > 1) __ Set(i.OutputRegister(1), 1); __ bind(&fail); break; } case kSSEInt32ToFloat64: if (instr->InputAt(0)->IsRegister()) { __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0)); } else { __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0)); } break; case kSSEInt32ToFloat32: if (instr->InputAt(0)->IsRegister()) { __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputRegister(0)); } else { __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputOperand(0)); } break; case kSSEInt64ToFloat32: if (instr->InputAt(0)->IsRegister()) { __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputRegister(0)); } else { __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputOperand(0)); } break; case kSSEInt64ToFloat64: if (instr->InputAt(0)->IsRegister()) { __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputRegister(0)); } else { __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputOperand(0)); } break; case kSSEUint64ToFloat32: if (instr->InputAt(0)->IsRegister()) { __ Cvtqui2ss(i.OutputDoubleRegister(), i.InputRegister(0)); } else { __ Cvtqui2ss(i.OutputDoubleRegister(), i.InputOperand(0)); } break; case kSSEUint64ToFloat64: if (instr->InputAt(0)->IsRegister()) { __ Cvtqui2sd(i.OutputDoubleRegister(), i.InputRegister(0)); } else { __ Cvtqui2sd(i.OutputDoubleRegister(), i.InputOperand(0)); } break; case kSSEUint32ToFloat64: if (instr->InputAt(0)->IsRegister()) { __ Cvtlui2sd(i.OutputDoubleRegister(), i.InputRegister(0)); } else { __ Cvtlui2sd(i.OutputDoubleRegister(), i.InputOperand(0)); } break; case kSSEUint32ToFloat32: if (instr->InputAt(0)->IsRegister()) { __ Cvtlui2ss(i.OutputDoubleRegister(), i.InputRegister(0)); } else { __ Cvtlui2ss(i.OutputDoubleRegister(), i.InputOperand(0)); } break; case kSSEFloat64ExtractLowWord32: if (instr->InputAt(0)->IsFPStackSlot()) { __ movl(i.OutputRegister(), i.InputOperand(0)); } else { __ Movd(i.OutputRegister(), i.InputDoubleRegister(0)); } break; case kSSEFloat64ExtractHighWord32: if (instr->InputAt(0)->IsFPStackSlot()) { __ movl(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2)); } else { __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1); } break; case kSSEFloat64InsertLowWord32: if (instr->InputAt(1)->IsRegister()) { __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 0); } else { __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0); } break; case kSSEFloat64InsertHighWord32: if (instr->InputAt(1)->IsRegister()) { __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 1); } else { __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1); } break; case kSSEFloat64LoadLowWord32: if (instr->InputAt(0)->IsRegister()) { __ Movd(i.OutputDoubleRegister(), i.InputRegister(0)); } else { __ Movd(i.OutputDoubleRegister(), i.InputOperand(0)); } break; case kAVXFloat32Cmp: { CpuFeatureScope avx_scope(tasm(), AVX); if (instr->InputAt(1)->IsFPRegister()) { __ vucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); } else { __ vucomiss(i.InputDoubleRegister(0), i.InputOperand(1)); } break; } case kAVXFloat32Add: ASSEMBLE_AVX_BINOP(vaddss); break; case kAVXFloat32Sub: ASSEMBLE_AVX_BINOP(vsubss); break; case kAVXFloat32Mul: ASSEMBLE_AVX_BINOP(vmulss); break; case kAVXFloat32Div: ASSEMBLE_AVX_BINOP(vdivss); // Don't delete this mov. It may improve performance on some CPUs, // when there is a (v)mulss depending on the result. __ Movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister()); break; case kAVXFloat64Cmp: { CpuFeatureScope avx_scope(tasm(), AVX); if (instr->InputAt(1)->IsFPRegister()) { __ vucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); } else { __ vucomisd(i.InputDoubleRegister(0), i.InputOperand(1)); } break; } case kAVXFloat64Add: ASSEMBLE_AVX_BINOP(vaddsd); break; case kAVXFloat64Sub: ASSEMBLE_AVX_BINOP(vsubsd); break; case kAVXFloat64Mul: ASSEMBLE_AVX_BINOP(vmulsd); break; case kAVXFloat64Div: ASSEMBLE_AVX_BINOP(vdivsd); // Don't delete this mov. It may improve performance on some CPUs, // when there is a (v)mulsd depending on the result. __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister()); break; case kAVXFloat32Abs: { // TODO(bmeurer): Use RIP relative 128-bit constants. CpuFeatureScope avx_scope(tasm(), AVX); __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 33); if (instr->InputAt(0)->IsFPRegister()) { __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputDoubleRegister(0)); } else { __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputOperand(0)); } break; } case kAVXFloat32Neg: { // TODO(bmeurer): Use RIP relative 128-bit constants. CpuFeatureScope avx_scope(tasm(), AVX); __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 31); if (instr->InputAt(0)->IsFPRegister()) { __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputDoubleRegister(0)); } else { __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputOperand(0)); } break; } case kAVXFloat64Abs: { // TODO(bmeurer): Use RIP relative 128-bit constants. CpuFeatureScope avx_scope(tasm(), AVX); __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 1); if (instr->InputAt(0)->IsFPRegister()) { __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputDoubleRegister(0)); } else { __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputOperand(0)); } break; } case kAVXFloat64Neg: { // TODO(bmeurer): Use RIP relative 128-bit constants. CpuFeatureScope avx_scope(tasm(), AVX); __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 63); if (instr->InputAt(0)->IsFPRegister()) { __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputDoubleRegister(0)); } else { __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg, i.InputOperand(0)); } break; } case kSSEFloat64SilenceNaN: __ Xorpd(kScratchDoubleReg, kScratchDoubleReg); __ Subsd(i.InputDoubleRegister(0), kScratchDoubleReg); break; case kX64Movsxbl: EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); ASSEMBLE_MOVX(movsxbl); __ AssertZeroExtended(i.OutputRegister()); EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kX64Movzxbl: EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); ASSEMBLE_MOVX(movzxbl); __ AssertZeroExtended(i.OutputRegister()); EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kX64Movsxbq: EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); ASSEMBLE_MOVX(movsxbq); EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kX64Movzxbq: EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); ASSEMBLE_MOVX(movzxbq); __ AssertZeroExtended(i.OutputRegister()); EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kX64Movb: { EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); size_t index = 0; Operand operand = i.MemoryOperand(&index); if (HasImmediateInput(instr, index)) { __ movb(operand, Immediate(i.InputInt8(index))); } else { __ movb(operand, i.InputRegister(index)); } EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; } case kX64Movsxwl: EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); ASSEMBLE_MOVX(movsxwl); __ AssertZeroExtended(i.OutputRegister()); EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kX64Movzxwl: EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); ASSEMBLE_MOVX(movzxwl); __ AssertZeroExtended(i.OutputRegister()); EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kX64Movsxwq: EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); ASSEMBLE_MOVX(movsxwq); break; case kX64Movzxwq: EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); ASSEMBLE_MOVX(movzxwq); __ AssertZeroExtended(i.OutputRegister()); EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kX64Movw: { EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); size_t index = 0; Operand operand = i.MemoryOperand(&index); if (HasImmediateInput(instr, index)) { __ movw(operand, Immediate(i.InputInt16(index))); } else { __ movw(operand, i.InputRegister(index)); } EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; } case kX64Movl: EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); if (instr->HasOutput()) { if (instr->addressing_mode() == kMode_None) { if (instr->InputAt(0)->IsRegister()) { __ movl(i.OutputRegister(), i.InputRegister(0)); } else { __ movl(i.OutputRegister(), i.InputOperand(0)); } } else { __ movl(i.OutputRegister(), i.MemoryOperand()); } __ AssertZeroExtended(i.OutputRegister()); } else { size_t index = 0; Operand operand = i.MemoryOperand(&index); if (HasImmediateInput(instr, index)) { __ movl(operand, i.InputImmediate(index)); } else { __ movl(operand, i.InputRegister(index)); } } EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kX64Movsxlq: EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); ASSEMBLE_MOVX(movsxlq); EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kX64Movq: EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); if (instr->HasOutput()) { __ movq(i.OutputRegister(), i.MemoryOperand()); } else { size_t index = 0; Operand operand = i.MemoryOperand(&index); if (HasImmediateInput(instr, index)) { __ movq(operand, i.InputImmediate(index)); } else { __ movq(operand, i.InputRegister(index)); } } EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kX64Movss: EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); if (instr->HasOutput()) { __ movss(i.OutputDoubleRegister(), i.MemoryOperand()); } else { size_t index = 0; Operand operand = i.MemoryOperand(&index); __ movss(operand, i.InputDoubleRegister(index)); } break; case kX64Movsd: { EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); if (instr->HasOutput()) { const MemoryAccessMode access_mode = static_cast(MiscField::decode(opcode)); if (access_mode == kMemoryAccessPoisoned) { // If we have to poison the loaded value, we load into a general // purpose register first, mask it with the poison, and move the // value from the general purpose register into the double register. __ movq(kScratchRegister, i.MemoryOperand()); __ andq(kScratchRegister, kSpeculationPoisonRegister); __ Movq(i.OutputDoubleRegister(), kScratchRegister); } else { __ Movsd(i.OutputDoubleRegister(), i.MemoryOperand()); } } else { size_t index = 0; Operand operand = i.MemoryOperand(&index); __ Movsd(operand, i.InputDoubleRegister(index)); } break; } case kX64Movdqu: { CpuFeatureScope sse_scope(tasm(), SSSE3); EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset()); if (instr->HasOutput()) { __ movdqu(i.OutputSimd128Register(), i.MemoryOperand()); } else { size_t index = 0; Operand operand = i.MemoryOperand(&index); __ movdqu(operand, i.InputSimd128Register(index)); } break; } case kX64BitcastFI: if (instr->InputAt(0)->IsFPStackSlot()) { __ movl(i.OutputRegister(), i.InputOperand(0)); } else { __ Movd(i.OutputRegister(), i.InputDoubleRegister(0)); } break; case kX64BitcastDL: if (instr->InputAt(0)->IsFPStackSlot()) { __ movq(i.OutputRegister(), i.InputOperand(0)); } else { __ Movq(i.OutputRegister(), i.InputDoubleRegister(0)); } break; case kX64BitcastIF: if (instr->InputAt(0)->IsRegister()) { __ Movd(i.OutputDoubleRegister(), i.InputRegister(0)); } else { __ movss(i.OutputDoubleRegister(), i.InputOperand(0)); } break; case kX64BitcastLD: if (instr->InputAt(0)->IsRegister()) { __ Movq(i.OutputDoubleRegister(), i.InputRegister(0)); } else { __ Movsd(i.OutputDoubleRegister(), i.InputOperand(0)); } break; case kX64Lea32: { AddressingMode mode = AddressingModeField::decode(instr->opcode()); // Shorten "leal" to "addl", "subl" or "shll" if the register allocation // and addressing mode just happens to work out. The "addl"/"subl" forms // in these cases are faster based on measurements. if (i.InputRegister(0) == i.OutputRegister()) { if (mode == kMode_MRI) { int32_t constant_summand = i.InputInt32(1); DCHECK_NE(0, constant_summand); if (constant_summand > 0) { __ addl(i.OutputRegister(), Immediate(constant_summand)); } else { __ subl(i.OutputRegister(), Immediate(-constant_summand)); } } else if (mode == kMode_MR1) { if (i.InputRegister(1) == i.OutputRegister()) { __ shll(i.OutputRegister(), Immediate(1)); } else { __ addl(i.OutputRegister(), i.InputRegister(1)); } } else if (mode == kMode_M2) { __ shll(i.OutputRegister(), Immediate(1)); } else if (mode == kMode_M4) { __ shll(i.OutputRegister(), Immediate(2)); } else if (mode == kMode_M8) { __ shll(i.OutputRegister(), Immediate(3)); } else { __ leal(i.OutputRegister(), i.MemoryOperand()); } } else if (mode == kMode_MR1 && i.InputRegister(1) == i.OutputRegister()) { __ addl(i.OutputRegister(), i.InputRegister(0)); } else { __ leal(i.OutputRegister(), i.MemoryOperand()); } __ AssertZeroExtended(i.OutputRegister()); break; } case kX64Lea: { AddressingMode mode = AddressingModeField::decode(instr->opcode()); // Shorten "leaq" to "addq", "subq" or "shlq" if the register allocation // and addressing mode just happens to work out. The "addq"/"subq" forms // in these cases are faster based on measurements. if (i.InputRegister(0) == i.OutputRegister()) { if (mode == kMode_MRI) { int32_t constant_summand = i.InputInt32(1); if (constant_summand > 0) { __ addq(i.OutputRegister(), Immediate(constant_summand)); } else if (constant_summand < 0) { __ subq(i.OutputRegister(), Immediate(-constant_summand)); } } else if (mode == kMode_MR1) { if (i.InputRegister(1) == i.OutputRegister()) { __ shlq(i.OutputRegister(), Immediate(1)); } else { __ addq(i.OutputRegister(), i.InputRegister(1)); } } else if (mode == kMode_M2) { __ shlq(i.OutputRegister(), Immediate(1)); } else if (mode == kMode_M4) { __ shlq(i.OutputRegister(), Immediate(2)); } else if (mode == kMode_M8) { __ shlq(i.OutputRegister(), Immediate(3)); } else { __ leaq(i.OutputRegister(), i.MemoryOperand()); } } else if (mode == kMode_MR1 && i.InputRegister(1) == i.OutputRegister()) { __ addq(i.OutputRegister(), i.InputRegister(0)); } else { __ leaq(i.OutputRegister(), i.MemoryOperand()); } break; } case kX64Dec32: __ decl(i.OutputRegister()); break; case kX64Inc32: __ incl(i.OutputRegister()); break; case kX64Push: if (AddressingModeField::decode(instr->opcode()) != kMode_None) { size_t index = 0; Operand operand = i.MemoryOperand(&index); __ pushq(operand); frame_access_state()->IncreaseSPDelta(1); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), kPointerSize); } else if (HasImmediateInput(instr, 0)) { __ pushq(i.InputImmediate(0)); frame_access_state()->IncreaseSPDelta(1); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), kPointerSize); } else if (instr->InputAt(0)->IsRegister()) { __ pushq(i.InputRegister(0)); frame_access_state()->IncreaseSPDelta(1); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), kPointerSize); } else if (instr->InputAt(0)->IsFloatRegister() || instr->InputAt(0)->IsDoubleRegister()) { // TODO(titzer): use another machine instruction? __ subq(rsp, Immediate(kDoubleSize)); frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), kDoubleSize); __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0)); } else if (instr->InputAt(0)->IsSimd128Register()) { // TODO(titzer): use another machine instruction? __ subq(rsp, Immediate(kSimd128Size)); frame_access_state()->IncreaseSPDelta(kSimd128Size / kPointerSize); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), kSimd128Size); __ Movups(Operand(rsp, 0), i.InputSimd128Register(0)); } else if (instr->InputAt(0)->IsStackSlot() || instr->InputAt(0)->IsFloatStackSlot() || instr->InputAt(0)->IsDoubleStackSlot()) { __ pushq(i.InputOperand(0)); frame_access_state()->IncreaseSPDelta(1); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), kPointerSize); } else { DCHECK(instr->InputAt(0)->IsSimd128StackSlot()); __ Movups(kScratchDoubleReg, i.InputOperand(0)); // TODO(titzer): use another machine instruction? __ subq(rsp, Immediate(kSimd128Size)); frame_access_state()->IncreaseSPDelta(kSimd128Size / kPointerSize); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), kSimd128Size); __ Movups(Operand(rsp, 0), kScratchDoubleReg); } break; case kX64Poke: { int slot = MiscField::decode(instr->opcode()); if (HasImmediateInput(instr, 0)) { __ movq(Operand(rsp, slot * kPointerSize), i.InputImmediate(0)); } else { __ movq(Operand(rsp, slot * kPointerSize), i.InputRegister(0)); } break; } case kX64Peek: { int reverse_slot = i.InputInt32(0); int offset = FrameSlotToFPOffset(frame()->GetTotalFrameSlotCount() - reverse_slot); if (instr->OutputAt(0)->IsFPRegister()) { LocationOperand* op = LocationOperand::cast(instr->OutputAt(0)); if (op->representation() == MachineRepresentation::kFloat64) { __ Movsd(i.OutputDoubleRegister(), Operand(rbp, offset)); } else { DCHECK_EQ(MachineRepresentation::kFloat32, op->representation()); __ Movss(i.OutputFloatRegister(), Operand(rbp, offset)); } } else { __ movq(i.OutputRegister(), Operand(rbp, offset)); } break; } // TODO(gdeepti): Get rid of redundant moves for F32x4Splat/Extract below case kX64F32x4Splat: { XMMRegister dst = i.OutputSimd128Register(); if (instr->InputAt(0)->IsFPRegister()) { __ movss(dst, i.InputDoubleRegister(0)); } else { __ movss(dst, i.InputOperand(0)); } __ shufps(dst, dst, 0x0); break; } case kX64F32x4ExtractLane: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ extractps(kScratchRegister, i.InputSimd128Register(0), i.InputInt8(1)); __ movd(i.OutputDoubleRegister(), kScratchRegister); break; } case kX64F32x4ReplaceLane: { CpuFeatureScope sse_scope(tasm(), SSE4_1); // The insertps instruction uses imm8[5:4] to indicate the lane // that needs to be replaced. byte select = i.InputInt8(1) << 4 & 0x30; __ insertps(i.OutputSimd128Register(), i.InputDoubleRegister(2), select); break; } case kX64F32x4Abs: { XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(0); if (dst == src) { __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); __ psrld(kScratchDoubleReg, 1); __ andps(i.OutputSimd128Register(), kScratchDoubleReg); } else { __ pcmpeqd(dst, dst); __ psrld(dst, 1); __ andps(dst, i.InputSimd128Register(0)); } break; } case kX64F32x4Neg: { XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(0); if (dst == src) { __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); __ pslld(kScratchDoubleReg, 31); __ xorps(i.OutputSimd128Register(), kScratchDoubleReg); } else { __ pcmpeqd(dst, dst); __ pslld(dst, 31); __ xorps(dst, i.InputSimd128Register(0)); } break; } case kX64F32x4RecipApprox: { __ rcpps(i.OutputSimd128Register(), i.InputSimd128Register(0)); break; } case kX64F32x4RecipSqrtApprox: { __ rsqrtps(i.OutputSimd128Register(), i.InputSimd128Register(0)); break; } case kX64F32x4Add: { DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); __ addps(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64F32x4AddHoriz: { DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); CpuFeatureScope sse_scope(tasm(), SSE3); __ haddps(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64F32x4Sub: { DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); __ subps(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64F32x4Mul: { DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); __ mulps(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64F32x4Min: { DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); __ minps(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64F32x4Max: { DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); __ maxps(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64F32x4Eq: { DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); __ cmpps(i.OutputSimd128Register(), i.InputSimd128Register(1), 0x0); break; } case kX64F32x4Ne: { DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); __ cmpps(i.OutputSimd128Register(), i.InputSimd128Register(1), 0x4); break; } case kX64F32x4Lt: { DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); __ cmpltps(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64F32x4Le: { DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0)); __ cmpleps(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I32x4Splat: { XMMRegister dst = i.OutputSimd128Register(); __ movd(dst, i.InputRegister(0)); __ pshufd(dst, dst, 0x0); break; } case kX64I32x4ExtractLane: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ Pextrd(i.OutputRegister(), i.InputSimd128Register(0), i.InputInt8(1)); break; } case kX64I32x4ReplaceLane: { CpuFeatureScope sse_scope(tasm(), SSE4_1); if (instr->InputAt(2)->IsRegister()) { __ Pinsrd(i.OutputSimd128Register(), i.InputRegister(2), i.InputInt8(1)); } else { __ Pinsrd(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1)); } break; } case kX64I32x4Neg: { CpuFeatureScope sse_scope(tasm(), SSSE3); XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(0); if (dst == src) { __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); __ psignd(dst, kScratchDoubleReg); } else { __ pxor(dst, dst); __ psubd(dst, src); } break; } case kX64I32x4Shl: { __ pslld(i.OutputSimd128Register(), i.InputInt8(1)); break; } case kX64I32x4ShrS: { __ psrad(i.OutputSimd128Register(), i.InputInt8(1)); break; } case kX64I32x4Add: { __ paddd(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I32x4AddHoriz: { CpuFeatureScope sse_scope(tasm(), SSSE3); __ phaddd(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I32x4Sub: { __ psubd(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I32x4Mul: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ pmulld(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I32x4MinS: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ pminsd(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I32x4MaxS: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ pmaxsd(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I32x4Eq: { __ pcmpeqd(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I32x4Ne: { __ pcmpeqd(i.OutputSimd128Register(), i.InputSimd128Register(1)); __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); __ pxor(i.OutputSimd128Register(), kScratchDoubleReg); break; } case kX64I32x4GtS: { __ pcmpgtd(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I32x4GeS: { CpuFeatureScope sse_scope(tasm(), SSE4_1); XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(1); __ pminsd(dst, src); __ pcmpeqd(dst, src); break; } case kX64I32x4ShrU: { __ psrld(i.OutputSimd128Register(), i.InputInt8(1)); break; } case kX64I32x4MinU: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ pminud(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I32x4MaxU: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ pmaxud(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I32x4GtU: { CpuFeatureScope sse_scope(tasm(), SSE4_1); XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(1); __ pmaxud(dst, src); __ pcmpeqd(dst, src); __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); __ pxor(dst, kScratchDoubleReg); break; } case kX64I32x4GeU: { CpuFeatureScope sse_scope(tasm(), SSE4_1); XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(1); __ pminud(dst, src); __ pcmpeqd(dst, src); break; } case kX64S128Zero: { XMMRegister dst = i.OutputSimd128Register(); __ xorps(dst, dst); break; } case kX64I16x8Splat: { XMMRegister dst = i.OutputSimd128Register(); __ movd(dst, i.InputRegister(0)); __ pshuflw(dst, dst, 0x0); __ pshufd(dst, dst, 0x0); break; } case kX64I16x8ExtractLane: { CpuFeatureScope sse_scope(tasm(), SSE4_1); Register dst = i.OutputRegister(); __ pextrw(dst, i.InputSimd128Register(0), i.InputInt8(1)); __ movsxwl(dst, dst); break; } case kX64I16x8ReplaceLane: { CpuFeatureScope sse_scope(tasm(), SSE4_1); if (instr->InputAt(2)->IsRegister()) { __ pinsrw(i.OutputSimd128Register(), i.InputRegister(2), i.InputInt8(1)); } else { __ pinsrw(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1)); } break; } case kX64I16x8Neg: { CpuFeatureScope sse_scope(tasm(), SSSE3); XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(0); if (dst == src) { __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); __ psignw(dst, kScratchDoubleReg); } else { __ pxor(dst, dst); __ psubw(dst, src); } break; } case kX64I16x8Shl: { __ psllw(i.OutputSimd128Register(), i.InputInt8(1)); break; } case kX64I16x8ShrS: { __ psraw(i.OutputSimd128Register(), i.InputInt8(1)); break; } case kX64I16x8Add: { __ paddw(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I16x8AddSaturateS: { __ paddsw(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I16x8AddHoriz: { CpuFeatureScope sse_scope(tasm(), SSSE3); __ phaddw(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I16x8Sub: { __ psubw(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I16x8SubSaturateS: { __ psubsw(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I16x8Mul: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ pmullw(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I16x8MinS: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ pminsw(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I16x8MaxS: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ pmaxsw(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I16x8Eq: { __ pcmpeqw(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I16x8Ne: { __ pcmpeqw(i.OutputSimd128Register(), i.InputSimd128Register(1)); __ pcmpeqw(kScratchDoubleReg, kScratchDoubleReg); __ pxor(i.OutputSimd128Register(), kScratchDoubleReg); break; } case kX64I16x8GtS: { __ pcmpgtw(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I16x8GeS: { CpuFeatureScope sse_scope(tasm(), SSE4_1); XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(1); __ pminsw(dst, src); __ pcmpeqw(dst, src); break; } case kX64I16x8ShrU: { __ psrlw(i.OutputSimd128Register(), i.InputInt8(1)); break; } case kX64I16x8AddSaturateU: { __ paddusw(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I16x8SubSaturateU: { __ psubusw(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I16x8MinU: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ pminuw(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I16x8MaxU: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ pmaxuw(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I16x8GtU: { CpuFeatureScope sse_scope(tasm(), SSE4_1); XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(1); __ pmaxuw(dst, src); __ pcmpeqw(dst, src); __ pcmpeqw(kScratchDoubleReg, kScratchDoubleReg); __ pxor(dst, kScratchDoubleReg); break; } case kX64I16x8GeU: { CpuFeatureScope sse_scope(tasm(), SSE4_1); XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(1); __ pminuw(dst, src); __ pcmpeqw(dst, src); break; } case kX64I8x16Splat: { CpuFeatureScope sse_scope(tasm(), SSSE3); XMMRegister dst = i.OutputSimd128Register(); __ movd(dst, i.InputRegister(0)); __ xorps(kScratchDoubleReg, kScratchDoubleReg); __ pshufb(dst, kScratchDoubleReg); break; } case kX64I8x16ExtractLane: { CpuFeatureScope sse_scope(tasm(), SSE4_1); Register dst = i.OutputRegister(); __ pextrb(dst, i.InputSimd128Register(0), i.InputInt8(1)); __ movsxbl(dst, dst); break; } case kX64I8x16ReplaceLane: { CpuFeatureScope sse_scope(tasm(), SSE4_1); if (instr->InputAt(2)->IsRegister()) { __ pinsrb(i.OutputSimd128Register(), i.InputRegister(2), i.InputInt8(1)); } else { __ pinsrb(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1)); } break; } case kX64I8x16Neg: { CpuFeatureScope sse_scope(tasm(), SSSE3); XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(0); if (dst == src) { __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); __ psignb(dst, kScratchDoubleReg); } else { __ pxor(dst, dst); __ psubb(dst, src); } break; } case kX64I8x16Add: { __ paddb(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I8x16AddSaturateS: { __ paddsb(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I8x16Sub: { __ psubb(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I8x16SubSaturateS: { __ psubsb(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I8x16MinS: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ pminsb(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I8x16MaxS: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ pmaxsb(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I8x16Eq: { __ pcmpeqb(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I8x16Ne: { __ pcmpeqb(i.OutputSimd128Register(), i.InputSimd128Register(1)); __ pcmpeqb(kScratchDoubleReg, kScratchDoubleReg); __ pxor(i.OutputSimd128Register(), kScratchDoubleReg); break; } case kX64I8x16GtS: { __ pcmpgtb(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I8x16GeS: { CpuFeatureScope sse_scope(tasm(), SSE4_1); XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(1); __ pminsb(dst, src); __ pcmpeqb(dst, src); break; } case kX64I8x16AddSaturateU: { __ paddusb(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I8x16SubSaturateU: { __ psubusb(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I8x16MinU: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ pminub(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I8x16MaxU: { CpuFeatureScope sse_scope(tasm(), SSE4_1); __ pmaxub(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64I8x16GtU: { CpuFeatureScope sse_scope(tasm(), SSE4_1); XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(1); __ pmaxub(dst, src); __ pcmpeqb(dst, src); __ pcmpeqb(kScratchDoubleReg, kScratchDoubleReg); __ pxor(dst, kScratchDoubleReg); break; } case kX64I8x16GeU: { CpuFeatureScope sse_scope(tasm(), SSE4_1); XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(1); __ pminub(dst, src); __ pcmpeqb(dst, src); break; } case kX64S128And: { __ pand(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64S128Or: { __ por(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64S128Xor: { __ pxor(i.OutputSimd128Register(), i.InputSimd128Register(1)); break; } case kX64S128Not: { XMMRegister dst = i.OutputSimd128Register(); XMMRegister src = i.InputSimd128Register(0); if (dst == src) { __ movaps(kScratchDoubleReg, dst); __ pcmpeqd(dst, dst); __ pxor(dst, kScratchDoubleReg); } else { __ pcmpeqd(dst, dst); __ pxor(dst, src); } break; } case kX64S128Select: { // Mask used here is stored in dst. XMMRegister dst = i.OutputSimd128Register(); __ movaps(kScratchDoubleReg, i.InputSimd128Register(1)); __ xorps(kScratchDoubleReg, i.InputSimd128Register(2)); __ andps(dst, kScratchDoubleReg); __ xorps(dst, i.InputSimd128Register(2)); break; } case kX64StackCheck: __ CompareRoot(rsp, Heap::kStackLimitRootIndex); break; case kWord32AtomicExchangeInt8: { __ xchgb(i.InputRegister(0), i.MemoryOperand(1)); __ movsxbl(i.InputRegister(0), i.InputRegister(0)); break; } case kWord32AtomicExchangeUint8: { __ xchgb(i.InputRegister(0), i.MemoryOperand(1)); __ movzxbl(i.InputRegister(0), i.InputRegister(0)); break; } case kWord32AtomicExchangeInt16: { __ xchgw(i.InputRegister(0), i.MemoryOperand(1)); __ movsxwl(i.InputRegister(0), i.InputRegister(0)); break; } case kWord32AtomicExchangeUint16: { __ xchgw(i.InputRegister(0), i.MemoryOperand(1)); __ movzxwl(i.InputRegister(0), i.InputRegister(0)); break; } case kWord32AtomicExchangeWord32: { __ xchgl(i.InputRegister(0), i.MemoryOperand(1)); break; } case kWord32AtomicCompareExchangeInt8: { __ lock(); __ cmpxchgb(i.MemoryOperand(2), i.InputRegister(1)); __ movsxbl(rax, rax); break; } case kWord32AtomicCompareExchangeUint8: { __ lock(); __ cmpxchgb(i.MemoryOperand(2), i.InputRegister(1)); __ movzxbl(rax, rax); break; } case kWord32AtomicCompareExchangeInt16: { __ lock(); __ cmpxchgw(i.MemoryOperand(2), i.InputRegister(1)); __ movsxwl(rax, rax); break; } case kWord32AtomicCompareExchangeUint16: { __ lock(); __ cmpxchgw(i.MemoryOperand(2), i.InputRegister(1)); __ movzxwl(rax, rax); break; } case kWord32AtomicCompareExchangeWord32: { __ lock(); __ cmpxchgl(i.MemoryOperand(2), i.InputRegister(1)); break; } #define ATOMIC_BINOP_CASE(op, inst) \ case kWord32Atomic##op##Int8: \ ASSEMBLE_ATOMIC_BINOP(inst, movb, cmpxchgb); \ __ movsxbl(rax, rax); \ break; \ case kWord32Atomic##op##Uint8: \ ASSEMBLE_ATOMIC_BINOP(inst, movb, cmpxchgb); \ __ movzxbl(rax, rax); \ break; \ case kWord32Atomic##op##Int16: \ ASSEMBLE_ATOMIC_BINOP(inst, movw, cmpxchgw); \ __ movsxwl(rax, rax); \ break; \ case kWord32Atomic##op##Uint16: \ ASSEMBLE_ATOMIC_BINOP(inst, movw, cmpxchgw); \ __ movzxwl(rax, rax); \ break; \ case kWord32Atomic##op##Word32: \ ASSEMBLE_ATOMIC_BINOP(inst, movl, cmpxchgl); \ break; ATOMIC_BINOP_CASE(Add, addl) ATOMIC_BINOP_CASE(Sub, subl) ATOMIC_BINOP_CASE(And, andl) ATOMIC_BINOP_CASE(Or, orl) ATOMIC_BINOP_CASE(Xor, xorl) #undef ATOMIC_BINOP_CASE case kX64Word64AtomicExchangeUint8: { __ xchgb(i.InputRegister(0), i.MemoryOperand(1)); __ movzxbq(i.InputRegister(0), i.InputRegister(0)); break; } case kX64Word64AtomicExchangeUint16: { __ xchgw(i.InputRegister(0), i.MemoryOperand(1)); __ movzxwq(i.InputRegister(0), i.InputRegister(0)); break; } case kX64Word64AtomicExchangeUint32: { __ xchgl(i.InputRegister(0), i.MemoryOperand(1)); break; } case kX64Word64AtomicExchangeUint64: { __ xchgq(i.InputRegister(0), i.MemoryOperand(1)); break; } case kX64Word64AtomicCompareExchangeUint8: { __ lock(); __ cmpxchgb(i.MemoryOperand(2), i.InputRegister(1)); __ movzxbq(rax, rax); break; } case kX64Word64AtomicCompareExchangeUint16: { __ lock(); __ cmpxchgw(i.MemoryOperand(2), i.InputRegister(1)); __ movzxwq(rax, rax); break; } case kX64Word64AtomicCompareExchangeUint32: { __ lock(); __ cmpxchgl(i.MemoryOperand(2), i.InputRegister(1)); break; } case kX64Word64AtomicCompareExchangeUint64: { __ lock(); __ cmpxchgq(i.MemoryOperand(2), i.InputRegister(1)); break; } #define ATOMIC64_BINOP_CASE(op, inst) \ case kX64Word64Atomic##op##Uint8: \ ASSEMBLE_ATOMIC64_BINOP(inst, movb, cmpxchgb); \ __ movzxbq(rax, rax); \ break; \ case kX64Word64Atomic##op##Uint16: \ ASSEMBLE_ATOMIC64_BINOP(inst, movw, cmpxchgw); \ __ movzxwq(rax, rax); \ break; \ case kX64Word64Atomic##op##Uint32: \ ASSEMBLE_ATOMIC64_BINOP(inst, movl, cmpxchgl); \ break; \ case kX64Word64Atomic##op##Uint64: \ ASSEMBLE_ATOMIC64_BINOP(inst, movq, cmpxchgq); \ break; ATOMIC64_BINOP_CASE(Add, addq) ATOMIC64_BINOP_CASE(Sub, subq) ATOMIC64_BINOP_CASE(And, andq) ATOMIC64_BINOP_CASE(Or, orq) ATOMIC64_BINOP_CASE(Xor, xorq) #undef ATOMIC64_BINOP_CASE case kWord32AtomicLoadInt8: case kWord32AtomicLoadUint8: case kWord32AtomicLoadInt16: case kWord32AtomicLoadUint16: case kWord32AtomicLoadWord32: case kWord32AtomicStoreWord8: case kWord32AtomicStoreWord16: case kWord32AtomicStoreWord32: case kX64Word64AtomicLoadUint8: case kX64Word64AtomicLoadUint16: case kX64Word64AtomicLoadUint32: case kX64Word64AtomicLoadUint64: case kX64Word64AtomicStoreWord8: case kX64Word64AtomicStoreWord16: case kX64Word64AtomicStoreWord32: case kX64Word64AtomicStoreWord64: UNREACHABLE(); // Won't be generated by instruction selector. break; } return kSuccess; } // NOLadability/fn_size) #undef ASSEMBLE_UNOP #undef ASSEMBLE_BINOP #undef ASSEMBLE_COMPARE #undef ASSEMBLE_MULT #undef ASSEMBLE_SHIFT #undef ASSEMBLE_MOVX #undef ASSEMBLE_SSE_BINOP #undef ASSEMBLE_SSE_UNOP #undef ASSEMBLE_AVX_BINOP #undef ASSEMBLE_IEEE754_BINOP #undef ASSEMBLE_IEEE754_UNOP #undef ASSEMBLE_ATOMIC_BINOP #undef ASSEMBLE_ATOMIC64_BINOP namespace { Condition FlagsConditionToCondition(FlagsCondition condition) { switch (condition) { case kUnorderedEqual: case kEqual: return equal; case kUnorderedNotEqual: case kNotEqual: return not_equal; case kSignedLessThan: return less; case kSignedGreaterThanOrEqual: return greater_equal; case kSignedLessThanOrEqual: return less_equal; case kSignedGreaterThan: return greater; case kUnsignedLessThan: return below; case kUnsignedGreaterThanOrEqual: return above_equal; case kUnsignedLessThanOrEqual: return below_equal; case kUnsignedGreaterThan: return above; case kOverflow: return overflow; case kNotOverflow: return no_overflow; default: break; } UNREACHABLE(); } } // namespace // Assembles branches after this instruction. void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) { Label::Distance flabel_distance = branch->fallthru ? Label::kNear : Label::kFar; Label* tlabel = branch->true_label; Label* flabel = branch->false_label; if (branch->condition == kUnorderedEqual) { __ j(parity_even, flabel, flabel_distance); } else if (branch->condition == kUnorderedNotEqual) { __ j(parity_even, tlabel); } __ j(FlagsConditionToCondition(branch->condition), tlabel); if (!branch->fallthru) __ jmp(flabel, flabel_distance); } void CodeGenerator::AssembleBranchPoisoning(FlagsCondition condition, Instruction* instr) { // TODO(jarin) Handle float comparisons (kUnordered[Not]Equal). if (condition == kUnorderedEqual || condition == kUnorderedNotEqual) { return; } condition = NegateFlagsCondition(condition); __ movl(kScratchRegister, Immediate(0)); __ cmovq(FlagsConditionToCondition(condition), kSpeculationPoisonRegister, kScratchRegister); } void CodeGenerator::AssembleArchDeoptBranch(Instruction* instr, BranchInfo* branch) { Label::Distance flabel_distance = branch->fallthru ? Label::kNear : Label::kFar; Label* tlabel = branch->true_label; Label* flabel = branch->false_label; Label nodeopt; if (branch->condition == kUnorderedEqual) { __ j(parity_even, flabel, flabel_distance); } else if (branch->condition == kUnorderedNotEqual) { __ j(parity_even, tlabel); } __ j(FlagsConditionToCondition(branch->condition), tlabel); if (FLAG_deopt_every_n_times > 0) { ExternalReference counter = ExternalReference::stress_deopt_count(isolate()); __ pushfq(); __ pushq(rax); __ load_rax(counter); __ decl(rax); __ j(not_zero, &nodeopt); __ Set(rax, FLAG_deopt_every_n_times); __ store_rax(counter); __ popq(rax); __ popfq(); __ jmp(tlabel); __ bind(&nodeopt); __ store_rax(counter); __ popq(rax); __ popfq(); } if (!branch->fallthru) { __ jmp(flabel, flabel_distance); } } void CodeGenerator::AssembleArchJump(RpoNumber target) { if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target)); } void CodeGenerator::AssembleArchTrap(Instruction* instr, FlagsCondition condition) { auto ool = new (zone()) WasmOutOfLineTrap(this, instr); Label* tlabel = ool->entry(); Label end; if (condition == kUnorderedEqual) { __ j(parity_even, &end); } else if (condition == kUnorderedNotEqual) { __ j(parity_even, tlabel); } __ j(FlagsConditionToCondition(condition), tlabel); __ bind(&end); } // Assembles boolean materializations after this instruction. void CodeGenerator::AssembleArchBoolean(Instruction* instr, FlagsCondition condition) { X64OperandConverter i(this, instr); Label done; // Materialize a full 64-bit 1 or 0 value. The result register is always the // last output of the instruction. Label check; DCHECK_NE(0u, instr->OutputCount()); Register reg = i.OutputRegister(instr->OutputCount() - 1); if (condition == kUnorderedEqual) { __ j(parity_odd, &check, Label::kNear); __ movl(reg, Immediate(0)); __ jmp(&done, Label::kNear); } else if (condition == kUnorderedNotEqual) { __ j(parity_odd, &check, Label::kNear); __ movl(reg, Immediate(1)); __ jmp(&done, Label::kNear); } __ bind(&check); __ setcc(FlagsConditionToCondition(condition), reg); __ movzxbl(reg, reg); __ bind(&done); } void CodeGenerator::AssembleArchBinarySearchSwitch(Instruction* instr) { X64OperandConverter i(this, instr); Register input = i.InputRegister(0); std::vector> cases; for (size_t index = 2; index < instr->InputCount(); index += 2) { cases.push_back({i.InputInt32(index + 0), GetLabel(i.InputRpo(index + 1))}); } AssembleArchBinarySearchSwitchRange(input, i.InputRpo(1), cases.data(), cases.data() + cases.size()); } void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) { X64OperandConverter i(this, instr); Register input = i.InputRegister(0); for (size_t index = 2; index < instr->InputCount(); index += 2) { __ cmpl(input, Immediate(i.InputInt32(index + 0))); __ j(equal, GetLabel(i.InputRpo(index + 1))); } AssembleArchJump(i.InputRpo(1)); } void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) { X64OperandConverter i(this, instr); Register input = i.InputRegister(0); int32_t const case_count = static_cast(instr->InputCount() - 2); Label** cases = zone()->NewArray(case_count); for (int32_t index = 0; index < case_count; ++index) { cases[index] = GetLabel(i.InputRpo(index + 2)); } Label* const table = AddJumpTable(cases, case_count); __ cmpl(input, Immediate(case_count)); __ j(above_equal, GetLabel(i.InputRpo(1))); __ leaq(kScratchRegister, Operand(table)); __ jmp(Operand(kScratchRegister, input, times_8, 0)); } namespace { static const int kQuadWordSize = 16; } // namespace void CodeGenerator::FinishFrame(Frame* frame) { auto call_descriptor = linkage()->GetIncomingDescriptor(); const RegList saves_fp = call_descriptor->CalleeSavedFPRegisters(); if (saves_fp != 0) { frame->AlignSavedCalleeRegisterSlots(); if (saves_fp != 0) { // Save callee-saved XMM registers. const uint32_t saves_fp_count = base::bits::CountPopulation(saves_fp); frame->AllocateSavedCalleeRegisterSlots(saves_fp_count * (kQuadWordSize / kPointerSize)); } } const RegList saves = call_descriptor->CalleeSavedRegisters(); if (saves != 0) { // Save callee-saved registers. int count = 0; for (int i = Register::kNumRegisters - 1; i >= 0; i--) { if (((1 << i) & saves)) { ++count; } } frame->AllocateSavedCalleeRegisterSlots(count); } } void CodeGenerator::AssembleConstructFrame() { auto call_descriptor = linkage()->GetIncomingDescriptor(); if (frame_access_state()->has_frame()) { int pc_base = __ pc_offset(); if (call_descriptor->IsCFunctionCall()) { __ pushq(rbp); __ movq(rbp, rsp); } else if (call_descriptor->IsJSFunctionCall()) { __ Prologue(); if (call_descriptor->PushArgumentCount()) { __ pushq(kJavaScriptCallArgCountRegister); } } else { __ StubPrologue(info()->GetOutputStackFrameType()); if (call_descriptor->IsWasmFunctionCall()) { __ pushq(kWasmInstanceRegister); } } unwinding_info_writer_.MarkFrameConstructed(pc_base); } int shrink_slots = frame()->GetTotalFrameSlotCount() - call_descriptor->CalculateFixedFrameSize(); if (info()->is_osr()) { // TurboFan OSR-compiled functions cannot be entered directly. __ Abort(AbortReason::kShouldNotDirectlyEnterOsrFunction); // Unoptimized code jumps directly to this entrypoint while the unoptimized // frame is still on the stack. Optimized code uses OSR values directly from // the unoptimized frame. Thus, all that needs to be done is to allocate the // remaining stack slots. if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --"); osr_pc_offset_ = __ pc_offset(); shrink_slots -= static_cast(osr_helper()->UnoptimizedFrameSlots()); ResetSpeculationPoison(); } const RegList saves = call_descriptor->CalleeSavedRegisters(); const RegList saves_fp = call_descriptor->CalleeSavedFPRegisters(); if (shrink_slots > 0) { DCHECK(frame_access_state()->has_frame()); if (info()->IsWasm() && shrink_slots > 128) { // For WebAssembly functions with big frames we have to do the stack // overflow check before we construct the frame. Otherwise we may not // have enough space on the stack to call the runtime for the stack // overflow. Label done; // If the frame is bigger than the stack, we throw the stack overflow // exception unconditionally. Thereby we can avoid the integer overflow // check in the condition code. if (shrink_slots * kPointerSize < FLAG_stack_size * 1024) { __ movq(kScratchRegister, FieldOperand(kWasmInstanceRegister, WasmInstanceObject::kRealStackLimitAddressOffset)); __ movq(kScratchRegister, Operand(kScratchRegister, 0)); __ addq(kScratchRegister, Immediate(shrink_slots * kPointerSize)); __ cmpq(rsp, kScratchRegister); __ j(above_equal, &done); } __ movp(rcx, FieldOperand(kWasmInstanceRegister, WasmInstanceObject::kCEntryStubOffset)); __ Move(rsi, Smi::kZero); __ CallRuntimeWithCEntry(Runtime::kThrowWasmStackOverflow, rcx); ReferenceMap* reference_map = new (zone()) ReferenceMap(zone()); RecordSafepoint(reference_map, Safepoint::kSimple, 0, Safepoint::kNoLazyDeopt); __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap); __ bind(&done); } // Skip callee-saved and return slots, which are created below. shrink_slots -= base::bits::CountPopulation(saves); shrink_slots -= base::bits::CountPopulation(saves_fp) * (kQuadWordSize / kPointerSize); shrink_slots -= frame()->GetReturnSlotCount(); if (shrink_slots > 0) { __ subq(rsp, Immediate(shrink_slots * kPointerSize)); } } if (saves_fp != 0) { // Save callee-saved XMM registers. const uint32_t saves_fp_count = base::bits::CountPopulation(saves_fp); const int stack_size = saves_fp_count * kQuadWordSize; // Adjust the stack pointer. __ subp(rsp, Immediate(stack_size)); // Store the registers on the stack. int slot_idx = 0; for (int i = 0; i < XMMRegister::kNumRegisters; i++) { if (!((1 << i) & saves_fp)) continue; __ movdqu(Operand(rsp, kQuadWordSize * slot_idx), XMMRegister::from_code(i)); slot_idx++; } } if (saves != 0) { // Save callee-saved registers. for (int i = Register::kNumRegisters - 1; i >= 0; i--) { if (!((1 << i) & saves)) continue; __ pushq(Register::from_code(i)); } } // Allocate return slots (located after callee-saved). if (frame()->GetReturnSlotCount() > 0) { __ subq(rsp, Immediate(frame()->GetReturnSlotCount() * kPointerSize)); } } void CodeGenerator::AssembleReturn(InstructionOperand* pop) { auto call_descriptor = linkage()->GetIncomingDescriptor(); // Restore registers. const RegList saves = call_descriptor->CalleeSavedRegisters(); if (saves != 0) { const int returns = frame()->GetReturnSlotCount(); if (returns != 0) { __ addq(rsp, Immediate(returns * kPointerSize)); } for (int i = 0; i < Register::kNumRegisters; i++) { if (!((1 << i) & saves)) continue; __ popq(Register::from_code(i)); } } const RegList saves_fp = call_descriptor->CalleeSavedFPRegisters(); if (saves_fp != 0) { const uint32_t saves_fp_count = base::bits::CountPopulation(saves_fp); const int stack_size = saves_fp_count * kQuadWordSize; // Load the registers from the stack. int slot_idx = 0; for (int i = 0; i < XMMRegister::kNumRegisters; i++) { if (!((1 << i) & saves_fp)) continue; __ movdqu(XMMRegister::from_code(i), Operand(rsp, kQuadWordSize * slot_idx)); slot_idx++; } // Adjust the stack pointer. __ addp(rsp, Immediate(stack_size)); } unwinding_info_writer_.MarkBlockWillExit(); // Might need rcx for scratch if pop_size is too big or if there is a variable // pop count. DCHECK_EQ(0u, call_descriptor->CalleeSavedRegisters() & rcx.bit()); DCHECK_EQ(0u, call_descriptor->CalleeSavedRegisters() & rdx.bit()); size_t pop_size = call_descriptor->StackParameterCount() * kPointerSize; X64OperandConverter g(this, nullptr); if (call_descriptor->IsCFunctionCall()) { AssembleDeconstructFrame(); } else if (frame_access_state()->has_frame()) { if (pop->IsImmediate() && g.ToConstant(pop).ToInt32() == 0) { // Canonicalize JSFunction return sites for now. if (return_label_.is_bound()) { __ jmp(&return_label_); return; } else { __ bind(&return_label_); AssembleDeconstructFrame(); } } else { AssembleDeconstructFrame(); } } if (pop->IsImmediate()) { pop_size += g.ToConstant(pop).ToInt32() * kPointerSize; CHECK_LT(pop_size, static_cast(std::numeric_limits::max())); __ Ret(static_cast(pop_size), rcx); } else { Register pop_reg = g.ToRegister(pop); Register scratch_reg = pop_reg == rcx ? rdx : rcx; __ popq(scratch_reg); __ leaq(rsp, Operand(rsp, pop_reg, times_8, static_cast(pop_size))); __ jmp(scratch_reg); } } void CodeGenerator::FinishCode() { tasm()->PatchConstPool(); } void CodeGenerator::AssembleMove(InstructionOperand* source, InstructionOperand* destination) { X64OperandConverter g(this, nullptr); // Helper function to write the given constant to the dst register. auto MoveConstantToRegister = [&](Register dst, Constant src) { switch (src.type()) { case Constant::kInt32: { if (RelocInfo::IsWasmPtrReference(src.rmode())) { __ movq(dst, src.ToInt64(), src.rmode()); } else { int32_t value = src.ToInt32(); if (value == 0) { __ xorl(dst, dst); } else { __ movl(dst, Immediate(value)); } } break; } case Constant::kInt64: if (RelocInfo::IsWasmPtrReference(src.rmode())) { __ movq(dst, src.ToInt64(), src.rmode()); } else { __ Set(dst, src.ToInt64()); } break; case Constant::kFloat32: __ MoveNumber(dst, src.ToFloat32()); break; case Constant::kFloat64: __ MoveNumber(dst, src.ToFloat64().value()); break; case Constant::kExternalReference: __ Move(dst, src.ToExternalReference()); break; case Constant::kHeapObject: { Handle src_object = src.ToHeapObject(); Heap::RootListIndex index; if (IsMaterializableFromRoot(src_object, &index)) { __ LoadRoot(dst, index); } else { __ Move(dst, src_object); } break; } case Constant::kRpoNumber: UNREACHABLE(); // TODO(dcarney): load of labels on x64. break; } }; // Helper function to write the given constant to the stack. auto MoveConstantToSlot = [&](Operand dst, Constant src) { if (!RelocInfo::IsWasmPtrReference(src.rmode())) { switch (src.type()) { case Constant::kInt32: __ movq(dst, Immediate(src.ToInt32())); return; case Constant::kInt64: __ Set(dst, src.ToInt64()); return; default: break; } } MoveConstantToRegister(kScratchRegister, src); __ movq(dst, kScratchRegister); }; // Dispatch on the source and destination operand kinds. switch (MoveType::InferMove(source, destination)) { case MoveType::kRegisterToRegister: if (source->IsRegister()) { __ movq(g.ToRegister(destination), g.ToRegister(source)); } else { DCHECK(source->IsFPRegister()); __ Movapd(g.ToDoubleRegister(destination), g.ToDoubleRegister(source)); } return; case MoveType::kRegisterToStack: { Operand dst = g.ToOperand(destination); if (source->IsRegister()) { __ movq(dst, g.ToRegister(source)); } else { DCHECK(source->IsFPRegister()); XMMRegister src = g.ToDoubleRegister(source); MachineRepresentation rep = LocationOperand::cast(source)->representation(); if (rep != MachineRepresentation::kSimd128) { __ Movsd(dst, src); } else { __ Movups(dst, src); } } return; } case MoveType::kStackToRegister: { Operand src = g.ToOperand(source); if (source->IsStackSlot()) { __ movq(g.ToRegister(destination), src); } else { DCHECK(source->IsFPStackSlot()); XMMRegister dst = g.ToDoubleRegister(destination); MachineRepresentation rep = LocationOperand::cast(source)->representation(); if (rep != MachineRepresentation::kSimd128) { __ Movsd(dst, src); } else { __ Movups(dst, src); } } return; } case MoveType::kStackToStack: { Operand src = g.ToOperand(source); Operand dst = g.ToOperand(destination); if (source->IsStackSlot()) { // Spill on demand to use a temporary register for memory-to-memory // moves. __ movq(kScratchRegister, src); __ movq(dst, kScratchRegister); } else { MachineRepresentation rep = LocationOperand::cast(source)->representation(); if (rep != MachineRepresentation::kSimd128) { __ Movsd(kScratchDoubleReg, src); __ Movsd(dst, kScratchDoubleReg); } else { DCHECK(source->IsSimd128StackSlot()); __ Movups(kScratchDoubleReg, src); __ Movups(dst, kScratchDoubleReg); } } return; } case MoveType::kConstantToRegister: { Constant src = g.ToConstant(source); if (destination->IsRegister()) { MoveConstantToRegister(g.ToRegister(destination), src); } else { DCHECK(destination->IsFPRegister()); XMMRegister dst = g.ToDoubleRegister(destination); if (src.type() == Constant::kFloat32) { // TODO(turbofan): Can we do better here? __ Move(dst, bit_cast(src.ToFloat32())); } else { DCHECK_EQ(src.type(), Constant::kFloat64); __ Move(dst, src.ToFloat64().AsUint64()); } } return; } case MoveType::kConstantToStack: { Constant src = g.ToConstant(source); Operand dst = g.ToOperand(destination); if (destination->IsStackSlot()) { MoveConstantToSlot(dst, src); } else { DCHECK(destination->IsFPStackSlot()); if (src.type() == Constant::kFloat32) { __ movl(dst, Immediate(bit_cast(src.ToFloat32()))); } else { DCHECK_EQ(src.type(), Constant::kFloat64); __ movq(kScratchRegister, src.ToFloat64().AsUint64()); __ movq(dst, kScratchRegister); } } return; } } UNREACHABLE(); } void CodeGenerator::AssembleSwap(InstructionOperand* source, InstructionOperand* destination) { X64OperandConverter g(this, nullptr); // Dispatch on the source and destination operand kinds. Not all // combinations are possible. switch (MoveType::InferSwap(source, destination)) { case MoveType::kRegisterToRegister: { if (source->IsRegister()) { Register src = g.ToRegister(source); Register dst = g.ToRegister(destination); __ movq(kScratchRegister, src); __ movq(src, dst); __ movq(dst, kScratchRegister); } else { DCHECK(source->IsFPRegister()); XMMRegister src = g.ToDoubleRegister(source); XMMRegister dst = g.ToDoubleRegister(destination); __ Movapd(kScratchDoubleReg, src); __ Movapd(src, dst); __ Movapd(dst, kScratchDoubleReg); } return; } case MoveType::kRegisterToStack: { if (source->IsRegister()) { Register src = g.ToRegister(source); __ pushq(src); frame_access_state()->IncreaseSPDelta(1); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), kPointerSize); __ movq(src, g.ToOperand(destination)); frame_access_state()->IncreaseSPDelta(-1); __ popq(g.ToOperand(destination)); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), -kPointerSize); } else { DCHECK(source->IsFPRegister()); XMMRegister src = g.ToDoubleRegister(source); Operand dst = g.ToOperand(destination); MachineRepresentation rep = LocationOperand::cast(source)->representation(); if (rep != MachineRepresentation::kSimd128) { __ Movsd(kScratchDoubleReg, src); __ Movsd(src, dst); __ Movsd(dst, kScratchDoubleReg); } else { __ Movups(kScratchDoubleReg, src); __ Movups(src, dst); __ Movups(dst, kScratchDoubleReg); } } return; } case MoveType::kStackToStack: { Operand src = g.ToOperand(source); Operand dst = g.ToOperand(destination); MachineRepresentation rep = LocationOperand::cast(source)->representation(); if (rep != MachineRepresentation::kSimd128) { Register tmp = kScratchRegister; __ movq(tmp, dst); __ pushq(src); // Then use stack to copy src to destination. unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), kPointerSize); __ popq(dst); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), -kPointerSize); __ movq(src, tmp); } else { // Without AVX, misaligned reads and writes will trap. Move using the // stack, in two parts. __ movups(kScratchDoubleReg, dst); // Save dst in scratch register. __ pushq(src); // Then use stack to copy src to destination. unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), kPointerSize); __ popq(dst); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), -kPointerSize); __ pushq(g.ToOperand(source, kPointerSize)); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), kPointerSize); __ popq(g.ToOperand(destination, kPointerSize)); unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(), -kPointerSize); __ movups(src, kScratchDoubleReg); } return; } default: UNREACHABLE(); break; } } void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) { for (size_t index = 0; index < target_count; ++index) { __ dq(targets[index]); } } #undef __ } // namespace compiler } // namespace internal } // namespace v8