1 /* 2 * Copyright (c) 2021-2025 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef COMPILER_OPTIMIZER_CODEGEN_TARGET_AMD64_TARGET_H 17 #define COMPILER_OPTIMIZER_CODEGEN_TARGET_AMD64_TARGET_H 18 19 #include "compiler/optimizer/code_generator/callconv.h" 20 21 #include "asmjit/x86.h" 22 #include "target_info.h" 23 24 namespace ark::compiler::amd64 { 25 const size_t MAX_SCALAR_PARAM_ID = 5; // %rdi, %rsi, %rdx, %rcx, %r8, %r9 26 const size_t MAX_VECTOR_PARAM_ID = 7; // %xmm0-%xmm7 27 28 class AsmJitErrorHandler : public asmjit::ErrorHandler { 29 public: 30 explicit AsmJitErrorHandler(Encoder *encoder); 31 NO_MOVE_SEMANTIC(AsmJitErrorHandler); 32 NO_COPY_SEMANTIC(AsmJitErrorHandler); 33 ~AsmJitErrorHandler() override = default; 34 35 void handleError([[maybe_unused]] asmjit::Error err, [[maybe_unused]] const char *message, 36 [[maybe_unused]] asmjit::BaseEmitter *origin) override; 37 38 private: 39 Encoder *encoder_ {nullptr}; 40 }; 41 42 class RegList { 43 public: RegList(size_t mask)44 explicit RegList(size_t mask) : mask_(mask) 45 { 46 for (size_t i = 0; i < sizeof(size_t) * BITS_PER_BYTE; ++i) { 47 if (Has(i)) { 48 ++count_; 49 } 50 } 51 } 52 53 DEFAULT_MOVE_SEMANTIC(RegList); 54 DEFAULT_COPY_SEMANTIC(RegList); 55 ~RegList() = default; 56 size_t()57 explicit operator size_t() const 58 { 59 return mask_; 60 } 61 IsEmpty()62 bool IsEmpty() const 63 { 64 return count_ == size_t(0); 65 } 66 GetCount()67 size_t GetCount() const 68 { 69 return count_; 70 } 71 Has(size_t i)72 bool Has(size_t i) const 73 { 74 return (mask_ & (size_t(1) << i)) != 0; 75 } 76 Add(size_t i)77 void Add(size_t i) 78 { 79 if (Has(i)) { 80 return; 81 } 82 mask_ |= size_t(1) << i; 83 ++count_; 84 } 85 Remove(size_t i)86 void Remove(size_t i) 87 { 88 if (!Has(i)) { 89 return; 90 } 91 mask_ &= ~(size_t(1) << i); 92 --count_; 93 } 94 Pop()95 size_t Pop() 96 { 97 ASSERT(!IsEmpty()); 98 size_t i = __builtin_ctzll(mask_); 99 Remove(i); 100 return i; 101 } 102 GetMask()103 size_t GetMask() const 104 { 105 return mask_; 106 } 107 108 private: 109 size_t mask_ {0}; 110 size_t count_ {0}; 111 }; 112 113 class ArchMem { 114 public: 115 explicit ArchMem(MemRef mem); 116 DEFAULT_MOVE_SEMANTIC(ArchMem); 117 DEFAULT_COPY_SEMANTIC(ArchMem); 118 ~ArchMem() = default; 119 120 asmjit::x86::Mem Prepare(asmjit::x86::Assembler *masm); 121 122 private: 123 int64_t bigShift_ {0}; 124 asmjit::x86::Mem mem_; 125 bool needExtendIndex_ {false}; 126 bool isPrepared_ {false}; 127 }; 128 129 /* 130 * Scalar registers mapping: 131 * +-----------+---------------------+ 132 * | AMD64 Reg | Panda Reg | 133 * +-----------+---------------------+ 134 * | rax | r0 | 135 * | rcx | r1 | 136 * | rdx | r2 | 137 * | rbx | r3 (renamed to r11) | 138 * | rsp | r4 (renamed to r10) | 139 * | rbp | r5 (renamed to r9) | 140 * | rsi | r6 | 141 * | rdi | r7 | 142 * | r8 | r8 | 143 * | r9 | r9 (renamed to r5) | 144 * | r10 | r10 (renamed to r4) | 145 * | r11 | r11 (renamed to r3) | 146 * | r12 | r12 | 147 * | r13 | r13 | 148 * | r14 | r14 | 149 * | r15 | r15 | 150 * | <no reg> | r16-r31 | 151 * +-----------+---------------------+ 152 * 153 * Vector registers mapping: 154 * xmm[i] <-> vreg[i], 0 <= i <= 15 155 */ 156 class Amd64RegisterDescription final : public RegistersDescription { 157 public: 158 explicit Amd64RegisterDescription(ArenaAllocator *allocator); 159 NO_MOVE_SEMANTIC(Amd64RegisterDescription); 160 NO_COPY_SEMANTIC(Amd64RegisterDescription); 161 ~Amd64RegisterDescription() override = default; 162 163 ArenaVector<Reg> GetCalleeSaved() override; 164 void SetCalleeSaved(const ArenaVector<Reg> ®s) override; 165 // Set used regs - change GetCallee 166 void SetUsedRegs(const ArenaVector<Reg> ®s) override; 167 168 RegMask GetCallerSavedRegMask() const override; 169 VRegMask GetCallerSavedVRegMask() const override; 170 bool IsCalleeRegister(Reg reg) override; 171 Reg GetZeroReg() const override; 172 bool IsZeroReg([[maybe_unused]] Reg reg) const override; 173 Reg::RegIDType GetTempReg() override; 174 Reg::RegIDType GetTempVReg() override; 175 RegMask GetDefaultRegMask() const override; 176 VRegMask GetVRegMask() override; 177 178 // Check register mapping 179 bool SupportMapping(uint32_t type) override; 180 bool IsValid() const override; 181 bool IsRegUsed(ArenaVector<Reg> vecReg, Reg reg) override; 182 183 public: 184 // Special implementation-specific getters 185 size_t GetCalleeSavedR(); 186 size_t GetCalleeSavedV(); 187 size_t GetCallerSavedR(); 188 size_t GetCallerSavedV(); 189 190 Reg AcquireScratchRegister(TypeInfo type); 191 void AcquireScratchRegister(Reg reg); 192 void ReleaseScratchRegister(Reg reg); 193 bool IsScratchRegisterReleased(Reg reg) const; 194 RegList GetScratchRegisters() const; 195 RegList GetScratchFPRegisters() const; 196 size_t GetScratchRegistersCount() const; 197 size_t GetScratchFPRegistersCount() const; 198 RegMask GetScratchRegistersMask() const; 199 RegMask GetScratchFpRegistersMask() const; 200 201 private: 202 ArenaVector<Reg> usedRegs_; 203 204 RegList calleeSaved_ {GetCalleeRegsMask(Arch::X86_64, false).GetValue()}; 205 RegList callerSaved_ {GetCallerRegsMask(Arch::X86_64, false).GetValue()}; 206 207 RegList calleeSavedv_ {GetCalleeRegsMask(Arch::X86_64, true).GetValue()}; 208 RegList callerSavedv_ {GetCallerRegsMask(Arch::X86_64, true).GetValue()}; 209 210 RegList scratch_ {compiler::arch_info::x86_64::TEMP_REGS.to_ulong()}; 211 RegList scratchv_ {compiler::arch_info::x86_64::TEMP_FP_REGS.to_ulong()}; 212 }; // Amd64RegisterDescription 213 214 class Amd64Encoder; 215 216 class Amd64LabelHolder final : public LabelHolder { 217 public: 218 using LabelType = asmjit::Label; 219 Amd64LabelHolder(Encoder * enc)220 explicit Amd64LabelHolder(Encoder *enc) : LabelHolder(enc), labels_(enc->GetAllocator()->Adapter()) {}; 221 NO_MOVE_SEMANTIC(Amd64LabelHolder); 222 NO_COPY_SEMANTIC(Amd64LabelHolder); 223 ~Amd64LabelHolder() override = default; 224 225 LabelId CreateLabel() override; 226 void CreateLabels(LabelId max) override; 227 void BindLabel(LabelId id) override; 228 LabelType *GetLabel(LabelId id); 229 LabelId Size() override; 230 231 private: 232 ArenaVector<LabelType *> labels_; 233 LabelId id_ {0}; 234 friend Amd64Encoder; 235 }; // Amd64LabelHolder 236 237 class Amd64Encoder final : public Encoder { 238 public: 239 using Encoder::Encoder; 240 explicit Amd64Encoder(ArenaAllocator *allocator); 241 ~Amd64Encoder() override; 242 NO_COPY_SEMANTIC(Amd64Encoder); 243 NO_MOVE_SEMANTIC(Amd64Encoder); 244 245 LabelHolder *GetLabels() const override; 246 bool IsValid() const override; 247 static constexpr auto GetTarget(); 248 249 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) 250 #define UNARY_OPERATION(opc) void Encode##opc(Reg dst, Reg src0) override; /* CC-OFF(G.PRE.09) code generation */ 251 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) 252 #define BINARY_OPERATION(opc) \ 253 void Encode##opc(Reg dst, Reg src0, Reg src1) override; \ 254 void Encode##opc(Reg dst, Reg src0, Imm src1) override; /* CC-OFF(G.PRE.09) code generation */ 255 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) 256 #define INST_DEF(OPCODE, MACRO) MACRO(OPCODE) 257 258 ENCODE_MATH_LIST(INST_DEF) 259 260 #undef UNARY_OPERATION 261 #undef BINARY_OPERATION 262 #undef INST_DEF 263 264 void EncodeNop() override; 265 266 // Additional special instructions 267 void EncodeAdd(Reg dst, Reg src0, Shift src1) override; 268 269 void EncodeCastToBool(Reg dst, Reg src) override; 270 void EncodeCast(Reg dst, bool dstSigned, Reg src, bool srcSigned) override; 271 void EncodeFastPathDynamicCast(Reg dst, Reg src, LabelHolder::LabelId slow) override; 272 void EncodeJsDoubleToCharCast(Reg dst, Reg src, Reg tmp, uint32_t failureResult) override; 273 void EncodeMin(Reg dst, bool dstSigned, Reg src0, Reg src1) override; 274 void EncodeDiv(Reg dst, bool dstSigned, Reg src0, Reg src1) override; 275 void EncodeMod(Reg dst, bool dstSigned, Reg src0, Reg src1) override; 276 void EncodeDiv(Reg dst, Reg src0, Imm imm, bool isSigned) override; 277 void EncodeSignedDiv(Reg dst, Reg src0, Imm imm); 278 void EncodeUnsignedDiv(Reg dst, Reg src0, Imm imm); 279 void EncodeMod(Reg dst, Reg src0, Imm imm, bool isSigned) override; 280 void EncodeMax(Reg dst, bool dstSigned, Reg src0, Reg src1) override; 281 282 void EncodeAddOverflow(compiler::LabelHolder::LabelId id, Reg dst, Reg src0, Reg src1, Condition cc) override; 283 void EncodeSubOverflow(compiler::LabelHolder::LabelId id, Reg dst, Reg src0, Reg src1, Condition cc) override; 284 void EncodeNegOverflowAndZero(compiler::LabelHolder::LabelId id, Reg dst, Reg src) override; 285 286 void EncodeLdr(Reg dst, bool dstSigned, MemRef mem) override; 287 void EncodeLdrAcquire(Reg dst, bool dstSigned, MemRef mem) override; 288 289 void EncodeMov(Reg dst, Imm src) override; 290 void EncodeStr(Reg src, MemRef mem) override; 291 void EncodeStrRelease(Reg src, MemRef mem) override; 292 // zerod high part: [reg.size, 64) 293 void EncodeStrz(Reg src, MemRef mem) override; 294 void EncodeSti(int64_t src, uint8_t srcSizeBytes, MemRef mem) override; 295 void EncodeSti(float src, MemRef mem) override; 296 void EncodeSti(double src, MemRef mem) override; 297 // size must be 8, 16,32 or 64 298 void EncodeMemCopy(MemRef memFrom, MemRef memTo, size_t size) override; 299 // size must be 8, 16,32 or 64 300 // zerod high part: [reg.size, 64) 301 void EncodeMemCopyz(MemRef memFrom, MemRef memTo, size_t size) override; 302 303 void EncodeCmp(Reg dst, Reg src0, Reg src1, Condition cc) override; 304 void EncodeCompare(Reg dst, Reg src0, Reg src1, Condition cc) override; 305 void EncodeCompareTest(Reg dst, Reg src0, Reg src1, Condition cc) override; 306 void EncodeAtomicByteOr(Reg addr, Reg value, bool fastEncoding) override; 307 308 void EncodeSelect(ArgsSelect &&args) override; 309 void EncodeSelect(ArgsSelectImm &&args) override; 310 void EncodeSelectTest(ArgsSelect &&args) override; 311 void EncodeSelectTest(ArgsSelectImm &&args) override; 312 313 void EncodeLdp(Reg dst0, Reg dst1, bool dstSigned, MemRef mem) override; 314 void EncodeStp(Reg src0, Reg src1, MemRef mem) override; 315 316 /* builtins-related encoders */ 317 void EncodeIsInf(Reg dst, Reg src) override; 318 void EncodeIsInteger(Reg dst, Reg src) override; 319 void EncodeIsSafeInteger(Reg dst, Reg src) override; 320 void EncodeBitCount(Reg dst, Reg src) override; 321 void EncodeCountLeadingZeroBits(Reg dst, Reg src) override; 322 void EncodeCountTrailingZeroBits(Reg dst, Reg src) override; 323 void EncodeCeil([[maybe_unused]] Reg dst, [[maybe_unused]] Reg src) override; 324 void EncodeFloor([[maybe_unused]] Reg dst, [[maybe_unused]] Reg src) override; 325 void EncodeRint([[maybe_unused]] Reg dst, [[maybe_unused]] Reg src) override; 326 void EncodeTrunc([[maybe_unused]] Reg dst, [[maybe_unused]] Reg src) override; 327 void EncodeRoundAway([[maybe_unused]] Reg dst, [[maybe_unused]] Reg src) override; 328 void EncodeRoundToPInfReturnFloat([[maybe_unused]] Reg dst, [[maybe_unused]] Reg src) override; 329 void EncodeRoundToPInfReturnScalar([[maybe_unused]] Reg dst, [[maybe_unused]] Reg src) override; 330 void EncodeReverseBytes(Reg dst, Reg src) override; 331 void EncodeReverseBits(Reg dst, Reg src) override; 332 void EncodeReverseHalfWords(Reg dst, Reg src) override; 333 void EncodeFpToBits(Reg dst, Reg src) override; 334 void EncodeMoveBitsRaw(Reg dst, Reg src) override; 335 void EncodeUnsignedExtendBytesToShorts(Reg dst, Reg src) override; 336 337 bool CanEncodeImmAddSubCmp(int64_t imm, uint32_t size, bool signedCompare) override; 338 bool CanEncodeImmLogical(uint64_t imm, uint32_t size) override; 339 bool CanEncodeScale(uint64_t imm, uint32_t size) override; 340 bool CanEncodeBitCount() override; 341 bool CanOptimizeImmDivMod(uint64_t imm, bool isSigned) const override; 342 343 void EncodeCompareAndSwap(Reg dst, Reg obj, Reg offset, Reg val, Reg newval) override; 344 void EncodeCompareAndSwap(Reg dst, Reg addr, Reg val, Reg newval) override; 345 void EncodeUnsafeGetAndSet(Reg dst, Reg obj, Reg offset, Reg val) override; 346 void EncodeUnsafeGetAndAdd(Reg dst, Reg obj, Reg offset, Reg val, Reg tmp) override; 347 void EncodeMemoryBarrier(memory_order::Order order) override; 348 void EncodeStackOverflowCheck(ssize_t offset) override; 349 350 void EncodeGetCurrentPc(Reg dst) override; 351 352 size_t GetCursorOffset() const override; 353 void SetCursorOffset(size_t offset) override; 354 355 Reg AcquireScratchRegister(TypeInfo type) override; 356 void AcquireScratchRegister(Reg reg) override; 357 void ReleaseScratchRegister(Reg reg) override; 358 bool IsScratchRegisterReleased(Reg reg) const override; 359 RegMask GetScratchRegistersMask() const override; 360 RegMask GetScratchFpRegistersMask() const override; 361 RegMask GetAvailableScratchRegisters() const override; 362 VRegMask GetAvailableScratchFpRegisters() const override; 363 TypeInfo GetRefType() override; 364 365 size_t DisasmInstr(std::ostream &stream, size_t pc, ssize_t codeOffset) const override; 366 367 void *BufferData() const override; 368 size_t BufferSize() const override; 369 370 bool InitMasm() override; 371 void Finalize() override; 372 373 void MakeCall(compiler::RelocationInfo *relocation) override; 374 void MakeCall(LabelHolder::LabelId id) override; 375 void MakeCall(const void *entryPoint) override; 376 void MakeCall(Reg reg) override; 377 void MakeCall(MemRef entryPoint) override; 378 void MakeCallAot(intptr_t offset) override; 379 void MakeCallByOffset(intptr_t offset) override; 380 void MakeLoadAotTable(intptr_t offset, Reg reg) override; 381 void MakeLoadAotTableAddr(intptr_t offset, Reg addr, Reg val) override; 382 bool CanMakeCallByOffset(intptr_t offset) override; 383 384 // Encode unconditional branch 385 void EncodeJump(LabelHolder::LabelId id) override; 386 387 // Encode jump with compare to zero 388 void EncodeJump(LabelHolder::LabelId id, Reg src, Condition cc) override; 389 390 // Compare reg and immediate and branch 391 void EncodeJump(LabelHolder::LabelId id, Reg src, Imm imm, Condition cc) override; 392 393 // Compare two regs and branch 394 void EncodeJump(LabelHolder::LabelId id, Reg src0, Reg src1, Condition cc) override; 395 396 // Compare reg and immediate and branch 397 void EncodeJumpTest(LabelHolder::LabelId id, Reg src, Imm imm, Condition cc) override; 398 399 // Compare two regs and branch 400 void EncodeJumpTest(LabelHolder::LabelId id, Reg src0, Reg src1, Condition cc) override; 401 402 // Encode jump by register value 403 void EncodeJump(Reg dst) override; 404 405 void EncodeJump(RelocationInfo *relocation) override; 406 void EncodeBitTestAndBranch(LabelHolder::LabelId id, compiler::Reg reg, uint32_t bitPos, bool bitValue) override; 407 void EncodeAbort() override; 408 void EncodeReturn() override; 409 410 void MakeLibCall(Reg dst, Reg src0, Reg src1, void *entryPoint); 411 412 void SaveRegisters(RegMask registers, ssize_t slot, size_t startReg, bool isFp) override; 413 void LoadRegisters(RegMask registers, ssize_t slot, size_t startReg, bool isFp) override; 414 void SaveRegisters(RegMask registers, bool isFp, ssize_t slot, Reg base, RegMask mask) override; 415 void LoadRegisters(RegMask registers, bool isFp, ssize_t slot, Reg base, RegMask mask) override; 416 void PushRegisters(RegMask registers, bool isFp) override; 417 void PopRegisters(RegMask registers, bool isFp) override; 418 419 template <typename Func> 420 void EncodeRelativePcMov(Reg reg, intptr_t offset, Func encodeInstruction); 421 422 asmjit::x86::Assembler *GetMasm() const; 423 size_t GetLabelAddress(LabelHolder::LabelId label) override; 424 bool LabelHasLinks(LabelHolder::LabelId label) override; 425 426 private: 427 template <bool IS_STORE> 428 void LoadStoreRegisters(RegMask registers, ssize_t slot, size_t startReg, bool isFp); 429 430 template <bool IS_STORE> 431 void LoadStoreRegisters(RegMask registers, bool isFp, int32_t slot, Reg base, RegMask mask); 432 433 inline Reg MakeShift(Shift shift); 434 435 template <typename T> 436 void EncodeReverseBitsImpl(Reg dst0, Reg src0); 437 438 void EncodeRoundToPInfFloat(Reg dst, Reg src); 439 void EncodeRoundToPInfDouble(Reg dst, Reg src); 440 441 void EncodeCastFloatToScalar(Reg dst, bool dstSigned, Reg src); 442 inline void EncodeCastFloatSignCheckRange(Reg dst, Reg src, const asmjit::Label &end); 443 inline void EncodeCastFloatUnsignCheckRange(Reg dst, Reg src, const asmjit::Label &end); 444 void EncodeCastFloatCheckNan(Reg dst, Reg src, const asmjit::Label &end); 445 void EncodeCastFloatCheckRange(Reg dst, Reg src, const asmjit::Label &end, int64_t minValue, uint64_t maxValue); 446 void EncodeCastFloat32ToUint64(Reg dst, Reg src); 447 void EncodeCastFloat64ToUint64(Reg dst, Reg src); 448 449 void EncodeCastScalarToFloat(Reg dst, Reg src, bool srcSigned); 450 void EncodeCastScalarToFloatUnsignDouble(Reg dst, Reg src); 451 void EncodeCastScalar(Reg dst, bool dstSigned, Reg src, bool srcSigned); 452 453 void EncodeDivFloat(Reg dst, Reg src0, Reg src1); 454 void EncodeModFloat(Reg dst, Reg src0, Reg src1); 455 template <bool IS_MAX> 456 void EncodeMinMaxFp(Reg dst, Reg src0, Reg src1); 457 458 template <typename T, size_t N> 459 void CopyArrayToXmm(Reg xmm, const std::array<T, N> &arr); 460 461 template <typename T> 462 void CopyImmToXmm(Reg xmm, T imm); 463 464 void EncodeCompareAndSwap(Reg dst, Reg obj, const Reg *offset, Reg val, Reg newval); 465 void EncodeCmpFracWithDelta(Reg src); 466 467 private: 468 Amd64LabelHolder *labels_ {nullptr}; 469 asmjit::ErrorHandler *errorHandler_ {nullptr}; 470 asmjit::CodeHolder *codeHolder_ {nullptr}; 471 asmjit::x86::Assembler *masm_ {nullptr}; 472 }; // Amd64Encoder 473 474 class Amd64ParameterInfo : public ParameterInfo { 475 public: 476 std::variant<Reg, uint8_t> GetNativeParam(const TypeInfo &type) override; 477 Location GetNextLocation(DataType::Type type) override; 478 }; 479 480 class Amd64CallingConvention : public CallingConvention { 481 public: 482 Amd64CallingConvention(ArenaAllocator *allocator, Encoder *enc, RegistersDescription *descr, CallConvMode mode); 483 NO_MOVE_SEMANTIC(Amd64CallingConvention); 484 NO_COPY_SEMANTIC(Amd64CallingConvention); 485 ~Amd64CallingConvention() override = default; 486 487 static constexpr auto GetTarget(); 488 bool IsValid() const override; 489 490 void GeneratePrologue(const FrameInfo &frameInfo) override; 491 void GenerateEpilogue(const FrameInfo &frameInfo, std::function<void()> postJob) override; 492 void GenerateNativePrologue(const FrameInfo &frameInfo) override; 493 void GenerateNativeEpilogue(const FrameInfo &frameInfo, std::function<void()> postJob) override; 494 495 void *GetCodeEntry() override; 496 uint32_t GetCodeSize() override; 497 498 // Pushes regs and returns number of regs(from boths vectors) 499 size_t PushRegs(RegList regs, RegList vregs); 500 // Pops regs and returns number of regs(from boths vectors) 501 size_t PopRegs(RegList regs, RegList vregs); 502 503 // Calculating information about parameters and save regs_offset registers for special needs 504 ParameterInfo *GetParameterInfo(uint8_t regsOffset) override; 505 asmjit::x86::Assembler *GetMasm(); 506 }; // Amd64CallingConvention 507 } // namespace ark::compiler::amd64 508 509 #endif // COMPILER_OPTIMIZER_CODEGEN_TARGET_AMD64_TARGET_H 510