1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 // x86_64 machine IR interface.
18
19 #ifndef BERBERIS_BACKEND_X86_64_MACHINE_IR_H_
20 #define BERBERIS_BACKEND_X86_64_MACHINE_IR_H_
21
22 #include <cstdint>
23 #include <string>
24
25 #include "berberis/assembler/x86_64.h"
26 #include "berberis/backend/code_emitter.h"
27 #include "berberis/backend/common/machine_ir.h"
28 #include "berberis/base/arena_alloc.h"
29 #include "berberis/guest_state/guest_state_arch.h"
30
31 namespace berberis {
32
33 enum MachineOpcode : int {
34 kMachineOpUndefined = 0,
35 kMachineOpCallImm,
36 kMachineOpCallImmArg,
37 kMachineOpPseudoBranch,
38 kMachineOpPseudoCondBranch,
39 kMachineOpPseudoCopy,
40 kMachineOpPseudoDefReg,
41 kMachineOpPseudoDefXReg,
42 kMachineOpPseudoIndirectJump,
43 kMachineOpPseudoJump,
44 kMachineOpPseudoReadFlags,
45 kMachineOpPseudoWriteFlags,
46 // Some frontends may need additional opcodes currently.
47 // Ideally we may want to separate froentend and backend, but for now only include
48 // berberis/backend/x86_64/machine_opcode_guest-inl.h if it exists.
49 #if __has_include("berberis/backend/x86_64/machine_opcode_guest-inl.h")
50 #include "berberis/backend/x86_64/machine_opcode_guest-inl.h"
51 #endif // __has_include("berberis/backend/x86_64/machine_opcode_guest-inl.h")
52 #include "machine_opcode_x86_64-inl.h" // NOLINT generated file!
53 };
54
55 namespace x86_64 {
56
57 constexpr const MachineReg kMachineRegR8{1};
58 constexpr const MachineReg kMachineRegR9{2};
59 constexpr const MachineReg kMachineRegR10{3};
60 constexpr const MachineReg kMachineRegR11{4};
61 constexpr const MachineReg kMachineRegRSI{5};
62 constexpr const MachineReg kMachineRegRDI{6};
63 constexpr const MachineReg kMachineRegRAX{7};
64 constexpr const MachineReg kMachineRegRBX{8};
65 constexpr const MachineReg kMachineRegRCX{9};
66 constexpr const MachineReg kMachineRegRDX{10};
67 constexpr const MachineReg kMachineRegRBP{11};
68 constexpr const MachineReg kMachineRegRSP{12};
69 constexpr const MachineReg kMachineRegR12{13};
70 constexpr const MachineReg kMachineRegR13{14};
71 constexpr const MachineReg kMachineRegR14{15};
72 constexpr const MachineReg kMachineRegR15{16};
73 constexpr const MachineReg kMachineRegFLAGS{19};
74 constexpr const MachineReg kMachineRegXMM0{20};
75 constexpr const MachineReg kMachineRegXMM1{21};
76 constexpr const MachineReg kMachineRegXMM2{22};
77 constexpr const MachineReg kMachineRegXMM3{23};
78 constexpr const MachineReg kMachineRegXMM4{24};
79 constexpr const MachineReg kMachineRegXMM5{25};
80 constexpr const MachineReg kMachineRegXMM6{26};
81 constexpr const MachineReg kMachineRegXMM7{27};
82 constexpr const MachineReg kMachineRegXMM8{28};
83 constexpr const MachineReg kMachineRegXMM9{29};
84 constexpr const MachineReg kMachineRegXMM10{30};
85 constexpr const MachineReg kMachineRegXMM11{31};
86 constexpr const MachineReg kMachineRegXMM12{32};
87 constexpr const MachineReg kMachineRegXMM13{33};
88 constexpr const MachineReg kMachineRegXMM14{34};
89 constexpr const MachineReg kMachineRegXMM15{35};
90
IsGReg(MachineReg r)91 inline bool IsGReg(MachineReg r) {
92 return r.reg() >= kMachineRegR8.reg() && r.reg() <= kMachineRegR15.reg();
93 }
94
IsXReg(MachineReg r)95 inline bool IsXReg(MachineReg r) {
96 return r.reg() >= kMachineRegXMM0.reg() && r.reg() <= kMachineRegXMM15.reg();
97 }
98
99 // rax, rdi, rsi, rdx, rcx, r8-r11, xmm0-xmm15, flags
100 const int kMaxMachineRegOperands = 26;
101
102 // Context loads and stores use rbp as base.
103 const MachineReg kCPUStatePointer = kMachineRegRBP;
104
105 struct MachineInsnInfo {
106 MachineOpcode opcode;
107 int num_reg_operands;
108 MachineRegKind reg_kinds[kMaxMachineRegOperands];
109 MachineInsnKind kind;
110 };
111
112 enum class MachineMemOperandScale {
113 kOne,
114 kTwo,
115 kFour,
116 kEight,
117 };
118
119 #include "machine_reg_class_x86_64-inl.h" // NOLINT generated file!
120
121 class MachineInsnX86_64 : public MachineInsn {
122 public:
123 static constexpr const auto kEAX = x86_64::kEAX;
124 static constexpr const auto kRAX = x86_64::kRAX;
125 static constexpr const auto kAL = x86_64::kAL;
126 static constexpr const auto kAX = x86_64::kAX;
127 static constexpr const auto kEBX = x86_64::kEBX;
128 static constexpr const auto kRBX = x86_64::kRBX;
129 static constexpr const auto kCL = x86_64::kCL;
130 static constexpr const auto kECX = x86_64::kECX;
131 static constexpr const auto kRCX = x86_64::kRCX;
132 static constexpr const auto kEDX = x86_64::kEDX;
133 static constexpr const auto kRDX = x86_64::kRDX;
134 static constexpr const auto kGeneralReg8 = x86_64::kGeneralReg8;
135 static constexpr const auto kGeneralReg16 = x86_64::kGeneralReg16;
136 static constexpr const auto kGeneralReg32 = x86_64::kGeneralReg32;
137 static constexpr const auto kGeneralReg64 = x86_64::kGeneralReg64;
138 static constexpr const auto kFpReg32 = x86_64::kFpReg32;
139 static constexpr const auto kFpReg64 = x86_64::kFpReg64;
140 static constexpr const auto kVecReg128 = x86_64::kVecReg128;
141 static constexpr const auto kXmmReg = x86_64::kXmmReg;
142 static constexpr const auto kFLAGS = x86_64::kFLAGS;
143
MachineInsnX86_64(const MachineInsnX86_64 & other)144 MachineInsnX86_64(const MachineInsnX86_64& other) : MachineInsn(other) {
145 for (int i = 0; i < kMaxMachineRegOperands; i++) {
146 regs_[i] = other.regs_[i];
147 }
148 scale_ = other.scale_;
149 disp_ = other.disp_;
150 imm_ = other.imm_;
151 cond_ = other.cond_;
152
153 SetRegs(regs_);
154 }
155
~MachineInsnX86_64()156 ~MachineInsnX86_64() override {
157 // No code here - will never be called!
158 }
159
scale()160 MachineMemOperandScale scale() const { return scale_; }
161
disp()162 uint32_t disp() const { return disp_; }
163
cond()164 Assembler::Condition cond() const { return cond_; }
165
imm()166 uint64_t imm() const { return imm_; }
167
IsCPUStateGet()168 bool IsCPUStateGet() {
169 if (opcode() != kMachineOpMovqRegMemBaseDisp && opcode() != kMachineOpMovdqaXRegMemBaseDisp &&
170 opcode() != kMachineOpMovwRegMemBaseDisp && opcode() != kMachineOpMovsdXRegMemBaseDisp) {
171 return false;
172 }
173
174 // Check that it is not for ThreadState fields outside of CPUState.
175 if (disp() >= sizeof(CPUState)) {
176 return false;
177 }
178
179 // reservation_value is loaded in HeavyOptimizerFrontend::AtomicLoad and written
180 // in HeavyOptimizerFrontend::AtomicStore partially (for performance
181 // reasons), which is not supported by our context optimizer.
182 auto reservation_value_offset = offsetof(ThreadState, cpu.reservation_value);
183 if (disp() >= reservation_value_offset &&
184 disp() < reservation_value_offset + sizeof(Reservation)) {
185 return false;
186 }
187
188 return RegAt(1) == kCPUStatePointer;
189 }
190
IsCPUStatePut()191 bool IsCPUStatePut() {
192 if (opcode() != kMachineOpMovqMemBaseDispReg && opcode() != kMachineOpMovdqaMemBaseDispXReg &&
193 opcode() != kMachineOpMovwMemBaseDispReg && opcode() != kMachineOpMovsdMemBaseDispXReg) {
194 return false;
195 }
196
197 // Check that it is not for ThreadState fields outside of CPUState.
198 if (disp() >= sizeof(CPUState)) {
199 return false;
200 }
201
202 // reservation_value is loaded in HeavyOptimizerFrontend::AtomicLoad and written
203 // in HeavyOptimizerFrontend::AtomicStore partially (for performance
204 // reasons), which is not supported by our context optimizer.
205 auto reservation_value_offset = offsetof(ThreadState, cpu.reservation_value);
206 if (disp() >= reservation_value_offset &&
207 disp() < reservation_value_offset + sizeof(Reservation)) {
208 return false;
209 }
210
211 return RegAt(0) == kCPUStatePointer;
212 }
213
214 protected:
MachineInsnX86_64(const MachineInsnInfo * info)215 explicit MachineInsnX86_64(const MachineInsnInfo* info)
216 : MachineInsn(info->opcode, info->num_reg_operands, info->reg_kinds, regs_, info->kind),
217 scale_(MachineMemOperandScale::kOne) {}
218
set_scale(MachineMemOperandScale scale)219 void set_scale(MachineMemOperandScale scale) { scale_ = scale; }
220
set_disp(uint32_t disp)221 void set_disp(uint32_t disp) { disp_ = disp; }
222
set_cond(Assembler::Condition cond)223 void set_cond(Assembler::Condition cond) { cond_ = cond; }
224
set_imm(uint64_t imm)225 void set_imm(uint64_t imm) { imm_ = imm; }
226
227 private:
228 MachineReg regs_[kMaxMachineRegOperands];
229 MachineMemOperandScale scale_;
230 uint32_t disp_;
231 uint64_t imm_;
232 Assembler::Condition cond_;
233 };
234
235 // Syntax sugar.
AsMachineInsnX86_64(const MachineInsn * insn)236 inline const MachineInsnX86_64* AsMachineInsnX86_64(const MachineInsn* insn) {
237 return static_cast<const MachineInsnX86_64*>(insn);
238 }
239
AsMachineInsnX86_64(MachineInsn * insn)240 inline MachineInsnX86_64* AsMachineInsnX86_64(MachineInsn* insn) {
241 return static_cast<MachineInsnX86_64*>(insn);
242 }
243
244 // Clobbered registers are described as DEF'ed.
245 // TODO(b/232598137): implement simpler support for clobbered registers?
246 class CallImm : public MachineInsnX86_64 {
247 public:
248 enum class RegType {
249 kIntType,
250 kXmmType,
251 };
252
253 static constexpr RegType kIntRegType = RegType::kIntType;
254 static constexpr RegType kXmmRegType = RegType::kXmmType;
255
256 struct Arg {
257 MachineReg reg;
258 RegType reg_type;
259 };
260
261 explicit CallImm(uint64_t imm);
262
263 [[nodiscard]] static int GetIntArgIndex(int i);
264 [[nodiscard]] static int GetXmmArgIndex(int i);
265 [[nodiscard]] static int GetFlagsArgIndex();
266
267 [[nodiscard]] MachineReg IntResultAt(int i) const;
268 [[nodiscard]] MachineReg XmmResultAt(int i) const;
269
270 [[nodiscard]] std::string GetDebugString() const override;
271 void Emit(CodeEmitter* as) const override;
EnableCustomAVX256ABI()272 void EnableCustomAVX256ABI() { custom_avx256_abi_ = true; };
273
274 private:
275 bool custom_avx256_abi_;
276 };
277
278 // An auxiliary instruction to express data-flow for CallImm arguments. It uses the same vreg as
279 // the corresponding operand in CallImm. The specific hard register assigned is defined by the
280 // register class of CallImm operand. MachineIRBuilder adds an extra PseudoCopy before this insn in
281 // case the same vreg holds values for several arguments (with non-intersecting register classes).
282 class CallImmArg : public MachineInsnX86_64 {
283 public:
284 explicit CallImmArg(MachineReg arg, CallImm::RegType reg_type);
285
286 std::string GetDebugString() const override;
Emit(CodeEmitter *)287 void Emit(CodeEmitter*) const override{
288 // It's an auxiliary instruction. Do not emit.
289 };
290 };
291
292 // This template is syntax sugar to group memory instructions with
293 // different addressing modes.
294 template <typename Absolute_, typename BaseDisp_, typename IndexDisp_, typename BaseIndexDisp_>
295 class MemInsns {
296 public:
297 using Absolute = Absolute_;
298 using BaseDisp = BaseDisp_;
299 using IndexDisp = IndexDisp_;
300 using BaseIndexDisp = BaseIndexDisp_;
301 };
302
303 using MachineInsnForArch = MachineInsnX86_64;
304
305 #include "gen_machine_ir_x86_64-inl.h" // NOLINT generated file!
306
307 class MachineInfo {
308 public:
309 #include "machine_info_x86_64-inl.h" // NOLINT generated file!
310 };
311
312 class MachineIR : public berberis::MachineIR {
313 public:
314 enum class BasicBlockOrder {
315 kUnordered,
316 kReversePostOrder,
317 };
318
319 explicit MachineIR(Arena* arena, int num_vreg = 0)
320 : berberis::MachineIR(arena, num_vreg, 0), bb_order_(BasicBlockOrder::kUnordered) {}
321
AddEdge(MachineBasicBlock * src,MachineBasicBlock * dst)322 void AddEdge(MachineBasicBlock* src, MachineBasicBlock* dst) {
323 MachineEdge* edge = NewInArena<MachineEdge>(arena(), arena(), src, dst);
324 src->out_edges().push_back(edge);
325 dst->in_edges().push_back(edge);
326 bb_order_ = BasicBlockOrder::kUnordered;
327 }
328
NewBasicBlock()329 [[nodiscard]] MachineBasicBlock* NewBasicBlock() {
330 return NewInArena<MachineBasicBlock>(arena(), arena(), ReserveBasicBlockId());
331 }
332
333 // Instruction iterators are preserved after splitting basic block and moving
334 // instructions to the new basic block.
SplitBasicBlock(MachineBasicBlock * bb,MachineInsnList::iterator insn_it)335 [[nodiscard]] MachineBasicBlock* SplitBasicBlock(MachineBasicBlock* bb,
336 MachineInsnList::iterator insn_it) {
337 MachineBasicBlock* new_bb = NewBasicBlock();
338
339 new_bb->insn_list().splice(
340 new_bb->insn_list().begin(), bb->insn_list(), insn_it, bb->insn_list().end());
341 bb->insn_list().push_back(NewInsn<PseudoBranch>(new_bb));
342
343 // Relink out edges from bb.
344 for (auto out_edge : bb->out_edges()) {
345 out_edge->set_src(new_bb);
346 }
347 new_bb->out_edges().swap(bb->out_edges());
348
349 AddEdge(bb, new_bb);
350 bb_list().push_back(new_bb);
351 return new_bb;
352 }
353
IsControlTransfer(MachineInsn * insn)354 [[nodiscard]] static bool IsControlTransfer(MachineInsn* insn) {
355 return insn->opcode() == kMachineOpPseudoBranch ||
356 insn->opcode() == kMachineOpPseudoCondBranch ||
357 insn->opcode() == kMachineOpPseudoIndirectJump || insn->opcode() == kMachineOpPseudoJump;
358 }
359
bb_order()360 [[nodiscard]] BasicBlockOrder bb_order() const { return bb_order_; }
361
set_bb_order(BasicBlockOrder order)362 void set_bb_order(BasicBlockOrder order) { bb_order_ = order; }
363
364 private:
365 BasicBlockOrder bb_order_;
366 };
367
368 } // namespace x86_64
369
370 } // namespace berberis
371
372 #endif // BERBERIS_BACKEND_X86_64_MACHINE_IR_H_
373