1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "gtest/gtest.h"
18
19 #include <csignal>
20 #include <cstdint>
21 #include <cstring>
22
23 #include "berberis/assembler/machine_code.h"
24 #include "berberis/backend/code_emitter.h"
25 #include "berberis/backend/common/reg_alloc.h"
26 #include "berberis/backend/x86_64/machine_ir.h"
27 #include "berberis/backend/x86_64/machine_ir_builder.h"
28 #include "berberis/backend/x86_64/machine_ir_check.h"
29 #include "berberis/base/arena_alloc.h"
30 #include "berberis/base/bit_util.h"
31 #include "berberis/code_gen_lib/code_gen_lib.h" // EmitFreeStackFrame
32 #include "berberis/test_utils/scoped_exec_region.h"
33
34 #include "x86_64/mem_operand.h"
35
36 namespace berberis {
37
38 namespace {
39
40 // TODO(b/232598137): Maybe share with
41 // heavy_optimizer/<guest>_to_<host>/call_intrinsic_tests.cc.
42 class ExecTest {
43 public:
44 ExecTest() = default;
45
Init(x86_64::MachineIR & machine_ir)46 void Init(x86_64::MachineIR& machine_ir) {
47 // Add exiting jump if not already.
48 auto* last_insn = machine_ir.bb_list().back()->insn_list().back();
49 if (!machine_ir.IsControlTransfer(last_insn)) {
50 auto* jump = machine_ir.template NewInsn<PseudoJump>(0);
51 machine_ir.bb_list().back()->insn_list().push_back(jump);
52 }
53
54 EXPECT_EQ(x86_64::CheckMachineIR(machine_ir), x86_64::kMachineIRCheckSuccess);
55
56 MachineCode machine_code;
57 CodeEmitter as(
58 &machine_code, machine_ir.FrameSize(), machine_ir.bb_list().size(), machine_ir.arena());
59
60 // We need to set exit_label_for_testing before Emit, which checks it.
61 auto* exit_label = as.MakeLabel();
62 as.set_exit_label_for_testing(exit_label);
63
64 // Save callee saved regs.
65 as.Push(as.rbp);
66 as.Push(as.rbx);
67 as.Push(as.r12);
68 as.Push(as.r13);
69 as.Push(as.r14);
70 as.Push(as.r15);
71 // Align stack for calls.
72 as.Subq(as.rsp, 8);
73
74 machine_ir.Emit(&as);
75
76 as.Bind(exit_label);
77 // Memorize returned rax.
78 as.Movq(as.rbp, bit_cast<int64_t>(&returned_rax_));
79 as.Movq({.base = as.rbp}, as.rax);
80
81 as.Addq(as.rsp, 8);
82 // Restore callee saved regs.
83 as.Pop(as.r15);
84 as.Pop(as.r14);
85 as.Pop(as.r13);
86 as.Pop(as.r12);
87 as.Pop(as.rbx);
88 as.Pop(as.rbp);
89
90 as.Ret();
91
92 as.Finalize();
93
94 exec_.Init(&machine_code);
95 }
96
Exec() const97 void Exec() const { exec_.get<void()>()(); }
98
recovery_map() const99 const RecoveryMap& recovery_map() const { return exec_.recovery_map(); }
100
returned_rax() const101 uint64_t returned_rax() const { return returned_rax_; }
102
103 private:
104 ScopedExecRegion exec_;
105 uint64_t returned_rax_;
106 };
107
108 // Convert flags to LAHF-compatible format.
MakeFlags(bool n,bool z,bool c,bool o)109 inline uint16_t MakeFlags(bool n, bool z, bool c, bool o) {
110 return (n ? (1 << 15) : 0) | (z ? (1 << 14) : 0) | (c ? (1 << 8) : 0) | (o ? 1 : 0);
111 }
112
MakeFlags(uint8_t nzco_bits)113 inline uint16_t MakeFlags(uint8_t nzco_bits) {
114 return MakeFlags(nzco_bits & 0b1000, nzco_bits & 0b0100, nzco_bits & 0b0010, nzco_bits & 0b0001);
115 }
116
TEST(ExecMachineIR,Smoke)117 TEST(ExecMachineIR, Smoke) {
118 struct Data {
119 uint64_t x;
120 uint64_t y;
121 } data;
122
123 Arena arena;
124 x86_64::MachineIR machine_ir(&arena);
125
126 x86_64::MachineIRBuilder builder(&machine_ir);
127 builder.StartBasicBlock(machine_ir.NewBasicBlock());
128
129 // Let RBP point to 'data'.
130 builder.Gen<x86_64::MovqRegImm>(x86_64::kMachineRegRBP, reinterpret_cast<uintptr_t>(&data));
131
132 // data.y = data.x;
133 builder.Gen<x86_64::MovqRegMemBaseDisp>(
134 x86_64::kMachineRegRAX, x86_64::kMachineRegRBP, offsetof(Data, x));
135 builder.Gen<x86_64::MovqMemBaseDispReg>(
136 x86_64::kMachineRegRBP, offsetof(Data, y), x86_64::kMachineRegRAX);
137
138 ExecTest test;
139 test.Init(machine_ir);
140
141 data.x = 1;
142 data.y = 2;
143 test.Exec();
144 EXPECT_EQ(1ULL, data.x);
145 EXPECT_EQ(1ULL, data.y);
146 }
147
TEST(ExecMachineIR,CallImm)148 TEST(ExecMachineIR, CallImm) {
149 Arena arena;
150 x86_64::MachineIR machine_ir(&arena);
151
152 x86_64::MachineIRBuilder builder(&machine_ir);
153 builder.StartBasicBlock(machine_ir.NewBasicBlock());
154
155 uint64_t data = 0xfeedf00d'feedf00dULL;
156 builder.Gen<x86_64::MovqRegImm>(x86_64::kMachineRegRDI, data);
157 auto* invert_func_ptr = +[](uint64_t arg) { return ~arg; };
158
159 MachineReg flag_register = machine_ir.AllocVReg();
160 builder.GenCallImm(bit_cast<uintptr_t>(invert_func_ptr), flag_register);
161
162 uint64_t result = 0;
163 builder.Gen<x86_64::MovqRegImm>(x86_64::kMachineRegRBP, reinterpret_cast<uintptr_t>(&result));
164 builder.Gen<x86_64::MovqMemBaseDispReg>(x86_64::kMachineRegRBP, 0, x86_64::kMachineRegRAX);
165
166 ExecTest test;
167 test.Init(machine_ir);
168 test.Exec();
169 EXPECT_EQ(result, ~data);
170 }
171
TEST(ExecMachineIR,CallImmAllocIntOperands)172 TEST(ExecMachineIR, CallImmAllocIntOperands) {
173 Arena arena;
174 x86_64::MachineIR machine_ir(&arena);
175
176 x86_64::MachineIRBuilder builder(&machine_ir);
177 builder.StartBasicBlock(machine_ir.NewBasicBlock());
178
179 uint64_t data = 0xfeedf00d'feedf00dULL;
180 struct Result {
181 uint64_t x;
182 uint64_t y;
183 } result = {0, 0};
184 MachineReg data_reg = machine_ir.AllocVReg();
185 MachineReg flag_register = machine_ir.AllocVReg();
186 auto* func_ptr = +[](uint64_t arg0,
187 uint64_t arg1,
188 uint64_t arg2,
189 uint64_t arg3,
190 uint64_t arg4,
191 uint64_t arg5) {
192 uint64_t res = arg0 + arg1 + arg2 + arg3 + arg4 + arg5;
193 return Result{res, res * 2};
194 };
195
196 builder.Gen<x86_64::MovqRegImm>(data_reg, data);
197 std::array<x86_64::CallImm::Arg, 6> args = {{
198 {data_reg, x86_64::CallImm::kIntRegType},
199 {data_reg, x86_64::CallImm::kIntRegType},
200 {data_reg, x86_64::CallImm::kIntRegType},
201 {data_reg, x86_64::CallImm::kIntRegType},
202 {data_reg, x86_64::CallImm::kIntRegType},
203 {data_reg, x86_64::CallImm::kIntRegType},
204 }};
205 auto* call = builder.GenCallImm(bit_cast<uintptr_t>(func_ptr), flag_register, args);
206 builder.Gen<x86_64::MovqRegImm>(x86_64::kMachineRegRBP, bit_cast<uintptr_t>(&result));
207 builder.Gen<x86_64::MovqMemBaseDispReg>(x86_64::kMachineRegRBP, 0, call->IntResultAt(0));
208 builder.Gen<x86_64::MovqMemBaseDispReg>(x86_64::kMachineRegRBP, 8, call->IntResultAt(1));
209
210 AllocRegs(&machine_ir);
211
212 ExecTest test;
213 test.Init(machine_ir);
214 test.Exec();
215 EXPECT_EQ(result.x, data * 6);
216 EXPECT_EQ(result.y, data * 12);
217 }
218
TEST(ExecMachineIR,CallImmAllocIntOperandsTupleResult)219 TEST(ExecMachineIR, CallImmAllocIntOperandsTupleResult) {
220 Arena arena;
221 x86_64::MachineIR machine_ir(&arena);
222
223 x86_64::MachineIRBuilder builder(&machine_ir);
224 builder.StartBasicBlock(machine_ir.NewBasicBlock());
225
226 uint64_t data = 0xfeedf00d'feedf00dULL;
227 using Result = std::tuple<uint64_t, uint64_t, uint64_t>;
228 Result result = std::make_tuple(0, 0, 0);
229 MachineReg data_reg = machine_ir.AllocVReg();
230 MachineReg result_ptr_reg = machine_ir.AllocVReg();
231 MachineReg flag_register = machine_ir.AllocVReg();
232 auto* func_ptr = +[](uint64_t arg0, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4) {
233 uint64_t one = arg0 + arg1 + arg2 + arg3 + arg4;
234 uint64_t two = one * 2;
235 uint64_t three = one * 3;
236 return std::make_tuple(one, two, three);
237 };
238
239 builder.Gen<x86_64::MovqRegImm>(data_reg, data);
240 builder.Gen<x86_64::MovqRegImm>(result_ptr_reg, bit_cast<uintptr_t>(&result));
241 std::array<x86_64::CallImm::Arg, 6> args = {{
242 {result_ptr_reg, x86_64::CallImm::kIntRegType},
243 {data_reg, x86_64::CallImm::kIntRegType},
244 {data_reg, x86_64::CallImm::kIntRegType},
245 {data_reg, x86_64::CallImm::kIntRegType},
246 {data_reg, x86_64::CallImm::kIntRegType},
247 {data_reg, x86_64::CallImm::kIntRegType},
248 }};
249 builder.GenCallImm(bit_cast<uintptr_t>(func_ptr), flag_register, args);
250
251 AllocRegs(&machine_ir);
252
253 ExecTest test;
254 test.Init(machine_ir);
255 test.Exec();
256 EXPECT_EQ(std::get<0>(result), data * 5);
257 EXPECT_EQ(std::get<1>(result), data * 10);
258 EXPECT_EQ(std::get<2>(result), data * 15);
259 }
260
TEST(ExecMachineIR,CallImmAllocXmmOperands)261 TEST(ExecMachineIR, CallImmAllocXmmOperands) {
262 Arena arena;
263 x86_64::MachineIR machine_ir(&arena);
264
265 x86_64::MachineIRBuilder builder(&machine_ir);
266 builder.StartBasicBlock(machine_ir.NewBasicBlock());
267
268 double data = 42.0;
269 struct Result {
270 double x;
271 double y;
272 } result = {0, 0};
273 MachineReg data_reg = machine_ir.AllocVReg();
274 MachineReg data_xreg = machine_ir.AllocVReg();
275 MachineReg flag_register = machine_ir.AllocVReg();
276 auto* func_ptr = +[](double arg0,
277 double arg1,
278 double arg2,
279 double arg3,
280 double arg4,
281 double arg5,
282 double arg6,
283 double arg7) {
284 double res = arg0 + arg1 + arg2 + arg3 + arg4 + arg5 + arg6 + arg7;
285 return Result{res, res * 2};
286 };
287
288 builder.Gen<x86_64::MovqRegImm>(data_reg, bit_cast<uint64_t>(data));
289 builder.Gen<x86_64::MovqXRegReg>(data_xreg, data_reg);
290
291 std::array<x86_64::CallImm::Arg, 8> args = {{
292 {data_xreg, x86_64::CallImm::kXmmRegType},
293 {data_xreg, x86_64::CallImm::kXmmRegType},
294 {data_xreg, x86_64::CallImm::kXmmRegType},
295 {data_xreg, x86_64::CallImm::kXmmRegType},
296 {data_xreg, x86_64::CallImm::kXmmRegType},
297 {data_xreg, x86_64::CallImm::kXmmRegType},
298 {data_xreg, x86_64::CallImm::kXmmRegType},
299 {data_xreg, x86_64::CallImm::kXmmRegType},
300 }};
301 auto* call = builder.GenCallImm(bit_cast<uintptr_t>(func_ptr), flag_register, args);
302 builder.Gen<x86_64::MovqRegImm>(x86_64::kMachineRegRBP, bit_cast<uintptr_t>(&result));
303 builder.Gen<x86_64::MovqRegXReg>(data_reg, call->XmmResultAt(0));
304 builder.Gen<x86_64::MovqMemBaseDispReg>(x86_64::kMachineRegRBP, 0, data_reg);
305 builder.Gen<x86_64::MovqRegXReg>(data_reg, call->XmmResultAt(1));
306 builder.Gen<x86_64::MovqMemBaseDispReg>(x86_64::kMachineRegRBP, 8, data_reg);
307
308 AllocRegs(&machine_ir);
309
310 ExecTest test;
311 test.Init(machine_ir);
312 test.Exec();
313 EXPECT_EQ(result.x, data * 8);
314 EXPECT_EQ(result.y, data * 16);
315 }
316
ClobberAllCallerSaved()317 void ClobberAllCallerSaved() {
318 constexpr uint64_t kClobberValue = 0xdeadbeef'deadbeefULL;
319 asm volatile(
320 "Movq %0, %%rax\n"
321 "Movq %0, %%rcx\n"
322 "Movq %0, %%rdx\n"
323 "Movq %0, %%rdi\n"
324 "Movq %0, %%rsi\n"
325 "Movq %0, %%r8\n"
326 "Movq %0, %%r9\n"
327 "Movq %0, %%r10\n"
328 "Movq %0, %%r11\n"
329 "Movq %%rax, %%xmm0\n"
330 "Movq %%rax, %%xmm1\n"
331 "Movq %%rax, %%xmm2\n"
332 "Movq %%rax, %%xmm3\n"
333 "Movq %%rax, %%xmm4\n"
334 "Movq %%rax, %%xmm5\n"
335 "Movq %%rax, %%xmm6\n"
336 "Movq %%rax, %%xmm7\n"
337 "Movq %%rax, %%xmm8\n"
338 "Movq %%rax, %%xmm9\n"
339 "Movq %%rax, %%xmm10\n"
340 "Movq %%rax, %%xmm11\n"
341 "Movq %%rax, %%xmm12\n"
342 "Movq %%rax, %%xmm13\n"
343 "Movq %%rax, %%xmm14\n"
344 "Movq %%rax, %%xmm15\n"
345 :
346 : "r"(kClobberValue)
347 : "rax",
348 "rcx",
349 "rdx",
350 "rdi",
351 "rsi",
352 "r8",
353 "r9",
354 "r10",
355 "r11",
356 "xmm0",
357 "xmm1",
358 "xmm2",
359 "xmm3",
360 "xmm4",
361 "xmm5",
362 "xmm6",
363 "xmm7",
364 "xmm8",
365 "xmm9",
366 "xmm10",
367 "xmm11",
368 "xmm12",
369 "xmm13",
370 "xmm14",
371 "xmm15");
372 }
373
374 template <bool kWithCallImm>
TestRegAlloc()375 void TestRegAlloc() {
376 constexpr int N = 128;
377
378 struct Data {
379 uint64_t in_array[N];
380 uint64_t out;
381 } data{};
382
383 Arena arena;
384 x86_64::MachineIR machine_ir(&arena);
385
386 x86_64::MachineIRBuilder builder(&machine_ir);
387 builder.StartBasicBlock(machine_ir.NewBasicBlock());
388
389 // Let rbp point to 'data'.
390 builder.Gen<x86_64::MovqRegImm>(x86_64::kMachineRegRBP, reinterpret_cast<uintptr_t>(&data));
391
392 // Read data.in_array into vregs, xor and write to data.out.
393
394 MachineReg vregs[N];
395 MachineReg xmm_vregs[N];
396
397 for (int i = 0; i < N; ++i) {
398 MachineReg v = machine_ir.AllocVReg();
399 vregs[i] = v;
400 builder.Gen<x86_64::MovqRegMemBaseDisp>(
401 v, x86_64::kMachineRegRBP, offsetof(Data, in_array) + i * sizeof(data.in_array[0]));
402 MachineReg vx = machine_ir.AllocVReg();
403 xmm_vregs[i] = vx;
404 builder.Gen<x86_64::MovqXRegReg>(vx, v);
405 }
406
407 if (kWithCallImm) {
408 // If there is no CallImm reg-alloc assigns vregs to hard-regs until available.
409 // When CallImm is here it must not allocate caller-saved regs to live across function call.
410 // Ideally we should have allocated hard-regs around the call explicitly and verify that
411 // reg-alloc would spill/fill them, but reg-alloc doesn't support that.
412 MachineReg flag_register = machine_ir.AllocVReg();
413 builder.GenCallImm(bit_cast<uintptr_t>(&ClobberAllCallerSaved), flag_register);
414 }
415
416 MachineReg v0 = machine_ir.AllocVReg();
417 builder.Gen<x86_64::MovqRegImm>(v0, 0);
418 MachineReg vx0 = machine_ir.AllocVReg();
419 builder.Gen<x86_64::XorpdXRegXReg>(vx0, vx0);
420
421 for (int i = 0; i < N; ++i) {
422 MachineReg vflags = machine_ir.AllocVReg();
423 builder.Gen<x86_64::XorqRegReg>(v0, vregs[i], vflags);
424 builder.Gen<x86_64::XorpdXRegXReg>(vx0, xmm_vregs[i]);
425 }
426
427 MachineReg v1 = machine_ir.AllocVReg();
428 builder.Gen<x86_64::MovqRegXReg>(v1, vx0);
429 MachineReg vflags = machine_ir.AllocVReg();
430 builder.Gen<x86_64::AddqRegReg>(v1, v0, vflags);
431 builder.Gen<x86_64::MovqMemBaseDispReg>(x86_64::kMachineRegRBP, offsetof(Data, out), v1);
432
433 AllocRegs(&machine_ir);
434
435 ExecTest test;
436 test.Init(machine_ir);
437
438 uint64_t res = 0;
439 for (int i = 0; i < N; ++i) {
440 // Add some irregularity to ensure the result isn't zero.
441 data.in_array[i] = i + (res << 4);
442 res ^= data.in_array[i];
443 }
444 // Sum for vregs and xmm_regs.
445 res *= 2;
446 test.Exec();
447 EXPECT_EQ(res, data.out);
448 }
449
TEST(ExecMachineIR,SmokeRegAlloc)450 TEST(ExecMachineIR, SmokeRegAlloc) {
451 TestRegAlloc<false>();
452 }
453
TEST(ExecMachineIR,RegAllocWithCallImm)454 TEST(ExecMachineIR, RegAllocWithCallImm) {
455 TestRegAlloc<true>();
456 }
457
TEST(ExecMachineIR,MemOperand)458 TEST(ExecMachineIR, MemOperand) {
459 struct Data {
460 uint64_t in_base_disp;
461 uint64_t in_index_disp;
462 uint64_t in_base_index_disp[3];
463
464 uint64_t out_base_disp;
465 uint64_t out_index_disp;
466 uint64_t out_base_index_disp;
467 } data = {};
468
469 Arena arena;
470 x86_64::MachineIR machine_ir(&arena);
471
472 x86_64::MachineIRBuilder builder(&machine_ir);
473 builder.StartBasicBlock(machine_ir.NewBasicBlock());
474
475 data.in_base_disp = 0xaaaabbbbccccddddULL;
476 data.in_index_disp = 0xdeadbeefdeadbeefULL;
477 data.in_base_index_disp[2] = 0xcafefeedf00dfeedULL;
478
479 // Base address.
480 MachineReg base_reg = machine_ir.AllocVReg();
481 builder.Gen<x86_64::MovqRegImm>(base_reg, reinterpret_cast<uintptr_t>(&data));
482
483 MachineReg data_reg;
484
485 // BaseDisp
486 x86_64::MemOperand mem_base_disp =
487 x86_64::MemOperand::MakeBaseDisp(base_reg, offsetof(Data, in_base_disp));
488 data_reg = machine_ir.AllocVReg();
489 x86_64::GenArgsMem<x86_64::MovzxblRegMemInsns>(&builder, mem_base_disp, data_reg);
490 builder.Gen<x86_64::MovqMemBaseDispReg>(base_reg, offsetof(Data, out_base_disp), data_reg);
491
492 // IndexDisp
493 MachineReg index_reg = machine_ir.AllocVReg();
494 static_assert(alignof(struct Data) >= 2);
495 builder.Gen<x86_64::MovqRegImm>(index_reg, reinterpret_cast<uintptr_t>(&data) / 2);
496 x86_64::MemOperand mem_index_disp =
497 x86_64::MemOperand::MakeIndexDisp<x86_64::MachineMemOperandScale::kTwo>(
498 index_reg, offsetof(Data, in_index_disp));
499 data_reg = machine_ir.AllocVReg();
500 x86_64::GenArgsMem<x86_64::MovzxblRegMemInsns>(&builder, mem_index_disp, data_reg);
501 builder.Gen<x86_64::MovqMemBaseDispReg>(base_reg, offsetof(Data, out_index_disp), data_reg);
502
503 // BaseIndexDisp
504 MachineReg tmp_base_reg = machine_ir.AllocVReg();
505 builder.Gen<x86_64::MovqRegImm>(tmp_base_reg,
506 reinterpret_cast<uintptr_t>(&data.in_base_index_disp[0]));
507 MachineReg tmp_index_reg = machine_ir.AllocVReg();
508 builder.Gen<x86_64::MovqRegImm>(tmp_index_reg, 2);
509 x86_64::MemOperand mem_base_index_disp =
510 x86_64::MemOperand::MakeBaseIndexDisp<x86_64::MachineMemOperandScale::kFour>(
511 tmp_base_reg, tmp_index_reg, 8);
512 data_reg = machine_ir.AllocVReg();
513 x86_64::GenArgsMem<x86_64::MovzxblRegMemInsns>(&builder, mem_base_index_disp, data_reg);
514 builder.Gen<x86_64::MovqMemBaseDispReg>(base_reg, offsetof(Data, out_base_index_disp), data_reg);
515
516 AllocRegs(&machine_ir);
517
518 ExecTest test;
519 test.Init(machine_ir);
520
521 test.Exec();
522 EXPECT_EQ(data.out_base_disp, 0xddU);
523 EXPECT_EQ(data.out_index_disp, 0xefU);
524 EXPECT_EQ(data.out_base_index_disp, 0xedU);
525 }
526
527 const MachineReg kGRegs[]{
528 x86_64::kMachineRegR8,
529 x86_64::kMachineRegR9,
530 x86_64::kMachineRegR10,
531 x86_64::kMachineRegR11,
532 x86_64::kMachineRegRSI,
533 x86_64::kMachineRegRDI,
534 x86_64::kMachineRegRAX,
535 x86_64::kMachineRegRBX,
536 x86_64::kMachineRegRCX,
537 x86_64::kMachineRegRDX,
538 x86_64::kMachineRegR12,
539 x86_64::kMachineRegR13,
540 x86_64::kMachineRegR14,
541 x86_64::kMachineRegR15,
542 };
543
544 const MachineReg kXmms[]{
545 x86_64::kMachineRegXMM0,
546 x86_64::kMachineRegXMM1,
547 x86_64::kMachineRegXMM2,
548 x86_64::kMachineRegXMM3,
549 x86_64::kMachineRegXMM4,
550 x86_64::kMachineRegXMM5,
551 x86_64::kMachineRegXMM6,
552 x86_64::kMachineRegXMM7,
553 x86_64::kMachineRegXMM8,
554 x86_64::kMachineRegXMM9,
555 x86_64::kMachineRegXMM10,
556 x86_64::kMachineRegXMM11,
557 x86_64::kMachineRegXMM12,
558 x86_64::kMachineRegXMM13,
559 x86_64::kMachineRegXMM14,
560 x86_64::kMachineRegXMM15,
561 };
562
563 class ExecMachineIRTest : public ::testing::Test {
564 protected:
565 struct Xmm {
566 uint64_t lo;
567 uint64_t hi;
568 };
569 static_assert(sizeof(Xmm) == 16, "bad xmm type");
570
571 struct Data {
572 uint64_t gregs[std::size(kGRegs)];
573 Xmm xmms[std::size(kXmms)];
574 Xmm slots[16];
575 };
576 static_assert(sizeof(Data) % sizeof(uint64_t) == 0, "bad data type");
577
InitData(Data * data)578 static void InitData(Data* data) {
579 // Try to have all 4-byte pieces different. This way we ensure that the
580 // upper half of gregs is also meaningful.
581 char* p = reinterpret_cast<char*>(data);
582 constexpr size_t kUnitSize = 4;
583 static_assert((sizeof(Data) % kUnitSize) == 0);
584 for (size_t i = 0; i < (sizeof(Data) / kUnitSize); ++i) {
585 static_assert(sizeof(i) >= kUnitSize);
586 memcpy(p + kUnitSize * i, &i, kUnitSize);
587 }
588 }
589
ExpectEqualData(const Data & x,const Data & y)590 static void ExpectEqualData(const Data& x, const Data& y) {
591 for (size_t i = 0; i < std::size(x.gregs); ++i) {
592 EXPECT_EQ(x.gregs[i], y.gregs[i]) << "gregs differ at index " << i;
593 }
594 for (size_t i = 0; i < std::size(x.xmms); ++i) {
595 EXPECT_EQ(x.xmms[i].lo, y.xmms[i].lo) << "xmms lo differ at index " << i;
596 EXPECT_EQ(x.xmms[i].hi, y.xmms[i].hi) << "xmms hi differ at index " << i;
597 }
598 for (size_t i = 0; i < std::size(x.slots); ++i) {
599 EXPECT_EQ(x.slots[i].lo, y.slots[i].lo) << "slots lo differ at index " << i;
600 EXPECT_EQ(x.slots[i].hi, y.slots[i].hi) << "slots hi differ at index " << i;
601 }
602 }
603
ExecMachineIRTest()604 ExecMachineIRTest() : machine_ir_(&arena_), builder_(&machine_ir_), data_{} {
605 bb_ = machine_ir_.NewBasicBlock();
606 builder_.StartBasicBlock(bb_);
607
608 // Let rbp point to 'data'.
609 builder_.Gen<x86_64::MovqRegImm>(x86_64::kMachineRegRBP, reinterpret_cast<uintptr_t>(&data_));
610
611 for (size_t i = 0; i < std::size(data_.slots); ++i) {
612 slots_[i] = MachineReg::CreateSpilledRegFromIndex(
613 machine_ir_.SpillSlotOffset(machine_ir_.AllocSpill()));
614
615 builder_.Gen<x86_64::MovdquXRegMemBaseDisp>(
616 x86_64::kMachineRegXMM0,
617 x86_64::kMachineRegRBP,
618 offsetof(Data, slots) + i * sizeof(data_.slots[0]));
619 builder_.Gen<PseudoCopy>(slots_[i], x86_64::kMachineRegXMM0, 16);
620 }
621
622 for (size_t i = 0; i < std::size(kXmms); ++i) {
623 builder_.Gen<x86_64::MovdquXRegMemBaseDisp>(
624 kXmms[i], x86_64::kMachineRegRBP, offsetof(Data, xmms) + i * sizeof(data_.xmms[0]));
625 }
626
627 for (size_t i = 0; i < std::size(kGRegs); ++i) {
628 builder_.Gen<x86_64::MovqRegMemBaseDisp>(
629 kGRegs[i], x86_64::kMachineRegRBP, offsetof(Data, gregs) + i * sizeof(data_.gregs[0]));
630 }
631 }
632
Finalize()633 void Finalize() {
634 for (size_t i = 0; i < std::size(kGRegs); ++i) {
635 builder_.Gen<x86_64::MovqMemBaseDispReg>(
636 x86_64::kMachineRegRBP, offsetof(Data, gregs) + i * sizeof(data_.gregs[0]), kGRegs[i]);
637 }
638
639 for (size_t i = 0; i < std::size(kXmms); ++i) {
640 builder_.Gen<x86_64::MovdquMemBaseDispXReg>(
641 x86_64::kMachineRegRBP, offsetof(Data, xmms) + i * sizeof(data_.xmms[0]), kXmms[i]);
642 }
643
644 for (size_t i = 0; i < std::size(data_.slots); ++i) {
645 builder_.Gen<PseudoCopy>(x86_64::kMachineRegXMM0, slots_[i], 16);
646 builder_.Gen<x86_64::MovdquMemBaseDispXReg>(
647 x86_64::kMachineRegRBP,
648 offsetof(Data, slots) + i * sizeof(data_.slots[0]),
649 x86_64::kMachineRegXMM0);
650 }
651
652 test_.Init(machine_ir_);
653 }
654
655 Arena arena_;
656 x86_64::MachineIR machine_ir_;
657 x86_64::MachineIRBuilder builder_;
658 MachineBasicBlock* bb_;
659 Data data_;
660 MachineReg slots_[std::size(Data{}.slots)];
661 ExecTest test_;
662 };
663
TEST_F(ExecMachineIRTest,Copy)664 TEST_F(ExecMachineIRTest, Copy) {
665 InitData(&data_);
666 Data dst_data = data_;
667
668 builder_.Gen<PseudoCopy>(kGRegs[1], kGRegs[0], 8);
669 dst_data.gregs[1] = data_.gregs[0];
670
671 builder_.Gen<PseudoCopy>(slots_[0], kXmms[0], 8);
672 dst_data.slots[0].lo = data_.xmms[0].lo;
673
674 builder_.Gen<PseudoCopy>(slots_[1], kXmms[1], 16);
675 dst_data.slots[1] = data_.xmms[1];
676
677 builder_.Gen<PseudoCopy>(kXmms[3], kXmms[2], 16);
678 dst_data.xmms[3] = data_.xmms[2];
679
680 // The minimum copy amount is 8 bytes. PseudoCopy of a smaller size will copy
681 // garbage in upper bytes. This is in compliance with MachineIR assumptions,
682 // but we cannot reliably test it.
683 builder_.Gen<PseudoCopy>(slots_[5], slots_[4], 8);
684 dst_data.slots[5].lo = data_.slots[4].lo;
685
686 builder_.Gen<PseudoCopy>(slots_[7], slots_[6], 16);
687 dst_data.slots[7] = data_.slots[6];
688
689 Finalize();
690 test_.Exec();
691 ExpectEqualData(data_, dst_data);
692 }
693
694 // TODO(b/200327919): Share with tests in runtime.
695 class ScopedSignalHandler {
696 public:
ScopedSignalHandler(int sig,void (* action)(int,siginfo_t *,void *))697 ScopedSignalHandler(int sig, void (*action)(int, siginfo_t*, void*)) : sig_(sig) {
698 struct sigaction act {};
699 act.sa_sigaction = action;
700 act.sa_flags = SA_SIGINFO;
701 sigaction(sig_, &act, &old_act_);
702 }
703
~ScopedSignalHandler()704 ~ScopedSignalHandler() { sigaction(sig_, &old_act_, nullptr); }
705
706 private:
707 int sig_;
708 struct sigaction old_act_;
709 };
710
711 const RecoveryMap* g_recovery_map;
712
SigsegvHandler(int sig,siginfo_t *,void * context)713 void SigsegvHandler(int sig, siginfo_t*, void* context) {
714 ASSERT_EQ(sig, SIGSEGV);
715
716 ucontext_t* ucontext = reinterpret_cast<ucontext_t*>(context);
717 uintptr_t rip = ucontext->uc_mcontext.gregs[REG_RIP];
718 auto it = g_recovery_map->find(rip);
719 ASSERT_TRUE(it != g_recovery_map->end());
720 ucontext->uc_mcontext.gregs[REG_RIP] = it->second;
721 }
722
TEST(ExecMachineIR,RecoveryBlock)723 TEST(ExecMachineIR, RecoveryBlock) {
724 ScopedSignalHandler handler(SIGSEGV, SigsegvHandler);
725
726 Arena arena;
727 x86_64::MachineIR machine_ir(&arena);
728 constexpr auto kScratchReg = x86_64::kMachineRegRBP;
729 auto* main_bb = machine_ir.NewBasicBlock();
730 auto* recovery_bb = machine_ir.NewBasicBlock();
731
732 x86_64::MachineIRBuilder builder(&machine_ir);
733 builder.StartBasicBlock(main_bb);
734 // Cause a SIGSEGV.
735 builder.Gen<x86_64::XorqRegReg>(kScratchReg, kScratchReg, x86_64::kMachineRegFLAGS);
736 builder.Gen<x86_64::MovqMemBaseDispReg>(kScratchReg, 0, kScratchReg);
737 builder.SetRecoveryPointAtLastInsn(recovery_bb);
738 builder.Gen<PseudoJump>(21ULL);
739
740 builder.StartBasicBlock(recovery_bb);
741 builder.Gen<PseudoJump>(42ULL);
742
743 machine_ir.AddEdge(main_bb, recovery_bb);
744
745 ExecTest test;
746 test.Init(machine_ir);
747 g_recovery_map = &test.recovery_map();
748
749 test.Exec();
750
751 // Guest PC for recovery is set in RAX.
752 EXPECT_EQ(test.returned_rax(), 42ULL);
753 }
754
TEST(ExecMachineIR,RecoveryWithGuestPC)755 TEST(ExecMachineIR, RecoveryWithGuestPC) {
756 ScopedSignalHandler handler(SIGSEGV, SigsegvHandler);
757
758 Arena arena;
759 x86_64::MachineIR machine_ir(&arena);
760 constexpr auto kScratchReg = x86_64::kMachineRegRBP;
761
762 x86_64::MachineIRBuilder builder(&machine_ir);
763 builder.StartBasicBlock(machine_ir.NewBasicBlock());
764 // Cause a SIGSEGV.
765 builder.Gen<x86_64::XorqRegReg>(kScratchReg, kScratchReg, x86_64::kMachineRegFLAGS);
766 builder.Gen<x86_64::MovqMemBaseDispReg>(kScratchReg, 0, kScratchReg);
767 builder.SetRecoveryWithGuestPCAtLastInsn(42ULL);
768
769 ExecTest test;
770 test.Init(machine_ir);
771 g_recovery_map = &test.recovery_map();
772
773 test.Exec();
774
775 // Guest PC for recovery is set to RAX.
776 EXPECT_EQ(test.returned_rax(), 42ULL);
777 }
778
TEST(ExecMachineIR,PseudoReadFlags)779 TEST(ExecMachineIR, PseudoReadFlags) {
780 struct Data {
781 uint64_t x;
782 uint64_t y;
783 } data{};
784 uint64_t res_flags;
785
786 Arena arena;
787 x86_64::MachineIR machine_ir(&arena);
788
789 x86_64::MachineIRBuilder builder(&machine_ir);
790 builder.StartBasicBlock(machine_ir.NewBasicBlock());
791
792 // Let RBP point to 'data'.
793 builder.Gen<x86_64::MovqRegImm>(x86_64::kMachineRegRBP, reinterpret_cast<uintptr_t>(&data));
794 builder.Gen<x86_64::MovqRegMemBaseDisp>(
795 x86_64::kMachineRegRAX, x86_64::kMachineRegRBP, offsetof(Data, x));
796 builder.Gen<x86_64::AddqRegMemBaseDisp>(
797 x86_64::kMachineRegRAX, x86_64::kMachineRegRBP, offsetof(Data, y), x86_64::kMachineRegFLAGS);
798 builder.Gen<PseudoReadFlags>(
799 PseudoReadFlags::kWithOverflow, x86_64::kMachineRegRAX, x86_64::kMachineRegFLAGS);
800 builder.Gen<x86_64::MovqRegImm>(x86_64::kMachineRegRBP, reinterpret_cast<uintptr_t>(&res_flags));
801 builder.Gen<x86_64::MovqMemBaseDispReg>(x86_64::kMachineRegRBP, 0, x86_64::kMachineRegRAX);
802
803 ExecTest test;
804 test.Init(machine_ir);
805
806 data.x = 1;
807 data.y = 1;
808 test.Exec();
809 EXPECT_EQ(res_flags & MakeFlags(0b1111), MakeFlags(0b0000));
810
811 data.x = ~0ULL;
812 data.y = 1;
813 test.Exec();
814 EXPECT_EQ(res_flags & MakeFlags(0b1111), MakeFlags(0b0110));
815
816 data.x = (~0ULL) >> 1;
817 data.y = 1;
818 test.Exec();
819 EXPECT_EQ(res_flags & MakeFlags(0b1111), MakeFlags(0b1001));
820 }
821
TEST(ExecMachineIR,PseudoReadFlagsWithoutOverflow)822 TEST(ExecMachineIR, PseudoReadFlagsWithoutOverflow) {
823 struct Data {
824 uint64_t x;
825 uint64_t y;
826 } data{};
827 uint64_t res_flags;
828
829 Arena arena;
830 x86_64::MachineIR machine_ir(&arena);
831
832 x86_64::MachineIRBuilder builder(&machine_ir);
833 builder.StartBasicBlock(machine_ir.NewBasicBlock());
834
835 // Let RBP point to 'data'.
836 builder.Gen<x86_64::MovqRegImm>(x86_64::kMachineRegRBP, reinterpret_cast<uintptr_t>(&data));
837 builder.Gen<x86_64::MovqRegMemBaseDisp>(
838 x86_64::kMachineRegRAX, x86_64::kMachineRegRBP, offsetof(Data, x));
839 builder.Gen<x86_64::AddqRegMemBaseDisp>(
840 x86_64::kMachineRegRAX, x86_64::kMachineRegRBP, offsetof(Data, y), x86_64::kMachineRegFLAGS);
841 // ReadFlags must reset overflow to zero, even if it's set in RAX.
842 builder.Gen<x86_64::MovqRegImm>(x86_64::kMachineRegRAX, MakeFlags(0b0001));
843 builder.Gen<PseudoReadFlags>(
844 PseudoReadFlags::kWithoutOverflow, x86_64::kMachineRegRAX, x86_64::kMachineRegFLAGS);
845 builder.Gen<x86_64::MovqRegImm>(x86_64::kMachineRegRBP, reinterpret_cast<uintptr_t>(&res_flags));
846 builder.Gen<x86_64::MovqMemBaseDispReg>(x86_64::kMachineRegRBP, 0, x86_64::kMachineRegRAX);
847
848 ExecTest test;
849 test.Init(machine_ir);
850
851 data.x = (~0ULL) >> 1;
852 data.y = 1;
853 test.Exec();
854 // Overflow happens but is not returned.
855 EXPECT_EQ(res_flags & MakeFlags(0b1111), MakeFlags(0b1000));
856 }
857
TEST(ExecMachineIR,PseudoWriteFlags)858 TEST(ExecMachineIR, PseudoWriteFlags) {
859 uint64_t arg_flags;
860 uint64_t res_flags;
861
862 Arena arena;
863 x86_64::MachineIR machine_ir(&arena);
864
865 x86_64::MachineIRBuilder builder(&machine_ir);
866 builder.StartBasicBlock(machine_ir.NewBasicBlock());
867
868 builder.Gen<x86_64::MovqRegImm>(x86_64::kMachineRegRBP, reinterpret_cast<uintptr_t>(&arg_flags));
869 builder.Gen<x86_64::MovqRegMemBaseDisp>(x86_64::kMachineRegRAX, x86_64::kMachineRegRBP, 0);
870 builder.Gen<PseudoWriteFlags>(x86_64::kMachineRegRAX, x86_64::kMachineRegFLAGS);
871 // Assume PseudoReadFlags is verified by another test.
872 builder.Gen<PseudoReadFlags>(
873 PseudoReadFlags::kWithOverflow, x86_64::kMachineRegRAX, x86_64::kMachineRegFLAGS);
874 builder.Gen<x86_64::MovqRegImm>(x86_64::kMachineRegRBP, reinterpret_cast<uintptr_t>(&res_flags));
875 builder.Gen<x86_64::MovqMemBaseDispReg>(x86_64::kMachineRegRBP, 0, x86_64::kMachineRegRAX);
876
877 ExecTest test;
878 test.Init(machine_ir);
879
880 arg_flags = MakeFlags(0b1111);
881 res_flags = 0;
882 test.Exec();
883 EXPECT_EQ(res_flags & MakeFlags(0b1111), MakeFlags(0b1111));
884
885 arg_flags = MakeFlags(0b1010);
886 res_flags = 0;
887 test.Exec();
888 EXPECT_EQ(res_flags & MakeFlags(0b1111), MakeFlags(0b1010));
889
890 arg_flags = MakeFlags(0b0101);
891 res_flags = 0;
892 test.Exec();
893 EXPECT_EQ(res_flags & MakeFlags(0b1111), MakeFlags(0b0101));
894 }
895
896 } // namespace
897
898 } // namespace berberis
899