1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "gtest/gtest.h"
18
19 #include <chrono> // chrono_literals::operator""ms
20 #include <initializer_list>
21 #include <string>
22 #include <thread> // this_thread::sleep_for
23
24 #include "berberis/base/config.h"
25 #include "berberis/guest_state/guest_addr.h"
26 #include "berberis/runtime_primitives/host_code.h"
27 #include "berberis/runtime_primitives/runtime_library.h" // kEntry*
28 #include "berberis/runtime_primitives/translation_cache.h"
29
30 namespace berberis {
31
32 namespace {
33
34 using std::chrono_literals::operator""ms;
35 // A test guest pc that is valid in both 32bit and 64bit modes.
36 constexpr GuestAddr kGuestPC = 0x12345678;
37
TEST(TranslationCacheTest,DefaultNotTranslated)38 TEST(TranslationCacheTest, DefaultNotTranslated) {
39 TranslationCache tc;
40
41 EXPECT_EQ(tc.GetHostCodePtr(kGuestPC)->load(), kEntryNotTranslated);
42 EXPECT_EQ(tc.GetHostCodePtr(kGuestPC + 1024)->load(), kEntryNotTranslated);
43 EXPECT_EQ(tc.GetInvocationCounter(kGuestPC), 0U);
44 }
45
TEST(TranslationCacheTest,UpdateInvocationCounter)46 TEST(TranslationCacheTest, UpdateInvocationCounter) {
47 TranslationCache tc;
48
49 // Create entry
50 GuestCodeEntry* entry = tc.AddAndLockForTranslation(kGuestPC, 0);
51 ASSERT_TRUE(entry);
52 EXPECT_EQ(entry->invocation_counter, 0U);
53 entry->invocation_counter = 42;
54 tc.SetTranslatedAndUnlock(
55 kGuestPC, entry, 1, GuestCodeEntry::Kind::kSpecialHandler, {kEntryNoExec, 0});
56
57 EXPECT_EQ(tc.GetInvocationCounter(kGuestPC), 42U);
58 }
59
TEST(TranslationCacheTest,AddAndLockForTranslation)60 TEST(TranslationCacheTest, AddAndLockForTranslation) {
61 TranslationCache tc;
62
63 // Cannot lock if counter is below the threshold, but entry is created anyway.
64 ASSERT_FALSE(tc.AddAndLockForTranslation(kGuestPC, 1));
65 GuestCodeEntry* entry = tc.LookupGuestCodeEntryUnsafeForTesting(kGuestPC);
66 ASSERT_TRUE(entry);
67 EXPECT_EQ(tc.GetHostCodePtr(kGuestPC)->load(), kEntryNotTranslated);
68 EXPECT_EQ(entry->kind, GuestCodeEntry::Kind::kInterpreted);
69 EXPECT_EQ(tc.GetInvocationCounter(kGuestPC), 1U);
70
71 // Lock when counter is equal or above the threshold.
72 entry = tc.AddAndLockForTranslation(kGuestPC, 1);
73 ASSERT_TRUE(entry);
74 EXPECT_EQ(tc.GetHostCodePtr(kGuestPC)->load(), kEntryTranslating);
75 EXPECT_EQ(entry->kind, GuestCodeEntry::Kind::kUnderProcessing);
76 EXPECT_EQ(tc.GetInvocationCounter(kGuestPC), 1U);
77
78 // Cannot lock locked.
79 ASSERT_FALSE(tc.AddAndLockForTranslation(kGuestPC, 0));
80
81 // Unlock.
82 tc.SetTranslatedAndUnlock(
83 kGuestPC, entry, 1, GuestCodeEntry::Kind::kSpecialHandler, {kEntryNoExec, 0});
84 EXPECT_EQ(tc.GetHostCodePtr(kGuestPC)->load(), kEntryNoExec);
85 EXPECT_EQ(entry->kind, GuestCodeEntry::Kind::kSpecialHandler);
86
87 // Cannot lock translated.
88 ASSERT_FALSE(tc.AddAndLockForTranslation(kGuestPC, 0));
89 }
90
91 constexpr bool kWrappedHostFunc = true;
92
TEST(TranslationCacheTest,AddAndLockForWrapping)93 TEST(TranslationCacheTest, AddAndLockForWrapping) {
94 TranslationCache tc;
95
96 // Add and lock nonexistent.
97 GuestCodeEntry* entry = tc.AddAndLockForWrapping(kGuestPC);
98 ASSERT_TRUE(entry);
99 ASSERT_EQ(kEntryWrapping, tc.GetHostCodePtr(kGuestPC)->load());
100 ASSERT_EQ(entry->kind, GuestCodeEntry::Kind::kUnderProcessing);
101
102 // Cannot lock locked.
103 ASSERT_FALSE(tc.AddAndLockForWrapping(kGuestPC));
104
105 // Unlock.
106 tc.SetWrappedAndUnlock(kGuestPC, entry, kWrappedHostFunc, {kEntryNoExec, 0});
107 ASSERT_EQ(kEntryNoExec, tc.GetHostCodePtr(kGuestPC)->load());
108 ASSERT_EQ(entry->kind, GuestCodeEntry::Kind::kHostWrapped);
109
110 // Cannot lock wrapped.
111 ASSERT_FALSE(tc.AddAndLockForWrapping(kGuestPC));
112
113 // Cannot lock not translated but already interpreted.
114 ASSERT_FALSE(tc.AddAndLockForTranslation(kGuestPC + 64, 1));
115 entry = tc.LookupGuestCodeEntryUnsafeForTesting(kGuestPC + 64);
116 ASSERT_TRUE(entry);
117 ASSERT_EQ(entry->kind, GuestCodeEntry::Kind::kInterpreted);
118 ASSERT_FALSE(tc.AddAndLockForWrapping(kGuestPC + 64));
119 }
120
121 HostCodeAddr kHostCodeStub = AsHostCodeAddr(AsHostCode(0xdeadbeef));
122
TestWrappingWorker(TranslationCache * tc,GuestAddr pc)123 void TestWrappingWorker(TranslationCache* tc, GuestAddr pc) {
124 while (true) {
125 GuestCodeEntry* entry = tc->AddAndLockForWrapping(pc);
126 if (entry) {
127 EXPECT_EQ(entry->host_code->load(), kEntryWrapping);
128 EXPECT_EQ(entry->kind, GuestCodeEntry::Kind::kUnderProcessing);
129 // Give other threads some time to run the loop. Typical test
130 // time is 1 ms, make sleep order of magnitude longer - 10ms.
131 std::this_thread::sleep_for(10ms);
132 tc->SetWrappedAndUnlock(pc, entry, kWrappedHostFunc, {kHostCodeStub, 0});
133 EXPECT_EQ(entry->host_code->load(), kHostCodeStub);
134 EXPECT_EQ(entry->kind, GuestCodeEntry::Kind::kHostWrapped);
135 return;
136 }
137
138 auto host_code = tc->GetHostCodePtr(pc)->load();
139
140 // Warning: the order of comparisons here is
141 // important since code can change in between.
142 if (host_code == kEntryWrapping) {
143 continue;
144 }
145
146 EXPECT_EQ(host_code, kHostCodeStub);
147 break;
148 }
149
150 return;
151 }
152
TestTranslationWorker(TranslationCache * tc,GuestAddr pc)153 void TestTranslationWorker(TranslationCache* tc, GuestAddr pc) {
154 while (true) {
155 GuestCodeEntry* entry = tc->AddAndLockForTranslation(pc, 0);
156 if (entry) {
157 EXPECT_EQ(entry->host_code->load(), kEntryTranslating);
158 EXPECT_EQ(entry->kind, GuestCodeEntry::Kind::kUnderProcessing);
159 // Give other threads some time to run the loop. Typical test
160 // time is 1 ms, make sleep order of magnitude longer - 10ms.
161 std::this_thread::sleep_for(10ms);
162 tc->SetTranslatedAndUnlock(
163 pc, entry, 1, GuestCodeEntry::Kind::kSpecialHandler, {kHostCodeStub, 0});
164 EXPECT_EQ(entry->host_code->load(), kHostCodeStub);
165 return;
166 }
167
168 auto host_code = tc->GetHostCodePtr(pc)->load();
169 if (host_code == kEntryTranslating) {
170 continue;
171 }
172 EXPECT_EQ(host_code, kHostCodeStub);
173 break;
174 }
175
176 return;
177 }
178
179 template <void(WorkerFunc)(TranslationCache*, GuestAddr)>
TranslationCacheTestRunThreads()180 void TranslationCacheTestRunThreads() {
181 TranslationCache tc;
182 constexpr uint32_t kNumThreads = 16;
183 std::array<std::thread, kNumThreads> threads;
184
185 // First test situation, when every thread has its own pc.
186 for (uint32_t i = 0; i < kNumThreads; i++) {
187 threads[i] = std::thread(WorkerFunc, &tc, i);
188 }
189
190 for (auto& thread : threads) {
191 thread.join();
192 }
193
194 // Now introduce heavy contention.
195 for (auto& thread : threads) {
196 thread = std::thread(WorkerFunc, &tc, kGuestPC);
197 }
198
199 for (auto& thread : threads) {
200 thread.join();
201 }
202 }
203
TEST(TranslationCacheTest,InvalidateNotTranslated)204 TEST(TranslationCacheTest, InvalidateNotTranslated) {
205 TranslationCache tc;
206
207 ASSERT_EQ(kEntryNotTranslated, tc.GetHostCodePtr(kGuestPC)->load());
208
209 tc.InvalidateGuestRange(kGuestPC, kGuestPC + 1);
210
211 // Not translated stays not translated
212 ASSERT_EQ(kEntryNotTranslated, tc.GetHostCodePtr(kGuestPC)->load());
213 ASSERT_FALSE(tc.LookupGuestCodeEntryUnsafeForTesting(kGuestPC));
214 }
215
TEST(TranslationCacheTest,InvalidateTranslated)216 TEST(TranslationCacheTest, InvalidateTranslated) {
217 TranslationCache tc;
218
219 GuestCodeEntry* entry = tc.AddAndLockForTranslation(kGuestPC, 0);
220 ASSERT_TRUE(entry);
221 ASSERT_EQ(kEntryTranslating, tc.GetHostCodePtr(kGuestPC)->load());
222
223 tc.SetTranslatedAndUnlock(
224 kGuestPC, entry, 1, GuestCodeEntry::Kind::kHeavyOptimized, {kHostCodeStub, 4});
225 ASSERT_EQ(kHostCodeStub, tc.GetHostCodePtr(kGuestPC)->load());
226
227 tc.InvalidateGuestRange(kGuestPC, kGuestPC + 1);
228
229 ASSERT_EQ(kEntryNotTranslated, tc.GetHostCodePtr(kGuestPC)->load());
230 ASSERT_FALSE(tc.LookupGuestCodeEntryUnsafeForTesting(kGuestPC));
231 }
232
TEST(TranslationCacheTest,InvalidateTranslating)233 TEST(TranslationCacheTest, InvalidateTranslating) {
234 TranslationCache tc;
235
236 GuestCodeEntry* entry = tc.AddAndLockForTranslation(kGuestPC, 0);
237 ASSERT_TRUE(entry);
238 ASSERT_EQ(kEntryTranslating, tc.GetHostCodePtr(kGuestPC)->load());
239
240 tc.InvalidateGuestRange(kGuestPC, kGuestPC + 1);
241 ASSERT_EQ(kEntryInvalidating, tc.GetHostCodePtr(kGuestPC)->load());
242 entry = tc.LookupGuestCodeEntryUnsafeForTesting(kGuestPC);
243 ASSERT_TRUE(entry);
244 ASSERT_EQ(entry->kind, GuestCodeEntry::Kind::kUnderProcessing);
245
246 tc.SetTranslatedAndUnlock(
247 kGuestPC, entry, 1, GuestCodeEntry::Kind::kSpecialHandler, {kHostCodeStub, 4});
248 ASSERT_EQ(kEntryNotTranslated, tc.GetHostCodePtr(kGuestPC)->load());
249 ASSERT_FALSE(tc.LookupGuestCodeEntryUnsafeForTesting(kGuestPC));
250 }
251
TEST(TranslationCacheTest,InvalidateTranslatingOutOfRange)252 TEST(TranslationCacheTest, InvalidateTranslatingOutOfRange) {
253 TranslationCache tc;
254
255 GuestCodeEntry* entry = tc.AddAndLockForTranslation(kGuestPC, 0);
256 ASSERT_TRUE(entry);
257 ASSERT_EQ(kEntryTranslating, tc.GetHostCodePtr(kGuestPC)->load());
258
259 // Invalidate range that does *not* contain translating address.
260 // The entry should still be invalidated, as translated region is only known after translation,
261 // and it might overlap with the invalidated range.
262 tc.InvalidateGuestRange(kGuestPC + 100, kGuestPC + 101);
263 ASSERT_EQ(kEntryInvalidating, tc.GetHostCodePtr(kGuestPC)->load());
264
265 tc.SetTranslatedAndUnlock(
266 kGuestPC, entry, 1, GuestCodeEntry::Kind::kSpecialHandler, {kHostCodeStub, 4});
267 ASSERT_EQ(kEntryNotTranslated, tc.GetHostCodePtr(kGuestPC)->load());
268 }
269
Translate(TranslationCache * tc,GuestAddr pc,uint32_t size,HostCodeAddr host_code)270 bool Translate(TranslationCache* tc, GuestAddr pc, uint32_t size, HostCodeAddr host_code) {
271 GuestCodeEntry* entry = tc->AddAndLockForTranslation(pc, 0);
272 if (!entry) {
273 return false;
274 }
275 tc->SetTranslatedAndUnlock(
276 pc, entry, size, GuestCodeEntry::Kind::kSpecialHandler, {host_code, 4});
277 return true;
278 }
279
TEST(TranslationCacheTest,LockForGearUpTranslation)280 TEST(TranslationCacheTest, LockForGearUpTranslation) {
281 TranslationCache tc;
282
283 // Cannot lock if not yet added.
284 ASSERT_FALSE(tc.LockForGearUpTranslation(kGuestPC));
285
286 ASSERT_TRUE(Translate(&tc, kGuestPC + 0, 1, kHostCodeStub));
287 GuestCodeEntry* entry = tc.LookupGuestCodeEntryUnsafeForTesting(kGuestPC);
288 ASSERT_TRUE(entry);
289 ASSERT_EQ(entry->kind, GuestCodeEntry::Kind::kSpecialHandler);
290
291 // Cannot lock if kind is not kLiteTranslated.
292 ASSERT_FALSE(tc.LockForGearUpTranslation(kGuestPC));
293
294 entry->kind = GuestCodeEntry::Kind::kLiteTranslated;
295
296 entry = tc.LockForGearUpTranslation(kGuestPC);
297 ASSERT_TRUE(entry);
298 ASSERT_EQ(kEntryTranslating, tc.GetHostCodePtr(kGuestPC)->load());
299 ASSERT_EQ(entry->kind, GuestCodeEntry::Kind::kUnderProcessing);
300
301 // Unlock.
302 tc.SetTranslatedAndUnlock(
303 kGuestPC, entry, 1, GuestCodeEntry::Kind::kHeavyOptimized, {kEntryNoExec, 0});
304 ASSERT_EQ(kEntryNoExec, tc.GetHostCodePtr(kGuestPC)->load());
305 ASSERT_EQ(entry->kind, GuestCodeEntry::Kind::kHeavyOptimized);
306
307 // Cannot lock translated.
308 ASSERT_FALSE(tc.AddAndLockForTranslation(kGuestPC, 0));
309 }
310
TEST(TranslationCacheTest,InvalidateRange)311 TEST(TranslationCacheTest, InvalidateRange) {
312 TranslationCache tc;
313
314 ASSERT_TRUE(Translate(&tc, kGuestPC + 0, 1, kHostCodeStub));
315 ASSERT_TRUE(Translate(&tc, kGuestPC + 1, 1, kHostCodeStub));
316 ASSERT_TRUE(Translate(&tc, kGuestPC + 2, 1, kHostCodeStub));
317
318 ASSERT_EQ(kHostCodeStub, tc.GetHostCodePtr(kGuestPC + 0)->load());
319 ASSERT_EQ(kHostCodeStub, tc.GetHostCodePtr(kGuestPC + 1)->load());
320 ASSERT_EQ(kHostCodeStub, tc.GetHostCodePtr(kGuestPC + 2)->load());
321
322 tc.InvalidateGuestRange(kGuestPC + 1, kGuestPC + 2);
323
324 ASSERT_EQ(kHostCodeStub, tc.GetHostCodePtr(kGuestPC + 0)->load());
325 ASSERT_EQ(kEntryNotTranslated, tc.GetHostCodePtr(kGuestPC + 1)->load());
326 ASSERT_EQ(kHostCodeStub, tc.GetHostCodePtr(kGuestPC + 2)->load());
327 }
328
Wrap(TranslationCache * tc,GuestAddr pc,HostCodeAddr host_code)329 bool Wrap(TranslationCache* tc, GuestAddr pc, HostCodeAddr host_code) {
330 GuestCodeEntry* entry = tc->AddAndLockForWrapping(pc);
331 if (!entry) {
332 return false;
333 }
334 tc->SetWrappedAndUnlock(pc, entry, kWrappedHostFunc, {host_code, 0});
335 return true;
336 }
337
TEST(TranslationCacheTest,InvalidateWrapped)338 TEST(TranslationCacheTest, InvalidateWrapped) {
339 TranslationCache tc;
340
341 ASSERT_TRUE(Wrap(&tc, kGuestPC, kEntryNoExec));
342
343 tc.InvalidateGuestRange(kGuestPC, kGuestPC + 1);
344
345 ASSERT_EQ(kEntryNotTranslated, tc.GetHostCodePtr(kGuestPC)->load());
346 }
347
TEST(TranslationCacheTest,InvalidateWrappingWrap)348 TEST(TranslationCacheTest, InvalidateWrappingWrap) {
349 TranslationCache tc;
350
351 GuestCodeEntry* entry = tc.AddAndLockForWrapping(kGuestPC);
352 ASSERT_TRUE(entry);
353
354 tc.InvalidateGuestRange(kGuestPC, kGuestPC + 1);
355 ASSERT_EQ(kEntryInvalidating, tc.GetHostCodePtr(kGuestPC)->load());
356
357 tc.SetWrappedAndUnlock(kGuestPC, entry, kWrappedHostFunc, {kEntryNoExec, 0});
358 ASSERT_EQ(kEntryNotTranslated, tc.GetHostCodePtr(kGuestPC)->load());
359
360 ASSERT_TRUE(Wrap(&tc, kGuestPC, kEntryNoExec));
361 }
362
TEST(TranslationCacheTest,WrapInvalidateWrap)363 TEST(TranslationCacheTest, WrapInvalidateWrap) {
364 TranslationCache tc;
365
366 ASSERT_TRUE(Wrap(&tc, kGuestPC, kEntryNoExec));
367
368 tc.InvalidateGuestRange(kGuestPC, kGuestPC + 1);
369
370 ASSERT_TRUE(Wrap(&tc, kGuestPC, kEntryNoExec));
371 }
372
TEST(TranslationCacheTest,WrapInvalidateTranslate)373 TEST(TranslationCacheTest, WrapInvalidateTranslate) {
374 TranslationCache tc;
375
376 ASSERT_TRUE(Wrap(&tc, kGuestPC, kEntryNoExec));
377
378 tc.InvalidateGuestRange(kGuestPC, kGuestPC + 1);
379
380 ASSERT_TRUE(Translate(&tc, kGuestPC, 1, kEntryNoExec));
381 }
382
TEST(TranslationCacheTest,WrappingStatesTest)383 TEST(TranslationCacheTest, WrappingStatesTest) {
384 TranslationCacheTestRunThreads<TestWrappingWorker>();
385 }
386
TEST(TranslationCacheTest,TranslationStatesTest)387 TEST(TranslationCacheTest, TranslationStatesTest) {
388 TranslationCacheTestRunThreads<TestTranslationWorker>();
389 }
390
391 constexpr size_t kGuestGearShiftRange = 64;
392
TestTriggerGearShiftForAddresses(GuestAddr pc,std::initializer_list<std::tuple<GuestAddr,uint32_t>> addr_and_expected_counter_list)393 void TestTriggerGearShiftForAddresses(
394 GuestAddr pc,
395 std::initializer_list<std::tuple<GuestAddr, uint32_t>> addr_and_expected_counter_list) {
396 TranslationCache tc;
397 // Lite translate interesting addresses.
398 for (auto [pc, unused_counter] : addr_and_expected_counter_list) {
399 ASSERT_TRUE(Translate(&tc, pc, 1, kHostCodeStub));
400 GuestCodeEntry* entry = tc.LookupGuestCodeEntryUnsafeForTesting(pc);
401 ASSERT_TRUE(entry);
402 ASSERT_EQ(entry->kind, GuestCodeEntry::Kind::kSpecialHandler);
403 ASSERT_EQ(entry->invocation_counter, 0u);
404 entry->kind = GuestCodeEntry::Kind::kLiteTranslated;
405 }
406
407 tc.TriggerGearShift(pc, kGuestGearShiftRange);
408
409 for (auto [pc, expected_counter] : addr_and_expected_counter_list) {
410 ASSERT_EQ(tc.LookupGuestCodeEntryUnsafeForTesting(pc)->invocation_counter, expected_counter)
411 << "pc=" << pc;
412 }
413 }
414
TEST(TranslationCacheTest,TriggerGearShift)415 TEST(TranslationCacheTest, TriggerGearShift) {
416 TestTriggerGearShiftForAddresses(kGuestPC,
417 {{kGuestPC, config::kGearSwitchThreshold},
418 {kGuestPC - kGuestGearShiftRange, config::kGearSwitchThreshold},
419 {kGuestPC - kGuestGearShiftRange - 1, 0},
420 {kGuestPC + kGuestGearShiftRange, config::kGearSwitchThreshold},
421 {kGuestPC + kGuestGearShiftRange + 1, 0}});
422 }
423
TEST(TranslationCacheTest,TriggerGearShiftTargetLessThanRange)424 TEST(TranslationCacheTest, TriggerGearShiftTargetLessThanRange) {
425 constexpr GuestAddr kSmallGuestPC = kGuestGearShiftRange / 2;
426 TestTriggerGearShiftForAddresses(
427 kSmallGuestPC,
428 {{kSmallGuestPC, config::kGearSwitchThreshold},
429 {kNullGuestAddr, config::kGearSwitchThreshold},
430 {kSmallGuestPC + kGuestGearShiftRange, config::kGearSwitchThreshold}});
431 }
432
TEST(TranslationCacheTest,TriggerGearShiftDoesNotAffectNotLiteTranslated)433 TEST(TranslationCacheTest, TriggerGearShiftDoesNotAffectNotLiteTranslated) {
434 TranslationCache tc;
435 ASSERT_TRUE(Translate(&tc, kGuestPC, 1, kHostCodeStub));
436 GuestCodeEntry* entry = tc.LookupGuestCodeEntryUnsafeForTesting(kGuestPC);
437 ASSERT_TRUE(entry);
438 ASSERT_EQ(entry->kind, GuestCodeEntry::Kind::kSpecialHandler);
439 ASSERT_EQ(entry->invocation_counter, 0u);
440
441 tc.TriggerGearShift(kGuestPC, kGuestGearShiftRange);
442
443 ASSERT_EQ(tc.LookupGuestCodeEntryUnsafeForTesting(kGuestPC)->invocation_counter, 0u);
444 }
445
446 } // namespace
447
448 } // namespace berberis
449