1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "translator_riscv64.h"
18 #include "berberis/runtime/translator.h"
19
20 #include <cstdint>
21 #include <cstdlib>
22 #include <tuple>
23
24 #include "berberis/assembler/machine_code.h"
25 #include "berberis/base/checks.h"
26 #include "berberis/base/config_globals.h"
27 #include "berberis/base/tracing.h"
28 #include "berberis/guest_os_primitives/guest_map_shadow.h"
29 #include "berberis/guest_state/guest_addr.h"
30 #include "berberis/guest_state/guest_state_opaque.h"
31 #include "berberis/heavy_optimizer/riscv64/heavy_optimize_region.h"
32 #include "berberis/interpreter/riscv64/interpreter.h"
33 #include "berberis/lite_translator/lite_translate_region.h"
34 #include "berberis/runtime_primitives/code_pool.h"
35 #include "berberis/runtime_primitives/host_code.h"
36 #include "berberis/runtime_primitives/profiler_interface.h"
37 #include "berberis/runtime_primitives/runtime_library.h"
38 #include "berberis/runtime_primitives/translation_cache.h"
39 #include "berberis/runtime_primitives/virtual_guest_call_frame.h"
40
41 namespace berberis {
42
43 namespace {
44
45 // Syntax sugar.
46 GuestCodeEntry::Kind kSpecialHandler = GuestCodeEntry::Kind::kSpecialHandler;
47 GuestCodeEntry::Kind kInterpreted = GuestCodeEntry::Kind::kInterpreted;
48 GuestCodeEntry::Kind kLightTranslated = GuestCodeEntry::Kind::kLightTranslated;
49 GuestCodeEntry::Kind kHeavyOptimized = GuestCodeEntry::Kind::kHeavyOptimized;
50
51 enum class TranslationMode {
52 kInterpretOnly,
53 kLiteTranslateOrFallbackToInterpret,
54 kHeavyOptimizeOrFallbackToInterpret,
55 kHeavyOptimizeOrFallbackToLiteTranslator,
56 kLightTranslateThenHeavyOptimize,
57 kTwoGear = kLightTranslateThenHeavyOptimize,
58 kNumModes
59 };
60
61 TranslationMode g_translation_mode = TranslationMode::kTwoGear;
62
UpdateTranslationMode()63 void UpdateTranslationMode() {
64 // Indices must match TranslationMode enum.
65 constexpr const char* kTranslationModeNames[] = {"interpret-only",
66 "lite-translate-or-interpret",
67 "heavy-optimize-or-interpret",
68 "heavy-optimize-or-lite-translate",
69 "two-gear"};
70 static_assert(static_cast<int>(TranslationMode::kNumModes) ==
71 sizeof(kTranslationModeNames) / sizeof(char*));
72
73 const char* config_mode = GetTranslationModeConfig();
74 if (!config_mode) {
75 return;
76 }
77
78 for (int i = 0; i < static_cast<int>(TranslationMode::kNumModes); i++) {
79 if (0 == strcmp(config_mode, kTranslationModeNames[i])) {
80 g_translation_mode = TranslationMode(i);
81 TRACE("translation mode is manually set to '%s'", config_mode);
82 return;
83 }
84 }
85
86 LOG_ALWAYS_FATAL("Unrecognized translation mode '%s'", config_mode);
87 }
88
89 // Use aligned address of this variable as the default stop address for guest execution.
90 // It should never coincide with any guest address or address of a wrapped host symbol.
91 // Unwinder might examine nearby insns.
92 alignas(4) uint32_t g_native_bridge_call_guest[] = {
93 // <native_bridge_call_guest>:
94 0xd503201f, // nop
95 0xd503201f, // nop <--
96 0xd503201f, // nop
97 };
98
99 enum class TranslationGear {
100 kFirst,
101 kSecond,
102 };
103
GetRiscv64InsnSize(GuestAddr pc)104 uint8_t GetRiscv64InsnSize(GuestAddr pc) {
105 constexpr uint16_t kInsnLenMask = uint16_t{0b11};
106 if ((*ToHostAddr<uint16_t>(pc) & kInsnLenMask) != kInsnLenMask) {
107 return 2;
108 }
109 return 4;
110 }
111
112 } // namespace
113
InstallTranslated(MachineCode * machine_code,GuestAddr pc,size_t size,const char * prefix)114 HostCodePiece InstallTranslated(MachineCode* machine_code,
115 GuestAddr pc,
116 size_t size,
117 const char* prefix) {
118 HostCode host_code = GetDefaultCodePoolInstance()->Add(machine_code);
119 ProfilerLogGeneratedCode(host_code, machine_code->install_size(), pc, size, prefix);
120 return {host_code, machine_code->install_size()};
121 }
122
InitTranslator()123 void InitTranslator() {
124 UpdateTranslationMode();
125 InitVirtualGuestCallFrameReturnAddress(ToGuestAddr(g_native_bridge_call_guest + 1));
126 InitInterpreter();
127 }
128
129 // Exported for testing only.
TryLiteTranslateAndInstallRegion(GuestAddr pc,const LiteTranslateParams & params)130 std::tuple<bool, HostCodePiece, size_t, GuestCodeEntry::Kind> TryLiteTranslateAndInstallRegion(
131 GuestAddr pc,
132 const LiteTranslateParams& params) {
133 MachineCode machine_code;
134
135 auto [success, stop_pc] = TryLiteTranslateRegion(pc, &machine_code, params);
136
137 size_t size = stop_pc - pc;
138
139 if (success) {
140 return {true, InstallTranslated(&machine_code, pc, size, "lite"), size, kLightTranslated};
141 }
142
143 if (size == 0) {
144 // Cannot translate even single instruction - the attempt failed.
145 return {false, {}, 0, {}};
146 }
147
148 MachineCode another_machine_code;
149 success = LiteTranslateRange(pc, stop_pc, &another_machine_code, params);
150 CHECK(success);
151
152 return {true,
153 InstallTranslated(&another_machine_code, pc, size, "lite_range"),
154 size,
155 kLightTranslated};
156 }
157
158 // Exported for testing only.
HeavyOptimizeRegion(GuestAddr pc)159 std::tuple<bool, HostCodePiece, size_t, GuestCodeEntry::Kind> HeavyOptimizeRegion(GuestAddr pc) {
160 MachineCode machine_code;
161 auto [stop_pc, success, unused_number_of_processed_instructions] =
162 HeavyOptimizeRegion(pc, &machine_code);
163 size_t size = stop_pc - pc;
164 if (success) {
165 return {true, InstallTranslated(&machine_code, pc, size, "heavy"), size, kHeavyOptimized};
166 }
167
168 if (size == 0) {
169 // Cannot translate even single instruction - the attempt failed.
170 return {false, {}, 0, {}};
171 }
172
173 // Report success because we at least translated some instructions.
174 return {true, InstallTranslated(&machine_code, pc, size, "heavy"), size, kHeavyOptimized};
175 }
176
177 template <TranslationGear kGear = TranslationGear::kFirst>
TranslateRegion(GuestAddr pc)178 void TranslateRegion(GuestAddr pc) {
179 TranslationCache* cache = TranslationCache::GetInstance();
180
181 GuestCodeEntry* entry;
182 if constexpr (kGear == TranslationGear::kFirst) {
183 entry = cache->AddAndLockForTranslation(pc, 0);
184 } else {
185 CHECK(g_translation_mode == TranslationMode::kTwoGear);
186 entry = cache->LockForGearUpTranslation(pc);
187 }
188 if (!entry) {
189 return;
190 }
191
192 GuestMapShadow* guest_map_shadow = GuestMapShadow::GetInstance();
193
194 // First check if the instruction would be in executable memory if it is compressed. This
195 // prevents dereferencing unknown memory to determine the size of the instruction.
196 constexpr uint8_t kMinimumInsnSize = 2;
197 if (!guest_map_shadow->IsExecutable(pc, kMinimumInsnSize)) {
198 cache->SetTranslatedAndUnlock(pc, entry, kMinimumInsnSize, kSpecialHandler, {kEntryNoExec, 0});
199 return;
200 }
201
202 // Now check the rest of the instruction based on its size. It is now safe to dereference the
203 // memory at pc because at least two bytes are within known executable memory.
204 uint8_t first_insn_size = GetRiscv64InsnSize(pc);
205 if (first_insn_size > kMinimumInsnSize &&
206 !guest_map_shadow->IsExecutable(pc + kMinimumInsnSize, first_insn_size - kMinimumInsnSize)) {
207 cache->SetTranslatedAndUnlock(pc, entry, first_insn_size, kSpecialHandler, {kEntryNoExec, 0});
208 return;
209 }
210
211 bool success;
212 HostCodePiece host_code_piece;
213 size_t size;
214 GuestCodeEntry::Kind kind;
215 if (g_translation_mode == TranslationMode::kInterpretOnly) {
216 std::tie(host_code_piece, size, kind) =
217 std::make_tuple(HostCodePiece{kEntryInterpret, 0}, first_insn_size, kInterpreted);
218 } else if (g_translation_mode == TranslationMode::kLiteTranslateOrFallbackToInterpret) {
219 std::tie(success, host_code_piece, size, kind) = TryLiteTranslateAndInstallRegion(pc);
220 if (!success) {
221 std::tie(host_code_piece, size, kind) =
222 std::make_tuple(HostCodePiece{kEntryInterpret, 0}, first_insn_size, kInterpreted);
223 }
224 } else if (g_translation_mode == TranslationMode::kTwoGear && kGear == TranslationGear::kFirst) {
225 std::tie(success, host_code_piece, size, kind) = TryLiteTranslateAndInstallRegion(
226 pc, {.enable_self_profiling = true, .counter_location = &(entry->invocation_counter)});
227 if (!success) {
228 // Heavy supports more insns than lite, so try to heavy optimize. If that fails, then
229 // fallback to interpret.
230 std::tie(success, host_code_piece, size, kind) = HeavyOptimizeRegion(pc);
231 if (!success) {
232 std::tie(host_code_piece, size, kind) =
233 std::make_tuple(HostCodePiece{kEntryInterpret, 0}, first_insn_size, kInterpreted);
234 }
235 }
236 } else if (g_translation_mode == TranslationMode::kHeavyOptimizeOrFallbackToInterpret ||
237 g_translation_mode == TranslationMode::kHeavyOptimizeOrFallbackToLiteTranslator ||
238 (g_translation_mode == TranslationMode::kTwoGear &&
239 kGear == TranslationGear::kSecond)) {
240 std::tie(success, host_code_piece, size, kind) = HeavyOptimizeRegion(pc);
241 if (!success) {
242 if (g_translation_mode == TranslationMode::kHeavyOptimizeOrFallbackToInterpret ||
243 // Lite might fail since not all insns are implemented. Fallback to interpret.
244 (g_translation_mode == TranslationMode::kTwoGear && kGear == TranslationGear::kSecond)) {
245 std::tie(host_code_piece, size, kind) =
246 std::make_tuple(HostCodePiece{kEntryInterpret, 0}, first_insn_size, kInterpreted);
247 } else if (g_translation_mode == TranslationMode::kHeavyOptimizeOrFallbackToLiteTranslator) {
248 std::tie(success, host_code_piece, size, kind) =
249 TryLiteTranslateAndInstallRegion(pc, {.enable_self_profiling = false});
250 // Lite might fail since not all insns are implemented. Fallback to interpret.
251 if (!success) {
252 std::tie(host_code_piece, size, kind) =
253 std::make_tuple(HostCodePiece{kEntryInterpret, 0}, first_insn_size, kInterpreted);
254 }
255 }
256 }
257 } else {
258 LOG_ALWAYS_FATAL("Unsupported translation mode %u", g_translation_mode);
259 }
260
261 // Now that we know the size of the translated block, make sure the entire memory block has
262 // executable permission before saving it to the cache.
263 // TODO(b/232598137): installing kEntryNoExec for the *current* pc is completely incorrect as
264 // we've checked that it's executable above. The straightforward thing to do would be to
265 // check executability of each instruction while translating, and generating signal raise
266 // for non-executable ones. This handles the case when region contains conditional branch
267 // to non-executable code.
268 if (!guest_map_shadow->IsExecutable(pc, size)) {
269 TRACE("setting partly executable region at [0x%zx, 0x%zx) as not executable!", pc, pc + size);
270 cache->SetTranslatedAndUnlock(pc, entry, size, kSpecialHandler, {kEntryNoExec, 0});
271 return;
272 }
273
274 cache->SetTranslatedAndUnlock(pc, entry, size, kind, host_code_piece);
275 }
276
277 // A wrapper to export a template function.
TranslateRegionAtFirstGear(GuestAddr pc)278 void TranslateRegionAtFirstGear(GuestAddr pc) {
279 TranslateRegion<TranslationGear::kFirst>(pc);
280 }
281
282 // ATTENTION: This symbol gets called directly, without PLT. To keep text
283 // sharable we should prevent preemption of this symbol, so do not export it!
284 // TODO(b/232598137): may be set default visibility to protected instead?
berberis_HandleNotTranslated(ThreadState * state)285 extern "C" __attribute__((used, __visibility__("hidden"))) void berberis_HandleNotTranslated(
286 ThreadState* state) {
287 TranslateRegion(state->cpu.insn_addr);
288 }
289
berberis_HandleInterpret(ThreadState * state)290 extern "C" __attribute__((used, __visibility__("hidden"))) void berberis_HandleInterpret(
291 ThreadState* state) {
292 InterpretInsn(state);
293 }
294
berberis_GetDispatchAddress(ThreadState * state)295 extern "C" __attribute__((used, __visibility__("hidden"))) const void* berberis_GetDispatchAddress(
296 ThreadState* state) {
297 CHECK(state);
298 if (ArePendingSignalsPresent(*state)) {
299 return kEntryExitGeneratedCode;
300 }
301 return TranslationCache::GetInstance()->GetHostCodePtr(state->cpu.insn_addr)->load();
302 }
303
304 extern "C" __attribute__((used, __visibility__("hidden"))) void
berberis_HandleLightCounterThresholdReached(ThreadState * state)305 berberis_HandleLightCounterThresholdReached(ThreadState* state) {
306 CHECK(g_translation_mode == TranslationMode::kTwoGear);
307 TranslateRegion<TranslationGear::kSecond>(state->cpu.insn_addr);
308 }
309
310 } // namespace berberis
311