1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "host/libs/vm_manager/crosvm_manager.h"
18
19 #include <android-base/file.h>
20 #include <android-base/logging.h>
21 #include <android-base/strings.h>
22 #include <sys/stat.h>
23 #include <sys/types.h>
24 #include <vulkan/vulkan.h>
25
26 #include <cassert>
27 #include <string>
28 #include <vector>
29
30 #include "common/libs/utils/environment.h"
31 #include "common/libs/utils/files.h"
32 #include "common/libs/utils/network.h"
33 #include "common/libs/utils/subprocess.h"
34 #include "host/libs/config/cuttlefish_config.h"
35 #include "host/libs/config/known_paths.h"
36 #include "host/libs/vm_manager/crosvm_builder.h"
37 #include "host/libs/vm_manager/qemu_manager.h"
38
39 namespace cuttlefish {
40 namespace vm_manager {
41
42 namespace {
43
GetControlSocketPath(const CuttlefishConfig::InstanceSpecific & instance,const std::string & socket_name)44 std::string GetControlSocketPath(
45 const CuttlefishConfig::InstanceSpecific& instance,
46 const std::string& socket_name) {
47 return instance.PerInstanceInternalPath(socket_name.c_str());
48 }
49
50 } // namespace
51
IsSupported()52 bool CrosvmManager::IsSupported() {
53 #ifdef __ANDROID__
54 return true;
55 #else
56 return HostSupportsQemuCli();
57 #endif
58 }
59
ConfigureGraphics(const CuttlefishConfig & config)60 std::vector<std::string> CrosvmManager::ConfigureGraphics(
61 const CuttlefishConfig& config) {
62 // Override the default HAL search paths in all cases. We do this because
63 // the HAL search path allows for fallbacks, and fallbacks in conjunction
64 // with properities lead to non-deterministic behavior while loading the
65 // HALs.
66 if (config.gpu_mode() == kGpuModeGuestSwiftshader) {
67 return {
68 "androidboot.cpuvulkan.version=" + std::to_string(VK_API_VERSION_1_2),
69 "androidboot.hardware.gralloc=minigbm",
70 "androidboot.hardware.hwcomposer="+ config.hwcomposer(),
71 "androidboot.hardware.egl=angle",
72 "androidboot.hardware.vulkan=pastel",
73 "androidboot.opengles.version=196609"}; // OpenGL ES 3.1
74 }
75
76 if (config.gpu_mode() == kGpuModeDrmVirgl) {
77 return {
78 "androidboot.cpuvulkan.version=0",
79 "androidboot.hardware.gralloc=minigbm",
80 "androidboot.hardware.hwcomposer=ranchu",
81 "androidboot.hardware.hwcomposer.mode=client",
82 "androidboot.hardware.egl=mesa",
83 // No "hardware" Vulkan support, yet
84 "androidboot.opengles.version=196608"}; // OpenGL ES 3.0
85 }
86 if (config.gpu_mode() == kGpuModeGfxStream) {
87 std::string gles_impl = config.enable_gpu_angle() ? "angle" : "emulation";
88 return {"androidboot.cpuvulkan.version=0",
89 "androidboot.hardware.gralloc=minigbm",
90 "androidboot.hardware.hwcomposer=" + config.hwcomposer(),
91 "androidboot.hardware.egl=" + gles_impl,
92 "androidboot.hardware.vulkan=ranchu",
93 "androidboot.hardware.gltransport=virtio-gpu-asg",
94 "androidboot.opengles.version=196608"}; // OpenGL ES 3.0
95 }
96 return {};
97 }
98
ConfigureBootDevices(int num_disks)99 std::string CrosvmManager::ConfigureBootDevices(int num_disks) {
100 // TODO There is no way to control this assignment with crosvm (yet)
101 if (HostArch() == Arch::X86_64) {
102 // crosvm has an additional PCI device for an ISA bridge
103 // virtio_gpu and virtio_wl precedes the first console or disk
104 return ConfigureMultipleBootDevices("pci0000:00/0000:00:", 3, num_disks);
105 } else {
106 // On ARM64 crosvm, block devices are on their own bridge, so we don't
107 // need to calculate it, and the path is always the same
108 return "androidboot.boot_devices=10000.pci";
109 }
110 }
111
112 constexpr auto crosvm_socket = "crosvm_control.sock";
113
StartCommands(const CuttlefishConfig & config)114 std::vector<Command> CrosvmManager::StartCommands(
115 const CuttlefishConfig& config) {
116 auto instance = config.ForDefaultInstance();
117 CrosvmBuilder crosvm_cmd;
118 crosvm_cmd.SetBinary(config.crosvm_binary());
119 crosvm_cmd.AddControlSocket(GetControlSocketPath(instance, crosvm_socket));
120
121 if (!config.smt()) {
122 crosvm_cmd.Cmd().AddParameter("--no-smt");
123 }
124
125 if (config.vhost_net()) {
126 crosvm_cmd.Cmd().AddParameter("--vhost-net");
127 }
128
129 #ifdef ENFORCE_MAC80211_HWSIM
130 if (!config.vhost_user_mac80211_hwsim().empty()) {
131 crosvm_cmd.Cmd().AddParameter("--vhost-user-mac80211-hwsim=",
132 config.vhost_user_mac80211_hwsim());
133 }
134 #endif
135
136 if (config.protected_vm()) {
137 crosvm_cmd.Cmd().AddParameter("--protected-vm");
138 }
139
140 if (config.gdb_port() > 0) {
141 CHECK(config.cpus() == 1) << "CPUs must be 1 for crosvm gdb mode";
142 crosvm_cmd.Cmd().AddParameter("--gdb=", config.gdb_port());
143 }
144
145 auto gpu_capture_enabled = !config.gpu_capture_binary().empty();
146 auto gpu_mode = config.gpu_mode();
147 auto udmabuf_string = config.enable_gpu_udmabuf() ? "true" : "false";
148 auto angle_string = config.enable_gpu_angle() ? ",angle=true" : "";
149 if (gpu_mode == kGpuModeGuestSwiftshader) {
150 crosvm_cmd.Cmd().AddParameter("--gpu=2D,udmabuf=", udmabuf_string);
151 } else if (gpu_mode == kGpuModeDrmVirgl || gpu_mode == kGpuModeGfxStream) {
152 crosvm_cmd.Cmd().AddParameter(
153 gpu_mode == kGpuModeGfxStream ? "--gpu=gfxstream," : "--gpu=",
154 "egl=true,surfaceless=true,glx=false,gles=true,udmabuf=", udmabuf_string,
155 angle_string);
156 }
157
158 for (const auto& display_config : config.display_configs()) {
159 crosvm_cmd.Cmd().AddParameter(
160 "--gpu-display=", "width=", display_config.width, ",",
161 "height=", display_config.height);
162 }
163
164 crosvm_cmd.Cmd().AddParameter("--wayland-sock=",
165 instance.frames_socket_path());
166
167 // crosvm_cmd.Cmd().AddParameter("--null-audio");
168 crosvm_cmd.Cmd().AddParameter("--mem=", config.memory_mb());
169 crosvm_cmd.Cmd().AddParameter("--cpus=", config.cpus());
170
171 auto disk_num = instance.virtual_disk_paths().size();
172 CHECK_GE(VmManager::kMaxDisks, disk_num)
173 << "Provided too many disks (" << disk_num << "), maximum "
174 << VmManager::kMaxDisks << "supported";
175 for (const auto& disk : instance.virtual_disk_paths()) {
176 crosvm_cmd.Cmd().AddParameter(
177 config.protected_vm() ? "--disk=" : "--rwdisk=", disk);
178 }
179
180 if (config.enable_webrtc()) {
181 auto touch_type_parameter =
182 config.enable_webrtc() ? "--multi-touch=" : "--single-touch=";
183
184 auto display_configs = config.display_configs();
185 CHECK_GE(display_configs.size(), 1);
186
187 for (int i = 0; i < display_configs.size(); ++i) {
188 auto display_config = display_configs[i];
189
190 crosvm_cmd.Cmd().AddParameter(
191 touch_type_parameter, instance.touch_socket_path(i), ":",
192 display_config.width, ":", display_config.height);
193 }
194 crosvm_cmd.Cmd().AddParameter("--keyboard=",
195 instance.keyboard_socket_path());
196 }
197 if (config.enable_webrtc()) {
198 crosvm_cmd.Cmd().AddParameter("--switches=",
199 instance.switches_socket_path());
200 }
201
202 SharedFD wifi_tap;
203 // GPU capture can only support named files and not file descriptors due to
204 // having to pass arguments to crosvm via a wrapper script.
205 if (!gpu_capture_enabled) {
206 crosvm_cmd.AddTap(instance.mobile_tap_name());
207 crosvm_cmd.AddTap(instance.ethernet_tap_name());
208
209 // TODO(b/199103204): remove this as well when
210 // PRODUCT_ENFORCE_MAC80211_HWSIM is removed
211 #ifndef ENFORCE_MAC80211_HWSIM
212 wifi_tap = crosvm_cmd.AddTap(instance.wifi_tap_name());
213 #endif
214 }
215
216 if (FileExists(instance.access_kregistry_path())) {
217 crosvm_cmd.Cmd().AddParameter("--rw-pmem-device=",
218 instance.access_kregistry_path());
219 }
220
221 if (FileExists(instance.hwcomposer_pmem_path())) {
222 crosvm_cmd.Cmd().AddParameter("--rw-pmem-device=",
223 instance.hwcomposer_pmem_path());
224 }
225
226 if (FileExists(instance.pstore_path())) {
227 crosvm_cmd.Cmd().AddParameter("--pstore=path=", instance.pstore_path(),
228 ",size=", FileSize(instance.pstore_path()));
229 }
230
231 if (config.enable_sandbox()) {
232 const bool seccomp_exists = DirectoryExists(config.seccomp_policy_dir());
233 const std::string& var_empty_dir = kCrosvmVarEmptyDir;
234 const bool var_empty_available = DirectoryExists(var_empty_dir);
235 if (!var_empty_available || !seccomp_exists) {
236 LOG(FATAL) << var_empty_dir << " is not an existing, empty directory."
237 << "seccomp-policy-dir, " << config.seccomp_policy_dir()
238 << " does not exist " << std::endl;
239 return {};
240 }
241 crosvm_cmd.Cmd().AddParameter("--seccomp-policy-dir=",
242 config.seccomp_policy_dir());
243 } else {
244 crosvm_cmd.Cmd().AddParameter("--disable-sandbox");
245 }
246
247 if (instance.vsock_guest_cid() >= 2) {
248 crosvm_cmd.Cmd().AddParameter("--cid=", instance.vsock_guest_cid());
249 }
250
251 // Use a virtio-console instance for the main kernel console. All
252 // messages will switch from earlycon to virtio-console after the driver
253 // is loaded, and crosvm will append to the kernel log automatically
254 crosvm_cmd.AddHvcConsoleReadOnly(instance.kernel_log_pipe_name());
255
256 if (config.console()) {
257 // stdin is the only currently supported way to write data to a serial port in
258 // crosvm. A file (named pipe) is used here instead of stdout to ensure only
259 // the serial port output is received by the console forwarder as crosvm may
260 // print other messages to stdout.
261 if (config.kgdb() || config.use_bootloader()) {
262 crosvm_cmd.AddSerialConsoleReadWrite(instance.console_out_pipe_name(),
263 instance.console_in_pipe_name());
264 // In kgdb mode, we have the interactive console on ttyS0 (both Android's
265 // console and kdb), so we can disable the virtio-console port usually
266 // allocated to Android's serial console, and redirect it to a sink. This
267 // ensures that that the PCI device assignments (and thus sepolicy) don't
268 // have to change
269 crosvm_cmd.AddHvcSink();
270 } else {
271 crosvm_cmd.AddSerialSink();
272 crosvm_cmd.AddHvcReadWrite(instance.console_out_pipe_name(),
273 instance.console_in_pipe_name());
274 }
275 } else {
276 // Use an 8250 UART (ISA or platform device) for earlycon, as the
277 // virtio-console driver may not be available for early messages
278 // In kgdb mode, earlycon is an interactive console, and so early
279 // dmesg will go there instead of the kernel.log
280 if (config.kgdb() || config.use_bootloader()) {
281 crosvm_cmd.AddSerialConsoleReadOnly(instance.kernel_log_pipe_name());
282 }
283
284 // as above, create a fake virtio-console 'sink' port when the serial
285 // console is disabled, so the PCI device ID assignments don't move
286 // around
287 crosvm_cmd.AddHvcSink();
288 }
289
290 auto crosvm_logs_path = instance.PerInstanceInternalPath("crosvm.fifo");
291 auto crosvm_logs = SharedFD::Fifo(crosvm_logs_path, 0666);
292 if (!crosvm_logs->IsOpen()) {
293 LOG(FATAL) << "Failed to create log fifo for crosvm's stdout/stderr: "
294 << crosvm_logs->StrError();
295 return {};
296 }
297
298 Command crosvm_log_tee_cmd(HostBinaryPath("log_tee"));
299 crosvm_log_tee_cmd.AddParameter("--process_name=crosvm");
300 crosvm_log_tee_cmd.AddParameter("--log_fd_in=", crosvm_logs);
301
302 // Serial port for logcat, redirected to a pipe
303 crosvm_cmd.AddHvcReadOnly(instance.logcat_pipe_name());
304
305 crosvm_cmd.AddHvcReadWrite(
306 instance.PerInstanceInternalPath("keymaster_fifo_vm.out"),
307 instance.PerInstanceInternalPath("keymaster_fifo_vm.in"));
308 crosvm_cmd.AddHvcReadWrite(
309 instance.PerInstanceInternalPath("gatekeeper_fifo_vm.out"),
310 instance.PerInstanceInternalPath("gatekeeper_fifo_vm.in"));
311
312 if (config.enable_host_bluetooth()) {
313 crosvm_cmd.AddHvcReadWrite(
314 instance.PerInstanceInternalPath("bt_fifo_vm.out"),
315 instance.PerInstanceInternalPath("bt_fifo_vm.in"));
316 } else {
317 crosvm_cmd.AddHvcSink();
318 }
319 if (config.enable_gnss_grpc_proxy()) {
320 crosvm_cmd.AddHvcReadWrite(
321 instance.PerInstanceInternalPath("gnsshvc_fifo_vm.out"),
322 instance.PerInstanceInternalPath("gnsshvc_fifo_vm.in"));
323 crosvm_cmd.AddHvcReadWrite(
324 instance.PerInstanceInternalPath("locationhvc_fifo_vm.out"),
325 instance.PerInstanceInternalPath("locationhvc_fifo_vm.in"));
326 } else {
327 for (auto i = 0; i < 2; i++) {
328 crosvm_cmd.AddHvcSink();
329 }
330 }
331
332 for (auto i = 0; i < VmManager::kMaxDisks - disk_num; i++) {
333 crosvm_cmd.AddHvcSink();
334 }
335 CHECK(crosvm_cmd.HvcNum() + disk_num ==
336 VmManager::kMaxDisks + VmManager::kDefaultNumHvcs)
337 << "HVC count (" << crosvm_cmd.HvcNum() << ") + disk count (" << disk_num
338 << ") is not the expected total of "
339 << VmManager::kMaxDisks + VmManager::kDefaultNumHvcs << " devices";
340
341 if (config.enable_audio()) {
342 crosvm_cmd.Cmd().AddParameter(
343 "--sound=", config.ForDefaultInstance().audio_server_path());
344 }
345
346 // TODO(b/162071003): virtiofs crashes without sandboxing, this should be fixed
347 if (0 && config.enable_sandbox()) {
348 // Set up directory shared with virtiofs
349 crosvm_cmd.Cmd().AddParameter(
350 "--shared-dir=", instance.PerInstancePath(kSharedDirName),
351 ":shared:type=fs");
352 }
353
354 // This needs to be the last parameter
355 crosvm_cmd.Cmd().AddParameter("--bios=", config.bootloader());
356
357 // TODO(b/199103204): remove this as well when PRODUCT_ENFORCE_MAC80211_HWSIM
358 // is removed
359 // Only run the leases workaround if we are not using the new network
360 // bridge architecture - in that case, we have a wider DHCP address
361 // space and stale leases should be much less of an issue
362 if (!FileExists("/var/run/cuttlefish-dnsmasq-cvd-wbr.leases") &&
363 wifi_tap->IsOpen()) {
364 // TODO(schuffelen): QEMU also needs this and this is not the best place for
365 // this code. Find a better place to put it.
366 auto lease_file =
367 ForCurrentInstance("/var/run/cuttlefish-dnsmasq-cvd-wbr-") + ".leases";
368
369 std::uint8_t dhcp_server_ip[] = {
370 192, 168, 96, (std::uint8_t)(ForCurrentInstance(1) * 4 - 3)};
371 if (!ReleaseDhcpLeases(lease_file, wifi_tap, dhcp_server_ip)) {
372 LOG(ERROR) << "Failed to release wifi DHCP leases. Connecting to the wifi "
373 << "network may not work.";
374 }
375 }
376
377 std::vector<Command> ret;
378
379 if (gpu_capture_enabled) {
380 const std::string gpu_capture_basename =
381 cpp_basename(config.gpu_capture_binary());
382
383 auto gpu_capture_logs_path =
384 instance.PerInstanceInternalPath("gpu_capture.fifo");
385 auto gpu_capture_logs = SharedFD::Fifo(gpu_capture_logs_path, 0666);
386 if (!gpu_capture_logs->IsOpen()) {
387 LOG(FATAL)
388 << "Failed to create log fifo for gpu capture's stdout/stderr: "
389 << gpu_capture_logs->StrError();
390 return {};
391 }
392
393 Command gpu_capture_log_tee_cmd(HostBinaryPath("log_tee"));
394 gpu_capture_log_tee_cmd.AddParameter("--process_name=",
395 gpu_capture_basename);
396 gpu_capture_log_tee_cmd.AddParameter("--log_fd_in=", gpu_capture_logs);
397
398 Command gpu_capture_command(config.gpu_capture_binary());
399 if (gpu_capture_basename == "ngfx") {
400 // Crosvm depends on command line arguments being passed as multiple
401 // arguments but ngfx only allows a single `--args`. To work around this,
402 // create a wrapper script that launches crosvm with all of the arguments
403 // and pass this wrapper script to ngfx.
404 const std::string crosvm_wrapper_path =
405 instance.PerInstanceInternalPath("crosvm_wrapper.sh");
406 const std::string crosvm_wrapper_content =
407 crosvm_cmd.Cmd().AsBashScript(crosvm_logs_path);
408
409 CHECK(android::base::WriteStringToFile(crosvm_wrapper_content,
410 crosvm_wrapper_path));
411 CHECK(MakeFileExecutable(crosvm_wrapper_path));
412
413 gpu_capture_command.AddParameter("--exe=", crosvm_wrapper_path);
414 gpu_capture_command.AddParameter("--launch-detached");
415 gpu_capture_command.AddParameter("--verbose");
416 gpu_capture_command.AddParameter("--activity=Frame Debugger");
417 } else {
418 // TODO(natsu): renderdoc
419 LOG(FATAL) << "Unhandled GPU capture binary: "
420 << config.gpu_capture_binary();
421 }
422
423 gpu_capture_command.RedirectStdIO(Subprocess::StdIOChannel::kStdOut,
424 gpu_capture_logs);
425 gpu_capture_command.RedirectStdIO(Subprocess::StdIOChannel::kStdErr,
426 gpu_capture_logs);
427
428 ret.push_back(std::move(gpu_capture_log_tee_cmd));
429 ret.push_back(std::move(gpu_capture_command));
430 } else {
431 crosvm_cmd.Cmd().RedirectStdIO(Subprocess::StdIOChannel::kStdOut,
432 crosvm_logs);
433 crosvm_cmd.Cmd().RedirectStdIO(Subprocess::StdIOChannel::kStdErr,
434 crosvm_logs);
435
436 ret.push_back(std::move(crosvm_cmd.Cmd()));
437 }
438
439 ret.push_back(std::move(crosvm_log_tee_cmd));
440 return ret;
441 }
442
443 } // namespace vm_manager
444 } // namespace cuttlefish
445
446