1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "host/libs/vm_manager/qemu_manager.h"
18
19 #include <string.h>
20 #include <sys/socket.h>
21 #include <sys/stat.h>
22 #include <sys/types.h>
23 #include <sys/un.h>
24 #include <sys/wait.h>
25 #include <unistd.h>
26
27 #include <cstdlib>
28 #include <iomanip>
29 #include <string>
30 #include <unordered_map>
31 #include <utility>
32 #include <vector>
33
34 #include <android-base/strings.h>
35 #include <android-base/logging.h>
36 #include <vulkan/vulkan.h>
37
38 #include "common/libs/utils/architecture.h"
39 #include "common/libs/utils/files.h"
40 #include "common/libs/utils/result.h"
41 #include "common/libs/utils/subprocess.h"
42 #include "host/libs/config/command_source.h"
43 #include "host/libs/config/cuttlefish_config.h"
44 #include "host/libs/vm_manager/vhost_user.h"
45
46 // This is the QEMU default, but set it explicitly just in case it
47 // changes upstream
48 static const int kMaxSerialPorts = 31;
49
50 namespace cuttlefish {
51 namespace vm_manager {
52 namespace {
53
GetMonitorPath(const CuttlefishConfig & config)54 std::string GetMonitorPath(const CuttlefishConfig& config) {
55 return config.ForDefaultInstance().PerInstanceInternalUdsPath(
56 "qemu_monitor.sock");
57 }
58
Stop()59 StopperResult Stop() {
60 auto config = CuttlefishConfig::Get();
61 auto monitor_path = GetMonitorPath(*config);
62 auto monitor_sock = SharedFD::SocketLocalClient(
63 monitor_path.c_str(), false, SOCK_STREAM);
64
65 if (!monitor_sock->IsOpen()) {
66 LOG(ERROR) << "The connection to qemu is closed, is it still running?";
67 return StopperResult::kStopFailure;
68 }
69 char msg[] = "{\"execute\":\"qmp_capabilities\"}{\"execute\":\"quit\"}";
70 ssize_t len = sizeof(msg) - 1;
71 while (len > 0) {
72 int tmp = monitor_sock->Write(msg, len);
73 if (tmp < 0) {
74 LOG(ERROR) << "Error writing to socket: " << monitor_sock->StrError();
75 return StopperResult::kStopFailure;
76 }
77 len -= tmp;
78 }
79 // Log the reply
80 char buff[1000];
81 while ((len = monitor_sock->Read(buff, sizeof(buff) - 1)) > 0) {
82 buff[len] = '\0';
83 LOG(INFO) << "From qemu monitor: " << buff;
84 }
85
86 return StopperResult::kStopSuccess;
87 }
88
GetQemuVersion(const std::string & qemu_binary)89 Result<std::pair<int, int>> GetQemuVersion(const std::string& qemu_binary) {
90 Command qemu_version_cmd(qemu_binary);
91 qemu_version_cmd.AddParameter("-version");
92
93 std::string qemu_version_input, qemu_version_output, qemu_version_error;
94 cuttlefish::SubprocessOptions options;
95 options.Verbose(false);
96 int qemu_version_ret = cuttlefish::RunWithManagedStdio(
97 std::move(qemu_version_cmd), &qemu_version_input, &qemu_version_output,
98 &qemu_version_error, std::move(options));
99 CF_EXPECT(qemu_version_ret == 0,
100 qemu_binary << " -version returned unexpected response "
101 << qemu_version_output << ". Stderr was "
102 << qemu_version_error);
103
104 // Snip around the extra text we don't care about
105 qemu_version_output.erase(0, std::string("QEMU emulator version ").length());
106 auto space_pos = qemu_version_output.find(" ", 0);
107 if (space_pos != std::string::npos) {
108 qemu_version_output.resize(space_pos);
109 }
110
111 auto qemu_version_bits = android::base::Split(qemu_version_output, ".");
112 return {{std::stoi(qemu_version_bits[0]), std::stoi(qemu_version_bits[1])}};
113 }
114
115 } // namespace
116
QemuManager(Arch arch)117 QemuManager::QemuManager(Arch arch) : arch_(arch) {}
118
IsSupported()119 bool QemuManager::IsSupported() {
120 return HostSupportsQemuCli();
121 }
122
123 Result<std::unordered_map<std::string, std::string>>
ConfigureGraphics(const CuttlefishConfig::InstanceSpecific & instance)124 QemuManager::ConfigureGraphics(
125 const CuttlefishConfig::InstanceSpecific& instance) {
126 // Override the default HAL search paths in all cases. We do this because
127 // the HAL search path allows for fallbacks, and fallbacks in conjunction
128 // with properties lead to non-deterministic behavior while loading the
129 // HALs.
130
131 std::unordered_map<std::string, std::string> bootconfig_args;
132 auto gpu_mode = instance.gpu_mode();
133 if (gpu_mode == kGpuModeGuestSwiftshader) {
134 bootconfig_args = {
135 {"androidboot.cpuvulkan.version", std::to_string(VK_API_VERSION_1_2)},
136 {"androidboot.hardware.gralloc", "minigbm"},
137 {"androidboot.hardware.hwcomposer", instance.hwcomposer()},
138 {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
139 {"androidboot.hardware.hwcomposer.display_framebuffer_format",
140 instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
141 {"androidboot.hardware.egl", "angle"},
142 {"androidboot.hardware.vulkan", "pastel"},
143 // OpenGL ES 3.1
144 {"androidboot.opengles.version", "196609"},
145 };
146 } else if (gpu_mode == kGpuModeDrmVirgl) {
147 bootconfig_args = {
148 {"androidboot.cpuvulkan.version", "0"},
149 {"androidboot.hardware.gralloc", "minigbm"},
150 {"androidboot.hardware.hwcomposer", "ranchu"},
151 {"androidboot.hardware.hwcomposer.mode", "client"},
152 {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
153 {"androidboot.hardware.hwcomposer.display_framebuffer_format",
154 instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
155 {"androidboot.hardware.egl", "mesa"},
156 // No "hardware" Vulkan support, yet
157 // OpenGL ES 3.0
158 {"androidboot.opengles.version", "196608"},
159 };
160 } else if (gpu_mode == kGpuModeGfxstream ||
161 gpu_mode == kGpuModeGfxstreamGuestAngle ||
162 gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader ||
163 gpu_mode == kGpuModeGfxstreamGuestAngleHostLavapipe) {
164 const bool uses_angle =
165 gpu_mode == kGpuModeGfxstreamGuestAngle ||
166 gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader ||
167 gpu_mode == kGpuModeGfxstreamGuestAngleHostLavapipe;
168 const std::string gles_impl = uses_angle ? "angle" : "emulation";
169 const std::string gltransport =
170 (instance.guest_android_version() == "11.0.0") ? "virtio-gpu-pipe"
171 : "virtio-gpu-asg";
172 bootconfig_args = {
173 {"androidboot.cpuvulkan.version", "0"},
174 {"androidboot.hardware.gralloc", "minigbm"},
175 {"androidboot.hardware.hwcomposer", instance.hwcomposer()},
176 {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
177 {"androidboot.hardware.hwcomposer.display_framebuffer_format",
178 instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
179 {"androidboot.hardware.egl", gles_impl},
180 {"androidboot.hardware.vulkan", "ranchu"},
181 {"androidboot.hardware.gltransport", gltransport},
182 {"androidboot.opengles.version", "196609"}, // OpenGL ES 3.1
183 };
184 } else if (instance.gpu_mode() == kGpuModeNone) {
185 return {};
186 } else {
187 return CF_ERR("Unhandled GPU mode: " << instance.gpu_mode());
188 }
189
190 if (!instance.gpu_angle_feature_overrides_enabled().empty()) {
191 bootconfig_args["androidboot.hardware.angle_feature_overrides_enabled"] =
192 instance.gpu_angle_feature_overrides_enabled();
193 }
194 if (!instance.gpu_angle_feature_overrides_disabled().empty()) {
195 bootconfig_args["androidboot.hardware.angle_feature_overrides_disabled"] =
196 instance.gpu_angle_feature_overrides_disabled();
197 }
198
199 return bootconfig_args;
200 }
201
202 Result<std::unordered_map<std::string, std::string>>
ConfigureBootDevices(const CuttlefishConfig::InstanceSpecific & instance)203 QemuManager::ConfigureBootDevices(
204 const CuttlefishConfig::InstanceSpecific& instance) {
205 const int num_disks = instance.virtual_disk_paths().size();
206 const int num_gpu = instance.hwcomposer() != kHwComposerNone;
207 switch (arch_) {
208 case Arch::Arm:
209 return {{{"androidboot.boot_devices", "3f000000.pcie"}}};
210 case Arch::Arm64:
211 #ifdef __APPLE__
212 return {{{"androidboot.boot_devices", "3f000000.pcie"}}};
213 #else
214 return {{{"androidboot.boot_devices", "4010000000.pcie"}}};
215 #endif
216 case Arch::RiscV64:
217 return {{{"androidboot.boot_devices", "soc/30000000.pci"}}};
218 case Arch::X86:
219 case Arch::X86_64: {
220 // QEMU has additional PCI devices for an ISA bridge and PIIX4
221 // virtio_gpu precedes the first console or disk
222 // TODO(schuffelen): Simplify this logic when crosvm uses multiport
223 int pci_offset = 3 + num_gpu - VmManager::kDefaultNumHvcs;
224 return ConfigureMultipleBootDevices("pci0000:00/0000:00:", pci_offset,
225 num_disks);
226 }
227 }
228 }
229
StartCommands(const CuttlefishConfig & config,std::vector<VmmDependencyCommand * > & dependency_commands)230 Result<std::vector<MonitorCommand>> QemuManager::StartCommands(
231 const CuttlefishConfig& config,
232 std::vector<VmmDependencyCommand*>& dependency_commands) {
233 std::vector<MonitorCommand> commands;
234 auto instance = config.ForDefaultInstance();
235 std::string qemu_binary = instance.qemu_binary_dir();
236 switch (arch_) {
237 case Arch::Arm:
238 qemu_binary += "/qemu-system-arm";
239 break;
240 case Arch::Arm64:
241 qemu_binary += "/qemu-system-aarch64";
242 break;
243 case Arch::RiscV64:
244 qemu_binary += "/qemu-system-riscv64";
245 break;
246 case Arch::X86:
247 qemu_binary += "/qemu-system-i386";
248 break;
249 case Arch::X86_64:
250 qemu_binary += "/qemu-system-x86_64";
251 break;
252 }
253
254 auto qemu_version = CF_EXPECT(GetQemuVersion(qemu_binary));
255 Command qemu_cmd(qemu_binary, KillSubprocessFallback(Stop));
256
257 qemu_cmd.AddPrerequisite([&dependency_commands]() -> Result<void> {
258 for (auto dependencyCommand : dependency_commands) {
259 CF_EXPECT(dependencyCommand->WaitForAvailability());
260 }
261
262 return {};
263 });
264
265 int hvc_num = 0;
266 int serial_num = 0;
267 auto add_hvc_sink = [&qemu_cmd, &hvc_num]() {
268 qemu_cmd.AddParameter("-chardev");
269 qemu_cmd.AddParameter("null,id=hvc", hvc_num);
270 qemu_cmd.AddParameter("-device");
271 qemu_cmd.AddParameter("virtconsole,bus=virtio-serial.0,chardev=hvc",
272 hvc_num);
273 hvc_num++;
274 };
275 auto add_serial_sink = [&qemu_cmd, &serial_num]() {
276 qemu_cmd.AddParameter("-chardev");
277 qemu_cmd.AddParameter("null,id=serial", serial_num);
278 qemu_cmd.AddParameter("-serial");
279 qemu_cmd.AddParameter("chardev:serial", serial_num);
280 serial_num++;
281 };
282 auto add_serial_console_ro = [&qemu_cmd,
283 &serial_num](const std::string& output) {
284 qemu_cmd.AddParameter("-chardev");
285 qemu_cmd.AddParameter("file,id=serial", serial_num, ",path=", output,
286 ",append=on");
287 qemu_cmd.AddParameter("-serial");
288 qemu_cmd.AddParameter("chardev:serial", serial_num);
289 serial_num++;
290 };
291 auto add_serial_console = [&qemu_cmd,
292 &serial_num](const std::string& prefix) {
293 qemu_cmd.AddParameter("-chardev");
294 qemu_cmd.AddParameter("pipe,id=serial", serial_num, ",path=", prefix);
295 qemu_cmd.AddParameter("-serial");
296 qemu_cmd.AddParameter("chardev:serial", serial_num);
297 serial_num++;
298 };
299 auto add_hvc_ro = [&qemu_cmd, &hvc_num](const std::string& output) {
300 qemu_cmd.AddParameter("-chardev");
301 qemu_cmd.AddParameter("file,id=hvc", hvc_num, ",path=", output,
302 ",append=on");
303 qemu_cmd.AddParameter("-device");
304 qemu_cmd.AddParameter("virtconsole,bus=virtio-serial.0,chardev=hvc",
305 hvc_num);
306 hvc_num++;
307 };
308 auto add_hvc = [&qemu_cmd, &hvc_num](const std::string& prefix) {
309 qemu_cmd.AddParameter("-chardev");
310 qemu_cmd.AddParameter("pipe,id=hvc", hvc_num, ",path=", prefix);
311 qemu_cmd.AddParameter("-device");
312 qemu_cmd.AddParameter("virtconsole,bus=virtio-serial.0,chardev=hvc",
313 hvc_num);
314 hvc_num++;
315 };
316 auto add_hvc_serial = [&qemu_cmd, &hvc_num](const std::string& prefix) {
317 qemu_cmd.AddParameter("-chardev");
318 qemu_cmd.AddParameter("serial,id=hvc", hvc_num, ",path=", prefix);
319 qemu_cmd.AddParameter("-device");
320 qemu_cmd.AddParameter("virtconsole,bus=virtio-serial.0,chardev=hvc",
321 hvc_num);
322 hvc_num++;
323 };
324
325 bool is_arm = arch_ == Arch::Arm || arch_ == Arch::Arm64;
326 bool is_x86 = arch_ == Arch::X86 || arch_ == Arch::X86_64;
327 bool is_riscv64 = arch_ == Arch::RiscV64;
328
329 auto access_kregistry_size_bytes = 0;
330 if (FileExists(instance.access_kregistry_path())) {
331 access_kregistry_size_bytes = FileSize(instance.access_kregistry_path());
332 CF_EXPECT((access_kregistry_size_bytes & (1024 * 1024 - 1)) == 0,
333 instance.access_kregistry_path()
334 << " file size (" << access_kregistry_size_bytes
335 << ") not a multiple of 1MB");
336 }
337
338 auto hwcomposer_pmem_size_bytes = 0;
339 if (instance.hwcomposer() != kHwComposerNone) {
340 if (FileExists(instance.hwcomposer_pmem_path())) {
341 hwcomposer_pmem_size_bytes = FileSize(instance.hwcomposer_pmem_path());
342 CF_EXPECT((hwcomposer_pmem_size_bytes & (1024 * 1024 - 1)) == 0,
343 instance.hwcomposer_pmem_path()
344 << " file size (" << hwcomposer_pmem_size_bytes
345 << ") not a multiple of 1MB");
346 }
347 }
348
349 auto pstore_size_bytes = 0;
350 if (FileExists(instance.pstore_path())) {
351 pstore_size_bytes = FileSize(instance.pstore_path());
352 CF_EXPECT((pstore_size_bytes & (1024 * 1024 - 1)) == 0,
353 instance.pstore_path() << " file size (" << pstore_size_bytes
354 << ") not a multiple of 1MB");
355 }
356
357 qemu_cmd.AddParameter("-name");
358 qemu_cmd.AddParameter("guest=", instance.instance_name(), ",debug-threads=on");
359
360 std::string machine = is_x86 ? "pc,nvdimm=on" : "virt";
361 if (is_arm) {
362 if (IsHostCompatible(arch_)) {
363 machine += ",gic-version=3";
364 } else {
365 // QEMU doesn't support GICv3 with TCG yet
366 machine += ",gic-version=2";
367 CF_EXPECT(instance.cpus() <= 8, "CPUs must be no more than 8 with GICv2");
368 }
369 }
370 if (instance.mte()) {
371 machine += ",mte=on";
372 }
373 qemu_cmd.AddParameter("-machine");
374 qemu_cmd.AddParameter(machine, ",usb=off,dump-guest-core=off,memory-backend=vm_ram");
375
376 if (IsHostCompatible(arch_)) {
377 qemu_cmd.AddParameter("-accel");
378 std::string accel;
379 #ifdef __linux__
380 accel = "kvm";
381 if (!config.kvm_path().empty()) {
382 accel += ",device=" + config.kvm_path();
383 }
384 #elif defined(__APPLE__)
385 accel = "hvf";
386 #else
387 #error "Unknown OS"
388 #endif
389 qemu_cmd.AddParameter(accel);
390 }
391
392 // Memory must be backed by a file for vhost-user to work correctly, otherwise
393 // qemu doesn't send the memory mappings necessary for the backend to access
394 // the virtqueues.
395 qemu_cmd.AddParameter("-object");
396 qemu_cmd.AddParameter("memory-backend-file,size=", instance.memory_mb(), "M",
397 ",prealloc=on,share=on,mem-path=",
398 instance.PerInstanceInternalPath("qemu.mem"),
399 ",id=vm_ram");
400
401 qemu_cmd.AddParameter("-m");
402 auto maxmem = instance.memory_mb() +
403 (access_kregistry_size_bytes / 1024 / 1024) +
404 (hwcomposer_pmem_size_bytes / 1024 / 1024) +
405 (is_x86 ? pstore_size_bytes / 1024 / 1024 : 0);
406 auto slots = is_x86 ? ",slots=2" : "";
407 qemu_cmd.AddParameter("size=", instance.memory_mb(), "M",
408 ",maxmem=", maxmem, "M", slots);
409
410 qemu_cmd.AddParameter("-overcommit");
411 qemu_cmd.AddParameter("mem-lock=off");
412
413 // Assume SMT is always 2 threads per core, which is how most hardware
414 // today is configured, and the way crosvm does it
415 qemu_cmd.AddParameter("-smp");
416 if (instance.smt()) {
417 CF_EXPECT(instance.cpus() % 2 == 0,
418 "CPUs must be a multiple of 2 in SMT mode");
419 qemu_cmd.AddParameter(instance.cpus(), ",cores=",
420 instance.cpus() / 2, ",threads=2");
421 } else {
422 qemu_cmd.AddParameter(instance.cpus(), ",cores=",
423 instance.cpus(), ",threads=1");
424 }
425
426 qemu_cmd.AddParameter("-uuid");
427 qemu_cmd.AddParameter(instance.uuid());
428
429 qemu_cmd.AddParameter("-no-user-config");
430 qemu_cmd.AddParameter("-nodefaults");
431 qemu_cmd.AddParameter("-no-shutdown");
432
433 qemu_cmd.AddParameter("-rtc");
434 qemu_cmd.AddParameter("base=utc");
435
436 qemu_cmd.AddParameter("-boot");
437 qemu_cmd.AddParameter("strict=on");
438
439 qemu_cmd.AddParameter("-chardev");
440 qemu_cmd.AddParameter("socket,id=charmonitor,path=", GetMonitorPath(config),
441 ",server=on,wait=off");
442
443 qemu_cmd.AddParameter("-mon");
444 qemu_cmd.AddParameter("chardev=charmonitor,id=monitor,mode=control");
445
446 auto gpu_mode = instance.gpu_mode();
447 if (gpu_mode == kGpuModeDrmVirgl) {
448 qemu_cmd.AddParameter("-display");
449 qemu_cmd.AddParameter("egl-headless");
450
451 qemu_cmd.AddParameter("-vnc");
452 qemu_cmd.AddParameter("127.0.0.1:", instance.qemu_vnc_server_port());
453 } else if (gpu_mode == kGpuModeGuestSwiftshader ||
454 gpu_mode == kGpuModeGfxstream ||
455 gpu_mode == kGpuModeGfxstreamGuestAngle ||
456 gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader ||
457 gpu_mode == kGpuModeGfxstreamGuestAngleHostLavapipe) {
458 qemu_cmd.AddParameter("-vnc");
459 qemu_cmd.AddParameter("127.0.0.1:", instance.qemu_vnc_server_port());
460 } else {
461 qemu_cmd.AddParameter("-display");
462 qemu_cmd.AddParameter("none");
463 }
464
465 if (instance.hwcomposer() != kHwComposerNone) {
466 auto display_configs = instance.display_configs();
467 CF_EXPECT(display_configs.size() >= 1);
468 auto display_config = display_configs[0];
469
470 qemu_cmd.AddParameter("-device");
471
472 std::string gpu_device;
473 if (gpu_mode == kGpuModeGuestSwiftshader || qemu_version.first < 6) {
474 gpu_device = "virtio-gpu-pci";
475 } else if (gpu_mode == kGpuModeDrmVirgl) {
476 gpu_device = "virtio-gpu-gl-pci";
477 } else if (gpu_mode == kGpuModeGfxstream) {
478 gpu_device =
479 "virtio-gpu-rutabaga,x-gfxstream-gles=on,gfxstream-vulkan=on,"
480 "x-gfxstream-composer=on,hostmem=256M";
481 } else if (gpu_mode == kGpuModeGfxstreamGuestAngle ||
482 gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader ||
483 gpu_mode == kGpuModeGfxstreamGuestAngleHostLavapipe) {
484 gpu_device =
485 "virtio-gpu-rutabaga,gfxstream-vulkan=on,"
486 "x-gfxstream-composer=on,hostmem=256M";
487
488 if (gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader) {
489 // See https://github.com/KhronosGroup/Vulkan-Loader.
490 const std::string swiftshader_icd_json =
491 HostUsrSharePath("vulkan/icd.d/vk_swiftshader_icd.json");
492 qemu_cmd.AddEnvironmentVariable("VK_DRIVER_FILES",
493 swiftshader_icd_json);
494 qemu_cmd.AddEnvironmentVariable("VK_ICD_FILENAMES",
495 swiftshader_icd_json);
496 } else if (gpu_mode == kGpuModeGfxstreamGuestAngleHostLavapipe) {
497 // See https://github.com/KhronosGroup/Vulkan-Loader.
498 const std::string lavapipe_icd_json =
499 HostUsrSharePath("vulkan/icd.d/vk_lavapipe_icd.cf.json");
500 qemu_cmd.AddEnvironmentVariable("VK_DRIVER_FILES", lavapipe_icd_json);
501 qemu_cmd.AddEnvironmentVariable("VK_ICD_FILENAMES", lavapipe_icd_json);
502 }
503 }
504
505 qemu_cmd.AddParameter(
506 gpu_device, ",id=gpu0",
507 fmt::format(",addr={:0>2x}.0", VmManager::kGpuPciSlotNum),
508 ",xres=", display_config.width, ",yres=", display_config.height);
509 }
510
511 if (!instance.console()) {
512 // In kgdb mode, earlycon is an interactive console, and so early
513 // dmesg will go there instead of the kernel.log. On QEMU, we do this
514 // bit of logic up before the hvc console is set up, so the command line
515 // flags appear in the right order and "append=on" does the right thing
516 if (instance.enable_kernel_log() &&
517 (instance.kgdb() || instance.use_bootloader())) {
518 add_serial_console_ro(instance.kernel_log_pipe_name());
519 }
520 }
521
522 qemu_cmd.AddParameter("-device");
523 qemu_cmd.AddParameter(
524 "virtio-serial-pci-non-transitional,max_ports=", kMaxSerialPorts,
525 ",id=virtio-serial");
526
527 // /dev/hvc0 = kernel console
528 // If kernel log is enabled, the virtio-console port will be specified as
529 // a true console for Linux, and kernel messages will be printed there.
530 // Otherwise, the port will still be set up for bootloader and userspace
531 // messages, but the kernel will not print anything here. This keeps our
532 // kernel log event features working. If an alternative "earlycon" boot
533 // console is configured above on a legacy serial port, it will control
534 // the main log until the virtio-console takes over.
535 // (Note that QEMU does not automatically generate console= parameters for
536 // the bootloader/kernel cmdline, so the control of whether this pipe is
537 // actually managed by the kernel as a console is handled elsewhere.)
538 add_hvc_ro(instance.kernel_log_pipe_name());
539
540 // /dev/hvc1 = serial console
541 if (instance.console()) {
542 if (instance.kgdb() || instance.use_bootloader()) {
543 add_serial_console(instance.console_pipe_prefix());
544
545 // In kgdb mode, we have the interactive console on ttyS0 (both Android's
546 // console and kdb), so we can disable the virtio-console port usually
547 // allocated to Android's serial console, and redirect it to a sink. This
548 // ensures that that the PCI device assignments (and thus sepolicy) don't
549 // have to change
550 add_hvc_sink();
551 } else {
552 add_serial_sink();
553 add_hvc(instance.console_pipe_prefix());
554 }
555 } else {
556 if (instance.kgdb() || instance.use_bootloader()) {
557 // The add_serial_console_ro() call above was applied by the time we reach
558 // this code, so we don't need another add_serial_*() call
559 }
560
561 // as above, create a fake virtio-console 'sink' port when the serial
562 // console is disabled, so the PCI device ID assignments don't move
563 // around
564 add_hvc_sink();
565 }
566
567 // /dev/hvc2 = serial logging
568 // Serial port for logcat, redirected to a pipe
569 add_hvc_ro(instance.logcat_pipe_name());
570
571 // /dev/hvc3 = keymaster (C++ implementation)
572 add_hvc(instance.PerInstanceInternalPath("keymaster_fifo_vm"));
573 // /dev/hvc4 = gatekeeper
574 add_hvc(instance.PerInstanceInternalPath("gatekeeper_fifo_vm"));
575 // /dev/hvc5 = bt
576 if (config.enable_host_bluetooth()) {
577 add_hvc(instance.PerInstanceInternalPath("bt_fifo_vm"));
578 } else {
579 add_hvc_sink();
580 }
581
582 // /dev/hvc6 = gnss
583 // /dev/hvc7 = location
584 if (instance.enable_gnss_grpc_proxy()) {
585 add_hvc(instance.PerInstanceInternalPath("gnsshvc_fifo_vm"));
586 add_hvc(instance.PerInstanceInternalPath("locationhvc_fifo_vm"));
587 } else {
588 for (auto i = 0; i < 2; i++) {
589 add_hvc_sink();
590 }
591 }
592
593 /* Added one for confirmation UI.
594 *
595 * b/237452165
596 *
597 * Confirmation UI is not supported with QEMU for now. In order
598 * to not conflict with confirmation UI-related configurations used
599 * w/ Crosvm, we should add one generic avc.
600 *
601 * confui_fifo_vm.{in/out} are created along with the streamer process,
602 * which is not created w/ QEMU.
603 */
604 // /dev/hvc8 = confirmationui
605 add_hvc_sink();
606
607 // /dev/hvc9 = uwb
608 if (config.enable_host_uwb()) {
609 add_hvc(instance.PerInstanceInternalPath("uwb_fifo_vm"));
610 } else {
611 add_hvc_sink();
612 }
613
614 // /dev/hvc10 = oemlock
615 add_hvc(instance.PerInstanceInternalPath("oemlock_fifo_vm"));
616
617 // /dev/hvc11 = keymint (Rust implementation)
618 add_hvc(instance.PerInstanceInternalPath("keymint_fifo_vm"));
619
620 // /dev/hvc12 = nfc
621 if (config.enable_host_nfc()) {
622 add_hvc(instance.PerInstanceInternalPath("nfc_fifo_vm"));
623 } else {
624 add_hvc_sink();
625 }
626
627 // /dev/hvc13 = sensors
628 add_hvc(instance.PerInstanceInternalPath("sensors_fifo_vm"));
629
630 // /dev/hvc14 = MCU CONTROL
631 if (instance.mcu()["control"]["type"].asString() == "serial") {
632 auto path = instance.PerInstanceInternalPath("mcu");
633 path += "/" + instance.mcu()["control"]["path"].asString();
634 add_hvc_serial(path);
635 } else {
636 add_hvc_sink();
637 }
638
639 // /dev/hvc15 = MCU UART
640 if (instance.mcu()["uart0"]["type"].asString() == "serial") {
641 auto path = instance.PerInstanceInternalPath("mcu");
642 path += "/" + instance.mcu()["uart0"]["path"].asString();
643 add_hvc_serial(path);
644 } else {
645 add_hvc_sink();
646 }
647
648 // /dev/hvc16 = Ti50 TPM FIFO
649 if (!instance.ti50_emulator().empty()) {
650 // TODO
651 // add_hvc_socket(instance.PerInstancePath("direct_tpm_fifo"));
652 add_hvc_sink();
653 } else {
654 add_hvc_sink();
655 }
656
657 auto disk_num = instance.virtual_disk_paths().size();
658
659 for (auto i = 0; i < VmManager::kMaxDisks - disk_num; i++) {
660 add_hvc_sink();
661 }
662
663 CF_EXPECT(
664 hvc_num + disk_num == VmManager::kMaxDisks + VmManager::kDefaultNumHvcs,
665 "HVC count (" << hvc_num << ") + disk count (" << disk_num << ") "
666 << "is not the expected total of "
667 << VmManager::kMaxDisks + VmManager::kDefaultNumHvcs
668 << " devices");
669
670 CF_EXPECT(VmManager::kMaxDisks >= disk_num,
671 "Provided too many disks (" << disk_num << "), maximum "
672 << VmManager::kMaxDisks << "supported");
673 auto readonly = instance.protected_vm() ? ",readonly" : "";
674 size_t i = 0;
675 for (const auto& disk : instance.virtual_disk_paths()) {
676 if (instance.vhost_user_block()) {
677 auto block = CF_EXPECT(VhostUserBlockDevice(config, i, disk));
678 commands.emplace_back(std::move(block.device_cmd));
679 commands.emplace_back(std::move(block.device_logs_cmd));
680 auto socket_path = std::move(block.socket_path);
681 qemu_cmd.AddPrerequisite([socket_path]() -> Result<void> {
682 #ifdef __linux__
683 return WaitForUnixSocketListeningWithoutConnect(socket_path,
684 /*timeoutSec=*/30);
685 #else
686 return CF_ERR("Unhandled check if vhost user block ready.");
687 #endif
688 });
689
690 qemu_cmd.AddParameter("-chardev");
691 qemu_cmd.AddParameter("socket,id=vhost-user-block-", i,
692 ",path=", socket_path);
693 qemu_cmd.AddParameter("-device");
694 qemu_cmd.AddParameter(
695 "vhost-user-blk-pci-non-transitional,chardev=vhost-user-block-", i);
696 } else {
697 qemu_cmd.AddParameter("-drive");
698 qemu_cmd.AddParameter("file=", disk, ",if=none,id=drive-virtio-disk", i,
699 ",aio=threads", readonly);
700 qemu_cmd.AddParameter("-device");
701 qemu_cmd.AddParameter(
702 "virtio-blk-pci-non-transitional,drive=drive-virtio-disk", i,
703 ",id=virtio-disk", i, (i == 0 ? ",bootindex=1" : ""));
704 }
705 ++i;
706 }
707
708 if (is_x86 && FileExists(instance.pstore_path())) {
709 // QEMU will assign the NVDIMM (ramoops pstore region) 150000000-1501fffff
710 // As we will pass this to ramoops, define this region first so it is always
711 // located at this address. This is currently x86 only.
712 qemu_cmd.AddParameter("-object");
713 qemu_cmd.AddParameter("memory-backend-file,id=objpmem0,share=on,mem-path=",
714 instance.pstore_path(), ",size=", pstore_size_bytes);
715
716 qemu_cmd.AddParameter("-device");
717 qemu_cmd.AddParameter("nvdimm,memdev=objpmem0,id=ramoops");
718 }
719
720 // QEMU does not implement virtio-pmem-pci for ARM64 or RISC-V yet; restore
721 // this when the device has been added
722 if (is_x86) {
723 if (access_kregistry_size_bytes > 0) {
724 qemu_cmd.AddParameter("-object");
725 qemu_cmd.AddParameter(
726 "memory-backend-file,id=objpmem1,share=on,mem-path=",
727 instance.access_kregistry_path(),
728 ",size=", access_kregistry_size_bytes);
729
730 qemu_cmd.AddParameter("-device");
731 qemu_cmd.AddParameter(
732 "virtio-pmem-pci,disable-legacy=on,memdev=objpmem1,id=pmem0");
733 }
734 if (hwcomposer_pmem_size_bytes > 0) {
735 qemu_cmd.AddParameter("-object");
736 qemu_cmd.AddParameter(
737 "memory-backend-file,id=objpmem2,share=on,mem-path=",
738 instance.hwcomposer_pmem_path(),
739 ",size=", hwcomposer_pmem_size_bytes);
740
741 qemu_cmd.AddParameter("-device");
742 qemu_cmd.AddParameter(
743 "virtio-pmem-pci,disable-legacy=on,memdev=objpmem2,id=pmem1");
744 }
745 }
746
747 qemu_cmd.AddParameter("-object");
748 qemu_cmd.AddParameter("rng-random,id=objrng0,filename=/dev/urandom");
749
750 qemu_cmd.AddParameter("-device");
751 qemu_cmd.AddParameter("virtio-rng-pci-non-transitional,rng=objrng0,id=rng0,",
752 "max-bytes=1024,period=2000");
753
754 // TODO: Use the vhost-user devices instead if/when qemu is accessed via
755 // webRTC instead of VNC.
756 qemu_cmd.AddParameter("-device");
757 qemu_cmd.AddParameter("virtio-mouse-pci,disable-legacy=on");
758
759 qemu_cmd.AddParameter("-device");
760 qemu_cmd.AddParameter("virtio-keyboard-pci,disable-legacy=on");
761
762 // device padding for unsupported "switches" input
763 qemu_cmd.AddParameter("-chardev");
764 qemu_cmd.AddParameter("socket,path=", instance.switches_socket_path(), ",id=switches0");
765 qemu_cmd.AddParameter("-device");
766 qemu_cmd.AddParameter("vhost-user-input-pci,chardev=switches0");
767
768 qemu_cmd.AddParameter("-chardev");
769 qemu_cmd.AddParameter("socket,path=", instance.rotary_socket_path(), ",id=rotary0");
770 qemu_cmd.AddParameter("-device");
771 qemu_cmd.AddParameter("vhost-user-input-pci,chardev=rotary0");
772
773 auto vhost_net = instance.vhost_net() ? ",vhost=on" : "";
774
775 qemu_cmd.AddParameter("-device");
776 qemu_cmd.AddParameter("virtio-balloon-pci-non-transitional,id=balloon0");
777
778 bool has_network_devices = false;
779 switch (instance.external_network_mode()) {
780 case ExternalNetworkMode::kTap:
781 if (instance.enable_tap_devices()) {
782 has_network_devices = true;
783 qemu_cmd.AddParameter("-netdev");
784 qemu_cmd.AddParameter(
785 "tap,id=hostnet0,ifname=", instance.mobile_tap_name(),
786 ",script=no,downscript=no", vhost_net);
787
788 qemu_cmd.AddParameter("-netdev");
789 qemu_cmd.AddParameter(
790 "tap,id=hostnet1,ifname=", instance.ethernet_tap_name(),
791 ",script=no,downscript=no", vhost_net);
792
793 if (!config.virtio_mac80211_hwsim()) {
794 qemu_cmd.AddParameter("-netdev");
795 qemu_cmd.AddParameter(
796 "tap,id=hostnet2,ifname=", instance.wifi_tap_name(),
797 ",script=no,downscript=no", vhost_net);
798 }
799 }
800 break;
801 case cuttlefish::ExternalNetworkMode::kSlirp: {
802 has_network_devices = true;
803 const std::string net =
804 fmt::format("{}/{}", instance.ril_ipaddr(), instance.ril_prefixlen());
805 const std::string& host = instance.ril_gateway();
806 qemu_cmd.AddParameter("-netdev");
807 // TODO(schuffelen): `dns` needs to match the first `nameserver` in
808 // `/etc/resolv.conf`. Implement something that generalizes beyond
809 // gLinux.
810 qemu_cmd.AddParameter("user,id=hostnet0,net=", net, ",host=", host,
811 ",dns=127.0.0.1");
812
813 qemu_cmd.AddParameter("-netdev");
814 qemu_cmd.AddParameter("user,id=hostnet1,net=10.0.1.1/24,dns=8.8.4.4");
815
816 if (!config.virtio_mac80211_hwsim()) {
817 qemu_cmd.AddParameter("-netdev");
818 qemu_cmd.AddParameter("user,id=hostnet2,net=10.0.2.1/24,dns=1.1.1.1");
819 }
820 break;
821 }
822 default:
823 return CF_ERRF("Unexpected net mode {}",
824 instance.external_network_mode());
825 }
826
827 if (has_network_devices) {
828 // The ordering of virtio-net devices is important. Make sure any change
829 // here is reflected in ethprime u-boot variable
830 qemu_cmd.AddParameter("-device");
831 qemu_cmd.AddParameter(
832 "virtio-net-pci-non-transitional,netdev=hostnet0,id=net0,mac=",
833 instance.mobile_mac());
834 qemu_cmd.AddParameter("-device");
835 qemu_cmd.AddParameter(
836 "virtio-net-pci-non-transitional,netdev=hostnet1,id=net1,mac=",
837 instance.ethernet_mac());
838 if (!config.virtio_mac80211_hwsim()) {
839 qemu_cmd.AddParameter("-device");
840 qemu_cmd.AddParameter(
841 "virtio-net-pci-non-transitional,netdev=hostnet2,id=net2,mac=",
842 instance.wifi_mac());
843 }
844 }
845
846 if (is_x86 || is_arm) {
847 qemu_cmd.AddParameter("-cpu");
848 qemu_cmd.AddParameter(IsHostCompatible(arch_) ? "host" : "max");
849 }
850
851 // Explicitly enable the optional extensions of interest, in case the default
852 // behavior changes upstream.
853 if (is_riscv64) {
854 qemu_cmd.AddParameter("-cpu");
855 qemu_cmd.AddParameter("rv64",
856 ",v=true,elen=64,vlen=128",
857 ",zba=true,zbb=true,zbs=true");
858 }
859
860 qemu_cmd.AddParameter("-msg");
861 qemu_cmd.AddParameter("timestamp=on");
862
863 #ifdef __linux__
864 qemu_cmd.AddParameter("-device");
865 qemu_cmd.AddParameter("vhost-vsock-pci-non-transitional,guest-cid=",
866 instance.vsock_guest_cid());
867 #endif
868
869 qemu_cmd.AddParameter("-device");
870 qemu_cmd.AddParameter("AC97,audiodev=audio_none");
871 qemu_cmd.AddParameter("-audiodev");
872 qemu_cmd.AddParameter("driver=none,id=audio_none");
873
874 qemu_cmd.AddParameter("-device");
875 qemu_cmd.AddParameter("qemu-xhci,id=xhci");
876
877 qemu_cmd.AddParameter("-L");
878 qemu_cmd.AddParameter(HostQemuBiosPath());
879
880 if (is_riscv64) {
881 qemu_cmd.AddParameter("-kernel");
882 qemu_cmd.AddParameter(instance.bootloader());
883 } else if (is_arm) {
884 qemu_cmd.AddParameter("-bios");
885 qemu_cmd.AddParameter(instance.bootloader());
886 } else {
887 qemu_cmd.AddParameter("-drive");
888 qemu_cmd.AddParameter("if=pflash,format=raw,readonly=on,file=",
889 instance.bootloader());
890 qemu_cmd.AddParameter("-drive");
891 qemu_cmd.AddParameter("if=pflash,format=raw,file=", instance.pflash_path());
892 }
893
894 if (instance.gdb_port() > 0) {
895 qemu_cmd.AddParameter("-S");
896 qemu_cmd.AddParameter("-gdb");
897 qemu_cmd.AddParameter("tcp::", instance.gdb_port());
898 }
899
900 // After all other devices are added, add some more console sinks
901 // so it doesn't upset any sepolicy, but works around a QEMU warning
902 // when U-Boot probes the ports between kDefaultNumHvcs and
903 // kMaxSerialPorts
904 while (hvc_num < kMaxSerialPorts) {
905 add_hvc_sink();
906 }
907
908 commands.emplace_back(std::move(qemu_cmd), true);
909 return commands;
910 }
911
912 } // namespace vm_manager
913 } // namespace cuttlefish
914