1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "host/libs/vm_manager/qemu_manager.h"
18
19 #include <string.h>
20 #include <sys/socket.h>
21 #include <sys/stat.h>
22 #include <sys/types.h>
23 #include <sys/un.h>
24 #include <sys/wait.h>
25 #include <unistd.h>
26
27 #include <cstdlib>
28 #include <sstream>
29 #include <string>
30 #include <thread>
31 #include <vector>
32
33 #include <android-base/strings.h>
34 #include <android-base/logging.h>
35 #include <vulkan/vulkan.h>
36
37 #include "common/libs/fs/shared_select.h"
38 #include "common/libs/utils/files.h"
39 #include "common/libs/utils/subprocess.h"
40 #include "common/libs/utils/users.h"
41 #include "host/libs/config/cuttlefish_config.h"
42 #include "host/libs/config/known_paths.h"
43
44 namespace cuttlefish {
45 namespace vm_manager {
46 namespace {
47
GetMonitorPath(const CuttlefishConfig & config)48 std::string GetMonitorPath(const CuttlefishConfig& config) {
49 return config.ForDefaultInstance()
50 .PerInstanceInternalPath("qemu_monitor.sock");
51 }
52
LogAndSetEnv(const char * key,const std::string & value)53 void LogAndSetEnv(const char* key, const std::string& value) {
54 setenv(key, value.c_str(), 1);
55 LOG(INFO) << key << "=" << value;
56 }
57
Stop()58 bool Stop() {
59 auto config = CuttlefishConfig::Get();
60 auto monitor_path = GetMonitorPath(*config);
61 auto monitor_sock = SharedFD::SocketLocalClient(
62 monitor_path.c_str(), false, SOCK_STREAM);
63
64 if (!monitor_sock->IsOpen()) {
65 LOG(ERROR) << "The connection to qemu is closed, is it still running?";
66 return false;
67 }
68 char msg[] = "{\"execute\":\"qmp_capabilities\"}{\"execute\":\"quit\"}";
69 ssize_t len = sizeof(msg) - 1;
70 while (len > 0) {
71 int tmp = monitor_sock->Write(msg, len);
72 if (tmp < 0) {
73 LOG(ERROR) << "Error writing to socket: " << monitor_sock->StrError();
74 return false;
75 }
76 len -= tmp;
77 }
78 // Log the reply
79 char buff[1000];
80 while ((len = monitor_sock->Read(buff, sizeof(buff) - 1)) > 0) {
81 buff[len] = '\0';
82 LOG(INFO) << "From qemu monitor: " << buff;
83 }
84
85 return true;
86 }
87
88 } // namespace
89
IsSupported()90 bool QemuManager::IsSupported() {
91 return HostSupportsQemuCli();
92 }
93
ConfigureGpuMode(const std::string & gpu_mode)94 std::vector<std::string> QemuManager::ConfigureGpuMode(
95 const std::string& gpu_mode) {
96 if (gpu_mode == kGpuModeGuestSwiftshader) {
97 // Override the default HAL search paths in all cases. We do this because
98 // the HAL search path allows for fallbacks, and fallbacks in conjunction
99 // with properities lead to non-deterministic behavior while loading the
100 // HALs.
101 return {
102 "androidboot.cpuvulkan.version=" + std::to_string(VK_API_VERSION_1_1),
103 "androidboot.hardware.gralloc=minigbm",
104 "androidboot.hardware.hwcomposer=ranchu",
105 "androidboot.hardware.egl=swiftshader",
106 "androidboot.hardware.vulkan=pastel",
107 };
108 }
109
110 if (gpu_mode == kGpuModeDrmVirgl) {
111 return {
112 "androidboot.cpuvulkan.version=0",
113 "androidboot.hardware.gralloc=minigbm",
114 "androidboot.hardware.hwcomposer=drm_minigbm",
115 "androidboot.hardware.egl=mesa",
116 };
117 }
118
119 return {};
120 }
121
ConfigureBootDevices(int num_disks)122 std::string QemuManager::ConfigureBootDevices(int num_disks) {
123 switch (arch_) {
124 case Arch::X86:
125 case Arch::X86_64: {
126 // QEMU has additional PCI devices for an ISA bridge and PIIX4
127 return ConfigureMultipleBootDevices("pci0000:00/0000:00:", 2, num_disks);
128 }
129 case Arch::Arm:
130 return "androidboot.boot_devices=3f000000.pcie";
131 case Arch::Arm64:
132 return "androidboot.boot_devices=4010000000.pcie";
133 }
134 }
135
StartCommands(const CuttlefishConfig & config)136 std::vector<Command> QemuManager::StartCommands(
137 const CuttlefishConfig& config) {
138 auto instance = config.ForDefaultInstance();
139
140 auto stop = [](Subprocess* proc) {
141 auto stopped = Stop();
142 if (stopped) {
143 return true;
144 }
145 LOG(WARNING) << "Failed to stop VMM nicely, "
146 << "attempting to KILL";
147 return KillSubprocess(proc);
148 };
149 std::string qemu_binary = config.qemu_binary_dir();
150 switch (arch_) {
151 case Arch::Arm:
152 qemu_binary += "/qemu-system-arm";
153 break;
154 case Arch::Arm64:
155 qemu_binary += "/qemu-system-aarch64";
156 break;
157 case Arch::X86:
158 qemu_binary += "/qemu-system-i386";
159 break;
160 case Arch::X86_64:
161 qemu_binary += "/qemu-system-x86_64";
162 break;
163 }
164 Command qemu_cmd(qemu_binary, stop);
165
166 int hvc_num = 0;
167 int serial_num = 0;
168 auto add_hvc_sink = [&qemu_cmd, &hvc_num]() {
169 qemu_cmd.AddParameter("-chardev");
170 qemu_cmd.AddParameter("null,id=hvc", hvc_num);
171 qemu_cmd.AddParameter("-device");
172 qemu_cmd.AddParameter(
173 "virtio-serial-pci-non-transitional,max_ports=1,id=virtio-serial",
174 hvc_num);
175 qemu_cmd.AddParameter("-device");
176 qemu_cmd.AddParameter("virtconsole,bus=virtio-serial", hvc_num,
177 ".0,chardev=hvc", hvc_num);
178 hvc_num++;
179 };
180 auto add_serial_sink = [&qemu_cmd, &serial_num]() {
181 qemu_cmd.AddParameter("-chardev");
182 qemu_cmd.AddParameter("null,id=serial", serial_num);
183 qemu_cmd.AddParameter("-serial");
184 qemu_cmd.AddParameter("chardev:serial", serial_num);
185 serial_num++;
186 };
187 auto add_serial_console_ro = [&qemu_cmd,
188 &serial_num](const std::string& output) {
189 qemu_cmd.AddParameter("-chardev");
190 qemu_cmd.AddParameter("file,id=serial", serial_num, ",path=", output,
191 ",append=on");
192 qemu_cmd.AddParameter("-serial");
193 qemu_cmd.AddParameter("chardev:serial", serial_num);
194 serial_num++;
195 };
196 auto add_serial_console = [&qemu_cmd,
197 &serial_num](const std::string& prefix) {
198 qemu_cmd.AddParameter("-chardev");
199 qemu_cmd.AddParameter("pipe,id=serial", serial_num, ",path=", prefix);
200 qemu_cmd.AddParameter("-serial");
201 qemu_cmd.AddParameter("chardev:serial", serial_num);
202 serial_num++;
203 };
204 auto add_hvc_ro = [&qemu_cmd, &hvc_num](const std::string& output) {
205 qemu_cmd.AddParameter("-chardev");
206 qemu_cmd.AddParameter("file,id=hvc", hvc_num, ",path=", output,
207 ",append=on");
208 qemu_cmd.AddParameter("-device");
209 qemu_cmd.AddParameter(
210 "virtio-serial-pci-non-transitional,max_ports=1,id=virtio-serial",
211 hvc_num);
212 qemu_cmd.AddParameter("-device");
213 qemu_cmd.AddParameter("virtconsole,bus=virtio-serial", hvc_num,
214 ".0,chardev=hvc", hvc_num);
215 hvc_num++;
216 };
217 auto add_hvc = [&qemu_cmd, &hvc_num](const std::string& prefix) {
218 qemu_cmd.AddParameter("-chardev");
219 qemu_cmd.AddParameter("pipe,id=hvc", hvc_num, ",path=", prefix);
220 qemu_cmd.AddParameter("-device");
221 qemu_cmd.AddParameter(
222 "virtio-serial-pci-non-transitional,max_ports=1,id=virtio-serial",
223 hvc_num);
224 qemu_cmd.AddParameter("-device");
225 qemu_cmd.AddParameter("virtconsole,bus=virtio-serial", hvc_num,
226 ".0,chardev=hvc", hvc_num);
227 hvc_num++;
228 };
229
230 bool is_arm = arch_ == Arch::Arm || arch_ == Arch::Arm64;
231
232 auto access_kregistry_size_bytes = 0;
233 if (FileExists(instance.access_kregistry_path())) {
234 access_kregistry_size_bytes = FileSize(instance.access_kregistry_path());
235 CHECK((access_kregistry_size_bytes & (1024 * 1024 - 1)) == 0)
236 << instance.access_kregistry_path() << " file size ("
237 << access_kregistry_size_bytes << ") not a multiple of 1MB";
238 }
239
240 auto pstore_size_bytes = 0;
241 if (FileExists(instance.pstore_path())) {
242 pstore_size_bytes = FileSize(instance.pstore_path());
243 CHECK((pstore_size_bytes & (1024 * 1024 - 1)) == 0)
244 << instance.pstore_path() << " file size ("
245 << pstore_size_bytes << ") not a multiple of 1MB";
246 }
247
248 qemu_cmd.AddParameter("-name");
249 qemu_cmd.AddParameter("guest=", instance.instance_name(), ",debug-threads=on");
250
251 qemu_cmd.AddParameter("-machine");
252 auto machine = is_arm ? "virt,gic-version=2,mte=on"
253 : "pc-i440fx-2.8,accel=kvm,nvdimm=on";
254 qemu_cmd.AddParameter(machine, ",usb=off,dump-guest-core=off");
255
256 qemu_cmd.AddParameter("-m");
257 auto maxmem = config.memory_mb() +
258 access_kregistry_size_bytes / 1024 / 1024 +
259 (is_arm ? 0 : pstore_size_bytes / 1024 / 1024);
260 auto slots = is_arm ? "" : ",slots=2";
261 qemu_cmd.AddParameter("size=", config.memory_mb(), "M",
262 ",maxmem=", maxmem, "M", slots);
263
264 qemu_cmd.AddParameter("-overcommit");
265 qemu_cmd.AddParameter("mem-lock=off");
266
267 // Assume SMT is always 2 threads per core, which is how most hardware
268 // today is configured, and the way crosvm does it
269 qemu_cmd.AddParameter("-smp");
270 if (config.smt()) {
271 CHECK(config.cpus() % 2 == 0)
272 << "CPUs must be a multiple of 2 in SMT mode";
273 qemu_cmd.AddParameter(config.cpus(), ",cores=",
274 config.cpus() / 2, ",threads=2");
275 } else {
276 qemu_cmd.AddParameter(config.cpus(), ",cores=",
277 config.cpus(), ",threads=1");
278 }
279
280 qemu_cmd.AddParameter("-uuid");
281 qemu_cmd.AddParameter(instance.uuid());
282
283 qemu_cmd.AddParameter("-no-user-config");
284 qemu_cmd.AddParameter("-nodefaults");
285 qemu_cmd.AddParameter("-no-shutdown");
286
287 qemu_cmd.AddParameter("-rtc");
288 qemu_cmd.AddParameter("base=utc");
289
290 qemu_cmd.AddParameter("-boot");
291 qemu_cmd.AddParameter("strict=on");
292
293 qemu_cmd.AddParameter("-chardev");
294 qemu_cmd.AddParameter("socket,id=charmonitor,path=", GetMonitorPath(config),
295 ",server,nowait");
296
297 qemu_cmd.AddParameter("-mon");
298 qemu_cmd.AddParameter("chardev=charmonitor,id=monitor,mode=control");
299
300 // In kgdb mode, earlycon is an interactive console, and so early
301 // dmesg will go there instead of the kernel.log. On QEMU, we do this
302 // bit of logic up before the hvc console is set up, so the command line
303 // flags appear in the right order and "append=on" does the right thing
304 if (!(config.console() && (config.kgdb() || config.use_bootloader()))) {
305 add_serial_console_ro(instance.kernel_log_pipe_name());
306 }
307
308 // Use a virtio-console instance for the main kernel console. All
309 // messages will switch from earlycon to virtio-console after the driver
310 // is loaded, and QEMU will append to the kernel log automatically
311 add_hvc_ro(instance.kernel_log_pipe_name());
312
313 if (config.console()) {
314 if (config.kgdb() || config.use_bootloader()) {
315 add_serial_console(instance.console_pipe_prefix());
316
317 // In kgdb mode, we have the interactive console on ttyS0 (both Android's
318 // console and kdb), so we can disable the virtio-console port usually
319 // allocated to Android's serial console, and redirect it to a sink. This
320 // ensures that that the PCI device assignments (and thus sepolicy) don't
321 // have to change
322 add_hvc_sink();
323 } else {
324 add_serial_sink();
325 add_hvc(instance.console_pipe_prefix());
326 }
327 } else {
328 if (config.kgdb() || config.use_bootloader()) {
329 // The add_serial_console_ro() call above was applied by the time we reach
330 // this code, so we don't need another add_serial_*() call
331 }
332
333 // as above, create a fake virtio-console 'sink' port when the serial
334 // console is disabled, so the PCI device ID assignments don't move
335 // around
336 add_hvc_sink();
337 }
338
339 if (config.enable_gnss_grpc_proxy()) {
340 add_serial_console(instance.gnss_pipe_prefix());
341 }
342
343 // Serial port for logcat, redirected to a pipe
344 add_hvc_ro(instance.logcat_pipe_name());
345
346 add_hvc(instance.PerInstanceInternalPath("keymaster_fifo_vm"));
347 add_hvc(instance.PerInstanceInternalPath("gatekeeper_fifo_vm"));
348 if (config.enable_host_bluetooth()) {
349 add_hvc(instance.PerInstanceInternalPath("bt_fifo_vm"));
350 } else {
351 add_hvc_sink();
352 }
353
354 auto disk_num = instance.virtual_disk_paths().size();
355
356 for (auto i = 0; i < VmManager::kMaxDisks - disk_num; i++) {
357 add_hvc_sink();
358 }
359
360 CHECK(hvc_num + disk_num == VmManager::kMaxDisks + VmManager::kDefaultNumHvcs)
361 << "HVC count (" << hvc_num << ") + disk count (" << disk_num << ") "
362 << "is not the expected total of "
363 << VmManager::kMaxDisks + VmManager::kDefaultNumHvcs << " devices";
364
365 CHECK_GE(VmManager::kMaxDisks, disk_num)
366 << "Provided too many disks (" << disk_num << "), maximum "
367 << VmManager::kMaxDisks << "supported";
368 auto readonly = config.protected_vm() ? ",readonly" : "";
369 for (size_t i = 0; i < disk_num; i++) {
370 auto bootindex = i == 0 ? ",bootindex=1" : "";
371 auto format = i == 0 ? "" : ",format=raw";
372 auto disk = instance.virtual_disk_paths()[i];
373 qemu_cmd.AddParameter("-drive");
374 qemu_cmd.AddParameter("file=", disk, ",if=none,id=drive-virtio-disk", i,
375 ",aio=threads", format, readonly);
376 qemu_cmd.AddParameter("-device");
377 qemu_cmd.AddParameter("virtio-blk-pci-non-transitional,scsi=off,drive=drive-virtio-disk", i,
378 ",id=virtio-disk", i, bootindex);
379 }
380
381 if (config.gpu_mode() == kGpuModeDrmVirgl) {
382 qemu_cmd.AddParameter("-display");
383 qemu_cmd.AddParameter("egl-headless");
384
385 qemu_cmd.AddParameter("-vnc");
386 qemu_cmd.AddParameter(":", instance.vnc_server_port() - 5900);
387 } else {
388 qemu_cmd.AddParameter("-display");
389 qemu_cmd.AddParameter("none");
390 }
391
392 if (!is_arm && FileExists(instance.pstore_path())) {
393 // QEMU will assign the NVDIMM (ramoops pstore region) 100000000-1001fffff
394 // As we will pass this to ramoops, define this region first so it is always
395 // located at this address. This is currently x86 only.
396 qemu_cmd.AddParameter("-object");
397 qemu_cmd.AddParameter("memory-backend-file,id=objpmem0,share,mem-path=",
398 instance.pstore_path(), ",size=", pstore_size_bytes);
399
400 qemu_cmd.AddParameter("-device");
401 qemu_cmd.AddParameter("nvdimm,memdev=objpmem0,id=ramoops");
402 }
403
404 // QEMU does not implement virtio-pmem-pci for ARM64 yet; restore this
405 // when the device has been added
406 if (!is_arm && FileExists(instance.access_kregistry_path())) {
407 qemu_cmd.AddParameter("-object");
408 qemu_cmd.AddParameter("memory-backend-file,id=objpmem1,share,mem-path=",
409 instance.access_kregistry_path(), ",size=",
410 access_kregistry_size_bytes);
411
412 qemu_cmd.AddParameter("-device");
413 qemu_cmd.AddParameter("virtio-pmem-pci,disable-legacy=on,memdev=objpmem1,id=pmem0");
414 }
415
416 qemu_cmd.AddParameter("-object");
417 qemu_cmd.AddParameter("rng-random,id=objrng0,filename=/dev/urandom");
418
419 qemu_cmd.AddParameter("-device");
420 qemu_cmd.AddParameter("virtio-rng-pci-non-transitional,rng=objrng0,id=rng0,",
421 "max-bytes=1024,period=2000");
422
423 qemu_cmd.AddParameter("-device");
424 qemu_cmd.AddParameter("virtio-mouse-pci");
425
426 qemu_cmd.AddParameter("-device");
427 qemu_cmd.AddParameter("virtio-keyboard-pci");
428
429 auto vhost_net = config.vhost_net() ? ",vhost=on" : "";
430
431 qemu_cmd.AddParameter("-device");
432 qemu_cmd.AddParameter("virtio-balloon-pci-non-transitional,id=balloon0");
433
434 qemu_cmd.AddParameter("-netdev");
435 qemu_cmd.AddParameter("tap,id=hostnet0,ifname=", instance.wifi_tap_name(),
436 ",script=no,downscript=no", vhost_net);
437
438 qemu_cmd.AddParameter("-device");
439 qemu_cmd.AddParameter("virtio-net-pci-non-transitional,netdev=hostnet0,id=net0");
440
441 qemu_cmd.AddParameter("-netdev");
442 qemu_cmd.AddParameter("tap,id=hostnet1,ifname=", instance.mobile_tap_name(),
443 ",script=no,downscript=no", vhost_net);
444
445 qemu_cmd.AddParameter("-device");
446 qemu_cmd.AddParameter("virtio-net-pci-non-transitional,netdev=hostnet1,id=net1");
447
448 qemu_cmd.AddParameter("-device");
449 qemu_cmd.AddParameter("virtio-gpu-pci,id=gpu0");
450
451 qemu_cmd.AddParameter("-cpu");
452 qemu_cmd.AddParameter(IsHostCompatible(arch_) ? "host" : "max");
453
454 qemu_cmd.AddParameter("-msg");
455 qemu_cmd.AddParameter("timestamp=on");
456
457 qemu_cmd.AddParameter("-device");
458 qemu_cmd.AddParameter("vhost-vsock-pci-non-transitional,guest-cid=",
459 instance.vsock_guest_cid());
460
461 qemu_cmd.AddParameter("-device");
462 qemu_cmd.AddParameter("AC97");
463
464 // TODO(b/172286896): This is temporarily optional, but should be made
465 // unconditional and moved up to the other network devices area
466 if (config.ethernet()) {
467 qemu_cmd.AddParameter("-netdev");
468 qemu_cmd.AddParameter("tap,id=hostnet2,ifname=", instance.ethernet_tap_name(),
469 ",script=no,downscript=no", vhost_net);
470
471 qemu_cmd.AddParameter("-device");
472 qemu_cmd.AddParameter("virtio-net-pci-non-transitional,netdev=hostnet2,id=net2");
473 }
474
475 qemu_cmd.AddParameter("-device");
476 qemu_cmd.AddParameter("qemu-xhci,id=xhci");
477
478 qemu_cmd.AddParameter("-bios");
479 qemu_cmd.AddParameter(config.bootloader());
480
481 if (config.gdb_port() > 0) {
482 qemu_cmd.AddParameter("-S");
483 qemu_cmd.AddParameter("-gdb");
484 qemu_cmd.AddParameter("tcp::", config.gdb_port());
485 }
486
487 LogAndSetEnv("QEMU_AUDIO_DRV", "none");
488
489 std::vector<Command> ret;
490 ret.push_back(std::move(qemu_cmd));
491 return ret;
492 }
493
494 } // namespace vm_manager
495 } // namespace cuttlefish
496
497