1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "host/libs/vm_manager/crosvm_manager.h"
18
19 #include <sys/stat.h>
20 #include <sys/types.h>
21
22 #include <cassert>
23 #include <string>
24 #include <vector>
25
26 #include <android-base/strings.h>
27 #include <android-base/logging.h>
28 #include <vulkan/vulkan.h>
29
30 #include "common/libs/utils/environment.h"
31 #include "common/libs/utils/network.h"
32 #include "common/libs/utils/subprocess.h"
33 #include "common/libs/utils/files.h"
34 #include "host/libs/config/cuttlefish_config.h"
35 #include "host/libs/config/known_paths.h"
36 #include "host/libs/vm_manager/qemu_manager.h"
37
38 namespace cuttlefish {
39 namespace vm_manager {
40
41 namespace {
42
GetControlSocketPath(const CuttlefishConfig & config)43 std::string GetControlSocketPath(const CuttlefishConfig& config) {
44 return config.ForDefaultInstance()
45 .PerInstanceInternalPath("crosvm_control.sock");
46 }
47
AddTapFdParameter(Command * crosvm_cmd,const std::string & tap_name)48 SharedFD AddTapFdParameter(Command* crosvm_cmd,
49 const std::string& tap_name) {
50 auto tap_fd = OpenTapInterface(tap_name);
51 if (tap_fd->IsOpen()) {
52 crosvm_cmd->AddParameter("--tap-fd=", tap_fd);
53 } else {
54 LOG(ERROR) << "Unable to connect to " << tap_name << ": "
55 << tap_fd->StrError();
56 }
57 return tap_fd;
58 }
59
ReleaseDhcpLeases(const std::string & lease_path,SharedFD tap_fd)60 bool ReleaseDhcpLeases(const std::string& lease_path, SharedFD tap_fd) {
61 auto lease_file_fd = SharedFD::Open(lease_path, O_RDONLY);
62 if (!lease_file_fd->IsOpen()) {
63 LOG(ERROR) << "Could not open leases file \"" << lease_path << '"';
64 return false;
65 }
66 bool success = true;
67 auto dhcp_leases = ParseDnsmasqLeases(lease_file_fd);
68 for (auto& lease : dhcp_leases) {
69 std::uint8_t dhcp_server_ip[] = {192, 168, 96, (std::uint8_t) (ForCurrentInstance(1) * 4 - 3)};
70 if (!ReleaseDhcp4(tap_fd, lease.mac_address, lease.ip_address, dhcp_server_ip)) {
71 LOG(ERROR) << "Failed to release " << lease;
72 success = false;
73 } else {
74 LOG(INFO) << "Successfully dropped " << lease;
75 }
76 }
77 return success;
78 }
79
Stop()80 bool Stop() {
81 auto config = CuttlefishConfig::Get();
82 Command command(config->crosvm_binary());
83 command.AddParameter("stop");
84 command.AddParameter(GetControlSocketPath(*config));
85
86 auto process = command.Start();
87
88 return process.Wait() == 0;
89 }
90
91 } // namespace
92
IsSupported()93 bool CrosvmManager::IsSupported() {
94 #ifdef __ANDROID__
95 return true;
96 #else
97 return HostSupportsQemuCli();
98 #endif
99 }
100
ConfigureGpuMode(const std::string & gpu_mode)101 std::vector<std::string> CrosvmManager::ConfigureGpuMode(
102 const std::string& gpu_mode) {
103 // Override the default HAL search paths in all cases. We do this because
104 // the HAL search path allows for fallbacks, and fallbacks in conjunction
105 // with properities lead to non-deterministic behavior while loading the
106 // HALs.
107 if (gpu_mode == kGpuModeGuestSwiftshader) {
108 return {
109 "androidboot.cpuvulkan.version=" + std::to_string(VK_API_VERSION_1_2),
110 "androidboot.hardware.gralloc=minigbm",
111 "androidboot.hardware.hwcomposer=ranchu",
112 "androidboot.hardware.egl=angle",
113 "androidboot.hardware.vulkan=pastel",
114 };
115 }
116
117 if (gpu_mode == kGpuModeDrmVirgl) {
118 return {
119 "androidboot.cpuvulkan.version=0",
120 "androidboot.hardware.gralloc=minigbm",
121 "androidboot.hardware.hwcomposer=drm_minigbm",
122 "androidboot.hardware.egl=mesa",
123 };
124 }
125 if (gpu_mode == kGpuModeGfxStream) {
126 return {
127 "androidboot.cpuvulkan.version=0",
128 "androidboot.hardware.gralloc=minigbm",
129 "androidboot.hardware.hwcomposer=ranchu",
130 "androidboot.hardware.egl=emulation",
131 "androidboot.hardware.vulkan=ranchu",
132 "androidboot.hardware.gltransport=virtio-gpu-asg",
133 };
134 }
135 return {};
136 }
137
ConfigureBootDevices(int num_disks)138 std::string CrosvmManager::ConfigureBootDevices(int num_disks) {
139 // TODO There is no way to control this assignment with crosvm (yet)
140 if (HostArch() == Arch::X86_64) {
141 // crosvm has an additional PCI device for an ISA bridge
142 return ConfigureMultipleBootDevices("pci0000:00/0000:00:", 1, num_disks);
143 } else {
144 // On ARM64 crosvm, block devices are on their own bridge, so we don't
145 // need to calculate it, and the path is always the same
146 return "androidboot.boot_devices=10000.pci";
147 }
148 }
149
StartCommands(const CuttlefishConfig & config)150 std::vector<Command> CrosvmManager::StartCommands(
151 const CuttlefishConfig& config) {
152 auto instance = config.ForDefaultInstance();
153 Command crosvm_cmd(config.crosvm_binary(), [](Subprocess* proc) {
154 auto stopped = Stop();
155 if (stopped) {
156 return true;
157 }
158 LOG(WARNING) << "Failed to stop VMM nicely, attempting to KILL";
159 return KillSubprocess(proc);
160 });
161
162 int hvc_num = 0;
163 int serial_num = 0;
164 auto add_hvc_sink = [&crosvm_cmd, &hvc_num]() {
165 crosvm_cmd.AddParameter("--serial=hardware=virtio-console,num=", ++hvc_num,
166 ",type=sink");
167 };
168 auto add_serial_sink = [&crosvm_cmd, &serial_num]() {
169 crosvm_cmd.AddParameter("--serial=hardware=serial,num=", ++serial_num,
170 ",type=sink");
171 };
172 auto add_hvc_console = [&crosvm_cmd, &hvc_num](const std::string& output) {
173 crosvm_cmd.AddParameter("--serial=hardware=virtio-console,num=", ++hvc_num,
174 ",type=file,path=", output, ",console=true");
175 };
176 auto add_serial_console_ro = [&crosvm_cmd,
177 &serial_num](const std::string& output) {
178 crosvm_cmd.AddParameter("--serial=hardware=serial,num=", ++serial_num,
179 ",type=file,path=", output, ",earlycon=true");
180 };
181 auto add_serial_console = [&crosvm_cmd, &serial_num](
182 const std::string& output,
183 const std::string& input) {
184 crosvm_cmd.AddParameter("--serial=hardware=serial,num=", ++serial_num,
185 ",type=file,path=", output, ",input=", input,
186 ",earlycon=true");
187 };
188 auto add_hvc_ro = [&crosvm_cmd, &hvc_num](const std::string& output) {
189 crosvm_cmd.AddParameter("--serial=hardware=virtio-console,num=", ++hvc_num,
190 ",type=file,path=", output);
191 };
192 auto add_hvc = [&crosvm_cmd, &hvc_num](const std::string& output,
193 const std::string& input) {
194 crosvm_cmd.AddParameter("--serial=hardware=virtio-console,num=", ++hvc_num,
195 ",type=file,path=", output, ",input=", input);
196 };
197 // Deprecated; do not add any more users
198 auto add_serial = [&crosvm_cmd, &serial_num](const std::string& output,
199 const std::string& input) {
200 crosvm_cmd.AddParameter("--serial=hardware=serial,num=", ++serial_num,
201 ",type=file,path=", output, ",input=", input);
202 };
203
204 crosvm_cmd.AddParameter("run");
205
206 if (!config.smt()) {
207 crosvm_cmd.AddParameter("--no-smt");
208 }
209
210 if (config.vhost_net()) {
211 crosvm_cmd.AddParameter("--vhost-net");
212 }
213
214 if (config.protected_vm()) {
215 crosvm_cmd.AddParameter("--protected-vm");
216 }
217
218 if (config.gdb_port() > 0) {
219 CHECK(config.cpus() == 1) << "CPUs must be 1 for crosvm gdb mode";
220 crosvm_cmd.AddParameter("--gdb=", config.gdb_port());
221 }
222
223 auto display_configs = config.display_configs();
224 CHECK_GE(display_configs.size(), 1);
225 auto display_config = display_configs[0];
226
227 auto gpu_mode = config.gpu_mode();
228
229 if (gpu_mode == kGpuModeGuestSwiftshader) {
230 crosvm_cmd.AddParameter("--gpu=2D,",
231 "width=", display_config.width, ",",
232 "height=", display_config.height);
233 } else if (gpu_mode == kGpuModeDrmVirgl || gpu_mode == kGpuModeGfxStream) {
234 crosvm_cmd.AddParameter(gpu_mode == kGpuModeGfxStream ?
235 "--gpu=gfxstream," : "--gpu=",
236 "width=", display_config.width, ",",
237 "height=", display_config.height, ",",
238 "egl=true,surfaceless=true,glx=false,gles=true");
239 }
240 crosvm_cmd.AddParameter("--wayland-sock=", instance.frames_socket_path());
241
242 // crosvm_cmd.AddParameter("--null-audio");
243 crosvm_cmd.AddParameter("--mem=", config.memory_mb());
244 crosvm_cmd.AddParameter("--cpus=", config.cpus());
245
246 auto disk_num = instance.virtual_disk_paths().size();
247 CHECK_GE(VmManager::kMaxDisks, disk_num)
248 << "Provided too many disks (" << disk_num << "), maximum "
249 << VmManager::kMaxDisks << "supported";
250 for (const auto& disk : instance.virtual_disk_paths()) {
251 crosvm_cmd.AddParameter(config.protected_vm() ? "--disk=" :
252 "--rwdisk=", disk);
253 }
254 crosvm_cmd.AddParameter("--socket=", GetControlSocketPath(config));
255
256 if (config.enable_vnc_server() || config.enable_webrtc()) {
257 auto touch_type_parameter =
258 config.enable_webrtc() ? "--multi-touch=" : "--single-touch=";
259 crosvm_cmd.AddParameter(touch_type_parameter, instance.touch_socket_path(),
260 ":", display_config.width, ":",
261 display_config.height);
262 crosvm_cmd.AddParameter("--keyboard=", instance.keyboard_socket_path());
263 }
264 if (config.enable_webrtc()) {
265 crosvm_cmd.AddParameter("--switches=", instance.switches_socket_path());
266 }
267
268 auto wifi_tap = AddTapFdParameter(&crosvm_cmd, instance.wifi_tap_name());
269 AddTapFdParameter(&crosvm_cmd, instance.mobile_tap_name());
270
271 if (FileExists(instance.access_kregistry_path())) {
272 crosvm_cmd.AddParameter("--rw-pmem-device=",
273 instance.access_kregistry_path());
274 }
275
276 if (FileExists(instance.pstore_path())) {
277 crosvm_cmd.AddParameter("--pstore=path=", instance.pstore_path(),
278 ",size=", FileSize(instance.pstore_path()));
279 }
280
281 if (config.enable_sandbox()) {
282 const bool seccomp_exists = DirectoryExists(config.seccomp_policy_dir());
283 const std::string& var_empty_dir = kCrosvmVarEmptyDir;
284 const bool var_empty_available = DirectoryExists(var_empty_dir);
285 if (!var_empty_available || !seccomp_exists) {
286 LOG(FATAL) << var_empty_dir << " is not an existing, empty directory."
287 << "seccomp-policy-dir, " << config.seccomp_policy_dir()
288 << " does not exist " << std::endl;
289 return {};
290 }
291 crosvm_cmd.AddParameter("--seccomp-policy-dir=", config.seccomp_policy_dir());
292 } else {
293 crosvm_cmd.AddParameter("--disable-sandbox");
294 }
295
296 if (instance.vsock_guest_cid() >= 2) {
297 crosvm_cmd.AddParameter("--cid=", instance.vsock_guest_cid());
298 }
299
300 // Use a virtio-console instance for the main kernel console. All
301 // messages will switch from earlycon to virtio-console after the driver
302 // is loaded, and crosvm will append to the kernel log automatically
303 add_hvc_console(instance.kernel_log_pipe_name());
304
305 if (config.console()) {
306 // stdin is the only currently supported way to write data to a serial port in
307 // crosvm. A file (named pipe) is used here instead of stdout to ensure only
308 // the serial port output is received by the console forwarder as crosvm may
309 // print other messages to stdout.
310 if (config.kgdb() || config.use_bootloader()) {
311 add_serial_console(instance.console_out_pipe_name(),
312 instance.console_in_pipe_name());
313 // In kgdb mode, we have the interactive console on ttyS0 (both Android's
314 // console and kdb), so we can disable the virtio-console port usually
315 // allocated to Android's serial console, and redirect it to a sink. This
316 // ensures that that the PCI device assignments (and thus sepolicy) don't
317 // have to change
318 add_hvc_sink();
319 } else {
320 add_serial_sink();
321 add_hvc(instance.console_out_pipe_name(),
322 instance.console_in_pipe_name());
323 }
324 } else {
325 // Use an 8250 UART (ISA or platform device) for earlycon, as the
326 // virtio-console driver may not be available for early messages
327 // In kgdb mode, earlycon is an interactive console, and so early
328 // dmesg will go there instead of the kernel.log
329 if (config.kgdb() || config.use_bootloader()) {
330 add_serial_console_ro(instance.kernel_log_pipe_name());
331 }
332
333 // as above, create a fake virtio-console 'sink' port when the serial
334 // console is disabled, so the PCI device ID assignments don't move
335 // around
336 add_hvc_sink();
337 }
338
339 if (config.enable_gnss_grpc_proxy()) {
340 add_serial(instance.gnss_out_pipe_name(), instance.gnss_in_pipe_name());
341 }
342
343 SharedFD log_out_rd, log_out_wr;
344 if (!SharedFD::Pipe(&log_out_rd, &log_out_wr)) {
345 LOG(ERROR) << "Failed to create log pipe for crosvm's stdout/stderr: "
346 << log_out_rd->StrError();
347 return {};
348 }
349 crosvm_cmd.RedirectStdIO(Subprocess::StdIOChannel::kStdOut, log_out_wr);
350 crosvm_cmd.RedirectStdIO(Subprocess::StdIOChannel::kStdErr, log_out_wr);
351
352 Command log_tee_cmd(HostBinaryPath("log_tee"));
353 log_tee_cmd.AddParameter("--process_name=crosvm");
354 log_tee_cmd.AddParameter("--log_fd_in=", log_out_rd);
355
356 // Serial port for logcat, redirected to a pipe
357 add_hvc_ro(instance.logcat_pipe_name());
358
359 add_hvc(instance.PerInstanceInternalPath("keymaster_fifo_vm.out"),
360 instance.PerInstanceInternalPath("keymaster_fifo_vm.in"));
361 add_hvc(instance.PerInstanceInternalPath("gatekeeper_fifo_vm.out"),
362 instance.PerInstanceInternalPath("gatekeeper_fifo_vm.in"));
363
364 if (config.enable_host_bluetooth()) {
365 add_hvc(instance.PerInstanceInternalPath("bt_fifo_vm.out"),
366 instance.PerInstanceInternalPath("bt_fifo_vm.in"));
367 } else {
368 add_hvc_sink();
369 }
370 for (auto i = 0; i < VmManager::kMaxDisks - disk_num; i++) {
371 add_hvc_sink();
372 }
373 CHECK(hvc_num + disk_num == VmManager::kMaxDisks + VmManager::kDefaultNumHvcs)
374 << "HVC count (" << hvc_num << ") + disk count (" << disk_num << ") "
375 << "is not the expected total of "
376 << VmManager::kMaxDisks + VmManager::kDefaultNumHvcs << " devices";
377
378 if (config.enable_audio()) {
379 crosvm_cmd.AddParameter("--ac97=backend=vios,server=" +
380 config.ForDefaultInstance().audio_server_path());
381 }
382
383 // TODO(b/172286896): This is temporarily optional, but should be made
384 // unconditional and moved up to the other network devices area
385 if (config.ethernet()) {
386 AddTapFdParameter(&crosvm_cmd, instance.ethernet_tap_name());
387 }
388
389 // TODO(b/162071003): virtiofs crashes without sandboxing, this should be fixed
390 if (config.enable_sandbox()) {
391 // Set up directory shared with virtiofs
392 crosvm_cmd.AddParameter("--shared-dir=", instance.PerInstancePath(kSharedDirName),
393 ":shared:type=fs");
394 }
395
396 // This needs to be the last parameter
397 crosvm_cmd.AddParameter("--bios=", config.bootloader());
398
399 // Only run the leases workaround if we are not using the new network
400 // bridge architecture - in that case, we have a wider DHCP address
401 // space and stale leases should be much less of an issue
402 if (!FileExists("/var/run/cuttlefish-dnsmasq-cvd-wbr.leases")) {
403 // TODO(schuffelen): QEMU also needs this and this is not the best place for
404 // this code. Find a better place to put it.
405 auto lease_file =
406 ForCurrentInstance("/var/run/cuttlefish-dnsmasq-cvd-wbr-") + ".leases";
407 if (!ReleaseDhcpLeases(lease_file, wifi_tap)) {
408 LOG(ERROR) << "Failed to release wifi DHCP leases. Connecting to the wifi "
409 << "network may not work.";
410 }
411 }
412
413 std::vector<Command> ret;
414 ret.push_back(std::move(crosvm_cmd));
415 ret.push_back(std::move(log_tee_cmd));
416 return ret;
417 }
418
419 } // namespace vm_manager
420 } // namespace cuttlefish
421
422