1 /*
2 * Copyright (C) 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <memory>
18
19 #include <android-base/logging.h>
20 #include <android-base/parseint.h>
21 #include <android-base/strings.h>
22 #include <fruit/fruit.h>
23 #include <gflags/gflags.h>
24 #include <libyuv.h>
25
26 #include "common/libs/fs/shared_fd.h"
27 #include "common/libs/utils/files.h"
28 #include "google/rpc/code.pb.h"
29 #include "host/frontend/webrtc/audio_handler.h"
30 #include "host/frontend/webrtc/client_server.h"
31 #include "host/frontend/webrtc/connection_observer.h"
32 #include "host/frontend/webrtc/display_handler.h"
33 #include "host/frontend/webrtc/kernel_log_events_handler.h"
34 #include "host/frontend/webrtc/libdevice/camera_controller.h"
35 #include "host/frontend/webrtc/libdevice/lights_observer.h"
36 #include "host/frontend/webrtc/libdevice/local_recorder.h"
37 #include "host/frontend/webrtc/libdevice/streamer.h"
38 #include "host/frontend/webrtc/libdevice/video_sink.h"
39 #include "host/frontend/webrtc/screenshot_handler.h"
40 #include "host/frontend/webrtc/webrtc_command_channel.h"
41 #include "host/libs/audio_connector/server.h"
42 #include "host/libs/config/cuttlefish_config.h"
43 #include "host/libs/config/logging.h"
44 #include "host/libs/config/openwrt_args.h"
45 #include "host/libs/confui/host_mode_ctrl.h"
46 #include "host/libs/confui/host_server.h"
47 #include "host/libs/input_connector/input_connector.h"
48 #include "host/libs/screen_connector/composition_manager.h"
49 #include "host/libs/screen_connector/screen_connector.h"
50 #include "webrtc_commands.pb.h"
51
52 DEFINE_bool(multitouch, true,
53 "Whether to send multi-touch or single-touch events");
54 DEFINE_string(touch_fds, "",
55 "A list of fds to listen on for touch connections.");
56 DEFINE_int32(mouse_fd, -1, "An fd to listen on for mouse connections.");
57 DEFINE_int32(rotary_fd, -1, "An fd to listen on for rotary connections.");
58 DEFINE_int32(keyboard_fd, -1, "An fd to listen on for keyboard connections.");
59 DEFINE_int32(switches_fd, -1, "An fd to listen on for switch connections.");
60 DEFINE_int32(frame_server_fd, -1, "An fd to listen on for frame updates");
61 DEFINE_int32(kernel_log_events_fd, -1,
62 "An fd to listen on for kernel log events.");
63 DEFINE_int32(command_fd, -1, "An fd to listen to for control messages");
64 DEFINE_int32(confui_in_fd, -1,
65 "Confirmation UI virtio-console from host to guest");
66 DEFINE_int32(confui_out_fd, -1,
67 "Confirmation UI virtio-console from guest to host");
68 DEFINE_string(action_servers, "",
69 "A comma-separated list of server_name:fd pairs, "
70 "where each entry corresponds to one custom action server.");
71 DEFINE_int32(audio_server_fd, -1, "An fd to listen on for audio frames");
72 DEFINE_int32(camera_streamer_fd, -1, "An fd to send client camera frames");
73 DEFINE_int32(sensors_fd, -1, "An fd to communicate with sensors_simulator.");
74 DEFINE_string(client_dir, "webrtc", "Location of the client files");
75 DEFINE_string(group_id, "", "The group id of device");
76
77 namespace cuttlefish {
78
79 using webrtc_streaming::RecordingManager;
80 using webrtc_streaming::ServerConfig;
81 using webrtc_streaming::Streamer;
82 using webrtc_streaming::StreamerConfig;
83 using webrtc_streaming::VideoSink;
84
85 constexpr auto kOpewnrtWanIpAddressName = "wan_ipaddr";
86 constexpr auto kTouchscreenPrefix = "display_";
87 constexpr auto kTouchpadPrefix = "touch_";
88
89 class CfOperatorObserver : public webrtc_streaming::OperatorObserver {
90 public:
91 virtual ~CfOperatorObserver() = default;
OnRegistered()92 virtual void OnRegistered() override {
93 LOG(VERBOSE) << "Registered with Operator";
94 }
OnClose()95 virtual void OnClose() override {
96 LOG(ERROR) << "Connection with Operator unexpectedly closed";
97 }
OnError()98 virtual void OnError() override {
99 LOG(ERROR) << "Error encountered in connection with Operator";
100 }
101 };
CreateAudioServer()102 std::unique_ptr<AudioServer> CreateAudioServer() {
103 SharedFD audio_server_fd = SharedFD::Dup(FLAGS_audio_server_fd);
104 close(FLAGS_audio_server_fd);
105 return std::make_unique<AudioServer>(audio_server_fd);
106 }
107
WebRtcComponent()108 fruit::Component<CustomActionConfigProvider> WebRtcComponent() {
109 return fruit::createComponent()
110 .install(ConfigFlagPlaceholder)
111 .install(CustomActionsComponent);
112 };
113
114 fruit::Component<ScreenConnector<DisplayHandler::WebRtcScProcessedFrame>,
115 confui::HostServer, confui::HostVirtualInput>
CreateConfirmationUIComponent(int * frames_fd,bool * frames_are_rgba,confui::PipeConnectionPair * pipe_io_pair,InputConnector * input_connector)116 CreateConfirmationUIComponent(int* frames_fd, bool* frames_are_rgba,
117 confui::PipeConnectionPair* pipe_io_pair,
118 InputConnector* input_connector) {
119 using ScreenConnector = DisplayHandler::ScreenConnector;
120 return fruit::createComponent()
121 .bindInstance<fruit::Annotated<WaylandScreenConnector::FramesFd, int>>(
122 *frames_fd)
123 .bindInstance<
124 fruit::Annotated<WaylandScreenConnector::FramesAreRgba, bool>>(
125 *frames_are_rgba)
126 .bindInstance(*pipe_io_pair)
127 .bind<ScreenConnectorFrameRenderer, ScreenConnector>()
128 .bindInstance(*input_connector);
129 }
130
ControlLoop(SharedFD control_socket,DisplayHandler & display_handler,RecordingManager & recording_manager,ScreenshotHandler & screenshot_handler)131 Result<void> ControlLoop(SharedFD control_socket,
132 DisplayHandler& display_handler,
133 RecordingManager& recording_manager,
134 ScreenshotHandler& screenshot_handler) {
135 WebrtcServerCommandChannel channel(control_socket);
136 while (true) {
137 webrtc::WebrtcCommandRequest request = CF_EXPECT(channel.ReceiveRequest());
138
139 Result<void> command_result = {};
140 if (request.has_start_recording_request()) {
141 LOG(INFO) << "Received command to start recording in main.cpp.";
142 recording_manager.Start();
143 } else if (request.has_stop_recording_request()) {
144 LOG(INFO) << "Received command to stop recording in main.cpp.";
145 recording_manager.Stop();
146 } else if (request.has_screenshot_display_request()) {
147 const auto& screenshot_request = request.screenshot_display_request();
148 LOG(INFO) << "Received command to screenshot display "
149 << screenshot_request.display_number() << "in main.cpp.";
150
151 display_handler.AddDisplayClient();
152
153 command_result =
154 screenshot_handler.Screenshot(screenshot_request.display_number(),
155 screenshot_request.screenshot_path());
156
157 display_handler.RemoveDisplayClient();
158
159 if (!command_result.ok()) {
160 LOG(ERROR) << "Failed to screenshot display "
161 << screenshot_request.display_number() << " to "
162 << screenshot_request.screenshot_path() << ":"
163 << command_result.error().Message();
164 }
165 } else {
166 LOG(FATAL) << "Unhandled request: " << request.DebugString();
167 }
168
169 webrtc::WebrtcCommandResponse response;
170 auto* response_status = response.mutable_status();
171 if (command_result.ok()) {
172 response_status->set_code(google::rpc::Code::OK);
173 } else {
174 response_status->set_code(google::rpc::Code::INTERNAL);
175 response_status->set_message(command_result.error().Message());
176 }
177
178 CF_EXPECT(channel.SendResponse(response));
179 }
180 }
181
CuttlefishMain()182 int CuttlefishMain() {
183 auto control_socket = SharedFD::Dup(FLAGS_command_fd);
184 close(FLAGS_command_fd);
185
186 auto cvd_config = CuttlefishConfig::Get();
187 auto instance = cvd_config->ForDefaultInstance();
188
189 cuttlefish::InputConnectorBuilder inputs_builder;
190
191 const auto display_count = instance.display_configs().size();
192 const auto touch_fds = android::base::Split(FLAGS_touch_fds, ",");
193 CHECK(touch_fds.size() == display_count + instance.touchpad_configs().size())
194 << "Number of touch FDs does not match the number of configured displays "
195 "and touchpads";
196 for (int i = 0; i < touch_fds.size(); i++) {
197 int touch_fd;
198 CHECK(android::base::ParseInt(touch_fds[i], &touch_fd))
199 << "Invalid touch_fd: " << touch_fds[i];
200 // Displays are listed first, then touchpads
201 auto label_prefix =
202 i < display_count ? kTouchscreenPrefix : kTouchpadPrefix;
203 auto device_idx = i < display_count ? i : i - display_count;
204 auto device_label = fmt::format("{}{}", label_prefix, device_idx);
205 auto touch_shared_fd = SharedFD::Dup(touch_fd);
206 if (FLAGS_multitouch) {
207 inputs_builder.WithMultitouchDevice(device_label, touch_shared_fd);
208 } else {
209 inputs_builder.WithTouchDevice(device_label, touch_shared_fd);
210 }
211 close(touch_fd);
212 }
213 if (FLAGS_rotary_fd >= 0) {
214 inputs_builder.WithRotary(SharedFD::Dup(FLAGS_rotary_fd));
215 close(FLAGS_rotary_fd);
216 }
217 if (FLAGS_mouse_fd >= 0) {
218 inputs_builder.WithMouse(SharedFD::Dup(FLAGS_mouse_fd));
219 close(FLAGS_mouse_fd);
220 }
221 if (FLAGS_keyboard_fd >= 0) {
222 inputs_builder.WithKeyboard(SharedFD::Dup(FLAGS_keyboard_fd));
223 close(FLAGS_keyboard_fd);
224 }
225 if (FLAGS_switches_fd >= 0) {
226 inputs_builder.WithSwitches(SharedFD::Dup(FLAGS_switches_fd));
227 close(FLAGS_switches_fd);
228 }
229
230 auto input_connector = std::move(inputs_builder).Build();
231
232 auto kernel_log_events_client = SharedFD::Dup(FLAGS_kernel_log_events_fd);
233 close(FLAGS_kernel_log_events_fd);
234
235 auto sensors_fd = cuttlefish::SharedFD::Dup(FLAGS_sensors_fd);
236 close(FLAGS_sensors_fd);
237
238 confui::PipeConnectionPair conf_ui_comm_fd_pair{
239 .from_guest_ = SharedFD::Dup(FLAGS_confui_out_fd),
240 .to_guest_ = SharedFD::Dup(FLAGS_confui_in_fd)};
241 close(FLAGS_confui_in_fd);
242 close(FLAGS_confui_out_fd);
243
244 int frames_fd = FLAGS_frame_server_fd;
245 bool frames_are_rgba = true;
246 fruit::Injector<ScreenConnector<DisplayHandler::WebRtcScProcessedFrame>,
247 confui::HostServer, confui::HostVirtualInput>
248 conf_ui_components_injector(CreateConfirmationUIComponent,
249 std::addressof(frames_fd),
250 std::addressof(frames_are_rgba),
251 &conf_ui_comm_fd_pair, input_connector.get());
252 auto& screen_connector =
253 conf_ui_components_injector.get<DisplayHandler::ScreenConnector&>();
254
255 auto client_server = ClientFilesServer::New(FLAGS_client_dir);
256 CHECK(client_server) << "Failed to initialize client files server";
257 auto& host_confui_server =
258 conf_ui_components_injector.get<confui::HostServer&>();
259 auto& confui_virtual_input =
260 conf_ui_components_injector.get<confui::HostVirtualInput&>();
261
262 StreamerConfig streamer_config;
263
264 streamer_config.device_id = instance.webrtc_device_id();
265 streamer_config.group_id = FLAGS_group_id;
266 streamer_config.client_files_port = client_server->port();
267 streamer_config.tcp_port_range = instance.webrtc_tcp_port_range();
268 streamer_config.udp_port_range = instance.webrtc_udp_port_range();
269 streamer_config.openwrt_device_id =
270 cvd_config->Instances()[0].webrtc_device_id();
271 streamer_config.openwrt_addr = OpenwrtArgsFromConfig(
272 cvd_config->Instances()[0])[kOpewnrtWanIpAddressName];
273 streamer_config.adb_port = instance.adb_host_port();
274 streamer_config.control_env_proxy_server_path =
275 instance.grpc_socket_path() + "/ControlEnvProxyServer.sock";
276 streamer_config.operator_server.addr = cvd_config->sig_server_address();
277 streamer_config.operator_server.port = cvd_config->sig_server_port();
278 streamer_config.operator_server.path = cvd_config->sig_server_path();
279 if (cvd_config->sig_server_secure()) {
280 streamer_config.operator_server.security =
281 cvd_config->sig_server_strict()
282 ? ServerConfig::Security::kStrict
283 : ServerConfig::Security::kAllowSelfSigned;
284 } else {
285 streamer_config.operator_server.security =
286 ServerConfig::Security::kInsecure;
287 }
288 streamer_config.enable_mouse = instance.enable_mouse();
289
290 KernelLogEventsHandler kernel_logs_event_handler(kernel_log_events_client);
291
292 std::shared_ptr<webrtc_streaming::LightsObserver> lights_observer;
293 if (instance.lights_server_port()) {
294 lights_observer = std::make_shared<webrtc_streaming::LightsObserver>(
295 instance.lights_server_port(), instance.vsock_guest_cid(),
296 instance.vhost_user_vsock());
297 lights_observer->Start();
298 }
299
300 webrtc_streaming::SensorsHandler sensors_handler(sensors_fd);
301
302 auto observer_factory = std::make_shared<CfConnectionObserverFactory>(
303 confui_virtual_input, kernel_logs_event_handler, sensors_handler,
304 lights_observer);
305
306 RecordingManager recording_manager;
307
308 ScreenshotHandler screenshot_handler;
309
310 auto streamer =
311 Streamer::Create(streamer_config, recording_manager, observer_factory);
312 CHECK(streamer) << "Could not create streamer";
313
314 // Determine whether to enable Display Composition feature.
315 // It's enabled via the multi-vd config file entry 'overlays'
316 std::optional<std::unique_ptr<CompositionManager>> composition_manager;
317
318 if (cvd_config->OverlaysEnabled()) {
319 Result<std::unique_ptr<CompositionManager>> composition_manager_result =
320 CompositionManager::Create();
321 if (composition_manager_result.ok() && *composition_manager_result) {
322 composition_manager = std::optional<std::unique_ptr<CompositionManager>>(
323 std::move(*composition_manager_result));
324 }
325 }
326
327 auto display_handler = std::make_shared<DisplayHandler>(
328 *streamer, screenshot_handler, screen_connector,
329 std::move(composition_manager));
330
331 if (instance.camera_server_port()) {
332 auto camera_controller = streamer->AddCamera(instance.camera_server_port(),
333 instance.vsock_guest_cid(),
334 instance.vhost_user_vsock());
335 observer_factory->SetCameraHandler(camera_controller);
336 streamer->SetHardwareSpec("camera_passthrough", true);
337 }
338
339 observer_factory->SetDisplayHandler(display_handler);
340
341 const auto touchpad_configs = instance.touchpad_configs();
342 for (int i = 0; i < touchpad_configs.size(); i++) {
343 streamer->AddTouchpad(kTouchpadPrefix + std::to_string(i),
344 touchpad_configs[i].width,
345 touchpad_configs[i].height);
346 }
347
348 streamer->SetHardwareSpec("CPUs", instance.cpus());
349 streamer->SetHardwareSpec("RAM",
350 std::to_string(instance.memory_mb()) + " mb");
351
352 std::string user_friendly_gpu_mode;
353 if (instance.gpu_mode() == kGpuModeGuestSwiftshader) {
354 user_friendly_gpu_mode = "SwiftShader (Guest CPU Rendering)";
355 } else if (instance.gpu_mode() == kGpuModeDrmVirgl) {
356 user_friendly_gpu_mode =
357 "VirglRenderer (Accelerated Rendering using Host OpenGL)";
358 } else if (instance.gpu_mode() == kGpuModeGfxstream) {
359 user_friendly_gpu_mode =
360 "Gfxstream (Accelerated Rendering using Host OpenGL and Vulkan)";
361 } else if (instance.gpu_mode() == kGpuModeGfxstreamGuestAngle) {
362 user_friendly_gpu_mode =
363 "Gfxstream (Accelerated Rendering using Host Vulkan)";
364 } else {
365 user_friendly_gpu_mode = instance.gpu_mode();
366 }
367 streamer->SetHardwareSpec("GPU Mode", user_friendly_gpu_mode);
368
369 std::shared_ptr<AudioHandler> audio_handler;
370 if (instance.enable_audio()) {
371 int output_streams_count = instance.audio_output_streams_count();
372 std::vector<std::shared_ptr<webrtc_streaming::AudioSink>> audio_streams(
373 output_streams_count);
374 for (int i = 0; i < audio_streams.size(); i++) {
375 audio_streams[i] = streamer->AddAudioStream("audio-" + std::to_string(i));
376 }
377 auto audio_server = CreateAudioServer();
378 auto audio_source = streamer->GetAudioSource();
379 audio_handler = std::make_shared<AudioHandler>(
380 std::move(audio_server), std::move(audio_streams), audio_source);
381 }
382
383 // Parse the -action_servers flag, storing a map of action server name -> fd
384 std::map<std::string, int> action_server_fds;
385 for (const std::string& action_server :
386 android::base::Split(FLAGS_action_servers, ",")) {
387 if (action_server.empty()) {
388 continue;
389 }
390 const std::vector<std::string> server_and_fd =
391 android::base::Split(action_server, ":");
392 CHECK(server_and_fd.size() == 2)
393 << "Wrong format for action server flag: " << action_server;
394 std::string server = server_and_fd[0];
395 int fd = std::stoi(server_and_fd[1]);
396 action_server_fds[server] = fd;
397 }
398
399 fruit::Injector<CustomActionConfigProvider> injector(WebRtcComponent);
400 for (auto& fragment : injector.getMultibindings<ConfigFragment>()) {
401 CHECK(cvd_config->LoadFragment(*fragment))
402 << "Failed to load config fragment";
403 }
404
405 const auto& actions_provider = injector.get<CustomActionConfigProvider&>();
406
407 for (const auto& custom_action :
408 actions_provider.CustomShellActions(instance.id())) {
409 const auto button = custom_action.button;
410 streamer->AddCustomControlPanelButtonWithShellCommand(
411 button.command, button.title, button.icon_name,
412 custom_action.shell_command);
413 }
414
415 for (const auto& custom_action :
416 actions_provider.CustomActionServers(instance.id())) {
417 if (action_server_fds.find(custom_action.server) ==
418 action_server_fds.end()) {
419 LOG(ERROR) << "Custom action server not provided as command line flag: "
420 << custom_action.server;
421 continue;
422 }
423 LOG(INFO) << "Connecting to custom action server " << custom_action.server;
424
425 int fd = action_server_fds[custom_action.server];
426 SharedFD custom_action_server = SharedFD::Dup(fd);
427 close(fd);
428
429 if (custom_action_server->IsOpen()) {
430 std::vector<std::string> commands_for_this_server;
431 for (const auto& button : custom_action.buttons) {
432 streamer->AddCustomControlPanelButton(button.command, button.title,
433 button.icon_name);
434 commands_for_this_server.push_back(button.command);
435 }
436 observer_factory->AddCustomActionServer(custom_action_server,
437 commands_for_this_server);
438 } else {
439 LOG(ERROR) << "Error connecting to custom action server: "
440 << custom_action.server;
441 }
442 }
443
444 for (const auto& custom_action :
445 actions_provider.CustomDeviceStateActions(instance.id())) {
446 const auto button = custom_action.button;
447 streamer->AddCustomControlPanelButtonWithDeviceStates(
448 button.command, button.title, button.icon_name,
449 custom_action.device_states);
450 }
451
452 std::shared_ptr<webrtc_streaming::OperatorObserver> operator_observer(
453 new CfOperatorObserver());
454 streamer->Register(operator_observer);
455
456 std::thread control_thread([&]() {
457 auto result = ControlLoop(control_socket, *display_handler,
458 recording_manager, screenshot_handler);
459 if (!result.ok()) {
460 LOG(ERROR) << "Webrtc control loop error: " << result.error().Message();
461 }
462 LOG(DEBUG) << "Webrtc control thread exiting.";
463 });
464
465 if (audio_handler) {
466 audio_handler->Start();
467 }
468 host_confui_server.Start();
469
470 if (instance.record_screen()) {
471 LOG(VERBOSE) << "Waiting for recording manager initializing.";
472 recording_manager.WaitForSources(instance.display_configs().size());
473 recording_manager.Start();
474 }
475
476 display_handler->Loop();
477
478 return 0;
479 }
480
481 } // namespace cuttlefish
482
main(int argc,char ** argv)483 int main(int argc, char** argv) {
484 cuttlefish::DefaultSubprocessLogging(argv);
485 ::gflags::ParseCommandLineFlags(&argc, &argv, true);
486 return cuttlefish::CuttlefishMain();
487 }
488