1 /*
2 * Copyright (C) 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "ConnectionObserver"
18
19 #include "host/frontend/webrtc/connection_observer.h"
20
21 #include <linux/input.h>
22
23 #include <chrono>
24 #include <map>
25 #include <set>
26 #include <thread>
27 #include <vector>
28
29 #include <json/json.h>
30
31 #include <android-base/logging.h>
32 #include <gflags/gflags.h>
33
34 #include "common/libs/confui/confui.h"
35 #include "common/libs/fs/shared_buf.h"
36 #include "host/frontend/webrtc/adb_handler.h"
37 #include "host/frontend/webrtc/bluetooth_handler.h"
38 #include "host/frontend/webrtc/lib/camera_controller.h"
39 #include "host/frontend/webrtc/lib/utils.h"
40 #include "host/libs/config/cuttlefish_config.h"
41
42 DECLARE_bool(write_virtio_input);
43
44 namespace cuttlefish {
45
46 // TODO (b/147511234): de-dup this from vnc server and here
47 struct virtio_input_event {
48 uint16_t type;
49 uint16_t code;
50 int32_t value;
51 };
52
53 struct multitouch_slot {
54 int32_t id;
55 int32_t slot;
56 int32_t x;
57 int32_t y;
58 };
59
60 struct InputEventBuffer {
61 virtual ~InputEventBuffer() = default;
62 virtual void AddEvent(uint16_t type, uint16_t code, int32_t value) = 0;
63 virtual size_t size() const = 0;
64 virtual const void *data() const = 0;
65 };
66
67 template <typename T>
68 struct InputEventBufferImpl : public InputEventBuffer {
InputEventBufferImplcuttlefish::InputEventBufferImpl69 InputEventBufferImpl() {
70 buffer_.reserve(6); // 6 is usually enough
71 }
AddEventcuttlefish::InputEventBufferImpl72 void AddEvent(uint16_t type, uint16_t code, int32_t value) override {
73 buffer_.push_back({.type = type, .code = code, .value = value});
74 }
datacuttlefish::InputEventBufferImpl75 T *data() { return buffer_.data(); }
datacuttlefish::InputEventBufferImpl76 const void *data() const override { return buffer_.data(); }
sizecuttlefish::InputEventBufferImpl77 std::size_t size() const override { return buffer_.size() * sizeof(T); }
78
79 private:
80 std::vector<T> buffer_;
81 };
82
83 // TODO: we could add an arg here to specify whether we want the multitouch buffer?
GetEventBuffer()84 std::unique_ptr<InputEventBuffer> GetEventBuffer() {
85 if (FLAGS_write_virtio_input) {
86 return std::unique_ptr<InputEventBuffer>(
87 new InputEventBufferImpl<virtio_input_event>());
88 } else {
89 return std::unique_ptr<InputEventBuffer>(
90 new InputEventBufferImpl<input_event>());
91 }
92 }
93
94 /**
95 * connection observer implementation for regular android mode.
96 * i.e. when it is not in the confirmation UI mode (or TEE),
97 * the control flow will fall back to this ConnectionObserverForAndroid
98 */
99 class ConnectionObserverImpl
100 : public cuttlefish::webrtc_streaming::ConnectionObserver {
101 public:
ConnectionObserverImpl(cuttlefish::InputSockets & input_sockets,cuttlefish::KernelLogEventsHandler * kernel_log_events_handler,std::map<std::string,cuttlefish::SharedFD> commands_to_custom_action_servers,std::weak_ptr<DisplayHandler> display_handler,CameraController * camera_controller,cuttlefish::confui::HostVirtualInput & confui_input)102 ConnectionObserverImpl(
103 cuttlefish::InputSockets &input_sockets,
104 cuttlefish::KernelLogEventsHandler *kernel_log_events_handler,
105 std::map<std::string, cuttlefish::SharedFD>
106 commands_to_custom_action_servers,
107 std::weak_ptr<DisplayHandler> display_handler,
108 CameraController *camera_controller,
109 cuttlefish::confui::HostVirtualInput &confui_input)
110 : input_sockets_(input_sockets),
111 kernel_log_events_handler_(kernel_log_events_handler),
112 commands_to_custom_action_servers_(commands_to_custom_action_servers),
113 weak_display_handler_(display_handler),
114 camera_controller_(camera_controller),
115 confui_input_(confui_input) {}
~ConnectionObserverImpl()116 virtual ~ConnectionObserverImpl() {
117 auto display_handler = weak_display_handler_.lock();
118 if (kernel_log_subscription_id_ != -1) {
119 kernel_log_events_handler_->Unsubscribe(kernel_log_subscription_id_);
120 }
121 }
122
OnConnected(std::function<void (const uint8_t *,size_t,bool)>)123 void OnConnected(std::function<void(const uint8_t *, size_t, bool)>
124 /*ctrl_msg_sender*/) override {
125 auto display_handler = weak_display_handler_.lock();
126 if (display_handler) {
127 std::thread th([this]() {
128 // The encoder in libwebrtc won't drop 5 consecutive frames due to frame
129 // size, so we make sure at least 5 frames are sent every time a client
130 // connects to ensure they receive at least one.
131 constexpr int kNumFrames = 5;
132 constexpr int kMillisPerFrame = 16;
133 for (int i = 0; i < kNumFrames; ++i) {
134 auto display_handler = weak_display_handler_.lock();
135 display_handler->SendLastFrame();
136 if (i < kNumFrames - 1) {
137 std::this_thread::sleep_for(std::chrono::milliseconds(kMillisPerFrame));
138 }
139 }
140 });
141 th.detach();
142 }
143 }
144
OnTouchEvent(const std::string & display_label,int x,int y,bool down)145 void OnTouchEvent(const std::string &display_label, int x, int y,
146 bool down) override {
147 if (confui_input_.IsConfUiActive()) {
148 ConfUiLog(DEBUG) << "delivering a touch event in confirmation UI mode";
149 confui_input_.TouchEvent(x, y, down);
150 return;
151 }
152 auto buffer = GetEventBuffer();
153 if (!buffer) {
154 LOG(ERROR) << "Failed to allocate event buffer";
155 return;
156 }
157 buffer->AddEvent(EV_ABS, ABS_X, x);
158 buffer->AddEvent(EV_ABS, ABS_Y, y);
159 buffer->AddEvent(EV_KEY, BTN_TOUCH, down);
160 buffer->AddEvent(EV_SYN, SYN_REPORT, 0);
161 cuttlefish::WriteAll(input_sockets_.GetTouchClientByLabel(display_label),
162 reinterpret_cast<const char *>(buffer->data()),
163 buffer->size());
164 }
165
OnMultiTouchEvent(const std::string & display_label,Json::Value id,Json::Value slot,Json::Value x,Json::Value y,bool down,int size)166 void OnMultiTouchEvent(const std::string &display_label, Json::Value id,
167 Json::Value slot, Json::Value x, Json::Value y,
168 bool down, int size) {
169 auto buffer = GetEventBuffer();
170 if (!buffer) {
171 LOG(ERROR) << "Failed to allocate event buffer";
172 return;
173 }
174
175 for (int i = 0; i < size; i++) {
176 auto this_slot = slot[i].asInt();
177 auto this_id = id[i].asInt();
178 auto this_x = x[i].asInt();
179 auto this_y = y[i].asInt();
180
181 if (confui_input_.IsConfUiActive()) {
182 if (down) {
183 ConfUiLog(DEBUG) << "Delivering event (" << x << ", " << y
184 << ") to conf ui";
185 }
186 confui_input_.TouchEvent(this_x, this_y, down);
187 continue;
188 }
189
190 buffer->AddEvent(EV_ABS, ABS_MT_SLOT, this_slot);
191 if (down) {
192 bool is_new = active_touch_slots_.insert(this_slot).second;
193 if (is_new) {
194 buffer->AddEvent(EV_ABS, ABS_MT_TRACKING_ID, this_id);
195 if (active_touch_slots_.size() == 1) {
196 buffer->AddEvent(EV_KEY, BTN_TOUCH, 1);
197 }
198 }
199 buffer->AddEvent(EV_ABS, ABS_MT_POSITION_X, this_x);
200 buffer->AddEvent(EV_ABS, ABS_MT_POSITION_Y, this_y);
201 // send ABS_X and ABS_Y for single-touch compatibility
202 buffer->AddEvent(EV_ABS, ABS_X, this_x);
203 buffer->AddEvent(EV_ABS, ABS_Y, this_y);
204 } else {
205 // released touch
206 buffer->AddEvent(EV_ABS, ABS_MT_TRACKING_ID, this_id);
207 active_touch_slots_.erase(this_slot);
208 if (active_touch_slots_.empty()) {
209 buffer->AddEvent(EV_KEY, BTN_TOUCH, 0);
210 }
211 }
212 }
213
214 buffer->AddEvent(EV_SYN, SYN_REPORT, 0);
215 cuttlefish::WriteAll(input_sockets_.GetTouchClientByLabel(display_label),
216 reinterpret_cast<const char *>(buffer->data()),
217 buffer->size());
218 }
219
OnKeyboardEvent(uint16_t code,bool down)220 void OnKeyboardEvent(uint16_t code, bool down) override {
221 if (confui_input_.IsConfUiActive()) {
222 ConfUiLog(DEBUG) << "keyboard event ignored in confirmation UI mode";
223 return;
224 }
225
226 auto buffer = GetEventBuffer();
227 if (!buffer) {
228 LOG(ERROR) << "Failed to allocate event buffer";
229 return;
230 }
231 buffer->AddEvent(EV_KEY, code, down);
232 buffer->AddEvent(EV_SYN, SYN_REPORT, 0);
233 cuttlefish::WriteAll(input_sockets_.keyboard_client,
234 reinterpret_cast<const char *>(buffer->data()),
235 buffer->size());
236 }
237
OnSwitchEvent(uint16_t code,bool state)238 void OnSwitchEvent(uint16_t code, bool state) override {
239 auto buffer = GetEventBuffer();
240 if (!buffer) {
241 LOG(ERROR) << "Failed to allocate event buffer";
242 return;
243 }
244 buffer->AddEvent(EV_SW, code, state);
245 buffer->AddEvent(EV_SYN, SYN_REPORT, 0);
246 cuttlefish::WriteAll(input_sockets_.switches_client,
247 reinterpret_cast<const char *>(buffer->data()),
248 buffer->size());
249 }
250
OnAdbChannelOpen(std::function<bool (const uint8_t *,size_t)> adb_message_sender)251 void OnAdbChannelOpen(std::function<bool(const uint8_t *, size_t)>
252 adb_message_sender) override {
253 LOG(VERBOSE) << "Adb Channel open";
254 adb_handler_.reset(new cuttlefish::webrtc_streaming::AdbHandler(
255 cuttlefish::CuttlefishConfig::Get()
256 ->ForDefaultInstance()
257 .adb_ip_and_port(),
258 adb_message_sender));
259 }
OnAdbMessage(const uint8_t * msg,size_t size)260 void OnAdbMessage(const uint8_t *msg, size_t size) override {
261 adb_handler_->handleMessage(msg, size);
262 }
OnControlChannelOpen(std::function<bool (const Json::Value)> control_message_sender)263 void OnControlChannelOpen(std::function<bool(const Json::Value)>
264 control_message_sender) override {
265 LOG(VERBOSE) << "Control Channel open";
266 if (camera_controller_) {
267 camera_controller_->SetMessageSender(control_message_sender);
268 }
269 kernel_log_subscription_id_ =
270 kernel_log_events_handler_->AddSubscriber(control_message_sender);
271 }
OnControlMessage(const uint8_t * msg,size_t size)272 void OnControlMessage(const uint8_t* msg, size_t size) override {
273 Json::Value evt;
274 const char* msg_str = reinterpret_cast<const char*>(msg);
275 Json::CharReaderBuilder builder;
276 std::unique_ptr<Json::CharReader> json_reader(builder.newCharReader());
277 std::string errorMessage;
278 if (!json_reader->parse(msg_str, msg_str + size, &evt, &errorMessage)) {
279 LOG(ERROR) << "Received invalid JSON object over control channel: " << errorMessage;
280 return;
281 }
282
283 auto result = webrtc_streaming::ValidationResult::ValidateJsonObject(
284 evt, "command",
285 /*required_fields=*/{{"command", Json::ValueType::stringValue}},
286 /*optional_fields=*/
287 {
288 {"button_state", Json::ValueType::stringValue},
289 {"lid_switch_open", Json::ValueType::booleanValue},
290 {"hinge_angle_value", Json::ValueType::intValue},
291 });
292 if (!result.ok()) {
293 LOG(ERROR) << result.error();
294 return;
295 }
296 auto command = evt["command"].asString();
297
298 if (command == "device_state") {
299 if (evt.isMember("lid_switch_open")) {
300 // InputManagerService treats a value of 0 as open and 1 as closed, so
301 // invert the lid_switch_open value that is sent to the input device.
302 OnSwitchEvent(SW_LID, !evt["lid_switch_open"].asBool());
303 }
304 if (evt.isMember("hinge_angle_value")) {
305 // TODO(b/181157794) Propagate hinge angle sensor data using a custom
306 // Sensor HAL.
307 }
308 return;
309 } else if (command.rfind("camera_", 0) == 0 && camera_controller_) {
310 // Handle commands starting with "camera_" by camera controller
311 camera_controller_->HandleMessage(evt);
312 return;
313 }
314
315 auto button_state = evt["button_state"].asString();
316 LOG(VERBOSE) << "Control command: " << command << " (" << button_state
317 << ")";
318 if (command == "power") {
319 OnKeyboardEvent(KEY_POWER, button_state == "down");
320 } else if (command == "home") {
321 OnKeyboardEvent(KEY_HOMEPAGE, button_state == "down");
322 } else if (command == "menu") {
323 OnKeyboardEvent(KEY_MENU, button_state == "down");
324 } else if (command == "volumedown") {
325 OnKeyboardEvent(KEY_VOLUMEDOWN, button_state == "down");
326 } else if (command == "volumeup") {
327 OnKeyboardEvent(KEY_VOLUMEUP, button_state == "down");
328 } else if (commands_to_custom_action_servers_.find(command) !=
329 commands_to_custom_action_servers_.end()) {
330 // Simple protocol for commands forwarded to action servers:
331 // - Always 128 bytes
332 // - Format: command:button_state
333 // - Example: my_button:down
334 std::string action_server_message = command + ":" + button_state;
335 cuttlefish::WriteAll(commands_to_custom_action_servers_[command],
336 action_server_message.c_str(), 128);
337 } else {
338 LOG(WARNING) << "Unsupported control command: " << command << " ("
339 << button_state << ")";
340 }
341 }
342
OnBluetoothChannelOpen(std::function<bool (const uint8_t *,size_t)> bluetooth_message_sender)343 void OnBluetoothChannelOpen(std::function<bool(const uint8_t *, size_t)>
344 bluetooth_message_sender) override {
345 LOG(VERBOSE) << "Bluetooth channel open";
346 auto config = cuttlefish::CuttlefishConfig::Get();
347 CHECK(config) << "Failed to get config";
348 bluetooth_handler_.reset(new cuttlefish::webrtc_streaming::BluetoothHandler(
349 config->rootcanal_test_port(), bluetooth_message_sender));
350 }
351
OnBluetoothMessage(const uint8_t * msg,size_t size)352 void OnBluetoothMessage(const uint8_t *msg, size_t size) override {
353 bluetooth_handler_->handleMessage(msg, size);
354 }
355
OnCameraData(const std::vector<char> & data)356 void OnCameraData(const std::vector<char> &data) override {
357 if (camera_controller_) {
358 camera_controller_->HandleMessage(data);
359 }
360 }
361
362 private:
363 cuttlefish::InputSockets& input_sockets_;
364 cuttlefish::KernelLogEventsHandler* kernel_log_events_handler_;
365 int kernel_log_subscription_id_ = -1;
366 std::shared_ptr<cuttlefish::webrtc_streaming::AdbHandler> adb_handler_;
367 std::shared_ptr<cuttlefish::webrtc_streaming::BluetoothHandler>
368 bluetooth_handler_;
369 std::map<std::string, cuttlefish::SharedFD> commands_to_custom_action_servers_;
370 std::weak_ptr<DisplayHandler> weak_display_handler_;
371 std::set<int32_t> active_touch_slots_;
372 cuttlefish::CameraController *camera_controller_;
373 cuttlefish::confui::HostVirtualInput &confui_input_;
374 };
375
CfConnectionObserverFactory(cuttlefish::InputSockets & input_sockets,cuttlefish::KernelLogEventsHandler * kernel_log_events_handler,cuttlefish::confui::HostVirtualInput & confui_input)376 CfConnectionObserverFactory::CfConnectionObserverFactory(
377 cuttlefish::InputSockets &input_sockets,
378 cuttlefish::KernelLogEventsHandler* kernel_log_events_handler,
379 cuttlefish::confui::HostVirtualInput &confui_input)
380 : input_sockets_(input_sockets),
381 kernel_log_events_handler_(kernel_log_events_handler),
382 confui_input_{confui_input} {}
383
384 std::shared_ptr<cuttlefish::webrtc_streaming::ConnectionObserver>
CreateObserver()385 CfConnectionObserverFactory::CreateObserver() {
386 return std::shared_ptr<cuttlefish::webrtc_streaming::ConnectionObserver>(
387 new ConnectionObserverImpl(input_sockets_, kernel_log_events_handler_,
388 commands_to_custom_action_servers_,
389 weak_display_handler_, camera_controller_,
390 confui_input_));
391 }
392
AddCustomActionServer(cuttlefish::SharedFD custom_action_server_fd,const std::vector<std::string> & commands)393 void CfConnectionObserverFactory::AddCustomActionServer(
394 cuttlefish::SharedFD custom_action_server_fd,
395 const std::vector<std::string>& commands) {
396 for (const std::string& command : commands) {
397 LOG(DEBUG) << "Action server is listening to command: " << command;
398 commands_to_custom_action_servers_[command] = custom_action_server_fd;
399 }
400 }
401
SetDisplayHandler(std::weak_ptr<DisplayHandler> display_handler)402 void CfConnectionObserverFactory::SetDisplayHandler(
403 std::weak_ptr<DisplayHandler> display_handler) {
404 weak_display_handler_ = display_handler;
405 }
406
SetCameraHandler(CameraController * controller)407 void CfConnectionObserverFactory::SetCameraHandler(
408 CameraController *controller) {
409 camera_controller_ = controller;
410 }
411 } // namespace cuttlefish
412