1 /*
2 * Copyright (C) 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "ConnectionObserver"
18
19 #include "host/frontend/webrtc/connection_observer.h"
20
21 #include <linux/input.h>
22
23 #include <chrono>
24 #include <map>
25 #include <set>
26 #include <thread>
27 #include <vector>
28
29 #include <json/json.h>
30
31 #include <android-base/logging.h>
32 #include <gflags/gflags.h>
33
34 #include "common/libs/confui/confui.h"
35 #include "common/libs/fs/shared_buf.h"
36 #include "host/frontend/webrtc/adb_handler.h"
37 #include "host/frontend/webrtc/bluetooth_handler.h"
38 #include "host/frontend/webrtc/gpx_locations_handler.h"
39 #include "host/frontend/webrtc/kml_locations_handler.h"
40 #include "host/frontend/webrtc/libdevice/camera_controller.h"
41 #include "host/frontend/webrtc/location_handler.h"
42 #include "host/libs/config/cuttlefish_config.h"
43
44 DECLARE_bool(write_virtio_input);
45
46 namespace cuttlefish {
47
48 // TODO (b/147511234): de-dup this from vnc server and here
49 struct virtio_input_event {
50 uint16_t type;
51 uint16_t code;
52 int32_t value;
53 };
54
55 struct multitouch_slot {
56 int32_t id;
57 int32_t slot;
58 int32_t x;
59 int32_t y;
60 };
61
62 struct InputEventBuffer {
63 virtual ~InputEventBuffer() = default;
64 virtual void AddEvent(uint16_t type, uint16_t code, int32_t value) = 0;
65 virtual size_t size() const = 0;
66 virtual const void *data() const = 0;
67 };
68
69 template <typename T>
70 struct InputEventBufferImpl : public InputEventBuffer {
InputEventBufferImplcuttlefish::InputEventBufferImpl71 InputEventBufferImpl() {
72 buffer_.reserve(6); // 6 is usually enough
73 }
AddEventcuttlefish::InputEventBufferImpl74 void AddEvent(uint16_t type, uint16_t code, int32_t value) override {
75 buffer_.push_back({.type = type, .code = code, .value = value});
76 }
datacuttlefish::InputEventBufferImpl77 T *data() { return buffer_.data(); }
datacuttlefish::InputEventBufferImpl78 const void *data() const override { return buffer_.data(); }
sizecuttlefish::InputEventBufferImpl79 std::size_t size() const override { return buffer_.size() * sizeof(T); }
80
81 private:
82 std::vector<T> buffer_;
83 };
84
85 // TODO: we could add an arg here to specify whether we want the multitouch buffer?
GetEventBuffer()86 std::unique_ptr<InputEventBuffer> GetEventBuffer() {
87 if (FLAGS_write_virtio_input) {
88 return std::unique_ptr<InputEventBuffer>(
89 new InputEventBufferImpl<virtio_input_event>());
90 } else {
91 return std::unique_ptr<InputEventBuffer>(
92 new InputEventBufferImpl<input_event>());
93 }
94 }
95
96 /**
97 * connection observer implementation for regular android mode.
98 * i.e. when it is not in the confirmation UI mode (or TEE),
99 * the control flow will fall back to this ConnectionObserverForAndroid
100 */
101 class ConnectionObserverImpl
102 : public cuttlefish::webrtc_streaming::ConnectionObserver {
103 public:
ConnectionObserverImpl(cuttlefish::InputSockets & input_sockets,cuttlefish::KernelLogEventsHandler * kernel_log_events_handler,std::map<std::string,cuttlefish::SharedFD> commands_to_custom_action_servers,std::weak_ptr<DisplayHandler> display_handler,CameraController * camera_controller,cuttlefish::confui::HostVirtualInput & confui_input)104 ConnectionObserverImpl(
105 cuttlefish::InputSockets &input_sockets,
106 cuttlefish::KernelLogEventsHandler *kernel_log_events_handler,
107 std::map<std::string, cuttlefish::SharedFD>
108 commands_to_custom_action_servers,
109 std::weak_ptr<DisplayHandler> display_handler,
110 CameraController *camera_controller,
111 cuttlefish::confui::HostVirtualInput &confui_input)
112 : input_sockets_(input_sockets),
113 kernel_log_events_handler_(kernel_log_events_handler),
114 commands_to_custom_action_servers_(commands_to_custom_action_servers),
115 weak_display_handler_(display_handler),
116 camera_controller_(camera_controller),
117 confui_input_(confui_input) {}
~ConnectionObserverImpl()118 virtual ~ConnectionObserverImpl() {
119 auto display_handler = weak_display_handler_.lock();
120 if (kernel_log_subscription_id_ != -1) {
121 kernel_log_events_handler_->Unsubscribe(kernel_log_subscription_id_);
122 }
123 }
124
OnConnected()125 void OnConnected() override {
126 auto display_handler = weak_display_handler_.lock();
127 if (display_handler) {
128 std::thread th([this]() {
129 // The encoder in libwebrtc won't drop 5 consecutive frames due to frame
130 // size, so we make sure at least 5 frames are sent every time a client
131 // connects to ensure they receive at least one.
132 constexpr int kNumFrames = 5;
133 constexpr int kMillisPerFrame = 16;
134 for (int i = 0; i < kNumFrames; ++i) {
135 auto display_handler = weak_display_handler_.lock();
136 display_handler->SendLastFrame();
137 if (i < kNumFrames - 1) {
138 std::this_thread::sleep_for(std::chrono::milliseconds(kMillisPerFrame));
139 }
140 }
141 });
142 th.detach();
143 }
144 }
145
OnTouchEvent(const std::string & display_label,int x,int y,bool down)146 void OnTouchEvent(const std::string &display_label, int x, int y,
147 bool down) override {
148 if (confui_input_.IsConfUiActive()) {
149 if (down) {
150 confui_input_.TouchEvent(x, y, down);
151 }
152 return;
153 }
154 auto buffer = GetEventBuffer();
155 if (!buffer) {
156 LOG(ERROR) << "Failed to allocate event buffer";
157 return;
158 }
159 buffer->AddEvent(EV_ABS, ABS_X, x);
160 buffer->AddEvent(EV_ABS, ABS_Y, y);
161 buffer->AddEvent(EV_KEY, BTN_TOUCH, down);
162 buffer->AddEvent(EV_SYN, SYN_REPORT, 0);
163 cuttlefish::WriteAll(input_sockets_.GetTouchClientByLabel(display_label),
164 reinterpret_cast<const char *>(buffer->data()),
165 buffer->size());
166 }
167
OnMultiTouchEvent(const std::string & display_label,Json::Value id,Json::Value slot,Json::Value x,Json::Value y,bool down,int size)168 void OnMultiTouchEvent(const std::string &display_label, Json::Value id,
169 Json::Value slot, Json::Value x, Json::Value y,
170 bool down, int size) {
171 auto buffer = GetEventBuffer();
172 if (!buffer) {
173 LOG(ERROR) << "Failed to allocate event buffer";
174 return;
175 }
176
177 for (int i = 0; i < size; i++) {
178 auto this_slot = slot[i].asInt();
179 auto this_id = id[i].asInt();
180 auto this_x = x[i].asInt();
181 auto this_y = y[i].asInt();
182
183 if (confui_input_.IsConfUiActive()) {
184 if (down) {
185 confui_input_.TouchEvent(this_x, this_y, down);
186 }
187 continue;
188 }
189
190 buffer->AddEvent(EV_ABS, ABS_MT_SLOT, this_slot);
191 if (down) {
192 bool is_new = active_touch_slots_.insert(this_slot).second;
193 if (is_new) {
194 buffer->AddEvent(EV_ABS, ABS_MT_TRACKING_ID, this_id);
195 if (active_touch_slots_.size() == 1) {
196 buffer->AddEvent(EV_KEY, BTN_TOUCH, 1);
197 }
198 }
199 buffer->AddEvent(EV_ABS, ABS_MT_POSITION_X, this_x);
200 buffer->AddEvent(EV_ABS, ABS_MT_POSITION_Y, this_y);
201 // send ABS_X and ABS_Y for single-touch compatibility
202 buffer->AddEvent(EV_ABS, ABS_X, this_x);
203 buffer->AddEvent(EV_ABS, ABS_Y, this_y);
204 } else {
205 // released touch
206 buffer->AddEvent(EV_ABS, ABS_MT_TRACKING_ID, this_id);
207 active_touch_slots_.erase(this_slot);
208 if (active_touch_slots_.empty()) {
209 buffer->AddEvent(EV_KEY, BTN_TOUCH, 0);
210 }
211 }
212 }
213
214 buffer->AddEvent(EV_SYN, SYN_REPORT, 0);
215 cuttlefish::WriteAll(input_sockets_.GetTouchClientByLabel(display_label),
216 reinterpret_cast<const char *>(buffer->data()),
217 buffer->size());
218 }
219
OnKeyboardEvent(uint16_t code,bool down)220 void OnKeyboardEvent(uint16_t code, bool down) override {
221 if (confui_input_.IsConfUiActive()) {
222 ConfUiLog(VERBOSE) << "keyboard event ignored in confirmation UI mode";
223 return;
224 }
225
226 auto buffer = GetEventBuffer();
227 if (!buffer) {
228 LOG(ERROR) << "Failed to allocate event buffer";
229 return;
230 }
231 buffer->AddEvent(EV_KEY, code, down);
232 buffer->AddEvent(EV_SYN, SYN_REPORT, 0);
233 cuttlefish::WriteAll(input_sockets_.keyboard_client,
234 reinterpret_cast<const char *>(buffer->data()),
235 buffer->size());
236 }
237
OnSwitchEvent(uint16_t code,bool state)238 void OnSwitchEvent(uint16_t code, bool state) {
239 auto buffer = GetEventBuffer();
240 if (!buffer) {
241 LOG(ERROR) << "Failed to allocate event buffer";
242 return;
243 }
244 buffer->AddEvent(EV_SW, code, state);
245 buffer->AddEvent(EV_SYN, SYN_REPORT, 0);
246 cuttlefish::WriteAll(input_sockets_.switches_client,
247 reinterpret_cast<const char *>(buffer->data()),
248 buffer->size());
249 }
250
OnAdbChannelOpen(std::function<bool (const uint8_t *,size_t)> adb_message_sender)251 void OnAdbChannelOpen(std::function<bool(const uint8_t *, size_t)>
252 adb_message_sender) override {
253 LOG(VERBOSE) << "Adb Channel open";
254 adb_handler_.reset(new cuttlefish::webrtc_streaming::AdbHandler(
255 cuttlefish::CuttlefishConfig::Get()
256 ->ForDefaultInstance()
257 .adb_ip_and_port(),
258 adb_message_sender));
259 }
OnAdbMessage(const uint8_t * msg,size_t size)260 void OnAdbMessage(const uint8_t *msg, size_t size) override {
261 adb_handler_->handleMessage(msg, size);
262 }
OnControlChannelOpen(std::function<bool (const Json::Value)> control_message_sender)263 void OnControlChannelOpen(std::function<bool(const Json::Value)>
264 control_message_sender) override {
265 LOG(VERBOSE) << "Control Channel open";
266 if (camera_controller_) {
267 camera_controller_->SetMessageSender(control_message_sender);
268 }
269 kernel_log_subscription_id_ =
270 kernel_log_events_handler_->AddSubscriber(control_message_sender);
271 }
272
OnLidStateChange(bool lid_open)273 void OnLidStateChange(bool lid_open) override {
274 // InputManagerService treats a value of 0 as open and 1 as closed, so
275 // invert the lid_switch_open value that is sent to the input device.
276 OnSwitchEvent(SW_LID, !lid_open);
277 }
OnHingeAngleChange(int)278 void OnHingeAngleChange(int /*hinge_angle*/) override {
279 // TODO(b/181157794) Propagate hinge angle sensor data using a custom
280 // Sensor HAL.
281 }
OnPowerButton(bool button_down)282 void OnPowerButton(bool button_down) override {
283 OnKeyboardEvent(KEY_POWER, button_down);
284 }
OnBackButton(bool button_down)285 void OnBackButton(bool button_down) override {
286 OnKeyboardEvent(KEY_BACK, button_down);
287 }
OnHomeButton(bool button_down)288 void OnHomeButton(bool button_down) override {
289 OnKeyboardEvent(KEY_HOMEPAGE, button_down);
290 }
OnMenuButton(bool button_down)291 void OnMenuButton(bool button_down) override {
292 OnKeyboardEvent(KEY_MENU, button_down);
293 }
OnVolumeDownButton(bool button_down)294 void OnVolumeDownButton(bool button_down) override {
295 OnKeyboardEvent(KEY_VOLUMEDOWN, button_down);
296 }
OnVolumeUpButton(bool button_down)297 void OnVolumeUpButton(bool button_down) override {
298 OnKeyboardEvent(KEY_VOLUMEUP, button_down);
299 }
OnCustomActionButton(const std::string & command,const std::string & button_state)300 void OnCustomActionButton(const std::string &command,
301 const std::string &button_state) override {
302 if (commands_to_custom_action_servers_.find(command) !=
303 commands_to_custom_action_servers_.end()) {
304 // Simple protocol for commands forwarded to action servers:
305 // - Always 128 bytes
306 // - Format: command:button_state
307 // - Example: my_button:down
308 std::string action_server_message = command + ":" + button_state;
309 cuttlefish::WriteAll(commands_to_custom_action_servers_[command],
310 action_server_message.c_str(), 128);
311 } else {
312 LOG(WARNING) << "Unsupported control command: " << command << " ("
313 << button_state << ")";
314 }
315 }
316
OnBluetoothChannelOpen(std::function<bool (const uint8_t *,size_t)> bluetooth_message_sender)317 void OnBluetoothChannelOpen(std::function<bool(const uint8_t *, size_t)>
318 bluetooth_message_sender) override {
319 LOG(VERBOSE) << "Bluetooth channel open";
320 auto config = cuttlefish::CuttlefishConfig::Get();
321 CHECK(config) << "Failed to get config";
322 bluetooth_handler_.reset(new cuttlefish::webrtc_streaming::BluetoothHandler(
323 config->rootcanal_test_port(), bluetooth_message_sender));
324 }
325
OnBluetoothMessage(const uint8_t * msg,size_t size)326 void OnBluetoothMessage(const uint8_t *msg, size_t size) override {
327 bluetooth_handler_->handleMessage(msg, size);
328 }
OnLocationChannelOpen(std::function<bool (const uint8_t *,size_t)> location_message_sender)329 void OnLocationChannelOpen(std::function<bool(const uint8_t *, size_t)>
330 location_message_sender) override {
331 LOG(VERBOSE) << "Location channel open";
332 auto config = cuttlefish::CuttlefishConfig::Get();
333 CHECK(config) << "Failed to get config";
334 location_handler_.reset(new cuttlefish::webrtc_streaming::LocationHandler(
335 location_message_sender));
336 }
OnLocationMessage(const uint8_t * msg,size_t size)337 void OnLocationMessage(const uint8_t *msg, size_t size) override {
338 std::string msgstr(msg, msg + size);
339
340 std::vector<std::string> inputs = android::base::Split(msgstr, ",");
341
342 if(inputs.size() != 3){
343 LOG(WARNING) << "Invalid location length , length = " << inputs.size();
344 return;
345 }
346
347 float longitude = std::stod(inputs.at(0));
348 float latitude = std::stod(inputs.at(1));
349 float elevation = std::stod(inputs.at(2));
350 location_handler_->HandleMessage(longitude, latitude, elevation);
351 }
352
OnKmlLocationsChannelOpen(std::function<bool (const uint8_t *,size_t)> kml_locations_message_sender)353 void OnKmlLocationsChannelOpen(std::function<bool(const uint8_t *, size_t)>
354 kml_locations_message_sender) override {
355 LOG(VERBOSE) << "Kml Locations channel open";
356 auto config = cuttlefish::CuttlefishConfig::Get();
357 CHECK(config) << "Failed to get config";
358 kml_locations_handler_.reset(
359 new cuttlefish::webrtc_streaming::KmlLocationsHandler(
360 kml_locations_message_sender));
361 }
OnKmlLocationsMessage(const uint8_t * msg,size_t size)362 void OnKmlLocationsMessage(const uint8_t *msg, size_t size) override {
363 kml_locations_handler_->HandleMessage(msg, size);
364 }
365
OnGpxLocationsChannelOpen(std::function<bool (const uint8_t *,size_t)> gpx_locations_message_sender)366 void OnGpxLocationsChannelOpen(std::function<bool(const uint8_t *, size_t)>
367 gpx_locations_message_sender) override {
368 LOG(VERBOSE) << "Gpx Locations channel open";
369 auto config = cuttlefish::CuttlefishConfig::Get();
370 CHECK(config) << "Failed to get config";
371 gpx_locations_handler_.reset(
372 new cuttlefish::webrtc_streaming::GpxLocationsHandler(
373 gpx_locations_message_sender));
374 }
OnGpxLocationsMessage(const uint8_t * msg,size_t size)375 void OnGpxLocationsMessage(const uint8_t *msg, size_t size) override {
376 gpx_locations_handler_->HandleMessage(msg, size);
377 }
378
OnCameraControlMsg(const Json::Value & msg)379 void OnCameraControlMsg(const Json::Value& msg) override {
380 if (camera_controller_) {
381 camera_controller_->HandleMessage(msg);
382 } else {
383 LOG(VERBOSE) << "Camera control message received but no camera "
384 "controller is available";
385 }
386 }
387
OnCameraData(const std::vector<char> & data)388 void OnCameraData(const std::vector<char> &data) override {
389 if (camera_controller_) {
390 camera_controller_->HandleMessage(data);
391 } else {
392 LOG(VERBOSE)
393 << "Camera data received but no camera controller is available";
394 }
395 }
396
397 private:
398 cuttlefish::InputSockets& input_sockets_;
399 cuttlefish::KernelLogEventsHandler* kernel_log_events_handler_;
400 int kernel_log_subscription_id_ = -1;
401 std::shared_ptr<cuttlefish::webrtc_streaming::AdbHandler> adb_handler_;
402 std::shared_ptr<cuttlefish::webrtc_streaming::BluetoothHandler>
403 bluetooth_handler_;
404 std::shared_ptr<cuttlefish::webrtc_streaming::LocationHandler>
405 location_handler_;
406 std::shared_ptr<cuttlefish::webrtc_streaming::KmlLocationsHandler>
407 kml_locations_handler_;
408 std::shared_ptr<cuttlefish::webrtc_streaming::GpxLocationsHandler>
409 gpx_locations_handler_;
410 std::map<std::string, cuttlefish::SharedFD> commands_to_custom_action_servers_;
411 std::weak_ptr<DisplayHandler> weak_display_handler_;
412 std::set<int32_t> active_touch_slots_;
413 cuttlefish::CameraController *camera_controller_;
414 cuttlefish::confui::HostVirtualInput &confui_input_;
415 };
416
CfConnectionObserverFactory(cuttlefish::InputSockets & input_sockets,cuttlefish::KernelLogEventsHandler * kernel_log_events_handler,cuttlefish::confui::HostVirtualInput & confui_input)417 CfConnectionObserverFactory::CfConnectionObserverFactory(
418 cuttlefish::InputSockets &input_sockets,
419 cuttlefish::KernelLogEventsHandler* kernel_log_events_handler,
420 cuttlefish::confui::HostVirtualInput &confui_input)
421 : input_sockets_(input_sockets),
422 kernel_log_events_handler_(kernel_log_events_handler),
423 confui_input_{confui_input} {}
424
425 std::shared_ptr<cuttlefish::webrtc_streaming::ConnectionObserver>
CreateObserver()426 CfConnectionObserverFactory::CreateObserver() {
427 return std::shared_ptr<cuttlefish::webrtc_streaming::ConnectionObserver>(
428 new ConnectionObserverImpl(input_sockets_, kernel_log_events_handler_,
429 commands_to_custom_action_servers_,
430 weak_display_handler_, camera_controller_,
431 confui_input_));
432 }
433
AddCustomActionServer(cuttlefish::SharedFD custom_action_server_fd,const std::vector<std::string> & commands)434 void CfConnectionObserverFactory::AddCustomActionServer(
435 cuttlefish::SharedFD custom_action_server_fd,
436 const std::vector<std::string>& commands) {
437 for (const std::string& command : commands) {
438 LOG(DEBUG) << "Action server is listening to command: " << command;
439 commands_to_custom_action_servers_[command] = custom_action_server_fd;
440 }
441 }
442
SetDisplayHandler(std::weak_ptr<DisplayHandler> display_handler)443 void CfConnectionObserverFactory::SetDisplayHandler(
444 std::weak_ptr<DisplayHandler> display_handler) {
445 weak_display_handler_ = display_handler;
446 }
447
SetCameraHandler(CameraController * controller)448 void CfConnectionObserverFactory::SetCameraHandler(
449 CameraController *controller) {
450 camera_controller_ = controller;
451 }
452 } // namespace cuttlefish
453