1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <cassert> 20 #include <chrono> 21 #include <cstdint> 22 #include <functional> 23 #include <memory> 24 #include <mutex> 25 #include <optional> 26 #include <string> 27 #include <thread> 28 #include <type_traits> 29 30 #include <android-base/logging.h> 31 #include "common/libs/concurrency/semaphore.h" 32 #include "common/libs/confui/confui.h" 33 #include "common/libs/fs/shared_fd.h" 34 #include "common/libs/utils/size_utils.h" 35 36 #include "host/libs/config/cuttlefish_config.h" 37 #include "host/libs/confui/host_mode_ctrl.h" 38 #include "host/libs/confui/host_utils.h" 39 #include "host/libs/screen_connector/screen_connector_common.h" 40 #include "host/libs/screen_connector/screen_connector_queue.h" 41 #include "host/libs/screen_connector/wayland_screen_connector.h" 42 43 namespace cuttlefish { 44 45 template <typename ProcessedFrameType> 46 class ScreenConnector : public ScreenConnectorInfo, 47 public ScreenConnectorFrameRenderer { 48 public: 49 static_assert(cuttlefish::is_movable<ProcessedFrameType>::value, 50 "ProcessedFrameType should be std::move-able."); 51 static_assert(std::is_base_of<ScreenConnectorFrameInfo, ProcessedFrameType>::value, 52 "ProcessedFrameType should inherit ScreenConnectorFrameInfo"); 53 54 /** 55 * This is the type of the callback function WebRTC/VNC is supposed to provide 56 * ScreenConnector with. 57 * 58 * The callback function should be defined so that the two parameters are 59 * given by the callback function caller (e.g. ScreenConnectorSource) and used 60 * to fill out the ProcessedFrameType object, msg. 61 * 62 * The ProcessedFrameType object is internally created by ScreenConnector, 63 * filled out by the ScreenConnectorSource, and returned via OnNextFrame() 64 * call. 65 */ 66 using GenerateProcessedFrameCallback = std::function<void( 67 std::uint32_t /*display_number*/, std::uint8_t* /*frame_pixels*/, 68 /* ScImpl enqueues this type into the Q */ 69 ProcessedFrameType& msg)>; 70 Get(const int frames_fd,HostModeCtrl & host_mode_ctrl)71 static std::unique_ptr<ScreenConnector<ProcessedFrameType>> Get( 72 const int frames_fd, HostModeCtrl& host_mode_ctrl) { 73 auto config = cuttlefish::CuttlefishConfig::Get(); 74 ScreenConnector<ProcessedFrameType>* raw_ptr = nullptr; 75 if (config->gpu_mode() == cuttlefish::kGpuModeDrmVirgl || 76 config->gpu_mode() == cuttlefish::kGpuModeGfxStream || 77 config->gpu_mode() == cuttlefish::kGpuModeGuestSwiftshader) { 78 raw_ptr = new ScreenConnector<ProcessedFrameType>( 79 std::make_unique<WaylandScreenConnector>(frames_fd), host_mode_ctrl); 80 } else { 81 LOG(FATAL) << "Invalid gpu mode: " << config->gpu_mode(); 82 } 83 return std::unique_ptr<ScreenConnector<ProcessedFrameType>>(raw_ptr); 84 } 85 86 virtual ~ScreenConnector() = default; 87 88 /** 89 * set the callback function to be eventually used by Wayland/Socket-Based Connectors 90 * 91 * @param[in] To tell how ScreenConnectorSource caches the frame & meta info 92 */ SetCallback(GenerateProcessedFrameCallback && frame_callback)93 void SetCallback(GenerateProcessedFrameCallback&& frame_callback) { 94 std::lock_guard<std::mutex> lock(streamer_callback_mutex_); 95 callback_from_streamer_ = std::move(frame_callback); 96 streamer_callback_set_cv_.notify_all(); 97 /* 98 * the first WaitForAtLeastOneClientConnection() call from VNC requires the 99 * Android-frame-processing thread starts beforehands (b/178504150) 100 */ 101 if (!sc_android_frame_fetching_thread_.joinable()) { 102 sc_android_frame_fetching_thread_ = cuttlefish::confui::thread::RunThread( 103 "AndroidFetcher", &ScreenConnector::AndroidFrameFetchingLoop, this); 104 } 105 } 106 IsCallbackSet()107 bool IsCallbackSet() const override { 108 if (callback_from_streamer_) { 109 return true; 110 } 111 return false; 112 } 113 114 /* returns the processed frame that also includes meta-info such as success/fail 115 * and display number from the guest 116 * 117 * NOTE THAT THIS IS THE ONLY CONSUMER OF THE TWO QUEUES 118 */ OnNextFrame()119 ProcessedFrameType OnNextFrame() { 120 on_next_frame_cnt_++; 121 while (true) { 122 ConfUiLog(VERBOSE) << "Streamer waiting Semaphore with host ctrl mode =" 123 << static_cast<std::uint32_t>( 124 host_mode_ctrl_.GetMode()) 125 << " and cnd = #" << on_next_frame_cnt_; 126 sc_sem_.SemWait(); 127 ConfUiLog(VERBOSE) 128 << "Streamer got Semaphore'ed resources with host ctrl mode =" 129 << static_cast<std::uint32_t>(host_mode_ctrl_.GetMode()) 130 << "and cnd = #" << on_next_frame_cnt_; 131 // do something 132 if (!sc_android_queue_.Empty()) { 133 auto mode = host_mode_ctrl_.GetMode(); 134 if (mode == HostModeCtrl::ModeType::kAndroidMode) { 135 ConfUiLog(VERBOSE) 136 << "Streamer gets Android frame with host ctrl mode =" 137 << static_cast<std::uint32_t>(mode) << "and cnd = #" 138 << on_next_frame_cnt_; 139 return sc_android_queue_.PopFront(); 140 } 141 // AndroidFrameFetchingLoop could have added 1 or 2 frames 142 // before it becomes Conf UI mode. 143 ConfUiLog(VERBOSE) 144 << "Streamer ignores Android frame with host ctrl mode =" 145 << static_cast<std::uint32_t>(mode) << "and cnd = #" 146 << on_next_frame_cnt_; 147 sc_android_queue_.PopFront(); 148 continue; 149 } 150 ConfUiLog(VERBOSE) << "Streamer gets Conf UI frame with host ctrl mode = " 151 << static_cast<std::uint32_t>( 152 host_mode_ctrl_.GetMode()) 153 << " and cnd = #" << on_next_frame_cnt_; 154 return sc_confui_queue_.PopFront(); 155 } 156 } 157 AndroidFrameFetchingLoop()158 [[noreturn]] void AndroidFrameFetchingLoop() { 159 unsigned long long int loop_cnt = 0; 160 cuttlefish::confui::thread::Set("AndroidFrameFetcher", 161 std::this_thread::get_id()); 162 while (true) { 163 loop_cnt++; 164 ProcessedFrameType processed_frame; 165 decltype(callback_from_streamer_) cp_of_streamer_callback; 166 { 167 std::lock_guard<std::mutex> lock(streamer_callback_mutex_); 168 cp_of_streamer_callback = callback_from_streamer_; 169 } 170 GenerateProcessedFrameCallbackImpl callback_for_sc_impl = 171 std::bind(cp_of_streamer_callback, std::placeholders::_1, 172 std::placeholders::_2, std::ref(processed_frame)); 173 ConfUiLog(VERBOSE) << cuttlefish::confui::thread::GetName( 174 std::this_thread::get_id()) 175 << " calling Android OnNextFrame. " 176 << " at loop #" << loop_cnt; 177 bool flag = sc_android_src_->OnNextFrame(callback_for_sc_impl); 178 processed_frame.is_success_ = flag && processed_frame.is_success_; 179 const bool is_confui_mode = host_mode_ctrl_.IsConfirmatioUiMode(); 180 if (!is_confui_mode) { 181 ConfUiLog(VERBOSE) << cuttlefish::confui::thread::GetName( 182 std::this_thread::get_id()) 183 << "is sending an Android Frame at loop_cnt #" 184 << loop_cnt; 185 sc_android_queue_.PushBack(std::move(processed_frame)); 186 continue; 187 } 188 ConfUiLog(VERBOSE) << cuttlefish::confui::thread::GetName( 189 std::this_thread::get_id()) 190 << "is skipping an Android Frame at loop_cnt #" 191 << loop_cnt; 192 } 193 } 194 195 /** 196 * ConfUi calls this when it has frames to render 197 * 198 * This won't be called if not by Confirmation UI. This won't affect rendering 199 * Android guest frames if Confirmation UI HAL is not active. 200 * 201 */ RenderConfirmationUi(const std::uint32_t display,std::uint8_t * raw_frame)202 bool RenderConfirmationUi(const std::uint32_t display, 203 std::uint8_t* raw_frame) override { 204 render_confui_cnt_++; 205 // wait callback is not set, the streamer is not ready 206 // return with LOG(ERROR) 207 if (!IsCallbackSet()) { 208 ConfUiLog(ERROR) << "callback function to process frames is not yet set"; 209 return false; 210 } 211 ProcessedFrameType processed_frame; 212 auto this_thread_name = cuttlefish::confui::thread::GetName(); 213 ConfUiLog(DEBUG) << this_thread_name 214 << "is sending a #" + std::to_string(render_confui_cnt_) 215 << "Conf UI frame"; 216 callback_from_streamer_(display, raw_frame, processed_frame); 217 // now add processed_frame to the queue 218 sc_confui_queue_.PushBack(std::move(processed_frame)); 219 return true; 220 } 221 222 // Let the screen connector know when there are clients connected ReportClientsConnected(bool have_clients)223 void ReportClientsConnected(bool have_clients) { 224 // screen connector implementation must implement ReportClientsConnected 225 sc_android_src_->ReportClientsConnected(have_clients); 226 return ; 227 } 228 229 protected: 230 template <typename T, 231 typename = std::enable_if_t< 232 std::is_base_of<ScreenConnectorSource, T>::value, void>> ScreenConnector(std::unique_ptr<T> && impl,HostModeCtrl & host_mode_ctrl)233 ScreenConnector(std::unique_ptr<T>&& impl, HostModeCtrl& host_mode_ctrl) 234 : sc_android_src_{std::move(impl)}, 235 host_mode_ctrl_{host_mode_ctrl}, 236 on_next_frame_cnt_{0}, 237 render_confui_cnt_{0}, 238 sc_android_queue_{sc_sem_}, 239 sc_confui_queue_{sc_sem_} {} 240 ScreenConnector() = delete; 241 242 private: 243 // either socket_based or wayland 244 std::unique_ptr<ScreenConnectorSource> sc_android_src_; 245 HostModeCtrl& host_mode_ctrl_; 246 unsigned long long int on_next_frame_cnt_; 247 unsigned long long int render_confui_cnt_; 248 Semaphore sc_sem_; 249 ScreenConnectorQueue<ProcessedFrameType> sc_android_queue_; 250 ScreenConnectorQueue<ProcessedFrameType> sc_confui_queue_; 251 GenerateProcessedFrameCallback callback_from_streamer_; 252 std::thread sc_android_frame_fetching_thread_; 253 std::mutex streamer_callback_mutex_; // mutex to set & read callback_from_streamer_ 254 std::condition_variable streamer_callback_set_cv_; 255 }; 256 257 } // namespace cuttlefish 258