1 /* 2 * Copyright (C) 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERARENDERTHREAD_H 18 #define ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERARENDERTHREAD_H 19 20 #include <atomic> 21 #include <cstdint> 22 #include <deque> 23 #include <future> 24 #include <memory> 25 #include <thread> 26 #include <variant> 27 #include <vector> 28 29 #include "VirtualCameraDevice.h" 30 #include "VirtualCameraSessionContext.h" 31 #include "aidl/android/hardware/camera/device/CameraMetadata.h" 32 #include "aidl/android/hardware/camera/device/ICameraDeviceCallback.h" 33 #include "android/binder_auto_utils.h" 34 #include "util/EglDisplayContext.h" 35 #include "util/EglFramebuffer.h" 36 #include "util/EglProgram.h" 37 #include "util/EglSurfaceTexture.h" 38 #include "util/Util.h" 39 40 namespace android { 41 namespace companion { 42 namespace virtualcamera { 43 44 // Represents single output buffer of capture request. 45 class CaptureRequestBuffer { 46 public: 47 CaptureRequestBuffer(int streamId, int bufferId, sp<Fence> fence = nullptr); 48 49 int getStreamId() const; 50 int getBufferId() const; 51 sp<Fence> getFence() const; 52 53 private: 54 const int mStreamId; 55 const int mBufferId; 56 const sp<Fence> mFence; 57 }; 58 59 struct RequestSettings { 60 int jpegQuality = VirtualCameraDevice::kDefaultJpegQuality; 61 int jpegOrientation = VirtualCameraDevice::kDefaultJpegOrientation; 62 Resolution thumbnailResolution = Resolution(0, 0); 63 int thumbnailJpegQuality = VirtualCameraDevice::kDefaultJpegQuality; 64 std::optional<FpsRange> fpsRange; 65 camera_metadata_enum_android_control_capture_intent_t captureIntent = 66 VirtualCameraDevice::kDefaultCaptureIntent; 67 std::optional<GpsCoordinates> gpsCoordinates; 68 std::optional<camera_metadata_enum_android_control_ae_precapture_trigger> 69 aePrecaptureTrigger; 70 }; 71 72 // Represents single capture request to fill set of buffers. 73 class ProcessCaptureRequestTask { 74 public: 75 ProcessCaptureRequestTask( 76 int frameNumber, const std::vector<CaptureRequestBuffer>& requestBuffers, 77 const RequestSettings& RequestSettings = {}); 78 79 // Returns frame number corresponding to the request. 80 int getFrameNumber() const; 81 82 // Get reference to vector describing output buffers corresponding 83 // to this request. 84 // 85 // Note that the vector is owned by the ProcessCaptureRequestTask instance, 86 // so it cannot be access outside of its lifetime. 87 const std::vector<CaptureRequestBuffer>& getBuffers() const; 88 89 const RequestSettings& getRequestSettings() const; 90 91 private: 92 const int mFrameNumber; 93 const std::vector<CaptureRequestBuffer> mBuffers; 94 const RequestSettings mRequestSettings; 95 }; 96 97 struct UpdateTextureTask {}; 98 99 struct RenderThreadTask 100 : public std::variant<std::unique_ptr<ProcessCaptureRequestTask>, 101 UpdateTextureTask> { 102 // Allow implicit conversion to bool. 103 // 104 // Returns false, if the RenderThreadTask consist of null 105 // ProcessCaptureRequestTask, which signals that the thread should terminate. 106 operator bool() const { 107 const bool isExitSignal = 108 std::holds_alternative<std::unique_ptr<ProcessCaptureRequestTask>>( 109 *this) && 110 std::get<std::unique_ptr<ProcessCaptureRequestTask>>(*this) == nullptr; 111 return !isExitSignal; 112 } 113 }; 114 115 // Wraps dedicated rendering thread and rendering business with corresponding 116 // input surface. 117 class VirtualCameraRenderThread { 118 public: 119 // Create VirtualCameraRenderThread instance: 120 // * sessionContext - VirtualCameraSessionContext reference for shared access 121 // to mapped buffers. 122 // * inputSurfaceSize - requested size of input surface. 123 // * reportedSensorSize - reported static sensor size of virtual camera. 124 // * cameraDeviceCallback - callback for corresponding camera instance 125 // * testMode - when set to true, test pattern is rendered to input surface 126 // before each capture request is processed to simulate client input. 127 VirtualCameraRenderThread( 128 VirtualCameraSessionContext& sessionContext, Resolution inputSurfaceSize, 129 Resolution reportedSensorSize, 130 std::shared_ptr< 131 ::aidl::android::hardware::camera::device::ICameraDeviceCallback> 132 cameraDeviceCallback); 133 134 ~VirtualCameraRenderThread(); 135 136 // Start rendering thread. 137 void start(); 138 // Stop rendering thread. 139 void stop(); 140 141 // Send request to render thread to update the texture. 142 // Currently queued buffers in the input surface will be consumed and the most 143 // recent buffer in the input surface will be attached to the texture), all 144 // other buffers will be returned to the buffer queue. 145 void requestTextureUpdate() EXCLUDES(mLock); 146 147 // Equeue capture task for processing on render thread. 148 void enqueueTask(std::unique_ptr<ProcessCaptureRequestTask> task) 149 EXCLUDES(mLock); 150 151 // Flush all in-flight requests. 152 void flush() EXCLUDES(mLock); 153 154 // Returns input surface corresponding to "virtual camera sensor". 155 sp<Surface> getInputSurface(); 156 157 private: 158 RenderThreadTask dequeueTask() EXCLUDES(mLock); 159 160 // Rendering thread entry point. 161 void threadLoop(); 162 163 // Process single capture request task (always called on render thread). 164 void processTask(const ProcessCaptureRequestTask& captureRequestTask); 165 166 // Flush single capture request task returning the error status immediately. 167 void flushCaptureRequest(const ProcessCaptureRequestTask& captureRequestTask); 168 169 // TODO(b/301023410) - Refactor the actual rendering logic off this class for 170 // easier testability. 171 172 // Create thumbnail with specified size for current image. 173 // The compressed image size is limited by 32KiB. 174 // Returns vector with compressed thumbnail if successful, 175 // empty vector otherwise. 176 std::vector<uint8_t> createThumbnail(Resolution resolution, int quality); 177 178 // Render current image to the BLOB buffer. 179 // If fence is specified, this function will block until the fence is cleared 180 // before writing to the buffer. 181 // Always called on render thread. 182 ndk::ScopedAStatus renderIntoBlobStreamBuffer( 183 const int streamId, const int bufferId, 184 const ::aidl::android::hardware::camera::device::CameraMetadata& 185 resultMetadata, 186 const RequestSettings& requestSettings, sp<Fence> fence = nullptr); 187 188 // Render current image to the YCbCr buffer. 189 // If fence is specified, this function will block until the fence is cleared 190 // before writing to the buffer. 191 // Always called on render thread. 192 ndk::ScopedAStatus renderIntoImageStreamBuffer(int streamId, int bufferId, 193 sp<Fence> fence = nullptr); 194 195 // Render current image into provided EglFramebuffer. 196 // If fence is specified, this function will block until the fence is cleared 197 // before writing to the buffer. 198 // Always called on the render thread. 199 ndk::ScopedAStatus renderIntoEglFramebuffer( 200 EglFrameBuffer& framebuffer, sp<Fence> fence = nullptr, 201 std::optional<Rect> viewport = std::nullopt); 202 203 // Camera callback 204 const std::shared_ptr< 205 ::aidl::android::hardware::camera::device::ICameraDeviceCallback> 206 mCameraDeviceCallback; 207 208 const Resolution mInputSurfaceSize; 209 const Resolution mReportedSensorSize; 210 211 VirtualCameraSessionContext& mSessionContext; 212 213 std::thread mThread; 214 215 // Blocking queue implementation. 216 std::mutex mLock; 217 std::deque<std::unique_ptr<ProcessCaptureRequestTask>> mQueue GUARDED_BY(mLock); 218 std::condition_variable mCondVar; 219 volatile bool mTextureUpdateRequested GUARDED_BY(mLock); 220 volatile bool mPendingExit GUARDED_BY(mLock); 221 222 // Acquisition timestamp of last frame. 223 std::atomic<uint64_t> mLastAcquisitionTimestampNanoseconds; 224 225 // EGL helpers - constructed and accessed only from rendering thread. 226 std::unique_ptr<EglDisplayContext> mEglDisplayContext; 227 std::unique_ptr<EglTextureProgram> mEglTextureYuvProgram; 228 std::unique_ptr<EglTextureProgram> mEglTextureRgbProgram; 229 std::unique_ptr<EglSurfaceTexture> mEglSurfaceTexture; 230 231 std::promise<sp<Surface>> mInputSurfacePromise; 232 std::shared_future<sp<Surface>> mInputSurfaceFuture; 233 }; 234 235 } // namespace virtualcamera 236 } // namespace companion 237 } // namespace android 238 239 #endif // ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERARENDERTHREAD_H 240