1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 18 #define ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 19 20 #include <queue> 21 22 #include <gui/IProducerListener.h> 23 #include <gui/CpuConsumer.h> 24 25 #include <media/hardware/VideoAPI.h> 26 #include <media/MediaCodecBuffer.h> 27 #include <media/stagefright/foundation/ALooper.h> 28 #include <media/stagefright/foundation/AMessage.h> 29 #include <media/stagefright/MediaCodec.h> 30 #include <media/stagefright/MediaMuxer.h> 31 32 #include "CompositeStream.h" 33 34 namespace android { 35 namespace camera3 { 36 37 class HeicCompositeStream : public CompositeStream, public Thread, 38 public CpuConsumer::FrameAvailableListener { 39 public: 40 HeicCompositeStream(sp<CameraDeviceBase> device, 41 wp<hardware::camera2::ICameraDeviceCallbacks> cb); 42 ~HeicCompositeStream() override; 43 44 static bool isHeicCompositeStream(const sp<Surface> &surface); 45 static bool isHeicCompositeStreamInfo(const OutputStreamInfo& streamInfo); 46 47 status_t createInternalStreams(const std::vector<sp<Surface>>& consumers, 48 bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, 49 camera_stream_rotation_t rotation, int *id, const std::string& physicalCameraId, 50 const std::unordered_set<int32_t> &sensorPixelModesUsed, 51 std::vector<int> *surfaceIds, int streamSetId, bool isShared, int32_t colorSpace, 52 int64_t dynamicProfile, int64_t streamUseCase, bool useReadoutTimestamp) override; 53 54 status_t deleteInternalStreams() override; 55 56 status_t configureStream() override; 57 58 status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector<int32_t>* /*out*/outputStreamIds, 59 int32_t* /*out*/currentStreamId) override; 60 61 status_t insertCompositeStreamIds(std::vector<int32_t>* compositeStreamIds /*out*/) override; 62 63 void onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) override; 64 getStreamId()65 int getStreamId() override { return mMainImageStreamId; } 66 67 // Use onShutter to keep track of frame number <-> timestamp mapping. 68 void onBufferReleased(const BufferInfo& bufferInfo) override; 69 void onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId, 70 const CameraMetadata& settings) override; 71 72 // CpuConsumer listener implementation 73 void onFrameAvailable(const BufferItem& item) override; 74 75 // Return stream information about the internal camera streams 76 static status_t getCompositeStreamInfo(const OutputStreamInfo &streamInfo, 77 const CameraMetadata& ch, std::vector<OutputStreamInfo>* compositeOutput /*out*/); 78 79 // Get composite stream stats getStreamStats(hardware::CameraStreamStats *)80 void getStreamStats(hardware::CameraStreamStats*) override {}; 81 82 static bool isSizeSupportedByHeifEncoder(int32_t width, int32_t height, 83 bool* useHeic, bool* useGrid, int64_t* stall, AString* hevcName = nullptr); 84 static bool isInMemoryTempFileSupported(); 85 protected: 86 87 bool threadLoop() override; 88 bool onStreamBufferError(const CaptureResultExtras& resultExtras) override; 89 void onResultError(const CaptureResultExtras& resultExtras) override; 90 void onRequestError(const CaptureResultExtras& resultExtras) override; 91 92 private: 93 // 94 // HEIC/HEVC Codec related structures, utility functions, and callbacks 95 // 96 struct CodecOutputBufferInfo { 97 int32_t index; 98 int32_t offset; 99 int32_t size; 100 int64_t timeUs; 101 uint32_t flags; 102 }; 103 104 struct CodecInputBufferInfo { 105 int32_t index; 106 int64_t timeUs; 107 size_t tileIndex; 108 }; 109 110 class CodecCallbackHandler : public AHandler { 111 public: CodecCallbackHandler(wp<HeicCompositeStream> parent)112 explicit CodecCallbackHandler(wp<HeicCompositeStream> parent) { 113 mParent = parent; 114 } 115 virtual void onMessageReceived(const sp<AMessage> &msg); 116 private: 117 wp<HeicCompositeStream> mParent; 118 }; 119 120 enum { 121 kWhatCallbackNotify, 122 }; 123 124 bool mUseHeic; 125 sp<MediaCodec> mCodec; 126 sp<ALooper> mCodecLooper, mCallbackLooper; 127 sp<CodecCallbackHandler> mCodecCallbackHandler; 128 sp<AMessage> mAsyncNotify; 129 sp<AMessage> mFormat; 130 size_t mNumOutputTiles; 131 132 int32_t mOutputWidth, mOutputHeight; 133 size_t mMaxHeicBufferSize; 134 int32_t mGridWidth, mGridHeight; 135 size_t mGridRows, mGridCols; 136 bool mUseGrid; // Whether to use framework YUV frame tiling. 137 138 static const int64_t kNoFrameDropMaxPtsGap = -1000000; 139 static const int32_t kNoGridOpRate = 30; 140 static const int32_t kGridOpRate = 120; 141 142 void onHeicOutputFrameAvailable(const CodecOutputBufferInfo& bufferInfo); 143 void onHeicInputFrameAvailable(int32_t index); // Only called for YUV input mode. 144 void onHeicFormatChanged(sp<AMessage>& newFormat); 145 void onHeicCodecError(); 146 147 status_t initializeCodec(uint32_t width, uint32_t height, 148 const sp<CameraDeviceBase>& cameraDevice); 149 void deinitCodec(); 150 151 // 152 // Composite stream related structures, utility functions and callbacks. 153 // 154 struct InputFrame { 155 int32_t orientation; 156 int32_t quality; 157 158 CpuConsumer::LockedBuffer appSegmentBuffer; 159 std::vector<CodecOutputBufferInfo> codecOutputBuffers; 160 std::unique_ptr<CameraMetadata> result; 161 162 // Fields that are only applicable to HEVC tiling. 163 CpuConsumer::LockedBuffer yuvBuffer; 164 std::vector<CodecInputBufferInfo> codecInputBuffers; 165 166 bool error; // Main input image buffer error 167 bool exifError; // Exif/APP_SEGMENT buffer error 168 int64_t timestamp; 169 int32_t requestId; 170 171 sp<AMessage> format; 172 sp<MediaMuxer> muxer; 173 int fenceFd; 174 int fileFd; 175 ssize_t trackIndex; 176 ANativeWindowBuffer *anb; 177 178 bool appSegmentWritten; 179 size_t pendingOutputTiles; 180 size_t codecInputCounter; 181 InputFrameInputFrame182 InputFrame() : orientation(0), quality(kDefaultJpegQuality), error(false), 183 exifError(false), timestamp(-1), requestId(-1), fenceFd(-1), 184 fileFd(-1), trackIndex(-1), anb(nullptr), appSegmentWritten(false), 185 pendingOutputTiles(0), codecInputCounter(0) { } 186 }; 187 188 void compilePendingInputLocked(); 189 // Find first complete and valid frame with smallest frame number 190 bool getNextReadyInputLocked(int64_t *frameNumber /*out*/); 191 // Find next failing frame number with smallest frame number and return respective frame number 192 int64_t getNextFailingInputLocked(); 193 194 status_t processInputFrame(int64_t frameNumber, InputFrame &inputFrame); 195 status_t processCodecInputFrame(InputFrame &inputFrame); 196 status_t startMuxerForInputFrame(int64_t frameNumber, InputFrame &inputFrame); 197 status_t processAppSegment(int64_t frameNumber, InputFrame &inputFrame); 198 status_t processOneCodecOutputFrame(int64_t frameNumber, InputFrame &inputFrame); 199 status_t processCompletedInputFrame(int64_t frameNumber, InputFrame &inputFrame); 200 201 void releaseInputFrameLocked(int64_t frameNumber, InputFrame *inputFrame /*out*/); 202 void releaseInputFramesLocked(); 203 204 size_t findAppSegmentsSize(const uint8_t* appSegmentBuffer, size_t maxSize, 205 size_t* app1SegmentSize); 206 status_t copyOneYuvTile(sp<MediaCodecBuffer>& codecBuffer, 207 const CpuConsumer::LockedBuffer& yuvBuffer, 208 size_t top, size_t left, size_t width, size_t height); 209 void initCopyRowFunction(int32_t width); 210 static size_t calcAppSegmentMaxSize(const CameraMetadata& info); 211 void updateCodecQualityLocked(int32_t quality); 212 213 static const nsecs_t kWaitDuration = 10000000; // 10 ms 214 static const int32_t kDefaultJpegQuality = 99; 215 static const auto kJpegDataSpace = HAL_DATASPACE_V0_JFIF; 216 static const android_dataspace kAppSegmentDataSpace = 217 static_cast<android_dataspace>(HAL_DATASPACE_JPEG_APP_SEGMENTS); 218 static const android_dataspace kHeifDataSpace = 219 static_cast<android_dataspace>(HAL_DATASPACE_HEIF); 220 // Use the limit of pipeline depth in the API sepc as maximum number of acquired 221 // app segment buffers. 222 static const uint32_t kMaxAcquiredAppSegment = 8; 223 224 int mAppSegmentStreamId, mAppSegmentSurfaceId; 225 sp<CpuConsumer> mAppSegmentConsumer; 226 sp<Surface> mAppSegmentSurface; 227 size_t mAppSegmentMaxSize; 228 std::queue<int64_t> mAppSegmentFrameNumbers; 229 CameraMetadata mStaticInfo; 230 231 int mMainImageStreamId, mMainImageSurfaceId; 232 sp<Surface> mMainImageSurface; 233 sp<CpuConsumer> mMainImageConsumer; // Only applicable for HEVC codec. 234 bool mYuvBufferAcquired; // Only applicable to HEVC codec 235 std::queue<int64_t> mMainImageFrameNumbers; 236 237 static const int32_t kMaxOutputSurfaceProducerCount = 1; 238 sp<Surface> mOutputSurface; 239 sp<ProducerListener> mProducerListener; 240 int32_t mDequeuedOutputBufferCnt; 241 242 // Map from frame number to JPEG setting of orientation+quality 243 struct HeicSettings { 244 int32_t orientation; 245 int32_t quality; 246 int64_t timestamp; 247 int32_t requestId; 248 bool shutterNotified; 249 HeicSettingsHeicSettings250 HeicSettings() : orientation(0), quality(95), timestamp(0), 251 requestId(-1), shutterNotified(false) {} HeicSettingsHeicSettings252 HeicSettings(int32_t _orientation, int32_t _quality) : 253 orientation(_orientation), 254 quality(_quality), timestamp(0), 255 requestId(-1), shutterNotified(false) {} 256 257 }; 258 std::map<int64_t, HeicSettings> mSettingsByFrameNumber; 259 260 // Keep all incoming APP segment Blob buffer pending further processing. 261 std::vector<int64_t> mInputAppSegmentBuffers; 262 263 // Keep all incoming HEIC blob buffer pending further processing. 264 std::vector<CodecOutputBufferInfo> mCodecOutputBuffers; 265 std::queue<int64_t> mCodecOutputBufferFrameNumbers; 266 size_t mCodecOutputCounter; 267 int32_t mQuality; 268 269 // Keep all incoming Yuv buffer pending tiling and encoding (for HEVC YUV tiling only) 270 std::vector<int64_t> mInputYuvBuffers; 271 // Keep all codec input buffers ready to be filled out (for HEVC YUV tiling only) 272 std::vector<int32_t> mCodecInputBuffers; 273 274 // Artificial strictly incremental YUV grid timestamp to make encoder happy. 275 int64_t mGridTimestampUs; 276 277 // Indexed by frame number. In most common use case, entries are accessed in order. 278 std::map<int64_t, InputFrame> mPendingInputFrames; 279 280 // Function pointer of libyuv row copy. 281 void (*mFnCopyRow)(const uint8_t* src, uint8_t* dst, int width); 282 283 // A set of APP_SEGMENT error frame numbers 284 std::set<int64_t> mExifErrorFrameNumbers; 285 void flagAnExifErrorFrameNumber(int64_t frameNumber); 286 287 // The status id for tracking the active/idle status of this composite stream 288 int mStatusId; 289 void markTrackerIdle(); 290 }; 291 292 }; // namespace camera3 293 }; // namespace android 294 295 #endif //ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 296