1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 18 #define ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 19 20 #include <queue> 21 22 #include <gui/IProducerListener.h> 23 #include <gui/CpuConsumer.h> 24 25 #include <media/hardware/VideoAPI.h> 26 #include <media/MediaCodecBuffer.h> 27 #include <media/stagefright/foundation/ALooper.h> 28 #include <media/stagefright/foundation/AMessage.h> 29 #include <media/stagefright/MediaCodec.h> 30 #include <media/stagefright/MediaMuxer.h> 31 32 #include "CompositeStream.h" 33 34 namespace android { 35 namespace camera3 { 36 37 class HeicCompositeStream : public CompositeStream, public Thread, 38 public CpuConsumer::FrameAvailableListener { 39 public: 40 HeicCompositeStream(sp<CameraDeviceBase> device, 41 wp<hardware::camera2::ICameraDeviceCallbacks> cb); 42 ~HeicCompositeStream() override; 43 44 static bool isHeicCompositeStream(const sp<Surface> &surface); 45 46 status_t createInternalStreams(const std::vector<sp<Surface>>& consumers, 47 bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, 48 camera_stream_rotation_t rotation, int *id, const String8& physicalCameraId, 49 const std::unordered_set<int32_t> &sensorPixelModesUsed, 50 std::vector<int> *surfaceIds, int streamSetId, bool isShared, int32_t colorSpace, 51 int64_t dynamicProfile, int64_t streamUseCase, bool useReadoutTimestamp) override; 52 53 status_t deleteInternalStreams() override; 54 55 status_t configureStream() override; 56 57 status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector<int32_t>* /*out*/outputStreamIds, 58 int32_t* /*out*/currentStreamId) override; 59 60 status_t insertCompositeStreamIds(std::vector<int32_t>* compositeStreamIds /*out*/) override; 61 62 void onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) override; 63 getStreamId()64 int getStreamId() override { return mMainImageStreamId; } 65 66 // Use onShutter to keep track of frame number <-> timestamp mapping. 67 void onBufferReleased(const BufferInfo& bufferInfo) override; 68 void onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId, 69 const CameraMetadata& settings) override; 70 71 // CpuConsumer listener implementation 72 void onFrameAvailable(const BufferItem& item) override; 73 74 // Return stream information about the internal camera streams 75 static status_t getCompositeStreamInfo(const OutputStreamInfo &streamInfo, 76 const CameraMetadata& ch, std::vector<OutputStreamInfo>* compositeOutput /*out*/); 77 78 // Get composite stream stats getStreamStats(hardware::CameraStreamStats *)79 void getStreamStats(hardware::CameraStreamStats*) override {}; 80 81 static bool isSizeSupportedByHeifEncoder(int32_t width, int32_t height, 82 bool* useHeic, bool* useGrid, int64_t* stall, AString* hevcName = nullptr); 83 static bool isInMemoryTempFileSupported(); 84 protected: 85 86 bool threadLoop() override; 87 bool onStreamBufferError(const CaptureResultExtras& resultExtras) override; 88 void onResultError(const CaptureResultExtras& resultExtras) override; 89 void onRequestError(const CaptureResultExtras& resultExtras) override; 90 91 private: 92 // 93 // HEIC/HEVC Codec related structures, utility functions, and callbacks 94 // 95 struct CodecOutputBufferInfo { 96 int32_t index; 97 int32_t offset; 98 int32_t size; 99 int64_t timeUs; 100 uint32_t flags; 101 }; 102 103 struct CodecInputBufferInfo { 104 int32_t index; 105 int64_t timeUs; 106 size_t tileIndex; 107 }; 108 109 class CodecCallbackHandler : public AHandler { 110 public: CodecCallbackHandler(wp<HeicCompositeStream> parent)111 explicit CodecCallbackHandler(wp<HeicCompositeStream> parent) { 112 mParent = parent; 113 } 114 virtual void onMessageReceived(const sp<AMessage> &msg); 115 private: 116 wp<HeicCompositeStream> mParent; 117 }; 118 119 enum { 120 kWhatCallbackNotify, 121 }; 122 123 bool mUseHeic; 124 sp<MediaCodec> mCodec; 125 sp<ALooper> mCodecLooper, mCallbackLooper; 126 sp<CodecCallbackHandler> mCodecCallbackHandler; 127 sp<AMessage> mAsyncNotify; 128 sp<AMessage> mFormat; 129 size_t mNumOutputTiles; 130 131 int32_t mOutputWidth, mOutputHeight; 132 size_t mMaxHeicBufferSize; 133 int32_t mGridWidth, mGridHeight; 134 size_t mGridRows, mGridCols; 135 bool mUseGrid; // Whether to use framework YUV frame tiling. 136 137 static const int64_t kNoFrameDropMaxPtsGap = -1000000; 138 static const int32_t kNoGridOpRate = 30; 139 static const int32_t kGridOpRate = 120; 140 141 void onHeicOutputFrameAvailable(const CodecOutputBufferInfo& bufferInfo); 142 void onHeicInputFrameAvailable(int32_t index); // Only called for YUV input mode. 143 void onHeicFormatChanged(sp<AMessage>& newFormat); 144 void onHeicCodecError(); 145 146 status_t initializeCodec(uint32_t width, uint32_t height, 147 const sp<CameraDeviceBase>& cameraDevice); 148 void deinitCodec(); 149 150 // 151 // Composite stream related structures, utility functions and callbacks. 152 // 153 struct InputFrame { 154 int32_t orientation; 155 int32_t quality; 156 157 CpuConsumer::LockedBuffer appSegmentBuffer; 158 std::vector<CodecOutputBufferInfo> codecOutputBuffers; 159 std::unique_ptr<CameraMetadata> result; 160 161 // Fields that are only applicable to HEVC tiling. 162 CpuConsumer::LockedBuffer yuvBuffer; 163 std::vector<CodecInputBufferInfo> codecInputBuffers; 164 165 bool error; // Main input image buffer error 166 bool exifError; // Exif/APP_SEGMENT buffer error 167 int64_t timestamp; 168 int32_t requestId; 169 170 sp<AMessage> format; 171 sp<MediaMuxer> muxer; 172 int fenceFd; 173 int fileFd; 174 ssize_t trackIndex; 175 ANativeWindowBuffer *anb; 176 177 bool appSegmentWritten; 178 size_t pendingOutputTiles; 179 size_t codecInputCounter; 180 InputFrameInputFrame181 InputFrame() : orientation(0), quality(kDefaultJpegQuality), error(false), 182 exifError(false), timestamp(-1), requestId(-1), fenceFd(-1), 183 fileFd(-1), trackIndex(-1), anb(nullptr), appSegmentWritten(false), 184 pendingOutputTiles(0), codecInputCounter(0) { } 185 }; 186 187 void compilePendingInputLocked(); 188 // Find first complete and valid frame with smallest frame number 189 bool getNextReadyInputLocked(int64_t *frameNumber /*out*/); 190 // Find next failing frame number with smallest frame number and return respective frame number 191 int64_t getNextFailingInputLocked(); 192 193 status_t processInputFrame(int64_t frameNumber, InputFrame &inputFrame); 194 status_t processCodecInputFrame(InputFrame &inputFrame); 195 status_t startMuxerForInputFrame(int64_t frameNumber, InputFrame &inputFrame); 196 status_t processAppSegment(int64_t frameNumber, InputFrame &inputFrame); 197 status_t processOneCodecOutputFrame(int64_t frameNumber, InputFrame &inputFrame); 198 status_t processCompletedInputFrame(int64_t frameNumber, InputFrame &inputFrame); 199 200 void releaseInputFrameLocked(int64_t frameNumber, InputFrame *inputFrame /*out*/); 201 void releaseInputFramesLocked(); 202 203 size_t findAppSegmentsSize(const uint8_t* appSegmentBuffer, size_t maxSize, 204 size_t* app1SegmentSize); 205 status_t copyOneYuvTile(sp<MediaCodecBuffer>& codecBuffer, 206 const CpuConsumer::LockedBuffer& yuvBuffer, 207 size_t top, size_t left, size_t width, size_t height); 208 void initCopyRowFunction(int32_t width); 209 static size_t calcAppSegmentMaxSize(const CameraMetadata& info); 210 void updateCodecQualityLocked(int32_t quality); 211 212 static const nsecs_t kWaitDuration = 10000000; // 10 ms 213 static const int32_t kDefaultJpegQuality = 99; 214 static const auto kJpegDataSpace = HAL_DATASPACE_V0_JFIF; 215 static const android_dataspace kAppSegmentDataSpace = 216 static_cast<android_dataspace>(HAL_DATASPACE_JPEG_APP_SEGMENTS); 217 static const android_dataspace kHeifDataSpace = 218 static_cast<android_dataspace>(HAL_DATASPACE_HEIF); 219 // Use the limit of pipeline depth in the API sepc as maximum number of acquired 220 // app segment buffers. 221 static const uint32_t kMaxAcquiredAppSegment = 8; 222 223 int mAppSegmentStreamId, mAppSegmentSurfaceId; 224 sp<CpuConsumer> mAppSegmentConsumer; 225 sp<Surface> mAppSegmentSurface; 226 size_t mAppSegmentMaxSize; 227 std::queue<int64_t> mAppSegmentFrameNumbers; 228 CameraMetadata mStaticInfo; 229 230 int mMainImageStreamId, mMainImageSurfaceId; 231 sp<Surface> mMainImageSurface; 232 sp<CpuConsumer> mMainImageConsumer; // Only applicable for HEVC codec. 233 bool mYuvBufferAcquired; // Only applicable to HEVC codec 234 std::queue<int64_t> mMainImageFrameNumbers; 235 236 static const int32_t kMaxOutputSurfaceProducerCount = 1; 237 sp<Surface> mOutputSurface; 238 sp<ProducerListener> mProducerListener; 239 int32_t mDequeuedOutputBufferCnt; 240 241 // Map from frame number to JPEG setting of orientation+quality 242 struct HeicSettings { 243 int32_t orientation; 244 int32_t quality; 245 int64_t timestamp; 246 int32_t requestId; 247 bool shutterNotified; 248 HeicSettingsHeicSettings249 HeicSettings() : orientation(0), quality(95), timestamp(0), 250 requestId(-1), shutterNotified(false) {} HeicSettingsHeicSettings251 HeicSettings(int32_t _orientation, int32_t _quality) : 252 orientation(_orientation), 253 quality(_quality), timestamp(0), 254 requestId(-1), shutterNotified(false) {} 255 256 }; 257 std::map<int64_t, HeicSettings> mSettingsByFrameNumber; 258 259 // Keep all incoming APP segment Blob buffer pending further processing. 260 std::vector<int64_t> mInputAppSegmentBuffers; 261 262 // Keep all incoming HEIC blob buffer pending further processing. 263 std::vector<CodecOutputBufferInfo> mCodecOutputBuffers; 264 std::queue<int64_t> mCodecOutputBufferFrameNumbers; 265 size_t mCodecOutputCounter; 266 int32_t mQuality; 267 268 // Keep all incoming Yuv buffer pending tiling and encoding (for HEVC YUV tiling only) 269 std::vector<int64_t> mInputYuvBuffers; 270 // Keep all codec input buffers ready to be filled out (for HEVC YUV tiling only) 271 std::vector<int32_t> mCodecInputBuffers; 272 273 // Artificial strictly incremental YUV grid timestamp to make encoder happy. 274 int64_t mGridTimestampUs; 275 276 // Indexed by frame number. In most common use case, entries are accessed in order. 277 std::map<int64_t, InputFrame> mPendingInputFrames; 278 279 // Function pointer of libyuv row copy. 280 void (*mFnCopyRow)(const uint8_t* src, uint8_t* dst, int width); 281 282 // A set of APP_SEGMENT error frame numbers 283 std::set<int64_t> mExifErrorFrameNumbers; 284 void flagAnExifErrorFrameNumber(int64_t frameNumber); 285 286 // The status id for tracking the active/idle status of this composite stream 287 int mStatusId; 288 void markTrackerIdle(); 289 }; 290 291 }; // namespace camera3 292 }; // namespace android 293 294 #endif //ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 295