1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 18 #define ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 19 20 #include <queue> 21 22 #include <gui/IProducerListener.h> 23 #include <gui/CpuConsumer.h> 24 25 #include <media/hardware/VideoAPI.h> 26 #include <media/MediaCodecBuffer.h> 27 #include <media/stagefright/foundation/ALooper.h> 28 #include <media/stagefright/foundation/AMessage.h> 29 #include <media/stagefright/MediaCodec.h> 30 #include <media/stagefright/MediaMuxer.h> 31 32 #include "CompositeStream.h" 33 34 namespace android { 35 namespace camera3 { 36 37 class HeicCompositeStream : public CompositeStream, public Thread, 38 public CpuConsumer::FrameAvailableListener { 39 public: 40 HeicCompositeStream(sp<CameraDeviceBase> device, 41 wp<hardware::camera2::ICameraDeviceCallbacks> cb); 42 ~HeicCompositeStream() override; 43 44 static bool isHeicCompositeStream(const sp<Surface> &surface); 45 46 status_t createInternalStreams(const std::vector<sp<Surface>>& consumers, 47 bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, 48 camera_stream_rotation_t rotation, int *id, const String8& physicalCameraId, 49 const std::unordered_set<int32_t> &sensorPixelModesUsed, 50 std::vector<int> *surfaceIds, 51 int streamSetId, bool isShared) override; 52 53 status_t deleteInternalStreams() override; 54 55 status_t configureStream() override; 56 57 status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector<int32_t>* /*out*/outputStreamIds, 58 int32_t* /*out*/currentStreamId) override; 59 60 status_t insertCompositeStreamIds(std::vector<int32_t>* compositeStreamIds /*out*/) override; 61 62 void onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) override; 63 getStreamId()64 int getStreamId() override { return mMainImageStreamId; } 65 66 // Use onShutter to keep track of frame number <-> timestamp mapping. 67 void onBufferReleased(const BufferInfo& bufferInfo) override; 68 void onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId, 69 const CameraMetadata& settings) override; 70 71 // CpuConsumer listener implementation 72 void onFrameAvailable(const BufferItem& item) override; 73 74 // Return stream information about the internal camera streams 75 static status_t getCompositeStreamInfo(const OutputStreamInfo &streamInfo, 76 const CameraMetadata& ch, std::vector<OutputStreamInfo>* compositeOutput /*out*/); 77 78 static bool isSizeSupportedByHeifEncoder(int32_t width, int32_t height, 79 bool* useHeic, bool* useGrid, int64_t* stall, AString* hevcName = nullptr); 80 static bool isInMemoryTempFileSupported(); 81 protected: 82 83 bool threadLoop() override; 84 bool onStreamBufferError(const CaptureResultExtras& resultExtras) override; 85 void onResultError(const CaptureResultExtras& resultExtras) override; 86 void onRequestError(const CaptureResultExtras& resultExtras) override; 87 88 private: 89 // 90 // HEIC/HEVC Codec related structures, utility functions, and callbacks 91 // 92 struct CodecOutputBufferInfo { 93 int32_t index; 94 int32_t offset; 95 int32_t size; 96 int64_t timeUs; 97 uint32_t flags; 98 }; 99 100 struct CodecInputBufferInfo { 101 int32_t index; 102 int64_t timeUs; 103 size_t tileIndex; 104 }; 105 106 class CodecCallbackHandler : public AHandler { 107 public: CodecCallbackHandler(wp<HeicCompositeStream> parent)108 explicit CodecCallbackHandler(wp<HeicCompositeStream> parent) { 109 mParent = parent; 110 } 111 virtual void onMessageReceived(const sp<AMessage> &msg); 112 private: 113 wp<HeicCompositeStream> mParent; 114 }; 115 116 enum { 117 kWhatCallbackNotify, 118 }; 119 120 bool mUseHeic; 121 sp<MediaCodec> mCodec; 122 sp<ALooper> mCodecLooper, mCallbackLooper; 123 sp<CodecCallbackHandler> mCodecCallbackHandler; 124 sp<AMessage> mAsyncNotify; 125 sp<AMessage> mFormat; 126 size_t mNumOutputTiles; 127 128 int32_t mOutputWidth, mOutputHeight; 129 size_t mMaxHeicBufferSize; 130 int32_t mGridWidth, mGridHeight; 131 size_t mGridRows, mGridCols; 132 bool mUseGrid; // Whether to use framework YUV frame tiling. 133 134 static const int64_t kNoFrameDropMaxPtsGap = -1000000; 135 static const int32_t kNoGridOpRate = 30; 136 static const int32_t kGridOpRate = 120; 137 138 void onHeicOutputFrameAvailable(const CodecOutputBufferInfo& bufferInfo); 139 void onHeicInputFrameAvailable(int32_t index); // Only called for YUV input mode. 140 void onHeicFormatChanged(sp<AMessage>& newFormat); 141 void onHeicCodecError(); 142 143 status_t initializeCodec(uint32_t width, uint32_t height, 144 const sp<CameraDeviceBase>& cameraDevice); 145 void deinitCodec(); 146 147 // 148 // Composite stream related structures, utility functions and callbacks. 149 // 150 struct InputFrame { 151 int32_t orientation; 152 int32_t quality; 153 154 CpuConsumer::LockedBuffer appSegmentBuffer; 155 std::vector<CodecOutputBufferInfo> codecOutputBuffers; 156 std::unique_ptr<CameraMetadata> result; 157 158 // Fields that are only applicable to HEVC tiling. 159 CpuConsumer::LockedBuffer yuvBuffer; 160 std::vector<CodecInputBufferInfo> codecInputBuffers; 161 162 bool error; // Main input image buffer error 163 bool exifError; // Exif/APP_SEGMENT buffer error 164 int64_t timestamp; 165 int32_t requestId; 166 167 sp<AMessage> format; 168 sp<MediaMuxer> muxer; 169 int fenceFd; 170 int fileFd; 171 ssize_t trackIndex; 172 ANativeWindowBuffer *anb; 173 174 bool appSegmentWritten; 175 size_t pendingOutputTiles; 176 size_t codecInputCounter; 177 InputFrameInputFrame178 InputFrame() : orientation(0), quality(kDefaultJpegQuality), error(false), 179 exifError(false), timestamp(-1), requestId(-1), fenceFd(-1), 180 fileFd(-1), trackIndex(-1), anb(nullptr), appSegmentWritten(false), 181 pendingOutputTiles(0), codecInputCounter(0) { } 182 }; 183 184 void compilePendingInputLocked(); 185 // Find first complete and valid frame with smallest frame number 186 bool getNextReadyInputLocked(int64_t *frameNumber /*out*/); 187 // Find next failing frame number with smallest frame number and return respective frame number 188 int64_t getNextFailingInputLocked(); 189 190 status_t processInputFrame(int64_t frameNumber, InputFrame &inputFrame); 191 status_t processCodecInputFrame(InputFrame &inputFrame); 192 status_t startMuxerForInputFrame(int64_t frameNumber, InputFrame &inputFrame); 193 status_t processAppSegment(int64_t frameNumber, InputFrame &inputFrame); 194 status_t processOneCodecOutputFrame(int64_t frameNumber, InputFrame &inputFrame); 195 status_t processCompletedInputFrame(int64_t frameNumber, InputFrame &inputFrame); 196 197 void releaseInputFrameLocked(int64_t frameNumber, InputFrame *inputFrame /*out*/); 198 void releaseInputFramesLocked(); 199 200 size_t findAppSegmentsSize(const uint8_t* appSegmentBuffer, size_t maxSize, 201 size_t* app1SegmentSize); 202 status_t copyOneYuvTile(sp<MediaCodecBuffer>& codecBuffer, 203 const CpuConsumer::LockedBuffer& yuvBuffer, 204 size_t top, size_t left, size_t width, size_t height); 205 void initCopyRowFunction(int32_t width); 206 static size_t calcAppSegmentMaxSize(const CameraMetadata& info); 207 void updateCodecQualityLocked(int32_t quality); 208 209 static const nsecs_t kWaitDuration = 10000000; // 10 ms 210 static const int32_t kDefaultJpegQuality = 99; 211 static const auto kJpegDataSpace = HAL_DATASPACE_V0_JFIF; 212 static const android_dataspace kAppSegmentDataSpace = 213 static_cast<android_dataspace>(HAL_DATASPACE_JPEG_APP_SEGMENTS); 214 static const android_dataspace kHeifDataSpace = 215 static_cast<android_dataspace>(HAL_DATASPACE_HEIF); 216 // Use the limit of pipeline depth in the API sepc as maximum number of acquired 217 // app segment buffers. 218 static const uint32_t kMaxAcquiredAppSegment = 8; 219 220 int mAppSegmentStreamId, mAppSegmentSurfaceId; 221 sp<CpuConsumer> mAppSegmentConsumer; 222 sp<Surface> mAppSegmentSurface; 223 size_t mAppSegmentMaxSize; 224 std::queue<int64_t> mAppSegmentFrameNumbers; 225 CameraMetadata mStaticInfo; 226 227 int mMainImageStreamId, mMainImageSurfaceId; 228 sp<Surface> mMainImageSurface; 229 sp<CpuConsumer> mMainImageConsumer; // Only applicable for HEVC codec. 230 bool mYuvBufferAcquired; // Only applicable to HEVC codec 231 std::queue<int64_t> mMainImageFrameNumbers; 232 233 static const int32_t kMaxOutputSurfaceProducerCount = 1; 234 sp<Surface> mOutputSurface; 235 sp<ProducerListener> mProducerListener; 236 int32_t mDequeuedOutputBufferCnt; 237 238 // Map from frame number to JPEG setting of orientation+quality 239 struct HeicSettings { 240 int32_t orientation; 241 int32_t quality; 242 int64_t timestamp; 243 int32_t requestId; 244 bool shutterNotified; 245 HeicSettingsHeicSettings246 HeicSettings() : orientation(0), quality(95), timestamp(0), 247 requestId(-1), shutterNotified(false) {} HeicSettingsHeicSettings248 HeicSettings(int32_t _orientation, int32_t _quality) : 249 orientation(_orientation), 250 quality(_quality), timestamp(0), 251 requestId(-1), shutterNotified(false) {} 252 253 }; 254 std::map<int64_t, HeicSettings> mSettingsByFrameNumber; 255 256 // Keep all incoming APP segment Blob buffer pending further processing. 257 std::vector<int64_t> mInputAppSegmentBuffers; 258 259 // Keep all incoming HEIC blob buffer pending further processing. 260 std::vector<CodecOutputBufferInfo> mCodecOutputBuffers; 261 std::queue<int64_t> mCodecOutputBufferFrameNumbers; 262 size_t mCodecOutputCounter; 263 int32_t mQuality; 264 265 // Keep all incoming Yuv buffer pending tiling and encoding (for HEVC YUV tiling only) 266 std::vector<int64_t> mInputYuvBuffers; 267 // Keep all codec input buffers ready to be filled out (for HEVC YUV tiling only) 268 std::vector<int32_t> mCodecInputBuffers; 269 270 // Artificial strictly incremental YUV grid timestamp to make encoder happy. 271 int64_t mGridTimestampUs; 272 273 // Indexed by frame number. In most common use case, entries are accessed in order. 274 std::map<int64_t, InputFrame> mPendingInputFrames; 275 276 // Function pointer of libyuv row copy. 277 void (*mFnCopyRow)(const uint8_t* src, uint8_t* dst, int width); 278 279 // A set of APP_SEGMENT error frame numbers 280 std::set<int64_t> mExifErrorFrameNumbers; 281 void flagAnExifErrorFrameNumber(int64_t frameNumber); 282 283 // The status id for tracking the active/idle status of this composite stream 284 int mStatusId; 285 void markTrackerIdle(); 286 }; 287 288 }; // namespace camera3 289 }; // namespace android 290 291 #endif //ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 292