1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 18 #define ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 19 20 #include <algorithm> 21 #include <android/data_space.h> 22 #include <memory> 23 #include <queue> 24 25 #include <gui/CpuConsumer.h> 26 27 #include <media/hardware/VideoAPI.h> 28 #include <media/MediaCodecBuffer.h> 29 #include <media/stagefright/foundation/ALooper.h> 30 #include <media/stagefright/foundation/AMessage.h> 31 #include <media/stagefright/MediaCodec.h> 32 #include <media/stagefright/MediaMuxer.h> 33 #include <ultrahdr/ultrahdrcommon.h> 34 #include <ultrahdr/gainmapmetadata.h> 35 36 #include "CompositeStream.h" 37 38 namespace android { 39 namespace camera3 { 40 41 class HeicCompositeStream : public CompositeStream, public Thread, 42 public CpuConsumer::FrameAvailableListener { 43 public: 44 HeicCompositeStream(sp<CameraDeviceBase> device, 45 wp<hardware::camera2::ICameraDeviceCallbacks> cb); 46 ~HeicCompositeStream() override; 47 48 static bool isHeicCompositeStream(const sp<Surface>& surface, bool isCompositeHeicDisabled, 49 bool isCompositeHeicUltraHDRDisabled); 50 static bool isHeicCompositeStreamInfo(const OutputStreamInfo& streamInfo, 51 bool isCompositeHeicDisabled, 52 bool isCompositeHeicUltraHDRDisabled); 53 54 status_t createInternalStreams(const std::vector<SurfaceHolder>& consumers, 55 bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, 56 camera_stream_rotation_t rotation, int *id, const std::string& physicalCameraId, 57 const std::unordered_set<int32_t> &sensorPixelModesUsed, 58 std::vector<int> *surfaceIds, int streamSetId, bool isShared, int32_t colorSpace, 59 int64_t dynamicProfile, int64_t streamUseCase, bool useReadoutTimestamp) override; 60 61 status_t deleteInternalStreams() override; 62 63 status_t configureStream() override; 64 65 status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector<int32_t>* /*out*/outputStreamIds, 66 int32_t* /*out*/currentStreamId) override; 67 68 status_t insertCompositeStreamIds(std::vector<int32_t>* compositeStreamIds /*out*/) override; 69 70 void onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) override; 71 getStreamId()72 int getStreamId() override { return mMainImageStreamId; } 73 74 // Use onShutter to keep track of frame number <-> timestamp mapping. 75 void onBufferReleased(const BufferInfo& bufferInfo) override; 76 void onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId, 77 const CameraMetadata& settings) override; 78 79 // CpuConsumer listener implementation 80 void onFrameAvailable(const BufferItem& item) override; 81 82 // Return stream information about the internal camera streams 83 static status_t getCompositeStreamInfo(const OutputStreamInfo &streamInfo, 84 const CameraMetadata& ch, std::vector<OutputStreamInfo>* compositeOutput /*out*/); 85 86 // Get composite stream stats getStreamStats(hardware::CameraStreamStats *)87 void getStreamStats(hardware::CameraStreamStats*) override {}; 88 89 static bool isSizeSupportedByHeifEncoder(int32_t width, int32_t height, 90 bool* useHeic, bool* useGrid, int64_t* stall, AString* hevcName = nullptr, 91 bool allowSWCodec = false); 92 static bool isInMemoryTempFileSupported(); 93 94 // HDR Gainmap subsampling 95 static constexpr size_t kGainmapScale = 4; 96 97 protected: 98 99 bool threadLoop() override; 100 bool onStreamBufferError(const CaptureResultExtras& resultExtras) override; 101 void onResultError(const CaptureResultExtras& resultExtras) override; 102 void onRequestError(const CaptureResultExtras& resultExtras) override; 103 104 private: 105 // 106 // HEIC/HEVC Codec related structures, utility functions, and callbacks 107 // 108 struct CodecOutputBufferInfo { 109 int32_t index; 110 int32_t offset; 111 int32_t size; 112 int64_t timeUs; 113 uint32_t flags; 114 }; 115 116 struct CodecInputBufferInfo { 117 int32_t index; 118 int64_t timeUs; 119 size_t tileIndex; 120 }; 121 122 class CodecCallbackHandler : public AHandler { 123 public: 124 explicit CodecCallbackHandler(wp<HeicCompositeStream> parent, bool isGainmap = false) : mParent(parent)125 mParent(parent), mIsGainmap(isGainmap) {} 126 virtual void onMessageReceived(const sp<AMessage> &msg); 127 private: 128 wp<HeicCompositeStream> mParent; 129 bool mIsGainmap; 130 }; 131 132 enum { 133 kWhatCallbackNotify, 134 }; 135 136 bool mUseHeic; 137 sp<MediaCodec> mCodec; 138 sp<MediaCodec> mGainmapCodec; 139 sp<ALooper> mCodecLooper, mCallbackLooper, mGainmapCallbackLooper; 140 sp<CodecCallbackHandler> mCodecCallbackHandler, mGainmapCodecCallbackHandler; 141 sp<AMessage> mAsyncNotify, mGainmapAsyncNotify; 142 sp<AMessage> mFormat, mGainmapFormat; 143 size_t mNumOutputTiles, mNumGainmapOutputTiles; 144 145 int32_t mOutputWidth, mOutputHeight, mGainmapOutputWidth, mGainmapOutputHeight; 146 size_t mMaxHeicBufferSize; 147 int32_t mGridWidth, mGridHeight, mGainmapGridWidth, mGainmapGridHeight; 148 size_t mGridRows, mGridCols, mGainmapGridRows, mGainmapGridCols; 149 bool mUseGrid, mGainmapUseGrid; // Whether to use framework YUV frame tiling. 150 151 static constexpr int64_t kNoFrameDropMaxPtsGap = -1000000; 152 static constexpr int32_t kNoGridOpRate = 30; 153 static constexpr int32_t kGridOpRate = 120; 154 155 void onHeicOutputFrameAvailable(const CodecOutputBufferInfo& bufferInfo, bool isGainmap); 156 void onHeicInputFrameAvailable(int32_t index, bool isGainmap);// Only called for YUV input mode. 157 void onHeicFormatChanged(sp<AMessage>& newFormat, bool isGainmap); 158 void onHeicGainmapFormatChanged(sp<AMessage>& newFormat); 159 void onHeicCodecError(); 160 161 status_t initializeCodec(uint32_t width, uint32_t height, 162 const sp<CameraDeviceBase>& cameraDevice); 163 void deinitCodec(); 164 status_t initializeGainmapCodec(); 165 void deinitGainmapCodec(); 166 167 // 168 // Composite stream related structures, utility functions and callbacks. 169 // 170 struct InputFrame { 171 int32_t orientation; 172 int32_t quality; 173 174 CpuConsumer::LockedBuffer appSegmentBuffer; 175 std::vector<CodecOutputBufferInfo> codecOutputBuffers, gainmapCodecOutputBuffers; 176 std::unique_ptr<CameraMetadata> result; 177 178 // Fields that are only applicable to HEVC tiling. 179 CpuConsumer::LockedBuffer yuvBuffer; 180 std::vector<CodecInputBufferInfo> codecInputBuffers, gainmapCodecInputBuffers; 181 182 bool error; // Main input image buffer error 183 bool exifError; // Exif/APP_SEGMENT buffer error 184 int64_t timestamp; 185 int32_t requestId; 186 187 sp<AMessage> format, gainmapFormat; 188 sp<MediaMuxer> muxer; 189 int fenceFd; 190 int fileFd; 191 ssize_t trackIndex, gainmapTrackIndex; 192 ANativeWindowBuffer *anb; 193 194 bool appSegmentWritten; 195 size_t pendingOutputTiles, gainmapPendingOutputTiles; 196 size_t codecInputCounter, gainmapCodecInputCounter; 197 198 std::unique_ptr<CpuConsumer::LockedBuffer> baseImage, gainmapImage; 199 std::unique_ptr<ultrahdr::uhdr_raw_image_ext> baseBuffer, gainmap; 200 std::unique_ptr<uint8_t[]> gainmapChroma; 201 std::vector<uint8_t> isoGainmapMetadata; 202 InputFrameInputFrame203 InputFrame() 204 : orientation(0), 205 quality(kDefaultJpegQuality), 206 error(false), 207 exifError(false), 208 timestamp(-1), 209 requestId(-1), 210 fenceFd(-1), 211 fileFd(-1), 212 trackIndex(-1), 213 gainmapTrackIndex(-1), 214 anb(nullptr), 215 appSegmentWritten(false), 216 pendingOutputTiles(0), 217 gainmapPendingOutputTiles(0), 218 codecInputCounter(0), 219 gainmapCodecInputCounter(0) {} 220 }; 221 222 void compilePendingInputLocked(); 223 // Find first complete and valid frame with smallest frame number 224 bool getNextReadyInputLocked(int64_t *frameNumber /*out*/); 225 // Find next failing frame number with smallest frame number and return respective frame number 226 int64_t getNextFailingInputLocked(); 227 228 status_t processInputFrame(int64_t frameNumber, InputFrame &inputFrame); 229 status_t processCodecInputFrame(InputFrame &inputFrame); 230 status_t processCodecGainmapInputFrame(InputFrame &inputFrame); 231 status_t startMuxerForInputFrame(int64_t frameNumber, InputFrame &inputFrame); 232 status_t processAppSegment(int64_t frameNumber, InputFrame &inputFrame); 233 status_t processOneCodecOutputFrame(int64_t frameNumber, InputFrame &inputFrame); 234 status_t processOneCodecGainmapOutputFrame(int64_t frameNumber, InputFrame &inputFrame); 235 status_t processCompletedInputFrame(int64_t frameNumber, InputFrame &inputFrame); 236 237 void releaseInputFrameLocked(int64_t frameNumber, InputFrame *inputFrame /*out*/); 238 void releaseInputFramesLocked(); 239 240 size_t findAppSegmentsSize(const uint8_t* appSegmentBuffer, size_t maxSize, 241 size_t* app1SegmentSize); 242 status_t copyOneYuvTile(sp<MediaCodecBuffer>& codecBuffer, 243 const CpuConsumer::LockedBuffer& yuvBuffer, 244 size_t top, size_t left, size_t width, size_t height); 245 void initCopyRowFunction(int32_t width); 246 static size_t calcAppSegmentMaxSize(const CameraMetadata& info); 247 void updateCodecQualityLocked(int32_t quality); 248 249 static constexpr nsecs_t kWaitDuration = 10000000; // 10 ms 250 static constexpr int32_t kDefaultJpegQuality = 99; 251 static constexpr auto kJpegDataSpace = HAL_DATASPACE_V0_JFIF; 252 static constexpr android_dataspace kAppSegmentDataSpace = 253 static_cast<android_dataspace>(HAL_DATASPACE_JPEG_APP_SEGMENTS); 254 static constexpr android_dataspace kHeifDataSpace = 255 static_cast<android_dataspace>(HAL_DATASPACE_HEIF); 256 android_dataspace mInternalDataSpace = kHeifDataSpace; 257 // Use the limit of pipeline depth in the API sepc as maximum number of acquired 258 // app segment buffers. 259 static constexpr uint32_t kMaxAcquiredAppSegment = 8; 260 261 int mAppSegmentStreamId, mAppSegmentSurfaceId; 262 sp<CpuConsumer> mAppSegmentConsumer; 263 sp<Surface> mAppSegmentSurface; 264 size_t mAppSegmentMaxSize; 265 std::queue<int64_t> mAppSegmentFrameNumbers; 266 CameraMetadata mStaticInfo; 267 268 int mMainImageStreamId, mMainImageSurfaceId; 269 sp<Surface> mMainImageSurface; 270 sp<CpuConsumer> mMainImageConsumer; // Only applicable for HEVC codec. 271 bool mYuvBufferAcquired; // Only applicable to HEVC codec 272 std::queue<int64_t> mMainImageFrameNumbers; 273 274 static constexpr int32_t kMaxOutputSurfaceProducerCount = 1; 275 sp<Surface> mOutputSurface; 276 sp<StreamSurfaceListener> mStreamSurfaceListener; 277 int32_t mDequeuedOutputBufferCnt; 278 279 // Map from frame number to JPEG setting of orientation+quality 280 struct HeicSettings { 281 int32_t orientation; 282 int32_t quality; 283 int64_t timestamp; 284 int32_t requestId; 285 bool shutterNotified; 286 HeicSettingsHeicSettings287 HeicSettings() : orientation(0), quality(95), timestamp(0), 288 requestId(-1), shutterNotified(false) {} HeicSettingsHeicSettings289 HeicSettings(int32_t _orientation, int32_t _quality) : 290 orientation(_orientation), 291 quality(_quality), timestamp(0), 292 requestId(-1), shutterNotified(false) {} 293 294 }; 295 std::map<int64_t, HeicSettings> mSettingsByFrameNumber; 296 297 // Keep all incoming APP segment Blob buffer pending further processing. 298 std::vector<int64_t> mInputAppSegmentBuffers; 299 300 // Keep all incoming HEIC blob buffer pending further processing. 301 std::vector<CodecOutputBufferInfo> mCodecOutputBuffers, mGainmapCodecOutputBuffers; 302 std::queue<int64_t> mCodecOutputBufferFrameNumbers, mCodecGainmapOutputBufferFrameNumbers; 303 size_t mCodecOutputCounter, mCodecGainmapOutputCounter; 304 int32_t mQuality; 305 306 // Keep all incoming Yuv buffer pending tiling and encoding (for HEVC YUV tiling only) 307 std::vector<int64_t> mInputYuvBuffers; 308 // Keep all codec input buffers ready to be filled out (for HEVC YUV tiling only) 309 std::vector<int32_t> mCodecInputBuffers, mGainmapCodecInputBuffers; 310 311 // Artificial strictly incremental YUV grid timestamp to make encoder happy. 312 int64_t mGridTimestampUs; 313 314 // Indexed by frame number. In most common use case, entries are accessed in order. 315 std::map<int64_t, InputFrame> mPendingInputFrames; 316 317 // Function pointer of libyuv row copy. 318 void (*mFnCopyRow)(const uint8_t* src, uint8_t* dst, int width); 319 320 // A set of APP_SEGMENT error frame numbers 321 std::set<int64_t> mExifErrorFrameNumbers; 322 void flagAnExifErrorFrameNumber(int64_t frameNumber); 323 324 // The status id for tracking the active/idle status of this composite stream 325 int mStatusId; 326 void markTrackerIdle(); 327 328 //APP_SEGMENT stream supported 329 bool mAppSegmentSupported = false; 330 331 bool mHDRGainmapEnabled = false; 332 333 // UltraHDR tonemap color and format aspects 334 static constexpr uhdr_img_fmt_t kUltraHdrInputFmt = UHDR_IMG_FMT_24bppYCbCrP010; 335 static constexpr uhdr_color_gamut kUltraHdrInputGamut = UHDR_CG_BT_2100; 336 static constexpr uhdr_color_transfer kUltraHdrInputTransfer = UHDR_CT_HLG; 337 static constexpr uhdr_color_range kUltraHdrInputRange = UHDR_CR_FULL_RANGE; 338 339 static constexpr uhdr_img_fmt_t kUltraHdrOutputFmt = UHDR_IMG_FMT_12bppYCbCr420; 340 static constexpr uhdr_color_gamut kUltraHdrOutputGamut = UHDR_CG_DISPLAY_P3; 341 static constexpr uhdr_color_transfer kUltraHdrOutputTransfer = UHDR_CT_SRGB; 342 static constexpr uhdr_color_range kUltraHdrOutputRange = UHDR_CR_FULL_RANGE; 343 344 static constexpr auto kUltraHDRDataSpace = 345 aidl::android::hardware::graphics::common::Dataspace::HEIF_ULTRAHDR; 346 347 // MediaMuxer/Codec color and format aspects for base image and gainmap metadata 348 static constexpr int32_t kCodecColorFormat = COLOR_FormatYUV420Flexible; 349 static constexpr ColorAspects::Primaries kCodecColorPrimaries = 350 ColorAspects::Primaries::PrimariesEG432; 351 static constexpr ColorAspects::MatrixCoeffs kCodecColorMatrix = 352 ColorAspects::MatrixCoeffs::MatrixUnspecified; 353 static constexpr ColorAspects::Transfer kCodecColorTransfer = 354 ColorAspects::Transfer::TransferSRGB; 355 static constexpr ColorAspects::Range kCodecColorRange = 356 ColorAspects::Range::RangeFull; 357 358 // MediaMuxer/Codec color and format aspects for gainmap as per ISO 23008-12:2024 359 static constexpr int32_t kCodecGainmapColorFormat = COLOR_FormatYUV420Flexible; 360 static constexpr ColorAspects::Primaries kCodecGainmapColorPrimaries = 361 ColorAspects::Primaries::PrimariesUnspecified; 362 static constexpr ColorAspects::MatrixCoeffs kCodecGainmapColorMatrix = 363 ColorAspects::MatrixCoeffs::MatrixUnspecified; 364 static constexpr ColorAspects::Transfer kCodecGainmapColorTransfer = 365 ColorAspects::Transfer::TransferUnspecified; 366 static constexpr ColorAspects::Range kCodecGainmapColorRange = 367 ColorAspects::Range::RangeFull; 368 369 370 status_t generateBaseImageAndGainmap(InputFrame &inputFrame); 371 }; 372 373 }; // namespace camera3 374 }; // namespace android 375 376 #endif //ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 377