1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 18 #define ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 19 20 #include <queue> 21 22 #include <gui/IProducerListener.h> 23 #include <gui/CpuConsumer.h> 24 25 #include <media/hardware/VideoAPI.h> 26 #include <media/MediaCodecBuffer.h> 27 #include <media/stagefright/foundation/ALooper.h> 28 #include <media/stagefright/MediaCodec.h> 29 #include <media/stagefright/MediaMuxer.h> 30 31 #include "CompositeStream.h" 32 33 namespace android { 34 namespace camera3 { 35 36 class HeicCompositeStream : public CompositeStream, public Thread, 37 public CpuConsumer::FrameAvailableListener { 38 public: 39 HeicCompositeStream(wp<CameraDeviceBase> device, 40 wp<hardware::camera2::ICameraDeviceCallbacks> cb); 41 ~HeicCompositeStream() override; 42 43 static bool isHeicCompositeStream(const sp<Surface> &surface); 44 45 status_t createInternalStreams(const std::vector<sp<Surface>>& consumers, 46 bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, 47 camera3_stream_rotation_t rotation, int *id, const String8& physicalCameraId, 48 std::vector<int> *surfaceIds, int streamSetId, bool isShared) override; 49 50 status_t deleteInternalStreams() override; 51 52 status_t configureStream() override; 53 54 status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector<int32_t>* /*out*/outputStreamIds, 55 int32_t* /*out*/currentStreamId) override; 56 57 void onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) override; 58 getStreamId()59 int getStreamId() override { return mMainImageStreamId; } 60 61 // Use onShutter to keep track of frame number <-> timestamp mapping. 62 void onBufferReleased(const BufferInfo& bufferInfo) override; 63 void onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId, 64 const CameraMetadata& settings) override; 65 66 // CpuConsumer listener implementation 67 void onFrameAvailable(const BufferItem& item) override; 68 69 // Return stream information about the internal camera streams 70 static status_t getCompositeStreamInfo(const OutputStreamInfo &streamInfo, 71 const CameraMetadata& ch, std::vector<OutputStreamInfo>* compositeOutput /*out*/); 72 73 static bool isSizeSupportedByHeifEncoder(int32_t width, int32_t height, 74 bool* useHeic, bool* useGrid, int64_t* stall, AString* hevcName = nullptr); 75 static bool isInMemoryTempFileSupported(); 76 protected: 77 78 bool threadLoop() override; 79 bool onStreamBufferError(const CaptureResultExtras& resultExtras) override; 80 void onResultError(const CaptureResultExtras& resultExtras) override; 81 82 private: 83 // 84 // HEIC/HEVC Codec related structures, utility functions, and callbacks 85 // 86 struct CodecOutputBufferInfo { 87 int32_t index; 88 int32_t offset; 89 int32_t size; 90 int64_t timeUs; 91 uint32_t flags; 92 }; 93 94 struct CodecInputBufferInfo { 95 int32_t index; 96 int64_t timeUs; 97 size_t tileIndex; 98 }; 99 100 class CodecCallbackHandler : public AHandler { 101 public: CodecCallbackHandler(wp<HeicCompositeStream> parent)102 explicit CodecCallbackHandler(wp<HeicCompositeStream> parent) { 103 mParent = parent; 104 } 105 virtual void onMessageReceived(const sp<AMessage> &msg); 106 private: 107 wp<HeicCompositeStream> mParent; 108 }; 109 110 enum { 111 kWhatCallbackNotify, 112 }; 113 114 bool mUseHeic; 115 sp<MediaCodec> mCodec; 116 sp<ALooper> mCodecLooper, mCallbackLooper; 117 sp<CodecCallbackHandler> mCodecCallbackHandler; 118 sp<AMessage> mAsyncNotify; 119 sp<AMessage> mFormat; 120 size_t mNumOutputTiles; 121 122 int32_t mOutputWidth, mOutputHeight; 123 size_t mMaxHeicBufferSize; 124 int32_t mGridWidth, mGridHeight; 125 size_t mGridRows, mGridCols; 126 bool mUseGrid; // Whether to use framework YUV frame tiling. 127 128 static const int64_t kNoFrameDropMaxPtsGap = -1000000; 129 static const int32_t kNoGridOpRate = 30; 130 static const int32_t kGridOpRate = 120; 131 132 void onHeicOutputFrameAvailable(const CodecOutputBufferInfo& bufferInfo); 133 void onHeicInputFrameAvailable(int32_t index); // Only called for YUV input mode. 134 void onHeicFormatChanged(sp<AMessage>& newFormat); 135 void onHeicCodecError(); 136 137 status_t initializeCodec(uint32_t width, uint32_t height, 138 const sp<CameraDeviceBase>& cameraDevice); 139 void deinitCodec(); 140 141 // 142 // Composite stream related structures, utility functions and callbacks. 143 // 144 struct InputFrame { 145 int32_t orientation; 146 int32_t quality; 147 148 CpuConsumer::LockedBuffer appSegmentBuffer; 149 std::vector<CodecOutputBufferInfo> codecOutputBuffers; 150 std::unique_ptr<CameraMetadata> result; 151 152 // Fields that are only applicable to HEVC tiling. 153 CpuConsumer::LockedBuffer yuvBuffer; 154 std::vector<CodecInputBufferInfo> codecInputBuffers; 155 156 bool error; 157 bool errorNotified; 158 int64_t frameNumber; 159 160 sp<MediaMuxer> muxer; 161 int fenceFd; 162 int fileFd; 163 ssize_t trackIndex; 164 ANativeWindowBuffer *anb; 165 166 bool appSegmentWritten; 167 size_t pendingOutputTiles; 168 size_t codecInputCounter; 169 InputFrameInputFrame170 InputFrame() : orientation(0), quality(kDefaultJpegQuality), error(false), 171 errorNotified(false), frameNumber(-1), fenceFd(-1), fileFd(-1), 172 trackIndex(-1), anb(nullptr), appSegmentWritten(false), 173 pendingOutputTiles(0), codecInputCounter(0) { } 174 }; 175 176 void compilePendingInputLocked(); 177 // Find first complete and valid frame with smallest timestamp 178 bool getNextReadyInputLocked(int64_t *currentTs /*out*/); 179 // Find next failing frame number with smallest timestamp and return respective frame number 180 int64_t getNextFailingInputLocked(int64_t *currentTs /*out*/); 181 182 status_t processInputFrame(nsecs_t timestamp, InputFrame &inputFrame); 183 status_t processCodecInputFrame(InputFrame &inputFrame); 184 status_t startMuxerForInputFrame(nsecs_t timestamp, InputFrame &inputFrame); 185 status_t processAppSegment(nsecs_t timestamp, InputFrame &inputFrame); 186 status_t processOneCodecOutputFrame(nsecs_t timestamp, InputFrame &inputFrame); 187 status_t processCompletedInputFrame(nsecs_t timestamp, InputFrame &inputFrame); 188 189 void releaseInputFrameLocked(InputFrame *inputFrame /*out*/); 190 void releaseInputFramesLocked(int64_t currentTs); 191 192 size_t findAppSegmentsSize(const uint8_t* appSegmentBuffer, size_t maxSize, 193 size_t* app1SegmentSize); 194 int64_t findTimestampInNsLocked(int64_t timeInUs); 195 status_t copyOneYuvTile(sp<MediaCodecBuffer>& codecBuffer, 196 const CpuConsumer::LockedBuffer& yuvBuffer, 197 size_t top, size_t left, size_t width, size_t height); 198 void initCopyRowFunction(int32_t width); 199 static size_t calcAppSegmentMaxSize(const CameraMetadata& info); 200 201 static const nsecs_t kWaitDuration = 10000000; // 10 ms 202 static const int32_t kDefaultJpegQuality = 99; 203 static const auto kJpegDataSpace = HAL_DATASPACE_V0_JFIF; 204 static const android_dataspace kAppSegmentDataSpace = 205 static_cast<android_dataspace>(HAL_DATASPACE_JPEG_APP_SEGMENTS); 206 static const android_dataspace kHeifDataSpace = 207 static_cast<android_dataspace>(HAL_DATASPACE_HEIF); 208 209 int mAppSegmentStreamId, mAppSegmentSurfaceId; 210 sp<CpuConsumer> mAppSegmentConsumer; 211 sp<Surface> mAppSegmentSurface; 212 bool mAppSegmentBufferAcquired; 213 size_t mAppSegmentMaxSize; 214 CameraMetadata mStaticInfo; 215 216 int mMainImageStreamId, mMainImageSurfaceId; 217 sp<Surface> mMainImageSurface; 218 sp<CpuConsumer> mMainImageConsumer; // Only applicable for HEVC codec. 219 bool mYuvBufferAcquired; // Only applicable to HEVC codec 220 221 sp<Surface> mOutputSurface; 222 sp<ProducerListener> mProducerListener; 223 224 225 // Map from frame number to JPEG setting of orientation+quality 226 std::map<int64_t, std::pair<int32_t, int32_t>> mSettingsByFrameNumber; 227 // Map from timestamp to JPEG setting of orientation+quality 228 std::map<int64_t, std::pair<int32_t, int32_t>> mSettingsByTimestamp; 229 230 // Keep all incoming APP segment Blob buffer pending further processing. 231 std::vector<int64_t> mInputAppSegmentBuffers; 232 233 // Keep all incoming HEIC blob buffer pending further processing. 234 std::vector<CodecOutputBufferInfo> mCodecOutputBuffers; 235 std::queue<int64_t> mCodecOutputBufferTimestamps; 236 size_t mOutputBufferCounter; 237 238 // Keep all incoming Yuv buffer pending tiling and encoding (for HEVC YUV tiling only) 239 std::vector<int64_t> mInputYuvBuffers; 240 // Keep all codec input buffers ready to be filled out (for HEVC YUV tiling only) 241 std::vector<int32_t> mCodecInputBuffers; 242 243 // Artificial strictly incremental YUV grid timestamp to make encoder happy. 244 int64_t mGridTimestampUs; 245 246 // In most common use case, entries are accessed in order. 247 std::map<int64_t, InputFrame> mPendingInputFrames; 248 249 // Function pointer of libyuv row copy. 250 void (*mFnCopyRow)(const uint8_t* src, uint8_t* dst, int width); 251 }; 252 253 }; // namespace camera3 254 }; // namespace android 255 256 #endif //ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 257