1 // Copyright 2020 The Android Open Source Project
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 #include <android-base/logging.h>
15 #include <android-base/strings.h>
16 #include <media/stagefright/MediaCodecConstants.h>
17
18 #include <fcntl.h>
19 #include <chrono>
20
21 #include "VideoDecoder.h"
22 #include "prebuilt_interface.h"
23
24 namespace android {
25 namespace automotive {
26 namespace computepipe {
27 namespace runner {
28 namespace input_manager {
29 namespace {
30
31 const int64_t kMicrosPerSecond = 1000 * 1000;
32 const int64_t kMediaCodecNonBlockingTimeoutUs = 5000; // 5ms.
33 int kMaxInUseBuffers = 50;
34
getCurrentTime()35 int64_t getCurrentTime() {
36 auto timePoint = std::chrono::system_clock::now();
37 return std::chrono::time_point_cast<std::chrono::microseconds>(timePoint)
38 .time_since_epoch()
39 .count();
40 }
41
toPixelFormat(int mediaFormat)42 PixelFormat toPixelFormat(int mediaFormat) {
43 switch (mediaFormat) {
44 case COLOR_FormatYUV420SemiPlanar:
45 return YUV_420;
46 default:
47 LOG(ERROR) << "Unsupported output format - " << mediaFormat;
48 return PIXELFORMAT_MAX;
49 }
50 }
51
52 } // namespace
53
VideoDecoder(const proto::InputStreamConfig & config,std::shared_ptr<InputEngineInterface> engineInterface)54 VideoDecoder::VideoDecoder(const proto::InputStreamConfig& config,
55 std::shared_ptr<InputEngineInterface> engineInterface) :
56 mEngine(engineInterface),
57 mConfig(config) {
58 if (config.has_video_config() && config.video_config().has_file_path()) {
59 mVideoPath = config.video_config().file_path();
60 }
61 }
62
~VideoDecoder()63 VideoDecoder::~VideoDecoder() {
64 stopDecoding();
65 }
66
getPlaybackFrameRate()67 float VideoDecoder::getPlaybackFrameRate() {
68 if (!mExtractor) {
69 if (initializeMediaExtractor() != Status::SUCCESS) {
70 LOG(ERROR) << "VideoDecoder: Received error initializing media extractor.";
71 return 0;
72 }
73 }
74 if (!mCodec) {
75 if (initializeMediaDecoder() != Status::SUCCESS) {
76 LOG(ERROR) << "VideoDecoder: Received error initializing media codec.";
77 return 0;
78 }
79 }
80 return mPlaybackFrameRate;
81 }
82
setInitialTimestamp(int64_t timestampMicros)83 void VideoDecoder::setInitialTimestamp(int64_t timestampMicros) {
84 mStartTimeMicros = timestampMicros;
85 }
86
startDecoding()87 Status VideoDecoder::startDecoding() {
88 mStopThread = false;
89 mDecoderThead = std::make_unique<std::thread>(&VideoDecoder::decoderThreadFunction, this);
90 return Status::SUCCESS;
91 }
92
stopDecoding()93 void VideoDecoder::stopDecoding() {
94 mStopThread = true;
95 if (mDecoderThead && mDecoderThead->joinable()) {
96 mDecoderThead->join();
97 mDecoderThead = nullptr;
98 }
99 releaseResources();
100 }
101
initializeMediaExtractor()102 Status VideoDecoder::initializeMediaExtractor() {
103 if (!mIsFdOpen) {
104 mFd = open(mVideoPath.c_str(), 0, O_RDONLY);
105 mIsFdOpen = true;
106 }
107 if (!mExtractor) {
108 mExtractor = AMediaExtractor_new();
109 int size = lseek(mFd, 0, SEEK_END);
110 // Reset the offset.
111 lseek(mFd, 0, SEEK_SET);
112 media_status_t status = AMediaExtractor_setDataSourceFd(mExtractor, mFd, 0, size);
113 if (status != AMEDIA_OK) {
114 LOG(ERROR) << "VideoDecoder: Received error when initializing media extractor.";
115 stopDecoding();
116 return Status::INTERNAL_ERROR;
117 }
118 }
119 return Status::SUCCESS;
120 }
121
initializeMediaDecoder()122 Status VideoDecoder::initializeMediaDecoder() {
123 int numTracks = AMediaExtractor_getTrackCount(mExtractor);
124 AMediaFormat* format;
125 const char* mime;
126 int i;
127 for (i = 0; i < numTracks; i++) {
128 format = AMediaExtractor_getTrackFormat(mExtractor, i);
129 if (!AMediaFormat_getString(format, AMEDIAFORMAT_KEY_MIME, &mime)) {
130 LOG(ERROR) << "VideoDecoder: Error in fetching format string";
131 }
132
133 if (android::base::StartsWith(mime, "video/")) {
134 media_status_t status = AMediaExtractor_selectTrack(mExtractor, i);
135 if (status != AMEDIA_OK) {
136 LOG(ERROR) << "VideoDecoder: Media extractor returned error to select track.";
137 return Status::INTERNAL_ERROR;
138 }
139 break;
140 }
141 AMediaFormat_delete(format);
142 }
143 if (i == numTracks) {
144 LOG(ERROR) << "VideoDecoder: No video track in " << mVideoPath;
145 return Status::INTERNAL_ERROR;
146 }
147
148 int frameRate;
149 AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_FRAME_RATE, &frameRate);
150 // TODO(b/156102135) - parse multiplier from input proto.
151 float playbackRateMultiplier = 1.0;
152 mPlaybackFrameRate = frameRate * playbackRateMultiplier;
153
154 mCodec = AMediaCodec_createDecoderByType(mime);
155 if (!mCodec) {
156 LOG(ERROR) << "VideoDecoder: Unable to create decoder.";
157 AMediaFormat_delete(format);
158 releaseResources();
159 return Status::INTERNAL_ERROR;
160 }
161
162 media_status_t status = AMediaCodec_configure(mCodec, format, nullptr, nullptr, 0);
163 if (status != AMEDIA_OK) {
164 LOG(ERROR) << "VideoDecoder: Received error in configuring mCodec.";
165 AMediaFormat_delete(format);
166 releaseResources();
167 return Status::INTERNAL_ERROR;
168 }
169 return Status::SUCCESS;
170 }
171
releaseResources()172 void VideoDecoder::releaseResources() {
173 if (mExtractor) {
174 (void)AMediaExtractor_delete(mExtractor);
175 mExtractor = nullptr;
176 }
177 if (mCodec) {
178 while (mDecodedBuffers.size()) {
179 std::pair<int, AMediaCodecBufferInfo> buffer = mDecodedBuffers.front();
180 AMediaCodec_releaseOutputBuffer(mCodec, buffer.first, false);
181 mDecodedBuffers.pop();
182 }
183 AMediaFormat* format = AMediaCodec_getOutputFormat(mCodec);
184 AMediaFormat_delete(format);
185 (void)AMediaCodec_delete(mCodec);
186 mCodec = nullptr;
187 }
188 if (mIsFdOpen) {
189 close(mFd);
190 mIsFdOpen = false;
191 }
192 }
193
decoderThreadFunction()194 void VideoDecoder::decoderThreadFunction() {
195 if (!mExtractor || !mCodec) {
196 CHECK(initializeMediaExtractor() == Status::SUCCESS);
197 CHECK(initializeMediaDecoder() == Status::SUCCESS);
198 }
199
200 media_status_t status = AMediaCodec_start(mCodec);
201 if (status != AMEDIA_OK) {
202 LOG(ERROR) << "VideoDecoder: Received error in starting decoder.";
203 mEngine->notifyInputError();
204 return;
205 }
206
207 int frameIx = 0;
208 int loopbackCount = mLoopbackCount;
209 if (loopbackCount == 0) {
210 sendEosFlag();
211 return;
212 }
213 while (!mStopThread) {
214 // Force 64bit integer arithmetic operations.
215 int64_t frameIntervalMicros = kMicrosPerSecond / mPlaybackFrameRate;
216 int64_t frameTimeMicros = frameIx * frameIntervalMicros + mStartTimeMicros;
217
218 if (getCurrentTime() > frameTimeMicros) {
219 if (readDecodedFrame(frameTimeMicros)) {
220 frameIx++;
221 }
222 }
223 addFramesToCodec();
224 popFramesFromCodec();
225 if (mExtractorFinished && (mCountQueuedBuffers == 0) && mDecodedBuffers.empty()) {
226 --loopbackCount;
227 if (loopbackCount == 0) {
228 sendEosFlag();
229 break;
230 }
231 LOG(ERROR) << "Remaining loopback count - " << loopbackCount;
232
233 AMediaExtractor_seekTo(mExtractor, 0, AMEDIAEXTRACTOR_SEEK_CLOSEST_SYNC);
234 AMediaCodec_flush(mCodec);
235
236 mStartTimeMicros = frameTimeMicros + frameIntervalMicros;
237 frameIx = 0;
238 mExtractorFinished = false;
239 }
240 std::this_thread::sleep_for(std::chrono::milliseconds(1));
241 }
242 releaseResources();
243 }
244
addFramesToCodec()245 void VideoDecoder::addFramesToCodec() {
246 if (mExtractorFinished) {
247 return;
248 }
249 while ((mCountQueuedBuffers + mDecodedBuffers.size()) <= kMaxInUseBuffers) {
250 size_t sampleSize = AMediaExtractor_getSampleSize(mExtractor);
251 int64_t presentationTime = AMediaExtractor_getSampleTime(mExtractor);
252 int bufferIx = AMediaCodec_dequeueInputBuffer(mCodec, kMediaCodecNonBlockingTimeoutUs);
253 if (bufferIx < 0) {
254 if (bufferIx != AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
255 LOG(ERROR) << "VideoDecoder: Received error in AMediaCodec_dequeueInputBuffer";
256 }
257 return;
258 }
259 size_t bufferSize;
260 uint8_t* buffer = AMediaCodec_getInputBuffer(mCodec, bufferIx, &bufferSize);
261 if (sampleSize > bufferSize) {
262 LOG(ERROR) << "VideoDecoder: Buffer is not large enough.";
263 }
264 if (presentationTime < 0) {
265 AMediaCodec_queueInputBuffer(mCodec, bufferIx, 0 /*offset*/, 0 /*size*/,
266 presentationTime, AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM);
267 mExtractorFinished = true;
268 return;
269 }
270 size_t read = AMediaExtractor_readSampleData(mExtractor, buffer, sampleSize);
271 media_status_t status = AMediaCodec_queueInputBuffer(mCodec, bufferIx, 0 /*offset*/, read,
272 presentationTime, 0 /*flags*/);
273 if (status != AMEDIA_OK) {
274 LOG(ERROR) << "VideoDecoder: Received error in queueing input buffer.";
275 }
276 mCountQueuedBuffers++;
277 AMediaExtractor_advance(mExtractor);
278 }
279 }
280
popFramesFromCodec()281 void VideoDecoder::popFramesFromCodec() {
282 while (mCountQueuedBuffers) {
283 AMediaCodecBufferInfo info;
284 int bufferIx = AMediaCodec_dequeueOutputBuffer(
285 mCodec, &info, kMediaCodecNonBlockingTimeoutUs);
286 if (bufferIx < 0) {
287 if (bufferIx != AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
288 LOG(ERROR) << "VideoDecoder: Received error in AMediaCodec_dequeueOutputBuffer";
289 }
290 return;
291 }
292 mDecodedBuffers.push(std::pair<int, AMediaCodecBufferInfo>(bufferIx, info));
293 mCountQueuedBuffers--;
294 }
295 }
296
readDecodedFrame(int64_t frameTimeMicros)297 bool VideoDecoder::readDecodedFrame(int64_t frameTimeMicros) {
298 if (mDecodedBuffers.empty()) {
299 return false;
300 }
301
302 AMediaFormat* format = AMediaCodec_getOutputFormat(mCodec);
303
304 int width, height, stride, outputFormat;
305 bool success = AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_WIDTH, &width);
306 success = success && AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_HEIGHT, &height);
307 success = success && AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_STRIDE, &stride);
308 success =
309 success && AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, &outputFormat);
310 if (!success) {
311 LOG(ERROR) << "Failure to find frame parameters, exiting.";
312 mEngine->notifyInputError();
313 return false;
314 }
315 PixelFormat prebuiltFormat = toPixelFormat(outputFormat);
316
317 std::pair<int, AMediaCodecBufferInfo> buffer = mDecodedBuffers.front();
318 size_t decodedOutSize;
319 uint8_t* outputBuffer = AMediaCodec_getOutputBuffer(mCodec, buffer.first, &decodedOutSize);
320
321 // Inject data to engine.
322 InputFrame inputFrame(height, width, prebuiltFormat, stride,
323 outputBuffer + buffer.second.offset);
324 mEngine->dispatchInputFrame(mConfig.stream_id(), frameTimeMicros, inputFrame);
325
326 media_status_t status = AMediaCodec_releaseOutputBuffer(mCodec, buffer.first, false);
327 if (status != AMEDIA_OK) {
328 LOG(ERROR) << "VideoDecoder: received error in releasing output buffer.";
329 }
330 mDecodedBuffers.pop();
331 return true;
332 }
333
sendEosFlag()334 void VideoDecoder::sendEosFlag() {
335 AMediaFormat* format = AMediaCodec_getOutputFormat(mCodec);
336 int outputFormat;
337 AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, &outputFormat);
338 PixelFormat prebuiltFormat = toPixelFormat(outputFormat);
339 InputFrame inputFrame(5, 5, prebuiltFormat, 5,
340 reinterpret_cast<unsigned char*>(
341 const_cast<char*>(kEndOfInputStreamFlag)));
342 mEngine->dispatchInputFrame(mConfig.stream_id(), 0, inputFrame);
343 }
344
345 } // namespace input_manager
346 } // namespace runner
347 } // namespace computepipe
348 } // namespace automotive
349 } // namespace android
350