1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "media/base/android/media_decoder_job.h"
6
7 #include "base/bind.h"
8 #include "base/callback_helpers.h"
9 #include "base/debug/trace_event.h"
10 #include "base/message_loop/message_loop.h"
11 #include "media/base/android/media_codec_bridge.h"
12 #include "media/base/bind_to_loop.h"
13 #include "media/base/buffers.h"
14
15 namespace media {
16
17 // Timeout value for media codec operations. Because the first
18 // DequeInputBuffer() can take about 150 milliseconds, use 250 milliseconds
19 // here. See http://b/9357571.
20 static const int kMediaCodecTimeoutInMilliseconds = 250;
21
MediaDecoderJob(const scoped_refptr<base::MessageLoopProxy> & decoder_loop,MediaCodecBridge * media_codec_bridge,const base::Closure & request_data_cb)22 MediaDecoderJob::MediaDecoderJob(
23 const scoped_refptr<base::MessageLoopProxy>& decoder_loop,
24 MediaCodecBridge* media_codec_bridge,
25 const base::Closure& request_data_cb)
26 : ui_loop_(base::MessageLoopProxy::current()),
27 decoder_loop_(decoder_loop),
28 media_codec_bridge_(media_codec_bridge),
29 needs_flush_(false),
30 input_eos_encountered_(false),
31 output_eos_encountered_(false),
32 skip_eos_enqueue_(true),
33 prerolling_(true),
34 weak_this_(this),
35 request_data_cb_(request_data_cb),
36 access_unit_index_(0),
37 input_buf_index_(-1),
38 stop_decode_pending_(false),
39 destroy_pending_(false) {
40 }
41
~MediaDecoderJob()42 MediaDecoderJob::~MediaDecoderJob() {}
43
OnDataReceived(const DemuxerData & data)44 void MediaDecoderJob::OnDataReceived(const DemuxerData& data) {
45 DVLOG(1) << __FUNCTION__ << ": " << data.access_units.size() << " units";
46 DCHECK(ui_loop_->BelongsToCurrentThread());
47 DCHECK(!on_data_received_cb_.is_null());
48
49 TRACE_EVENT_ASYNC_END2(
50 "media", "MediaDecoderJob::RequestData", this,
51 "Data type", data.type == media::DemuxerStream::AUDIO ? "AUDIO" : "VIDEO",
52 "Units read", data.access_units.size());
53
54 base::Closure done_cb = base::ResetAndReturn(&on_data_received_cb_);
55
56 if (stop_decode_pending_) {
57 OnDecodeCompleted(MEDIA_CODEC_STOPPED, kNoTimestamp(), 0);
58 return;
59 }
60
61 access_unit_index_ = 0;
62 received_data_ = data;
63 done_cb.Run();
64 }
65
Prefetch(const base::Closure & prefetch_cb)66 void MediaDecoderJob::Prefetch(const base::Closure& prefetch_cb) {
67 DCHECK(ui_loop_->BelongsToCurrentThread());
68 DCHECK(on_data_received_cb_.is_null());
69 DCHECK(decode_cb_.is_null());
70
71 if (HasData()) {
72 DVLOG(1) << __FUNCTION__ << " : using previously received data";
73 ui_loop_->PostTask(FROM_HERE, prefetch_cb);
74 return;
75 }
76
77 DVLOG(1) << __FUNCTION__ << " : requesting data";
78 RequestData(prefetch_cb);
79 }
80
Decode(const base::TimeTicks & start_time_ticks,const base::TimeDelta & start_presentation_timestamp,const DecoderCallback & callback)81 bool MediaDecoderJob::Decode(
82 const base::TimeTicks& start_time_ticks,
83 const base::TimeDelta& start_presentation_timestamp,
84 const DecoderCallback& callback) {
85 DCHECK(decode_cb_.is_null());
86 DCHECK(on_data_received_cb_.is_null());
87 DCHECK(ui_loop_->BelongsToCurrentThread());
88
89 decode_cb_ = callback;
90
91 if (!HasData()) {
92 RequestData(base::Bind(&MediaDecoderJob::DecodeNextAccessUnit,
93 base::Unretained(this),
94 start_time_ticks,
95 start_presentation_timestamp));
96 return true;
97 }
98
99 if (DemuxerStream::kConfigChanged ==
100 received_data_.access_units[access_unit_index_].status) {
101 // Clear received data because we need to handle a config change.
102 decode_cb_.Reset();
103 received_data_ = DemuxerData();
104 access_unit_index_ = 0;
105 return false;
106 }
107
108 DecodeNextAccessUnit(start_time_ticks, start_presentation_timestamp);
109 return true;
110 }
111
StopDecode()112 void MediaDecoderJob::StopDecode() {
113 DCHECK(ui_loop_->BelongsToCurrentThread());
114 DCHECK(is_decoding());
115 stop_decode_pending_ = true;
116 }
117
Flush()118 void MediaDecoderJob::Flush() {
119 DCHECK(decode_cb_.is_null());
120
121 // Do nothing, flush when the next Decode() happens.
122 needs_flush_ = true;
123 received_data_ = DemuxerData();
124 input_eos_encountered_ = false;
125 access_unit_index_ = 0;
126 on_data_received_cb_.Reset();
127 }
128
BeginPrerolling(const base::TimeDelta & preroll_timestamp)129 void MediaDecoderJob::BeginPrerolling(
130 const base::TimeDelta& preroll_timestamp) {
131 DVLOG(1) << __FUNCTION__ << "(" << preroll_timestamp.InSecondsF() << ")";
132 DCHECK(ui_loop_->BelongsToCurrentThread());
133 DCHECK(!is_decoding());
134
135 preroll_timestamp_ = preroll_timestamp;
136 prerolling_ = true;
137 }
138
Release()139 void MediaDecoderJob::Release() {
140 DCHECK(ui_loop_->BelongsToCurrentThread());
141 DVLOG(1) << __FUNCTION__;
142
143 // If the decoder job is not waiting for data, and is still decoding, we
144 // cannot delete the job immediately.
145 destroy_pending_ = on_data_received_cb_.is_null() && is_decoding();
146
147 request_data_cb_.Reset();
148 on_data_received_cb_.Reset();
149 decode_cb_.Reset();
150
151 if (destroy_pending_) {
152 DVLOG(1) << __FUNCTION__ << " : delete is pending decode completion";
153 return;
154 }
155
156 delete this;
157 }
158
QueueInputBuffer(const AccessUnit & unit)159 MediaCodecStatus MediaDecoderJob::QueueInputBuffer(const AccessUnit& unit) {
160 DVLOG(1) << __FUNCTION__;
161 DCHECK(decoder_loop_->BelongsToCurrentThread());
162 TRACE_EVENT0("media", __FUNCTION__);
163
164 int input_buf_index = input_buf_index_;
165 input_buf_index_ = -1;
166
167 // TODO(xhwang): Hide DequeueInputBuffer() and the index in MediaCodecBridge.
168 if (input_buf_index == -1) {
169 base::TimeDelta timeout = base::TimeDelta::FromMilliseconds(
170 kMediaCodecTimeoutInMilliseconds);
171 MediaCodecStatus status =
172 media_codec_bridge_->DequeueInputBuffer(timeout, &input_buf_index);
173 if (status != MEDIA_CODEC_OK) {
174 DVLOG(1) << "DequeueInputBuffer fails: " << status;
175 return status;
176 }
177 }
178
179 // TODO(qinmin): skip frames if video is falling far behind.
180 DCHECK_GE(input_buf_index, 0);
181 if (unit.end_of_stream || unit.data.empty()) {
182 media_codec_bridge_->QueueEOS(input_buf_index);
183 return MEDIA_CODEC_INPUT_END_OF_STREAM;
184 }
185
186 if (unit.key_id.empty() || unit.iv.empty()) {
187 DCHECK(unit.iv.empty() || !unit.key_id.empty());
188 return media_codec_bridge_->QueueInputBuffer(
189 input_buf_index, &unit.data[0], unit.data.size(), unit.timestamp);
190 }
191
192 MediaCodecStatus status = media_codec_bridge_->QueueSecureInputBuffer(
193 input_buf_index,
194 &unit.data[0], unit.data.size(),
195 reinterpret_cast<const uint8*>(&unit.key_id[0]), unit.key_id.size(),
196 reinterpret_cast<const uint8*>(&unit.iv[0]), unit.iv.size(),
197 unit.subsamples.empty() ? NULL : &unit.subsamples[0],
198 unit.subsamples.size(),
199 unit.timestamp);
200
201 // In case of MEDIA_CODEC_NO_KEY, we must reuse the |input_buf_index_|.
202 // Otherwise MediaDrm will report errors.
203 if (status == MEDIA_CODEC_NO_KEY)
204 input_buf_index_ = input_buf_index;
205
206 return status;
207 }
208
HasData() const209 bool MediaDecoderJob::HasData() const {
210 DCHECK(ui_loop_->BelongsToCurrentThread());
211 // When |input_eos_encountered_| is set, |access_units| must not be empty and
212 // |access_unit_index_| must be pointing to an EOS unit. We'll reuse this
213 // unit to flush the decoder until we hit output EOS.
214 DCHECK(!input_eos_encountered_ ||
215 (received_data_.access_units.size() > 0 &&
216 access_unit_index_ < received_data_.access_units.size()))
217 << " (access_units.size(): " << received_data_.access_units.size()
218 << ", access_unit_index_: " << access_unit_index_ << ")";
219 return access_unit_index_ < received_data_.access_units.size() ||
220 input_eos_encountered_;
221 }
222
RequestData(const base::Closure & done_cb)223 void MediaDecoderJob::RequestData(const base::Closure& done_cb) {
224 DVLOG(1) << __FUNCTION__;
225 DCHECK(ui_loop_->BelongsToCurrentThread());
226 DCHECK(on_data_received_cb_.is_null());
227 DCHECK(!input_eos_encountered_);
228
229 TRACE_EVENT_ASYNC_BEGIN0("media", "MediaDecoderJob::RequestData", this);
230
231 received_data_ = DemuxerData();
232 access_unit_index_ = 0;
233 on_data_received_cb_ = done_cb;
234
235 request_data_cb_.Run();
236 }
237
DecodeNextAccessUnit(const base::TimeTicks & start_time_ticks,const base::TimeDelta & start_presentation_timestamp)238 void MediaDecoderJob::DecodeNextAccessUnit(
239 const base::TimeTicks& start_time_ticks,
240 const base::TimeDelta& start_presentation_timestamp) {
241 DCHECK(ui_loop_->BelongsToCurrentThread());
242 DCHECK(!decode_cb_.is_null());
243
244 // If the first access unit is a config change, request the player to dequeue
245 // the input buffer again so that it can request config data.
246 if (received_data_.access_units[access_unit_index_].status ==
247 DemuxerStream::kConfigChanged) {
248 ui_loop_->PostTask(FROM_HERE,
249 base::Bind(&MediaDecoderJob::OnDecodeCompleted,
250 base::Unretained(this),
251 MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER,
252 kNoTimestamp(),
253 0));
254 return;
255 }
256
257 decoder_loop_->PostTask(FROM_HERE, base::Bind(
258 &MediaDecoderJob::DecodeInternal, base::Unretained(this),
259 received_data_.access_units[access_unit_index_],
260 start_time_ticks, start_presentation_timestamp, needs_flush_,
261 media::BindToLoop(ui_loop_, base::Bind(
262 &MediaDecoderJob::OnDecodeCompleted, base::Unretained(this)))));
263 needs_flush_ = false;
264 }
265
DecodeInternal(const AccessUnit & unit,const base::TimeTicks & start_time_ticks,const base::TimeDelta & start_presentation_timestamp,bool needs_flush,const MediaDecoderJob::DecoderCallback & callback)266 void MediaDecoderJob::DecodeInternal(
267 const AccessUnit& unit,
268 const base::TimeTicks& start_time_ticks,
269 const base::TimeDelta& start_presentation_timestamp,
270 bool needs_flush,
271 const MediaDecoderJob::DecoderCallback& callback) {
272 DVLOG(1) << __FUNCTION__;
273 DCHECK(decoder_loop_->BelongsToCurrentThread());
274 TRACE_EVENT0("media", __FUNCTION__);
275
276 if (needs_flush) {
277 DVLOG(1) << "DecodeInternal needs flush.";
278 input_eos_encountered_ = false;
279 output_eos_encountered_ = false;
280 MediaCodecStatus reset_status = media_codec_bridge_->Reset();
281 if (MEDIA_CODEC_OK != reset_status) {
282 callback.Run(reset_status, kNoTimestamp(), 0);
283 return;
284 }
285 }
286
287 // Once output EOS has occurred, we should not be asked to decode again.
288 // MediaCodec has undefined behavior if similarly asked to decode after output
289 // EOS.
290 DCHECK(!output_eos_encountered_);
291
292 // For aborted access unit, just skip it and inform the player.
293 if (unit.status == DemuxerStream::kAborted) {
294 // TODO(qinmin): use a new enum instead of MEDIA_CODEC_STOPPED.
295 callback.Run(MEDIA_CODEC_STOPPED, kNoTimestamp(), 0);
296 return;
297 }
298
299 if (skip_eos_enqueue_) {
300 if (unit.end_of_stream || unit.data.empty()) {
301 input_eos_encountered_ = true;
302 output_eos_encountered_ = true;
303 callback.Run(MEDIA_CODEC_OUTPUT_END_OF_STREAM, kNoTimestamp(), 0);
304 return;
305 }
306
307 skip_eos_enqueue_ = false;
308 }
309
310 MediaCodecStatus input_status = MEDIA_CODEC_INPUT_END_OF_STREAM;
311 if (!input_eos_encountered_) {
312 input_status = QueueInputBuffer(unit);
313 if (input_status == MEDIA_CODEC_INPUT_END_OF_STREAM) {
314 input_eos_encountered_ = true;
315 } else if (input_status != MEDIA_CODEC_OK) {
316 callback.Run(input_status, kNoTimestamp(), 0);
317 return;
318 }
319 }
320
321 int buffer_index = 0;
322 size_t offset = 0;
323 size_t size = 0;
324 base::TimeDelta presentation_timestamp;
325
326 base::TimeDelta timeout = base::TimeDelta::FromMilliseconds(
327 kMediaCodecTimeoutInMilliseconds);
328
329 MediaCodecStatus status =
330 media_codec_bridge_->DequeueOutputBuffer(timeout,
331 &buffer_index,
332 &offset,
333 &size,
334 &presentation_timestamp,
335 &output_eos_encountered_,
336 NULL);
337
338 if (status != MEDIA_CODEC_OK) {
339 if (status == MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED &&
340 !media_codec_bridge_->GetOutputBuffers()) {
341 status = MEDIA_CODEC_ERROR;
342 }
343 callback.Run(status, kNoTimestamp(), 0);
344 return;
345 }
346
347 // TODO(xhwang/qinmin): This logic is correct but strange. Clean it up.
348 if (output_eos_encountered_)
349 status = MEDIA_CODEC_OUTPUT_END_OF_STREAM;
350 else if (input_status == MEDIA_CODEC_INPUT_END_OF_STREAM)
351 status = MEDIA_CODEC_INPUT_END_OF_STREAM;
352
353 // Check whether we need to render the output.
354 // TODO(qinmin): comparing most recently queued input's |unit.timestamp| with
355 // |preroll_timestamp_| is not accurate due to data reordering and possible
356 // input queueing without immediate dequeue when |input_status| !=
357 // |MEDIA_CODEC_OK|. Need to use the |presentation_timestamp| for video, and
358 // use |size| to calculate the timestamp for audio. See
359 // http://crbug.com/310823 and http://b/11356652.
360 bool render_output = unit.timestamp >= preroll_timestamp_ &&
361 (status != MEDIA_CODEC_OUTPUT_END_OF_STREAM || size != 0u);
362 base::TimeDelta time_to_render;
363 DCHECK(!start_time_ticks.is_null());
364 if (render_output && ComputeTimeToRender()) {
365 time_to_render = presentation_timestamp - (base::TimeTicks::Now() -
366 start_time_ticks + start_presentation_timestamp);
367 }
368
369 if (time_to_render > base::TimeDelta()) {
370 decoder_loop_->PostDelayedTask(
371 FROM_HERE,
372 base::Bind(&MediaDecoderJob::ReleaseOutputBuffer,
373 weak_this_.GetWeakPtr(), buffer_index, size, render_output,
374 base::Bind(callback, status, presentation_timestamp)),
375 time_to_render);
376 return;
377 }
378
379 // TODO(qinmin): The codec is lagging behind, need to recalculate the
380 // |start_presentation_timestamp_| and |start_time_ticks_| in
381 // media_source_player.cc.
382 DVLOG(1) << "codec is lagging behind :" << time_to_render.InMicroseconds();
383 if (render_output) {
384 // The player won't expect a timestamp smaller than the
385 // |start_presentation_timestamp|. However, this could happen due to decoder
386 // errors.
387 presentation_timestamp = std::max(
388 presentation_timestamp, start_presentation_timestamp);
389 } else {
390 presentation_timestamp = kNoTimestamp();
391 }
392 ReleaseOutputCompletionCallback completion_callback = base::Bind(
393 callback, status, presentation_timestamp);
394 ReleaseOutputBuffer(buffer_index, size, render_output, completion_callback);
395 }
396
OnDecodeCompleted(MediaCodecStatus status,const base::TimeDelta & presentation_timestamp,size_t audio_output_bytes)397 void MediaDecoderJob::OnDecodeCompleted(
398 MediaCodecStatus status, const base::TimeDelta& presentation_timestamp,
399 size_t audio_output_bytes) {
400 DCHECK(ui_loop_->BelongsToCurrentThread());
401
402 if (destroy_pending_) {
403 DVLOG(1) << __FUNCTION__ << " : completing pending deletion";
404 delete this;
405 return;
406 }
407
408 DCHECK(!decode_cb_.is_null());
409
410 // If output was queued for rendering, then we have completed prerolling.
411 if (presentation_timestamp != kNoTimestamp())
412 prerolling_ = false;
413
414 switch (status) {
415 case MEDIA_CODEC_OK:
416 case MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER:
417 case MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED:
418 case MEDIA_CODEC_OUTPUT_FORMAT_CHANGED:
419 case MEDIA_CODEC_OUTPUT_END_OF_STREAM:
420 if (!input_eos_encountered_)
421 access_unit_index_++;
422 break;
423
424 case MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER:
425 case MEDIA_CODEC_INPUT_END_OF_STREAM:
426 case MEDIA_CODEC_NO_KEY:
427 case MEDIA_CODEC_STOPPED:
428 case MEDIA_CODEC_ERROR:
429 // Do nothing.
430 break;
431 };
432
433 stop_decode_pending_ = false;
434 base::ResetAndReturn(&decode_cb_).Run(status, presentation_timestamp,
435 audio_output_bytes);
436 }
437
438 } // namespace media
439