1 /*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "MediaCodec"
19 #include <utils/Log.h>
20
21 #include <inttypes.h>
22 #include <stdlib.h>
23
24 #include "include/SecureBuffer.h"
25 #include "include/SharedMemoryBuffer.h"
26 #include "include/SoftwareRenderer.h"
27 #include "StagefrightPluginLoader.h"
28
29 #include <android/hardware/cas/native/1.0/IDescrambler.h>
30
31 #include <binder/IMemory.h>
32 #include <binder/IPCThreadState.h>
33 #include <binder/IServiceManager.h>
34 #include <binder/MemoryDealer.h>
35 #include <cutils/properties.h>
36 #include <gui/BufferQueue.h>
37 #include <gui/Surface.h>
38 #include <media/ICrypto.h>
39 #include <media/IOMX.h>
40 #include <media/IResourceManagerService.h>
41 #include <media/MediaCodecBuffer.h>
42 #include <media/MediaAnalyticsItem.h>
43 #include <media/stagefright/foundation/ABuffer.h>
44 #include <media/stagefright/foundation/ADebug.h>
45 #include <media/stagefright/foundation/AMessage.h>
46 #include <media/stagefright/foundation/AString.h>
47 #include <media/stagefright/foundation/AUtils.h>
48 #include <media/stagefright/foundation/avc_utils.h>
49 #include <media/stagefright/foundation/hexdump.h>
50 #include <media/stagefright/ACodec.h>
51 #include <media/stagefright/BufferProducerWrapper.h>
52 #include <media/stagefright/MediaCodec.h>
53 #include <media/stagefright/MediaCodecList.h>
54 #include <media/stagefright/MediaDefs.h>
55 #include <media/stagefright/MediaErrors.h>
56 #include <media/stagefright/MediaFilter.h>
57 #include <media/stagefright/OMXClient.h>
58 #include <media/stagefright/PersistentSurface.h>
59 #include <media/stagefright/SurfaceUtils.h>
60 #include <mediautils/BatteryNotifier.h>
61 #include <private/android_filesystem_config.h>
62 #include <utils/Singleton.h>
63
64 namespace android {
65
66 // key for media statistics
67 static const char *kCodecKeyName = "codec";
68 // attrs for media statistics
69 // NB: these are matched with public Java API constants defined
70 // in frameworks/base/media/java/android/media/MediaCodec.java
71 // These must be kept synchronized with the constants there.
72 static const char *kCodecCodec = "android.media.mediacodec.codec"; /* e.g. OMX.google.aac.decoder */
73 static const char *kCodecMime = "android.media.mediacodec.mime"; /* e.g. audio/mime */
74 static const char *kCodecMode = "android.media.mediacodec.mode"; /* audio, video */
75 static const char *kCodecModeVideo = "video"; /* values returned for kCodecMode */
76 static const char *kCodecModeAudio = "audio";
77 static const char *kCodecEncoder = "android.media.mediacodec.encoder"; /* 0,1 */
78 static const char *kCodecSecure = "android.media.mediacodec.secure"; /* 0, 1 */
79 static const char *kCodecWidth = "android.media.mediacodec.width"; /* 0..n */
80 static const char *kCodecHeight = "android.media.mediacodec.height"; /* 0..n */
81 static const char *kCodecRotation = "android.media.mediacodec.rotation-degrees"; /* 0/90/180/270 */
82
83 // NB: These are not yet exposed as public Java API constants.
84 static const char *kCodecCrypto = "android.media.mediacodec.crypto"; /* 0,1 */
85 static const char *kCodecProfile = "android.media.mediacodec.profile"; /* 0..n */
86 static const char *kCodecLevel = "android.media.mediacodec.level"; /* 0..n */
87 static const char *kCodecMaxWidth = "android.media.mediacodec.maxwidth"; /* 0..n */
88 static const char *kCodecMaxHeight = "android.media.mediacodec.maxheight"; /* 0..n */
89 static const char *kCodecError = "android.media.mediacodec.errcode";
90 static const char *kCodecErrorState = "android.media.mediacodec.errstate";
91 static const char *kCodecLatencyMax = "android.media.mediacodec.latency.max"; /* in us */
92 static const char *kCodecLatencyMin = "android.media.mediacodec.latency.min"; /* in us */
93 static const char *kCodecLatencyAvg = "android.media.mediacodec.latency.avg"; /* in us */
94 static const char *kCodecLatencyCount = "android.media.mediacodec.latency.n";
95 static const char *kCodecLatencyHist = "android.media.mediacodec.latency.hist"; /* in us */
96 static const char *kCodecLatencyUnknown = "android.media.mediacodec.latency.unknown";
97
98 // the kCodecRecent* fields appear only in getMetrics() results
99 static const char *kCodecRecentLatencyMax = "android.media.mediacodec.recent.max"; /* in us */
100 static const char *kCodecRecentLatencyMin = "android.media.mediacodec.recent.min"; /* in us */
101 static const char *kCodecRecentLatencyAvg = "android.media.mediacodec.recent.avg"; /* in us */
102 static const char *kCodecRecentLatencyCount = "android.media.mediacodec.recent.n";
103 static const char *kCodecRecentLatencyHist = "android.media.mediacodec.recent.hist"; /* in us */
104
105 // XXX suppress until we get our representation right
106 static bool kEmitHistogram = false;
107
108
getId(const sp<IResourceManagerClient> & client)109 static int64_t getId(const sp<IResourceManagerClient> &client) {
110 return (int64_t) client.get();
111 }
112
isResourceError(status_t err)113 static bool isResourceError(status_t err) {
114 return (err == NO_MEMORY);
115 }
116
117 static const int kMaxRetry = 2;
118 static const int kMaxReclaimWaitTimeInUs = 500000; // 0.5s
119 static const int kNumBuffersAlign = 16;
120
121 ////////////////////////////////////////////////////////////////////////////////
122
123 struct ResourceManagerClient : public BnResourceManagerClient {
ResourceManagerClientandroid::ResourceManagerClient124 explicit ResourceManagerClient(MediaCodec* codec) : mMediaCodec(codec) {}
125
reclaimResourceandroid::ResourceManagerClient126 virtual bool reclaimResource() {
127 sp<MediaCodec> codec = mMediaCodec.promote();
128 if (codec == NULL) {
129 // codec is already gone.
130 return true;
131 }
132 status_t err = codec->reclaim();
133 if (err == WOULD_BLOCK) {
134 ALOGD("Wait for the client to release codec.");
135 usleep(kMaxReclaimWaitTimeInUs);
136 ALOGD("Try to reclaim again.");
137 err = codec->reclaim(true /* force */);
138 }
139 if (err != OK) {
140 ALOGW("ResourceManagerClient failed to release codec with err %d", err);
141 }
142 return (err == OK);
143 }
144
getNameandroid::ResourceManagerClient145 virtual String8 getName() {
146 String8 ret;
147 sp<MediaCodec> codec = mMediaCodec.promote();
148 if (codec == NULL) {
149 // codec is already gone.
150 return ret;
151 }
152
153 AString name;
154 if (codec->getName(&name) == OK) {
155 ret.setTo(name.c_str());
156 }
157 return ret;
158 }
159
160 protected:
~ResourceManagerClientandroid::ResourceManagerClient161 virtual ~ResourceManagerClient() {}
162
163 private:
164 wp<MediaCodec> mMediaCodec;
165
166 DISALLOW_EVIL_CONSTRUCTORS(ResourceManagerClient);
167 };
168
ResourceManagerServiceProxy(pid_t pid)169 MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy(pid_t pid)
170 : mPid(pid) {
171 if (mPid == MediaCodec::kNoPid) {
172 mPid = IPCThreadState::self()->getCallingPid();
173 }
174 }
175
~ResourceManagerServiceProxy()176 MediaCodec::ResourceManagerServiceProxy::~ResourceManagerServiceProxy() {
177 if (mService != NULL) {
178 IInterface::asBinder(mService)->unlinkToDeath(this);
179 }
180 }
181
init()182 void MediaCodec::ResourceManagerServiceProxy::init() {
183 sp<IServiceManager> sm = defaultServiceManager();
184 sp<IBinder> binder = sm->getService(String16("media.resource_manager"));
185 mService = interface_cast<IResourceManagerService>(binder);
186 if (mService == NULL) {
187 ALOGE("Failed to get ResourceManagerService");
188 return;
189 }
190 IInterface::asBinder(mService)->linkToDeath(this);
191 }
192
binderDied(const wp<IBinder> &)193 void MediaCodec::ResourceManagerServiceProxy::binderDied(const wp<IBinder>& /*who*/) {
194 ALOGW("ResourceManagerService died.");
195 Mutex::Autolock _l(mLock);
196 mService.clear();
197 }
198
addResource(int64_t clientId,const sp<IResourceManagerClient> & client,const Vector<MediaResource> & resources)199 void MediaCodec::ResourceManagerServiceProxy::addResource(
200 int64_t clientId,
201 const sp<IResourceManagerClient> &client,
202 const Vector<MediaResource> &resources) {
203 Mutex::Autolock _l(mLock);
204 if (mService == NULL) {
205 return;
206 }
207 mService->addResource(mPid, clientId, client, resources);
208 }
209
removeResource(int64_t clientId)210 void MediaCodec::ResourceManagerServiceProxy::removeResource(int64_t clientId) {
211 Mutex::Autolock _l(mLock);
212 if (mService == NULL) {
213 return;
214 }
215 mService->removeResource(mPid, clientId);
216 }
217
reclaimResource(const Vector<MediaResource> & resources)218 bool MediaCodec::ResourceManagerServiceProxy::reclaimResource(
219 const Vector<MediaResource> &resources) {
220 Mutex::Autolock _l(mLock);
221 if (mService == NULL) {
222 return false;
223 }
224 return mService->reclaimResource(mPid, resources);
225 }
226
227 ////////////////////////////////////////////////////////////////////////////////
228
BufferInfo()229 MediaCodec::BufferInfo::BufferInfo() : mOwnedByClient(false) {}
230
231 ////////////////////////////////////////////////////////////////////////////////
232
233 namespace {
234
235 enum {
236 kWhatFillThisBuffer = 'fill',
237 kWhatDrainThisBuffer = 'drai',
238 kWhatEOS = 'eos ',
239 kWhatStartCompleted = 'Scom',
240 kWhatStopCompleted = 'scom',
241 kWhatReleaseCompleted = 'rcom',
242 kWhatFlushCompleted = 'fcom',
243 kWhatError = 'erro',
244 kWhatComponentAllocated = 'cAll',
245 kWhatComponentConfigured = 'cCon',
246 kWhatInputSurfaceCreated = 'isfc',
247 kWhatInputSurfaceAccepted = 'isfa',
248 kWhatSignaledInputEOS = 'seos',
249 kWhatOutputFramesRendered = 'outR',
250 kWhatOutputBuffersChanged = 'outC',
251 };
252
253 class BufferCallback : public CodecBase::BufferCallback {
254 public:
255 explicit BufferCallback(const sp<AMessage> ¬ify);
256 virtual ~BufferCallback() = default;
257
258 virtual void onInputBufferAvailable(
259 size_t index, const sp<MediaCodecBuffer> &buffer) override;
260 virtual void onOutputBufferAvailable(
261 size_t index, const sp<MediaCodecBuffer> &buffer) override;
262 private:
263 const sp<AMessage> mNotify;
264 };
265
BufferCallback(const sp<AMessage> & notify)266 BufferCallback::BufferCallback(const sp<AMessage> ¬ify)
267 : mNotify(notify) {}
268
onInputBufferAvailable(size_t index,const sp<MediaCodecBuffer> & buffer)269 void BufferCallback::onInputBufferAvailable(
270 size_t index, const sp<MediaCodecBuffer> &buffer) {
271 sp<AMessage> notify(mNotify->dup());
272 notify->setInt32("what", kWhatFillThisBuffer);
273 notify->setSize("index", index);
274 notify->setObject("buffer", buffer);
275 notify->post();
276 }
277
onOutputBufferAvailable(size_t index,const sp<MediaCodecBuffer> & buffer)278 void BufferCallback::onOutputBufferAvailable(
279 size_t index, const sp<MediaCodecBuffer> &buffer) {
280 sp<AMessage> notify(mNotify->dup());
281 notify->setInt32("what", kWhatDrainThisBuffer);
282 notify->setSize("index", index);
283 notify->setObject("buffer", buffer);
284 notify->post();
285 }
286
287 class CodecCallback : public CodecBase::CodecCallback {
288 public:
289 explicit CodecCallback(const sp<AMessage> ¬ify);
290 virtual ~CodecCallback() = default;
291
292 virtual void onEos(status_t err) override;
293 virtual void onStartCompleted() override;
294 virtual void onStopCompleted() override;
295 virtual void onReleaseCompleted() override;
296 virtual void onFlushCompleted() override;
297 virtual void onError(status_t err, enum ActionCode actionCode) override;
298 virtual void onComponentAllocated(const char *componentName) override;
299 virtual void onComponentConfigured(
300 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) override;
301 virtual void onInputSurfaceCreated(
302 const sp<AMessage> &inputFormat,
303 const sp<AMessage> &outputFormat,
304 const sp<BufferProducerWrapper> &inputSurface) override;
305 virtual void onInputSurfaceCreationFailed(status_t err) override;
306 virtual void onInputSurfaceAccepted(
307 const sp<AMessage> &inputFormat,
308 const sp<AMessage> &outputFormat) override;
309 virtual void onInputSurfaceDeclined(status_t err) override;
310 virtual void onSignaledInputEOS(status_t err) override;
311 virtual void onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) override;
312 virtual void onOutputBuffersChanged() override;
313 private:
314 const sp<AMessage> mNotify;
315 };
316
CodecCallback(const sp<AMessage> & notify)317 CodecCallback::CodecCallback(const sp<AMessage> ¬ify) : mNotify(notify) {}
318
onEos(status_t err)319 void CodecCallback::onEos(status_t err) {
320 sp<AMessage> notify(mNotify->dup());
321 notify->setInt32("what", kWhatEOS);
322 notify->setInt32("err", err);
323 notify->post();
324 }
325
onStartCompleted()326 void CodecCallback::onStartCompleted() {
327 sp<AMessage> notify(mNotify->dup());
328 notify->setInt32("what", kWhatStartCompleted);
329 notify->post();
330 }
331
onStopCompleted()332 void CodecCallback::onStopCompleted() {
333 sp<AMessage> notify(mNotify->dup());
334 notify->setInt32("what", kWhatStopCompleted);
335 notify->post();
336 }
337
onReleaseCompleted()338 void CodecCallback::onReleaseCompleted() {
339 sp<AMessage> notify(mNotify->dup());
340 notify->setInt32("what", kWhatReleaseCompleted);
341 notify->post();
342 }
343
onFlushCompleted()344 void CodecCallback::onFlushCompleted() {
345 sp<AMessage> notify(mNotify->dup());
346 notify->setInt32("what", kWhatFlushCompleted);
347 notify->post();
348 }
349
onError(status_t err,enum ActionCode actionCode)350 void CodecCallback::onError(status_t err, enum ActionCode actionCode) {
351 sp<AMessage> notify(mNotify->dup());
352 notify->setInt32("what", kWhatError);
353 notify->setInt32("err", err);
354 notify->setInt32("actionCode", actionCode);
355 notify->post();
356 }
357
onComponentAllocated(const char * componentName)358 void CodecCallback::onComponentAllocated(const char *componentName) {
359 sp<AMessage> notify(mNotify->dup());
360 notify->setInt32("what", kWhatComponentAllocated);
361 notify->setString("componentName", componentName);
362 notify->post();
363 }
364
onComponentConfigured(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat)365 void CodecCallback::onComponentConfigured(
366 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) {
367 sp<AMessage> notify(mNotify->dup());
368 notify->setInt32("what", kWhatComponentConfigured);
369 notify->setMessage("input-format", inputFormat);
370 notify->setMessage("output-format", outputFormat);
371 notify->post();
372 }
373
onInputSurfaceCreated(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat,const sp<BufferProducerWrapper> & inputSurface)374 void CodecCallback::onInputSurfaceCreated(
375 const sp<AMessage> &inputFormat,
376 const sp<AMessage> &outputFormat,
377 const sp<BufferProducerWrapper> &inputSurface) {
378 sp<AMessage> notify(mNotify->dup());
379 notify->setInt32("what", kWhatInputSurfaceCreated);
380 notify->setMessage("input-format", inputFormat);
381 notify->setMessage("output-format", outputFormat);
382 notify->setObject("input-surface", inputSurface);
383 notify->post();
384 }
385
onInputSurfaceCreationFailed(status_t err)386 void CodecCallback::onInputSurfaceCreationFailed(status_t err) {
387 sp<AMessage> notify(mNotify->dup());
388 notify->setInt32("what", kWhatInputSurfaceCreated);
389 notify->setInt32("err", err);
390 notify->post();
391 }
392
onInputSurfaceAccepted(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat)393 void CodecCallback::onInputSurfaceAccepted(
394 const sp<AMessage> &inputFormat,
395 const sp<AMessage> &outputFormat) {
396 sp<AMessage> notify(mNotify->dup());
397 notify->setInt32("what", kWhatInputSurfaceAccepted);
398 notify->setMessage("input-format", inputFormat);
399 notify->setMessage("output-format", outputFormat);
400 notify->post();
401 }
402
onInputSurfaceDeclined(status_t err)403 void CodecCallback::onInputSurfaceDeclined(status_t err) {
404 sp<AMessage> notify(mNotify->dup());
405 notify->setInt32("what", kWhatInputSurfaceAccepted);
406 notify->setInt32("err", err);
407 notify->post();
408 }
409
onSignaledInputEOS(status_t err)410 void CodecCallback::onSignaledInputEOS(status_t err) {
411 sp<AMessage> notify(mNotify->dup());
412 notify->setInt32("what", kWhatSignaledInputEOS);
413 if (err != OK) {
414 notify->setInt32("err", err);
415 }
416 notify->post();
417 }
418
onOutputFramesRendered(const std::list<FrameRenderTracker::Info> & done)419 void CodecCallback::onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) {
420 sp<AMessage> notify(mNotify->dup());
421 notify->setInt32("what", kWhatOutputFramesRendered);
422 if (MediaCodec::CreateFramesRenderedMessage(done, notify)) {
423 notify->post();
424 }
425 }
426
onOutputBuffersChanged()427 void CodecCallback::onOutputBuffersChanged() {
428 sp<AMessage> notify(mNotify->dup());
429 notify->setInt32("what", kWhatOutputBuffersChanged);
430 notify->post();
431 }
432
433 } // namespace
434
435 ////////////////////////////////////////////////////////////////////////////////
436
437 // static
CreateByType(const sp<ALooper> & looper,const AString & mime,bool encoder,status_t * err,pid_t pid,uid_t uid)438 sp<MediaCodec> MediaCodec::CreateByType(
439 const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err, pid_t pid,
440 uid_t uid) {
441 Vector<AString> matchingCodecs;
442
443 MediaCodecList::findMatchingCodecs(
444 mime.c_str(),
445 encoder,
446 0,
447 &matchingCodecs);
448
449 if (err != NULL) {
450 *err = NAME_NOT_FOUND;
451 }
452 for (size_t i = 0; i < matchingCodecs.size(); ++i) {
453 sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);
454 AString componentName = matchingCodecs[i];
455 status_t ret = codec->init(componentName);
456 if (err != NULL) {
457 *err = ret;
458 }
459 if (ret == OK) {
460 return codec;
461 }
462 ALOGD("Allocating component '%s' failed (%d), try next one.",
463 componentName.c_str(), ret);
464 }
465 return NULL;
466 }
467
468 // static
CreateByComponentName(const sp<ALooper> & looper,const AString & name,status_t * err,pid_t pid,uid_t uid)469 sp<MediaCodec> MediaCodec::CreateByComponentName(
470 const sp<ALooper> &looper, const AString &name, status_t *err, pid_t pid, uid_t uid) {
471 sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);
472
473 const status_t ret = codec->init(name);
474 if (err != NULL) {
475 *err = ret;
476 }
477 return ret == OK ? codec : NULL; // NULL deallocates codec.
478 }
479
480 // static
CreatePersistentInputSurface()481 sp<PersistentSurface> MediaCodec::CreatePersistentInputSurface() {
482 // allow plugin to create surface
483 sp<PersistentSurface> pluginSurface =
484 StagefrightPluginLoader::GetCCodecInstance()->createInputSurface();
485 if (pluginSurface != nullptr) {
486 return pluginSurface;
487 }
488
489 OMXClient client;
490 if (client.connect() != OK) {
491 ALOGE("Failed to connect to OMX to create persistent input surface.");
492 return NULL;
493 }
494
495 sp<IOMX> omx = client.interface();
496
497 sp<IGraphicBufferProducer> bufferProducer;
498 sp<IGraphicBufferSource> bufferSource;
499
500 status_t err = omx->createInputSurface(&bufferProducer, &bufferSource);
501
502 if (err != OK) {
503 ALOGE("Failed to create persistent input surface.");
504 return NULL;
505 }
506
507 return new PersistentSurface(bufferProducer, bufferSource);
508 }
509
MediaCodec(const sp<ALooper> & looper,pid_t pid,uid_t uid)510 MediaCodec::MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid)
511 : mState(UNINITIALIZED),
512 mReleasedByResourceManager(false),
513 mLooper(looper),
514 mCodec(NULL),
515 mReplyID(0),
516 mFlags(0),
517 mStickyError(OK),
518 mSoftRenderer(NULL),
519 mAnalyticsItem(NULL),
520 mResourceManagerClient(new ResourceManagerClient(this)),
521 mResourceManagerService(new ResourceManagerServiceProxy(pid)),
522 mBatteryStatNotified(false),
523 mIsVideo(false),
524 mVideoWidth(0),
525 mVideoHeight(0),
526 mRotationDegrees(0),
527 mDequeueInputTimeoutGeneration(0),
528 mDequeueInputReplyID(0),
529 mDequeueOutputTimeoutGeneration(0),
530 mDequeueOutputReplyID(0),
531 mHaveInputSurface(false),
532 mHavePendingInputBuffers(false),
533 mCpuBoostRequested(false),
534 mLatencyUnknown(0) {
535 if (uid == kNoUid) {
536 mUid = IPCThreadState::self()->getCallingUid();
537 } else {
538 mUid = uid;
539 }
540
541 initAnalyticsItem();
542 }
543
~MediaCodec()544 MediaCodec::~MediaCodec() {
545 CHECK_EQ(mState, UNINITIALIZED);
546 mResourceManagerService->removeResource(getId(mResourceManagerClient));
547
548 flushAnalyticsItem();
549 }
550
initAnalyticsItem()551 void MediaCodec::initAnalyticsItem() {
552 if (mAnalyticsItem == NULL) {
553 mAnalyticsItem = MediaAnalyticsItem::create(kCodecKeyName);
554 }
555
556 mLatencyHist.setup(kLatencyHistBuckets, kLatencyHistWidth, kLatencyHistFloor);
557
558 {
559 Mutex::Autolock al(mRecentLock);
560 for (int i = 0; i<kRecentLatencyFrames; i++) {
561 mRecentSamples[i] = kRecentSampleInvalid;
562 }
563 mRecentHead = 0;
564 }
565 }
566
updateAnalyticsItem()567 void MediaCodec::updateAnalyticsItem() {
568 ALOGV("MediaCodec::updateAnalyticsItem");
569 if (mAnalyticsItem == NULL) {
570 return;
571 }
572
573 if (mLatencyHist.getCount() != 0 ) {
574 mAnalyticsItem->setInt64(kCodecLatencyMax, mLatencyHist.getMax());
575 mAnalyticsItem->setInt64(kCodecLatencyMin, mLatencyHist.getMin());
576 mAnalyticsItem->setInt64(kCodecLatencyAvg, mLatencyHist.getAvg());
577 mAnalyticsItem->setInt64(kCodecLatencyCount, mLatencyHist.getCount());
578
579 if (kEmitHistogram) {
580 // and the histogram itself
581 std::string hist = mLatencyHist.emit();
582 mAnalyticsItem->setCString(kCodecLatencyHist, hist.c_str());
583 }
584 }
585 if (mLatencyUnknown > 0) {
586 mAnalyticsItem->setInt64(kCodecLatencyUnknown, mLatencyUnknown);
587 }
588
589 #if 0
590 // enable for short term, only while debugging
591 updateEphemeralAnalytics(mAnalyticsItem);
592 #endif
593 }
594
updateEphemeralAnalytics(MediaAnalyticsItem * item)595 void MediaCodec::updateEphemeralAnalytics(MediaAnalyticsItem *item) {
596 ALOGD("MediaCodec::updateEphemeralAnalytics()");
597
598 if (item == NULL) {
599 return;
600 }
601
602 Histogram recentHist;
603
604 // build an empty histogram
605 recentHist.setup(kLatencyHistBuckets, kLatencyHistWidth, kLatencyHistFloor);
606
607 // stuff it with the samples in the ring buffer
608 {
609 Mutex::Autolock al(mRecentLock);
610
611 for (int i=0; i<kRecentLatencyFrames; i++) {
612 if (mRecentSamples[i] != kRecentSampleInvalid) {
613 recentHist.insert(mRecentSamples[i]);
614 }
615 }
616 }
617
618
619 // spit the data (if any) into the supplied analytics record
620 if (recentHist.getCount()!= 0 ) {
621 item->setInt64(kCodecRecentLatencyMax, recentHist.getMax());
622 item->setInt64(kCodecRecentLatencyMin, recentHist.getMin());
623 item->setInt64(kCodecRecentLatencyAvg, recentHist.getAvg());
624 item->setInt64(kCodecRecentLatencyCount, recentHist.getCount());
625
626 if (kEmitHistogram) {
627 // and the histogram itself
628 std::string hist = recentHist.emit();
629 item->setCString(kCodecRecentLatencyHist, hist.c_str());
630 }
631 }
632 }
633
flushAnalyticsItem()634 void MediaCodec::flushAnalyticsItem() {
635 updateAnalyticsItem();
636 if (mAnalyticsItem != NULL) {
637 // don't log empty records
638 if (mAnalyticsItem->count() > 0) {
639 mAnalyticsItem->selfrecord();
640 }
641 delete mAnalyticsItem;
642 mAnalyticsItem = NULL;
643 }
644 }
645
setup(int nbuckets,int64_t width,int64_t floor)646 bool MediaCodec::Histogram::setup(int nbuckets, int64_t width, int64_t floor)
647 {
648 if (nbuckets <= 0 || width <= 0) {
649 return false;
650 }
651
652 // get histogram buckets
653 if (nbuckets == mBucketCount && mBuckets != NULL) {
654 // reuse our existing buffer
655 memset(mBuckets, 0, sizeof(*mBuckets) * mBucketCount);
656 } else {
657 // get a new pre-zeroed buffer
658 int64_t *newbuckets = (int64_t *)calloc(nbuckets, sizeof (*mBuckets));
659 if (newbuckets == NULL) {
660 goto bad;
661 }
662 if (mBuckets != NULL)
663 free(mBuckets);
664 mBuckets = newbuckets;
665 }
666
667 mWidth = width;
668 mFloor = floor;
669 mCeiling = floor + nbuckets * width;
670 mBucketCount = nbuckets;
671
672 mMin = INT64_MAX;
673 mMax = INT64_MIN;
674 mSum = 0;
675 mCount = 0;
676 mBelow = mAbove = 0;
677
678 return true;
679
680 bad:
681 if (mBuckets != NULL) {
682 free(mBuckets);
683 mBuckets = NULL;
684 }
685
686 return false;
687 }
688
insert(int64_t sample)689 void MediaCodec::Histogram::insert(int64_t sample)
690 {
691 // histogram is not set up
692 if (mBuckets == NULL) {
693 return;
694 }
695
696 mCount++;
697 mSum += sample;
698 if (mMin > sample) mMin = sample;
699 if (mMax < sample) mMax = sample;
700
701 if (sample < mFloor) {
702 mBelow++;
703 } else if (sample >= mCeiling) {
704 mAbove++;
705 } else {
706 int64_t slot = (sample - mFloor) / mWidth;
707 CHECK(slot < mBucketCount);
708 mBuckets[slot]++;
709 }
710 return;
711 }
712
emit()713 std::string MediaCodec::Histogram::emit()
714 {
715 std::string value;
716 char buffer[64];
717
718 // emits: width,Below{bucket0,bucket1,...., bucketN}above
719 // unconfigured will emit: 0,0{}0
720 // XXX: is this best representation?
721 snprintf(buffer, sizeof(buffer), "%" PRId64 ",%" PRId64 ",%" PRId64 "{",
722 mFloor, mWidth, mBelow);
723 value = buffer;
724 for (int i = 0; i < mBucketCount; i++) {
725 if (i != 0) {
726 value = value + ",";
727 }
728 snprintf(buffer, sizeof(buffer), "%" PRId64, mBuckets[i]);
729 value = value + buffer;
730 }
731 snprintf(buffer, sizeof(buffer), "}%" PRId64 , mAbove);
732 value = value + buffer;
733 return value;
734 }
735
736 // when we send a buffer to the codec;
statsBufferSent(int64_t presentationUs)737 void MediaCodec::statsBufferSent(int64_t presentationUs) {
738
739 // only enqueue if we have a legitimate time
740 if (presentationUs <= 0) {
741 ALOGV("presentation time: %" PRId64, presentationUs);
742 return;
743 }
744
745 const int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
746 BufferFlightTiming_t startdata = { presentationUs, nowNs };
747
748 {
749 // mutex access to mBuffersInFlight and other stats
750 Mutex::Autolock al(mLatencyLock);
751
752
753 // XXX: we *could* make sure that the time is later than the end of queue
754 // as part of a consistency check...
755 mBuffersInFlight.push_back(startdata);
756 }
757 }
758
759 // when we get a buffer back from the codec
statsBufferReceived(int64_t presentationUs)760 void MediaCodec::statsBufferReceived(int64_t presentationUs) {
761
762 CHECK_NE(mState, UNINITIALIZED);
763
764 // mutex access to mBuffersInFlight and other stats
765 Mutex::Autolock al(mLatencyLock);
766
767 // how long this buffer took for the round trip through the codec
768 // NB: pipelining can/will make these times larger. e.g., if each packet
769 // is always 2 msec and we have 3 in flight at any given time, we're going to
770 // see "6 msec" as an answer.
771
772 // ignore stuff with no presentation time
773 if (presentationUs <= 0) {
774 ALOGV("-- returned buffer timestamp %" PRId64 " <= 0, ignore it", presentationUs);
775 mLatencyUnknown++;
776 return;
777 }
778
779 BufferFlightTiming_t startdata;
780 bool valid = false;
781 while (mBuffersInFlight.size() > 0) {
782 startdata = *mBuffersInFlight.begin();
783 ALOGV("-- Looking at startdata. presentation %" PRId64 ", start %" PRId64,
784 startdata.presentationUs, startdata.startedNs);
785 if (startdata.presentationUs == presentationUs) {
786 // a match
787 ALOGV("-- match entry for %" PRId64 ", hits our frame of %" PRId64,
788 startdata.presentationUs, presentationUs);
789 mBuffersInFlight.pop_front();
790 valid = true;
791 break;
792 } else if (startdata.presentationUs < presentationUs) {
793 // we must have missed the match for this, drop it and keep looking
794 ALOGV("-- drop entry for %" PRId64 ", before our frame of %" PRId64,
795 startdata.presentationUs, presentationUs);
796 mBuffersInFlight.pop_front();
797 continue;
798 } else {
799 // head is after, so we don't have a frame for ourselves
800 ALOGV("-- found entry for %" PRId64 ", AFTER our frame of %" PRId64
801 " we have nothing to pair with",
802 startdata.presentationUs, presentationUs);
803 mLatencyUnknown++;
804 return;
805 }
806 }
807 if (!valid) {
808 ALOGV("-- empty queue, so ignore that.");
809 mLatencyUnknown++;
810 return;
811 }
812
813 // nowNs start our calculations
814 const int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
815 int64_t latencyUs = (nowNs - startdata.startedNs + 500) / 1000;
816
817 mLatencyHist.insert(latencyUs);
818
819 // push into the recent samples
820 {
821 Mutex::Autolock al(mRecentLock);
822
823 if (mRecentHead >= kRecentLatencyFrames) {
824 mRecentHead = 0;
825 }
826 mRecentSamples[mRecentHead++] = latencyUs;
827 }
828 }
829
830 // static
PostAndAwaitResponse(const sp<AMessage> & msg,sp<AMessage> * response)831 status_t MediaCodec::PostAndAwaitResponse(
832 const sp<AMessage> &msg, sp<AMessage> *response) {
833 status_t err = msg->postAndAwaitResponse(response);
834
835 if (err != OK) {
836 return err;
837 }
838
839 if (!(*response)->findInt32("err", &err)) {
840 err = OK;
841 }
842
843 return err;
844 }
845
PostReplyWithError(const sp<AReplyToken> & replyID,int32_t err)846 void MediaCodec::PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err) {
847 int32_t finalErr = err;
848 if (mReleasedByResourceManager) {
849 // override the err code if MediaCodec has been released by ResourceManager.
850 finalErr = DEAD_OBJECT;
851 }
852
853 sp<AMessage> response = new AMessage;
854 response->setInt32("err", finalErr);
855 response->postReply(replyID);
856 }
857
CreateCCodec()858 static CodecBase *CreateCCodec() {
859 return StagefrightPluginLoader::GetCCodecInstance()->createCodec();
860 }
861
862 //static
GetCodecBase(const AString & name,const char * owner)863 sp<CodecBase> MediaCodec::GetCodecBase(const AString &name, const char *owner) {
864 if (owner) {
865 if (strcmp(owner, "default") == 0) {
866 return new ACodec;
867 } else if (strncmp(owner, "codec2", 6) == 0) {
868 return CreateCCodec();
869 }
870 }
871
872 if (name.startsWithIgnoreCase("c2.")) {
873 return CreateCCodec();
874 } else if (name.startsWithIgnoreCase("omx.")) {
875 // at this time only ACodec specifies a mime type.
876 return new ACodec;
877 } else if (name.startsWithIgnoreCase("android.filter.")) {
878 return new MediaFilter;
879 } else {
880 return NULL;
881 }
882 }
883
init(const AString & name)884 status_t MediaCodec::init(const AString &name) {
885 mResourceManagerService->init();
886
887 // save init parameters for reset
888 mInitName = name;
889
890 // Current video decoders do not return from OMX_FillThisBuffer
891 // quickly, violating the OpenMAX specs, until that is remedied
892 // we need to invest in an extra looper to free the main event
893 // queue.
894
895 mCodecInfo.clear();
896
897 bool secureCodec = false;
898 AString tmp = name;
899 if (tmp.endsWith(".secure")) {
900 secureCodec = true;
901 tmp.erase(tmp.size() - 7, 7);
902 }
903 const sp<IMediaCodecList> mcl = MediaCodecList::getInstance();
904 if (mcl == NULL) {
905 mCodec = NULL; // remove the codec.
906 return NO_INIT; // if called from Java should raise IOException
907 }
908 for (const AString &codecName : { name, tmp }) {
909 ssize_t codecIdx = mcl->findCodecByName(codecName.c_str());
910 if (codecIdx < 0) {
911 continue;
912 }
913 mCodecInfo = mcl->getCodecInfo(codecIdx);
914 Vector<AString> mediaTypes;
915 mCodecInfo->getSupportedMediaTypes(&mediaTypes);
916 for (size_t i = 0; i < mediaTypes.size(); i++) {
917 if (mediaTypes[i].startsWith("video/")) {
918 mIsVideo = true;
919 break;
920 }
921 }
922 break;
923 }
924 if (mCodecInfo == nullptr) {
925 return NAME_NOT_FOUND;
926 }
927
928 mCodec = GetCodecBase(name, mCodecInfo->getOwnerName());
929 if (mCodec == NULL) {
930 return NAME_NOT_FOUND;
931 }
932
933 if (mIsVideo) {
934 // video codec needs dedicated looper
935 if (mCodecLooper == NULL) {
936 mCodecLooper = new ALooper;
937 mCodecLooper->setName("CodecLooper");
938 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
939 }
940
941 mCodecLooper->registerHandler(mCodec);
942 } else {
943 mLooper->registerHandler(mCodec);
944 }
945
946 mLooper->registerHandler(this);
947
948 mCodec->setCallback(
949 std::unique_ptr<CodecBase::CodecCallback>(
950 new CodecCallback(new AMessage(kWhatCodecNotify, this))));
951 mBufferChannel = mCodec->getBufferChannel();
952 mBufferChannel->setCallback(
953 std::unique_ptr<CodecBase::BufferCallback>(
954 new BufferCallback(new AMessage(kWhatCodecNotify, this))));
955
956 sp<AMessage> msg = new AMessage(kWhatInit, this);
957 msg->setObject("codecInfo", mCodecInfo);
958 // name may be different from mCodecInfo->getCodecName() if we stripped
959 // ".secure"
960 msg->setString("name", name);
961
962 if (mAnalyticsItem != NULL) {
963 mAnalyticsItem->setCString(kCodecCodec, name.c_str());
964 mAnalyticsItem->setCString(kCodecMode, mIsVideo ? kCodecModeVideo : kCodecModeAudio);
965 }
966
967 status_t err;
968 Vector<MediaResource> resources;
969 MediaResource::Type type =
970 secureCodec ? MediaResource::kSecureCodec : MediaResource::kNonSecureCodec;
971 MediaResource::SubType subtype =
972 mIsVideo ? MediaResource::kVideoCodec : MediaResource::kAudioCodec;
973 resources.push_back(MediaResource(type, subtype, 1));
974 for (int i = 0; i <= kMaxRetry; ++i) {
975 if (i > 0) {
976 // Don't try to reclaim resource for the first time.
977 if (!mResourceManagerService->reclaimResource(resources)) {
978 break;
979 }
980 }
981
982 sp<AMessage> response;
983 err = PostAndAwaitResponse(msg, &response);
984 if (!isResourceError(err)) {
985 break;
986 }
987 }
988 return err;
989 }
990
setCallback(const sp<AMessage> & callback)991 status_t MediaCodec::setCallback(const sp<AMessage> &callback) {
992 sp<AMessage> msg = new AMessage(kWhatSetCallback, this);
993 msg->setMessage("callback", callback);
994
995 sp<AMessage> response;
996 return PostAndAwaitResponse(msg, &response);
997 }
998
setOnFrameRenderedNotification(const sp<AMessage> & notify)999 status_t MediaCodec::setOnFrameRenderedNotification(const sp<AMessage> ¬ify) {
1000 sp<AMessage> msg = new AMessage(kWhatSetNotification, this);
1001 msg->setMessage("on-frame-rendered", notify);
1002 return msg->post();
1003 }
1004
configure(const sp<AMessage> & format,const sp<Surface> & nativeWindow,const sp<ICrypto> & crypto,uint32_t flags)1005 status_t MediaCodec::configure(
1006 const sp<AMessage> &format,
1007 const sp<Surface> &nativeWindow,
1008 const sp<ICrypto> &crypto,
1009 uint32_t flags) {
1010 return configure(format, nativeWindow, crypto, NULL, flags);
1011 }
1012
configure(const sp<AMessage> & format,const sp<Surface> & surface,const sp<ICrypto> & crypto,const sp<IDescrambler> & descrambler,uint32_t flags)1013 status_t MediaCodec::configure(
1014 const sp<AMessage> &format,
1015 const sp<Surface> &surface,
1016 const sp<ICrypto> &crypto,
1017 const sp<IDescrambler> &descrambler,
1018 uint32_t flags) {
1019 sp<AMessage> msg = new AMessage(kWhatConfigure, this);
1020
1021 if (mAnalyticsItem != NULL) {
1022 int32_t profile = 0;
1023 if (format->findInt32("profile", &profile)) {
1024 mAnalyticsItem->setInt32(kCodecProfile, profile);
1025 }
1026 int32_t level = 0;
1027 if (format->findInt32("level", &level)) {
1028 mAnalyticsItem->setInt32(kCodecLevel, level);
1029 }
1030 mAnalyticsItem->setInt32(kCodecEncoder, (flags & CONFIGURE_FLAG_ENCODE) ? 1 : 0);
1031 }
1032
1033 if (mIsVideo) {
1034 format->findInt32("width", &mVideoWidth);
1035 format->findInt32("height", &mVideoHeight);
1036 if (!format->findInt32("rotation-degrees", &mRotationDegrees)) {
1037 mRotationDegrees = 0;
1038 }
1039
1040 if (mAnalyticsItem != NULL) {
1041 mAnalyticsItem->setInt32(kCodecWidth, mVideoWidth);
1042 mAnalyticsItem->setInt32(kCodecHeight, mVideoHeight);
1043 mAnalyticsItem->setInt32(kCodecRotation, mRotationDegrees);
1044 int32_t maxWidth = 0;
1045 if (format->findInt32("max-width", &maxWidth)) {
1046 mAnalyticsItem->setInt32(kCodecMaxWidth, maxWidth);
1047 }
1048 int32_t maxHeight = 0;
1049 if (format->findInt32("max-height", &maxHeight)) {
1050 mAnalyticsItem->setInt32(kCodecMaxHeight, maxHeight);
1051 }
1052 }
1053
1054 // Prevent possible integer overflow in downstream code.
1055 if (mVideoWidth < 0 || mVideoHeight < 0 ||
1056 (uint64_t)mVideoWidth * mVideoHeight > (uint64_t)INT32_MAX / 4) {
1057 ALOGE("Invalid size(s), width=%d, height=%d", mVideoWidth, mVideoHeight);
1058 return BAD_VALUE;
1059 }
1060 }
1061
1062 msg->setMessage("format", format);
1063 msg->setInt32("flags", flags);
1064 msg->setObject("surface", surface);
1065
1066 if (crypto != NULL || descrambler != NULL) {
1067 if (crypto != NULL) {
1068 msg->setPointer("crypto", crypto.get());
1069 } else {
1070 msg->setPointer("descrambler", descrambler.get());
1071 }
1072 if (mAnalyticsItem != NULL) {
1073 mAnalyticsItem->setInt32(kCodecCrypto, 1);
1074 }
1075 } else if (mFlags & kFlagIsSecure) {
1076 ALOGW("Crypto or descrambler should be given for secure codec");
1077 }
1078
1079 // save msg for reset
1080 mConfigureMsg = msg;
1081
1082 status_t err;
1083 Vector<MediaResource> resources;
1084 MediaResource::Type type = (mFlags & kFlagIsSecure) ?
1085 MediaResource::kSecureCodec : MediaResource::kNonSecureCodec;
1086 MediaResource::SubType subtype =
1087 mIsVideo ? MediaResource::kVideoCodec : MediaResource::kAudioCodec;
1088 resources.push_back(MediaResource(type, subtype, 1));
1089 // Don't know the buffer size at this point, but it's fine to use 1 because
1090 // the reclaimResource call doesn't consider the requester's buffer size for now.
1091 resources.push_back(MediaResource(MediaResource::kGraphicMemory, 1));
1092 for (int i = 0; i <= kMaxRetry; ++i) {
1093 if (i > 0) {
1094 // Don't try to reclaim resource for the first time.
1095 if (!mResourceManagerService->reclaimResource(resources)) {
1096 break;
1097 }
1098 }
1099
1100 sp<AMessage> response;
1101 err = PostAndAwaitResponse(msg, &response);
1102 if (err != OK && err != INVALID_OPERATION) {
1103 // MediaCodec now set state to UNINITIALIZED upon any fatal error.
1104 // To maintain backward-compatibility, do a reset() to put codec
1105 // back into INITIALIZED state.
1106 // But don't reset if the err is INVALID_OPERATION, which means
1107 // the configure failure is due to wrong state.
1108
1109 ALOGE("configure failed with err 0x%08x, resetting...", err);
1110 reset();
1111 }
1112 if (!isResourceError(err)) {
1113 break;
1114 }
1115 }
1116
1117 return err;
1118 }
1119
releaseCrypto()1120 status_t MediaCodec::releaseCrypto()
1121 {
1122 ALOGV("releaseCrypto");
1123
1124 sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this);
1125
1126 sp<AMessage> response;
1127 status_t status = msg->postAndAwaitResponse(&response);
1128
1129 if (status == OK && response != NULL) {
1130 CHECK(response->findInt32("status", &status));
1131 ALOGV("releaseCrypto ret: %d ", status);
1132 }
1133 else {
1134 ALOGE("releaseCrypto err: %d", status);
1135 }
1136
1137 return status;
1138 }
1139
onReleaseCrypto(const sp<AMessage> & msg)1140 void MediaCodec::onReleaseCrypto(const sp<AMessage>& msg)
1141 {
1142 status_t status = INVALID_OPERATION;
1143 if (mCrypto != NULL) {
1144 ALOGV("onReleaseCrypto: mCrypto: %p (%d)", mCrypto.get(), mCrypto->getStrongCount());
1145 mBufferChannel->setCrypto(NULL);
1146 // TODO change to ALOGV
1147 ALOGD("onReleaseCrypto: [before clear] mCrypto: %p (%d)",
1148 mCrypto.get(), mCrypto->getStrongCount());
1149 mCrypto.clear();
1150
1151 status = OK;
1152 }
1153 else {
1154 ALOGW("onReleaseCrypto: No mCrypto. err: %d", status);
1155 }
1156
1157 sp<AMessage> response = new AMessage;
1158 response->setInt32("status", status);
1159
1160 sp<AReplyToken> replyID;
1161 CHECK(msg->senderAwaitsResponse(&replyID));
1162 response->postReply(replyID);
1163 }
1164
setInputSurface(const sp<PersistentSurface> & surface)1165 status_t MediaCodec::setInputSurface(
1166 const sp<PersistentSurface> &surface) {
1167 sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this);
1168 msg->setObject("input-surface", surface.get());
1169
1170 sp<AMessage> response;
1171 return PostAndAwaitResponse(msg, &response);
1172 }
1173
setSurface(const sp<Surface> & surface)1174 status_t MediaCodec::setSurface(const sp<Surface> &surface) {
1175 sp<AMessage> msg = new AMessage(kWhatSetSurface, this);
1176 msg->setObject("surface", surface);
1177
1178 sp<AMessage> response;
1179 return PostAndAwaitResponse(msg, &response);
1180 }
1181
createInputSurface(sp<IGraphicBufferProducer> * bufferProducer)1182 status_t MediaCodec::createInputSurface(
1183 sp<IGraphicBufferProducer>* bufferProducer) {
1184 sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, this);
1185
1186 sp<AMessage> response;
1187 status_t err = PostAndAwaitResponse(msg, &response);
1188 if (err == NO_ERROR) {
1189 // unwrap the sp<IGraphicBufferProducer>
1190 sp<RefBase> obj;
1191 bool found = response->findObject("input-surface", &obj);
1192 CHECK(found);
1193 sp<BufferProducerWrapper> wrapper(
1194 static_cast<BufferProducerWrapper*>(obj.get()));
1195 *bufferProducer = wrapper->getBufferProducer();
1196 } else {
1197 ALOGW("createInputSurface failed, err=%d", err);
1198 }
1199 return err;
1200 }
1201
getGraphicBufferSize()1202 uint64_t MediaCodec::getGraphicBufferSize() {
1203 if (!mIsVideo) {
1204 return 0;
1205 }
1206
1207 uint64_t size = 0;
1208 size_t portNum = sizeof(mPortBuffers) / sizeof((mPortBuffers)[0]);
1209 for (size_t i = 0; i < portNum; ++i) {
1210 // TODO: this is just an estimation, we should get the real buffer size from ACodec.
1211 size += mPortBuffers[i].size() * mVideoWidth * mVideoHeight * 3 / 2;
1212 }
1213 return size;
1214 }
1215
addResource(MediaResource::Type type,MediaResource::SubType subtype,uint64_t value)1216 void MediaCodec::addResource(
1217 MediaResource::Type type, MediaResource::SubType subtype, uint64_t value) {
1218 Vector<MediaResource> resources;
1219 resources.push_back(MediaResource(type, subtype, value));
1220 mResourceManagerService->addResource(
1221 getId(mResourceManagerClient), mResourceManagerClient, resources);
1222 }
1223
start()1224 status_t MediaCodec::start() {
1225 sp<AMessage> msg = new AMessage(kWhatStart, this);
1226
1227 status_t err;
1228 Vector<MediaResource> resources;
1229 MediaResource::Type type = (mFlags & kFlagIsSecure) ?
1230 MediaResource::kSecureCodec : MediaResource::kNonSecureCodec;
1231 MediaResource::SubType subtype =
1232 mIsVideo ? MediaResource::kVideoCodec : MediaResource::kAudioCodec;
1233 resources.push_back(MediaResource(type, subtype, 1));
1234 // Don't know the buffer size at this point, but it's fine to use 1 because
1235 // the reclaimResource call doesn't consider the requester's buffer size for now.
1236 resources.push_back(MediaResource(MediaResource::kGraphicMemory, 1));
1237 for (int i = 0; i <= kMaxRetry; ++i) {
1238 if (i > 0) {
1239 // Don't try to reclaim resource for the first time.
1240 if (!mResourceManagerService->reclaimResource(resources)) {
1241 break;
1242 }
1243 // Recover codec from previous error before retry start.
1244 err = reset();
1245 if (err != OK) {
1246 ALOGE("retrying start: failed to reset codec");
1247 break;
1248 }
1249 sp<AMessage> response;
1250 err = PostAndAwaitResponse(mConfigureMsg, &response);
1251 if (err != OK) {
1252 ALOGE("retrying start: failed to configure codec");
1253 break;
1254 }
1255 }
1256
1257 sp<AMessage> response;
1258 err = PostAndAwaitResponse(msg, &response);
1259 if (!isResourceError(err)) {
1260 break;
1261 }
1262 }
1263 return err;
1264 }
1265
stop()1266 status_t MediaCodec::stop() {
1267 sp<AMessage> msg = new AMessage(kWhatStop, this);
1268
1269 sp<AMessage> response;
1270 return PostAndAwaitResponse(msg, &response);
1271 }
1272
hasPendingBuffer(int portIndex)1273 bool MediaCodec::hasPendingBuffer(int portIndex) {
1274 return std::any_of(
1275 mPortBuffers[portIndex].begin(), mPortBuffers[portIndex].end(),
1276 [](const BufferInfo &info) { return info.mOwnedByClient; });
1277 }
1278
hasPendingBuffer()1279 bool MediaCodec::hasPendingBuffer() {
1280 return hasPendingBuffer(kPortIndexInput) || hasPendingBuffer(kPortIndexOutput);
1281 }
1282
reclaim(bool force)1283 status_t MediaCodec::reclaim(bool force) {
1284 ALOGD("MediaCodec::reclaim(%p) %s", this, mInitName.c_str());
1285 sp<AMessage> msg = new AMessage(kWhatRelease, this);
1286 msg->setInt32("reclaimed", 1);
1287 msg->setInt32("force", force ? 1 : 0);
1288
1289 sp<AMessage> response;
1290 status_t ret = PostAndAwaitResponse(msg, &response);
1291 if (ret == -ENOENT) {
1292 ALOGD("MediaCodec looper is gone, skip reclaim");
1293 ret = OK;
1294 }
1295 return ret;
1296 }
1297
release()1298 status_t MediaCodec::release() {
1299 sp<AMessage> msg = new AMessage(kWhatRelease, this);
1300
1301 sp<AMessage> response;
1302 return PostAndAwaitResponse(msg, &response);
1303 }
1304
reset()1305 status_t MediaCodec::reset() {
1306 /* When external-facing MediaCodec object is created,
1307 it is already initialized. Thus, reset is essentially
1308 release() followed by init(), plus clearing the state */
1309
1310 status_t err = release();
1311
1312 // unregister handlers
1313 if (mCodec != NULL) {
1314 if (mCodecLooper != NULL) {
1315 mCodecLooper->unregisterHandler(mCodec->id());
1316 } else {
1317 mLooper->unregisterHandler(mCodec->id());
1318 }
1319 mCodec = NULL;
1320 }
1321 mLooper->unregisterHandler(id());
1322
1323 mFlags = 0; // clear all flags
1324 mStickyError = OK;
1325
1326 // reset state not reset by setState(UNINITIALIZED)
1327 mReplyID = 0;
1328 mDequeueInputReplyID = 0;
1329 mDequeueOutputReplyID = 0;
1330 mDequeueInputTimeoutGeneration = 0;
1331 mDequeueOutputTimeoutGeneration = 0;
1332 mHaveInputSurface = false;
1333
1334 if (err == OK) {
1335 err = init(mInitName);
1336 }
1337 return err;
1338 }
1339
queueInputBuffer(size_t index,size_t offset,size_t size,int64_t presentationTimeUs,uint32_t flags,AString * errorDetailMsg)1340 status_t MediaCodec::queueInputBuffer(
1341 size_t index,
1342 size_t offset,
1343 size_t size,
1344 int64_t presentationTimeUs,
1345 uint32_t flags,
1346 AString *errorDetailMsg) {
1347 if (errorDetailMsg != NULL) {
1348 errorDetailMsg->clear();
1349 }
1350
1351 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
1352 msg->setSize("index", index);
1353 msg->setSize("offset", offset);
1354 msg->setSize("size", size);
1355 msg->setInt64("timeUs", presentationTimeUs);
1356 msg->setInt32("flags", flags);
1357 msg->setPointer("errorDetailMsg", errorDetailMsg);
1358
1359 sp<AMessage> response;
1360 return PostAndAwaitResponse(msg, &response);
1361 }
1362
queueSecureInputBuffer(size_t index,size_t offset,const CryptoPlugin::SubSample * subSamples,size_t numSubSamples,const uint8_t key[16],const uint8_t iv[16],CryptoPlugin::Mode mode,const CryptoPlugin::Pattern & pattern,int64_t presentationTimeUs,uint32_t flags,AString * errorDetailMsg)1363 status_t MediaCodec::queueSecureInputBuffer(
1364 size_t index,
1365 size_t offset,
1366 const CryptoPlugin::SubSample *subSamples,
1367 size_t numSubSamples,
1368 const uint8_t key[16],
1369 const uint8_t iv[16],
1370 CryptoPlugin::Mode mode,
1371 const CryptoPlugin::Pattern &pattern,
1372 int64_t presentationTimeUs,
1373 uint32_t flags,
1374 AString *errorDetailMsg) {
1375 if (errorDetailMsg != NULL) {
1376 errorDetailMsg->clear();
1377 }
1378
1379 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
1380 msg->setSize("index", index);
1381 msg->setSize("offset", offset);
1382 msg->setPointer("subSamples", (void *)subSamples);
1383 msg->setSize("numSubSamples", numSubSamples);
1384 msg->setPointer("key", (void *)key);
1385 msg->setPointer("iv", (void *)iv);
1386 msg->setInt32("mode", mode);
1387 msg->setInt32("encryptBlocks", pattern.mEncryptBlocks);
1388 msg->setInt32("skipBlocks", pattern.mSkipBlocks);
1389 msg->setInt64("timeUs", presentationTimeUs);
1390 msg->setInt32("flags", flags);
1391 msg->setPointer("errorDetailMsg", errorDetailMsg);
1392
1393 sp<AMessage> response;
1394 status_t err = PostAndAwaitResponse(msg, &response);
1395
1396 return err;
1397 }
1398
dequeueInputBuffer(size_t * index,int64_t timeoutUs)1399 status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) {
1400 sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, this);
1401 msg->setInt64("timeoutUs", timeoutUs);
1402
1403 sp<AMessage> response;
1404 status_t err;
1405 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1406 return err;
1407 }
1408
1409 CHECK(response->findSize("index", index));
1410
1411 return OK;
1412 }
1413
dequeueOutputBuffer(size_t * index,size_t * offset,size_t * size,int64_t * presentationTimeUs,uint32_t * flags,int64_t timeoutUs)1414 status_t MediaCodec::dequeueOutputBuffer(
1415 size_t *index,
1416 size_t *offset,
1417 size_t *size,
1418 int64_t *presentationTimeUs,
1419 uint32_t *flags,
1420 int64_t timeoutUs) {
1421 sp<AMessage> msg = new AMessage(kWhatDequeueOutputBuffer, this);
1422 msg->setInt64("timeoutUs", timeoutUs);
1423
1424 sp<AMessage> response;
1425 status_t err;
1426 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1427 return err;
1428 }
1429
1430 CHECK(response->findSize("index", index));
1431 CHECK(response->findSize("offset", offset));
1432 CHECK(response->findSize("size", size));
1433 CHECK(response->findInt64("timeUs", presentationTimeUs));
1434 CHECK(response->findInt32("flags", (int32_t *)flags));
1435
1436 return OK;
1437 }
1438
renderOutputBufferAndRelease(size_t index)1439 status_t MediaCodec::renderOutputBufferAndRelease(size_t index) {
1440 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
1441 msg->setSize("index", index);
1442 msg->setInt32("render", true);
1443
1444 sp<AMessage> response;
1445 return PostAndAwaitResponse(msg, &response);
1446 }
1447
renderOutputBufferAndRelease(size_t index,int64_t timestampNs)1448 status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) {
1449 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
1450 msg->setSize("index", index);
1451 msg->setInt32("render", true);
1452 msg->setInt64("timestampNs", timestampNs);
1453
1454 sp<AMessage> response;
1455 return PostAndAwaitResponse(msg, &response);
1456 }
1457
releaseOutputBuffer(size_t index)1458 status_t MediaCodec::releaseOutputBuffer(size_t index) {
1459 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
1460 msg->setSize("index", index);
1461
1462 sp<AMessage> response;
1463 return PostAndAwaitResponse(msg, &response);
1464 }
1465
signalEndOfInputStream()1466 status_t MediaCodec::signalEndOfInputStream() {
1467 sp<AMessage> msg = new AMessage(kWhatSignalEndOfInputStream, this);
1468
1469 sp<AMessage> response;
1470 return PostAndAwaitResponse(msg, &response);
1471 }
1472
getOutputFormat(sp<AMessage> * format) const1473 status_t MediaCodec::getOutputFormat(sp<AMessage> *format) const {
1474 sp<AMessage> msg = new AMessage(kWhatGetOutputFormat, this);
1475
1476 sp<AMessage> response;
1477 status_t err;
1478 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1479 return err;
1480 }
1481
1482 CHECK(response->findMessage("format", format));
1483
1484 return OK;
1485 }
1486
getInputFormat(sp<AMessage> * format) const1487 status_t MediaCodec::getInputFormat(sp<AMessage> *format) const {
1488 sp<AMessage> msg = new AMessage(kWhatGetInputFormat, this);
1489
1490 sp<AMessage> response;
1491 status_t err;
1492 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1493 return err;
1494 }
1495
1496 CHECK(response->findMessage("format", format));
1497
1498 return OK;
1499 }
1500
getName(AString * name) const1501 status_t MediaCodec::getName(AString *name) const {
1502 sp<AMessage> msg = new AMessage(kWhatGetName, this);
1503
1504 sp<AMessage> response;
1505 status_t err;
1506 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1507 return err;
1508 }
1509
1510 CHECK(response->findString("name", name));
1511
1512 return OK;
1513 }
1514
getCodecInfo(sp<MediaCodecInfo> * codecInfo) const1515 status_t MediaCodec::getCodecInfo(sp<MediaCodecInfo> *codecInfo) const {
1516 sp<AMessage> msg = new AMessage(kWhatGetCodecInfo, this);
1517
1518 sp<AMessage> response;
1519 status_t err;
1520 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1521 return err;
1522 }
1523
1524 sp<RefBase> obj;
1525 CHECK(response->findObject("codecInfo", &obj));
1526 *codecInfo = static_cast<MediaCodecInfo *>(obj.get());
1527
1528 return OK;
1529 }
1530
getMetrics(MediaAnalyticsItem * & reply)1531 status_t MediaCodec::getMetrics(MediaAnalyticsItem * &reply) {
1532
1533 reply = NULL;
1534
1535 // shouldn't happen, but be safe
1536 if (mAnalyticsItem == NULL) {
1537 return UNKNOWN_ERROR;
1538 }
1539
1540 // update any in-flight data that's not carried within the record
1541 updateAnalyticsItem();
1542
1543 // send it back to the caller.
1544 reply = mAnalyticsItem->dup();
1545
1546 updateEphemeralAnalytics(reply);
1547
1548 return OK;
1549 }
1550
getInputBuffers(Vector<sp<MediaCodecBuffer>> * buffers) const1551 status_t MediaCodec::getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const {
1552 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this);
1553 msg->setInt32("portIndex", kPortIndexInput);
1554 msg->setPointer("buffers", buffers);
1555
1556 sp<AMessage> response;
1557 return PostAndAwaitResponse(msg, &response);
1558 }
1559
getOutputBuffers(Vector<sp<MediaCodecBuffer>> * buffers) const1560 status_t MediaCodec::getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const {
1561 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this);
1562 msg->setInt32("portIndex", kPortIndexOutput);
1563 msg->setPointer("buffers", buffers);
1564
1565 sp<AMessage> response;
1566 return PostAndAwaitResponse(msg, &response);
1567 }
1568
getOutputBuffer(size_t index,sp<MediaCodecBuffer> * buffer)1569 status_t MediaCodec::getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer) {
1570 sp<AMessage> format;
1571 return getBufferAndFormat(kPortIndexOutput, index, buffer, &format);
1572 }
1573
getOutputFormat(size_t index,sp<AMessage> * format)1574 status_t MediaCodec::getOutputFormat(size_t index, sp<AMessage> *format) {
1575 sp<MediaCodecBuffer> buffer;
1576 return getBufferAndFormat(kPortIndexOutput, index, &buffer, format);
1577 }
1578
getInputBuffer(size_t index,sp<MediaCodecBuffer> * buffer)1579 status_t MediaCodec::getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer) {
1580 sp<AMessage> format;
1581 return getBufferAndFormat(kPortIndexInput, index, buffer, &format);
1582 }
1583
isExecuting() const1584 bool MediaCodec::isExecuting() const {
1585 return mState == STARTED || mState == FLUSHED;
1586 }
1587
getBufferAndFormat(size_t portIndex,size_t index,sp<MediaCodecBuffer> * buffer,sp<AMessage> * format)1588 status_t MediaCodec::getBufferAndFormat(
1589 size_t portIndex, size_t index,
1590 sp<MediaCodecBuffer> *buffer, sp<AMessage> *format) {
1591 // use mutex instead of a context switch
1592 if (mReleasedByResourceManager) {
1593 ALOGE("getBufferAndFormat - resource already released");
1594 return DEAD_OBJECT;
1595 }
1596
1597 if (buffer == NULL) {
1598 ALOGE("getBufferAndFormat - null MediaCodecBuffer");
1599 return INVALID_OPERATION;
1600 }
1601
1602 if (format == NULL) {
1603 ALOGE("getBufferAndFormat - null AMessage");
1604 return INVALID_OPERATION;
1605 }
1606
1607 buffer->clear();
1608 format->clear();
1609
1610 if (!isExecuting()) {
1611 ALOGE("getBufferAndFormat - not executing");
1612 return INVALID_OPERATION;
1613 }
1614
1615 // we do not want mPortBuffers to change during this section
1616 // we also don't want mOwnedByClient to change during this
1617 Mutex::Autolock al(mBufferLock);
1618
1619 std::vector<BufferInfo> &buffers = mPortBuffers[portIndex];
1620 if (index >= buffers.size()) {
1621 ALOGE("getBufferAndFormat - trying to get buffer with "
1622 "bad index (index=%zu buffer_size=%zu)", index, buffers.size());
1623 return INVALID_OPERATION;
1624 }
1625
1626 const BufferInfo &info = buffers[index];
1627 if (!info.mOwnedByClient) {
1628 ALOGE("getBufferAndFormat - invalid operation "
1629 "(the index %zu is not owned by client)", index);
1630 return INVALID_OPERATION;
1631 }
1632
1633 *buffer = info.mData;
1634 *format = info.mData->format();
1635
1636 return OK;
1637 }
1638
flush()1639 status_t MediaCodec::flush() {
1640 sp<AMessage> msg = new AMessage(kWhatFlush, this);
1641
1642 sp<AMessage> response;
1643 return PostAndAwaitResponse(msg, &response);
1644 }
1645
requestIDRFrame()1646 status_t MediaCodec::requestIDRFrame() {
1647 (new AMessage(kWhatRequestIDRFrame, this))->post();
1648
1649 return OK;
1650 }
1651
requestActivityNotification(const sp<AMessage> & notify)1652 void MediaCodec::requestActivityNotification(const sp<AMessage> ¬ify) {
1653 sp<AMessage> msg = new AMessage(kWhatRequestActivityNotification, this);
1654 msg->setMessage("notify", notify);
1655 msg->post();
1656 }
1657
requestCpuBoostIfNeeded()1658 void MediaCodec::requestCpuBoostIfNeeded() {
1659 if (mCpuBoostRequested) {
1660 return;
1661 }
1662 int32_t colorFormat;
1663 if (mOutputFormat->contains("hdr-static-info")
1664 && mOutputFormat->findInt32("color-format", &colorFormat)
1665 // check format for OMX only, for C2 the format is always opaque since the
1666 // software rendering doesn't go through client
1667 && ((mSoftRenderer != NULL && colorFormat == OMX_COLOR_FormatYUV420Planar16)
1668 || mOwnerName.equalsIgnoreCase("codec2::software"))) {
1669 int32_t left, top, right, bottom, width, height;
1670 int64_t totalPixel = 0;
1671 if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {
1672 totalPixel = (right - left + 1) * (bottom - top + 1);
1673 } else if (mOutputFormat->findInt32("width", &width)
1674 && mOutputFormat->findInt32("height", &height)) {
1675 totalPixel = width * height;
1676 }
1677 if (totalPixel >= 1920 * 1080) {
1678 addResource(MediaResource::kCpuBoost,
1679 MediaResource::kUnspecifiedSubType, 1);
1680 mCpuBoostRequested = true;
1681 }
1682 }
1683 }
1684
1685 ////////////////////////////////////////////////////////////////////////////////
1686
cancelPendingDequeueOperations()1687 void MediaCodec::cancelPendingDequeueOperations() {
1688 if (mFlags & kFlagDequeueInputPending) {
1689 PostReplyWithError(mDequeueInputReplyID, INVALID_OPERATION);
1690
1691 ++mDequeueInputTimeoutGeneration;
1692 mDequeueInputReplyID = 0;
1693 mFlags &= ~kFlagDequeueInputPending;
1694 }
1695
1696 if (mFlags & kFlagDequeueOutputPending) {
1697 PostReplyWithError(mDequeueOutputReplyID, INVALID_OPERATION);
1698
1699 ++mDequeueOutputTimeoutGeneration;
1700 mDequeueOutputReplyID = 0;
1701 mFlags &= ~kFlagDequeueOutputPending;
1702 }
1703 }
1704
handleDequeueInputBuffer(const sp<AReplyToken> & replyID,bool newRequest)1705 bool MediaCodec::handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest) {
1706 if (!isExecuting() || (mFlags & kFlagIsAsync)
1707 || (newRequest && (mFlags & kFlagDequeueInputPending))) {
1708 PostReplyWithError(replyID, INVALID_OPERATION);
1709 return true;
1710 } else if (mFlags & kFlagStickyError) {
1711 PostReplyWithError(replyID, getStickyError());
1712 return true;
1713 }
1714
1715 ssize_t index = dequeuePortBuffer(kPortIndexInput);
1716
1717 if (index < 0) {
1718 CHECK_EQ(index, -EAGAIN);
1719 return false;
1720 }
1721
1722 sp<AMessage> response = new AMessage;
1723 response->setSize("index", index);
1724 response->postReply(replyID);
1725
1726 return true;
1727 }
1728
handleDequeueOutputBuffer(const sp<AReplyToken> & replyID,bool newRequest)1729 bool MediaCodec::handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest) {
1730 if (!isExecuting() || (mFlags & kFlagIsAsync)
1731 || (newRequest && (mFlags & kFlagDequeueOutputPending))) {
1732 PostReplyWithError(replyID, INVALID_OPERATION);
1733 } else if (mFlags & kFlagStickyError) {
1734 PostReplyWithError(replyID, getStickyError());
1735 } else if (mFlags & kFlagOutputBuffersChanged) {
1736 PostReplyWithError(replyID, INFO_OUTPUT_BUFFERS_CHANGED);
1737 mFlags &= ~kFlagOutputBuffersChanged;
1738 } else if (mFlags & kFlagOutputFormatChanged) {
1739 PostReplyWithError(replyID, INFO_FORMAT_CHANGED);
1740 mFlags &= ~kFlagOutputFormatChanged;
1741 } else {
1742 sp<AMessage> response = new AMessage;
1743 ssize_t index = dequeuePortBuffer(kPortIndexOutput);
1744
1745 if (index < 0) {
1746 CHECK_EQ(index, -EAGAIN);
1747 return false;
1748 }
1749
1750 const sp<MediaCodecBuffer> &buffer =
1751 mPortBuffers[kPortIndexOutput][index].mData;
1752
1753 response->setSize("index", index);
1754 response->setSize("offset", buffer->offset());
1755 response->setSize("size", buffer->size());
1756
1757 int64_t timeUs;
1758 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1759
1760 statsBufferReceived(timeUs);
1761
1762 response->setInt64("timeUs", timeUs);
1763
1764 int32_t flags;
1765 CHECK(buffer->meta()->findInt32("flags", &flags));
1766
1767 response->setInt32("flags", flags);
1768 response->postReply(replyID);
1769 }
1770
1771 return true;
1772 }
1773
onMessageReceived(const sp<AMessage> & msg)1774 void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
1775 switch (msg->what()) {
1776 case kWhatCodecNotify:
1777 {
1778 int32_t what;
1779 CHECK(msg->findInt32("what", &what));
1780
1781 switch (what) {
1782 case kWhatError:
1783 {
1784 int32_t err, actionCode;
1785 CHECK(msg->findInt32("err", &err));
1786 CHECK(msg->findInt32("actionCode", &actionCode));
1787
1788 ALOGE("Codec reported err %#x, actionCode %d, while in state %d",
1789 err, actionCode, mState);
1790 if (err == DEAD_OBJECT) {
1791 mFlags |= kFlagSawMediaServerDie;
1792 mFlags &= ~kFlagIsComponentAllocated;
1793 }
1794
1795 bool sendErrorResponse = true;
1796
1797 switch (mState) {
1798 case INITIALIZING:
1799 {
1800 setState(UNINITIALIZED);
1801 break;
1802 }
1803
1804 case CONFIGURING:
1805 {
1806 if (actionCode == ACTION_CODE_FATAL) {
1807 mAnalyticsItem->setInt32(kCodecError, err);
1808 mAnalyticsItem->setCString(kCodecErrorState, stateString(mState).c_str());
1809 flushAnalyticsItem();
1810 initAnalyticsItem();
1811 }
1812 setState(actionCode == ACTION_CODE_FATAL ?
1813 UNINITIALIZED : INITIALIZED);
1814 break;
1815 }
1816
1817 case STARTING:
1818 {
1819 if (actionCode == ACTION_CODE_FATAL) {
1820 mAnalyticsItem->setInt32(kCodecError, err);
1821 mAnalyticsItem->setCString(kCodecErrorState, stateString(mState).c_str());
1822 flushAnalyticsItem();
1823 initAnalyticsItem();
1824 }
1825 setState(actionCode == ACTION_CODE_FATAL ?
1826 UNINITIALIZED : CONFIGURED);
1827 break;
1828 }
1829
1830 case RELEASING:
1831 {
1832 // Ignore the error, assuming we'll still get
1833 // the shutdown complete notification. If we
1834 // don't, we'll timeout and force release.
1835 sendErrorResponse = false;
1836 FALLTHROUGH_INTENDED;
1837 }
1838 case STOPPING:
1839 {
1840 if (mFlags & kFlagSawMediaServerDie) {
1841 // MediaServer died, there definitely won't
1842 // be a shutdown complete notification after
1843 // all.
1844
1845 // note that we're directly going from
1846 // STOPPING->UNINITIALIZED, instead of the
1847 // usual STOPPING->INITIALIZED state.
1848 setState(UNINITIALIZED);
1849 if (mState == RELEASING) {
1850 mComponentName.clear();
1851 }
1852 (new AMessage)->postReply(mReplyID);
1853 sendErrorResponse = false;
1854 }
1855 break;
1856 }
1857
1858 case FLUSHING:
1859 {
1860 if (actionCode == ACTION_CODE_FATAL) {
1861 mAnalyticsItem->setInt32(kCodecError, err);
1862 mAnalyticsItem->setCString(kCodecErrorState, stateString(mState).c_str());
1863 flushAnalyticsItem();
1864 initAnalyticsItem();
1865
1866 setState(UNINITIALIZED);
1867 } else {
1868 setState(
1869 (mFlags & kFlagIsAsync) ? FLUSHED : STARTED);
1870 }
1871 break;
1872 }
1873
1874 case FLUSHED:
1875 case STARTED:
1876 {
1877 sendErrorResponse = false;
1878
1879 setStickyError(err);
1880 postActivityNotificationIfPossible();
1881
1882 cancelPendingDequeueOperations();
1883
1884 if (mFlags & kFlagIsAsync) {
1885 onError(err, actionCode);
1886 }
1887 switch (actionCode) {
1888 case ACTION_CODE_TRANSIENT:
1889 break;
1890 case ACTION_CODE_RECOVERABLE:
1891 setState(INITIALIZED);
1892 break;
1893 default:
1894 mAnalyticsItem->setInt32(kCodecError, err);
1895 mAnalyticsItem->setCString(kCodecErrorState, stateString(mState).c_str());
1896 flushAnalyticsItem();
1897 initAnalyticsItem();
1898 setState(UNINITIALIZED);
1899 break;
1900 }
1901 break;
1902 }
1903
1904 default:
1905 {
1906 sendErrorResponse = false;
1907
1908 setStickyError(err);
1909 postActivityNotificationIfPossible();
1910
1911 // actionCode in an uninitialized state is always fatal.
1912 if (mState == UNINITIALIZED) {
1913 actionCode = ACTION_CODE_FATAL;
1914 }
1915 if (mFlags & kFlagIsAsync) {
1916 onError(err, actionCode);
1917 }
1918 switch (actionCode) {
1919 case ACTION_CODE_TRANSIENT:
1920 break;
1921 case ACTION_CODE_RECOVERABLE:
1922 setState(INITIALIZED);
1923 break;
1924 default:
1925 setState(UNINITIALIZED);
1926 break;
1927 }
1928 break;
1929 }
1930 }
1931
1932 if (sendErrorResponse) {
1933 PostReplyWithError(mReplyID, err);
1934 }
1935 break;
1936 }
1937
1938 case kWhatComponentAllocated:
1939 {
1940 if (mState == RELEASING || mState == UNINITIALIZED) {
1941 // In case a kWhatError or kWhatRelease message came in and replied,
1942 // we log a warning and ignore.
1943 ALOGW("allocate interrupted by error or release, current state %d",
1944 mState);
1945 break;
1946 }
1947 CHECK_EQ(mState, INITIALIZING);
1948 setState(INITIALIZED);
1949 mFlags |= kFlagIsComponentAllocated;
1950
1951 CHECK(msg->findString("componentName", &mComponentName));
1952
1953 if (mComponentName.c_str()) {
1954 mAnalyticsItem->setCString(kCodecCodec, mComponentName.c_str());
1955 }
1956
1957 const char *owner = mCodecInfo->getOwnerName();
1958 if (mComponentName.startsWith("OMX.google.")
1959 && (owner == nullptr || strncmp(owner, "default", 8) == 0)) {
1960 mFlags |= kFlagUsesSoftwareRenderer;
1961 } else {
1962 mFlags &= ~kFlagUsesSoftwareRenderer;
1963 }
1964 mOwnerName = owner;
1965
1966 MediaResource::Type resourceType;
1967 if (mComponentName.endsWith(".secure")) {
1968 mFlags |= kFlagIsSecure;
1969 resourceType = MediaResource::kSecureCodec;
1970 mAnalyticsItem->setInt32(kCodecSecure, 1);
1971 } else {
1972 mFlags &= ~kFlagIsSecure;
1973 resourceType = MediaResource::kNonSecureCodec;
1974 mAnalyticsItem->setInt32(kCodecSecure, 0);
1975 }
1976
1977 if (mIsVideo) {
1978 // audio codec is currently ignored.
1979 addResource(resourceType, MediaResource::kVideoCodec, 1);
1980 }
1981
1982 (new AMessage)->postReply(mReplyID);
1983 break;
1984 }
1985
1986 case kWhatComponentConfigured:
1987 {
1988 if (mState == RELEASING || mState == UNINITIALIZED || mState == INITIALIZED) {
1989 // In case a kWhatError or kWhatRelease message came in and replied,
1990 // we log a warning and ignore.
1991 ALOGW("configure interrupted by error or release, current state %d",
1992 mState);
1993 break;
1994 }
1995 CHECK_EQ(mState, CONFIGURING);
1996
1997 // reset input surface flag
1998 mHaveInputSurface = false;
1999
2000 CHECK(msg->findMessage("input-format", &mInputFormat));
2001 CHECK(msg->findMessage("output-format", &mOutputFormat));
2002
2003 // limit to confirming the opt-in behavior to minimize any behavioral change
2004 if (mSurface != nullptr && !mAllowFrameDroppingBySurface) {
2005 // signal frame dropping mode in the input format as this may also be
2006 // meaningful and confusing for an encoder in a transcoder scenario
2007 mInputFormat->setInt32("allow-frame-drop", mAllowFrameDroppingBySurface);
2008 }
2009 ALOGV("[%s] configured as input format: %s, output format: %s",
2010 mComponentName.c_str(),
2011 mInputFormat->debugString(4).c_str(),
2012 mOutputFormat->debugString(4).c_str());
2013 int32_t usingSwRenderer;
2014 if (mOutputFormat->findInt32("using-sw-renderer", &usingSwRenderer)
2015 && usingSwRenderer) {
2016 mFlags |= kFlagUsesSoftwareRenderer;
2017 }
2018 setState(CONFIGURED);
2019 (new AMessage)->postReply(mReplyID);
2020
2021 // augment our media metrics info, now that we know more things
2022 if (mAnalyticsItem != NULL) {
2023 sp<AMessage> format;
2024 if (mConfigureMsg != NULL &&
2025 mConfigureMsg->findMessage("format", &format)) {
2026 // format includes: mime
2027 AString mime;
2028 if (format->findString("mime", &mime)) {
2029 mAnalyticsItem->setCString(kCodecMime, mime.c_str());
2030 }
2031 }
2032 }
2033 break;
2034 }
2035
2036 case kWhatInputSurfaceCreated:
2037 {
2038 // response to initiateCreateInputSurface()
2039 status_t err = NO_ERROR;
2040 sp<AMessage> response = new AMessage;
2041 if (!msg->findInt32("err", &err)) {
2042 sp<RefBase> obj;
2043 msg->findObject("input-surface", &obj);
2044 CHECK(msg->findMessage("input-format", &mInputFormat));
2045 CHECK(msg->findMessage("output-format", &mOutputFormat));
2046 ALOGV("[%s] input surface created as input format: %s, output format: %s",
2047 mComponentName.c_str(),
2048 mInputFormat->debugString(4).c_str(),
2049 mOutputFormat->debugString(4).c_str());
2050 CHECK(obj != NULL);
2051 response->setObject("input-surface", obj);
2052 mHaveInputSurface = true;
2053 } else {
2054 response->setInt32("err", err);
2055 }
2056 response->postReply(mReplyID);
2057 break;
2058 }
2059
2060 case kWhatInputSurfaceAccepted:
2061 {
2062 // response to initiateSetInputSurface()
2063 status_t err = NO_ERROR;
2064 sp<AMessage> response = new AMessage();
2065 if (!msg->findInt32("err", &err)) {
2066 CHECK(msg->findMessage("input-format", &mInputFormat));
2067 CHECK(msg->findMessage("output-format", &mOutputFormat));
2068 mHaveInputSurface = true;
2069 } else {
2070 response->setInt32("err", err);
2071 }
2072 response->postReply(mReplyID);
2073 break;
2074 }
2075
2076 case kWhatSignaledInputEOS:
2077 {
2078 // response to signalEndOfInputStream()
2079 sp<AMessage> response = new AMessage;
2080 status_t err;
2081 if (msg->findInt32("err", &err)) {
2082 response->setInt32("err", err);
2083 }
2084 response->postReply(mReplyID);
2085 break;
2086 }
2087
2088 case kWhatStartCompleted:
2089 {
2090 if (mState == RELEASING || mState == UNINITIALIZED) {
2091 // In case a kWhatRelease message came in and replied,
2092 // we log a warning and ignore.
2093 ALOGW("start interrupted by release, current state %d", mState);
2094 break;
2095 }
2096
2097 CHECK_EQ(mState, STARTING);
2098 if (mIsVideo) {
2099 addResource(
2100 MediaResource::kGraphicMemory,
2101 MediaResource::kUnspecifiedSubType,
2102 getGraphicBufferSize());
2103 }
2104 setState(STARTED);
2105 (new AMessage)->postReply(mReplyID);
2106 break;
2107 }
2108
2109 case kWhatOutputBuffersChanged:
2110 {
2111 mFlags |= kFlagOutputBuffersChanged;
2112 postActivityNotificationIfPossible();
2113 break;
2114 }
2115
2116 case kWhatOutputFramesRendered:
2117 {
2118 // ignore these in all states except running, and check that we have a
2119 // notification set
2120 if (mState == STARTED && mOnFrameRenderedNotification != NULL) {
2121 sp<AMessage> notify = mOnFrameRenderedNotification->dup();
2122 notify->setMessage("data", msg);
2123 notify->post();
2124 }
2125 break;
2126 }
2127
2128 case kWhatFillThisBuffer:
2129 {
2130 /* size_t index = */updateBuffers(kPortIndexInput, msg);
2131
2132 if (mState == FLUSHING
2133 || mState == STOPPING
2134 || mState == RELEASING) {
2135 returnBuffersToCodecOnPort(kPortIndexInput);
2136 break;
2137 }
2138
2139 if (!mCSD.empty()) {
2140 ssize_t index = dequeuePortBuffer(kPortIndexInput);
2141 CHECK_GE(index, 0);
2142
2143 // If codec specific data had been specified as
2144 // part of the format in the call to configure and
2145 // if there's more csd left, we submit it here
2146 // clients only get access to input buffers once
2147 // this data has been exhausted.
2148
2149 status_t err = queueCSDInputBuffer(index);
2150
2151 if (err != OK) {
2152 ALOGE("queueCSDInputBuffer failed w/ error %d",
2153 err);
2154
2155 setStickyError(err);
2156 postActivityNotificationIfPossible();
2157
2158 cancelPendingDequeueOperations();
2159 }
2160 break;
2161 }
2162
2163 if (mFlags & kFlagIsAsync) {
2164 if (!mHaveInputSurface) {
2165 if (mState == FLUSHED) {
2166 mHavePendingInputBuffers = true;
2167 } else {
2168 onInputBufferAvailable();
2169 }
2170 }
2171 } else if (mFlags & kFlagDequeueInputPending) {
2172 CHECK(handleDequeueInputBuffer(mDequeueInputReplyID));
2173
2174 ++mDequeueInputTimeoutGeneration;
2175 mFlags &= ~kFlagDequeueInputPending;
2176 mDequeueInputReplyID = 0;
2177 } else {
2178 postActivityNotificationIfPossible();
2179 }
2180 break;
2181 }
2182
2183 case kWhatDrainThisBuffer:
2184 {
2185 /* size_t index = */updateBuffers(kPortIndexOutput, msg);
2186
2187 if (mState == FLUSHING
2188 || mState == STOPPING
2189 || mState == RELEASING) {
2190 returnBuffersToCodecOnPort(kPortIndexOutput);
2191 break;
2192 }
2193
2194 sp<RefBase> obj;
2195 CHECK(msg->findObject("buffer", &obj));
2196 sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
2197
2198 if (mOutputFormat != buffer->format()) {
2199 mOutputFormat = buffer->format();
2200 ALOGV("[%s] output format changed to: %s",
2201 mComponentName.c_str(), mOutputFormat->debugString(4).c_str());
2202
2203 if (mSoftRenderer == NULL &&
2204 mSurface != NULL &&
2205 (mFlags & kFlagUsesSoftwareRenderer)) {
2206 AString mime;
2207 CHECK(mOutputFormat->findString("mime", &mime));
2208
2209 // TODO: propagate color aspects to software renderer to allow better
2210 // color conversion to RGB. For now, just mark dataspace for YUV
2211 // rendering.
2212 int32_t dataSpace;
2213 if (mOutputFormat->findInt32("android._dataspace", &dataSpace)) {
2214 ALOGD("[%s] setting dataspace on output surface to #%x",
2215 mComponentName.c_str(), dataSpace);
2216 int err = native_window_set_buffers_data_space(
2217 mSurface.get(), (android_dataspace)dataSpace);
2218 ALOGW_IF(err != 0, "failed to set dataspace on surface (%d)", err);
2219 }
2220 if (mOutputFormat->contains("hdr-static-info")) {
2221 HDRStaticInfo info;
2222 if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info)) {
2223 setNativeWindowHdrMetadata(mSurface.get(), &info);
2224 }
2225 }
2226
2227 sp<ABuffer> hdr10PlusInfo;
2228 if (mOutputFormat->findBuffer("hdr10-plus-info", &hdr10PlusInfo)
2229 && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) {
2230 native_window_set_buffers_hdr10_plus_metadata(mSurface.get(),
2231 hdr10PlusInfo->size(), hdr10PlusInfo->data());
2232 }
2233
2234 if (mime.startsWithIgnoreCase("video/")) {
2235 mSurface->setDequeueTimeout(-1);
2236 mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees);
2237 }
2238 }
2239
2240 requestCpuBoostIfNeeded();
2241
2242 if (mFlags & kFlagIsEncoder) {
2243 // Before we announce the format change we should
2244 // collect codec specific data and amend the output
2245 // format as necessary.
2246 int32_t flags = 0;
2247 (void) buffer->meta()->findInt32("flags", &flags);
2248 if (flags & BUFFER_FLAG_CODECCONFIG) {
2249 status_t err =
2250 amendOutputFormatWithCodecSpecificData(buffer);
2251
2252 if (err != OK) {
2253 ALOGE("Codec spit out malformed codec "
2254 "specific data!");
2255 }
2256 }
2257 }
2258 if (mFlags & kFlagIsAsync) {
2259 onOutputFormatChanged();
2260 } else {
2261 mFlags |= kFlagOutputFormatChanged;
2262 postActivityNotificationIfPossible();
2263 }
2264
2265 // Notify mCrypto of video resolution changes
2266 if (mCrypto != NULL) {
2267 int32_t left, top, right, bottom, width, height;
2268 if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {
2269 mCrypto->notifyResolution(right - left + 1, bottom - top + 1);
2270 } else if (mOutputFormat->findInt32("width", &width)
2271 && mOutputFormat->findInt32("height", &height)) {
2272 mCrypto->notifyResolution(width, height);
2273 }
2274 }
2275 }
2276
2277 if (mFlags & kFlagIsAsync) {
2278 onOutputBufferAvailable();
2279 } else if (mFlags & kFlagDequeueOutputPending) {
2280 CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID));
2281
2282 ++mDequeueOutputTimeoutGeneration;
2283 mFlags &= ~kFlagDequeueOutputPending;
2284 mDequeueOutputReplyID = 0;
2285 } else {
2286 postActivityNotificationIfPossible();
2287 }
2288
2289 break;
2290 }
2291
2292 case kWhatEOS:
2293 {
2294 // We already notify the client of this by using the
2295 // corresponding flag in "onOutputBufferReady".
2296 break;
2297 }
2298
2299 case kWhatStopCompleted:
2300 {
2301 if (mState != STOPPING) {
2302 ALOGW("Received kWhatStopCompleted in state %d", mState);
2303 break;
2304 }
2305 setState(INITIALIZED);
2306 (new AMessage)->postReply(mReplyID);
2307 break;
2308 }
2309
2310 case kWhatReleaseCompleted:
2311 {
2312 if (mState != RELEASING) {
2313 ALOGW("Received kWhatReleaseCompleted in state %d", mState);
2314 break;
2315 }
2316 setState(UNINITIALIZED);
2317 mComponentName.clear();
2318
2319 mFlags &= ~kFlagIsComponentAllocated;
2320
2321 mResourceManagerService->removeResource(getId(mResourceManagerClient));
2322
2323 (new AMessage)->postReply(mReplyID);
2324 break;
2325 }
2326
2327 case kWhatFlushCompleted:
2328 {
2329 if (mState != FLUSHING) {
2330 ALOGW("received FlushCompleted message in state %d",
2331 mState);
2332 break;
2333 }
2334
2335 if (mFlags & kFlagIsAsync) {
2336 setState(FLUSHED);
2337 } else {
2338 setState(STARTED);
2339 mCodec->signalResume();
2340 }
2341
2342 (new AMessage)->postReply(mReplyID);
2343 break;
2344 }
2345
2346 default:
2347 TRESPASS();
2348 }
2349 break;
2350 }
2351
2352 case kWhatInit:
2353 {
2354 sp<AReplyToken> replyID;
2355 CHECK(msg->senderAwaitsResponse(&replyID));
2356
2357 if (mState != UNINITIALIZED) {
2358 PostReplyWithError(replyID, INVALID_OPERATION);
2359 break;
2360 }
2361
2362 mReplyID = replyID;
2363 setState(INITIALIZING);
2364
2365 sp<RefBase> codecInfo;
2366 CHECK(msg->findObject("codecInfo", &codecInfo));
2367 AString name;
2368 CHECK(msg->findString("name", &name));
2369
2370 sp<AMessage> format = new AMessage;
2371 format->setObject("codecInfo", codecInfo);
2372 format->setString("componentName", name);
2373
2374 mCodec->initiateAllocateComponent(format);
2375 break;
2376 }
2377
2378 case kWhatSetNotification:
2379 {
2380 sp<AMessage> notify;
2381 if (msg->findMessage("on-frame-rendered", ¬ify)) {
2382 mOnFrameRenderedNotification = notify;
2383 }
2384 break;
2385 }
2386
2387 case kWhatSetCallback:
2388 {
2389 sp<AReplyToken> replyID;
2390 CHECK(msg->senderAwaitsResponse(&replyID));
2391
2392 if (mState == UNINITIALIZED
2393 || mState == INITIALIZING
2394 || isExecuting()) {
2395 // callback can't be set after codec is executing,
2396 // or before it's initialized (as the callback
2397 // will be cleared when it goes to INITIALIZED)
2398 PostReplyWithError(replyID, INVALID_OPERATION);
2399 break;
2400 }
2401
2402 sp<AMessage> callback;
2403 CHECK(msg->findMessage("callback", &callback));
2404
2405 mCallback = callback;
2406
2407 if (mCallback != NULL) {
2408 ALOGI("MediaCodec will operate in async mode");
2409 mFlags |= kFlagIsAsync;
2410 } else {
2411 mFlags &= ~kFlagIsAsync;
2412 }
2413
2414 sp<AMessage> response = new AMessage;
2415 response->postReply(replyID);
2416 break;
2417 }
2418
2419 case kWhatConfigure:
2420 {
2421 sp<AReplyToken> replyID;
2422 CHECK(msg->senderAwaitsResponse(&replyID));
2423
2424 if (mState != INITIALIZED) {
2425 PostReplyWithError(replyID, INVALID_OPERATION);
2426 break;
2427 }
2428
2429 sp<RefBase> obj;
2430 CHECK(msg->findObject("surface", &obj));
2431
2432 sp<AMessage> format;
2433 CHECK(msg->findMessage("format", &format));
2434
2435 int32_t push;
2436 if (msg->findInt32("push-blank-buffers-on-shutdown", &push) && push != 0) {
2437 mFlags |= kFlagPushBlankBuffersOnShutdown;
2438 }
2439
2440 if (obj != NULL) {
2441 if (!format->findInt32("allow-frame-drop", &mAllowFrameDroppingBySurface)) {
2442 // allow frame dropping by surface by default
2443 mAllowFrameDroppingBySurface = true;
2444 }
2445
2446 format->setObject("native-window", obj);
2447 status_t err = handleSetSurface(static_cast<Surface *>(obj.get()));
2448 if (err != OK) {
2449 PostReplyWithError(replyID, err);
2450 break;
2451 }
2452 } else {
2453 // we are not using surface so this variable is not used, but initialize sensibly anyway
2454 mAllowFrameDroppingBySurface = false;
2455
2456 handleSetSurface(NULL);
2457 }
2458
2459 mReplyID = replyID;
2460 setState(CONFIGURING);
2461
2462 void *crypto;
2463 if (!msg->findPointer("crypto", &crypto)) {
2464 crypto = NULL;
2465 }
2466
2467 ALOGV("kWhatConfigure: Old mCrypto: %p (%d)",
2468 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0));
2469
2470 mCrypto = static_cast<ICrypto *>(crypto);
2471 mBufferChannel->setCrypto(mCrypto);
2472
2473 ALOGV("kWhatConfigure: New mCrypto: %p (%d)",
2474 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0));
2475
2476 void *descrambler;
2477 if (!msg->findPointer("descrambler", &descrambler)) {
2478 descrambler = NULL;
2479 }
2480
2481 mDescrambler = static_cast<IDescrambler *>(descrambler);
2482 mBufferChannel->setDescrambler(mDescrambler);
2483
2484 uint32_t flags;
2485 CHECK(msg->findInt32("flags", (int32_t *)&flags));
2486
2487 if (flags & CONFIGURE_FLAG_ENCODE) {
2488 format->setInt32("encoder", true);
2489 mFlags |= kFlagIsEncoder;
2490 }
2491
2492 extractCSD(format);
2493
2494 mCodec->initiateConfigureComponent(format);
2495 break;
2496 }
2497
2498 case kWhatSetSurface:
2499 {
2500 sp<AReplyToken> replyID;
2501 CHECK(msg->senderAwaitsResponse(&replyID));
2502
2503 status_t err = OK;
2504
2505 switch (mState) {
2506 case CONFIGURED:
2507 case STARTED:
2508 case FLUSHED:
2509 {
2510 sp<RefBase> obj;
2511 (void)msg->findObject("surface", &obj);
2512 sp<Surface> surface = static_cast<Surface *>(obj.get());
2513 if (mSurface == NULL) {
2514 // do not support setting surface if it was not set
2515 err = INVALID_OPERATION;
2516 } else if (obj == NULL) {
2517 // do not support unsetting surface
2518 err = BAD_VALUE;
2519 } else {
2520 err = connectToSurface(surface);
2521 if (err == ALREADY_EXISTS) {
2522 // reconnecting to same surface
2523 err = OK;
2524 } else {
2525 if (err == OK) {
2526 if (mFlags & kFlagUsesSoftwareRenderer) {
2527 if (mSoftRenderer != NULL
2528 && (mFlags & kFlagPushBlankBuffersOnShutdown)) {
2529 pushBlankBuffersToNativeWindow(mSurface.get());
2530 }
2531 surface->setDequeueTimeout(-1);
2532 mSoftRenderer = new SoftwareRenderer(surface);
2533 // TODO: check if this was successful
2534 } else {
2535 err = mCodec->setSurface(surface);
2536 }
2537 }
2538 if (err == OK) {
2539 (void)disconnectFromSurface();
2540 mSurface = surface;
2541 }
2542 }
2543 }
2544 break;
2545 }
2546
2547 default:
2548 err = INVALID_OPERATION;
2549 break;
2550 }
2551
2552 PostReplyWithError(replyID, err);
2553 break;
2554 }
2555
2556 case kWhatCreateInputSurface:
2557 case kWhatSetInputSurface:
2558 {
2559 sp<AReplyToken> replyID;
2560 CHECK(msg->senderAwaitsResponse(&replyID));
2561
2562 // Must be configured, but can't have been started yet.
2563 if (mState != CONFIGURED) {
2564 PostReplyWithError(replyID, INVALID_OPERATION);
2565 break;
2566 }
2567
2568 mReplyID = replyID;
2569 if (msg->what() == kWhatCreateInputSurface) {
2570 mCodec->initiateCreateInputSurface();
2571 } else {
2572 sp<RefBase> obj;
2573 CHECK(msg->findObject("input-surface", &obj));
2574
2575 mCodec->initiateSetInputSurface(
2576 static_cast<PersistentSurface *>(obj.get()));
2577 }
2578 break;
2579 }
2580 case kWhatStart:
2581 {
2582 sp<AReplyToken> replyID;
2583 CHECK(msg->senderAwaitsResponse(&replyID));
2584
2585 if (mState == FLUSHED) {
2586 setState(STARTED);
2587 if (mHavePendingInputBuffers) {
2588 onInputBufferAvailable();
2589 mHavePendingInputBuffers = false;
2590 }
2591 mCodec->signalResume();
2592 PostReplyWithError(replyID, OK);
2593 break;
2594 } else if (mState != CONFIGURED) {
2595 PostReplyWithError(replyID, INVALID_OPERATION);
2596 break;
2597 }
2598
2599 mReplyID = replyID;
2600 setState(STARTING);
2601
2602 mCodec->initiateStart();
2603 break;
2604 }
2605
2606 case kWhatStop:
2607 case kWhatRelease:
2608 {
2609 State targetState =
2610 (msg->what() == kWhatStop) ? INITIALIZED : UNINITIALIZED;
2611
2612 sp<AReplyToken> replyID;
2613 CHECK(msg->senderAwaitsResponse(&replyID));
2614
2615 // already stopped/released
2616 if (mState == UNINITIALIZED && mReleasedByResourceManager) {
2617 sp<AMessage> response = new AMessage;
2618 response->setInt32("err", OK);
2619 response->postReply(replyID);
2620 break;
2621 }
2622
2623 int32_t reclaimed = 0;
2624 msg->findInt32("reclaimed", &reclaimed);
2625 if (reclaimed) {
2626 mReleasedByResourceManager = true;
2627
2628 int32_t force = 0;
2629 msg->findInt32("force", &force);
2630 if (!force && hasPendingBuffer()) {
2631 ALOGW("Can't reclaim codec right now due to pending buffers.");
2632
2633 // return WOULD_BLOCK to ask resource manager to retry later.
2634 sp<AMessage> response = new AMessage;
2635 response->setInt32("err", WOULD_BLOCK);
2636 response->postReply(replyID);
2637
2638 // notify the async client
2639 if (mFlags & kFlagIsAsync) {
2640 onError(DEAD_OBJECT, ACTION_CODE_FATAL);
2641 }
2642 break;
2643 }
2644 }
2645
2646 bool isReleasingAllocatedComponent =
2647 (mFlags & kFlagIsComponentAllocated) && targetState == UNINITIALIZED;
2648 if (!isReleasingAllocatedComponent // See 1
2649 && mState != INITIALIZED
2650 && mState != CONFIGURED && !isExecuting()) {
2651 // 1) Permit release to shut down the component if allocated.
2652 //
2653 // 2) We may be in "UNINITIALIZED" state already and
2654 // also shutdown the encoder/decoder without the
2655 // client being aware of this if media server died while
2656 // we were being stopped. The client would assume that
2657 // after stop() returned, it would be safe to call release()
2658 // and it should be in this case, no harm to allow a release()
2659 // if we're already uninitialized.
2660 sp<AMessage> response = new AMessage;
2661 // TODO: we shouldn't throw an exception for stop/release. Change this to wait until
2662 // the previous stop/release completes and then reply with OK.
2663 status_t err = mState == targetState ? OK : INVALID_OPERATION;
2664 response->setInt32("err", err);
2665 if (err == OK && targetState == UNINITIALIZED) {
2666 mComponentName.clear();
2667 }
2668 response->postReply(replyID);
2669 break;
2670 }
2671
2672 // If we're flushing, stopping, configuring or starting but
2673 // received a release request, post the reply for the pending call
2674 // first, and consider it done. The reply token will be replaced
2675 // after this, and we'll no longer be able to reply.
2676 if (mState == FLUSHING || mState == STOPPING
2677 || mState == CONFIGURING || mState == STARTING) {
2678 (new AMessage)->postReply(mReplyID);
2679 }
2680
2681 if (mFlags & kFlagSawMediaServerDie) {
2682 // It's dead, Jim. Don't expect initiateShutdown to yield
2683 // any useful results now...
2684 setState(UNINITIALIZED);
2685 if (targetState == UNINITIALIZED) {
2686 mComponentName.clear();
2687 }
2688 (new AMessage)->postReply(replyID);
2689 break;
2690 }
2691
2692 // If we already have an error, component may not be able to
2693 // complete the shutdown properly. If we're stopping, post the
2694 // reply now with an error to unblock the client, client can
2695 // release after the failure (instead of ANR).
2696 if (msg->what() == kWhatStop && (mFlags & kFlagStickyError)) {
2697 PostReplyWithError(replyID, getStickyError());
2698 break;
2699 }
2700
2701 mReplyID = replyID;
2702 setState(msg->what() == kWhatStop ? STOPPING : RELEASING);
2703
2704 mCodec->initiateShutdown(
2705 msg->what() == kWhatStop /* keepComponentAllocated */);
2706
2707 returnBuffersToCodec(reclaimed);
2708
2709 if (mSoftRenderer != NULL && (mFlags & kFlagPushBlankBuffersOnShutdown)) {
2710 pushBlankBuffersToNativeWindow(mSurface.get());
2711 }
2712
2713 break;
2714 }
2715
2716 case kWhatDequeueInputBuffer:
2717 {
2718 sp<AReplyToken> replyID;
2719 CHECK(msg->senderAwaitsResponse(&replyID));
2720
2721 if (mFlags & kFlagIsAsync) {
2722 ALOGE("dequeueInputBuffer can't be used in async mode");
2723 PostReplyWithError(replyID, INVALID_OPERATION);
2724 break;
2725 }
2726
2727 if (mHaveInputSurface) {
2728 ALOGE("dequeueInputBuffer can't be used with input surface");
2729 PostReplyWithError(replyID, INVALID_OPERATION);
2730 break;
2731 }
2732
2733 if (handleDequeueInputBuffer(replyID, true /* new request */)) {
2734 break;
2735 }
2736
2737 int64_t timeoutUs;
2738 CHECK(msg->findInt64("timeoutUs", &timeoutUs));
2739
2740 if (timeoutUs == 0LL) {
2741 PostReplyWithError(replyID, -EAGAIN);
2742 break;
2743 }
2744
2745 mFlags |= kFlagDequeueInputPending;
2746 mDequeueInputReplyID = replyID;
2747
2748 if (timeoutUs > 0LL) {
2749 sp<AMessage> timeoutMsg =
2750 new AMessage(kWhatDequeueInputTimedOut, this);
2751 timeoutMsg->setInt32(
2752 "generation", ++mDequeueInputTimeoutGeneration);
2753 timeoutMsg->post(timeoutUs);
2754 }
2755 break;
2756 }
2757
2758 case kWhatDequeueInputTimedOut:
2759 {
2760 int32_t generation;
2761 CHECK(msg->findInt32("generation", &generation));
2762
2763 if (generation != mDequeueInputTimeoutGeneration) {
2764 // Obsolete
2765 break;
2766 }
2767
2768 CHECK(mFlags & kFlagDequeueInputPending);
2769
2770 PostReplyWithError(mDequeueInputReplyID, -EAGAIN);
2771
2772 mFlags &= ~kFlagDequeueInputPending;
2773 mDequeueInputReplyID = 0;
2774 break;
2775 }
2776
2777 case kWhatQueueInputBuffer:
2778 {
2779 sp<AReplyToken> replyID;
2780 CHECK(msg->senderAwaitsResponse(&replyID));
2781
2782 if (!isExecuting()) {
2783 PostReplyWithError(replyID, INVALID_OPERATION);
2784 break;
2785 } else if (mFlags & kFlagStickyError) {
2786 PostReplyWithError(replyID, getStickyError());
2787 break;
2788 }
2789
2790 status_t err = onQueueInputBuffer(msg);
2791
2792 PostReplyWithError(replyID, err);
2793 break;
2794 }
2795
2796 case kWhatDequeueOutputBuffer:
2797 {
2798 sp<AReplyToken> replyID;
2799 CHECK(msg->senderAwaitsResponse(&replyID));
2800
2801 if (mFlags & kFlagIsAsync) {
2802 ALOGE("dequeueOutputBuffer can't be used in async mode");
2803 PostReplyWithError(replyID, INVALID_OPERATION);
2804 break;
2805 }
2806
2807 if (handleDequeueOutputBuffer(replyID, true /* new request */)) {
2808 break;
2809 }
2810
2811 int64_t timeoutUs;
2812 CHECK(msg->findInt64("timeoutUs", &timeoutUs));
2813
2814 if (timeoutUs == 0LL) {
2815 PostReplyWithError(replyID, -EAGAIN);
2816 break;
2817 }
2818
2819 mFlags |= kFlagDequeueOutputPending;
2820 mDequeueOutputReplyID = replyID;
2821
2822 if (timeoutUs > 0LL) {
2823 sp<AMessage> timeoutMsg =
2824 new AMessage(kWhatDequeueOutputTimedOut, this);
2825 timeoutMsg->setInt32(
2826 "generation", ++mDequeueOutputTimeoutGeneration);
2827 timeoutMsg->post(timeoutUs);
2828 }
2829 break;
2830 }
2831
2832 case kWhatDequeueOutputTimedOut:
2833 {
2834 int32_t generation;
2835 CHECK(msg->findInt32("generation", &generation));
2836
2837 if (generation != mDequeueOutputTimeoutGeneration) {
2838 // Obsolete
2839 break;
2840 }
2841
2842 CHECK(mFlags & kFlagDequeueOutputPending);
2843
2844 PostReplyWithError(mDequeueOutputReplyID, -EAGAIN);
2845
2846 mFlags &= ~kFlagDequeueOutputPending;
2847 mDequeueOutputReplyID = 0;
2848 break;
2849 }
2850
2851 case kWhatReleaseOutputBuffer:
2852 {
2853 sp<AReplyToken> replyID;
2854 CHECK(msg->senderAwaitsResponse(&replyID));
2855
2856 if (!isExecuting()) {
2857 PostReplyWithError(replyID, INVALID_OPERATION);
2858 break;
2859 } else if (mFlags & kFlagStickyError) {
2860 PostReplyWithError(replyID, getStickyError());
2861 break;
2862 }
2863
2864 status_t err = onReleaseOutputBuffer(msg);
2865
2866 PostReplyWithError(replyID, err);
2867 break;
2868 }
2869
2870 case kWhatSignalEndOfInputStream:
2871 {
2872 sp<AReplyToken> replyID;
2873 CHECK(msg->senderAwaitsResponse(&replyID));
2874
2875 if (!isExecuting() || !mHaveInputSurface) {
2876 PostReplyWithError(replyID, INVALID_OPERATION);
2877 break;
2878 } else if (mFlags & kFlagStickyError) {
2879 PostReplyWithError(replyID, getStickyError());
2880 break;
2881 }
2882
2883 mReplyID = replyID;
2884 mCodec->signalEndOfInputStream();
2885 break;
2886 }
2887
2888 case kWhatGetBuffers:
2889 {
2890 sp<AReplyToken> replyID;
2891 CHECK(msg->senderAwaitsResponse(&replyID));
2892 if (!isExecuting() || (mFlags & kFlagIsAsync)) {
2893 PostReplyWithError(replyID, INVALID_OPERATION);
2894 break;
2895 } else if (mFlags & kFlagStickyError) {
2896 PostReplyWithError(replyID, getStickyError());
2897 break;
2898 }
2899
2900 int32_t portIndex;
2901 CHECK(msg->findInt32("portIndex", &portIndex));
2902
2903 Vector<sp<MediaCodecBuffer> > *dstBuffers;
2904 CHECK(msg->findPointer("buffers", (void **)&dstBuffers));
2905
2906 dstBuffers->clear();
2907 // If we're using input surface (either non-persistent created by
2908 // createInputSurface(), or persistent set by setInputSurface()),
2909 // give the client an empty input buffers array.
2910 if (portIndex != kPortIndexInput || !mHaveInputSurface) {
2911 if (portIndex == kPortIndexInput) {
2912 mBufferChannel->getInputBufferArray(dstBuffers);
2913 } else {
2914 mBufferChannel->getOutputBufferArray(dstBuffers);
2915 }
2916 }
2917
2918 (new AMessage)->postReply(replyID);
2919 break;
2920 }
2921
2922 case kWhatFlush:
2923 {
2924 sp<AReplyToken> replyID;
2925 CHECK(msg->senderAwaitsResponse(&replyID));
2926
2927 if (!isExecuting()) {
2928 PostReplyWithError(replyID, INVALID_OPERATION);
2929 break;
2930 } else if (mFlags & kFlagStickyError) {
2931 PostReplyWithError(replyID, getStickyError());
2932 break;
2933 }
2934
2935 mReplyID = replyID;
2936 // TODO: skip flushing if already FLUSHED
2937 setState(FLUSHING);
2938
2939 mCodec->signalFlush();
2940 returnBuffersToCodec();
2941 break;
2942 }
2943
2944 case kWhatGetInputFormat:
2945 case kWhatGetOutputFormat:
2946 {
2947 sp<AMessage> format =
2948 (msg->what() == kWhatGetOutputFormat ? mOutputFormat : mInputFormat);
2949
2950 sp<AReplyToken> replyID;
2951 CHECK(msg->senderAwaitsResponse(&replyID));
2952
2953 if ((mState != CONFIGURED && mState != STARTING &&
2954 mState != STARTED && mState != FLUSHING &&
2955 mState != FLUSHED)
2956 || format == NULL) {
2957 PostReplyWithError(replyID, INVALID_OPERATION);
2958 break;
2959 } else if (mFlags & kFlagStickyError) {
2960 PostReplyWithError(replyID, getStickyError());
2961 break;
2962 }
2963
2964 sp<AMessage> response = new AMessage;
2965 response->setMessage("format", format);
2966 response->postReply(replyID);
2967 break;
2968 }
2969
2970 case kWhatRequestIDRFrame:
2971 {
2972 mCodec->signalRequestIDRFrame();
2973 break;
2974 }
2975
2976 case kWhatRequestActivityNotification:
2977 {
2978 CHECK(mActivityNotify == NULL);
2979 CHECK(msg->findMessage("notify", &mActivityNotify));
2980
2981 postActivityNotificationIfPossible();
2982 break;
2983 }
2984
2985 case kWhatGetName:
2986 {
2987 sp<AReplyToken> replyID;
2988 CHECK(msg->senderAwaitsResponse(&replyID));
2989
2990 if (mComponentName.empty()) {
2991 PostReplyWithError(replyID, INVALID_OPERATION);
2992 break;
2993 }
2994
2995 sp<AMessage> response = new AMessage;
2996 response->setString("name", mComponentName.c_str());
2997 response->postReply(replyID);
2998 break;
2999 }
3000
3001 case kWhatGetCodecInfo:
3002 {
3003 sp<AReplyToken> replyID;
3004 CHECK(msg->senderAwaitsResponse(&replyID));
3005
3006 sp<AMessage> response = new AMessage;
3007 response->setObject("codecInfo", mCodecInfo);
3008 response->postReply(replyID);
3009 break;
3010 }
3011
3012 case kWhatSetParameters:
3013 {
3014 sp<AReplyToken> replyID;
3015 CHECK(msg->senderAwaitsResponse(&replyID));
3016
3017 sp<AMessage> params;
3018 CHECK(msg->findMessage("params", ¶ms));
3019
3020 status_t err = onSetParameters(params);
3021
3022 PostReplyWithError(replyID, err);
3023 break;
3024 }
3025
3026 case kWhatDrmReleaseCrypto:
3027 {
3028 onReleaseCrypto(msg);
3029 break;
3030 }
3031
3032 default:
3033 TRESPASS();
3034 }
3035 }
3036
extractCSD(const sp<AMessage> & format)3037 void MediaCodec::extractCSD(const sp<AMessage> &format) {
3038 mCSD.clear();
3039
3040 size_t i = 0;
3041 for (;;) {
3042 sp<ABuffer> csd;
3043 if (!format->findBuffer(AStringPrintf("csd-%u", i).c_str(), &csd)) {
3044 break;
3045 }
3046 if (csd->size() == 0) {
3047 ALOGW("csd-%zu size is 0", i);
3048 }
3049
3050 mCSD.push_back(csd);
3051 ++i;
3052 }
3053
3054 ALOGV("Found %zu pieces of codec specific data.", mCSD.size());
3055 }
3056
queueCSDInputBuffer(size_t bufferIndex)3057 status_t MediaCodec::queueCSDInputBuffer(size_t bufferIndex) {
3058 CHECK(!mCSD.empty());
3059
3060 const BufferInfo &info = mPortBuffers[kPortIndexInput][bufferIndex];
3061
3062 sp<ABuffer> csd = *mCSD.begin();
3063 mCSD.erase(mCSD.begin());
3064
3065 const sp<MediaCodecBuffer> &codecInputData = info.mData;
3066
3067 if (csd->size() > codecInputData->capacity()) {
3068 return -EINVAL;
3069 }
3070 if (codecInputData->data() == NULL) {
3071 ALOGV("Input buffer %zu is not properly allocated", bufferIndex);
3072 return -EINVAL;
3073 }
3074
3075 memcpy(codecInputData->data(), csd->data(), csd->size());
3076
3077 AString errorDetailMsg;
3078
3079 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
3080 msg->setSize("index", bufferIndex);
3081 msg->setSize("offset", 0);
3082 msg->setSize("size", csd->size());
3083 msg->setInt64("timeUs", 0LL);
3084 msg->setInt32("flags", BUFFER_FLAG_CODECCONFIG);
3085 msg->setPointer("errorDetailMsg", &errorDetailMsg);
3086
3087 return onQueueInputBuffer(msg);
3088 }
3089
setState(State newState)3090 void MediaCodec::setState(State newState) {
3091 if (newState == INITIALIZED || newState == UNINITIALIZED) {
3092 delete mSoftRenderer;
3093 mSoftRenderer = NULL;
3094
3095 if ( mCrypto != NULL ) {
3096 ALOGV("setState: ~mCrypto: %p (%d)",
3097 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0));
3098 }
3099 mCrypto.clear();
3100 mDescrambler.clear();
3101 handleSetSurface(NULL);
3102
3103 mInputFormat.clear();
3104 mOutputFormat.clear();
3105 mFlags &= ~kFlagOutputFormatChanged;
3106 mFlags &= ~kFlagOutputBuffersChanged;
3107 mFlags &= ~kFlagStickyError;
3108 mFlags &= ~kFlagIsEncoder;
3109 mFlags &= ~kFlagIsAsync;
3110 mStickyError = OK;
3111
3112 mActivityNotify.clear();
3113 mCallback.clear();
3114 }
3115
3116 if (newState == UNINITIALIZED) {
3117 // return any straggling buffers, e.g. if we got here on an error
3118 returnBuffersToCodec();
3119
3120 // The component is gone, mediaserver's probably back up already
3121 // but should definitely be back up should we try to instantiate
3122 // another component.. and the cycle continues.
3123 mFlags &= ~kFlagSawMediaServerDie;
3124 }
3125
3126 mState = newState;
3127
3128 cancelPendingDequeueOperations();
3129
3130 updateBatteryStat();
3131 }
3132
returnBuffersToCodec(bool isReclaim)3133 void MediaCodec::returnBuffersToCodec(bool isReclaim) {
3134 returnBuffersToCodecOnPort(kPortIndexInput, isReclaim);
3135 returnBuffersToCodecOnPort(kPortIndexOutput, isReclaim);
3136 }
3137
returnBuffersToCodecOnPort(int32_t portIndex,bool isReclaim)3138 void MediaCodec::returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim) {
3139 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
3140 Mutex::Autolock al(mBufferLock);
3141
3142 for (size_t i = 0; i < mPortBuffers[portIndex].size(); ++i) {
3143 BufferInfo *info = &mPortBuffers[portIndex][i];
3144
3145 if (info->mData != nullptr) {
3146 sp<MediaCodecBuffer> buffer = info->mData;
3147 if (isReclaim && info->mOwnedByClient) {
3148 ALOGD("port %d buffer %zu still owned by client when codec is reclaimed",
3149 portIndex, i);
3150 } else {
3151 info->mOwnedByClient = false;
3152 info->mData.clear();
3153 }
3154 mBufferChannel->discardBuffer(buffer);
3155 }
3156 }
3157
3158 mAvailPortBuffers[portIndex].clear();
3159 }
3160
updateBuffers(int32_t portIndex,const sp<AMessage> & msg)3161 size_t MediaCodec::updateBuffers(
3162 int32_t portIndex, const sp<AMessage> &msg) {
3163 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
3164 size_t index;
3165 CHECK(msg->findSize("index", &index));
3166 sp<RefBase> obj;
3167 CHECK(msg->findObject("buffer", &obj));
3168 sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
3169
3170 {
3171 Mutex::Autolock al(mBufferLock);
3172 if (mPortBuffers[portIndex].size() <= index) {
3173 mPortBuffers[portIndex].resize(align(index + 1, kNumBuffersAlign));
3174 }
3175 mPortBuffers[portIndex][index].mData = buffer;
3176 }
3177 mAvailPortBuffers[portIndex].push_back(index);
3178
3179 return index;
3180 }
3181
onQueueInputBuffer(const sp<AMessage> & msg)3182 status_t MediaCodec::onQueueInputBuffer(const sp<AMessage> &msg) {
3183 size_t index;
3184 size_t offset;
3185 size_t size;
3186 int64_t timeUs;
3187 uint32_t flags;
3188 CHECK(msg->findSize("index", &index));
3189 CHECK(msg->findSize("offset", &offset));
3190 CHECK(msg->findInt64("timeUs", &timeUs));
3191 CHECK(msg->findInt32("flags", (int32_t *)&flags));
3192
3193 const CryptoPlugin::SubSample *subSamples;
3194 size_t numSubSamples;
3195 const uint8_t *key;
3196 const uint8_t *iv;
3197 CryptoPlugin::Mode mode = CryptoPlugin::kMode_Unencrypted;
3198
3199 // We allow the simpler queueInputBuffer API to be used even in
3200 // secure mode, by fabricating a single unencrypted subSample.
3201 CryptoPlugin::SubSample ss;
3202 CryptoPlugin::Pattern pattern;
3203
3204 if (msg->findSize("size", &size)) {
3205 if (hasCryptoOrDescrambler()) {
3206 ss.mNumBytesOfClearData = size;
3207 ss.mNumBytesOfEncryptedData = 0;
3208
3209 subSamples = &ss;
3210 numSubSamples = 1;
3211 key = NULL;
3212 iv = NULL;
3213 pattern.mEncryptBlocks = 0;
3214 pattern.mSkipBlocks = 0;
3215 }
3216 } else {
3217 if (!hasCryptoOrDescrambler()) {
3218 ALOGE("[%s] queuing secure buffer without mCrypto or mDescrambler!",
3219 mComponentName.c_str());
3220 return -EINVAL;
3221 }
3222
3223 CHECK(msg->findPointer("subSamples", (void **)&subSamples));
3224 CHECK(msg->findSize("numSubSamples", &numSubSamples));
3225 CHECK(msg->findPointer("key", (void **)&key));
3226 CHECK(msg->findPointer("iv", (void **)&iv));
3227 CHECK(msg->findInt32("encryptBlocks", (int32_t *)&pattern.mEncryptBlocks));
3228 CHECK(msg->findInt32("skipBlocks", (int32_t *)&pattern.mSkipBlocks));
3229
3230 int32_t tmp;
3231 CHECK(msg->findInt32("mode", &tmp));
3232
3233 mode = (CryptoPlugin::Mode)tmp;
3234
3235 size = 0;
3236 for (size_t i = 0; i < numSubSamples; ++i) {
3237 size += subSamples[i].mNumBytesOfClearData;
3238 size += subSamples[i].mNumBytesOfEncryptedData;
3239 }
3240 }
3241
3242 if (index >= mPortBuffers[kPortIndexInput].size()) {
3243 return -ERANGE;
3244 }
3245
3246 BufferInfo *info = &mPortBuffers[kPortIndexInput][index];
3247
3248 if (info->mData == nullptr || !info->mOwnedByClient) {
3249 return -EACCES;
3250 }
3251
3252 if (offset + size > info->mData->capacity()) {
3253 return -EINVAL;
3254 }
3255
3256 info->mData->setRange(offset, size);
3257 info->mData->meta()->setInt64("timeUs", timeUs);
3258 if (flags & BUFFER_FLAG_EOS) {
3259 info->mData->meta()->setInt32("eos", true);
3260 }
3261
3262 if (flags & BUFFER_FLAG_CODECCONFIG) {
3263 info->mData->meta()->setInt32("csd", true);
3264 }
3265
3266 sp<MediaCodecBuffer> buffer = info->mData;
3267 status_t err = OK;
3268 if (hasCryptoOrDescrambler()) {
3269 AString *errorDetailMsg;
3270 CHECK(msg->findPointer("errorDetailMsg", (void **)&errorDetailMsg));
3271
3272 err = mBufferChannel->queueSecureInputBuffer(
3273 buffer,
3274 (mFlags & kFlagIsSecure),
3275 key,
3276 iv,
3277 mode,
3278 pattern,
3279 subSamples,
3280 numSubSamples,
3281 errorDetailMsg);
3282 } else {
3283 err = mBufferChannel->queueInputBuffer(buffer);
3284 }
3285
3286 if (err == OK) {
3287 // synchronization boundary for getBufferAndFormat
3288 Mutex::Autolock al(mBufferLock);
3289 info->mOwnedByClient = false;
3290 info->mData.clear();
3291
3292 statsBufferSent(timeUs);
3293 }
3294
3295 return err;
3296 }
3297
3298 //static
CreateFramesRenderedMessage(const std::list<FrameRenderTracker::Info> & done,sp<AMessage> & msg)3299 size_t MediaCodec::CreateFramesRenderedMessage(
3300 const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg) {
3301 size_t index = 0;
3302
3303 for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();
3304 it != done.cend(); ++it) {
3305 if (it->getRenderTimeNs() < 0) {
3306 continue; // dropped frame from tracking
3307 }
3308 msg->setInt64(AStringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs());
3309 msg->setInt64(AStringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs());
3310 ++index;
3311 }
3312 return index;
3313 }
3314
onReleaseOutputBuffer(const sp<AMessage> & msg)3315 status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
3316 size_t index;
3317 CHECK(msg->findSize("index", &index));
3318
3319 int32_t render;
3320 if (!msg->findInt32("render", &render)) {
3321 render = 0;
3322 }
3323
3324 if (!isExecuting()) {
3325 return -EINVAL;
3326 }
3327
3328 if (index >= mPortBuffers[kPortIndexOutput].size()) {
3329 return -ERANGE;
3330 }
3331
3332 BufferInfo *info = &mPortBuffers[kPortIndexOutput][index];
3333
3334 if (info->mData == nullptr || !info->mOwnedByClient) {
3335 return -EACCES;
3336 }
3337
3338 // synchronization boundary for getBufferAndFormat
3339 sp<MediaCodecBuffer> buffer;
3340 {
3341 Mutex::Autolock al(mBufferLock);
3342 info->mOwnedByClient = false;
3343 buffer = info->mData;
3344 info->mData.clear();
3345 }
3346
3347 if (render && buffer->size() != 0) {
3348 int64_t mediaTimeUs = -1;
3349 buffer->meta()->findInt64("timeUs", &mediaTimeUs);
3350
3351 int64_t renderTimeNs = 0;
3352 if (!msg->findInt64("timestampNs", &renderTimeNs)) {
3353 // use media timestamp if client did not request a specific render timestamp
3354 ALOGV("using buffer PTS of %lld", (long long)mediaTimeUs);
3355 renderTimeNs = mediaTimeUs * 1000;
3356 }
3357
3358 if (mSoftRenderer != NULL) {
3359 std::list<FrameRenderTracker::Info> doneFrames = mSoftRenderer->render(
3360 buffer->data(), buffer->size(), mediaTimeUs, renderTimeNs,
3361 mPortBuffers[kPortIndexOutput].size(), buffer->format());
3362
3363 // if we are running, notify rendered frames
3364 if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) {
3365 sp<AMessage> notify = mOnFrameRenderedNotification->dup();
3366 sp<AMessage> data = new AMessage;
3367 if (CreateFramesRenderedMessage(doneFrames, data)) {
3368 notify->setMessage("data", data);
3369 notify->post();
3370 }
3371 }
3372 }
3373 mBufferChannel->renderOutputBuffer(buffer, renderTimeNs);
3374 } else {
3375 mBufferChannel->discardBuffer(buffer);
3376 }
3377
3378 return OK;
3379 }
3380
dequeuePortBuffer(int32_t portIndex)3381 ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {
3382 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
3383
3384 List<size_t> *availBuffers = &mAvailPortBuffers[portIndex];
3385
3386 if (availBuffers->empty()) {
3387 return -EAGAIN;
3388 }
3389
3390 size_t index = *availBuffers->begin();
3391 availBuffers->erase(availBuffers->begin());
3392
3393 BufferInfo *info = &mPortBuffers[portIndex][index];
3394 CHECK(!info->mOwnedByClient);
3395 {
3396 Mutex::Autolock al(mBufferLock);
3397 info->mOwnedByClient = true;
3398
3399 // set image-data
3400 if (info->mData->format() != NULL) {
3401 sp<ABuffer> imageData;
3402 if (info->mData->format()->findBuffer("image-data", &imageData)) {
3403 info->mData->meta()->setBuffer("image-data", imageData);
3404 }
3405 int32_t left, top, right, bottom;
3406 if (info->mData->format()->findRect("crop", &left, &top, &right, &bottom)) {
3407 info->mData->meta()->setRect("crop-rect", left, top, right, bottom);
3408 }
3409 }
3410 }
3411
3412 return index;
3413 }
3414
connectToSurface(const sp<Surface> & surface)3415 status_t MediaCodec::connectToSurface(const sp<Surface> &surface) {
3416 status_t err = OK;
3417 if (surface != NULL) {
3418 uint64_t oldId, newId;
3419 if (mSurface != NULL
3420 && surface->getUniqueId(&newId) == NO_ERROR
3421 && mSurface->getUniqueId(&oldId) == NO_ERROR
3422 && newId == oldId) {
3423 ALOGI("[%s] connecting to the same surface. Nothing to do.", mComponentName.c_str());
3424 return ALREADY_EXISTS;
3425 }
3426
3427 err = nativeWindowConnect(surface.get(), "connectToSurface");
3428 if (err == OK) {
3429 // Require a fresh set of buffers after each connect by using a unique generation
3430 // number. Rely on the fact that max supported process id by Linux is 2^22.
3431 // PID is never 0 so we don't have to worry that we use the default generation of 0.
3432 // TODO: come up with a unique scheme if other producers also set the generation number.
3433 static uint32_t mSurfaceGeneration = 0;
3434 uint32_t generation = (getpid() << 10) | (++mSurfaceGeneration & ((1 << 10) - 1));
3435 surface->setGenerationNumber(generation);
3436 ALOGI("[%s] setting surface generation to %u", mComponentName.c_str(), generation);
3437
3438 // HACK: clear any free buffers. Remove when connect will automatically do this.
3439 // This is needed as the consumer may be holding onto stale frames that it can reattach
3440 // to this surface after disconnect/connect, and those free frames would inherit the new
3441 // generation number. Disconnecting after setting a unique generation prevents this.
3442 nativeWindowDisconnect(surface.get(), "connectToSurface(reconnect)");
3443 err = nativeWindowConnect(surface.get(), "connectToSurface(reconnect)");
3444 }
3445
3446 if (err != OK) {
3447 ALOGE("nativeWindowConnect returned an error: %s (%d)", strerror(-err), err);
3448 } else {
3449 if (!mAllowFrameDroppingBySurface) {
3450 disableLegacyBufferDropPostQ(surface);
3451 }
3452 }
3453 }
3454 // do not return ALREADY_EXISTS unless surfaces are the same
3455 return err == ALREADY_EXISTS ? BAD_VALUE : err;
3456 }
3457
disconnectFromSurface()3458 status_t MediaCodec::disconnectFromSurface() {
3459 status_t err = OK;
3460 if (mSurface != NULL) {
3461 // Resetting generation is not technically needed, but there is no need to keep it either
3462 mSurface->setGenerationNumber(0);
3463 err = nativeWindowDisconnect(mSurface.get(), "disconnectFromSurface");
3464 if (err != OK) {
3465 ALOGW("nativeWindowDisconnect returned an error: %s (%d)", strerror(-err), err);
3466 }
3467 // assume disconnected even on error
3468 mSurface.clear();
3469 }
3470 return err;
3471 }
3472
handleSetSurface(const sp<Surface> & surface)3473 status_t MediaCodec::handleSetSurface(const sp<Surface> &surface) {
3474 status_t err = OK;
3475 if (mSurface != NULL) {
3476 (void)disconnectFromSurface();
3477 }
3478 if (surface != NULL) {
3479 err = connectToSurface(surface);
3480 if (err == OK) {
3481 mSurface = surface;
3482 }
3483 }
3484 return err;
3485 }
3486
onInputBufferAvailable()3487 void MediaCodec::onInputBufferAvailable() {
3488 int32_t index;
3489 while ((index = dequeuePortBuffer(kPortIndexInput)) >= 0) {
3490 sp<AMessage> msg = mCallback->dup();
3491 msg->setInt32("callbackID", CB_INPUT_AVAILABLE);
3492 msg->setInt32("index", index);
3493 msg->post();
3494 }
3495 }
3496
onOutputBufferAvailable()3497 void MediaCodec::onOutputBufferAvailable() {
3498 int32_t index;
3499 while ((index = dequeuePortBuffer(kPortIndexOutput)) >= 0) {
3500 const sp<MediaCodecBuffer> &buffer =
3501 mPortBuffers[kPortIndexOutput][index].mData;
3502 sp<AMessage> msg = mCallback->dup();
3503 msg->setInt32("callbackID", CB_OUTPUT_AVAILABLE);
3504 msg->setInt32("index", index);
3505 msg->setSize("offset", buffer->offset());
3506 msg->setSize("size", buffer->size());
3507
3508 int64_t timeUs;
3509 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
3510
3511 msg->setInt64("timeUs", timeUs);
3512
3513 statsBufferReceived(timeUs);
3514
3515 int32_t flags;
3516 CHECK(buffer->meta()->findInt32("flags", &flags));
3517
3518 msg->setInt32("flags", flags);
3519
3520 msg->post();
3521 }
3522 }
3523
onError(status_t err,int32_t actionCode,const char * detail)3524 void MediaCodec::onError(status_t err, int32_t actionCode, const char *detail) {
3525 if (mCallback != NULL) {
3526 sp<AMessage> msg = mCallback->dup();
3527 msg->setInt32("callbackID", CB_ERROR);
3528 msg->setInt32("err", err);
3529 msg->setInt32("actionCode", actionCode);
3530
3531 if (detail != NULL) {
3532 msg->setString("detail", detail);
3533 }
3534
3535 msg->post();
3536 }
3537 }
3538
onOutputFormatChanged()3539 void MediaCodec::onOutputFormatChanged() {
3540 if (mCallback != NULL) {
3541 sp<AMessage> msg = mCallback->dup();
3542 msg->setInt32("callbackID", CB_OUTPUT_FORMAT_CHANGED);
3543 msg->setMessage("format", mOutputFormat);
3544 msg->post();
3545 }
3546 }
3547
postActivityNotificationIfPossible()3548 void MediaCodec::postActivityNotificationIfPossible() {
3549 if (mActivityNotify == NULL) {
3550 return;
3551 }
3552
3553 bool isErrorOrOutputChanged =
3554 (mFlags & (kFlagStickyError
3555 | kFlagOutputBuffersChanged
3556 | kFlagOutputFormatChanged));
3557
3558 if (isErrorOrOutputChanged
3559 || !mAvailPortBuffers[kPortIndexInput].empty()
3560 || !mAvailPortBuffers[kPortIndexOutput].empty()) {
3561 mActivityNotify->setInt32("input-buffers",
3562 mAvailPortBuffers[kPortIndexInput].size());
3563
3564 if (isErrorOrOutputChanged) {
3565 // we want consumer to dequeue as many times as it can
3566 mActivityNotify->setInt32("output-buffers", INT32_MAX);
3567 } else {
3568 mActivityNotify->setInt32("output-buffers",
3569 mAvailPortBuffers[kPortIndexOutput].size());
3570 }
3571 mActivityNotify->post();
3572 mActivityNotify.clear();
3573 }
3574 }
3575
setParameters(const sp<AMessage> & params)3576 status_t MediaCodec::setParameters(const sp<AMessage> ¶ms) {
3577 sp<AMessage> msg = new AMessage(kWhatSetParameters, this);
3578 msg->setMessage("params", params);
3579
3580 sp<AMessage> response;
3581 return PostAndAwaitResponse(msg, &response);
3582 }
3583
onSetParameters(const sp<AMessage> & params)3584 status_t MediaCodec::onSetParameters(const sp<AMessage> ¶ms) {
3585 mCodec->signalSetParameters(params);
3586
3587 return OK;
3588 }
3589
amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> & buffer)3590 status_t MediaCodec::amendOutputFormatWithCodecSpecificData(
3591 const sp<MediaCodecBuffer> &buffer) {
3592 AString mime;
3593 CHECK(mOutputFormat->findString("mime", &mime));
3594
3595 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) {
3596 // Codec specific data should be SPS and PPS in a single buffer,
3597 // each prefixed by a startcode (0x00 0x00 0x00 0x01).
3598 // We separate the two and put them into the output format
3599 // under the keys "csd-0" and "csd-1".
3600
3601 unsigned csdIndex = 0;
3602
3603 const uint8_t *data = buffer->data();
3604 size_t size = buffer->size();
3605
3606 const uint8_t *nalStart;
3607 size_t nalSize;
3608 while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) {
3609 sp<ABuffer> csd = new ABuffer(nalSize + 4);
3610 memcpy(csd->data(), "\x00\x00\x00\x01", 4);
3611 memcpy(csd->data() + 4, nalStart, nalSize);
3612
3613 mOutputFormat->setBuffer(
3614 AStringPrintf("csd-%u", csdIndex).c_str(), csd);
3615
3616 ++csdIndex;
3617 }
3618
3619 if (csdIndex != 2) {
3620 return ERROR_MALFORMED;
3621 }
3622 } else {
3623 // For everything else we just stash the codec specific data into
3624 // the output format as a single piece of csd under "csd-0".
3625 sp<ABuffer> csd = new ABuffer(buffer->size());
3626 memcpy(csd->data(), buffer->data(), buffer->size());
3627 csd->setRange(0, buffer->size());
3628 mOutputFormat->setBuffer("csd-0", csd);
3629 }
3630
3631 return OK;
3632 }
3633
updateBatteryStat()3634 void MediaCodec::updateBatteryStat() {
3635 if (!mIsVideo) {
3636 return;
3637 }
3638
3639 if (mState == CONFIGURED && !mBatteryStatNotified) {
3640 BatteryNotifier::getInstance().noteStartVideo(mUid);
3641 mBatteryStatNotified = true;
3642 } else if (mState == UNINITIALIZED && mBatteryStatNotified) {
3643 BatteryNotifier::getInstance().noteStopVideo(mUid);
3644 mBatteryStatNotified = false;
3645 }
3646 }
3647
stateString(State state)3648 std::string MediaCodec::stateString(State state) {
3649 const char *rval = NULL;
3650 char rawbuffer[16]; // room for "%d"
3651
3652 switch (state) {
3653 case UNINITIALIZED: rval = "UNINITIALIZED"; break;
3654 case INITIALIZING: rval = "INITIALIZING"; break;
3655 case INITIALIZED: rval = "INITIALIZED"; break;
3656 case CONFIGURING: rval = "CONFIGURING"; break;
3657 case CONFIGURED: rval = "CONFIGURED"; break;
3658 case STARTING: rval = "STARTING"; break;
3659 case STARTED: rval = "STARTED"; break;
3660 case FLUSHING: rval = "FLUSHING"; break;
3661 case FLUSHED: rval = "FLUSHED"; break;
3662 case STOPPING: rval = "STOPPING"; break;
3663 case RELEASING: rval = "RELEASING"; break;
3664 default:
3665 snprintf(rawbuffer, sizeof(rawbuffer), "%d", state);
3666 rval = rawbuffer;
3667 break;
3668 }
3669 return rval;
3670 }
3671
3672 } // namespace android
3673