1 /*
2 * Copyright (C) 2009 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "MPEG4Writer"
19
20 #include <algorithm>
21
22 #include <arpa/inet.h>
23 #include <fcntl.h>
24 #include <inttypes.h>
25 #include <pthread.h>
26 #include <sys/prctl.h>
27 #include <sys/stat.h>
28 #include <sys/types.h>
29 #include <unistd.h>
30
31 #include <utils/Log.h>
32
33 #include <functional>
34
35 #include <media/stagefright/MediaSource.h>
36 #include <media/stagefright/foundation/ADebug.h>
37 #include <media/stagefright/foundation/AMessage.h>
38 #include <media/stagefright/foundation/ALookup.h>
39 #include <media/stagefright/foundation/AUtils.h>
40 #include <media/stagefright/foundation/ByteUtils.h>
41 #include <media/stagefright/foundation/ColorUtils.h>
42 #include <media/stagefright/foundation/avc_utils.h>
43 #include <media/stagefright/MPEG4Writer.h>
44 #include <media/stagefright/MediaBuffer.h>
45 #include <media/stagefright/MetaData.h>
46 #include <media/stagefright/MediaDefs.h>
47 #include <media/stagefright/MediaCodecConstants.h>
48 #include <media/stagefright/MediaErrors.h>
49 #include <media/stagefright/Utils.h>
50 #include <media/mediarecorder.h>
51 #include <cutils/properties.h>
52
53 #include <media/esds/ESDS.h>
54 #include "include/HevcUtils.h"
55
56 #include <com_android_internal_camera_flags.h>
57 #include <com_android_media_editing_flags.h>
58 namespace editing_flags = com::android::media::editing::flags;
59
60 #ifndef __predict_false
61 #define __predict_false(exp) __builtin_expect((exp) != 0, 0)
62 #endif
63
64 #define WARN_UNLESS(condition, message, ...) \
65 ( (__predict_false(condition)) ? false : ({ \
66 ALOGW("Condition %s failed " message, #condition, ##__VA_ARGS__); \
67 true; \
68 }))
69
70 namespace flags_camera = com::android::internal::camera::flags;
71
72 namespace android {
73
74 static const int64_t kMinStreamableFileSizeInBytes = 5 * 1024 * 1024;
75 static const uint8_t kNalUnitTypeSeqParamSet = 0x07;
76 static const uint8_t kNalUnitTypePicParamSet = 0x08;
77 static const int64_t kInitialDelayTimeUs = 700000LL;
78 static const int64_t kMaxMetadataSize = 0x4000000LL; // 64MB max per-frame metadata size
79 static const int64_t kMaxCttsOffsetTimeUs = 30 * 60 * 1000000LL; // 30 minutes
80 static const size_t kESDSScratchBufferSize = 10; // kMaxAtomSize in Mpeg4Extractor 64MB
81 // Allow up to 100 milli second, which is safely above the maximum delay observed in manual testing
82 // between posting from setNextFd and handling it
83 static const int64_t kFdCondWaitTimeoutNs = 100000000;
84
85 static const char kMetaKey_Version[] = "com.android.version";
86 static const char kMetaKey_Manufacturer[] = "com.android.manufacturer";
87 static const char kMetaKey_Model[] = "com.android.model";
88
89 #ifdef SHOW_BUILD
90 static const char kMetaKey_Build[] = "com.android.build";
91 #endif
92 static const char kMetaKey_CaptureFps[] = "com.android.capture.fps";
93 static const char kMetaKey_TemporalLayerCount[] = "com.android.video.temporal_layers_count";
94
95 static const int kTimestampDebugCount = 10;
96 static const int kItemIdBase = 10000;
97 static const char kExifHeader[] = {'E', 'x', 'i', 'f', '\0', '\0'};
98 static const char kGainmapMetaHeader[] = {'t', 'm', 'a', 'p', '\0', '\0'};
99 static const char kGainmapHeader[] = {'g', 'm', 'a', 'p', '\0', '\0'};
100 static const uint8_t kExifApp1Marker[] = {'E', 'x', 'i', 'f', 0xff, 0xe1};
101
102 static const uint8_t kMandatoryHevcNalUnitTypes[3] = {
103 kHevcNalUnitTypeVps,
104 kHevcNalUnitTypeSps,
105 kHevcNalUnitTypePps,
106 };
107 static const uint8_t kHevcNalUnitTypes[5] = {
108 kHevcNalUnitTypeVps,
109 kHevcNalUnitTypeSps,
110 kHevcNalUnitTypePps,
111 kHevcNalUnitTypePrefixSei,
112 kHevcNalUnitTypeSuffixSei,
113 };
114 /* uncomment to include build in meta */
115 //#define SHOW_MODEL_BUILD 1
116
117 class MPEG4Writer::Track {
118 struct TrackId {
TrackIdandroid::MPEG4Writer::Track::TrackId119 TrackId(uint32_t aId)
120 :mId(aId),
121 mTrackIdValid(false) {
122 }
isValidandroid::MPEG4Writer::Track::TrackId123 bool isValid(bool akKey4BitTrackIds) {
124 // trackId cannot be zero, ISO/IEC 14496-12 8.3.2.3
125 if (mId == 0) {
126 return false;
127 }
128 /* MediaRecorder uses only 4 bit to represent track ids during notifying clients.
129 * MediaMuxer's track ids are restricted by container allowed size only.
130 * MPEG4 Container defines unsigned int (32), ISO/IEC 14496-12 8.3.2.2
131 */
132 if (akKey4BitTrackIds && mId > 15) {
133 return false;
134 }
135 mTrackIdValid = true;
136 return true;
137 }
getIdandroid::MPEG4Writer::Track::TrackId138 uint32_t getId() const {
139 CHECK(mTrackIdValid);
140 return mId;
141 }
142 TrackId() = delete;
143 DISALLOW_EVIL_CONSTRUCTORS(TrackId);
144 private:
145 // unsigned int (32), ISO/IEC 14496-12 8.3.2.2
146 uint32_t mId;
147 bool mTrackIdValid;
148 };
149
150 public:
151 Track(MPEG4Writer *owner, const sp<MediaSource> &source, uint32_t aTrackId);
152
153 ~Track();
154
155 status_t start(MetaData *params);
156 status_t stop(bool stopSource = true);
157 status_t pause();
158 bool reachedEOS();
159
160 int64_t getDurationUs() const;
161 int64_t getEstimatedTrackSizeBytes() const;
162 int32_t getMetaSizeIncrease(int32_t angle, int32_t trackCount) const;
163 void writeTrackHeader();
164 int64_t getMinCttsOffsetTimeUs();
165 void bufferChunk(int64_t timestampUs);
isAvc() const166 bool isAvc() const { return mIsAvc; }
isHevc() const167 bool isHevc() const { return mIsHevc; }
isAv1() const168 bool isAv1() const { return mIsAv1; }
isApv() const169 bool isApv() const { return mIsApv; }
isHeic() const170 bool isHeic() const { return mIsHeic; }
isAvif() const171 bool isAvif() const { return mIsAvif; }
isHeif() const172 bool isHeif() const { return mIsHeif; }
isAudio() const173 bool isAudio() const { return mIsAudio; }
isMPEG4() const174 bool isMPEG4() const { return mIsMPEG4; }
usePrefix() const175 bool usePrefix() const { return mIsAvc || mIsHevc || mIsHeic || mIsDovi; }
176 bool isExifData(MediaBufferBase *buffer, uint32_t *tiffHdrOffset) const;
177 bool isGainmapMetaData(MediaBufferBase* buffer, uint32_t* offset) const;
178 bool isGainmapData(MediaBufferBase* buffer, uint32_t* offset) const;
179 void addChunkOffset(off64_t offset);
180 void addItemOffsetAndSize(off64_t offset, size_t size, bool isExif,
181 bool isGainmapMeta = false, bool isGainmap = false);
182 void flushItemRefs();
getTrackId()183 TrackId& getTrackId() { return mTrackId; }
184 status_t dump(int fd, const Vector<String16>& args) const;
185 static const char *getFourCCForMime(const char *mime);
186 const char *getDoviFourCC() const;
187 const char *getTrackType() const;
188 void resetInternal();
189 int64_t trackMetaDataSize();
190 bool isTimestampValid(int64_t timeUs);
getImageItemId()191 uint16_t getImageItemId() { return mImageItemId; };
getGainmapItemId()192 uint16_t getGainmapItemId() { return mGainmapItemId; };
getGainmapMetaItemId()193 uint16_t getGainmapMetaItemId() { return mGainmapMetadataItemId; };
194
195 private:
196 // A helper class to handle faster write box with table entries
197 template<class TYPE, unsigned ENTRY_SIZE>
198 // ENTRY_SIZE: # of values in each entry
199 struct ListTableEntries {
200 static_assert(ENTRY_SIZE > 0, "ENTRY_SIZE must be positive");
ListTableEntriesandroid::MPEG4Writer::Track::ListTableEntries201 ListTableEntries(uint32_t elementCapacity)
202 : mElementCapacity(elementCapacity),
203 mTotalNumTableEntries(0),
204 mNumValuesInCurrEntry(0),
205 mCurrTableEntriesElement(NULL) {
206 CHECK_GT(mElementCapacity, 0u);
207 // Ensure no integer overflow on allocation in add().
208 CHECK_LT(ENTRY_SIZE, UINT32_MAX / mElementCapacity);
209 }
210
211 // Free the allocated memory.
~ListTableEntriesandroid::MPEG4Writer::Track::ListTableEntries212 ~ListTableEntries() {
213 while (!mTableEntryList.empty()) {
214 typename List<TYPE *>::iterator it = mTableEntryList.begin();
215 delete[] (*it);
216 mTableEntryList.erase(it);
217 }
218 }
219
220 // Replace the value at the given position by the given value.
221 // There must be an existing value at the given position.
222 // @arg value must be in network byte order
223 // @arg pos location the value must be in.
setandroid::MPEG4Writer::Track::ListTableEntries224 void set(const TYPE& value, uint32_t pos) {
225 CHECK_LT(pos, mTotalNumTableEntries * ENTRY_SIZE);
226
227 typename List<TYPE *>::iterator it = mTableEntryList.begin();
228 uint32_t iterations = (pos / (mElementCapacity * ENTRY_SIZE));
229 while (it != mTableEntryList.end() && iterations > 0) {
230 ++it;
231 --iterations;
232 }
233 CHECK(it != mTableEntryList.end());
234 CHECK_EQ(iterations, 0u);
235
236 (*it)[(pos % (mElementCapacity * ENTRY_SIZE))] = value;
237 }
238
239 // Get the value at the given position by the given value.
240 // @arg value the retrieved value at the position in network byte order.
241 // @arg pos location the value must be in.
242 // @return true if a value is found.
getandroid::MPEG4Writer::Track::ListTableEntries243 bool get(TYPE& value, uint32_t pos) const {
244 if (pos >= mTotalNumTableEntries * ENTRY_SIZE) {
245 return false;
246 }
247
248 typename List<TYPE *>::iterator it = mTableEntryList.begin();
249 uint32_t iterations = (pos / (mElementCapacity * ENTRY_SIZE));
250 while (it != mTableEntryList.end() && iterations > 0) {
251 ++it;
252 --iterations;
253 }
254 CHECK(it != mTableEntryList.end());
255 CHECK_EQ(iterations, 0u);
256
257 value = (*it)[(pos % (mElementCapacity * ENTRY_SIZE))];
258 return true;
259 }
260
261 // adjusts all values by |adjust(value)|
adjustEntriesandroid::MPEG4Writer::Track::ListTableEntries262 void adjustEntries(
263 std::function<void(size_t /* ix */, TYPE(& /* entry */)[ENTRY_SIZE])> update) {
264 size_t nEntries = mTotalNumTableEntries + mNumValuesInCurrEntry / ENTRY_SIZE;
265 size_t ix = 0;
266 for (TYPE *entryArray : mTableEntryList) {
267 size_t num = std::min(nEntries, (size_t)mElementCapacity);
268 for (size_t i = 0; i < num; ++i) {
269 update(ix++, (TYPE(&)[ENTRY_SIZE])(*entryArray));
270 entryArray += ENTRY_SIZE;
271 }
272 nEntries -= num;
273 }
274 }
275
276 // Store a single value.
277 // @arg value must be in network byte order.
addandroid::MPEG4Writer::Track::ListTableEntries278 void add(const TYPE& value) {
279 CHECK_LT(mNumValuesInCurrEntry, mElementCapacity);
280 uint32_t nEntries = mTotalNumTableEntries % mElementCapacity;
281 uint32_t nValues = mNumValuesInCurrEntry % ENTRY_SIZE;
282 if (nEntries == 0 && nValues == 0) {
283 mCurrTableEntriesElement = new TYPE[ENTRY_SIZE * mElementCapacity];
284 CHECK(mCurrTableEntriesElement != NULL);
285 mTableEntryList.push_back(mCurrTableEntriesElement);
286 }
287
288 uint32_t pos = nEntries * ENTRY_SIZE + nValues;
289 mCurrTableEntriesElement[pos] = value;
290
291 ++mNumValuesInCurrEntry;
292 if ((mNumValuesInCurrEntry % ENTRY_SIZE) == 0) {
293 ++mTotalNumTableEntries;
294 mNumValuesInCurrEntry = 0;
295 }
296 }
297
298 // Write out the table entries:
299 // 1. the number of entries goes first
300 // 2. followed by the values in the table enties in order
301 // @arg writer the writer to actual write to the storage
writeandroid::MPEG4Writer::Track::ListTableEntries302 void write(MPEG4Writer *writer) const {
303 CHECK_EQ(mNumValuesInCurrEntry % ENTRY_SIZE, 0u);
304 uint32_t nEntries = mTotalNumTableEntries;
305 writer->writeInt32(nEntries);
306 for (typename List<TYPE *>::iterator it = mTableEntryList.begin();
307 it != mTableEntryList.end(); ++it) {
308 CHECK_GT(nEntries, 0u);
309 if (nEntries >= mElementCapacity) {
310 writer->write(*it, sizeof(TYPE) * ENTRY_SIZE, mElementCapacity);
311 nEntries -= mElementCapacity;
312 } else {
313 writer->write(*it, sizeof(TYPE) * ENTRY_SIZE, nEntries);
314 break;
315 }
316 }
317 }
318
319 // Return the number of entries in the table.
countandroid::MPEG4Writer::Track::ListTableEntries320 uint32_t count() const { return mTotalNumTableEntries; }
321
322 private:
323 uint32_t mElementCapacity; // # entries in an element
324 uint32_t mTotalNumTableEntries;
325 uint32_t mNumValuesInCurrEntry; // up to ENTRY_SIZE
326 TYPE *mCurrTableEntriesElement;
327 mutable List<TYPE *> mTableEntryList;
328
329 DISALLOW_EVIL_CONSTRUCTORS(ListTableEntries);
330 };
331
332
333
334 MPEG4Writer *mOwner;
335 sp<MetaData> mMeta;
336 sp<MediaSource> mSource;
337 volatile bool mDone;
338 volatile bool mPaused;
339 volatile bool mResumed;
340 volatile bool mStarted;
341 bool mIsAvc;
342 bool mIsHevc;
343 bool mIsAv1;
344 bool mIsApv;
345 bool mIsDovi;
346 bool mIsAudio;
347 bool mIsVideo;
348 bool mIsHeic;
349 bool mIsAvif;
350 bool mIsHeif;
351 bool mIsMPEG4;
352 bool mGotStartKeyFrame;
353 bool mIsMalformed;
354 TrackId mTrackId;
355 int64_t mTrackDurationUs;
356 int64_t mMaxChunkDurationUs;
357 int64_t mLastDecodingTimeUs;
358 int64_t mEstimatedTrackSizeBytes;
359 int64_t mMdatSizeBytes;
360 int32_t mTimeScale;
361
362 pthread_t mThread;
363
364 List<MediaBuffer *> mChunkSamples;
365
366 bool mSamplesHaveSameSize;
367 ListTableEntries<uint32_t, 1> *mStszTableEntries;
368 ListTableEntries<off64_t, 1> *mCo64TableEntries;
369 ListTableEntries<uint32_t, 3> *mStscTableEntries;
370 ListTableEntries<uint32_t, 1> *mStssTableEntries;
371 ListTableEntries<uint32_t, 2> *mSttsTableEntries;
372 ListTableEntries<uint32_t, 2> *mCttsTableEntries;
373 ListTableEntries<uint32_t, 3> *mElstTableEntries; // 3columns: segDuration, mediaTime, mediaRate
374
375 int64_t mMinCttsOffsetTimeUs;
376 int64_t mMinCttsOffsetTicks;
377 int64_t mMaxCttsOffsetTicks;
378
379 // Save the last 10 frames' timestamp and frame type for debug.
380 struct TimestampDebugHelperEntry {
381 int64_t pts;
382 int64_t dts;
383 std::string frameType;
384 };
385
386 std::list<TimestampDebugHelperEntry> mTimestampDebugHelper;
387
388 // Sequence parameter set or picture parameter set
389 struct AVCParamSet {
AVCParamSetandroid::MPEG4Writer::Track::AVCParamSet390 AVCParamSet(uint16_t length, const uint8_t *data)
391 : mLength(length), mData(data) {}
392
393 uint16_t mLength;
394 const uint8_t *mData;
395 };
396 List<AVCParamSet> mSeqParamSets;
397 List<AVCParamSet> mPicParamSets;
398 uint8_t mProfileIdc;
399 uint8_t mProfileCompatible;
400 uint8_t mLevelIdc;
401
402 int32_t mDoviProfile;
403
404 void *mCodecSpecificData;
405 size_t mCodecSpecificDataSize;
406 bool mGotAllCodecSpecificData;
407 bool mTrackingProgressStatus;
408
409 bool mReachedEOS;
410 int64_t mStartTimestampUs;
411 int64_t mStartTimeRealUs;
412 int64_t mFirstSampleTimeRealUs;
413 // Captures negative start offset of a track(track starttime < 0).
414 int64_t mFirstSampleStartOffsetUs;
415 int64_t mPreviousTrackTimeUs;
416 int64_t mTrackEveryTimeDurationUs;
417
418 int32_t mRotation;
419
420 Vector<uint16_t> mProperties;
421 ItemRefs mDimgRefs;
422 ItemRefs mGainmapDimgRefs;
423 Vector<uint16_t> mExifList;
424 uint16_t mImageItemId;
425 uint16_t mItemIdBase;
426 int32_t mIsPrimary;
427 int32_t mWidth, mHeight;
428 int32_t mTileWidth, mTileHeight;
429 int32_t mGridRows, mGridCols;
430 size_t mNumTiles, mTileIndex;
431 uint16_t mGainmapItemId, mGainmapMetadataItemId;
432 ColorAspects mColorAspects;
433 bool mColorAspectsValid;
434 Vector<uint8_t> mBitsPerChannel;
435
436 // Update the audio track's drift information.
437 void updateDriftTime(const sp<MetaData>& meta);
438
439 void dumpTimeStamps();
440
441 int64_t getStartTimeOffsetTimeUs() const;
442 int32_t getStartTimeOffsetScaledTime() const;
443
444 static void *ThreadWrapper(void *me);
445 status_t threadEntry();
446
447 const uint8_t *parseParamSet(
448 const uint8_t *data, size_t length, int type, size_t *paramSetLen);
449
450 status_t copyCodecSpecificData(const uint8_t *data, size_t size, size_t minLength = 0);
451
452 status_t makeAVCCodecSpecificData(const uint8_t *data, size_t size);
453 status_t copyAVCCodecSpecificData(const uint8_t *data, size_t size);
454 status_t parseAVCCodecSpecificData(const uint8_t *data, size_t size);
455
456 status_t makeHEVCCodecSpecificData(const uint8_t *data, size_t size);
457 status_t copyHEVCCodecSpecificData(const uint8_t *data, size_t size);
458 status_t parseHEVCCodecSpecificData(
459 const uint8_t *data, size_t size, HevcParameterSets ¶mSets);
460
461 status_t getDolbyVisionProfile();
462
463 // Track authoring progress status
464 void trackProgressStatus(int64_t timeUs, status_t err = OK);
465 void initTrackingProgressStatus(MetaData *params);
466
467 void getCodecSpecificDataFromInputFormatIfPossible();
468
469 // Determine the track time scale
470 // If it is an audio track, try to use the sampling rate as
471 // the time scale; however, if user chooses the overwrite
472 // value, the user-supplied time scale will be used.
473 void setTimeScale();
474
475 // Simple validation on the codec specific data
476 status_t checkCodecSpecificData() const;
477
478 void updateTrackSizeEstimate();
479 void addOneStscTableEntry(size_t chunkId, size_t sampleId);
480 void addOneStssTableEntry(size_t sampleId);
481 void addOneSttsTableEntry(size_t sampleCount, int32_t delta /* media time scale based */);
482 void addOneCttsTableEntry(size_t sampleCount, int32_t sampleOffset);
483 void addOneElstTableEntry(uint32_t segmentDuration, int32_t mediaTime,
484 int16_t mediaRate, int16_t mediaRateFraction);
485
486 bool isTrackMalFormed();
487 void sendTrackSummary(bool hasMultipleTracks);
488
489 // Write the boxes
490 void writeCo64Box();
491 void writeStscBox();
492 void writeStszBox();
493 void writeStssBox();
494 void writeSttsBox();
495 void writeCttsBox();
496 void writeD263Box();
497 void writePaspBox();
498 void writeAvccBox();
499 void writeHvccBox();
500 void writeAv1cBox();
501 void writeApvcBox();
502 void writeDoviConfigBox();
503 void writeUrlBox();
504 void writeDrefBox();
505 void writeDinfBox();
506 void writeDamrBox();
507 void writeMdhdBox(uint32_t now);
508 void writeSmhdBox();
509 void writeVmhdBox();
510 void writeNmhdBox();
511 void writeHdlrBox();
512 void writeTkhdBox(uint32_t now);
513 void writeColrBox();
514 void writeMdcvAndClliBoxes();
515 void writeMp4aEsdsBox();
516 void writeMp4vEsdsBox();
517 void writeAudioFourCCBox();
518 void writeVideoFourCCBox();
519 void writeMetadataFourCCBox();
520 void writeStblBox();
521 void writeEdtsBox();
522
523 Track(const Track &);
524 Track &operator=(const Track &);
525 };
526
MPEG4Writer(int fd)527 MPEG4Writer::MPEG4Writer(int fd) {
528 initInternal(dup(fd), true /*isFirstSession*/);
529 }
530
~MPEG4Writer()531 MPEG4Writer::~MPEG4Writer() {
532 reset();
533
534 while (!mTracks.empty()) {
535 List<Track *>::iterator it = mTracks.begin();
536 delete *it;
537 (*it) = NULL;
538 mTracks.erase(it);
539 }
540 mTracks.clear();
541
542 if (mNextFd != -1) {
543 close(mNextFd);
544 }
545 }
546
initInternal(int fd,bool isFirstSession)547 void MPEG4Writer::initInternal(int fd, bool isFirstSession) {
548 ALOGV("initInternal");
549 mFd = fd;
550 mNextFd = -1;
551 mInitCheck = mFd < 0? NO_INIT: OK;
552
553 mInterleaveDurationUs = 1000000;
554
555 mStartTimestampUs = -1LL;
556 mStartTimeOffsetMs = -1;
557 mStartTimeOffsetBFramesUs = 0;
558 mPaused = false;
559 mStarted = false;
560 mWriterThreadStarted = false;
561 mSendNotify = false;
562 mWriteSeekErr = false;
563 mFallocateErr = false;
564 // Reset following variables for all the sessions and they will be
565 // initialized in start(MetaData *param).
566 mIsRealTimeRecording = true;
567 mIsBackgroundMode = false;
568 mUse4ByteNalLength = true;
569 mOffset = 0;
570 mMaxOffsetAppend = 0;
571 mPreAllocateFileEndOffset = 0;
572 mMdatOffset = 0;
573 mMdatEndOffset = 0;
574 mInMemoryCache = NULL;
575 mInMemoryCacheOffset = 0;
576 mInMemoryCacheSize = 0;
577 mWriteBoxToMemory = false;
578 mFreeBoxOffset = 0;
579 mStreamableFile = false;
580 mTimeScale = -1;
581 mHasFileLevelMeta = false;
582 mIsAvif = false;
583 mHasGainmap = false;
584 mFileLevelMetaDataSize = 0;
585 mPrimaryItemId = 0;
586 mAssociationEntryCount = 0;
587 mNumGrids = 0;
588 mNextItemId = kItemIdBase;
589 mHasRefs = false;
590 mResetStatus = OK;
591 mPreAllocFirstTime = true;
592 mPrevAllTracksTotalMetaDataSizeEstimate = 0;
593 mIsFirstChunk = false;
594 mDone = false;
595 mThread = 0;
596 mDriftTimeUs = 0;
597 mHasDolbyVision = false;
598
599 // Following variables only need to be set for the first recording session.
600 // And they will stay the same for all the recording sessions.
601 if (isFirstSession) {
602 mMoovExtraSize = 0;
603 mHasMoovBox = false;
604 mMetaKeys = new AMessage();
605 addDeviceMeta();
606 mLatitudex10000 = 0;
607 mLongitudex10000 = 0;
608 mAreGeoTagsAvailable = false;
609 mSwitchPending = false;
610 mIsFileSizeLimitExplicitlyRequested = false;
611 }
612
613 // Verify mFd is seekable
614 off64_t off = lseek64(mFd, 0, SEEK_SET);
615 if (off < 0) {
616 ALOGE("cannot seek mFd: %s (%d) %lld", strerror(errno), errno, (long long)mFd);
617 release();
618 }
619
620 if (fallocate64(mFd, FALLOC_FL_KEEP_SIZE, 0, 1) == 0) {
621 ALOGD("PreAllocation enabled");
622 mPreAllocationEnabled = true;
623 } else {
624 ALOGD("PreAllocation disabled. fallocate : %s, %d", strerror(errno), errno);
625 mPreAllocationEnabled = false;
626 }
627
628 for (List<Track *>::iterator it = mTracks.begin();
629 it != mTracks.end(); ++it) {
630 (*it)->resetInternal();
631 }
632 }
633
dump(int fd,const Vector<String16> & args)634 status_t MPEG4Writer::dump(
635 int fd, const Vector<String16>& args) {
636 const size_t SIZE = 256;
637 char buffer[SIZE];
638 String8 result;
639 snprintf(buffer, SIZE, " MPEG4Writer %p\n", this);
640 result.append(buffer);
641 snprintf(buffer, SIZE, " mStarted: %s\n", mStarted? "true": "false");
642 result.append(buffer);
643 ::write(fd, result.c_str(), result.size());
644 for (List<Track *>::iterator it = mTracks.begin();
645 it != mTracks.end(); ++it) {
646 (*it)->dump(fd, args);
647 }
648 return OK;
649 }
650
dump(int fd,const Vector<String16> &) const651 status_t MPEG4Writer::Track::dump(
652 int fd, const Vector<String16>& /* args */) const {
653 const size_t SIZE = 256;
654 char buffer[SIZE];
655 String8 result;
656 snprintf(buffer, SIZE, " %s track\n", getTrackType());
657 result.append(buffer);
658 snprintf(buffer, SIZE, " reached EOS: %s\n",
659 mReachedEOS? "true": "false");
660 result.append(buffer);
661 snprintf(buffer, SIZE, " frames encoded : %d\n", mStszTableEntries->count());
662 result.append(buffer);
663 snprintf(buffer, SIZE, " duration encoded : %" PRId64 " us\n", mTrackDurationUs);
664 result.append(buffer);
665 ::write(fd, result.c_str(), result.size());
666 return OK;
667 }
668
getDoviFourCC() const669 const char *MPEG4Writer::Track::getDoviFourCC() const {
670 if (mDoviProfile == DolbyVisionProfileDvheStn) {
671 return "dvh1";
672 } else if (mDoviProfile == DolbyVisionProfileDvheSt) {
673 return "hvc1";
674 } else if (mDoviProfile == DolbyVisionProfileDvavSe) {
675 return "avc1";
676 }
677 return nullptr;
678 }
679
680 // static
getFourCCForMime(const char * mime)681 const char *MPEG4Writer::Track::getFourCCForMime(const char *mime) {
682 if (mime == NULL) {
683 return NULL;
684 }
685 if (!strncasecmp(mime, "audio/", 6)) {
686 if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_AMR_NB, mime)) {
687 return "samr";
688 } else if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_AMR_WB, mime)) {
689 return "sawb";
690 } else if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_AAC, mime)) {
691 return "mp4a";
692 }
693 } else if (!strncasecmp(mime, "video/", 6)) {
694 if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_MPEG4, mime)) {
695 return "mp4v";
696 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_H263, mime)) {
697 return "s263";
698 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime)) {
699 return "avc1";
700 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_HEVC, mime)) {
701 return "hvc1";
702 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_AV1, mime)) {
703 return "av01";
704 } else if (editing_flags::muxer_mp4_enable_apv() &&
705 !strcasecmp(MEDIA_MIMETYPE_VIDEO_APV, mime)) {
706 return "apv1";
707 }
708 } else if (!strncasecmp(mime, "application/", 12)) {
709 return "mett";
710 } else if (!strcasecmp(MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC, mime)) {
711 return "heic";
712 } else if (!strcasecmp(MEDIA_MIMETYPE_IMAGE_AVIF, mime)) {
713 return "avif";
714 } else {
715 ALOGE("Track (%s) other than video/audio/metadata is not supported", mime);
716 }
717 return NULL;
718 }
719
addSource(const sp<MediaSource> & source)720 status_t MPEG4Writer::addSource(const sp<MediaSource> &source) {
721 Mutex::Autolock l(mLock);
722 if (mStarted) {
723 ALOGE("Attempt to add source AFTER recording is started");
724 return UNKNOWN_ERROR;
725 }
726
727 CHECK(source.get() != NULL);
728
729 const char *mime = NULL;
730 sp<MetaData> meta = source->getFormat();
731 meta->findCString(kKeyMIMEType, &mime);
732
733
734 // Background mode for media transcoding. If either audio or video track signal this is in
735 // background mode, we will set all the threads to run in background priority.
736 int32_t isBackgroundMode;
737 if (meta && meta->findInt32(kKeyBackgroundMode, &isBackgroundMode)) {
738 mIsBackgroundMode |= isBackgroundMode;
739 }
740
741 if (flags_camera::camera_heif_gainmap()) {
742 int32_t gainmap = 0;
743 if (meta && meta->findInt32(kKeyGainmap, &gainmap)) {
744 mHasGainmap |= gainmap;
745 }
746 }
747
748 if (!strcmp(mime, MEDIA_MIMETYPE_VIDEO_DOLBY_VISION)) {
749 // For MEDIA_MIMETYPE_VIDEO_DOLBY_VISION,
750 // getFourCCForMime() requires profile information
751 // to decide the final FourCC codes.
752 // So we let the creation of the new track now and
753 // assign FourCC codes later using getDoviFourCC()
754 ALOGV("Add source mime '%s'", mime);
755 mHasDolbyVision = true;
756 } else if (Track::getFourCCForMime(mime) == NULL) {
757 ALOGE("Unsupported mime '%s'", mime);
758 return ERROR_UNSUPPORTED;
759 }
760
761 // This is a metadata track or the first track of either audio or video
762 // Go ahead to add the track.
763 Track *track = new Track(this, source, 1 + mTracks.size());
764 mTracks.push_back(track);
765
766 mHasMoovBox |= !track->isHeif();
767 mHasFileLevelMeta |= track->isHeif();
768 mIsAvif |= track->isAvif();
769
770 return OK;
771 }
772
startTracks(MetaData * params)773 status_t MPEG4Writer::startTracks(MetaData *params) {
774 if (mTracks.empty()) {
775 ALOGE("No source added");
776 return INVALID_OPERATION;
777 }
778
779 for (List<Track *>::iterator it = mTracks.begin();
780 it != mTracks.end(); ++it) {
781 status_t err = (*it)->start(params);
782
783 if (err != OK) {
784 for (List<Track *>::iterator it2 = mTracks.begin();
785 it2 != it; ++it2) {
786 (*it2)->stop();
787 }
788
789 return err;
790 }
791 }
792 return OK;
793 }
794
addDeviceMeta()795 void MPEG4Writer::addDeviceMeta() {
796 // add device info and estimate space in 'moov'
797 char val[PROPERTY_VALUE_MAX];
798 size_t n;
799 // meta size is estimated by adding up the following:
800 // - meta header structures, which occur only once (total 66 bytes)
801 // - size for each key, which consists of a fixed header (32 bytes),
802 // plus key length and data length.
803 mMoovExtraSize += 66;
804 if (property_get("ro.build.version.release", val, NULL)
805 && (n = strlen(val)) > 0) {
806 mMetaKeys->setString(kMetaKey_Version, val, n + 1);
807 mMoovExtraSize += sizeof(kMetaKey_Version) + n + 32;
808 }
809
810 if (property_get_bool("media.recorder.show_manufacturer_and_model", false)) {
811 if (property_get("ro.product.manufacturer", val, NULL)
812 && (n = strlen(val)) > 0) {
813 mMetaKeys->setString(kMetaKey_Manufacturer, val, n + 1);
814 mMoovExtraSize += sizeof(kMetaKey_Manufacturer) + n + 32;
815 }
816 if (property_get("ro.product.model", val, NULL)
817 && (n = strlen(val)) > 0) {
818 mMetaKeys->setString(kMetaKey_Model, val, n + 1);
819 mMoovExtraSize += sizeof(kMetaKey_Model) + n + 32;
820 }
821 }
822 #ifdef SHOW_MODEL_BUILD
823 if (property_get("ro.build.display.id", val, NULL)
824 && (n = strlen(val)) > 0) {
825 mMetaKeys->setString(kMetaKey_Build, val, n + 1);
826 mMoovExtraSize += sizeof(kMetaKey_Build) + n + 32;
827 }
828 #endif
829 }
830
estimateFileLevelMetaSize(MetaData * params)831 int64_t MPEG4Writer::estimateFileLevelMetaSize(MetaData *params) {
832 int32_t rotation;
833 if (!params || !params->findInt32(kKeyRotation, &rotation)) {
834 rotation = 0;
835 }
836
837 // base meta size
838 int64_t metaSize = 12 // meta fullbox header
839 + 33 // hdlr box
840 + 14 // pitm box
841 + 16 // iloc box (fixed size portion)
842 + 14 // iinf box (fixed size portion)
843 + 32 // iprp box (fixed size protion)
844 + 8 // idat box (when empty)
845 + 12 // iref box (when empty)
846 ;
847
848 if (flags_camera::camera_heif_gainmap()) {
849 metaSize += 36; // grpl box (when empty)
850 }
851
852 for (List<Track *>::iterator it = mTracks.begin();
853 it != mTracks.end(); ++it) {
854 if ((*it)->isHeif()) {
855 metaSize += (*it)->getMetaSizeIncrease(rotation, mTracks.size());
856 }
857 }
858
859 ALOGV("estimated meta size: %lld", (long long) metaSize);
860
861 // Need at least 8-byte padding at the end, otherwise the left-over
862 // freebox may become malformed
863 return metaSize + 8;
864 }
865
estimateMoovBoxSize(int32_t bitRate)866 int64_t MPEG4Writer::estimateMoovBoxSize(int32_t bitRate) {
867 // This implementation is highly experimental/heurisitic.
868 //
869 // Statistical analysis shows that metadata usually accounts
870 // for a small portion of the total file size, usually < 0.6%.
871
872 // The default MIN_MOOV_BOX_SIZE is set to 0.6% x 1MB / 2,
873 // where 1MB is the common file size limit for MMS application.
874 // The default MAX _MOOV_BOX_SIZE value is based on about 3
875 // minute video recording with a bit rate about 3 Mbps, because
876 // statistics show that most captured videos are less than 3 minutes.
877
878 // If the estimation is wrong, we will pay the price of wasting
879 // some reserved space. This should not happen so often statistically.
880 static const int64_t MIN_MOOV_BOX_SIZE = 3 * 1024; // 3 KibiBytes
881 static const int64_t MAX_MOOV_BOX_SIZE = (180 * 3000000 * 6LL / 8000); // 395.5 KibiBytes
882 int64_t size = MIN_MOOV_BOX_SIZE;
883
884 // Max file size limit is set
885 if (mMaxFileSizeLimitBytes != 0 && mIsFileSizeLimitExplicitlyRequested) {
886 size = mMaxFileSizeLimitBytes / 1000 * 6;
887 }
888
889 // Max file duration limit is set
890 if (mMaxFileDurationLimitUs != 0) {
891 if (bitRate > 0) {
892 int64_t size2 =
893 ((mMaxFileDurationLimitUs / 1000) * bitRate * 6) / 8000000;
894 if (mMaxFileSizeLimitBytes != 0 && mIsFileSizeLimitExplicitlyRequested) {
895 // When both file size and duration limits are set,
896 // we use the smaller limit of the two.
897 if (size > size2) {
898 size = size2;
899 }
900 } else {
901 // Only max file duration limit is set
902 size = size2;
903 }
904 }
905 }
906
907 if (size < MIN_MOOV_BOX_SIZE) {
908 size = MIN_MOOV_BOX_SIZE;
909 }
910
911 // Any long duration recording will be probably end up with
912 // non-streamable mp4 file.
913 if (size > MAX_MOOV_BOX_SIZE) {
914 size = MAX_MOOV_BOX_SIZE;
915 }
916
917 // Account for the extra stuff (Geo, meta keys, etc.)
918 size += mMoovExtraSize;
919
920 ALOGI("limits: %" PRId64 "/%" PRId64 " bytes/us, bit rate: %d bps and the"
921 " estimated moov size %" PRId64 " bytes",
922 mMaxFileSizeLimitBytes, mMaxFileDurationLimitUs, bitRate, size);
923
924 return size;
925 }
926
validateAllTracksId(bool akKey4BitTrackIds)927 status_t MPEG4Writer::validateAllTracksId(bool akKey4BitTrackIds) {
928 for (List<Track *>::iterator it = mTracks.begin(); it != mTracks.end(); ++it) {
929 if (!(*it)->getTrackId().isValid(akKey4BitTrackIds)) {
930 return BAD_VALUE;
931 }
932 }
933 return OK;
934 }
935
start(MetaData * param)936 status_t MPEG4Writer::start(MetaData *param) {
937 if (mInitCheck != OK) {
938 return UNKNOWN_ERROR;
939 }
940 mStartMeta = param;
941
942 /*
943 * Check mMaxFileSizeLimitBytes at the beginning since mMaxFileSizeLimitBytes may be implicitly
944 * changed later as per filesizebits of filesystem even if user does not set it explicitly.
945 */
946 if (mMaxFileSizeLimitBytes != 0) {
947 mIsFileSizeLimitExplicitlyRequested = true;
948 }
949
950 /* mMaxFileSizeLimitBytes has to be set everytime fd is switched, hence the following code is
951 * appropriate in start() method.
952 */
953 int32_t fileSizeBits = fpathconf(mFd, _PC_FILESIZEBITS);
954 ALOGD("fpathconf _PC_FILESIZEBITS:%" PRId32, fileSizeBits);
955 fileSizeBits = std::min(fileSizeBits, 52 /* cap it below 4 peta bytes */);
956 int64_t maxFileSizeBytes = ((int64_t)1 << fileSizeBits) - 1;
957 if (mMaxFileSizeLimitBytes > maxFileSizeBytes) {
958 mMaxFileSizeLimitBytes = maxFileSizeBytes;
959 ALOGD("File size limit (%" PRId64 " bytes) too big. It is changed to %" PRId64 " bytes",
960 mMaxFileSizeLimitBytes, maxFileSizeBytes);
961 } else if (mMaxFileSizeLimitBytes == 0) {
962 mMaxFileSizeLimitBytes = maxFileSizeBytes;
963 ALOGD("File size limit set to %" PRId64 " bytes implicitly", maxFileSizeBytes);
964 }
965
966 int32_t use2ByteNalLength;
967 if (param &&
968 param->findInt32(kKey2ByteNalLength, &use2ByteNalLength) &&
969 use2ByteNalLength) {
970 mUse4ByteNalLength = false;
971 }
972
973 int32_t isRealTimeRecording;
974 if (param && param->findInt32(kKeyRealTimeRecording, &isRealTimeRecording)) {
975 mIsRealTimeRecording = isRealTimeRecording;
976 }
977
978 mStartTimestampUs = -1;
979
980 if (mStarted) {
981 if (mPaused) {
982 mPaused = false;
983 return startTracks(param);
984 }
985 return OK;
986 }
987
988 if (!param ||
989 !param->findInt32(kKeyTimeScale, &mTimeScale)) {
990 // Increased by a factor of 10 to improve precision of segment duration in edit list entry.
991 mTimeScale = 10000;
992 }
993 CHECK_GT(mTimeScale, 0);
994 ALOGV("movie time scale: %d", mTimeScale);
995
996 /*
997 * When the requested file size limit is small, the priority
998 * is to meet the file size limit requirement, rather than
999 * to make the file streamable. mStreamableFile does not tell
1000 * whether the actual recorded file is streamable or not.
1001 */
1002 mStreamableFile =
1003 (mMaxFileSizeLimitBytes != 0 &&
1004 mMaxFileSizeLimitBytes >= kMinStreamableFileSizeInBytes);
1005
1006 /*
1007 * mWriteBoxToMemory is true if the amount of data in a file-level meta or
1008 * moov box is smaller than the reserved free space at the beginning of a
1009 * file, AND when the content of the box is constructed. Note that video/
1010 * audio frame data is always written to the file but not in the memory.
1011 *
1012 * Before stop()/reset() is called, mWriteBoxToMemory is always
1013 * false. When reset() is called at the end of a recording session,
1014 * file-level meta and/or moov box needs to be constructed.
1015 *
1016 * 1) Right before the box is constructed, mWriteBoxToMemory to set to
1017 * mStreamableFile so that if the file is intended to be streamable, it
1018 * is set to true; otherwise, it is set to false. When the value is set
1019 * to false, all the content of that box is written immediately to
1020 * the end of the file. When the value is set to true, all the
1021 * content of that box is written to an in-memory cache,
1022 * mInMemoryCache, util the following condition happens. Note
1023 * that the size of the in-memory cache is the same as the
1024 * reserved free space at the beginning of the file.
1025 *
1026 * 2) While the data of the box is written to an in-memory
1027 * cache, the data size is checked against the reserved space.
1028 * If the data size surpasses the reserved space, subsequent box data
1029 * could no longer be hold in the in-memory cache. This also
1030 * indicates that the reserved space was too small. At this point,
1031 * _all_ subsequent box data must be written to the end of the file.
1032 * mWriteBoxToMemory must be set to false to direct the write
1033 * to the file.
1034 *
1035 * 3) If the data size in the box is smaller than the reserved
1036 * space after the box is completely constructed, the in-memory
1037 * cache copy of the box is written to the reserved free space.
1038 * mWriteBoxToMemory is always set to false after all boxes that
1039 * using the in-memory cache have been constructed.
1040 */
1041 mWriteBoxToMemory = false;
1042 mInMemoryCache = NULL;
1043 mInMemoryCacheOffset = 0;
1044
1045 status_t err = OK;
1046 int32_t is4bitTrackId = false;
1047 if (param && param->findInt32(kKey4BitTrackIds, &is4bitTrackId) && is4bitTrackId) {
1048 err = validateAllTracksId(true);
1049 } else {
1050 err = validateAllTracksId(false);
1051 }
1052 if (err != OK) {
1053 return err;
1054 }
1055
1056 ALOGV("muxer starting: mHasMoovBox %d, mHasFileLevelMeta %d, mIsAvif %d",
1057 mHasMoovBox, mHasFileLevelMeta, mIsAvif);
1058
1059 err = startWriterThread();
1060 if (err != OK) {
1061 return err;
1062 }
1063
1064 err = setupAndStartLooper();
1065 if (err != OK) {
1066 return err;
1067 }
1068
1069 writeFtypBox(param);
1070
1071 mFreeBoxOffset = mOffset;
1072
1073 if (mInMemoryCacheSize == 0) {
1074 int32_t bitRate = -1;
1075 if (mHasFileLevelMeta) {
1076 mFileLevelMetaDataSize = estimateFileLevelMetaSize(param);
1077 mInMemoryCacheSize += mFileLevelMetaDataSize;
1078 }
1079 if (mHasMoovBox) {
1080 if (param) {
1081 param->findInt32(kKeyBitRate, &bitRate);
1082 }
1083 mInMemoryCacheSize += estimateMoovBoxSize(bitRate);
1084 }
1085 }
1086 if (mStreamableFile) {
1087 // Reserve a 'free' box only for streamable file
1088 seekOrPostError(mFd, mFreeBoxOffset, SEEK_SET);
1089 writeInt32(mInMemoryCacheSize);
1090 write("free", 4);
1091 if (mInMemoryCacheSize >= 8) {
1092 off64_t bufSize = mInMemoryCacheSize - 8;
1093 char* zeroBuffer = new (std::nothrow) char[bufSize];
1094 if (zeroBuffer) {
1095 std::fill_n(zeroBuffer, bufSize, '0');
1096 writeOrPostError(mFd, zeroBuffer, bufSize);
1097 delete [] zeroBuffer;
1098 } else {
1099 ALOGW("freebox in file isn't initialized to 0");
1100 }
1101 } else {
1102 ALOGW("freebox size is less than 8:%" PRId64, mInMemoryCacheSize);
1103 }
1104 mMdatOffset = mFreeBoxOffset + mInMemoryCacheSize;
1105 } else {
1106 mMdatOffset = mOffset;
1107 }
1108
1109 mOffset = mMdatOffset;
1110 seekOrPostError(mFd, mMdatOffset, SEEK_SET);
1111 write("\x00\x00\x00\x01mdat????????", 16);
1112
1113 /* Confirm whether the writing of the initial file atoms, ftyp and free,
1114 * are written to the file properly by posting kWhatNoIOErrorSoFar to the
1115 * MP4WtrCtrlHlpLooper that's handling write and seek errors also. If there
1116 * was kWhatIOError, the following two scenarios should be handled.
1117 * 1) If kWhatIOError was delivered and processed, MP4WtrCtrlHlpLooper
1118 * would have stopped all threads gracefully already and posting
1119 * kWhatNoIOErrorSoFar would fail.
1120 * 2) If kWhatIOError wasn't delivered or getting processed,
1121 * kWhatNoIOErrorSoFar should get posted successfully. Wait for
1122 * response from MP4WtrCtrlHlpLooper.
1123 */
1124 sp<AMessage> msg = new AMessage(kWhatNoIOErrorSoFar, mReflector);
1125 sp<AMessage> response;
1126 err = msg->postAndAwaitResponse(&response);
1127 if (err != OK || !response->findInt32("err", &err) || err != OK) {
1128 return ERROR_IO;
1129 }
1130
1131 err = startTracks(param);
1132 if (err != OK) {
1133 return err;
1134 }
1135
1136 mStarted = true;
1137 return OK;
1138 }
1139
stop()1140 status_t MPEG4Writer::stop() {
1141 // If reset was in progress, wait for it to complete.
1142 return reset(true, true);
1143 }
1144
pause()1145 status_t MPEG4Writer::pause() {
1146 ALOGW("MPEG4Writer: pause is not supported");
1147 return ERROR_UNSUPPORTED;
1148 }
1149
stopWriterThread()1150 status_t MPEG4Writer::stopWriterThread() {
1151 ALOGV("Stopping writer thread");
1152 if (!mWriterThreadStarted) {
1153 ALOGD("Writer thread not started");
1154 return OK;
1155 }
1156 {
1157 Mutex::Autolock autolock(mLock);
1158 mDone = true;
1159 mChunkReadyCondition.signal();
1160 }
1161
1162 void *dummy;
1163 status_t err = OK;
1164 int retVal = pthread_join(mThread, &dummy);
1165 if (retVal == 0) {
1166 err = static_cast<status_t>(reinterpret_cast<uintptr_t>(dummy));
1167 ALOGD("WriterThread stopped. Status:%d", err);
1168 } else {
1169 ALOGE("stopWriterThread pthread_join status:%d", retVal);
1170 err = UNKNOWN_ERROR;
1171 }
1172 mWriterThreadStarted = false;
1173 return err;
1174 }
1175
1176 /*
1177 * MP4 file standard defines a composition matrix:
1178 * | a b u |
1179 * | c d v |
1180 * | x y w |
1181 *
1182 * the element in the matrix is stored in the following
1183 * order: {a, b, u, c, d, v, x, y, w},
1184 * where a, b, c, d, x, and y is in 16.16 format, while
1185 * u, v and w is in 2.30 format.
1186 */
writeCompositionMatrix(int degrees)1187 void MPEG4Writer::writeCompositionMatrix(int degrees) {
1188 ALOGV("writeCompositionMatrix");
1189 uint32_t a = 0x00010000;
1190 uint32_t b = 0;
1191 uint32_t c = 0;
1192 uint32_t d = 0x00010000;
1193 switch (degrees) {
1194 case 0:
1195 break;
1196 case 90:
1197 a = 0;
1198 b = 0x00010000;
1199 c = 0xFFFF0000;
1200 d = 0;
1201 break;
1202 case 180:
1203 a = 0xFFFF0000;
1204 d = 0xFFFF0000;
1205 break;
1206 case 270:
1207 a = 0;
1208 b = 0xFFFF0000;
1209 c = 0x00010000;
1210 d = 0;
1211 break;
1212 default:
1213 CHECK(!"Should never reach this unknown rotation");
1214 break;
1215 }
1216
1217 writeInt32(a); // a
1218 writeInt32(b); // b
1219 writeInt32(0); // u
1220 writeInt32(c); // c
1221 writeInt32(d); // d
1222 writeInt32(0); // v
1223 writeInt32(0); // x
1224 writeInt32(0); // y
1225 writeInt32(0x40000000); // w
1226 }
1227
printWriteDurations()1228 void MPEG4Writer::printWriteDurations() {
1229 if (mWriteDurationPQ.empty()) {
1230 return;
1231 }
1232 std::string writeDurationsString =
1233 "Top " + std::to_string(mWriteDurationPQ.size()) + " write durations(microseconds):";
1234 uint8_t i = 0;
1235 while (!mWriteDurationPQ.empty()) {
1236 writeDurationsString +=
1237 " #" + std::to_string(++i) + ":" + std::to_string(mWriteDurationPQ.top().count());
1238 mWriteDurationPQ.pop();
1239 }
1240 ALOGD("%s", writeDurationsString.c_str());
1241 }
1242
release()1243 status_t MPEG4Writer::release() {
1244 ALOGD("release()");
1245 status_t err = OK;
1246 if (!truncatePreAllocation()) {
1247 if (err == OK) { err = ERROR_IO; }
1248 }
1249
1250 // TODO(b/174770856) remove this measurement (and perhaps the fsync)
1251 nsecs_t sync_started = systemTime(SYSTEM_TIME_REALTIME);
1252 if (fsync(mFd) != 0) {
1253 ALOGW("(ignored)fsync err:%s(%d)", std::strerror(errno), errno);
1254 // Don't bubble up fsync error, b/157291505.
1255 // if (err == OK) { err = ERROR_IO; }
1256 }
1257 nsecs_t sync_finished = systemTime(SYSTEM_TIME_REALTIME);
1258 nsecs_t sync_elapsed_ns = sync_finished - sync_started;
1259 int64_t filesize = -1;
1260 struct stat statbuf;
1261 if (fstat(mFd, &statbuf) == 0) {
1262 filesize = statbuf.st_size;
1263 }
1264 ALOGD("final fsync() takes %" PRId64 " ms, file size %" PRId64,
1265 sync_elapsed_ns / 1000000, (int64_t) filesize);
1266
1267 if (close(mFd) != 0) {
1268 ALOGE("close err:%s(%d)", std::strerror(errno), errno);
1269 if (err == OK) { err = ERROR_IO; }
1270 }
1271 mFd = -1;
1272 if (mNextFd != -1) {
1273 if (close(mNextFd) != 0) {
1274 ALOGE("close(mNextFd) error:%s(%d)", std::strerror(errno), errno);
1275 }
1276 if (err == OK) { err = ERROR_IO; }
1277 mNextFd = -1;
1278 }
1279 stopAndReleaseLooper();
1280 mInitCheck = NO_INIT;
1281 mStarted = false;
1282 free(mInMemoryCache);
1283 mInMemoryCache = NULL;
1284
1285 printWriteDurations();
1286
1287 return err;
1288 }
1289
finishCurrentSession()1290 status_t MPEG4Writer::finishCurrentSession() {
1291 ALOGV("finishCurrentSession");
1292 /* Don't wait if reset is in progress already, that avoids deadlock
1293 * as finishCurrentSession() is called from control looper thread.
1294 */
1295 return reset(false, false);
1296 }
1297
switchFd()1298 status_t MPEG4Writer::switchFd() {
1299 ALOGV("switchFd");
1300 Mutex::Autolock l(mLock);
1301 if (mSwitchPending) {
1302 return OK;
1303 }
1304
1305 // Wait for the signal only if the new file is not available.
1306 if (mNextFd == -1) {
1307 status_t res = mFdCond.waitRelative(mLock, kFdCondWaitTimeoutNs);
1308 if (res != OK) {
1309 ALOGW("No FileDescriptor for next recording");
1310 return INVALID_OPERATION;
1311 }
1312 }
1313
1314 mSwitchPending = true;
1315 sp<AMessage> msg = new AMessage(kWhatSwitch, mReflector);
1316 status_t err = msg->post();
1317
1318 return err;
1319 }
1320
reset(bool stopSource,bool waitForAnyPreviousCallToComplete)1321 status_t MPEG4Writer::reset(bool stopSource, bool waitForAnyPreviousCallToComplete) {
1322 ALOGD("reset()");
1323 std::unique_lock<std::mutex> lk(mResetMutex, std::defer_lock);
1324 if (waitForAnyPreviousCallToComplete) {
1325 /* stop=>reset from client needs the return value of reset call, hence wait here
1326 * if a reset was in process already.
1327 */
1328 lk.lock();
1329 } else if (!lk.try_lock()) {
1330 /* Internal reset from control looper thread shouldn't wait for any reset in
1331 * process already.
1332 */
1333 return INVALID_OPERATION;
1334 }
1335
1336 if (mResetStatus != OK) {
1337 /* Don't have to proceed if reset has finished with an error before.
1338 * If there was no error before, proceeding reset would be harmless, as the
1339 * the call would return from the mInitCheck condition below.
1340 */
1341 return mResetStatus;
1342 }
1343
1344 if (mInitCheck != OK) {
1345 mResetStatus = OK;
1346 return mResetStatus;
1347 } else {
1348 if (!mWriterThreadStarted ||
1349 !mStarted) {
1350 status_t writerErr = OK;
1351 if (mWriterThreadStarted) {
1352 writerErr = stopWriterThread();
1353 }
1354 status_t retErr = release();
1355 if (writerErr != OK) {
1356 retErr = writerErr;
1357 }
1358 mResetStatus = retErr;
1359 return mResetStatus;
1360 }
1361 }
1362
1363 status_t err = OK;
1364 int64_t maxDurationUs = 0;
1365 int64_t minDurationUs = 0x7fffffffffffffffLL;
1366 int32_t nonImageTrackCount = 0;
1367 for (List<Track *>::iterator it = mTracks.begin();
1368 it != mTracks.end(); ++it) {
1369 status_t trackErr = (*it)->stop(stopSource);
1370 WARN_UNLESS(trackErr == OK, "%s track stopped with an error",
1371 (*it)->getTrackType());
1372 if (err == OK && trackErr != OK) {
1373 err = trackErr;
1374 }
1375
1376 // skip image tracks
1377 if ((*it)->isHeif()) continue;
1378 nonImageTrackCount++;
1379
1380 int64_t durationUs = (*it)->getDurationUs();
1381 if (durationUs > maxDurationUs) {
1382 maxDurationUs = durationUs;
1383 }
1384 if (durationUs < minDurationUs) {
1385 minDurationUs = durationUs;
1386 }
1387 }
1388
1389 if (nonImageTrackCount > 1) {
1390 ALOGD("Duration from tracks range is [%" PRId64 ", %" PRId64 "] us",
1391 minDurationUs, maxDurationUs);
1392 }
1393
1394 status_t writerErr = stopWriterThread();
1395
1396 // Propagating writer error
1397 if (err == OK && writerErr != OK) {
1398 err = writerErr;
1399 }
1400
1401 // Do not write out movie header on error except malformed track.
1402 // TODO: Remove samples of malformed tracks added in mdat.
1403 if (err != OK && err != ERROR_MALFORMED) {
1404 // Ignoring release() return value as there was an "err" already.
1405 release();
1406 mResetStatus = err;
1407 return mResetStatus;
1408 }
1409
1410 // Fix up the size of the 'mdat' chunk.
1411 seekOrPostError(mFd, mMdatOffset + 8, SEEK_SET);
1412 uint64_t size = mOffset - mMdatOffset;
1413 size = hton64(size);
1414 writeOrPostError(mFd, &size, 8);
1415 seekOrPostError(mFd, mOffset, SEEK_SET);
1416 mMdatEndOffset = mOffset;
1417
1418 // Construct file-level meta and moov box now
1419 mInMemoryCacheOffset = 0;
1420 mWriteBoxToMemory = mStreamableFile;
1421 if (mWriteBoxToMemory) {
1422 // There is no need to allocate in-memory cache
1423 // if the file is not streamable.
1424
1425 mInMemoryCache = (uint8_t *) malloc(mInMemoryCacheSize);
1426 CHECK(mInMemoryCache != NULL);
1427 }
1428
1429 if (mHasFileLevelMeta) {
1430 writeFileLevelMetaBox();
1431 if (mWriteBoxToMemory) {
1432 writeCachedBoxToFile("meta");
1433 } else {
1434 ALOGI("The file meta box is written at the end.");
1435 }
1436 }
1437
1438 if (mHasMoovBox) {
1439 writeMoovBox(maxDurationUs);
1440 // mWriteBoxToMemory could be set to false in
1441 // MPEG4Writer::write() method
1442 if (mWriteBoxToMemory) {
1443 writeCachedBoxToFile("moov");
1444 } else {
1445 ALOGI("The mp4 file will not be streamable.");
1446 }
1447 ALOGI("MOOV atom was written to the file");
1448 }
1449 mWriteBoxToMemory = false;
1450
1451 // Free in-memory cache for box writing
1452 if (mInMemoryCache != NULL) {
1453 free(mInMemoryCache);
1454 mInMemoryCache = NULL;
1455 mInMemoryCacheOffset = 0;
1456 }
1457
1458 CHECK(mBoxes.empty());
1459
1460 status_t errRelease = release();
1461 // Prioritize the error that occurred before release().
1462 if (err == OK) {
1463 err = errRelease;
1464 }
1465 mResetStatus = err;
1466 return mResetStatus;
1467 }
1468
1469 /*
1470 * Writes currently cached box into file.
1471 *
1472 * Must be called while mWriteBoxToMemory is true, and will not modify
1473 * mWriteBoxToMemory. After the call, remaining cache size will be
1474 * reduced and buffer offset will be set to the beginning of the cache.
1475 */
writeCachedBoxToFile(const char * type)1476 void MPEG4Writer::writeCachedBoxToFile(const char *type) {
1477 CHECK(mWriteBoxToMemory);
1478
1479 mWriteBoxToMemory = false;
1480 // Content of the box is saved in the cache, and the in-memory
1481 // box needs to be written to the file in a single shot.
1482
1483 CHECK_LE(mInMemoryCacheOffset + 8, mInMemoryCacheSize);
1484
1485 // Cached box
1486 seekOrPostError(mFd, mFreeBoxOffset, SEEK_SET);
1487 mOffset = mFreeBoxOffset;
1488 write(mInMemoryCache, 1, mInMemoryCacheOffset);
1489
1490 // Free box
1491 seekOrPostError(mFd, mOffset, SEEK_SET);
1492 mFreeBoxOffset = mOffset;
1493 writeInt32(mInMemoryCacheSize - mInMemoryCacheOffset);
1494 write("free", 4);
1495
1496 // Rewind buffering to the beginning, and restore mWriteBoxToMemory flag
1497 mInMemoryCacheSize -= mInMemoryCacheOffset;
1498 mInMemoryCacheOffset = 0;
1499 mWriteBoxToMemory = true;
1500
1501 ALOGV("dumped out %s box, estimated size remaining %lld",
1502 type, (long long)mInMemoryCacheSize);
1503 }
1504
getMpeg4Time()1505 uint32_t MPEG4Writer::getMpeg4Time() {
1506 time_t now = time(NULL);
1507 // MP4 file uses time counting seconds since midnight, Jan. 1, 1904
1508 // while time function returns Unix epoch values which starts
1509 // at 1970-01-01. Lets add the number of seconds between them
1510 static const uint32_t delta = (66 * 365 + 17) * (24 * 60 * 60);
1511 if (now < 0 || uint32_t(now) > UINT32_MAX - delta) {
1512 return 0;
1513 }
1514 uint32_t mpeg4Time = uint32_t(now) + delta;
1515 return mpeg4Time;
1516 }
1517
writeMvhdBox(int64_t durationUs)1518 void MPEG4Writer::writeMvhdBox(int64_t durationUs) {
1519 uint32_t now = getMpeg4Time();
1520 beginBox("mvhd");
1521 writeInt32(0); // version=0, flags=0
1522 writeInt32(now); // creation time
1523 writeInt32(now); // modification time
1524 writeInt32(mTimeScale); // mvhd timescale
1525 int32_t duration = (durationUs * mTimeScale + 5E5) / 1E6;
1526 writeInt32(duration);
1527 writeInt32(0x10000); // rate: 1.0
1528 writeInt16(0x100); // volume
1529 writeInt16(0); // reserved
1530 writeInt32(0); // reserved
1531 writeInt32(0); // reserved
1532 writeCompositionMatrix(0); // matrix
1533 writeInt32(0); // predefined
1534 writeInt32(0); // predefined
1535 writeInt32(0); // predefined
1536 writeInt32(0); // predefined
1537 writeInt32(0); // predefined
1538 writeInt32(0); // predefined
1539 writeInt32(mTracks.size() + 1); // nextTrackID
1540 endBox(); // mvhd
1541 }
1542
writeMoovBox(int64_t durationUs)1543 void MPEG4Writer::writeMoovBox(int64_t durationUs) {
1544 beginBox("moov");
1545 writeMvhdBox(durationUs);
1546 if (mAreGeoTagsAvailable) {
1547 writeUdtaBox();
1548 }
1549 writeMoovLevelMetaBox();
1550 // Loop through all the tracks to get the global time offset if there is
1551 // any ctts table appears in a video track.
1552 int64_t minCttsOffsetTimeUs = kMaxCttsOffsetTimeUs;
1553 for (List<Track *>::iterator it = mTracks.begin();
1554 it != mTracks.end(); ++it) {
1555 if (!(*it)->isHeif()) {
1556 minCttsOffsetTimeUs =
1557 std::min(minCttsOffsetTimeUs, (*it)->getMinCttsOffsetTimeUs());
1558 }
1559 }
1560 ALOGI("Adjust the moov start time from %lld us -> %lld us", (long long)mStartTimestampUs,
1561 (long long)(mStartTimestampUs + minCttsOffsetTimeUs - kMaxCttsOffsetTimeUs));
1562 // Adjust movie start time.
1563 mStartTimestampUs += minCttsOffsetTimeUs - kMaxCttsOffsetTimeUs;
1564
1565 // Add mStartTimeOffsetBFramesUs(-ve or zero) to the start offset of tracks.
1566 mStartTimeOffsetBFramesUs = minCttsOffsetTimeUs - kMaxCttsOffsetTimeUs;
1567 ALOGV("mStartTimeOffsetBFramesUs :%" PRId32, mStartTimeOffsetBFramesUs);
1568
1569 for (List<Track *>::iterator it = mTracks.begin();
1570 it != mTracks.end(); ++it) {
1571 if (!(*it)->isHeif()) {
1572 (*it)->writeTrackHeader();
1573 }
1574 }
1575 endBox(); // moov
1576 }
1577
writeFtypBox(MetaData * param)1578 void MPEG4Writer::writeFtypBox(MetaData *param) {
1579 beginBox("ftyp");
1580
1581 int32_t fileType;
1582 if (!param || !param->findInt32(kKeyFileType, &fileType)) {
1583 fileType = OUTPUT_FORMAT_MPEG_4;
1584 }
1585 if (fileType != OUTPUT_FORMAT_MPEG_4 && fileType != OUTPUT_FORMAT_HEIF) {
1586 writeFourcc("3gp4");
1587 writeInt32(0);
1588 writeFourcc("isom");
1589 writeFourcc("3gp4");
1590 } else {
1591 // Only write "heic"/"avif" as major brand if the client specified HEIF/AVIF
1592 // AND we indeed receive some image heic/avif tracks.
1593 if (fileType == OUTPUT_FORMAT_HEIF && mHasFileLevelMeta) {
1594 if (mIsAvif) {
1595 writeFourcc("avif");
1596 } else {
1597 writeFourcc("heic");
1598 }
1599 } else {
1600 writeFourcc("mp42");
1601 }
1602 writeInt32(0);
1603 if (mHasFileLevelMeta) {
1604 if (mIsAvif) {
1605 writeFourcc("mif1");
1606 writeFourcc("miaf");
1607 writeFourcc("avif");
1608 } else {
1609 writeFourcc("mif1");
1610 writeFourcc("heic");
1611 if (flags_camera::camera_heif_gainmap() && mHasGainmap) {
1612 writeFourcc("tmap");
1613 }
1614 }
1615 }
1616 if (mHasMoovBox) {
1617 writeFourcc("isom");
1618 writeFourcc("mp42");
1619 }
1620 // If an AV1 video track is present, write "av01" as one of the
1621 // compatible brands.
1622 for (List<Track *>::iterator it = mTracks.begin(); it != mTracks.end();
1623 ++it) {
1624 if ((*it)->isAv1()) {
1625 writeFourcc("av01");
1626 break;
1627 }
1628 }
1629 // The brand ‘dby1’ should be used in the compatible_brands field to indicate that the file
1630 // is compliant with all Dolby Extensions. For details, refer to
1631 // https://professional.dolby.com/siteassets/content-creation/dolby-vision-for-content-creators/dolby_vision_bitstreams_within_the_iso_base_media_file_format_dec2017.pdf
1632 // Chapter 7, Dolby Vision Files.
1633 if (fileType == OUTPUT_FORMAT_MPEG_4 && mHasDolbyVision) {
1634 writeFourcc("dby1");
1635 }
1636 }
1637
1638 endBox();
1639 }
1640
isTestModeEnabled()1641 static bool isTestModeEnabled() {
1642 #if (PROPERTY_VALUE_MAX < 5)
1643 #error "PROPERTY_VALUE_MAX must be at least 5"
1644 #endif
1645
1646 // Test mode is enabled only if rw.media.record.test system
1647 // property is enabled.
1648 if (property_get_bool("rw.media.record.test", false)) {
1649 return true;
1650 }
1651 return false;
1652 }
1653
sendSessionSummary()1654 void MPEG4Writer::sendSessionSummary() {
1655 // Send session summary only if test mode is enabled
1656 if (!isTestModeEnabled()) {
1657 return;
1658 }
1659
1660 for (List<ChunkInfo>::iterator it = mChunkInfos.begin();
1661 it != mChunkInfos.end(); ++it) {
1662 uint32_t trackNum = (it->mTrack->getTrackId().getId() << 28);
1663 notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
1664 trackNum | MEDIA_RECORDER_TRACK_INTER_CHUNK_TIME_MS,
1665 it->mMaxInterChunkDurUs);
1666 }
1667 }
1668
setInterleaveDuration(uint32_t durationUs)1669 status_t MPEG4Writer::setInterleaveDuration(uint32_t durationUs) {
1670 mInterleaveDurationUs = durationUs;
1671 return OK;
1672 }
1673
lock()1674 void MPEG4Writer::lock() {
1675 mLock.lock();
1676 }
1677
unlock()1678 void MPEG4Writer::unlock() {
1679 mLock.unlock();
1680 }
1681
addSample_l(MediaBuffer * buffer,bool usePrefix,uint32_t tiffHdrOffset,size_t * bytesWritten)1682 off64_t MPEG4Writer::addSample_l(
1683 MediaBuffer *buffer, bool usePrefix,
1684 uint32_t tiffHdrOffset, size_t *bytesWritten) {
1685 off64_t old_offset = mOffset;
1686 int64_t offset;
1687 ALOGV("buffer->range_length:%lld", (long long)buffer->range_length());
1688 if (buffer->meta_data().findInt64(kKeySampleFileOffset, &offset)) {
1689 ALOGV("offset:%lld, old_offset:%lld", (long long)offset, (long long)old_offset);
1690 if (mMaxOffsetAppend > offset) {
1691 // This has already been appended, skip updating mOffset value.
1692 *bytesWritten = buffer->range_length();
1693 return offset;
1694 }
1695 if (old_offset == offset) {
1696 mOffset += buffer->range_length();
1697 } else {
1698 ALOGV("offset and old_offset are not equal! diff:%lld", (long long)offset - old_offset);
1699 mOffset = offset + buffer->range_length();
1700 // mOffset += buffer->range_length() + offset - old_offset;
1701 }
1702 *bytesWritten = buffer->range_length();
1703 ALOGV("mOffset:%lld, mMaxOffsetAppend:%lld, bytesWritten:%lld", (long long)mOffset,
1704 (long long)mMaxOffsetAppend, (long long)*bytesWritten);
1705 mMaxOffsetAppend = std::max(mOffset, mMaxOffsetAppend);
1706 seekOrPostError(mFd, mMaxOffsetAppend, SEEK_SET);
1707 return offset;
1708 }
1709
1710 ALOGV("mOffset:%lld, mMaxOffsetAppend:%lld", (long long)mOffset, (long long)mMaxOffsetAppend);
1711
1712 if (usePrefix) {
1713 addMultipleLengthPrefixedSamples_l(buffer);
1714 } else {
1715 if (tiffHdrOffset > 0) {
1716 tiffHdrOffset = htonl(tiffHdrOffset);
1717 writeOrPostError(mFd, &tiffHdrOffset, 4); // exif_tiff_header_offset field
1718 mOffset += 4;
1719 }
1720
1721 writeOrPostError(mFd, (const uint8_t*)buffer->data() + buffer->range_offset(),
1722 buffer->range_length());
1723
1724 mOffset += buffer->range_length();
1725 }
1726 *bytesWritten = mOffset - old_offset;
1727
1728 ALOGV("mOffset:%lld, old_offset:%lld, bytesWritten:%lld", (long long)mOffset,
1729 (long long)old_offset, (long long)*bytesWritten);
1730
1731 return old_offset;
1732 }
1733
StripStartcode(MediaBuffer * buffer)1734 static void StripStartcode(MediaBuffer *buffer) {
1735 if (buffer->range_length() < 4) {
1736 return;
1737 }
1738
1739 const uint8_t *ptr =
1740 (const uint8_t *)buffer->data() + buffer->range_offset();
1741
1742 if (!memcmp(ptr, "\x00\x00\x00\x01", 4)) {
1743 ALOGV("stripping start code");
1744 buffer->set_range(
1745 buffer->range_offset() + 4, buffer->range_length() - 4);
1746 }
1747 }
1748
addMultipleLengthPrefixedSamples_l(MediaBuffer * buffer)1749 void MPEG4Writer::addMultipleLengthPrefixedSamples_l(MediaBuffer *buffer) {
1750 const uint8_t *dataStart = (const uint8_t *)buffer->data() + buffer->range_offset();
1751 const uint8_t *currentNalStart = dataStart;
1752 const uint8_t *nextNalStart;
1753 const uint8_t *data = dataStart;
1754 size_t nextNalSize;
1755 size_t searchSize = buffer->range_length();
1756
1757 while (getNextNALUnit(&data, &searchSize, &nextNalStart,
1758 &nextNalSize, true) == OK) {
1759 size_t currentNalSize = nextNalStart - currentNalStart - 4 /* strip start-code */;
1760 MediaBuffer *nalBuf = new MediaBuffer((void *)currentNalStart, currentNalSize);
1761 addLengthPrefixedSample_l(nalBuf);
1762 nalBuf->release();
1763
1764 currentNalStart = nextNalStart;
1765 }
1766
1767 size_t currentNalOffset = currentNalStart - dataStart;
1768 buffer->set_range(buffer->range_offset() + currentNalOffset,
1769 buffer->range_length() - currentNalOffset);
1770 addLengthPrefixedSample_l(buffer);
1771 }
1772
addLengthPrefixedSample_l(MediaBuffer * buffer)1773 void MPEG4Writer::addLengthPrefixedSample_l(MediaBuffer *buffer) {
1774 ALOGV("alp:buffer->range_length:%lld", (long long)buffer->range_length());
1775 size_t length = buffer->range_length();
1776 if (mUse4ByteNalLength) {
1777 ALOGV("mUse4ByteNalLength");
1778 uint8_t x[4];
1779 x[0] = length >> 24;
1780 x[1] = (length >> 16) & 0xff;
1781 x[2] = (length >> 8) & 0xff;
1782 x[3] = length & 0xff;
1783 writeOrPostError(mFd, &x, 4);
1784 writeOrPostError(mFd, (const uint8_t*)buffer->data() + buffer->range_offset(), length);
1785 mOffset += length + 4;
1786 } else {
1787 ALOGV("mUse2ByteNalLength");
1788 CHECK_LT(length, 65536u);
1789
1790 uint8_t x[2];
1791 x[0] = length >> 8;
1792 x[1] = length & 0xff;
1793 writeOrPostError(mFd, &x, 2);
1794 writeOrPostError(mFd, (const uint8_t*)buffer->data() + buffer->range_offset(), length);
1795 mOffset += length + 2;
1796 }
1797 }
1798
write(const void * ptr,size_t size,size_t nmemb)1799 size_t MPEG4Writer::write(
1800 const void *ptr, size_t size, size_t nmemb) {
1801
1802 const size_t bytes = size * nmemb;
1803 if (mWriteBoxToMemory) {
1804
1805 off64_t boxSize = 8 + mInMemoryCacheOffset + bytes;
1806 if (boxSize > mInMemoryCacheSize) {
1807 // The reserved free space at the beginning of the file is not big
1808 // enough. Boxes should be written to the end of the file from now
1809 // on, but not to the in-memory cache.
1810
1811 // We write partial box that is in the memory to the file first.
1812 for (List<off64_t>::iterator it = mBoxes.begin();
1813 it != mBoxes.end(); ++it) {
1814 (*it) += mOffset;
1815 }
1816 seekOrPostError(mFd, mOffset, SEEK_SET);
1817 writeOrPostError(mFd, mInMemoryCache, mInMemoryCacheOffset);
1818 writeOrPostError(mFd, ptr, bytes);
1819 mOffset += (bytes + mInMemoryCacheOffset);
1820
1821 // All subsequent boxes will be written to the end of the file.
1822 mWriteBoxToMemory = false;
1823 } else {
1824 memcpy(mInMemoryCache + mInMemoryCacheOffset, ptr, bytes);
1825 mInMemoryCacheOffset += bytes;
1826 }
1827 } else {
1828 writeOrPostError(mFd, ptr, bytes);
1829 mOffset += bytes;
1830 }
1831 return bytes;
1832 }
1833
writeOrPostError(int fd,const void * buf,size_t count)1834 void MPEG4Writer::writeOrPostError(int fd, const void* buf, size_t count) {
1835 if (mWriteSeekErr == true)
1836 return;
1837
1838 auto beforeTP = std::chrono::high_resolution_clock::now();
1839 ssize_t bytesWritten = ::write(fd, buf, count);
1840 auto afterTP = std::chrono::high_resolution_clock::now();
1841 auto writeDuration =
1842 std::chrono::duration_cast<std::chrono::microseconds>(afterTP - beforeTP).count();
1843 mWriteDurationPQ.emplace(writeDuration);
1844 if (mWriteDurationPQ.size() > kWriteDurationsCount) {
1845 mWriteDurationPQ.pop();
1846 }
1847
1848 /* Write as much as possible during stop() execution when there was an error
1849 * (mWriteSeekErr == true) in the previous call to write() or lseek64().
1850 */
1851 if (bytesWritten == count)
1852 return;
1853 mWriteSeekErr = true;
1854 // Note that errno is not changed even when bytesWritten < count.
1855 ALOGE("writeOrPostError bytesWritten:%zd, count:%zu, error:%s(%d)", bytesWritten, count,
1856 std::strerror(errno), errno);
1857
1858 // Can't guarantee that file is usable or write would succeed anymore, hence signal to stop.
1859 sp<AMessage> msg = new AMessage(kWhatIOError, mReflector);
1860 msg->setInt32("err", ERROR_IO);
1861 WARN_UNLESS(msg->post() == OK, "writeOrPostError:error posting ERROR_IO");
1862 }
1863
seekOrPostError(int fd,off64_t offset,int whence)1864 void MPEG4Writer::seekOrPostError(int fd, off64_t offset, int whence) {
1865 if (mWriteSeekErr == true)
1866 return;
1867 off64_t resOffset = lseek64(fd, offset, whence);
1868 /* Allow to seek during stop() execution even when there was an error
1869 * (mWriteSeekErr == true) in the previous call to write() or lseek64().
1870 */
1871 if (resOffset == offset)
1872 return;
1873 mWriteSeekErr = true;
1874 ALOGE("seekOrPostError resOffset:%" PRIu64 ", offset:%" PRIu64 ", error:%s(%d)", resOffset,
1875 offset, std::strerror(errno), errno);
1876
1877 // Can't guarantee that file is usable or seek would succeed anymore, hence signal to stop.
1878 sp<AMessage> msg = new AMessage(kWhatIOError, mReflector);
1879 msg->setInt32("err", ERROR_IO);
1880 WARN_UNLESS(msg->post() == OK, "seekOrPostError:error posting ERROR_IO");
1881 }
1882
beginBox(uint32_t id)1883 void MPEG4Writer::beginBox(uint32_t id) {
1884 ALOGV("beginBox:%" PRIu32, id);
1885
1886 mBoxes.push_back(mWriteBoxToMemory?
1887 mInMemoryCacheOffset: mOffset);
1888
1889 writeInt32(0);
1890 writeInt32(id);
1891 }
1892
beginBox(const char * fourcc)1893 void MPEG4Writer::beginBox(const char *fourcc) {
1894 ALOGV("beginBox:%s", fourcc);
1895 CHECK_EQ(strlen(fourcc), 4u);
1896
1897 mBoxes.push_back(mWriteBoxToMemory?
1898 mInMemoryCacheOffset: mOffset);
1899
1900 writeInt32(0);
1901 writeFourcc(fourcc);
1902 }
1903
endBox()1904 void MPEG4Writer::endBox() {
1905 CHECK(!mBoxes.empty());
1906
1907 off64_t offset = *--mBoxes.end();
1908 mBoxes.erase(--mBoxes.end());
1909
1910 if (mWriteBoxToMemory) {
1911 int32_t x = htonl(mInMemoryCacheOffset - offset);
1912 memcpy(mInMemoryCache + offset, &x, 4);
1913 } else {
1914 seekOrPostError(mFd, offset, SEEK_SET);
1915 writeInt32(mOffset - offset);
1916 ALOGV("box size:%" PRIu64, mOffset - offset);
1917 mOffset -= 4;
1918 seekOrPostError(mFd, mOffset, SEEK_SET);
1919 }
1920 }
1921
writeInt8(int8_t x)1922 void MPEG4Writer::writeInt8(int8_t x) {
1923 write(&x, 1, 1);
1924 }
1925
writeInt16(int16_t x)1926 void MPEG4Writer::writeInt16(int16_t x) {
1927 x = htons(x);
1928 write(&x, 1, 2);
1929 }
1930
writeInt32(int32_t x)1931 void MPEG4Writer::writeInt32(int32_t x) {
1932 x = htonl(x);
1933 write(&x, 1, 4);
1934 }
1935
writeInt64(int64_t x)1936 void MPEG4Writer::writeInt64(int64_t x) {
1937 x = hton64(x);
1938 write(&x, 1, 8);
1939 }
1940
writeCString(const char * s)1941 void MPEG4Writer::writeCString(const char *s) {
1942 size_t n = strlen(s);
1943 write(s, 1, n + 1);
1944 }
1945
writeFourcc(const char * s)1946 void MPEG4Writer::writeFourcc(const char *s) {
1947 CHECK_EQ(strlen(s), 4u);
1948 write(s, 1, 4);
1949 }
1950
1951
1952 // Written in +/-DD.DDDD format
writeLatitude(int degreex10000)1953 void MPEG4Writer::writeLatitude(int degreex10000) {
1954 bool isNegative = (degreex10000 < 0);
1955 char sign = isNegative? '-': '+';
1956
1957 // Handle the whole part
1958 char str[9];
1959 int wholePart = degreex10000 / 10000;
1960 if (wholePart == 0) {
1961 snprintf(str, 5, "%c%.2d.", sign, wholePart);
1962 } else {
1963 snprintf(str, 5, "%+.2d.", wholePart);
1964 }
1965
1966 // Handle the fractional part
1967 int fractionalPart = degreex10000 - (wholePart * 10000);
1968 if (fractionalPart < 0) {
1969 fractionalPart = -fractionalPart;
1970 }
1971 snprintf(&str[4], 5, "%.4d", fractionalPart);
1972
1973 // Do not write the null terminator
1974 write(str, 1, 8);
1975 }
1976
1977 // Written in +/- DDD.DDDD format
writeLongitude(int degreex10000)1978 void MPEG4Writer::writeLongitude(int degreex10000) {
1979 bool isNegative = (degreex10000 < 0);
1980 char sign = isNegative? '-': '+';
1981
1982 // Handle the whole part
1983 char str[10];
1984 int wholePart = degreex10000 / 10000;
1985 if (wholePart == 0) {
1986 snprintf(str, 6, "%c%.3d.", sign, wholePart);
1987 } else {
1988 snprintf(str, 6, "%+.3d.", wholePart);
1989 }
1990
1991 // Handle the fractional part
1992 int fractionalPart = degreex10000 - (wholePart * 10000);
1993 if (fractionalPart < 0) {
1994 fractionalPart = -fractionalPart;
1995 }
1996 snprintf(&str[5], 5, "%.4d", fractionalPart);
1997
1998 // Do not write the null terminator
1999 write(str, 1, 9);
2000 }
2001
2002 /*
2003 * Geodata is stored according to ISO-6709 standard.
2004 * latitudex10000 is latitude in degrees times 10000, and
2005 * longitudex10000 is longitude in degrees times 10000.
2006 * The range for the latitude is in [-90, +90], and
2007 * The range for the longitude is in [-180, +180]
2008 */
setGeoData(int latitudex10000,int longitudex10000)2009 status_t MPEG4Writer::setGeoData(int latitudex10000, int longitudex10000) {
2010 // Is latitude or longitude out of range?
2011 if (latitudex10000 < -900000 || latitudex10000 > 900000 ||
2012 longitudex10000 < -1800000 || longitudex10000 > 1800000) {
2013 return BAD_VALUE;
2014 }
2015
2016 mLatitudex10000 = latitudex10000;
2017 mLongitudex10000 = longitudex10000;
2018 mAreGeoTagsAvailable = true;
2019 mMoovExtraSize += 30;
2020 return OK;
2021 }
2022
setCaptureRate(float captureFps)2023 status_t MPEG4Writer::setCaptureRate(float captureFps) {
2024 if (captureFps <= 0.0f) {
2025 return BAD_VALUE;
2026 }
2027
2028 // Increase moovExtraSize once only irrespective of how many times
2029 // setCaptureRate is called.
2030 bool containsCaptureFps = mMetaKeys->contains(kMetaKey_CaptureFps);
2031 mMetaKeys->setFloat(kMetaKey_CaptureFps, captureFps);
2032 if (!containsCaptureFps) {
2033 mMoovExtraSize += sizeof(kMetaKey_CaptureFps) + 4 + 32;
2034 }
2035
2036 return OK;
2037 }
2038
setTemporalLayerCount(uint32_t layerCount)2039 status_t MPEG4Writer::setTemporalLayerCount(uint32_t layerCount) {
2040 if (layerCount > 9) {
2041 return BAD_VALUE;
2042 }
2043
2044 if (layerCount > 0) {
2045 mMetaKeys->setInt32(kMetaKey_TemporalLayerCount, layerCount);
2046 mMoovExtraSize += sizeof(kMetaKey_TemporalLayerCount) + 4 + 32;
2047 }
2048
2049 return OK;
2050 }
2051
notifyApproachingLimit()2052 void MPEG4Writer::notifyApproachingLimit() {
2053 Mutex::Autolock autolock(mLock);
2054 // Only notify once.
2055 if (mSendNotify) {
2056 return;
2057 }
2058 ALOGW("Recorded file size is approaching limit %" PRId64 "bytes",
2059 mMaxFileSizeLimitBytes);
2060 notify(MEDIA_RECORDER_EVENT_INFO, MEDIA_RECORDER_INFO_MAX_FILESIZE_APPROACHING, 0);
2061 mSendNotify = true;
2062 }
2063
write(const void * data,size_t size)2064 void MPEG4Writer::write(const void *data, size_t size) {
2065 write(data, 1, size);
2066 }
2067
isFileStreamable() const2068 bool MPEG4Writer::isFileStreamable() const {
2069 return mStreamableFile;
2070 }
2071
preAllocate(uint64_t wantSize)2072 bool MPEG4Writer::preAllocate(uint64_t wantSize) {
2073 if (!mPreAllocationEnabled)
2074 return true;
2075
2076 std::lock_guard<std::mutex> l(mFallocMutex);
2077
2078 if (mFallocateErr == true)
2079 return false;
2080
2081 // approxMOOVHeadersSize has to be changed whenever its needed in the future.
2082 uint64_t approxMOOVHeadersSize = 500;
2083 // approxTrackHeadersSize has to be changed whenever its needed in the future.
2084 const uint64_t approxTrackHeadersSize = 800;
2085
2086 uint64_t approxMOOVBoxSize = 0;
2087 if (mPreAllocFirstTime) {
2088 mPreAllocFirstTime = false;
2089 approxMOOVBoxSize = approxMOOVHeadersSize + mFileLevelMetaDataSize + mMoovExtraSize +
2090 (approxTrackHeadersSize * numTracks());
2091 ALOGV("firstTimeAllocation approxMOOVBoxSize:%" PRIu64, approxMOOVBoxSize);
2092 }
2093
2094 uint64_t allTracksTotalMetaDataSizeEstimate = 0;
2095 for (List<Track *>::iterator it = mTracks.begin(); it != mTracks.end(); ++it) {
2096 allTracksTotalMetaDataSizeEstimate += ((*it)->trackMetaDataSize());
2097 }
2098 ALOGV(" allTracksTotalMetaDataSizeEstimate:%" PRIu64, allTracksTotalMetaDataSizeEstimate);
2099
2100 /* MOOVBoxSize will increase whenever a sample gets written to the file. Enough to allocate
2101 * the delta increase for each sample after the very first allocation.
2102 */
2103 uint64_t approxMetaDataSizeIncrease =
2104 allTracksTotalMetaDataSizeEstimate - mPrevAllTracksTotalMetaDataSizeEstimate;
2105 ALOGV("approxMetaDataSizeIncrease:%" PRIu64 " wantSize:%" PRIu64, approxMetaDataSizeIncrease,
2106 wantSize);
2107 mPrevAllTracksTotalMetaDataSizeEstimate = allTracksTotalMetaDataSizeEstimate;
2108 ALOGV("mPreAllocateFileEndOffset:%" PRIu64 " mOffset:%" PRIu64, mPreAllocateFileEndOffset,
2109 mOffset);
2110 off64_t lastFileEndOffset = std::max(mPreAllocateFileEndOffset, mOffset);
2111 uint64_t preAllocateSize = wantSize + approxMOOVBoxSize + approxMetaDataSizeIncrease;
2112 ALOGV("preAllocateSize :%" PRIu64 " lastFileEndOffset:%" PRIu64, preAllocateSize,
2113 lastFileEndOffset);
2114
2115 int res = fallocate64(mFd, FALLOC_FL_KEEP_SIZE, lastFileEndOffset, preAllocateSize);
2116 if (res == -1) {
2117 ALOGE("fallocate err:%s, %d, fd:%d", strerror(errno), errno, mFd);
2118 sp<AMessage> msg = new AMessage(kWhatFallocateError, mReflector);
2119 msg->setInt32("err", ERROR_IO);
2120 status_t err = msg->post();
2121 mFallocateErr = true;
2122 ALOGD("preAllocation post:%d", err);
2123 } else {
2124 mPreAllocateFileEndOffset = lastFileEndOffset + preAllocateSize;
2125 ALOGV("mPreAllocateFileEndOffset:%" PRIu64, mPreAllocateFileEndOffset);
2126 }
2127 return (res == -1) ? false : true;
2128 }
2129
truncatePreAllocation()2130 bool MPEG4Writer::truncatePreAllocation() {
2131 if (!mPreAllocationEnabled)
2132 return true;
2133
2134 bool status = true;
2135 off64_t endOffset = std::max(mMdatEndOffset, mOffset);
2136 /* if mPreAllocateFileEndOffset >= endOffset, then preallocation logic works good. (diff >= 0).
2137 * Otherwise, the logic needs to be modified.
2138 */
2139 ALOGD("ftruncate mPreAllocateFileEndOffset:%" PRId64 " mOffset:%" PRIu64
2140 " mMdatEndOffset:%" PRIu64 " diff:%" PRId64, mPreAllocateFileEndOffset, mOffset,
2141 mMdatEndOffset, mPreAllocateFileEndOffset - endOffset);
2142 if (ftruncate64(mFd, endOffset) == -1) {
2143 ALOGE("ftruncate err:%s, %d, fd:%d", strerror(errno), errno, mFd);
2144 status = false;
2145 /* No need to post and handle(stop & notify client) error like it's done in preAllocate(),
2146 * because ftruncate() is called during release() only and the error here would be
2147 * reported from there as this function is returning false on any error in ftruncate().
2148 */
2149 }
2150 return status;
2151 }
2152
exceedsFileSizeLimit()2153 bool MPEG4Writer::exceedsFileSizeLimit() {
2154 // No limit
2155 if (mMaxFileSizeLimitBytes == 0) {
2156 return false;
2157 }
2158 int64_t nTotalBytesEstimate = static_cast<int64_t>(mInMemoryCacheSize);
2159 for (List<Track *>::iterator it = mTracks.begin();
2160 it != mTracks.end(); ++it) {
2161 nTotalBytesEstimate += (*it)->getEstimatedTrackSizeBytes();
2162 }
2163
2164 if (!mStreamableFile) {
2165 // Add 1024 bytes as error tolerance
2166 return nTotalBytesEstimate + 1024 >= mMaxFileSizeLimitBytes;
2167 }
2168
2169 // Be conservative in the estimate: do not exceed 95% of
2170 // the target file limit. For small target file size limit, though,
2171 // this will not help.
2172 return (nTotalBytesEstimate >= (95 * mMaxFileSizeLimitBytes) / 100);
2173 }
2174
approachingFileSizeLimit()2175 bool MPEG4Writer::approachingFileSizeLimit() {
2176 // No limit
2177 if (mMaxFileSizeLimitBytes == 0) {
2178 return false;
2179 }
2180
2181 int64_t nTotalBytesEstimate = static_cast<int64_t>(mInMemoryCacheSize);
2182 for (List<Track *>::iterator it = mTracks.begin();
2183 it != mTracks.end(); ++it) {
2184 nTotalBytesEstimate += (*it)->getEstimatedTrackSizeBytes();
2185 }
2186
2187 if (!mStreamableFile) {
2188 // Add 1024 bytes as error tolerance
2189 return nTotalBytesEstimate + 1024 >= (90 * mMaxFileSizeLimitBytes) / 100;
2190 }
2191
2192 return (nTotalBytesEstimate >= (90 * mMaxFileSizeLimitBytes) / 100);
2193 }
2194
exceedsFileDurationLimit()2195 bool MPEG4Writer::exceedsFileDurationLimit() {
2196 // No limit
2197 if (mMaxFileDurationLimitUs == 0) {
2198 return false;
2199 }
2200
2201 for (List<Track *>::iterator it = mTracks.begin();
2202 it != mTracks.end(); ++it) {
2203 if (!(*it)->isHeif() &&
2204 (*it)->getDurationUs() >= mMaxFileDurationLimitUs) {
2205 return true;
2206 }
2207 }
2208 return false;
2209 }
2210
reachedEOS()2211 bool MPEG4Writer::reachedEOS() {
2212 bool allDone = true;
2213 for (List<Track *>::iterator it = mTracks.begin();
2214 it != mTracks.end(); ++it) {
2215 if (!(*it)->reachedEOS()) {
2216 allDone = false;
2217 break;
2218 }
2219 }
2220
2221 return allDone;
2222 }
2223
setStartTimestampUs(int64_t timeUs)2224 void MPEG4Writer::setStartTimestampUs(int64_t timeUs) {
2225 ALOGI("setStartTimestampUs: %" PRId64, timeUs);
2226 CHECK_GE(timeUs, 0LL);
2227 Mutex::Autolock autoLock(mLock);
2228 if (mStartTimestampUs < 0 || mStartTimestampUs > timeUs) {
2229 mStartTimestampUs = timeUs;
2230 ALOGI("Earliest track starting time: %" PRId64, mStartTimestampUs);
2231 }
2232 }
2233
getStartTimestampUs()2234 int64_t MPEG4Writer::getStartTimestampUs() {
2235 Mutex::Autolock autoLock(mLock);
2236 return mStartTimestampUs;
2237 }
2238
2239 /* Returns negative when reordering is needed because of BFrames or zero otherwise.
2240 * CTTS values for tracks with BFrames offsets this negative value.
2241 */
getStartTimeOffsetBFramesUs()2242 int32_t MPEG4Writer::getStartTimeOffsetBFramesUs() {
2243 Mutex::Autolock autoLock(mLock);
2244 return mStartTimeOffsetBFramesUs;
2245 }
2246
numTracks()2247 size_t MPEG4Writer::numTracks() {
2248 Mutex::Autolock autolock(mLock);
2249 return mTracks.size();
2250 }
2251
2252 ////////////////////////////////////////////////////////////////////////////////
2253
Track(MPEG4Writer * owner,const sp<MediaSource> & source,uint32_t aTrackId)2254 MPEG4Writer::Track::Track(MPEG4Writer* owner, const sp<MediaSource>& source, uint32_t aTrackId)
2255 : mOwner(owner),
2256 mMeta(source->getFormat()),
2257 mSource(source),
2258 mDone(false),
2259 mPaused(false),
2260 mResumed(false),
2261 mStarted(false),
2262 mGotStartKeyFrame(false),
2263 mIsMalformed(false),
2264 mTrackId(aTrackId),
2265 mTrackDurationUs(0),
2266 mEstimatedTrackSizeBytes(0),
2267 mSamplesHaveSameSize(true),
2268 mStszTableEntries(new ListTableEntries<uint32_t, 1>(1000)),
2269 mCo64TableEntries(new ListTableEntries<off64_t, 1>(1000)),
2270 mStscTableEntries(new ListTableEntries<uint32_t, 3>(1000)),
2271 mStssTableEntries(new ListTableEntries<uint32_t, 1>(1000)),
2272 mSttsTableEntries(new ListTableEntries<uint32_t, 2>(1000)),
2273 mCttsTableEntries(new ListTableEntries<uint32_t, 2>(1000)),
2274 mElstTableEntries(new ListTableEntries<uint32_t, 3>(3)), // Reserve 3 rows, a row has 3 items
2275 mMinCttsOffsetTimeUs(0),
2276 mMinCttsOffsetTicks(0),
2277 mMaxCttsOffsetTicks(0),
2278 mDoviProfile(0),
2279 mCodecSpecificData(NULL),
2280 mCodecSpecificDataSize(0),
2281 mGotAllCodecSpecificData(false),
2282 mReachedEOS(false),
2283 mStartTimestampUs(-1),
2284 mFirstSampleTimeRealUs(0),
2285 mFirstSampleStartOffsetUs(0),
2286 mRotation(0),
2287 mDimgRefs("dimg"),
2288 mGainmapDimgRefs("dimg"),
2289 mImageItemId(0),
2290 mItemIdBase(0),
2291 mIsPrimary(0),
2292 mWidth(0),
2293 mHeight(0),
2294 mTileWidth(0),
2295 mTileHeight(0),
2296 mGridRows(0),
2297 mGridCols(0),
2298 mNumTiles(1),
2299 mTileIndex(0),
2300 mGainmapItemId(0),
2301 mGainmapMetadataItemId(0),
2302 mColorAspectsValid(false) {
2303 getCodecSpecificDataFromInputFormatIfPossible();
2304
2305 const char *mime;
2306 mMeta->findCString(kKeyMIMEType, &mime);
2307 mIsAvc = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC);
2308 mIsHevc = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_HEVC);
2309 mIsAv1 = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AV1);
2310 mIsApv = editing_flags::muxer_mp4_enable_apv() && !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_APV);
2311 mIsDovi = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_DOLBY_VISION);
2312 mIsAudio = !strncasecmp(mime, "audio/", 6);
2313 mIsVideo = !strncasecmp(mime, "video/", 6);
2314 mIsHeic = !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC);
2315 mIsAvif = !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF);
2316 mIsHeif = mIsHeic || mIsAvif;
2317 mIsMPEG4 = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_MPEG4) ||
2318 !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AAC);
2319
2320 // store temporal layer count
2321 if (mIsVideo) {
2322 int32_t count;
2323 if (mMeta->findInt32(kKeyTemporalLayerCount, &count) && count > 1) {
2324 mOwner->setTemporalLayerCount(count);
2325 }
2326 }
2327
2328 if (!mIsHeif) {
2329 setTimeScale();
2330 } else {
2331 CHECK(mMeta->findInt32(kKeyWidth, &mWidth) && (mWidth > 0));
2332 CHECK(mMeta->findInt32(kKeyHeight, &mHeight) && (mHeight > 0));
2333
2334 int32_t tileWidth, tileHeight, gridRows, gridCols;
2335 if (mMeta->findInt32(kKeyTileWidth, &tileWidth) && (tileWidth > 0) &&
2336 mMeta->findInt32(kKeyTileHeight, &tileHeight) && (tileHeight > 0) &&
2337 mMeta->findInt32(kKeyGridRows, &gridRows) && (gridRows > 0) &&
2338 mMeta->findInt32(kKeyGridCols, &gridCols) && (gridCols > 0)) {
2339 mTileWidth = tileWidth;
2340 mTileHeight = tileHeight;
2341 mGridRows = gridRows;
2342 mGridCols = gridCols;
2343 mNumTiles = gridRows * gridCols;
2344 }
2345 if (!mMeta->findInt32(kKeyTrackIsDefault, &mIsPrimary)) {
2346 mIsPrimary = false;
2347 }
2348 }
2349 }
2350
2351 // Clear all the internal states except the CSD data.
resetInternal()2352 void MPEG4Writer::Track::resetInternal() {
2353 mDone = false;
2354 mPaused = false;
2355 mResumed = false;
2356 mStarted = false;
2357 mGotStartKeyFrame = false;
2358 mIsMalformed = false;
2359 mTrackDurationUs = 0;
2360 mEstimatedTrackSizeBytes = 0;
2361 mSamplesHaveSameSize = false;
2362 if (mStszTableEntries != NULL) {
2363 delete mStszTableEntries;
2364 mStszTableEntries = new ListTableEntries<uint32_t, 1>(1000);
2365 }
2366 if (mCo64TableEntries != NULL) {
2367 delete mCo64TableEntries;
2368 mCo64TableEntries = new ListTableEntries<off64_t, 1>(1000);
2369 }
2370 if (mStscTableEntries != NULL) {
2371 delete mStscTableEntries;
2372 mStscTableEntries = new ListTableEntries<uint32_t, 3>(1000);
2373 }
2374 if (mStssTableEntries != NULL) {
2375 delete mStssTableEntries;
2376 mStssTableEntries = new ListTableEntries<uint32_t, 1>(1000);
2377 }
2378 if (mSttsTableEntries != NULL) {
2379 delete mSttsTableEntries;
2380 mSttsTableEntries = new ListTableEntries<uint32_t, 2>(1000);
2381 }
2382 if (mCttsTableEntries != NULL) {
2383 delete mCttsTableEntries;
2384 mCttsTableEntries = new ListTableEntries<uint32_t, 2>(1000);
2385 }
2386 if (mElstTableEntries != NULL) {
2387 delete mElstTableEntries;
2388 mElstTableEntries = new ListTableEntries<uint32_t, 3>(3);
2389 }
2390 mReachedEOS = false;
2391 }
2392
trackMetaDataSize()2393 int64_t MPEG4Writer::Track::trackMetaDataSize() {
2394 int64_t co64BoxSizeBytes = mCo64TableEntries->count() * 8;
2395 int64_t stszBoxSizeBytes = mStszTableEntries->count() * 4;
2396 int64_t trackMetaDataSize = mStscTableEntries->count() * 12 + // stsc box size
2397 mStssTableEntries->count() * 4 + // stss box size
2398 mSttsTableEntries->count() * 8 + // stts box size
2399 mCttsTableEntries->count() * 8 + // ctts box size
2400 mElstTableEntries->count() * 12 + // elst box size
2401 co64BoxSizeBytes + // stco box size
2402 stszBoxSizeBytes; // stsz box size
2403 return trackMetaDataSize;
2404 }
2405
2406
updateTrackSizeEstimate()2407 void MPEG4Writer::Track::updateTrackSizeEstimate() {
2408 mEstimatedTrackSizeBytes = mMdatSizeBytes; // media data size
2409 if (!isHeif() && !mOwner->isFileStreamable()) {
2410 mEstimatedTrackSizeBytes += trackMetaDataSize();
2411 }
2412 }
2413
addOneStscTableEntry(size_t chunkId,size_t sampleId)2414 void MPEG4Writer::Track::addOneStscTableEntry(
2415 size_t chunkId, size_t sampleId) {
2416 mStscTableEntries->add(htonl(chunkId));
2417 mStscTableEntries->add(htonl(sampleId));
2418 mStscTableEntries->add(htonl(1));
2419 }
2420
addOneStssTableEntry(size_t sampleId)2421 void MPEG4Writer::Track::addOneStssTableEntry(size_t sampleId) {
2422 mStssTableEntries->add(htonl(sampleId));
2423 }
2424
addOneSttsTableEntry(size_t sampleCount,int32_t delta)2425 void MPEG4Writer::Track::addOneSttsTableEntry(size_t sampleCount, int32_t delta) {
2426 if (delta == 0) {
2427 ALOGW("0-duration samples found: %zu", sampleCount);
2428 }
2429 mSttsTableEntries->add(htonl(sampleCount));
2430 mSttsTableEntries->add(htonl(delta));
2431 }
2432
addOneCttsTableEntry(size_t sampleCount,int32_t sampleOffset)2433 void MPEG4Writer::Track::addOneCttsTableEntry(size_t sampleCount, int32_t sampleOffset) {
2434 if (!mIsVideo) {
2435 return;
2436 }
2437 mCttsTableEntries->add(htonl(sampleCount));
2438 mCttsTableEntries->add(htonl(sampleOffset));
2439 }
2440
addOneElstTableEntry(uint32_t segmentDuration,int32_t mediaTime,int16_t mediaRate,int16_t mediaRateFraction)2441 void MPEG4Writer::Track::addOneElstTableEntry(
2442 uint32_t segmentDuration, int32_t mediaTime, int16_t mediaRate, int16_t mediaRateFraction) {
2443 ALOGV("segmentDuration:%u, mediaTime:%d", segmentDuration, mediaTime);
2444 ALOGV("mediaRate :%" PRId16 ", mediaRateFraction :%" PRId16 ", Ored %u", mediaRate,
2445 mediaRateFraction, ((((uint32_t)mediaRate) << 16) | ((uint32_t)mediaRateFraction)));
2446 mElstTableEntries->add(htonl(segmentDuration));
2447 mElstTableEntries->add(htonl(mediaTime));
2448 mElstTableEntries->add(htonl((((uint32_t)mediaRate) << 16) | (uint32_t)mediaRateFraction));
2449 }
2450
setupAndStartLooper()2451 status_t MPEG4Writer::setupAndStartLooper() {
2452 status_t err = OK;
2453 if (mLooper == nullptr) {
2454 mLooper = new ALooper;
2455 mLooper->setName("MP4WtrCtrlHlpLooper");
2456 if (mIsBackgroundMode) {
2457 err = mLooper->start(false, false, ANDROID_PRIORITY_BACKGROUND);
2458 } else {
2459 err = mLooper->start();
2460 }
2461 mReflector = new AHandlerReflector<MPEG4Writer>(this);
2462 mLooper->registerHandler(mReflector);
2463 }
2464 ALOGD("MP4WtrCtrlHlpLooper Started");
2465 return err;
2466 }
2467
stopAndReleaseLooper()2468 void MPEG4Writer::stopAndReleaseLooper() {
2469 if (mLooper != nullptr) {
2470 if (mReflector != nullptr) {
2471 mLooper->unregisterHandler(mReflector->id());
2472 mReflector.clear();
2473 }
2474 mLooper->stop();
2475 mLooper.clear();
2476 ALOGD("MP4WtrCtrlHlpLooper stopped");
2477 }
2478 }
2479
setNextFd(int fd)2480 status_t MPEG4Writer::setNextFd(int fd) {
2481 Mutex::Autolock l(mLock);
2482 if (mNextFd != -1) {
2483 // No need to set a new FD yet.
2484 return INVALID_OPERATION;
2485 }
2486 mNextFd = dup(fd);
2487 mFdCond.signal();
2488 return OK;
2489 }
2490
isGainmapMetaData(MediaBufferBase * buffer,uint32_t * offset) const2491 bool MPEG4Writer::Track::isGainmapMetaData(MediaBufferBase* buffer, uint32_t* offset) const {
2492 if (!mIsHeif) {
2493 return false;
2494 }
2495
2496 // Gainmap metadata block starting with 'tmap\0\0'
2497 size_t length = buffer->range_length();
2498 uint8_t *data = (uint8_t *)buffer->data() + buffer->range_offset();
2499 if ((length > sizeof(kGainmapMetaHeader)) &&
2500 !memcmp(data, kGainmapMetaHeader, sizeof(kGainmapMetaHeader))) {
2501 *offset = sizeof(kGainmapMetaHeader);
2502 return true;
2503 }
2504
2505 return false;
2506 }
2507
isGainmapData(MediaBufferBase * buffer,uint32_t * offset) const2508 bool MPEG4Writer::Track::isGainmapData(MediaBufferBase* buffer, uint32_t* offset) const {
2509 if (!mIsHeif) {
2510 return false;
2511 }
2512
2513 // Gainmap block starting with 'gmap\0\0'
2514 size_t length = buffer->range_length();
2515 uint8_t* data = (uint8_t*)buffer->data() + buffer->range_offset();
2516 if ((length > sizeof(kGainmapHeader)) &&
2517 !memcmp(data, kGainmapHeader, sizeof(kGainmapHeader))) {
2518 *offset = sizeof(kGainmapHeader);
2519 return true;
2520 }
2521
2522 return false;
2523 }
2524
isExifData(MediaBufferBase * buffer,uint32_t * tiffHdrOffset) const2525 bool MPEG4Writer::Track::isExifData(MediaBufferBase* buffer, uint32_t* tiffHdrOffset) const {
2526 if (!mIsHeif) {
2527 return false;
2528 }
2529
2530 // Exif block starting with 'Exif\0\0'
2531 size_t length = buffer->range_length();
2532 uint8_t* data = (uint8_t*)buffer->data() + buffer->range_offset();
2533 if ((length > sizeof(kExifHeader)) && !memcmp(data, kExifHeader, sizeof(kExifHeader))) {
2534 *tiffHdrOffset = sizeof(kExifHeader);
2535 return true;
2536 }
2537
2538 // Exif block starting with fourcc 'Exif' followed by APP1 marker
2539 if ((length > sizeof(kExifApp1Marker) + 2 + sizeof(kExifHeader)) &&
2540 !memcmp(data, kExifApp1Marker, sizeof(kExifApp1Marker)) &&
2541 !memcmp(data + sizeof(kExifApp1Marker) + 2, kExifHeader, sizeof(kExifHeader))) {
2542 // skip 'Exif' fourcc
2543 buffer->set_range(4, buffer->range_length() - 4);
2544
2545 // 2-byte APP1 + 2-byte size followed by kExifHeader
2546 *tiffHdrOffset = 2 + 2 + sizeof(kExifHeader);
2547 return true;
2548 }
2549
2550 return false;
2551 }
2552
addChunkOffset(off64_t offset)2553 void MPEG4Writer::Track::addChunkOffset(off64_t offset) {
2554 CHECK(!mIsHeif);
2555 mCo64TableEntries->add(hton64(offset));
2556 }
2557
addItemOffsetAndSize(off64_t offset,size_t size,bool isExif,bool isGainmapMeta,bool isGainmap)2558 void MPEG4Writer::Track::addItemOffsetAndSize(off64_t offset, size_t size, bool isExif,
2559 bool isGainmapMeta, bool isGainmap) {
2560 CHECK(mIsHeif);
2561
2562 if (offset > UINT32_MAX || size > UINT32_MAX) {
2563 ALOGE("offset or size is out of range: %lld, %lld",
2564 (long long) offset, (long long) size);
2565 mIsMalformed = true;
2566 }
2567 if (mIsMalformed) {
2568 return;
2569 }
2570
2571 if (isExif) {
2572 uint16_t exifItemId;
2573 if (mOwner->reserveItemId_l(1, &exifItemId) != OK) {
2574 return;
2575 }
2576
2577 mExifList.push_back(mOwner->addItem_l({
2578 .itemType = "Exif",
2579 .itemId = exifItemId,
2580 .isPrimary = false,
2581 .isHidden = false,
2582 .offset = (uint32_t)offset,
2583 .size = (uint32_t)size,
2584 }));
2585 return;
2586 }
2587
2588 bool hasGrid = (mTileWidth > 0);
2589
2590 if (isGainmapMeta && flags_camera::camera_heif_gainmap()) {
2591 uint16_t metaItemId;
2592 if (mOwner->reserveItemId_l(1, &metaItemId) != OK) {
2593 return;
2594 }
2595
2596 Vector<uint16_t> props;
2597 if (mColorAspectsValid) {
2598 ItemProperty property;
2599 property.type = FOURCC('c', 'o', 'l', 'r');
2600 ColorUtils::convertCodecColorAspectsToIsoAspects(
2601 mColorAspects, &property.colorPrimaries, &property.colorTransfer,
2602 &property.colorMatrix, &property.colorRange);
2603 props.push_back(mOwner->addProperty_l(property));
2604 }
2605 if (!mBitsPerChannel.empty()) {
2606 ItemProperty property;
2607 property.type = FOURCC('p', 'i', 'x', 'i');
2608 property.bitsPerChannel.appendVector(mBitsPerChannel);
2609 props.push_back(mOwner->addProperty_l(property));
2610 }
2611 props.push_back(mOwner->addProperty_l({
2612 .type = FOURCC('i', 's', 'p', 'e'),
2613 .width = hasGrid ? mTileWidth : mWidth,
2614 .height = hasGrid ? mTileHeight : mHeight,
2615 }));
2616 mGainmapMetadataItemId = mOwner->addItem_l({
2617 .itemType = "tmap",
2618 .itemId = metaItemId,
2619 .isPrimary = false,
2620 .isHidden = false,
2621 .offset = (uint32_t)offset,
2622 .size = (uint32_t)size,
2623 .properties = props,
2624 });
2625 return;
2626 }
2627
2628 if (mTileIndex >= mNumTiles) {
2629 ALOGW("Ignoring excess tiles!");
2630 return;
2631 }
2632
2633 // Rotation angle in HEIF is CCW, framework angle is CW.
2634 int32_t heifRotation = 0;
2635 switch(mRotation) {
2636 case 90: heifRotation = 3; break;
2637 case 180: heifRotation = 2; break;
2638 case 270: heifRotation = 1; break;
2639 default: break; // don't set if invalid
2640 }
2641
2642 if (mProperties.empty()) {
2643 // Min length of hvcC CSD is 23. (ISO/IEC 14496-15:2014 Chapter 8.4.1.1.2)
2644 if (mIsHeif && mCodecSpecificDataSize < 23) {
2645 ALOGE("hvcC csd size is less than 23 bytes");
2646 return;
2647 }
2648 mProperties.push_back(mOwner->addProperty_l({
2649 .type = static_cast<uint32_t>(mIsAvif ?
2650 FOURCC('a', 'v', '1', 'C') :
2651 FOURCC('h', 'v', 'c', 'C')),
2652 .data = ABuffer::CreateAsCopy(mCodecSpecificData, mCodecSpecificDataSize)
2653 }));
2654
2655 mProperties.push_back(mOwner->addProperty_l({
2656 .type = FOURCC('i', 's', 'p', 'e'),
2657 .width = hasGrid ? mTileWidth : mWidth,
2658 .height = hasGrid ? mTileHeight : mHeight,
2659 }));
2660
2661 if (!hasGrid && heifRotation > 0) {
2662 mProperties.push_back(mOwner->addProperty_l({
2663 .type = FOURCC('i', 'r', 'o', 't'),
2664 .rotation = heifRotation,
2665 }));
2666 }
2667 }
2668
2669 mTileIndex++;
2670 if (hasGrid) {
2671 uint16_t id = mOwner->addItem_l({
2672 .itemType = mIsAvif ? "av01" : "hvc1",
2673 .itemId = mItemIdBase++,
2674 .isPrimary = false,
2675 .isHidden = true,
2676 .offset = (uint32_t)offset,
2677 .size = (uint32_t)size,
2678 .properties = mProperties,
2679 });
2680 if (isGainmap && flags_camera::camera_heif_gainmap()) {
2681 mGainmapDimgRefs.value.push_back(id);
2682 } else {
2683 mDimgRefs.value.push_back(id);
2684 }
2685
2686 if (mTileIndex == mNumTiles) {
2687 mProperties.clear();
2688 mProperties.push_back(mOwner->addProperty_l({
2689 .type = FOURCC('i', 's', 'p', 'e'),
2690 .width = mWidth,
2691 .height = mHeight,
2692 }));
2693 if (heifRotation > 0) {
2694 mProperties.push_back(mOwner->addProperty_l({
2695 .type = FOURCC('i', 'r', 'o', 't'),
2696 .rotation = heifRotation,
2697 }));
2698 }
2699 if (mColorAspectsValid && flags_camera::camera_heif_gainmap()) {
2700 ItemProperty property;
2701 property.type = FOURCC('c', 'o', 'l', 'r');
2702 ColorUtils::convertCodecColorAspectsToIsoAspects(
2703 mColorAspects, &property.colorPrimaries, &property.colorTransfer,
2704 &property.colorMatrix, &property.colorRange);
2705 mProperties.push_back(mOwner->addProperty_l(property));
2706 }
2707 if (!mBitsPerChannel.empty() && flags_camera::camera_heif_gainmap()) {
2708 ItemProperty property;
2709 property.type = FOURCC('p', 'i', 'x', 'i');
2710 property.bitsPerChannel.appendVector(mBitsPerChannel);
2711 mProperties.push_back(mOwner->addProperty_l(property));
2712 }
2713 uint16_t itemId = mOwner->addItem_l({
2714 .itemType = "grid",
2715 .itemId = mItemIdBase++,
2716 .isPrimary = isGainmap && flags_camera::camera_heif_gainmap()
2717 ? false
2718 : (mIsPrimary != 0),
2719 .isHidden = false,
2720 .rows = (uint32_t)mGridRows,
2721 .cols = (uint32_t)mGridCols,
2722 .width = (uint32_t)mWidth,
2723 .height = (uint32_t)mHeight,
2724 .properties = mProperties,
2725 });
2726
2727 if (isGainmap && flags_camera::camera_heif_gainmap()) {
2728 mGainmapItemId = itemId;
2729 } else {
2730 mImageItemId = itemId;
2731 }
2732 }
2733 } else {
2734 if (mColorAspectsValid && flags_camera::camera_heif_gainmap()) {
2735 ItemProperty property;
2736 property.type = FOURCC('c', 'o', 'l', 'r');
2737 ColorUtils::convertCodecColorAspectsToIsoAspects(
2738 mColorAspects, &property.colorPrimaries, &property.colorTransfer,
2739 &property.colorMatrix, &property.colorRange);
2740 mProperties.push_back(mOwner->addProperty_l(property));
2741 }
2742 if (!mBitsPerChannel.empty() && flags_camera::camera_heif_gainmap()) {
2743 ItemProperty property;
2744 property.type = FOURCC('p', 'i', 'x', 'i');
2745 property.bitsPerChannel.appendVector(mBitsPerChannel);
2746 mProperties.push_back(mOwner->addProperty_l(property));
2747 }
2748 uint16_t itemId = mOwner->addItem_l({
2749 .itemType = mIsAvif ? "av01" : "hvc1",
2750 .itemId = mItemIdBase++,
2751 .isPrimary = (isGainmap && flags_camera::camera_heif_gainmap()) ? false
2752 : (mIsPrimary != 0),
2753 .isHidden = false,
2754 .offset = (uint32_t)offset,
2755 .size = (uint32_t)size,
2756 .properties = mProperties,
2757 });
2758
2759 if (isGainmap && flags_camera::camera_heif_gainmap()) {
2760 mGainmapItemId = itemId;
2761 } else {
2762 mImageItemId = itemId;
2763 }
2764 }
2765 }
2766
2767 // Flush out the item refs for this track. Note that it must be called after the
2768 // writer thread has stopped, because there might be pending items in the last
2769 // few chunks written by the writer thread (as opposed to the track). In particular,
2770 // it affects the 'dimg' refs for tiled image, as we only have the refs after the
2771 // last tile sample is written.
flushItemRefs()2772 void MPEG4Writer::Track::flushItemRefs() {
2773 CHECK(mIsHeif);
2774
2775 if (mImageItemId > 0) {
2776 mOwner->addRefs_l(mImageItemId, mDimgRefs);
2777
2778 if (!mExifList.empty()) {
2779 // The "cdsc" ref is from the metadata/exif item to the image item.
2780 // So the refs all contain the image item.
2781 ItemRefs cdscRefs("cdsc");
2782 cdscRefs.value.push_back(mImageItemId);
2783 for (uint16_t exifItem : mExifList) {
2784 mOwner->addRefs_l(exifItem, cdscRefs);
2785 }
2786 }
2787 }
2788
2789 if ((mGainmapItemId > 0) && flags_camera::camera_heif_gainmap()) {
2790 mOwner->addRefs_l(mGainmapItemId, mGainmapDimgRefs);
2791 }
2792 }
2793
setTimeScale()2794 void MPEG4Writer::Track::setTimeScale() {
2795 ALOGV("setTimeScale");
2796 // Default time scale
2797 mTimeScale = 90000;
2798
2799 if (mIsAudio) {
2800 // Use the sampling rate as the default time scale for audio track.
2801 int32_t sampleRate;
2802 bool success = mMeta->findInt32(kKeySampleRate, &sampleRate);
2803 CHECK(success);
2804 mTimeScale = sampleRate;
2805 }
2806
2807 // If someone would like to overwrite the timescale, use user-supplied value.
2808 int32_t timeScale;
2809 if (mMeta->findInt32(kKeyTimeScale, &timeScale)) {
2810 mTimeScale = timeScale;
2811 }
2812
2813 CHECK_GT(mTimeScale, 0);
2814 }
2815
onMessageReceived(const sp<AMessage> & msg)2816 void MPEG4Writer::onMessageReceived(const sp<AMessage> &msg) {
2817 switch (msg->what()) {
2818 case kWhatSwitch:
2819 {
2820 mLock.lock();
2821 int fd = mNextFd;
2822 mNextFd = -1;
2823 mLock.unlock();
2824 if (finishCurrentSession() == OK) {
2825 initInternal(fd, false /*isFirstSession*/);
2826 status_t status = start(mStartMeta.get());
2827 mSwitchPending = false;
2828 if (status == OK) {
2829 notify(MEDIA_RECORDER_EVENT_INFO,
2830 MEDIA_RECORDER_INFO_NEXT_OUTPUT_FILE_STARTED, 0);
2831 }
2832 }
2833 break;
2834 }
2835 /* ::write() or lseek64() wasn't a success, file could be malformed.
2836 * Or fallocate() failed. reset() and notify client on both the cases.
2837 */
2838 case kWhatFallocateError: // fallthrough
2839 case kWhatIOError: {
2840 int32_t err;
2841 CHECK(msg->findInt32("err", &err));
2842 // If reset already in process, don't wait for it complete to avoid deadlock.
2843 reset(true, false);
2844 //TODO: new MEDIA_RECORDER_ERROR_**** instead MEDIA_RECORDER_ERROR_UNKNOWN ?
2845 notify(MEDIA_RECORDER_EVENT_ERROR, MEDIA_RECORDER_ERROR_UNKNOWN, err);
2846 break;
2847 }
2848 /* Response to kWhatNoIOErrorSoFar would be OK always as of now.
2849 * Responding with other options could be added later if required.
2850 */
2851 case kWhatNoIOErrorSoFar: {
2852 ALOGV("kWhatNoIOErrorSoFar");
2853 sp<AMessage> response = new AMessage;
2854 response->setInt32("err", OK);
2855 sp<AReplyToken> replyID;
2856 CHECK(msg->senderAwaitsResponse(&replyID));
2857 response->postReply(replyID);
2858 break;
2859 }
2860 default:
2861 TRESPASS();
2862 }
2863 }
2864
getCodecSpecificDataFromInputFormatIfPossible()2865 void MPEG4Writer::Track::getCodecSpecificDataFromInputFormatIfPossible() {
2866 const char *mime;
2867
2868 CHECK(mMeta->findCString(kKeyMIMEType, &mime));
2869
2870 uint32_t type;
2871 const void *data = NULL;
2872 size_t size = 0;
2873 if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC)) {
2874 mMeta->findData(kKeyAVCC, &type, &data, &size);
2875 } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_HEVC) ||
2876 !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC)) {
2877 mMeta->findData(kKeyHVCC, &type, &data, &size);
2878 } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AV1) ||
2879 !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF)) {
2880 mMeta->findData(kKeyAV1C, &type, &data, &size);
2881 } else if (editing_flags::muxer_mp4_enable_apv() &&
2882 !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_APV)) {
2883 mMeta->findData(kKeyAPVC, &type, &data, &size);
2884 } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_DOLBY_VISION)) {
2885 getDolbyVisionProfile();
2886 if (!mMeta->findData(kKeyAVCC, &type, &data, &size) &&
2887 !mMeta->findData(kKeyHVCC, &type, &data, &size)) {
2888 ALOGE("Failed: No HVCC/AVCC for Dolby Vision ..\n");
2889 return;
2890 }
2891 } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_MPEG4) ||
2892 !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AAC)) {
2893 if (mMeta->findData(kKeyESDS, &type, &data, &size)) {
2894 ESDS esds(data, size);
2895 if (esds.getCodecSpecificInfo(&data, &size) == OK &&
2896 data != NULL &&
2897 copyCodecSpecificData((uint8_t*)data, size) == OK) {
2898 mGotAllCodecSpecificData = true;
2899 }
2900 return;
2901 }
2902 }
2903 if (data != NULL && copyCodecSpecificData((uint8_t *)data, size) == OK) {
2904 mGotAllCodecSpecificData = true;
2905 }
2906 }
2907
~Track()2908 MPEG4Writer::Track::~Track() {
2909 stop();
2910
2911 delete mStszTableEntries;
2912 delete mCo64TableEntries;
2913 delete mStscTableEntries;
2914 delete mSttsTableEntries;
2915 delete mStssTableEntries;
2916 delete mCttsTableEntries;
2917 delete mElstTableEntries;
2918
2919 mStszTableEntries = NULL;
2920 mCo64TableEntries = NULL;
2921 mStscTableEntries = NULL;
2922 mSttsTableEntries = NULL;
2923 mStssTableEntries = NULL;
2924 mCttsTableEntries = NULL;
2925 mElstTableEntries = NULL;
2926
2927 if (mCodecSpecificData != NULL) {
2928 free(mCodecSpecificData);
2929 mCodecSpecificData = NULL;
2930 }
2931
2932 }
2933
initTrackingProgressStatus(MetaData * params)2934 void MPEG4Writer::Track::initTrackingProgressStatus(MetaData *params) {
2935 ALOGV("initTrackingProgressStatus");
2936 mPreviousTrackTimeUs = -1;
2937 mTrackingProgressStatus = false;
2938 mTrackEveryTimeDurationUs = 0;
2939 {
2940 int64_t timeUs;
2941 if (params && params->findInt64(kKeyTrackTimeStatus, &timeUs)) {
2942 ALOGV("Receive request to track progress status for every %" PRId64 " us", timeUs);
2943 mTrackEveryTimeDurationUs = timeUs;
2944 mTrackingProgressStatus = true;
2945 }
2946 }
2947 }
2948
2949 // static
ThreadWrapper(void * me)2950 void *MPEG4Writer::ThreadWrapper(void *me) {
2951 ALOGV("ThreadWrapper: %p", me);
2952 MPEG4Writer *writer = static_cast<MPEG4Writer *>(me);
2953 writer->threadFunc();
2954 return NULL;
2955 }
2956
bufferChunk(const Chunk & chunk)2957 void MPEG4Writer::bufferChunk(const Chunk& chunk) {
2958 ALOGV("bufferChunk: %p", chunk.mTrack);
2959 Mutex::Autolock autolock(mLock);
2960 CHECK_EQ(mDone, false);
2961
2962 for (List<ChunkInfo>::iterator it = mChunkInfos.begin();
2963 it != mChunkInfos.end(); ++it) {
2964
2965 if (chunk.mTrack == it->mTrack) { // Found owner
2966 it->mChunks.push_back(chunk);
2967 mChunkReadyCondition.signal();
2968 return;
2969 }
2970 }
2971
2972 CHECK(!"Received a chunk for a unknown track");
2973 }
2974
writeChunkToFile(Chunk * chunk)2975 void MPEG4Writer::writeChunkToFile(Chunk* chunk) {
2976 ALOGV("writeChunkToFile: %" PRId64 " from %s track",
2977 chunk->mTimeStampUs, chunk->mTrack->getTrackType());
2978
2979 int32_t isFirstSample = true;
2980 while (!chunk->mSamples.empty()) {
2981 List<MediaBuffer *>::iterator it = chunk->mSamples.begin();
2982
2983 uint32_t tiffHdrOffset;
2984 if (!(*it)->meta_data().findInt32(
2985 kKeyExifTiffOffset, (int32_t*)&tiffHdrOffset)) {
2986 tiffHdrOffset = 0;
2987 }
2988 bool isExif = (tiffHdrOffset > 0);
2989 bool usePrefix = chunk->mTrack->usePrefix() && !isExif;
2990
2991 size_t bytesWritten;
2992 off64_t offset = addSample_l(*it, usePrefix, tiffHdrOffset, &bytesWritten);
2993
2994 if (chunk->mTrack->isHeif()) {
2995 chunk->mTrack->addItemOffsetAndSize(offset, bytesWritten, isExif);
2996 } else if (isFirstSample) {
2997 chunk->mTrack->addChunkOffset(offset);
2998 isFirstSample = false;
2999 }
3000
3001 (*it)->release();
3002 (*it) = NULL;
3003 chunk->mSamples.erase(it);
3004 }
3005 chunk->mSamples.clear();
3006 }
3007
writeAllChunks()3008 void MPEG4Writer::writeAllChunks() {
3009 ALOGV("writeAllChunks");
3010 size_t outstandingChunks = 0;
3011 Chunk chunk;
3012 while (findChunkToWrite(&chunk)) {
3013 writeChunkToFile(&chunk);
3014 ++outstandingChunks;
3015 }
3016
3017 sendSessionSummary();
3018
3019 mChunkInfos.clear();
3020 ALOGD("%zu chunks are written in the last batch", outstandingChunks);
3021 }
3022
findChunkToWrite(Chunk * chunk)3023 bool MPEG4Writer::findChunkToWrite(Chunk *chunk) {
3024 ALOGV("findChunkToWrite");
3025
3026 int64_t minTimestampUs = 0x7FFFFFFFFFFFFFFFLL;
3027 Track *track = NULL;
3028 for (List<ChunkInfo>::iterator it = mChunkInfos.begin();
3029 it != mChunkInfos.end(); ++it) {
3030 if (!it->mChunks.empty()) {
3031 List<Chunk>::iterator chunkIt = it->mChunks.begin();
3032 if (chunkIt->mTimeStampUs < minTimestampUs) {
3033 minTimestampUs = chunkIt->mTimeStampUs;
3034 track = it->mTrack;
3035 }
3036 }
3037 }
3038
3039 if (track == NULL) {
3040 ALOGV("Nothing to be written after all");
3041 return false;
3042 }
3043
3044 if (mIsFirstChunk) {
3045 mIsFirstChunk = false;
3046 }
3047
3048 for (List<ChunkInfo>::iterator it = mChunkInfos.begin();
3049 it != mChunkInfos.end(); ++it) {
3050 if (it->mTrack == track) {
3051 *chunk = *(it->mChunks.begin());
3052 it->mChunks.erase(it->mChunks.begin());
3053 CHECK_EQ(chunk->mTrack, track);
3054
3055 int64_t interChunkTimeUs =
3056 chunk->mTimeStampUs - it->mPrevChunkTimestampUs;
3057 if (interChunkTimeUs > it->mPrevChunkTimestampUs) {
3058 it->mMaxInterChunkDurUs = interChunkTimeUs;
3059 }
3060 return true;
3061 }
3062 }
3063
3064 return false;
3065 }
3066
threadFunc()3067 void MPEG4Writer::threadFunc() {
3068 ALOGV("threadFunc");
3069
3070 prctl(PR_SET_NAME, (unsigned long)"MPEG4Writer", 0, 0, 0);
3071
3072 if (mIsBackgroundMode) {
3073 // Background priority for media transcoding.
3074 androidSetThreadPriority(0 /* tid (0 = current) */, ANDROID_PRIORITY_BACKGROUND);
3075 }
3076
3077 Mutex::Autolock autoLock(mLock);
3078 while (!mDone) {
3079 Chunk chunk;
3080 bool chunkFound = false;
3081
3082 while (!mDone && !(chunkFound = findChunkToWrite(&chunk))) {
3083 mChunkReadyCondition.wait(mLock);
3084 }
3085
3086 // In real time recording mode, write without holding the lock in order
3087 // to reduce the blocking time for media track threads.
3088 // Otherwise, hold the lock until the existing chunks get written to the
3089 // file.
3090 if (chunkFound) {
3091 if (mIsRealTimeRecording) {
3092 mLock.unlock();
3093 }
3094 writeChunkToFile(&chunk);
3095 if (mIsRealTimeRecording) {
3096 mLock.lock();
3097 }
3098 }
3099 }
3100
3101 writeAllChunks();
3102 ALOGV("threadFunc mOffset:%lld, mMaxOffsetAppend:%lld", (long long)mOffset,
3103 (long long)mMaxOffsetAppend);
3104 mOffset = std::max(mOffset, mMaxOffsetAppend);
3105 }
3106
startWriterThread()3107 status_t MPEG4Writer::startWriterThread() {
3108 ALOGV("startWriterThread");
3109
3110 mDone = false;
3111 mIsFirstChunk = true;
3112 mDriftTimeUs = 0;
3113 for (List<Track *>::iterator it = mTracks.begin();
3114 it != mTracks.end(); ++it) {
3115 ChunkInfo info;
3116 info.mTrack = *it;
3117 info.mPrevChunkTimestampUs = 0;
3118 info.mMaxInterChunkDurUs = 0;
3119 mChunkInfos.push_back(info);
3120 }
3121
3122 pthread_attr_t attr;
3123 pthread_attr_init(&attr);
3124 pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
3125 pthread_create(&mThread, &attr, ThreadWrapper, this);
3126 pthread_attr_destroy(&attr);
3127 mWriterThreadStarted = true;
3128 return OK;
3129 }
3130
3131
start(MetaData * params)3132 status_t MPEG4Writer::Track::start(MetaData *params) {
3133 if (!mDone && mPaused) {
3134 mPaused = false;
3135 mResumed = true;
3136 return OK;
3137 }
3138
3139 int64_t startTimeUs;
3140 if (params == NULL || !params->findInt64(kKeyTime, &startTimeUs)) {
3141 startTimeUs = 0;
3142 }
3143 mStartTimeRealUs = startTimeUs;
3144
3145 int32_t rotationDegrees;
3146 if ((mIsVideo || mIsHeif) && params &&
3147 params->findInt32(kKeyRotation, &rotationDegrees)) {
3148 mRotation = rotationDegrees;
3149 }
3150 if (mIsHeif) {
3151 // Reserve the item ids, so that the item ids are ordered in the same
3152 // order that the image tracks are added.
3153 // If we leave the item ids to be assigned when the sample is written out,
3154 // the original track order may not be preserved, if two image tracks
3155 // have data around the same time. (This could happen especially when
3156 // we're encoding with single tile.) The reordering may be undesirable,
3157 // even if the file is well-formed and the primary picture is correct.
3158
3159 // Reserve item ids for samples + grid
3160 size_t numItemsToReserve = mNumTiles + (mNumTiles > 0);
3161 status_t err = mOwner->reserveItemId_l(numItemsToReserve, &mItemIdBase);
3162 if (err != OK) {
3163 return err;
3164 }
3165 }
3166
3167 initTrackingProgressStatus(params);
3168
3169 sp<MetaData> meta = new MetaData;
3170 if (mOwner->isRealTimeRecording() && mOwner->numTracks() > 1) {
3171 /*
3172 * This extra delay of accepting incoming audio/video signals
3173 * helps to align a/v start time at the beginning of a recording
3174 * session, and it also helps eliminate the "recording" sound for
3175 * camcorder applications.
3176 *
3177 * If client does not set the start time offset, we fall back to
3178 * use the default initial delay value.
3179 */
3180 int64_t startTimeOffsetUs = mOwner->getStartTimeOffsetMs() * 1000LL;
3181 if (startTimeOffsetUs < 0) { // Start time offset was not set
3182 startTimeOffsetUs = kInitialDelayTimeUs;
3183 }
3184 startTimeUs += startTimeOffsetUs;
3185 ALOGI("Start time offset: %" PRId64 " us", startTimeOffsetUs);
3186 }
3187
3188 meta->setInt64(kKeyTime, startTimeUs);
3189
3190 status_t err = mSource->start(meta.get());
3191 if (err != OK) {
3192 mDone = mReachedEOS = true;
3193 return err;
3194 }
3195
3196 pthread_attr_t attr;
3197 pthread_attr_init(&attr);
3198 pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
3199
3200 mDone = false;
3201 mStarted = true;
3202 mTrackDurationUs = 0;
3203 mReachedEOS = false;
3204 mEstimatedTrackSizeBytes = 0;
3205 mMdatSizeBytes = 0;
3206 mMaxChunkDurationUs = 0;
3207 mLastDecodingTimeUs = -1;
3208
3209 pthread_create(&mThread, &attr, ThreadWrapper, this);
3210 pthread_attr_destroy(&attr);
3211
3212 return OK;
3213 }
3214
pause()3215 status_t MPEG4Writer::Track::pause() {
3216 mPaused = true;
3217 return OK;
3218 }
3219
stop(bool stopSource)3220 status_t MPEG4Writer::Track::stop(bool stopSource) {
3221 ALOGD("%s track stopping. %s source", getTrackType(), stopSource ? "Stop" : "Not Stop");
3222 if (!mStarted) {
3223 ALOGE("Stop() called but track is not started or stopped");
3224 return ERROR_END_OF_STREAM;
3225 }
3226
3227 if (mDone) {
3228 return OK;
3229 }
3230
3231 if (stopSource) {
3232 ALOGD("%s track source stopping", getTrackType());
3233 mSource->stop();
3234 ALOGD("%s track source stopped", getTrackType());
3235 }
3236
3237 // Set mDone to be true after sucessfully stop mSource as mSource may be still outputting
3238 // buffers to the writer.
3239 mDone = true;
3240
3241 void *dummy;
3242 status_t err = OK;
3243 int retVal = pthread_join(mThread, &dummy);
3244 if (retVal == 0) {
3245 err = static_cast<status_t>(reinterpret_cast<uintptr_t>(dummy));
3246 ALOGD("%s track stopped. Status:%d. %s source",
3247 getTrackType(), err, stopSource ? "Stop" : "Not Stop");
3248 } else {
3249 ALOGE("track::stop: pthread_join retVal:%d", retVal);
3250 err = UNKNOWN_ERROR;
3251 }
3252 mStarted = false;
3253 return err;
3254 }
3255
reachedEOS()3256 bool MPEG4Writer::Track::reachedEOS() {
3257 return mReachedEOS;
3258 }
3259
3260 // static
ThreadWrapper(void * me)3261 void *MPEG4Writer::Track::ThreadWrapper(void *me) {
3262 Track *track = static_cast<Track *>(me);
3263
3264 status_t err = track->threadEntry();
3265 return (void *)(uintptr_t)err;
3266 }
3267
getNalUnitType(uint8_t byte,uint8_t * type)3268 static void getNalUnitType(uint8_t byte, uint8_t* type) {
3269 ALOGV("getNalUnitType: %d", byte);
3270
3271 // nal_unit_type: 5-bit unsigned integer
3272 *type = (byte & 0x1F);
3273 }
3274
parseParamSet(const uint8_t * data,size_t length,int type,size_t * paramSetLen)3275 const uint8_t *MPEG4Writer::Track::parseParamSet(
3276 const uint8_t *data, size_t length, int type, size_t *paramSetLen) {
3277
3278 ALOGV("parseParamSet");
3279 CHECK(type == kNalUnitTypeSeqParamSet ||
3280 type == kNalUnitTypePicParamSet);
3281
3282 const uint8_t *nextStartCode = findNextNalStartCode(data, length);
3283 *paramSetLen = nextStartCode - data;
3284 if (*paramSetLen == 0) {
3285 ALOGE("Param set is malformed, since its length is 0");
3286 return NULL;
3287 }
3288
3289 AVCParamSet paramSet(*paramSetLen, data);
3290 if (type == kNalUnitTypeSeqParamSet) {
3291 if (*paramSetLen < 4) {
3292 ALOGE("Seq parameter set malformed");
3293 return NULL;
3294 }
3295 if (mSeqParamSets.empty()) {
3296 mProfileIdc = data[1];
3297 mProfileCompatible = data[2];
3298 mLevelIdc = data[3];
3299 } else {
3300 if (mProfileIdc != data[1] ||
3301 mProfileCompatible != data[2] ||
3302 mLevelIdc != data[3]) {
3303 // COULD DO: set profile/level to the lowest required to support all SPSs
3304 ALOGE("Inconsistent profile/level found in seq parameter sets");
3305 return NULL;
3306 }
3307 }
3308 mSeqParamSets.push_back(paramSet);
3309 } else {
3310 mPicParamSets.push_back(paramSet);
3311 }
3312 return nextStartCode;
3313 }
3314
copyAVCCodecSpecificData(const uint8_t * data,size_t size)3315 status_t MPEG4Writer::Track::copyAVCCodecSpecificData(
3316 const uint8_t *data, size_t size) {
3317 ALOGV("copyAVCCodecSpecificData");
3318
3319 // 2 bytes for each of the parameter set length field
3320 // plus the 7 bytes for the header
3321 return copyCodecSpecificData(data, size, 4 + 7);
3322 }
3323
copyHEVCCodecSpecificData(const uint8_t * data,size_t size)3324 status_t MPEG4Writer::Track::copyHEVCCodecSpecificData(
3325 const uint8_t *data, size_t size) {
3326 ALOGV("copyHEVCCodecSpecificData");
3327
3328 // Min length of HEVC CSD is 23. (ISO/IEC 14496-15:2014 Chapter 8.3.3.1.2)
3329 return copyCodecSpecificData(data, size, 23);
3330 }
3331
copyCodecSpecificData(const uint8_t * data,size_t size,size_t minLength)3332 status_t MPEG4Writer::Track::copyCodecSpecificData(
3333 const uint8_t *data, size_t size, size_t minLength) {
3334 if (size < minLength) {
3335 ALOGE("Codec specific data length too short: %zu", size);
3336 return ERROR_MALFORMED;
3337 }
3338
3339 mCodecSpecificData = malloc(size);
3340 if (mCodecSpecificData == NULL) {
3341 ALOGE("Failed allocating codec specific data");
3342 return NO_MEMORY;
3343 }
3344 mCodecSpecificDataSize = size;
3345 memcpy(mCodecSpecificData, data, size);
3346 return OK;
3347 }
3348
parseAVCCodecSpecificData(const uint8_t * data,size_t size)3349 status_t MPEG4Writer::Track::parseAVCCodecSpecificData(
3350 const uint8_t *data, size_t size) {
3351
3352 ALOGV("parseAVCCodecSpecificData");
3353 // Data starts with a start code.
3354 // SPS and PPS are separated with start codes.
3355 // Also, SPS must come before PPS
3356 uint8_t type = kNalUnitTypeSeqParamSet;
3357 bool gotSps = false;
3358 bool gotPps = false;
3359 const uint8_t *tmp = data;
3360 const uint8_t *nextStartCode = data;
3361 size_t bytesLeft = size;
3362 size_t paramSetLen = 0;
3363 mCodecSpecificDataSize = 0;
3364 while (bytesLeft > 4 && !memcmp("\x00\x00\x00\x01", tmp, 4)) {
3365 getNalUnitType(*(tmp + 4), &type);
3366 if (type == kNalUnitTypeSeqParamSet) {
3367 if (gotPps) {
3368 ALOGE("SPS must come before PPS");
3369 return ERROR_MALFORMED;
3370 }
3371 if (!gotSps) {
3372 gotSps = true;
3373 }
3374 nextStartCode = parseParamSet(tmp + 4, bytesLeft - 4, type, ¶mSetLen);
3375 } else if (type == kNalUnitTypePicParamSet) {
3376 if (!gotSps) {
3377 ALOGE("SPS must come before PPS");
3378 return ERROR_MALFORMED;
3379 }
3380 if (!gotPps) {
3381 gotPps = true;
3382 }
3383 nextStartCode = parseParamSet(tmp + 4, bytesLeft - 4, type, ¶mSetLen);
3384 } else {
3385 ALOGE("Only SPS and PPS Nal units are expected");
3386 return ERROR_MALFORMED;
3387 }
3388
3389 if (nextStartCode == NULL) {
3390 ALOGE("nextStartCode is null");
3391 return ERROR_MALFORMED;
3392 }
3393
3394 // Move on to find the next parameter set
3395 bytesLeft -= nextStartCode - tmp;
3396 tmp = nextStartCode;
3397 mCodecSpecificDataSize += (2 + paramSetLen);
3398 }
3399
3400 {
3401 // Check on the number of seq parameter sets
3402 size_t nSeqParamSets = mSeqParamSets.size();
3403 if (nSeqParamSets == 0) {
3404 ALOGE("Cound not find sequence parameter set");
3405 return ERROR_MALFORMED;
3406 }
3407
3408 if (nSeqParamSets > 0x1F) {
3409 ALOGE("Too many seq parameter sets (%zu) found", nSeqParamSets);
3410 return ERROR_MALFORMED;
3411 }
3412 }
3413
3414 {
3415 // Check on the number of pic parameter sets
3416 size_t nPicParamSets = mPicParamSets.size();
3417 if (nPicParamSets == 0) {
3418 ALOGE("Cound not find picture parameter set");
3419 return ERROR_MALFORMED;
3420 }
3421 if (nPicParamSets > 0xFF) {
3422 ALOGE("Too many pic parameter sets (%zd) found", nPicParamSets);
3423 return ERROR_MALFORMED;
3424 }
3425 }
3426 // FIXME:
3427 // Add chromat_format_idc, bit depth values, etc for AVC/h264 high profile and above
3428 // and remove #if 0
3429 #if 0
3430 {
3431 // Check on the profiles
3432 // These profiles requires additional parameter set extensions
3433 if (mProfileIdc == 100 || mProfileIdc == 110 ||
3434 mProfileIdc == 122 || mProfileIdc == 144) {
3435 ALOGE("Sorry, no support for profile_idc: %d!", mProfileIdc);
3436 return BAD_VALUE;
3437 }
3438 }
3439 #endif
3440 return OK;
3441 }
3442
makeAVCCodecSpecificData(const uint8_t * data,size_t size)3443 status_t MPEG4Writer::Track::makeAVCCodecSpecificData(
3444 const uint8_t *data, size_t size) {
3445
3446 if (mCodecSpecificData != NULL) {
3447 ALOGE("Already have codec specific data");
3448 return ERROR_MALFORMED;
3449 }
3450
3451 if (size < 4) {
3452 ALOGE("Codec specific data length too short: %zu", size);
3453 return ERROR_MALFORMED;
3454 }
3455
3456 // Data is in the form of AVCCodecSpecificData
3457 if (memcmp("\x00\x00\x00\x01", data, 4)) {
3458 return copyAVCCodecSpecificData(data, size);
3459 }
3460
3461 if (parseAVCCodecSpecificData(data, size) != OK) {
3462 return ERROR_MALFORMED;
3463 }
3464
3465 // ISO 14496-15: AVC file format
3466 mCodecSpecificDataSize += 7; // 7 more bytes in the header
3467 mCodecSpecificData = malloc(mCodecSpecificDataSize);
3468 if (mCodecSpecificData == NULL) {
3469 mCodecSpecificDataSize = 0;
3470 ALOGE("Failed allocating codec specific data");
3471 return NO_MEMORY;
3472 }
3473 uint8_t *header = (uint8_t *)mCodecSpecificData;
3474 header[0] = 1; // version
3475 header[1] = mProfileIdc; // profile indication
3476 header[2] = mProfileCompatible; // profile compatibility
3477 header[3] = mLevelIdc;
3478
3479 // 6-bit '111111' followed by 2-bit to lengthSizeMinuusOne
3480 if (mOwner->useNalLengthFour()) {
3481 header[4] = 0xfc | 3; // length size == 4 bytes
3482 } else {
3483 header[4] = 0xfc | 1; // length size == 2 bytes
3484 }
3485
3486 // 3-bit '111' followed by 5-bit numSequenceParameterSets
3487 int nSequenceParamSets = mSeqParamSets.size();
3488 header[5] = 0xe0 | nSequenceParamSets;
3489 header += 6;
3490 for (List<AVCParamSet>::iterator it = mSeqParamSets.begin();
3491 it != mSeqParamSets.end(); ++it) {
3492 // 16-bit sequence parameter set length
3493 uint16_t seqParamSetLength = it->mLength;
3494 header[0] = seqParamSetLength >> 8;
3495 header[1] = seqParamSetLength & 0xff;
3496
3497 // SPS NAL unit (sequence parameter length bytes)
3498 memcpy(&header[2], it->mData, seqParamSetLength);
3499 header += (2 + seqParamSetLength);
3500 }
3501
3502 // 8-bit nPictureParameterSets
3503 int nPictureParamSets = mPicParamSets.size();
3504 header[0] = nPictureParamSets;
3505 header += 1;
3506 for (List<AVCParamSet>::iterator it = mPicParamSets.begin();
3507 it != mPicParamSets.end(); ++it) {
3508 // 16-bit picture parameter set length
3509 uint16_t picParamSetLength = it->mLength;
3510 header[0] = picParamSetLength >> 8;
3511 header[1] = picParamSetLength & 0xff;
3512
3513 // PPS Nal unit (picture parameter set length bytes)
3514 memcpy(&header[2], it->mData, picParamSetLength);
3515 header += (2 + picParamSetLength);
3516 }
3517
3518 return OK;
3519 }
3520
3521
parseHEVCCodecSpecificData(const uint8_t * data,size_t size,HevcParameterSets & paramSets)3522 status_t MPEG4Writer::Track::parseHEVCCodecSpecificData(
3523 const uint8_t *data, size_t size, HevcParameterSets ¶mSets) {
3524
3525 ALOGV("parseHEVCCodecSpecificData");
3526 const uint8_t *tmp = data;
3527 const uint8_t *nextStartCode = data;
3528 size_t bytesLeft = size;
3529 while (bytesLeft > 4 && !memcmp("\x00\x00\x00\x01", tmp, 4)) {
3530 nextStartCode = findNextNalStartCode(tmp + 4, bytesLeft - 4);
3531 status_t err = paramSets.addNalUnit(tmp + 4, (nextStartCode - tmp) - 4);
3532 if (err != OK) {
3533 return ERROR_MALFORMED;
3534 }
3535
3536 // Move on to find the next parameter set
3537 bytesLeft -= nextStartCode - tmp;
3538 tmp = nextStartCode;
3539 }
3540
3541 size_t csdSize = 23;
3542 const size_t numNalUnits = paramSets.getNumNalUnits();
3543 for (size_t i = 0; i < ARRAY_SIZE(kMandatoryHevcNalUnitTypes); ++i) {
3544 int type = kMandatoryHevcNalUnitTypes[i];
3545 size_t numParamSets = paramSets.getNumNalUnitsOfType(type);
3546 if (numParamSets == 0) {
3547 ALOGE("Cound not find NAL unit of type %d", type);
3548 return ERROR_MALFORMED;
3549 }
3550 }
3551 for (size_t i = 0; i < ARRAY_SIZE(kHevcNalUnitTypes); ++i) {
3552 int type = kHevcNalUnitTypes[i];
3553 size_t numParamSets = paramSets.getNumNalUnitsOfType(type);
3554 if (numParamSets > 0xffff) {
3555 ALOGE("Too many seq parameter sets (%zu) found", numParamSets);
3556 return ERROR_MALFORMED;
3557 }
3558 csdSize += 3;
3559 for (size_t j = 0; j < numNalUnits; ++j) {
3560 if (paramSets.getType(j) != type) {
3561 continue;
3562 }
3563 csdSize += 2 + paramSets.getSize(j);
3564 }
3565 }
3566 mCodecSpecificDataSize = csdSize;
3567 return OK;
3568 }
3569
makeHEVCCodecSpecificData(const uint8_t * data,size_t size)3570 status_t MPEG4Writer::Track::makeHEVCCodecSpecificData(
3571 const uint8_t *data, size_t size) {
3572
3573 if (mCodecSpecificData != NULL) {
3574 ALOGE("Already have codec specific data");
3575 return ERROR_MALFORMED;
3576 }
3577
3578 if (size < 4) {
3579 ALOGE("Codec specific data length too short: %zu", size);
3580 return ERROR_MALFORMED;
3581 }
3582
3583 // Data is in the form of HEVCCodecSpecificData
3584 if (memcmp("\x00\x00\x00\x01", data, 4)) {
3585 return copyHEVCCodecSpecificData(data, size);
3586 }
3587
3588 HevcParameterSets paramSets;
3589 if (parseHEVCCodecSpecificData(data, size, paramSets) != OK) {
3590 ALOGE("failed parsing codec specific data");
3591 return ERROR_MALFORMED;
3592 }
3593
3594 mCodecSpecificData = malloc(mCodecSpecificDataSize);
3595 if (mCodecSpecificData == NULL) {
3596 mCodecSpecificDataSize = 0;
3597 ALOGE("Failed allocating codec specific data");
3598 return NO_MEMORY;
3599 }
3600 status_t err = paramSets.makeHvcc((uint8_t *)mCodecSpecificData,
3601 &mCodecSpecificDataSize, mOwner->useNalLengthFour() ? 4 : 2);
3602 if (err != OK) {
3603 ALOGE("failed constructing HVCC atom");
3604 return err;
3605 }
3606
3607 return OK;
3608 }
3609
getDolbyVisionProfile()3610 status_t MPEG4Writer::Track::getDolbyVisionProfile() {
3611 uint32_t type;
3612 const void *data = NULL;
3613 size_t size = 0;
3614
3615 if (!mMeta->findData(kKeyDVCC, &type, &data, &size) &&
3616 !mMeta->findData(kKeyDVVC, &type, &data, &size) &&
3617 !mMeta->findData(kKeyDVWC, &type, &data, &size)) {
3618 ALOGE("Failed getting Dovi config for Dolby Vision %d", (int)size);
3619 return ERROR_MALFORMED;
3620 }
3621 static const ALookup<uint8_t, int32_t> dolbyVisionProfileMap = {
3622 {1, DolbyVisionProfileDvavPen},
3623 {3, DolbyVisionProfileDvheDen},
3624 {4, DolbyVisionProfileDvheDtr},
3625 {5, DolbyVisionProfileDvheStn},
3626 {6, DolbyVisionProfileDvheDth},
3627 {7, DolbyVisionProfileDvheDtb},
3628 {8, DolbyVisionProfileDvheSt},
3629 {9, DolbyVisionProfileDvavSe},
3630 {10, DolbyVisionProfileDvav110}
3631 };
3632
3633 // Dolby Vision profile information is extracted as per
3634 // https://dolby.my.salesforce.com/sfc/p/#700000009YuG/a/4u000000l6FB/076wHYEmyEfz09m0V1bo85_25hlUJjaiWTbzorNmYY4
3635 uint8_t dv_profile = ((((uint8_t *)data)[2] >> 1) & 0x7f);
3636
3637 if (!dolbyVisionProfileMap.map(dv_profile, &mDoviProfile)) {
3638 ALOGE("Failed to get Dolby Profile from DV Config data");
3639 return ERROR_MALFORMED;
3640 }
3641 return OK;
3642 }
3643
3644 /*
3645 * Updates the drift time from the audio track so that
3646 * the video track can get the updated drift time information
3647 * from the file writer. The fluctuation of the drift time of the audio
3648 * encoding path is smoothed out with a simple filter by giving a larger
3649 * weight to more recently drift time. The filter coefficients, 0.5 and 0.5,
3650 * are heuristically determined.
3651 */
updateDriftTime(const sp<MetaData> & meta)3652 void MPEG4Writer::Track::updateDriftTime(const sp<MetaData>& meta) {
3653 int64_t driftTimeUs = 0;
3654 if (meta->findInt64(kKeyDriftTime, &driftTimeUs)) {
3655 int64_t prevDriftTimeUs = mOwner->getDriftTimeUs();
3656 int64_t timeUs = (driftTimeUs + prevDriftTimeUs) >> 1;
3657 mOwner->setDriftTimeUs(timeUs);
3658 }
3659 }
3660
dumpTimeStamps()3661 void MPEG4Writer::Track::dumpTimeStamps() {
3662 if (!mTimestampDebugHelper.empty()) {
3663 std::string timeStampString = "Dumping " + std::string(getTrackType()) + " track's last " +
3664 std::to_string(mTimestampDebugHelper.size()) +
3665 " frames' timestamps(pts, dts) and frame type : ";
3666 for (const TimestampDebugHelperEntry& entry : mTimestampDebugHelper) {
3667 timeStampString += "\n(" + std::to_string(entry.pts) + "us, " +
3668 std::to_string(entry.dts) + "us " + entry.frameType + ") ";
3669 }
3670 ALOGE("%s", timeStampString.c_str());
3671 } else {
3672 ALOGE("0 frames to dump timeStamps in %s track ", getTrackType());
3673 }
3674 }
3675
threadEntry()3676 status_t MPEG4Writer::Track::threadEntry() {
3677 int32_t count = 0;
3678 const int64_t interleaveDurationUs = mOwner->interleaveDuration();
3679 const bool hasMultipleTracks = (mOwner->numTracks() > 1);
3680 int64_t chunkTimestampUs = 0;
3681 int32_t nChunks = 0;
3682 int32_t nActualFrames = 0; // frames containing non-CSD data (non-0 length)
3683 int32_t nZeroLengthFrames = 0;
3684 int64_t lastTimestampUs = 0; // Previous sample time stamp
3685 int64_t previousSampleTimestampWithoutFudgeUs = 0; // Timestamp received/without fudge for STTS
3686 int64_t lastDurationUs = 0; // Between the previous two samples
3687 int64_t currDurationTicks = 0; // Timescale based ticks
3688 int64_t lastDurationTicks = 0; // Timescale based ticks
3689 int32_t sampleCount = 1; // Sample count in the current stts table entry
3690 uint32_t previousSampleSize = 0; // Size of the previous sample
3691 int64_t previousPausedDurationUs = 0;
3692 int64_t timestampUs = 0;
3693 int64_t cttsOffsetTimeUs = 0;
3694 int64_t currCttsOffsetTimeTicks = 0; // Timescale based ticks
3695 int64_t lastCttsOffsetTimeTicks = -1; // Timescale based ticks
3696 int32_t cttsSampleCount = 0; // Sample count in the current ctts table entry
3697 uint32_t lastSamplesPerChunk = 0;
3698 int64_t lastSampleDurationUs = -1; // Duration calculated from EOS buffer and its timestamp
3699 int64_t lastSampleDurationTicks = -1; // Timescale based ticks
3700 int64_t sampleFileOffset = -1;
3701
3702 if (mIsAudio) {
3703 prctl(PR_SET_NAME, (unsigned long)"MP4WtrAudTrkThread", 0, 0, 0);
3704 } else if (mIsVideo) {
3705 prctl(PR_SET_NAME, (unsigned long)"MP4WtrVidTrkThread", 0, 0, 0);
3706 } else {
3707 prctl(PR_SET_NAME, (unsigned long)"MP4WtrMetaTrkThread", 0, 0, 0);
3708 }
3709
3710 if (mOwner->isRealTimeRecording()) {
3711 androidSetThreadPriority(0, ANDROID_PRIORITY_AUDIO);
3712 } else if (mOwner->isBackgroundMode()) {
3713 // Background priority for media transcoding.
3714 androidSetThreadPriority(0 /* tid (0 = current) */, ANDROID_PRIORITY_BACKGROUND);
3715 }
3716
3717 sp<MetaData> meta_data;
3718
3719 status_t err = OK;
3720 MediaBufferBase *buffer;
3721 const char *trackName = getTrackType();
3722 while (!mDone && (err = mSource->read(&buffer)) == OK) {
3723 ALOGV("read:buffer->range_length:%lld", (long long)buffer->range_length());
3724 int32_t isEOS = false;
3725 if (buffer->range_length() == 0) {
3726 if (buffer->meta_data().findInt32(kKeyIsEndOfStream, &isEOS) && isEOS) {
3727 int64_t eosSampleTimestampUs = -1;
3728 CHECK(buffer->meta_data().findInt64(kKeyTime, &eosSampleTimestampUs));
3729 if (eosSampleTimestampUs > 0) {
3730 lastSampleDurationUs = eosSampleTimestampUs -
3731 previousSampleTimestampWithoutFudgeUs -
3732 previousPausedDurationUs;
3733 if (lastSampleDurationUs >= 0) {
3734 lastSampleDurationTicks = (lastSampleDurationUs * mTimeScale + 500000LL) /
3735 1000000LL;
3736 } else {
3737 ALOGW("lastSampleDurationUs %" PRId64 " is negative", lastSampleDurationUs);
3738 }
3739 }
3740 buffer->release();
3741 buffer = nullptr;
3742 mSource->stop();
3743 break;
3744 } else {
3745 buffer->release();
3746 buffer = nullptr;
3747 ++nZeroLengthFrames;
3748 continue;
3749 }
3750 }
3751
3752 // If the codec specific data has not been received yet, delay pause.
3753 // After the codec specific data is received, discard what we received
3754 // when the track is to be paused.
3755 if (mPaused && !mResumed) {
3756 buffer->release();
3757 buffer = NULL;
3758 continue;
3759 }
3760
3761 ++count;
3762
3763 int32_t isCodecConfig;
3764 if (buffer->meta_data().findInt32(kKeyIsCodecConfig, &isCodecConfig)
3765 && isCodecConfig) {
3766 // if config format (at track addition) already had CSD, keep that
3767 // UNLESS we have not received any frames yet.
3768 // TODO: for now the entire CSD has to come in one frame for encoders, even though
3769 // they need to be spread out for decoders.
3770 if (mGotAllCodecSpecificData && nActualFrames > 0) {
3771 ALOGI("ignoring additional CSD for video track after first frame");
3772 } else {
3773 mMeta = mSource->getFormat(); // get output format after format change
3774 status_t err;
3775 if (mIsAvc) {
3776 err = makeAVCCodecSpecificData(
3777 (const uint8_t *)buffer->data()
3778 + buffer->range_offset(),
3779 buffer->range_length());
3780 } else if (mIsHevc || mIsHeic) {
3781 err = makeHEVCCodecSpecificData(
3782 (const uint8_t *)buffer->data()
3783 + buffer->range_offset(),
3784 buffer->range_length());
3785 } else if (mIsMPEG4 || mIsAv1 || mIsApv) {
3786 err = copyCodecSpecificData((const uint8_t *)buffer->data() + buffer->range_offset(),
3787 buffer->range_length());
3788 }
3789 if (mIsDovi) {
3790 err = getDolbyVisionProfile();
3791 if(err == OK) {
3792 const void *data = NULL;
3793 size_t size = 0;
3794 uint32_t type = 0;
3795 if (mDoviProfile == DolbyVisionProfileDvavSe) {
3796 mMeta->findData(kKeyAVCC, &type, &data, &size);
3797 } else if (mDoviProfile < DolbyVisionProfileDvavSe) {
3798 mMeta->findData(kKeyHVCC, &type, &data, &size);
3799 } else {
3800 ALOGW("DV Profiles > DolbyVisionProfileDvavSe are not supported");
3801 err = ERROR_MALFORMED;
3802 }
3803 if (err == OK && data != NULL &&
3804 copyCodecSpecificData((uint8_t *)data, size) == OK) {
3805 mGotAllCodecSpecificData = true;
3806 }
3807 }
3808 }
3809 }
3810 buffer->release();
3811 buffer = NULL;
3812 if (OK != err) {
3813 mSource->stop();
3814 mIsMalformed = true;
3815 uint32_t trackNum = (mTrackId.getId() << 28);
3816 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_ERROR,
3817 trackNum | MEDIA_RECORDER_TRACK_ERROR_GENERAL, err);
3818 break;
3819 }
3820
3821 mGotAllCodecSpecificData = true;
3822 continue;
3823 }
3824
3825 // Per-frame metadata sample's size must be smaller than max allowed.
3826 if (!mIsVideo && !mIsAudio && !mIsHeif &&
3827 buffer->range_length() >= kMaxMetadataSize) {
3828 ALOGW("Buffer size is %zu. Maximum metadata buffer size is %lld for %s track",
3829 buffer->range_length(), (long long)kMaxMetadataSize, trackName);
3830 buffer->release();
3831 mSource->stop();
3832 mIsMalformed = true;
3833 break;
3834 }
3835
3836 bool isGainmapMeta = false;
3837 bool isGainmap = false;
3838 bool isExif = false;
3839 uint32_t tiffHdrOffset = 0;
3840 uint32_t gainmapOffset = 0;
3841 int32_t isMuxerData;
3842 if (buffer->meta_data().findInt32(kKeyIsMuxerData, &isMuxerData) && isMuxerData) {
3843 if (flags_camera::camera_heif_gainmap()) {
3844 isGainmapMeta = isGainmapMetaData(buffer, &gainmapOffset);
3845 isGainmap = isGainmapData(buffer, &gainmapOffset);
3846 if ((isGainmap || isGainmapMeta) && (gainmapOffset > 0) &&
3847 (gainmapOffset < buffer->range_length())) {
3848 // Don't include the tmap/gmap header
3849 buffer->set_range(gainmapOffset, buffer->range_length() - gainmapOffset);
3850 }
3851 }
3852 isExif = isExifData(buffer, &tiffHdrOffset);
3853 if (!isExif && !isGainmap && !isGainmapMeta) {
3854 ALOGW("Ignoring bad muxer data block");
3855 buffer->release();
3856 buffer = NULL;
3857 continue;
3858 }
3859 }
3860 if (flags_camera::camera_heif_gainmap()) {
3861 int32_t val32;
3862 if (buffer->meta_data().findInt32(kKeyColorPrimaries, &val32)) {
3863 mColorAspects.mPrimaries = static_cast<ColorAspects::Primaries>(val32);
3864 mColorAspectsValid = true;
3865 } else {
3866 mColorAspectsValid = false;
3867 }
3868 if (buffer->meta_data().findInt32(kKeyTransferFunction, &val32)) {
3869 mColorAspects.mTransfer = static_cast<ColorAspects::Transfer>(val32);
3870 } else {
3871 mColorAspectsValid = false;
3872 }
3873 if (buffer->meta_data().findInt32(kKeyColorMatrix, &val32)) {
3874 mColorAspects.mMatrixCoeffs = static_cast<ColorAspects::MatrixCoeffs>(val32);
3875 } else {
3876 mColorAspectsValid = false;
3877 }
3878 if (buffer->meta_data().findInt32(kKeyColorRange, &val32)) {
3879 mColorAspects.mRange = static_cast<ColorAspects::Range>(val32);
3880 } else {
3881 mColorAspectsValid = false;
3882 }
3883 if (mBitsPerChannel.empty() && buffer->meta_data().findInt32(kKeyColorFormat, &val32)) {
3884 switch (val32) {
3885 case COLOR_FormatYUV420Flexible:
3886 case COLOR_FormatYUV420Planar:
3887 case COLOR_FormatYUV420SemiPlanar: {
3888 uint8_t bitsPerChannel[] = {8, 8, 8};
3889 mBitsPerChannel.appendArray(bitsPerChannel, sizeof(bitsPerChannel));
3890 }
3891 break;
3892 default:
3893 break;
3894 }
3895 }
3896 }
3897
3898 if (!buffer->meta_data().findInt64(kKeySampleFileOffset, &sampleFileOffset)) {
3899 sampleFileOffset = -1;
3900 }
3901 int64_t lastSample = -1;
3902 if (!buffer->meta_data().findInt64(kKeyLastSampleIndexInChunk, &lastSample)) {
3903 lastSample = -1;
3904 }
3905 ALOGV("sampleFileOffset:%lld", (long long)sampleFileOffset);
3906
3907 /*
3908 * Reserve space in the file for the current sample + to be written MOOV box. If reservation
3909 * for a new sample fails, preAllocate(...) stops muxing session completely. Stop() could
3910 * write MOOV box successfully as space for the same was reserved in the prior call.
3911 * Release the current buffer/sample here.
3912 */
3913 if (sampleFileOffset == -1 && !mOwner->preAllocate(buffer->range_length())) {
3914 buffer->release();
3915 buffer = nullptr;
3916 break;
3917 }
3918
3919 ++nActualFrames;
3920
3921 // Make a deep copy of the MediaBuffer and Metadata and release
3922 // the original as soon as we can
3923 MediaBuffer* copy = new MediaBuffer(buffer->range_length());
3924 if (sampleFileOffset != -1) {
3925 copy->meta_data().setInt64(kKeySampleFileOffset, sampleFileOffset);
3926 } else {
3927 memcpy(copy->data(), (uint8_t*)buffer->data() + buffer->range_offset(),
3928 buffer->range_length());
3929 }
3930 copy->set_range(0, buffer->range_length());
3931
3932 meta_data = new MetaData(buffer->meta_data());
3933 buffer->release();
3934 buffer = NULL;
3935 if (isExif) {
3936 copy->meta_data().setInt32(kKeyExifTiffOffset, tiffHdrOffset);
3937 }
3938 bool usePrefix = this->usePrefix() && !isExif && !isGainmapMeta;
3939 if (sampleFileOffset == -1 && usePrefix) {
3940 StripStartcode(copy);
3941 }
3942 size_t sampleSize = copy->range_length();
3943 if (sampleFileOffset == -1 && usePrefix) {
3944 if (mOwner->useNalLengthFour()) {
3945 ALOGV("nallength4");
3946 sampleSize += 4;
3947 } else {
3948 ALOGV("nallength2");
3949 sampleSize += 2;
3950 }
3951 }
3952
3953 // Max file size or duration handling
3954 mMdatSizeBytes += sampleSize;
3955 updateTrackSizeEstimate();
3956
3957 if (mOwner->exceedsFileSizeLimit()) {
3958 copy->release();
3959 if (mOwner->switchFd() != OK) {
3960 ALOGW("Recorded file size exceeds limit %" PRId64 "bytes",
3961 mOwner->mMaxFileSizeLimitBytes);
3962 mSource->stop();
3963 mOwner->notify(
3964 MEDIA_RECORDER_EVENT_INFO, MEDIA_RECORDER_INFO_MAX_FILESIZE_REACHED, 0);
3965 } else {
3966 ALOGV("%s Current recorded file size exceeds limit %" PRId64 "bytes. Switching output",
3967 getTrackType(), mOwner->mMaxFileSizeLimitBytes);
3968 }
3969 break;
3970 }
3971
3972 if (mOwner->exceedsFileDurationLimit()) {
3973 ALOGW("Recorded file duration exceeds limit %" PRId64 "microseconds",
3974 mOwner->mMaxFileDurationLimitUs);
3975 copy->release();
3976 mSource->stop();
3977 mOwner->notify(MEDIA_RECORDER_EVENT_INFO, MEDIA_RECORDER_INFO_MAX_DURATION_REACHED, 0);
3978 break;
3979 }
3980
3981 if (mOwner->approachingFileSizeLimit()) {
3982 mOwner->notifyApproachingLimit();
3983 }
3984 int32_t isSync = false;
3985 meta_data->findInt32(kKeyIsSyncFrame, &isSync);
3986 CHECK(meta_data->findInt64(kKeyTime, ×tampUs));
3987 timestampUs += mFirstSampleStartOffsetUs;
3988
3989 // For video, skip the first several non-key frames until getting the first key frame.
3990 if (mIsVideo && !mGotStartKeyFrame && !isSync) {
3991 ALOGD("Video skip non-key frame");
3992 copy->release();
3993 continue;
3994 }
3995 if (mIsVideo && isSync) {
3996 mGotStartKeyFrame = true;
3997 }
3998 ////////////////////////////////////////////////////////////////////////////////
3999 if (!mIsHeif) {
4000 if (mStszTableEntries->count() == 0) {
4001 mFirstSampleTimeRealUs = systemTime() / 1000;
4002 if (timestampUs < 0 && mFirstSampleStartOffsetUs == 0) {
4003 if (WARN_UNLESS(timestampUs != INT64_MIN, "for %s track", trackName)) {
4004 copy->release();
4005 mSource->stop();
4006 mIsMalformed = true;
4007 break;
4008 }
4009 mFirstSampleStartOffsetUs = -timestampUs;
4010 timestampUs = 0;
4011 }
4012 mOwner->setStartTimestampUs(timestampUs);
4013 mStartTimestampUs = timestampUs;
4014 previousPausedDurationUs = mStartTimestampUs;
4015 }
4016
4017 if (mResumed) {
4018 int64_t durExcludingEarlierPausesUs = timestampUs - previousPausedDurationUs;
4019 if (WARN_UNLESS(durExcludingEarlierPausesUs >= 0LL, "for %s track", trackName)) {
4020 copy->release();
4021 mSource->stop();
4022 mIsMalformed = true;
4023 break;
4024 }
4025
4026 int64_t pausedDurationUs = durExcludingEarlierPausesUs - mTrackDurationUs;
4027 if (WARN_UNLESS(pausedDurationUs >= lastDurationUs, "for %s track", trackName)) {
4028 copy->release();
4029 mSource->stop();
4030 mIsMalformed = true;
4031 break;
4032 }
4033
4034 previousPausedDurationUs += pausedDurationUs - lastDurationUs;
4035 mResumed = false;
4036 }
4037 TimestampDebugHelperEntry timestampDebugEntry;
4038 timestampUs -= previousPausedDurationUs;
4039 timestampDebugEntry.pts = timestampUs;
4040 if (WARN_UNLESS(timestampUs >= 0LL, "for %s track", trackName)) {
4041 copy->release();
4042 mSource->stop();
4043 mIsMalformed = true;
4044 break;
4045 }
4046
4047 if (mIsVideo) {
4048 /*
4049 * Composition time: timestampUs
4050 * Decoding time: decodingTimeUs
4051 * Composition time offset = composition time - decoding time
4052 */
4053 int64_t decodingTimeUs;
4054 CHECK(meta_data->findInt64(kKeyDecodingTime, &decodingTimeUs));
4055 decodingTimeUs -= previousPausedDurationUs;
4056
4057 // ensure non-negative, monotonic decoding time
4058 if (mLastDecodingTimeUs < 0) {
4059 decodingTimeUs = std::max((int64_t)0, decodingTimeUs);
4060 } else {
4061 // increase decoding time by at least the larger vaule of 1 tick and
4062 // 0.1 milliseconds. This needs to take into account the possible
4063 // delta adjustment in DurationTicks in below.
4064 decodingTimeUs = std::max(mLastDecodingTimeUs +
4065 std::max(100, divUp(1000000, mTimeScale)), decodingTimeUs);
4066 }
4067
4068 mLastDecodingTimeUs = decodingTimeUs;
4069 timestampDebugEntry.dts = decodingTimeUs;
4070 timestampDebugEntry.frameType = isSync ? "Key frame" : "Non-Key frame";
4071 // Insert the timestamp into the mTimestampDebugHelper
4072 if (mTimestampDebugHelper.size() >= kTimestampDebugCount) {
4073 mTimestampDebugHelper.pop_front();
4074 }
4075 mTimestampDebugHelper.push_back(timestampDebugEntry);
4076
4077 cttsOffsetTimeUs =
4078 timestampUs + kMaxCttsOffsetTimeUs - decodingTimeUs;
4079 if (WARN_UNLESS(cttsOffsetTimeUs >= 0LL, "for %s track", trackName)) {
4080 copy->release();
4081 mSource->stop();
4082 mIsMalformed = true;
4083 break;
4084 }
4085
4086 timestampUs = decodingTimeUs;
4087 ALOGV("decoding time: %" PRId64 " and ctts offset time: %" PRId64,
4088 timestampUs, cttsOffsetTimeUs);
4089
4090 // Update ctts box table if necessary
4091 currCttsOffsetTimeTicks =
4092 (cttsOffsetTimeUs * mTimeScale + 500000LL) / 1000000LL;
4093 if (WARN_UNLESS(currCttsOffsetTimeTicks <= 0x0FFFFFFFFLL, "for %s track", trackName)) {
4094 copy->release();
4095 mSource->stop();
4096 mIsMalformed = true;
4097 break;
4098 }
4099
4100 if (mStszTableEntries->count() == 0) {
4101 // Force the first ctts table entry to have one single entry
4102 // so that we can do adjustment for the initial track start
4103 // time offset easily in writeCttsBox().
4104 lastCttsOffsetTimeTicks = currCttsOffsetTimeTicks;
4105 addOneCttsTableEntry(1, currCttsOffsetTimeTicks);
4106 cttsSampleCount = 0; // No sample in ctts box is pending
4107 } else {
4108 if (currCttsOffsetTimeTicks != lastCttsOffsetTimeTicks) {
4109 addOneCttsTableEntry(cttsSampleCount, lastCttsOffsetTimeTicks);
4110 lastCttsOffsetTimeTicks = currCttsOffsetTimeTicks;
4111 cttsSampleCount = 1; // One sample in ctts box is pending
4112 } else {
4113 ++cttsSampleCount;
4114 }
4115 }
4116
4117 // Update ctts time offset range
4118 if (mStszTableEntries->count() == 0) {
4119 mMinCttsOffsetTicks = currCttsOffsetTimeTicks;
4120 mMaxCttsOffsetTicks = currCttsOffsetTimeTicks;
4121 } else {
4122 if (currCttsOffsetTimeTicks > mMaxCttsOffsetTicks) {
4123 mMaxCttsOffsetTicks = currCttsOffsetTimeTicks;
4124 } else if (currCttsOffsetTimeTicks < mMinCttsOffsetTicks) {
4125 mMinCttsOffsetTicks = currCttsOffsetTimeTicks;
4126 mMinCttsOffsetTimeUs = cttsOffsetTimeUs;
4127 }
4128 }
4129 }
4130
4131 if (mOwner->isRealTimeRecording()) {
4132 if (mIsAudio) {
4133 updateDriftTime(meta_data);
4134 }
4135 }
4136
4137 if (WARN_UNLESS(timestampUs >= 0LL, "for %s track", trackName)) {
4138 copy->release();
4139 mSource->stop();
4140 mIsMalformed = true;
4141 break;
4142 }
4143
4144 ALOGV("%s media time stamp: %" PRId64 " and previous paused duration %" PRId64,
4145 trackName, timestampUs, previousPausedDurationUs);
4146 if (timestampUs > mTrackDurationUs) {
4147 mTrackDurationUs = timestampUs;
4148 }
4149
4150 // We need to use the time scale based ticks, rather than the
4151 // timestamp itself to determine whether we have to use a new
4152 // stts entry, since we may have rounding errors.
4153 // The calculation is intended to reduce the accumulated
4154 // rounding errors.
4155 currDurationTicks =
4156 ((timestampUs * mTimeScale + 500000LL) / 1000000LL -
4157 (lastTimestampUs * mTimeScale + 500000LL) / 1000000LL);
4158 if (currDurationTicks < 0LL) {
4159 ALOGE("do not support out of order frames (timestamp: %lld < last: %lld for %s track",
4160 (long long)timestampUs, (long long)lastTimestampUs, trackName);
4161 copy->release();
4162 mSource->stop();
4163 mIsMalformed = true;
4164 break;
4165 }
4166
4167 previousSampleTimestampWithoutFudgeUs = timestampUs;
4168
4169 // if the duration is different for this sample, see if it is close enough to the previous
4170 // duration that we can fudge it and use the same value, to avoid filling the stts table
4171 // with lots of near-identical entries.
4172 // "close enough" here means that the current duration needs to be adjusted by less
4173 // than 0.1 milliseconds
4174 if (lastDurationTicks && (currDurationTicks != lastDurationTicks)) {
4175 int64_t deltaUs = ((lastDurationTicks - currDurationTicks) * 1000000LL
4176 + (mTimeScale / 2)) / mTimeScale;
4177 if (deltaUs > -100 && deltaUs < 100) {
4178 // use previous ticks, and adjust timestamp as if it was actually that number
4179 // of ticks
4180 currDurationTicks = lastDurationTicks;
4181 timestampUs += deltaUs;
4182 }
4183 }
4184 mStszTableEntries->add(htonl(sampleSize));
4185
4186 if (mStszTableEntries->count() > 2) {
4187
4188 // Force the first sample to have its own stts entry so that
4189 // we can adjust its value later to maintain the A/V sync.
4190 if (lastDurationTicks && currDurationTicks != lastDurationTicks) {
4191 addOneSttsTableEntry(sampleCount, lastDurationTicks);
4192 sampleCount = 1;
4193 } else {
4194 ++sampleCount;
4195 }
4196 }
4197 if (mSamplesHaveSameSize) {
4198 if (mStszTableEntries->count() >= 2 && previousSampleSize != sampleSize) {
4199 mSamplesHaveSameSize = false;
4200 }
4201 previousSampleSize = sampleSize;
4202 }
4203 ALOGV("%s timestampUs/lastTimestampUs: %" PRId64 "/%" PRId64,
4204 trackName, timestampUs, lastTimestampUs);
4205 lastDurationUs = timestampUs - lastTimestampUs;
4206 lastDurationTicks = currDurationTicks;
4207 lastTimestampUs = timestampUs;
4208
4209 if (isSync != 0) {
4210 addOneStssTableEntry(mStszTableEntries->count());
4211 }
4212
4213 if (mTrackingProgressStatus) {
4214 if (mPreviousTrackTimeUs <= 0) {
4215 mPreviousTrackTimeUs = mStartTimestampUs;
4216 }
4217 trackProgressStatus(timestampUs);
4218 }
4219 }
4220
4221 if (flags_camera::camera_heif_gainmap() && mOwner->mHasGainmap) {
4222 Mutex::Autolock lock(mOwner->mLock);
4223 size_t bytesWritten;
4224 off64_t offset = mOwner->addSample_l(copy, usePrefix, tiffHdrOffset, &bytesWritten);
4225 addItemOffsetAndSize(offset, bytesWritten, isExif, isGainmapMeta, isGainmap);
4226 copy->release();
4227 copy = NULL;
4228 continue;
4229 }
4230
4231 if (!hasMultipleTracks) {
4232 size_t bytesWritten;
4233 off64_t offset = mOwner->addSample_l(
4234 copy, usePrefix, tiffHdrOffset, &bytesWritten);
4235 if (mIsHeif) {
4236 addItemOffsetAndSize(offset, bytesWritten, isExif);
4237 } else {
4238 if (mCo64TableEntries->count() == 0) {
4239 addChunkOffset(offset);
4240 }
4241 }
4242 copy->release();
4243 copy = NULL;
4244 continue;
4245 }
4246
4247 mChunkSamples.push_back(copy);
4248 if (mIsHeif) {
4249 bufferChunk(0 /*timestampUs*/);
4250 ++nChunks;
4251 } else if (interleaveDurationUs == 0) {
4252 addOneStscTableEntry(++nChunks, 1);
4253 bufferChunk(timestampUs);
4254 } else {
4255 if (chunkTimestampUs == 0) {
4256 chunkTimestampUs = timestampUs;
4257 } else {
4258 int64_t chunkDurationUs = timestampUs - chunkTimestampUs;
4259 if (chunkDurationUs > interleaveDurationUs || lastSample > 1) {
4260 ALOGV("lastSample:%lld", (long long)lastSample);
4261 if (chunkDurationUs > mMaxChunkDurationUs) {
4262 mMaxChunkDurationUs = chunkDurationUs;
4263 }
4264 ++nChunks;
4265 if (nChunks == 1 || // First chunk
4266 lastSamplesPerChunk != mChunkSamples.size()) {
4267 lastSamplesPerChunk = mChunkSamples.size();
4268 addOneStscTableEntry(nChunks, lastSamplesPerChunk);
4269 }
4270 bufferChunk(timestampUs);
4271 chunkTimestampUs = timestampUs;
4272 }
4273 }
4274 }
4275 }
4276
4277 if (isTrackMalFormed()) {
4278 dumpTimeStamps();
4279 err = ERROR_MALFORMED;
4280 }
4281
4282 mOwner->trackProgressStatus(mTrackId.getId(), -1, err);
4283
4284 // Add final entries only for non-empty tracks.
4285 if (mStszTableEntries->count() > 0) {
4286 if (mIsHeif) {
4287 if (!mChunkSamples.empty()) {
4288 bufferChunk(0);
4289 ++nChunks;
4290 }
4291 } else {
4292 // Last chunk
4293 if (!hasMultipleTracks) {
4294 addOneStscTableEntry(1, mStszTableEntries->count());
4295 } else if (!mChunkSamples.empty()) {
4296 addOneStscTableEntry(++nChunks, mChunkSamples.size());
4297 bufferChunk(timestampUs);
4298 }
4299
4300 // We don't really know how long the last frame lasts, since
4301 // there is no frame time after it, just repeat the previous
4302 // frame's duration.
4303 if (mStszTableEntries->count() == 1) {
4304 if (lastSampleDurationUs >= 0) {
4305 addOneSttsTableEntry(sampleCount, lastSampleDurationTicks);
4306 } else {
4307 lastDurationUs = 0; // A single sample's duration
4308 lastDurationTicks = 0;
4309 addOneSttsTableEntry(sampleCount, lastDurationTicks);
4310 }
4311 } else if (lastSampleDurationUs >= 0) {
4312 addOneSttsTableEntry(sampleCount, lastDurationTicks);
4313 addOneSttsTableEntry(1, lastSampleDurationTicks);
4314 } else {
4315 ++sampleCount; // Count for the last sample
4316 addOneSttsTableEntry(sampleCount, lastDurationTicks);
4317 }
4318
4319 // The last ctts box entry may not have been written yet, and this
4320 // is to make sure that we write out the last ctts box entry.
4321 if (currCttsOffsetTimeTicks == lastCttsOffsetTimeTicks) {
4322 if (cttsSampleCount > 0) {
4323 addOneCttsTableEntry(cttsSampleCount, lastCttsOffsetTimeTicks);
4324 }
4325 }
4326 if (lastSampleDurationUs >= 0) {
4327 mTrackDurationUs += lastSampleDurationUs;
4328 } else {
4329 mTrackDurationUs += lastDurationUs;
4330 }
4331 }
4332 }
4333 mReachedEOS = true;
4334
4335 sendTrackSummary(hasMultipleTracks);
4336
4337 ALOGI("Received total/0-length (%d/%d) buffers and encoded %d frames. - %s",
4338 count, nZeroLengthFrames, mStszTableEntries->count(), trackName);
4339 if (mIsAudio) {
4340 ALOGI("Audio track drift time: %" PRId64 " us", mOwner->getDriftTimeUs());
4341 }
4342
4343 if (err == ERROR_END_OF_STREAM) {
4344 return OK;
4345 }
4346 return err;
4347 }
4348
isTrackMalFormed()4349 bool MPEG4Writer::Track::isTrackMalFormed() {
4350 if (mIsMalformed) {
4351 return true;
4352 }
4353
4354 int32_t emptyTrackMalformed = false;
4355 if (mOwner->mStartMeta &&
4356 mOwner->mStartMeta->findInt32(kKeyEmptyTrackMalFormed, &emptyTrackMalformed) &&
4357 emptyTrackMalformed) {
4358 // MediaRecorder(sets kKeyEmptyTrackMalFormed by default) report empty tracks as malformed.
4359 if (!mIsHeif && mStszTableEntries->count() == 0) { // no samples written
4360 ALOGE("The number of recorded samples is 0");
4361 mIsMalformed = true;
4362 return true;
4363 }
4364 if (mIsVideo && mStssTableEntries->count() == 0) { // no sync frames for video
4365 ALOGE("There are no sync frames for video track");
4366 mIsMalformed = true;
4367 return true;
4368 }
4369 } else {
4370 // Through MediaMuxer, empty tracks can be added. No sync frames for video.
4371 if (mIsVideo && mStszTableEntries->count() > 0 && mStssTableEntries->count() == 0) {
4372 ALOGE("There are no sync frames for video track");
4373 mIsMalformed = true;
4374 return true;
4375 }
4376 }
4377 // Don't check for CodecSpecificData when track is empty.
4378 if (mStszTableEntries->count() > 0 && OK != checkCodecSpecificData()) {
4379 // No codec specific data.
4380 mIsMalformed = true;
4381 return true;
4382 }
4383
4384 return false;
4385 }
4386
sendTrackSummary(bool hasMultipleTracks)4387 void MPEG4Writer::Track::sendTrackSummary(bool hasMultipleTracks) {
4388
4389 // Send track summary only if test mode is enabled.
4390 if (!isTestModeEnabled()) {
4391 return;
4392 }
4393
4394 uint32_t trackNum = (mTrackId.getId() << 28);
4395
4396 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4397 trackNum | MEDIA_RECORDER_TRACK_INFO_TYPE,
4398 mIsAudio ? 0: 1);
4399
4400 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4401 trackNum | MEDIA_RECORDER_TRACK_INFO_DURATION_MS,
4402 mTrackDurationUs / 1000);
4403
4404 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4405 trackNum | MEDIA_RECORDER_TRACK_INFO_ENCODED_FRAMES,
4406 mStszTableEntries->count());
4407
4408 {
4409 // The system delay time excluding the requested initial delay that
4410 // is used to eliminate the recording sound.
4411 int64_t startTimeOffsetUs = mOwner->getStartTimeOffsetMs() * 1000LL;
4412 if (startTimeOffsetUs < 0) { // Start time offset was not set
4413 startTimeOffsetUs = kInitialDelayTimeUs;
4414 }
4415 int64_t initialDelayUs =
4416 mFirstSampleTimeRealUs - mStartTimeRealUs - startTimeOffsetUs;
4417
4418 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4419 trackNum | MEDIA_RECORDER_TRACK_INFO_INITIAL_DELAY_MS,
4420 (initialDelayUs) / 1000);
4421 }
4422
4423 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4424 trackNum | MEDIA_RECORDER_TRACK_INFO_DATA_KBYTES,
4425 mMdatSizeBytes / 1024);
4426
4427 if (hasMultipleTracks) {
4428 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4429 trackNum | MEDIA_RECORDER_TRACK_INFO_MAX_CHUNK_DUR_MS,
4430 mMaxChunkDurationUs / 1000);
4431
4432 int64_t moovStartTimeUs = mOwner->getStartTimestampUs();
4433 if (mStartTimestampUs != moovStartTimeUs) {
4434 int64_t startTimeOffsetUs = mStartTimestampUs - moovStartTimeUs;
4435 mOwner->notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4436 trackNum | MEDIA_RECORDER_TRACK_INFO_START_OFFSET_MS,
4437 startTimeOffsetUs / 1000);
4438 }
4439 }
4440 }
4441
trackProgressStatus(int64_t timeUs,status_t err)4442 void MPEG4Writer::Track::trackProgressStatus(int64_t timeUs, status_t err) {
4443 ALOGV("trackProgressStatus: %" PRId64 " us", timeUs);
4444
4445 if (mTrackEveryTimeDurationUs > 0 &&
4446 timeUs - mPreviousTrackTimeUs >= mTrackEveryTimeDurationUs) {
4447 ALOGV("Fire time tracking progress status at %" PRId64 " us", timeUs);
4448 mOwner->trackProgressStatus(mTrackId.getId(), timeUs - mPreviousTrackTimeUs, err);
4449 mPreviousTrackTimeUs = timeUs;
4450 }
4451 }
4452
trackProgressStatus(uint32_t trackId,int64_t timeUs,status_t err)4453 void MPEG4Writer::trackProgressStatus(
4454 uint32_t trackId, int64_t timeUs, status_t err) {
4455 Mutex::Autolock lock(mLock);
4456 uint32_t trackNum = (trackId << 28);
4457
4458 // Error notification
4459 // Do not consider ERROR_END_OF_STREAM an error
4460 if (err != OK && err != ERROR_END_OF_STREAM) {
4461 notify(MEDIA_RECORDER_TRACK_EVENT_ERROR,
4462 trackNum | MEDIA_RECORDER_TRACK_ERROR_GENERAL,
4463 err);
4464 return;
4465 }
4466
4467 if (timeUs == -1) {
4468 // Send completion notification
4469 notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4470 trackNum | MEDIA_RECORDER_TRACK_INFO_COMPLETION_STATUS,
4471 err);
4472 } else {
4473 // Send progress status
4474 notify(MEDIA_RECORDER_TRACK_EVENT_INFO,
4475 trackNum | MEDIA_RECORDER_TRACK_INFO_PROGRESS_IN_TIME,
4476 timeUs / 1000);
4477 }
4478 }
4479
setDriftTimeUs(int64_t driftTimeUs)4480 void MPEG4Writer::setDriftTimeUs(int64_t driftTimeUs) {
4481 ALOGV("setDriftTimeUs: %" PRId64 " us", driftTimeUs);
4482 Mutex::Autolock autolock(mLock);
4483 mDriftTimeUs = driftTimeUs;
4484 }
4485
getDriftTimeUs()4486 int64_t MPEG4Writer::getDriftTimeUs() {
4487 ALOGV("getDriftTimeUs: %" PRId64 " us", mDriftTimeUs);
4488 Mutex::Autolock autolock(mLock);
4489 return mDriftTimeUs;
4490 }
4491
isRealTimeRecording() const4492 bool MPEG4Writer::isRealTimeRecording() const {
4493 return mIsRealTimeRecording;
4494 }
4495
isBackgroundMode() const4496 bool MPEG4Writer::isBackgroundMode() const {
4497 return mIsBackgroundMode;
4498 }
4499
useNalLengthFour()4500 bool MPEG4Writer::useNalLengthFour() {
4501 return mUse4ByteNalLength;
4502 }
4503
bufferChunk(int64_t timestampUs)4504 void MPEG4Writer::Track::bufferChunk(int64_t timestampUs) {
4505 ALOGV("bufferChunk");
4506
4507 Chunk chunk(this, timestampUs, mChunkSamples);
4508 mOwner->bufferChunk(chunk);
4509 mChunkSamples.clear();
4510 }
4511
getDurationUs() const4512 int64_t MPEG4Writer::Track::getDurationUs() const {
4513 return mTrackDurationUs + getStartTimeOffsetTimeUs() + mOwner->getStartTimeOffsetBFramesUs();
4514 }
4515
getEstimatedTrackSizeBytes() const4516 int64_t MPEG4Writer::Track::getEstimatedTrackSizeBytes() const {
4517 return mEstimatedTrackSizeBytes;
4518 }
4519
getMetaSizeIncrease(int32_t angle,int32_t trackCount) const4520 int32_t MPEG4Writer::Track::getMetaSizeIncrease(
4521 int32_t angle, int32_t trackCount) const {
4522 CHECK(mIsHeif);
4523
4524 int32_t grid = (mTileWidth > 0);
4525 int32_t rotate = (angle > 0);
4526
4527 // Note that the rotation angle is in the file meta, and we don't have
4528 // it until start, so here the calculation has to assume rotation.
4529
4530 // increase to ipco
4531 int32_t increase = 20 * (grid + 1) // 'ispe' property
4532 + (8 + mCodecSpecificDataSize) // 'hvcC' property
4533 ;
4534
4535 if (rotate) {
4536 increase += 9; // 'irot' property (worst case)
4537 }
4538
4539 if (flags_camera::camera_heif_gainmap()) {
4540 // assume we have HDR gainmap and associated metadata
4541 increase += (8 + mCodecSpecificDataSize) // 'hvcC' property (HDR gainmap)
4542 + (2 * 20) // 'ispe' property
4543 + (2 * 16) // 'pixi' property
4544 + (2 * 19) // 'colr' property
4545 ;
4546 }
4547
4548 // increase to iref and idat
4549 if (grid) {
4550 increase += (12 + mNumTiles * 2) // 'dimg' in iref
4551 + 12; // ImageGrid in 'idat' (worst case)
4552 }
4553
4554 increase += (12 + 2); // 'cdsc' in iref
4555
4556 // increase to iloc, iinf
4557 increase += (16 // increase to 'iloc'
4558 + 21) // increase to 'iinf'
4559 * (mNumTiles + grid + 1); // "+1" is for 'Exif'
4560
4561 if (flags_camera::camera_heif_gainmap()) {
4562 increase += (16 // increase to 'iloc'
4563 + 21) // increase to 'iinf'
4564 * 2; // "2" is for 'tmap', 'gmap'
4565 }
4566
4567 // When total # of properties is > 127, the properties id becomes 2-byte.
4568 // We write 4 properties at most for each image (2x'ispe', 1x'hvcC', 1x'irot').
4569 // Set the threshold to be 30.
4570 int32_t propBytes = trackCount > 30 ? 2 : 1;
4571
4572 // increase to ipma
4573 increase += (3 + 2 * propBytes) * mNumTiles // 'ispe' + 'hvcC'
4574 + grid * (3 + propBytes) // 'ispe' for grid
4575 + rotate * propBytes; // 'irot' (either on grid or tile)
4576
4577 return increase;
4578 }
4579
checkCodecSpecificData() const4580 status_t MPEG4Writer::Track::checkCodecSpecificData() const {
4581 const char *mime;
4582 CHECK(mMeta->findCString(kKeyMIMEType, &mime));
4583 if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_AAC, mime) ||
4584 !strcasecmp(MEDIA_MIMETYPE_VIDEO_MPEG4, mime) ||
4585 !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime) ||
4586 !strcasecmp(MEDIA_MIMETYPE_VIDEO_HEVC, mime) ||
4587 !strcasecmp(MEDIA_MIMETYPE_VIDEO_AV1, mime) ||
4588 (editing_flags::muxer_mp4_enable_apv() && !strcasecmp(MEDIA_MIMETYPE_VIDEO_APV, mime)) ||
4589 !strcasecmp(MEDIA_MIMETYPE_VIDEO_DOLBY_VISION, mime) ||
4590 !strcasecmp(MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC, mime) ||
4591 !strcasecmp(MEDIA_MIMETYPE_IMAGE_AVIF, mime)) {
4592 if (!mCodecSpecificData ||
4593 mCodecSpecificDataSize <= 0) {
4594 ALOGE("Missing codec specific data");
4595 return ERROR_MALFORMED;
4596 }
4597 } else {
4598 if (mCodecSpecificData ||
4599 mCodecSpecificDataSize > 0) {
4600 ALOGE("Unexepected codec specific data found");
4601 return ERROR_MALFORMED;
4602 }
4603 }
4604 return OK;
4605 }
4606
getTrackType() const4607 const char *MPEG4Writer::Track::getTrackType() const {
4608 return mIsAudio ? "Audio" :
4609 mIsVideo ? "Video" :
4610 mIsHeif ? "Image" :
4611 "Metadata";
4612 }
4613
writeTrackHeader()4614 void MPEG4Writer::Track::writeTrackHeader() {
4615 uint32_t now = getMpeg4Time();
4616 mOwner->beginBox("trak");
4617 writeTkhdBox(now);
4618 writeEdtsBox();
4619 mOwner->beginBox("mdia");
4620 writeMdhdBox(now);
4621 writeHdlrBox();
4622 mOwner->beginBox("minf");
4623 if (mIsAudio) {
4624 writeSmhdBox();
4625 } else if (mIsVideo) {
4626 writeVmhdBox();
4627 } else {
4628 writeNmhdBox();
4629 }
4630 writeDinfBox();
4631 writeStblBox();
4632 mOwner->endBox(); // minf
4633 mOwner->endBox(); // mdia
4634 mOwner->endBox(); // trak
4635 }
4636
getMinCttsOffsetTimeUs()4637 int64_t MPEG4Writer::Track::getMinCttsOffsetTimeUs() {
4638 // For video tracks with ctts table, this should return the minimum ctts
4639 // offset in the table. For non-video tracks or video tracks without ctts
4640 // table, this will return kMaxCttsOffsetTimeUs.
4641 if (mMinCttsOffsetTicks == mMaxCttsOffsetTicks) {
4642 return kMaxCttsOffsetTimeUs;
4643 }
4644 return mMinCttsOffsetTimeUs;
4645 }
4646
writeStblBox()4647 void MPEG4Writer::Track::writeStblBox() {
4648 mOwner->beginBox("stbl");
4649 // Add subboxes for only non-empty and well-formed tracks.
4650 if (mStszTableEntries->count() > 0 && !isTrackMalFormed()) {
4651 mOwner->beginBox("stsd");
4652 mOwner->writeInt32(0); // version=0, flags=0
4653 mOwner->writeInt32(1); // entry count
4654 if (mIsAudio) {
4655 writeAudioFourCCBox();
4656 } else if (mIsVideo) {
4657 writeVideoFourCCBox();
4658 } else {
4659 writeMetadataFourCCBox();
4660 }
4661 mOwner->endBox(); // stsd
4662 writeSttsBox();
4663 if (mIsVideo) {
4664 writeCttsBox();
4665 writeStssBox();
4666 }
4667 writeStszBox();
4668 writeStscBox();
4669 writeCo64Box();
4670 }
4671 mOwner->endBox(); // stbl
4672 }
4673
writeMetadataFourCCBox()4674 void MPEG4Writer::Track::writeMetadataFourCCBox() {
4675 const char *mime;
4676 bool success = mMeta->findCString(kKeyMIMEType, &mime);
4677 CHECK(success);
4678 const char *fourcc = getFourCCForMime(mime);
4679 if (fourcc == NULL) {
4680 ALOGE("Unknown mime type '%s'.", mime);
4681 TRESPASS();
4682 }
4683 mOwner->beginBox(fourcc); // TextMetaDataSampleEntry
4684
4685 // HACK to make the metadata track compliant with the ISO standard.
4686 //
4687 // Metadata track is added from API 26 and the original implementation does not
4688 // fully followed the TextMetaDataSampleEntry specified in ISO/IEC 14496-12-2015
4689 // in that only the mime_format is written out. content_encoding and
4690 // data_reference_index have not been written out. This leads to the failure
4691 // when some MP4 parser tries to parse the metadata track according to the
4692 // standard. The hack here will make the metadata track compliant with the
4693 // standard while still maintaining backwards compatibility. This would enable
4694 // Android versions before API 29 to be able to read out the standard compliant
4695 // Metadata track generated with Android API 29 and upward. The trick is based
4696 // on the fact that the Metadata track must start with prefix “application/” and
4697 // those missing fields are not used in Android's Metadata track. By writting
4698 // out the mime_format twice, the first mime_format will be used to fill out the
4699 // missing reserved, data_reference_index and content encoding fields. On the
4700 // parser side, the extracter before API 29 will read out the first mime_format
4701 // correctly and drop the second mime_format. The extractor from API 29 will
4702 // check if the reserved, data_reference_index and content encoding are filled
4703 // with “application” to detect if this is a standard compliant metadata track
4704 // and read out the data accordingly.
4705 mOwner->writeCString(mime);
4706
4707 mOwner->writeCString(mime); // metadata mime_format
4708 mOwner->endBox(); // mett
4709 }
4710
writeVideoFourCCBox()4711 void MPEG4Writer::Track::writeVideoFourCCBox() {
4712 const char *mime;
4713 bool success = mMeta->findCString(kKeyMIMEType, &mime);
4714 CHECK(success);
4715 const char *fourcc;
4716 if (!strcmp(mime, MEDIA_MIMETYPE_VIDEO_DOLBY_VISION)) {
4717 fourcc = getDoviFourCC();
4718 } else {
4719 fourcc = getFourCCForMime(mime);
4720 }
4721
4722 if (fourcc == NULL) {
4723 ALOGE("Unknown mime type '%s'.", mime);
4724 TRESPASS();
4725 }
4726
4727 mOwner->beginBox(fourcc); // video format
4728 mOwner->writeInt32(0); // reserved
4729 mOwner->writeInt16(0); // reserved
4730 mOwner->writeInt16(1); // data ref index
4731 mOwner->writeInt16(0); // predefined
4732 mOwner->writeInt16(0); // reserved
4733 mOwner->writeInt32(0); // predefined
4734 mOwner->writeInt32(0); // predefined
4735 mOwner->writeInt32(0); // predefined
4736
4737 int32_t width, height;
4738 success = mMeta->findInt32(kKeyWidth, &width);
4739 success = success && mMeta->findInt32(kKeyHeight, &height);
4740 CHECK(success);
4741
4742 mOwner->writeInt16(width);
4743 mOwner->writeInt16(height);
4744 mOwner->writeInt32(0x480000); // horiz resolution
4745 mOwner->writeInt32(0x480000); // vert resolution
4746 mOwner->writeInt32(0); // reserved
4747 mOwner->writeInt16(1); // frame count
4748 mOwner->writeInt8(0); // compressor string length
4749 mOwner->write(" ", 31);
4750 mOwner->writeInt16(0x18); // depth
4751 mOwner->writeInt16(-1); // predefined
4752
4753 if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_MPEG4, mime)) {
4754 writeMp4vEsdsBox();
4755 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_H263, mime)) {
4756 writeD263Box();
4757 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime)) {
4758 writeAvccBox();
4759 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_HEVC, mime)) {
4760 writeHvccBox();
4761 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_AV1, mime)) {
4762 writeAv1cBox();
4763 } else if (editing_flags::muxer_mp4_enable_apv() &&
4764 !strcasecmp(MEDIA_MIMETYPE_VIDEO_APV, mime)) {
4765 writeApvcBox();
4766 } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_DOLBY_VISION, mime)) {
4767 if (mDoviProfile <= DolbyVisionProfileDvheSt) {
4768 writeHvccBox();
4769 } else if (mDoviProfile == DolbyVisionProfileDvavSe) {
4770 writeAvccBox();
4771 } else {
4772 TRESPASS("Unsupported Dolby Vision profile");
4773 }
4774 writeDoviConfigBox();
4775 }
4776
4777 writePaspBox();
4778 writeColrBox();
4779 writeMdcvAndClliBoxes();
4780 mOwner->endBox(); // mp4v, s263 or avc1
4781 }
4782
writeColrBox()4783 void MPEG4Writer::Track::writeColrBox() {
4784 ColorAspects aspects;
4785 memset(&aspects, 0, sizeof(aspects));
4786 // Color metadata may have changed.
4787 sp<MetaData> meta = mSource->getFormat();
4788 bool findPrimaries = meta->findInt32(kKeyColorPrimaries, (int32_t*)&aspects.mPrimaries);
4789 bool findTransfer = meta->findInt32(kKeyTransferFunction, (int32_t*)&aspects.mTransfer);
4790 bool findMatrix = meta->findInt32(kKeyColorMatrix, (int32_t*)&aspects.mMatrixCoeffs);
4791 bool findRange = meta->findInt32(kKeyColorRange, (int32_t*)&aspects.mRange);
4792 if (!findPrimaries && !findTransfer && !findMatrix && !findRange) {
4793 ALOGV("no color information");
4794 return;
4795 }
4796
4797 int32_t primaries, transfer, coeffs;
4798 bool fullRange;
4799 ALOGV("primaries=%s transfer=%s matrix=%s range=%s",
4800 asString(aspects.mPrimaries),
4801 asString(aspects.mTransfer),
4802 asString(aspects.mMatrixCoeffs),
4803 asString(aspects.mRange));
4804 ColorUtils::convertCodecColorAspectsToIsoAspects(
4805 aspects, &primaries, &transfer, &coeffs, &fullRange);
4806 mOwner->beginBox("colr");
4807 mOwner->writeFourcc("nclx");
4808 mOwner->writeInt16(primaries);
4809 mOwner->writeInt16(transfer);
4810 mOwner->writeInt16(coeffs);
4811 mOwner->writeInt8(int8_t(fullRange ? 0x80 : 0x0));
4812 mOwner->endBox(); // colr
4813 }
4814
writeMdcvAndClliBoxes()4815 void MPEG4Writer::Track::writeMdcvAndClliBoxes() {
4816 sp<MetaData> meta = mSource->getFormat();
4817 uint32_t type;
4818 const uint8_t* data;
4819 size_t size;
4820 bool found =
4821 meta->findData(kKeyHdrStaticInfo, &type, reinterpret_cast<const void**>(&data), &size);
4822 if (!found) {
4823 return; // Nothing to encode.
4824 }
4825 if (size != 25) {
4826 ALOGW("Ignoring HDR static info with unexpected size %d", (int)size);
4827 return;
4828 }
4829 uint16_t displayPrimariesRX = U16LE_AT(&data[1]);
4830 uint16_t displayPrimariesRY = U16LE_AT(&data[3]);
4831
4832 uint16_t displayPrimariesGX = U16LE_AT(&data[5]);
4833 uint16_t displayPrimariesGY = U16LE_AT(&data[7]);
4834
4835 uint16_t displayPrimariesBX = U16LE_AT(&data[9]);
4836 uint16_t displayPrimariesBY = U16LE_AT(&data[11]);
4837
4838 uint16_t whitePointX = U16LE_AT(&data[13]);
4839 uint16_t whitePointY = U16LE_AT(&data[15]);
4840
4841 uint16_t maxDisplayMasteringLuminance = U16LE_AT(&data[17]);
4842 uint16_t minDisplayMasteringLuminance = U16LE_AT(&data[19]);
4843
4844 uint16_t maxContentLightLevel = U16LE_AT(&data[21]);
4845 uint16_t maxPicAverageLightLevel = U16LE_AT(&data[23]);
4846
4847 mOwner->beginBox("mdcv");
4848 mOwner->writeInt16(displayPrimariesGX);
4849 mOwner->writeInt16(displayPrimariesGY);
4850 mOwner->writeInt16(displayPrimariesBX);
4851 mOwner->writeInt16(displayPrimariesBY);
4852 mOwner->writeInt16(displayPrimariesRX);
4853 mOwner->writeInt16(displayPrimariesRY);
4854 mOwner->writeInt16(whitePointX);
4855 mOwner->writeInt16(whitePointY);
4856 mOwner->writeInt32(maxDisplayMasteringLuminance * 10000);
4857 mOwner->writeInt32(minDisplayMasteringLuminance * 10000);
4858 mOwner->endBox(); // mdcv.
4859
4860 mOwner->beginBox("clli");
4861 mOwner->writeInt16(maxContentLightLevel);
4862 mOwner->writeInt16(maxPicAverageLightLevel);
4863 mOwner->endBox(); // clli.
4864 }
4865
writeAudioFourCCBox()4866 void MPEG4Writer::Track::writeAudioFourCCBox() {
4867 const char *mime;
4868 bool success = mMeta->findCString(kKeyMIMEType, &mime);
4869 CHECK(success);
4870 const char *fourcc = getFourCCForMime(mime);
4871 if (fourcc == NULL) {
4872 ALOGE("Unknown mime type '%s'.", mime);
4873 TRESPASS();
4874 }
4875
4876 mOwner->beginBox(fourcc); // audio format
4877 mOwner->writeInt32(0); // reserved
4878 mOwner->writeInt16(0); // reserved
4879 mOwner->writeInt16(0x1); // data ref index
4880 mOwner->writeInt32(0); // reserved
4881 mOwner->writeInt32(0); // reserved
4882 int32_t nChannels;
4883 CHECK_EQ(true, mMeta->findInt32(kKeyChannelCount, &nChannels));
4884 mOwner->writeInt16(nChannels); // channel count
4885 mOwner->writeInt16(16); // sample size
4886 mOwner->writeInt16(0); // predefined
4887 mOwner->writeInt16(0); // reserved
4888
4889 int32_t samplerate;
4890 success = mMeta->findInt32(kKeySampleRate, &samplerate);
4891 CHECK(success);
4892 mOwner->writeInt32(samplerate << 16);
4893 if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_AAC, mime)) {
4894 writeMp4aEsdsBox();
4895 } else if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_AMR_NB, mime) ||
4896 !strcasecmp(MEDIA_MIMETYPE_AUDIO_AMR_WB, mime)) {
4897 writeDamrBox();
4898 }
4899 mOwner->endBox();
4900 }
4901
generateEsdsSize(size_t dataLength,size_t * sizeGenerated,uint8_t * buffer)4902 static void generateEsdsSize(size_t dataLength, size_t* sizeGenerated, uint8_t* buffer) {
4903 size_t offset = 0, cur = 0;
4904 size_t more = 0x00;
4905 *sizeGenerated = 0;
4906 /* Start with the LSB(7 bits) of dataLength and build the byte sequence upto MSB.
4907 * Continuation flag(most significant bit) will be set on the first N-1 bytes.
4908 */
4909 do {
4910 buffer[cur++] = (dataLength & 0x7f) | more;
4911 dataLength >>= 7;
4912 more = 0x80;
4913 ++(*sizeGenerated);
4914 } while (dataLength > 0u);
4915 --cur;
4916 // Reverse the newly formed byte sequence.
4917 while (cur > offset) {
4918 uint8_t tmp = buffer[cur];
4919 buffer[cur--] = buffer[offset];
4920 buffer[offset++] = tmp;
4921 }
4922 }
4923
writeMp4aEsdsBox()4924 void MPEG4Writer::Track::writeMp4aEsdsBox() {
4925 CHECK(mCodecSpecificData);
4926 CHECK_GT(mCodecSpecificDataSize, 0u);
4927
4928 uint8_t sizeESDBuffer[kESDSScratchBufferSize];
4929 uint8_t sizeDCDBuffer[kESDSScratchBufferSize];
4930 uint8_t sizeDSIBuffer[kESDSScratchBufferSize];
4931 size_t sizeESD = 0;
4932 size_t sizeDCD = 0;
4933 size_t sizeDSI = 0;
4934 generateEsdsSize(mCodecSpecificDataSize, &sizeDSI, sizeDSIBuffer);
4935 generateEsdsSize(mCodecSpecificDataSize + sizeDSI + 14, &sizeDCD, sizeDCDBuffer);
4936 generateEsdsSize(mCodecSpecificDataSize + sizeDSI + sizeDCD + 21, &sizeESD, sizeESDBuffer);
4937
4938 mOwner->beginBox("esds");
4939
4940 mOwner->writeInt32(0); // version=0, flags=0
4941 mOwner->writeInt8(0x03); // ES_DescrTag
4942 mOwner->write(sizeESDBuffer, sizeESD);
4943 mOwner->writeInt16(0x0000);// ES_ID
4944 mOwner->writeInt8(0x00);
4945
4946 mOwner->writeInt8(0x04); // DecoderConfigDescrTag
4947 mOwner->write(sizeDCDBuffer, sizeDCD);
4948 mOwner->writeInt8(0x40); // objectTypeIndication ISO/IEC 14492-2
4949 mOwner->writeInt8(0x15); // streamType AudioStream
4950
4951 mOwner->writeInt16(0x03); // XXX
4952 mOwner->writeInt8(0x00); // buffer size 24-bit (0x300)
4953
4954 int32_t avgBitrate = 0;
4955 (void)mMeta->findInt32(kKeyBitRate, &avgBitrate);
4956 int32_t maxBitrate = 0;
4957 (void)mMeta->findInt32(kKeyMaxBitRate, &maxBitrate);
4958 mOwner->writeInt32(maxBitrate);
4959 mOwner->writeInt32(avgBitrate);
4960
4961 mOwner->writeInt8(0x05); // DecoderSpecificInfoTag
4962 mOwner->write(sizeDSIBuffer, sizeDSI);
4963 mOwner->write(mCodecSpecificData, mCodecSpecificDataSize);
4964
4965 static const uint8_t kData2[] = {
4966 0x06, // SLConfigDescriptorTag
4967 0x01,
4968 0x02
4969 };
4970 mOwner->write(kData2, sizeof(kData2));
4971
4972 mOwner->endBox(); // esds
4973 }
4974
writeMp4vEsdsBox()4975 void MPEG4Writer::Track::writeMp4vEsdsBox() {
4976 CHECK(mCodecSpecificData);
4977 CHECK_GT(mCodecSpecificDataSize, 0u);
4978
4979 uint8_t sizeESDBuffer[kESDSScratchBufferSize];
4980 uint8_t sizeDCDBuffer[kESDSScratchBufferSize];
4981 uint8_t sizeDSIBuffer[kESDSScratchBufferSize];
4982 size_t sizeESD = 0;
4983 size_t sizeDCD = 0;
4984 size_t sizeDSI = 0;
4985 generateEsdsSize(mCodecSpecificDataSize, &sizeDSI, sizeDSIBuffer);
4986 generateEsdsSize(mCodecSpecificDataSize + sizeDSI + 14, &sizeDCD, sizeDCDBuffer);
4987 generateEsdsSize(mCodecSpecificDataSize + sizeDSI + sizeDCD + 21, &sizeESD, sizeESDBuffer);
4988
4989 mOwner->beginBox("esds");
4990
4991 mOwner->writeInt32(0); // version=0, flags=0
4992
4993 mOwner->writeInt8(0x03); // ES_DescrTag
4994 mOwner->write(sizeESDBuffer, sizeESD);
4995 mOwner->writeInt16(0x0000); // ES_ID
4996 mOwner->writeInt8(0x1f);
4997
4998 mOwner->writeInt8(0x04); // DecoderConfigDescrTag
4999 mOwner->write(sizeDCDBuffer, sizeDCD);
5000 mOwner->writeInt8(0x20); // objectTypeIndication ISO/IEC 14492-2
5001 mOwner->writeInt8(0x11); // streamType VisualStream
5002
5003 static const uint8_t kData[] = {
5004 0x01, 0x77, 0x00, // buffer size 96000 bytes
5005 };
5006 mOwner->write(kData, sizeof(kData));
5007
5008 int32_t avgBitrate = 0;
5009 (void)mMeta->findInt32(kKeyBitRate, &avgBitrate);
5010 int32_t maxBitrate = 0;
5011 (void)mMeta->findInt32(kKeyMaxBitRate, &maxBitrate);
5012 mOwner->writeInt32(maxBitrate);
5013 mOwner->writeInt32(avgBitrate);
5014
5015 mOwner->writeInt8(0x05); // DecoderSpecificInfoTag
5016
5017 mOwner->write(sizeDSIBuffer, sizeDSI);
5018 mOwner->write(mCodecSpecificData, mCodecSpecificDataSize);
5019
5020 static const uint8_t kData2[] = {
5021 0x06, // SLConfigDescriptorTag
5022 0x01,
5023 0x02
5024 };
5025 mOwner->write(kData2, sizeof(kData2));
5026
5027 mOwner->endBox(); // esds
5028 }
5029
writeTkhdBox(uint32_t now)5030 void MPEG4Writer::Track::writeTkhdBox(uint32_t now) {
5031 mOwner->beginBox("tkhd");
5032 // Flags = 7 to indicate that the track is enabled, and
5033 // part of the presentation
5034 mOwner->writeInt32(0x07); // version=0, flags=7
5035 mOwner->writeInt32(now); // creation time
5036 mOwner->writeInt32(now); // modification time
5037 mOwner->writeInt32(mTrackId.getId()); // track id starts with 1
5038 mOwner->writeInt32(0); // reserved
5039 int64_t trakDurationUs = getDurationUs();
5040 int32_t mvhdTimeScale = mOwner->getTimeScale();
5041 int32_t tkhdDuration =
5042 (trakDurationUs * mvhdTimeScale + 5E5) / 1E6;
5043 mOwner->writeInt32(tkhdDuration); // in mvhd timescale
5044 mOwner->writeInt32(0); // reserved
5045 mOwner->writeInt32(0); // reserved
5046 mOwner->writeInt16(0); // layer
5047 mOwner->writeInt16(0); // alternate group
5048 mOwner->writeInt16(mIsAudio ? 0x100 : 0); // volume
5049 mOwner->writeInt16(0); // reserved
5050
5051 mOwner->writeCompositionMatrix(mRotation); // matrix
5052
5053 if (!mIsVideo) {
5054 mOwner->writeInt32(0);
5055 mOwner->writeInt32(0);
5056 } else {
5057 int32_t width, height;
5058 bool success = mMeta->findInt32(kKeyDisplayWidth, &width);
5059 success = success && mMeta->findInt32(kKeyDisplayHeight, &height);
5060
5061 // Use width/height if display width/height are not present.
5062 if (!success) {
5063 success = mMeta->findInt32(kKeyWidth, &width);
5064 success = success && mMeta->findInt32(kKeyHeight, &height);
5065 }
5066 CHECK(success);
5067
5068 mOwner->writeInt32(width << 16); // 32-bit fixed-point value
5069 mOwner->writeInt32(height << 16); // 32-bit fixed-point value
5070 }
5071 mOwner->endBox(); // tkhd
5072 }
5073
writeVmhdBox()5074 void MPEG4Writer::Track::writeVmhdBox() {
5075 mOwner->beginBox("vmhd");
5076 mOwner->writeInt32(0x01); // version=0, flags=1
5077 mOwner->writeInt16(0); // graphics mode
5078 mOwner->writeInt16(0); // opcolor
5079 mOwner->writeInt16(0);
5080 mOwner->writeInt16(0);
5081 mOwner->endBox();
5082 }
5083
writeSmhdBox()5084 void MPEG4Writer::Track::writeSmhdBox() {
5085 mOwner->beginBox("smhd");
5086 mOwner->writeInt32(0); // version=0, flags=0
5087 mOwner->writeInt16(0); // balance
5088 mOwner->writeInt16(0); // reserved
5089 mOwner->endBox();
5090 }
5091
writeNmhdBox()5092 void MPEG4Writer::Track::writeNmhdBox() {
5093 mOwner->beginBox("nmhd");
5094 mOwner->writeInt32(0); // version=0, flags=0
5095 mOwner->endBox();
5096 }
5097
writeHdlrBox()5098 void MPEG4Writer::Track::writeHdlrBox() {
5099 mOwner->beginBox("hdlr");
5100 mOwner->writeInt32(0); // version=0, flags=0
5101 mOwner->writeInt32(0); // component type: should be mhlr
5102 mOwner->writeFourcc(mIsAudio ? "soun" : (mIsVideo ? "vide" : "meta")); // component subtype
5103 mOwner->writeInt32(0); // reserved
5104 mOwner->writeInt32(0); // reserved
5105 mOwner->writeInt32(0); // reserved
5106 // Removing "r" for the name string just makes the string 4 byte aligned
5107 mOwner->writeCString(mIsAudio ? "SoundHandle": (mIsVideo ? "VideoHandle" : "MetadHandle"));
5108 mOwner->endBox();
5109 }
5110
writeEdtsBox()5111 void MPEG4Writer::Track::writeEdtsBox() {
5112 ALOGV("%s : getStartTimeOffsetTimeUs of track:%" PRId64 " us", getTrackType(),
5113 getStartTimeOffsetTimeUs());
5114
5115 int32_t mvhdTimeScale = mOwner->getTimeScale();
5116 ALOGV("mvhdTimeScale:%" PRId32, mvhdTimeScale);
5117 /* trackStartOffsetUs of this track is the sum of longest offset needed by a track among all
5118 * tracks with B frames in this movie and the start offset of this track.
5119 */
5120 int64_t trackStartOffsetUs = getStartTimeOffsetTimeUs();
5121 ALOGV("trackStartOffsetUs:%" PRIu64, trackStartOffsetUs);
5122
5123 // Longest offset needed by a track among all tracks with B frames.
5124 int32_t movieStartOffsetBFramesUs = mOwner->getStartTimeOffsetBFramesUs();
5125 ALOGV("movieStartOffsetBFramesUs:%" PRId32, movieStartOffsetBFramesUs);
5126
5127 // This media/track's real duration (sum of duration of all samples in this track).
5128 uint32_t tkhdDurationTicks = (mTrackDurationUs * mvhdTimeScale + 5E5) / 1E6;
5129 ALOGV("mTrackDurationUs:%" PRId64 "us", mTrackDurationUs);
5130
5131 int64_t movieStartTimeUs = mOwner->getStartTimestampUs();
5132 ALOGV("movieStartTimeUs:%" PRId64, movieStartTimeUs);
5133
5134 int64_t trackStartTimeUs = movieStartTimeUs + trackStartOffsetUs;
5135 ALOGV("trackStartTimeUs:%" PRId64, trackStartTimeUs);
5136
5137 if (movieStartOffsetBFramesUs == 0) {
5138 // No B frames in any tracks.
5139 if (trackStartOffsetUs > 0) {
5140 // Track with positive start offset.
5141 uint32_t segDuration = (trackStartOffsetUs * mvhdTimeScale + 5E5) / 1E6;
5142 ALOGV("segDuration:%" PRIu64 "us", trackStartOffsetUs);
5143 /* The first entry is an empty edit (indicated by media_time equal to -1), and its
5144 * duration (segment_duration) is equal to the difference of the presentation times of
5145 * the earliest media sample among all tracks and the earliest media sample of the track.
5146 */
5147 ALOGV("Empty edit list entry");
5148 addOneElstTableEntry(segDuration, -1, 1, 0);
5149 addOneElstTableEntry(tkhdDurationTicks, 0, 1, 0);
5150 } else if (mFirstSampleStartOffsetUs > 0) {
5151 // Track with start time < 0 / negative start offset.
5152 ALOGV("Normal edit list entry");
5153 int32_t mediaTime = (mFirstSampleStartOffsetUs * mTimeScale + 5E5) / 1E6;
5154 int32_t firstSampleOffsetTicks =
5155 (mFirstSampleStartOffsetUs * mvhdTimeScale + 5E5) / 1E6;
5156 if (tkhdDurationTicks >= firstSampleOffsetTicks) {
5157 // samples before 0 don't count in for duration, hence subtract
5158 // firstSampleOffsetTicks.
5159 addOneElstTableEntry(tkhdDurationTicks - firstSampleOffsetTicks, mediaTime, 1, 0);
5160 } else {
5161 ALOGW("The track header duration %" PRId64
5162 " is smaller than the first sample offset %" PRId64,
5163 mTrackDurationUs, mFirstSampleStartOffsetUs);
5164 }
5165 } else {
5166 // Track starting at zero.
5167 ALOGV("No edit list entry required for this track");
5168 }
5169 } else if (movieStartOffsetBFramesUs < 0) {
5170 // B frames present in at least one of the tracks.
5171 ALOGV("writeEdtsBox - Reordered frames(B frames) present");
5172 if (trackStartOffsetUs == std::abs(movieStartOffsetBFramesUs)) {
5173 // Track starting at 0, no start offset.
5174 // TODO : need to take care of mFirstSampleStartOffsetUs > 0 and trackStartOffsetUs > 0
5175 // separately
5176 if (mMinCttsOffsetTicks == mMaxCttsOffsetTicks) {
5177 // Video with no B frame or non-video track.
5178 if (mFirstSampleStartOffsetUs > 0) {
5179 // Track with start time < 0 / negative start offset.
5180 ALOGV("Normal edit list entry");
5181 ALOGV("mFirstSampleStartOffsetUs:%" PRId64 "us", mFirstSampleStartOffsetUs);
5182 int32_t mediaTimeTicks = (mFirstSampleStartOffsetUs * mTimeScale + 5E5) / 1E6;
5183 int32_t firstSampleOffsetTicks =
5184 (mFirstSampleStartOffsetUs * mvhdTimeScale + 5E5) / 1E6;
5185 // Samples before 0 don't count for duration, subtract firstSampleOffsetTicks.
5186 addOneElstTableEntry(tkhdDurationTicks - firstSampleOffsetTicks, mediaTimeTicks,
5187 1, 0);
5188 }
5189 } else {
5190 // Track with B Frames.
5191 int32_t mediaTimeTicks = (trackStartOffsetUs * mTimeScale + 5E5) / 1E6;
5192 ALOGV("mediaTime:%" PRId64 "us", trackStartOffsetUs);
5193 ALOGV("Normal edit list entry to negate start offset by B Frames in others tracks");
5194 addOneElstTableEntry(tkhdDurationTicks, mediaTimeTicks, 1, 0);
5195 }
5196 } else if (trackStartOffsetUs > std::abs(movieStartOffsetBFramesUs)) {
5197 // Track with start offset.
5198 ALOGV("Tracks starting > 0");
5199 int32_t editDurationTicks = 0;
5200 int32_t trackStartOffsetBFramesUs = getMinCttsOffsetTimeUs() - kMaxCttsOffsetTimeUs;
5201 ALOGV("trackStartOffsetBFramesUs:%" PRId32, trackStartOffsetBFramesUs);
5202 if (mMinCttsOffsetTicks == mMaxCttsOffsetTicks) {
5203 // Video with no B frame or non-video track.
5204 editDurationTicks =
5205 ((trackStartOffsetUs + movieStartOffsetBFramesUs) * mvhdTimeScale + 5E5) /
5206 1E6;
5207 ALOGV("editDuration:%" PRId64 "us", (trackStartOffsetUs + movieStartOffsetBFramesUs));
5208 } else {
5209 // Track with B frame.
5210 editDurationTicks =
5211 ((trackStartOffsetUs + movieStartOffsetBFramesUs +
5212 trackStartOffsetBFramesUs) * mvhdTimeScale + 5E5) / 1E6;
5213 ALOGV("editDuration:%" PRId64 "us", (trackStartOffsetUs + movieStartOffsetBFramesUs + trackStartOffsetBFramesUs));
5214 }
5215 ALOGV("editDurationTicks:%" PRIu32, editDurationTicks);
5216 if (editDurationTicks > 0) {
5217 ALOGV("Empty edit list entry");
5218 addOneElstTableEntry(editDurationTicks, -1, 1, 0);
5219 addOneElstTableEntry(tkhdDurationTicks, 0, 1, 0);
5220 } else if (editDurationTicks < 0) {
5221 // Only video tracks with B Frames would hit this case.
5222 ALOGV("Edit list entry to negate start offset by B frames in other tracks");
5223 if (com::android::media::editing::flags::
5224 stagefrightrecorder_enable_b_frames()) {
5225 int32_t mediaTimeTicks =
5226 ((trackStartOffsetUs + movieStartOffsetBFramesUs +
5227 trackStartOffsetBFramesUs) * mTimeScale - 5E5) / 1E6;
5228 addOneElstTableEntry(tkhdDurationTicks, std::abs(mediaTimeTicks), 1, 0);
5229 } else {
5230 addOneElstTableEntry(tkhdDurationTicks, std::abs(editDurationTicks), 1, 0);
5231 }
5232 } else {
5233 ALOGV("No edit list entry needed for this track");
5234 }
5235 } else {
5236 // Not expecting this case as we adjust negative start timestamps to zero.
5237 ALOGW("trackStartOffsetUs < std::abs(movieStartOffsetBFramesUs)");
5238 }
5239 } else {
5240 // Neither B frames present nor absent! or any other case?.
5241 ALOGW("movieStartOffsetBFramesUs > 0");
5242 }
5243
5244 if (mElstTableEntries->count() == 0) {
5245 return;
5246 }
5247
5248 mOwner->beginBox("edts");
5249 mOwner->beginBox("elst");
5250 mOwner->writeInt32(0); // version=0, flags=0
5251 mElstTableEntries->write(mOwner);
5252 mOwner->endBox(); // elst;
5253 mOwner->endBox(); // edts
5254 }
5255
writeMdhdBox(uint32_t now)5256 void MPEG4Writer::Track::writeMdhdBox(uint32_t now) {
5257 int64_t trakDurationUs = getDurationUs();
5258 int64_t mdhdDuration = (trakDurationUs * mTimeScale + 5E5) / 1E6;
5259 mOwner->beginBox("mdhd");
5260
5261 if (mdhdDuration > UINT32_MAX) {
5262 mOwner->writeInt32((1 << 24)); // version=1, flags=0
5263 mOwner->writeInt64((int64_t)now); // creation time
5264 mOwner->writeInt64((int64_t)now); // modification time
5265 mOwner->writeInt32(mTimeScale); // media timescale
5266 mOwner->writeInt64(mdhdDuration); // media timescale
5267 } else {
5268 mOwner->writeInt32(0); // version=0, flags=0
5269 mOwner->writeInt32(now); // creation time
5270 mOwner->writeInt32(now); // modification time
5271 mOwner->writeInt32(mTimeScale); // media timescale
5272 mOwner->writeInt32((int32_t)mdhdDuration); // use media timescale
5273 }
5274 // Language follows the three letter standard ISO-639-2/T
5275 // 'e', 'n', 'g' for "English", for instance.
5276 // Each character is packed as the difference between its ASCII value and 0x60.
5277 // For "English", these are 00101, 01110, 00111.
5278 // XXX: Where is the padding bit located: 0x15C7?
5279 const char *lang = NULL;
5280 int16_t langCode = 0;
5281 if (mMeta->findCString(kKeyMediaLanguage, &lang) && lang && strnlen(lang, 3) > 2) {
5282 langCode = ((lang[0] & 0x1f) << 10) | ((lang[1] & 0x1f) << 5) | (lang[2] & 0x1f);
5283 }
5284 mOwner->writeInt16(langCode); // language code
5285 mOwner->writeInt16(0); // predefined
5286 mOwner->endBox();
5287 }
5288
writeDamrBox()5289 void MPEG4Writer::Track::writeDamrBox() {
5290 // 3gpp2 Spec AMRSampleEntry fields
5291 mOwner->beginBox("damr");
5292 mOwner->writeCString(" "); // vendor: 4 bytes
5293 mOwner->writeInt8(0); // decoder version
5294 mOwner->writeInt16(0x83FF); // mode set: all enabled
5295 mOwner->writeInt8(0); // mode change period
5296 mOwner->writeInt8(1); // frames per sample
5297 mOwner->endBox();
5298 }
5299
writeUrlBox()5300 void MPEG4Writer::Track::writeUrlBox() {
5301 // The table index here refers to the sample description index
5302 // in the sample table entries.
5303 mOwner->beginBox("url ");
5304 mOwner->writeInt32(1); // version=0, flags=1 (self-contained)
5305 mOwner->endBox(); // url
5306 }
5307
writeDrefBox()5308 void MPEG4Writer::Track::writeDrefBox() {
5309 mOwner->beginBox("dref");
5310 mOwner->writeInt32(0); // version=0, flags=0
5311 mOwner->writeInt32(1); // entry count (either url or urn)
5312 writeUrlBox();
5313 mOwner->endBox(); // dref
5314 }
5315
writeDinfBox()5316 void MPEG4Writer::Track::writeDinfBox() {
5317 mOwner->beginBox("dinf");
5318 writeDrefBox();
5319 mOwner->endBox(); // dinf
5320 }
5321
writeAvccBox()5322 void MPEG4Writer::Track::writeAvccBox() {
5323 CHECK(mCodecSpecificData);
5324 CHECK_GE(mCodecSpecificDataSize, 5u);
5325
5326 // Patch avcc's lengthSize field to match the number
5327 // of bytes we use to indicate the size of a nal unit.
5328 uint8_t *ptr = (uint8_t *)mCodecSpecificData;
5329 ptr[4] = (ptr[4] & 0xfc) | (mOwner->useNalLengthFour() ? 3 : 1);
5330 mOwner->beginBox("avcC");
5331 mOwner->write(mCodecSpecificData, mCodecSpecificDataSize);
5332 mOwner->endBox(); // avcC
5333 }
5334
writeHvccBox()5335 void MPEG4Writer::Track::writeHvccBox() {
5336 CHECK(mCodecSpecificData);
5337 CHECK_GE(mCodecSpecificDataSize, 5u);
5338
5339 // Patch hvcc's lengthSize field to match the number
5340 // of bytes we use to indicate the size of a nal unit.
5341 uint8_t *ptr = (uint8_t *)mCodecSpecificData;
5342 ptr[21] = (ptr[21] & 0xfc) | (mOwner->useNalLengthFour() ? 3 : 1);
5343 mOwner->beginBox("hvcC");
5344 mOwner->write(mCodecSpecificData, mCodecSpecificDataSize);
5345 mOwner->endBox(); // hvcC
5346 }
5347
writeAv1cBox()5348 void MPEG4Writer::Track::writeAv1cBox() {
5349 CHECK(mCodecSpecificData);
5350 CHECK_GE(mCodecSpecificDataSize, 4u);
5351
5352 mOwner->beginBox("av1C");
5353 mOwner->write(mCodecSpecificData, mCodecSpecificDataSize);
5354 mOwner->endBox(); // av1C
5355 }
5356
writeApvcBox()5357 void MPEG4Writer::Track::writeApvcBox() {
5358 CHECK(mCodecSpecificData);
5359 CHECK_GE(mCodecSpecificDataSize, 4u);
5360
5361 mOwner->beginBox("apvC");
5362 // apvC extends FullBox and hence the need to write first
5363 // 4 bytes here when compared with av1C which extends Box.
5364 mOwner->writeInt32(0); // version=0, flags=0
5365 mOwner->write(mCodecSpecificData, mCodecSpecificDataSize);
5366 mOwner->endBox(); // apvC
5367 }
5368
writeDoviConfigBox()5369 void MPEG4Writer::Track::writeDoviConfigBox() {
5370 CHECK_NE(mDoviProfile, 0u);
5371
5372 uint32_t type = 0;
5373 const void *data = nullptr;
5374 size_t size = 0;
5375 // check to see which key has the configuration box.
5376 if (mMeta->findData(kKeyDVCC, &type, &data, &size) ||
5377 mMeta->findData(kKeyDVVC, &type, &data, &size) ||
5378 mMeta->findData(kKeyDVWC, &type, &data, &size)) {
5379
5380 // if this box is present we write the box, or
5381 // this mp4 will be interpreted as a backward
5382 // compatible stream.
5383 if (mDoviProfile > DolbyVisionProfileDvav110) {
5384 mOwner->beginBox("dvwC");
5385 } else if (mDoviProfile > DolbyVisionProfileDvheDtb) {
5386 mOwner->beginBox("dvvC");
5387 } else {
5388 mOwner->beginBox("dvcC");
5389 }
5390 mOwner->write(data, size);
5391 mOwner->endBox(); // dvwC/dvvC/dvcC
5392 }
5393 }
5394
writeD263Box()5395 void MPEG4Writer::Track::writeD263Box() {
5396 mOwner->beginBox("d263");
5397 mOwner->writeInt32(0); // vendor
5398 mOwner->writeInt8(0); // decoder version
5399 mOwner->writeInt8(10); // level: 10
5400 mOwner->writeInt8(0); // profile: 0
5401 mOwner->endBox(); // d263
5402 }
5403
5404 // This is useful if the pixel is not square
writePaspBox()5405 void MPEG4Writer::Track::writePaspBox() {
5406 // Do not write 'pasp' box unless the track format specifies it.
5407 // According to ISO/IEC 14496-12 (ISO base media file format), 'pasp' box
5408 // is optional. If present, it overrides the SAR from the video CSD. Only
5409 // set it if the track format specifically requests that.
5410 int32_t hSpacing, vSpacing;
5411 if (mMeta->findInt32(kKeySARWidth, &hSpacing) && (hSpacing > 0)
5412 && mMeta->findInt32(kKeySARHeight, &vSpacing) && (vSpacing > 0)) {
5413 mOwner->beginBox("pasp");
5414 mOwner->writeInt32(hSpacing); // hspacing
5415 mOwner->writeInt32(vSpacing); // vspacing
5416 mOwner->endBox(); // pasp
5417 }
5418 }
5419
getStartTimeOffsetTimeUs() const5420 int64_t MPEG4Writer::Track::getStartTimeOffsetTimeUs() const {
5421 int64_t trackStartTimeOffsetUs = 0;
5422 int64_t moovStartTimeUs = mOwner->getStartTimestampUs();
5423 if (mStartTimestampUs != -1 && mStartTimestampUs != moovStartTimeUs) {
5424 CHECK_GT(mStartTimestampUs, moovStartTimeUs);
5425 trackStartTimeOffsetUs = mStartTimestampUs - moovStartTimeUs;
5426 }
5427 return trackStartTimeOffsetUs;
5428 }
5429
getStartTimeOffsetScaledTime() const5430 int32_t MPEG4Writer::Track::getStartTimeOffsetScaledTime() const {
5431 return (getStartTimeOffsetTimeUs() * mTimeScale + 500000LL) / 1000000LL;
5432 }
5433
writeSttsBox()5434 void MPEG4Writer::Track::writeSttsBox() {
5435 mOwner->beginBox("stts");
5436 mOwner->writeInt32(0); // version=0, flags=0
5437 mSttsTableEntries->write(mOwner);
5438 mOwner->endBox(); // stts
5439 }
5440
writeCttsBox()5441 void MPEG4Writer::Track::writeCttsBox() {
5442 // There is no B frame at all
5443 if (mMinCttsOffsetTicks == mMaxCttsOffsetTicks) {
5444 return;
5445 }
5446
5447 // Do not write ctts box when there is no need to have it.
5448 if (mCttsTableEntries->count() == 0) {
5449 return;
5450 }
5451
5452 ALOGV("ctts box has %d entries with range [%" PRId64 ", %" PRId64 "]",
5453 mCttsTableEntries->count(), mMinCttsOffsetTicks, mMaxCttsOffsetTicks);
5454
5455 mOwner->beginBox("ctts");
5456 mOwner->writeInt32(0); // version=0, flags=0
5457 // Adjust ctts entries to have only offset needed for reordering frames.
5458 int64_t deltaTimeUs = mMinCttsOffsetTimeUs;
5459 ALOGV("ctts deltaTimeUs:%" PRId64, deltaTimeUs);
5460 int64_t delta = (deltaTimeUs * mTimeScale + 500000LL) / 1000000LL;
5461 mCttsTableEntries->adjustEntries([delta](size_t /* ix */, uint32_t (&value)[2]) {
5462 // entries are <count, ctts> pairs; adjust only ctts
5463 uint32_t duration = htonl(value[1]); // back to host byte order
5464 // Prevent overflow and underflow
5465 if (delta > duration) {
5466 duration = 0;
5467 } else if (delta < 0 && UINT32_MAX + delta < duration) {
5468 duration = UINT32_MAX;
5469 } else {
5470 duration -= delta;
5471 }
5472 value[1] = htonl(duration);
5473 });
5474 mCttsTableEntries->write(mOwner);
5475 mOwner->endBox(); // ctts
5476 }
5477
writeStssBox()5478 void MPEG4Writer::Track::writeStssBox() {
5479 mOwner->beginBox("stss");
5480 mOwner->writeInt32(0); // version=0, flags=0
5481 mStssTableEntries->write(mOwner);
5482 mOwner->endBox(); // stss
5483 }
5484
writeStszBox()5485 void MPEG4Writer::Track::writeStszBox() {
5486 mOwner->beginBox("stsz");
5487 mOwner->writeInt32(0); // version=0, flags=0
5488 mOwner->writeInt32(0);
5489 mStszTableEntries->write(mOwner);
5490 mOwner->endBox(); // stsz
5491 }
5492
writeStscBox()5493 void MPEG4Writer::Track::writeStscBox() {
5494 mOwner->beginBox("stsc");
5495 mOwner->writeInt32(0); // version=0, flags=0
5496 mStscTableEntries->write(mOwner);
5497 mOwner->endBox(); // stsc
5498 }
5499
writeCo64Box()5500 void MPEG4Writer::Track::writeCo64Box() {
5501 mOwner->beginBox("co64");
5502 mOwner->writeInt32(0); // version=0, flags=0
5503 mCo64TableEntries->write(mOwner);
5504 mOwner->endBox(); // stco or co64
5505 }
5506
writeUdtaBox()5507 void MPEG4Writer::writeUdtaBox() {
5508 beginBox("udta");
5509 writeGeoDataBox();
5510 endBox();
5511 }
5512
writeHdlr(const char * handlerType)5513 void MPEG4Writer::writeHdlr(const char *handlerType) {
5514 beginBox("hdlr");
5515 writeInt32(0); // Version, Flags
5516 writeInt32(0); // Predefined
5517 writeFourcc(handlerType);
5518 writeInt32(0); // Reserved[0]
5519 writeInt32(0); // Reserved[1]
5520 writeInt32(0); // Reserved[2]
5521 writeInt8(0); // Name (empty)
5522 endBox();
5523 }
5524
writeKeys()5525 void MPEG4Writer::writeKeys() {
5526 size_t count = mMetaKeys->countEntries();
5527
5528 beginBox("keys");
5529 writeInt32(0); // Version, Flags
5530 writeInt32(count); // Entry_count
5531 for (size_t i = 0; i < count; i++) {
5532 AMessage::Type type;
5533 const char *key = mMetaKeys->getEntryNameAt(i, &type);
5534 size_t n = strlen(key);
5535 writeInt32(n + 8);
5536 writeFourcc("mdta");
5537 write(key, n); // write without the \0
5538 }
5539 endBox();
5540 }
5541
writeIlst()5542 void MPEG4Writer::writeIlst() {
5543 size_t count = mMetaKeys->countEntries();
5544
5545 beginBox("ilst");
5546 for (size_t i = 0; i < count; i++) {
5547 beginBox(i + 1); // key id (1-based)
5548 beginBox("data");
5549 AMessage::Type type;
5550 const char *key = mMetaKeys->getEntryNameAt(i, &type);
5551 switch (type) {
5552 case AMessage::kTypeString:
5553 {
5554 AString val;
5555 CHECK(mMetaKeys->findString(key, &val));
5556 writeInt32(1); // type = UTF8
5557 writeInt32(0); // default country/language
5558 write(val.c_str(), strlen(val.c_str())); // write without \0
5559 break;
5560 }
5561
5562 case AMessage::kTypeFloat:
5563 {
5564 float val;
5565 CHECK(mMetaKeys->findFloat(key, &val));
5566 writeInt32(23); // type = float32
5567 writeInt32(0); // default country/language
5568 writeInt32(*reinterpret_cast<int32_t *>(&val));
5569 break;
5570 }
5571
5572 case AMessage::kTypeInt32:
5573 {
5574 int32_t val;
5575 CHECK(mMetaKeys->findInt32(key, &val));
5576 writeInt32(67); // type = signed int32
5577 writeInt32(0); // default country/language
5578 writeInt32(val);
5579 break;
5580 }
5581
5582 default:
5583 {
5584 ALOGW("Unsupported key type, writing 0 instead");
5585 writeInt32(77); // type = unsigned int32
5586 writeInt32(0); // default country/language
5587 writeInt32(0);
5588 break;
5589 }
5590 }
5591 endBox(); // data
5592 endBox(); // key id
5593 }
5594 endBox(); // ilst
5595 }
5596
writeMoovLevelMetaBox()5597 void MPEG4Writer::writeMoovLevelMetaBox() {
5598 size_t count = mMetaKeys->countEntries();
5599 if (count == 0) {
5600 return;
5601 }
5602
5603 beginBox("meta");
5604 writeHdlr("mdta");
5605 writeKeys();
5606 writeIlst();
5607 endBox();
5608 }
5609
writeIlocBox()5610 void MPEG4Writer::writeIlocBox() {
5611 beginBox("iloc");
5612 // Use version 1 to allow construction method 1 that refers to
5613 // data in idat box inside meta box.
5614 writeInt32(0x01000000); // Version = 1, Flags = 0
5615 writeInt16(0x4400); // offset_size = length_size = 4
5616 // base_offset_size = index_size = 0
5617
5618 // 16-bit item_count
5619 size_t itemCount = mItems.size();
5620 if (itemCount > 65535) {
5621 ALOGW("Dropping excess items: itemCount %zu", itemCount);
5622 itemCount = 65535;
5623 }
5624 writeInt16((uint16_t)itemCount);
5625
5626 for (auto it = mItems.begin(); it != mItems.end(); it++) {
5627 ItemInfo &item = it->second;
5628
5629 writeInt16(item.itemId);
5630 bool isGrid = item.isGrid();
5631
5632 writeInt16(isGrid ? 1 : 0); // construction_method
5633 writeInt16(0); // data_reference_index = 0
5634 writeInt16(1); // extent_count = 1
5635
5636 if (isGrid) {
5637 // offset into the 'idat' box
5638 writeInt32(mNumGrids++ * 8);
5639 writeInt32(8);
5640 } else {
5641 writeInt32(item.offset);
5642 writeInt32(item.size);
5643 }
5644 }
5645 endBox();
5646 }
5647
writeInfeBox(uint16_t itemId,const char * itemType,uint32_t flags)5648 void MPEG4Writer::writeInfeBox(
5649 uint16_t itemId, const char *itemType, uint32_t flags) {
5650 beginBox("infe");
5651 writeInt32(0x02000000 | flags); // Version = 2, Flags = 0
5652 writeInt16(itemId);
5653 writeInt16(0); //item_protection_index = 0
5654 writeFourcc(itemType);
5655 writeCString(""); // item_name
5656 endBox();
5657 }
5658
writeIinfBox()5659 void MPEG4Writer::writeIinfBox() {
5660 beginBox("iinf");
5661 writeInt32(0); // Version = 0, Flags = 0
5662
5663 // 16-bit item_count
5664 size_t itemCount = mItems.size();
5665 if (itemCount > 65535) {
5666 ALOGW("Dropping excess items: itemCount %zu", itemCount);
5667 itemCount = 65535;
5668 }
5669
5670 writeInt16((uint16_t)itemCount);
5671 for (auto it = mItems.begin(); it != mItems.end(); it++) {
5672 ItemInfo &item = it->second;
5673
5674 writeInfeBox(item.itemId, item.itemType,
5675 (item.isImage() && item.isHidden) ? 1 : 0);
5676 }
5677
5678 endBox();
5679 }
5680
writeIdatBox()5681 void MPEG4Writer::writeIdatBox() {
5682 beginBox("idat");
5683
5684 for (auto it = mItems.begin(); it != mItems.end(); it++) {
5685 ItemInfo &item = it->second;
5686
5687 if (item.isGrid()) {
5688 writeInt8(0); // version
5689 // flags == 1 means 32-bit width,height
5690 int8_t flags = (item.width > 65535 || item.height > 65535);
5691 writeInt8(flags);
5692 writeInt8(item.rows - 1);
5693 writeInt8(item.cols - 1);
5694 if (flags) {
5695 writeInt32(item.width);
5696 writeInt32(item.height);
5697 } else {
5698 writeInt16((uint16_t)item.width);
5699 writeInt16((uint16_t)item.height);
5700 }
5701 }
5702 }
5703
5704 endBox();
5705 }
5706
writeIrefBox()5707 void MPEG4Writer::writeIrefBox() {
5708 beginBox("iref");
5709 writeInt32(0); // Version = 0, Flags = 0
5710 {
5711 for (auto it = mItems.begin(); it != mItems.end(); it++) {
5712 ItemInfo &item = it->second;
5713
5714 for (size_t r = 0; r < item.refsList.size(); r++) {
5715 const ItemRefs &refs = item.refsList[r];
5716 beginBox(refs.key);
5717 writeInt16(item.itemId);
5718 size_t refCount = refs.value.size();
5719 if (refCount > 65535) {
5720 ALOGW("too many entries in %s", refs.key);
5721 refCount = 65535;
5722 }
5723 writeInt16((uint16_t)refCount);
5724 for (size_t refIndex = 0; refIndex < refCount; refIndex++) {
5725 writeInt16(refs.value[refIndex]);
5726 }
5727 endBox();
5728 }
5729 }
5730 }
5731 endBox();
5732 }
5733
writePitmBox()5734 void MPEG4Writer::writePitmBox() {
5735 beginBox("pitm");
5736 writeInt32(0); // Version = 0, Flags = 0
5737 writeInt16(mPrimaryItemId);
5738 endBox();
5739 }
5740
writeGrplBox(const Vector<uint16_t> & items)5741 void MPEG4Writer::writeGrplBox(const Vector<uint16_t> &items) {
5742 if (flags_camera::camera_heif_gainmap()) {
5743 beginBox("grpl");
5744 beginBox("altr");
5745 writeInt32(0); // Version = 0, Flags = 0
5746 writeInt32(1); // Group Id
5747 writeInt32(items.size());// Number of entities
5748 for (size_t i = 0; i < items.size(); i++) {
5749 writeInt32(items[i]);// Item Id
5750 }
5751 endBox();
5752 endBox();
5753 }
5754 }
5755
writeIpcoBox()5756 void MPEG4Writer::writeIpcoBox() {
5757 beginBox("ipco");
5758 size_t numProperties = mProperties.size();
5759 if (numProperties > 32767) {
5760 ALOGW("Dropping excess properties: numProperties %zu", numProperties);
5761 numProperties = 32767;
5762 }
5763 for (size_t propIndex = 0; propIndex < numProperties; propIndex++) {
5764 switch (mProperties[propIndex].type) {
5765 case FOURCC('h', 'v', 'c', 'C'):
5766 {
5767 beginBox("hvcC");
5768 sp<ABuffer> hvcc = mProperties[propIndex].data;
5769 // Patch avcc's lengthSize field to match the number
5770 // of bytes we use to indicate the size of a nal unit.
5771 uint8_t *ptr = (uint8_t *)hvcc->data();
5772 ptr[21] = (ptr[21] & 0xfc) | (useNalLengthFour() ? 3 : 1);
5773 write(hvcc->data(), hvcc->size());
5774 endBox();
5775 break;
5776 }
5777 case FOURCC('a', 'v', '1', 'C'):
5778 {
5779 beginBox("av1C");
5780 sp<ABuffer> av1c = mProperties[propIndex].data;
5781 write(av1c->data(), av1c->size());
5782 endBox();
5783 break;
5784 }
5785 case FOURCC('i', 's', 'p', 'e'):
5786 {
5787 beginBox("ispe");
5788 writeInt32(0); // Version = 0, Flags = 0
5789 writeInt32(mProperties[propIndex].width);
5790 writeInt32(mProperties[propIndex].height);
5791 endBox();
5792 break;
5793 }
5794 case FOURCC('i', 'r', 'o', 't'):
5795 {
5796 beginBox("irot");
5797 writeInt8(mProperties[propIndex].rotation);
5798 endBox();
5799 break;
5800 }
5801 case FOURCC('c', 'o', 'l', 'r'):
5802 {
5803 if (flags_camera::camera_heif_gainmap()) {
5804 beginBox("colr");
5805 writeFourcc("nclx");
5806 writeInt16(mProperties[propIndex].colorPrimaries);
5807 writeInt16(mProperties[propIndex].colorTransfer);
5808 writeInt16(mProperties[propIndex].colorMatrix);
5809 writeInt8(int8_t(mProperties[propIndex].colorRange ? 0x80 : 0x0));
5810 endBox();
5811 }
5812 break;
5813 }
5814 case FOURCC('p', 'i', 'x', 'i'):
5815 {
5816 if (flags_camera::camera_heif_gainmap()) {
5817 beginBox("pixi");
5818 writeInt32(0); // Version = 0, Flags = 0
5819 writeInt8(mProperties[propIndex].bitsPerChannel.size()); // Number of channels
5820 for (size_t i = 0; i < mProperties[propIndex].bitsPerChannel.size(); i++) {
5821 writeInt8(mProperties[propIndex].bitsPerChannel[i]); // Channel bit depth
5822 }
5823 endBox();
5824 }
5825 break;
5826 }
5827 default:
5828 ALOGW("Skipping unrecognized property: type 0x%08x",
5829 mProperties[propIndex].type);
5830 }
5831 }
5832 endBox();
5833 }
5834
writeIpmaBox()5835 void MPEG4Writer::writeIpmaBox() {
5836 beginBox("ipma");
5837 uint32_t flags = (mProperties.size() > 127) ? 1 : 0;
5838 writeInt32(flags); // Version = 0
5839
5840 writeInt32(mAssociationEntryCount);
5841 for (auto it = mItems.begin(); it != mItems.end(); it++) {
5842 ItemInfo &item = it->second;
5843
5844 const Vector<uint16_t> &properties = item.properties;
5845 if (properties.empty()) {
5846 continue;
5847 }
5848 writeInt16(item.itemId);
5849
5850 size_t entryCount = properties.size();
5851 if (entryCount > 255) {
5852 ALOGW("Dropping excess associations: entryCount %zu", entryCount);
5853 entryCount = 255;
5854 }
5855 writeInt8((uint8_t)entryCount);
5856 for (size_t propIndex = 0; propIndex < entryCount; propIndex++) {
5857 if (flags & 1) {
5858 writeInt16((1 << 15) | properties[propIndex]);
5859 } else {
5860 writeInt8((1 << 7) | properties[propIndex]);
5861 }
5862 }
5863 }
5864 endBox();
5865 }
5866
writeIprpBox()5867 void MPEG4Writer::writeIprpBox() {
5868 beginBox("iprp");
5869 writeIpcoBox();
5870 writeIpmaBox();
5871 endBox();
5872 }
5873
writeFileLevelMetaBox()5874 void MPEG4Writer::writeFileLevelMetaBox() {
5875 // patch up the mPrimaryItemId and count items with prop associations
5876 uint16_t firstVisibleItemId = 0;
5877 uint16_t firstImageItemId = 0;
5878 for (auto it = mItems.begin(); it != mItems.end(); it++) {
5879 ItemInfo &item = it->second;
5880
5881 if (item.isGainmapMeta() && !item.properties.empty() &&
5882 flags_camera::camera_heif_gainmap()) {
5883 mAssociationEntryCount++;
5884 continue;
5885 }
5886
5887 if (!item.isImage()) continue;
5888
5889 if (item.isPrimary) {
5890 mPrimaryItemId = item.itemId;
5891 }
5892 if (!firstImageItemId) {
5893 firstImageItemId = item.itemId;
5894 }
5895 if (!firstVisibleItemId && !item.isHidden) {
5896 firstVisibleItemId = item.itemId;
5897 }
5898 if (!item.properties.empty()) {
5899 mAssociationEntryCount++;
5900 }
5901 }
5902
5903 if (!firstImageItemId) {
5904 ALOGE("no valid image was found");
5905 return;
5906 }
5907
5908 if (mPrimaryItemId == 0) {
5909 if (firstVisibleItemId > 0) {
5910 ALOGW("didn't find primary, using first visible image");
5911 mPrimaryItemId = firstVisibleItemId;
5912 } else {
5913 ALOGW("no primary and no visible item, using first image");
5914 mPrimaryItemId = firstImageItemId;
5915 }
5916 }
5917
5918 uint16_t gainmapItemId = 0;
5919 uint16_t gainmapMetaItemId = 0;
5920 for (List<Track *>::iterator it = mTracks.begin();
5921 it != mTracks.end(); ++it) {
5922 if ((*it)->isHeif()) {
5923 (*it)->flushItemRefs();
5924 }
5925 if (flags_camera::camera_heif_gainmap()) {
5926 if ((*it)->getGainmapItemId() > 0) {
5927 gainmapItemId = (*it)->getGainmapItemId();
5928 }
5929 if ((*it)->getGainmapMetaItemId() > 0) {
5930 gainmapMetaItemId = (*it)->getGainmapMetaItemId();
5931 }
5932 }
5933 }
5934 if ((gainmapItemId > 0) && (gainmapMetaItemId > 0) && flags_camera::camera_heif_gainmap()) {
5935 ItemRefs gainmapRefs("dimg");
5936 gainmapRefs.value.push_back(mPrimaryItemId);
5937 gainmapRefs.value.push_back(gainmapItemId);
5938 addRefs_l(gainmapMetaItemId, gainmapRefs);
5939 }
5940
5941 beginBox("meta");
5942 writeInt32(0); // Version = 0, Flags = 0
5943 writeHdlr("pict");
5944 writeIlocBox();
5945 writeIinfBox();
5946 writePitmBox();
5947 writeIprpBox();
5948 if (mNumGrids > 0) {
5949 writeIdatBox();
5950 }
5951 if (mHasRefs) {
5952 writeIrefBox();
5953 }
5954 if ((gainmapItemId > 0) && (gainmapMetaItemId > 0) && flags_camera::camera_heif_gainmap()) {
5955 Vector<uint16_t> itemIds;
5956 itemIds.push_back(gainmapMetaItemId);
5957 itemIds.push_back(mPrimaryItemId);
5958 writeGrplBox(itemIds);
5959 }
5960 endBox();
5961 }
5962
addProperty_l(const ItemProperty & prop)5963 uint16_t MPEG4Writer::addProperty_l(const ItemProperty &prop) {
5964 char typeStr[5];
5965 MakeFourCCString(prop.type, typeStr);
5966 ALOGV("addProperty_l: %s", typeStr);
5967
5968 mProperties.push_back(prop);
5969
5970 // returning 1-based property index
5971 return mProperties.size();
5972 }
5973
reserveItemId_l(size_t numItems,uint16_t * itemIdBase)5974 status_t MPEG4Writer::reserveItemId_l(size_t numItems, uint16_t *itemIdBase) {
5975 if (numItems > UINT16_MAX - mNextItemId) {
5976 ALOGE("couldn't reserve item ids for %zu items", numItems);
5977 return ERROR_OUT_OF_RANGE;
5978 }
5979 *itemIdBase = mNextItemId;
5980 mNextItemId += numItems;
5981 return OK;
5982 }
5983
addItem_l(const ItemInfo & info)5984 uint16_t MPEG4Writer::addItem_l(const ItemInfo &info) {
5985 ALOGV("addItem_l: type %s, offset %u, size %u",
5986 info.itemType, info.offset, info.size);
5987
5988 if (info.itemId < kItemIdBase || info.itemId >= mNextItemId) {
5989 ALOGW("Item id %u is used without reservation!", info.itemId);
5990 }
5991
5992 mItems[info.itemId] = info;
5993
5994 #if (LOG_NDEBUG==0)
5995 if (!info.properties.empty()) {
5996 AString str;
5997 for (size_t i = 0; i < info.properties.size(); i++) {
5998 if (i > 0) {
5999 str.append(", ");
6000 }
6001 str.append(info.properties[i]);
6002 }
6003 ALOGV("addItem_l: id %d, properties: %s", info.itemId, str.c_str());
6004 }
6005 #endif // (LOG_NDEBUG==0)
6006
6007 return info.itemId;
6008 }
6009
addRefs_l(uint16_t itemId,const ItemRefs & refs)6010 void MPEG4Writer::addRefs_l(uint16_t itemId, const ItemRefs &refs) {
6011 if (refs.value.empty()) {
6012 return;
6013 }
6014 if (itemId < kItemIdBase || itemId >= mNextItemId) {
6015 ALOGW("itemId %u for ref is invalid!", itemId);
6016 return;
6017 }
6018
6019 auto it = mItems.find(itemId);
6020 if (it == mItems.end()) {
6021 ALOGW("itemId %u was not added yet", itemId);
6022 return;
6023 }
6024 it->second.refsList.push_back(refs);
6025 mHasRefs = true;
6026 }
6027
6028 /*
6029 * Geodata is stored according to ISO-6709 standard.
6030 */
writeGeoDataBox()6031 void MPEG4Writer::writeGeoDataBox() {
6032 beginBox("\xA9xyz");
6033 /*
6034 * For historical reasons, any user data start
6035 * with "\0xA9", must be followed by its assoicated
6036 * language code.
6037 * 0x0012: text string length
6038 * 0x15c7: lang (locale) code: en
6039 */
6040 writeInt32(0x001215c7);
6041 writeLatitude(mLatitudex10000);
6042 writeLongitude(mLongitudex10000);
6043 writeInt8(0x2F);
6044 endBox();
6045 }
6046
6047 } // namespace android
6048