1 /**
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <algorithm>
18 #include <VideoJitterBuffer.h>
19 #include <ImsMediaDataQueue.h>
20 #include <ImsMediaTrace.h>
21 #include <ImsMediaVideoUtil.h>
22 #include <ImsMediaTimer.h>
23
24 #define DEFAULT_MAX_SAVE_FRAME_NUM (5)
25 #define DEFAULT_IDR_FRAME_CHECK_INTRERVAL (3)
26 #define DEFAULT_VIDEO_JITTER_IDR_WAIT_DELAY (200)
27 #define RTCPNACK_SEQ_INCREASE(seq) ((seq) == 0xffff ? 0 : (seq) + 1)
28 #define RTCPNACK_SEQ_ROUND_COMPARE(a, b) (((a) > (b)) && ((a) > 0xfff0) && ((b) < 0x000f))
29 #define DEFAULT_PACKET_LOSS_MONITORING_TIME (5) // sec
30 #define PACKET_LOSS_RATIO (2) // percentage
31 #define BITRATE_ADAPTIVE_RATIO (0.1f) // ratio
32
VideoJitterBuffer()33 VideoJitterBuffer::VideoJitterBuffer() :
34 BaseJitterBuffer()
35 {
36 mCodecType = kVideoCodecAvc;
37 mNewInputData = false;
38 mFramerate = 15;
39 mFrameInterval = 1000 / mFramerate;
40 mMaxSaveFrameNum = DEFAULT_MAX_SAVE_FRAME_NUM;
41 mSavedFrameNum = 0;
42 mMarkedFrameNum = 0;
43 InitLostPktList();
44 mResponseWaitTime = 0;
45 mLastPlayedTime = 0;
46 mNumAddedPacket = 0;
47 mNumLossPacket = 0;
48 mAccumulatedPacketSize = 0;
49 mLastAddedTimestamp = 0;
50 mLastAddedSeqNum = 0;
51 mIDRCheckCnt = DEFAULT_IDR_FRAME_CHECK_INTRERVAL;
52 mFirTimeStamp = 0;
53 mMaxBitrate = 0;
54 mIncomingBitrate = 0;
55 mLossDuration = DEFAULT_PACKET_LOSS_MONITORING_TIME;
56 mLossRateThreshold = 0;
57 mCountTimerExpired = 0;
58 mTimer = nullptr;
59 }
60
~VideoJitterBuffer()61 VideoJitterBuffer::~VideoJitterBuffer()
62 {
63 InitLostPktList();
64
65 if (mTimer != nullptr)
66 {
67 IMLOGD0("[~VideoJitterBuffer] stop timer");
68 ImsMediaTimer::TimerStop(mTimer, nullptr);
69 mTimer = nullptr;
70 }
71 }
72
SetJitterBufferSize(uint32_t nInit,uint32_t nMin,uint32_t nMax)73 void VideoJitterBuffer::SetJitterBufferSize(uint32_t nInit, uint32_t nMin, uint32_t nMax)
74 {
75 BaseJitterBuffer::SetJitterBufferSize(nInit, nMin, nMax);
76 mMaxSaveFrameNum = mMaxJitterBufferSize * 20 / mFrameInterval;
77 mIDRCheckCnt = DEFAULT_VIDEO_JITTER_IDR_WAIT_DELAY / mFrameInterval;
78 mFirTimeStamp = 0;
79 IMLOGD2("[SetJitterBufferSize] maxSaveFrameNum[%u], IDRCheckCnt[%d]", mMaxSaveFrameNum,
80 mIDRCheckCnt);
81 }
82
SetCodecType(uint32_t type)83 void VideoJitterBuffer::SetCodecType(uint32_t type)
84 {
85 mCodecType = type;
86 }
87
SetFramerate(uint32_t framerate)88 void VideoJitterBuffer::SetFramerate(uint32_t framerate)
89 {
90 mFramerate = framerate;
91 mFrameInterval = 1000 / mFramerate;
92 IMLOGD2("[SetFramerate] framerate[%u], frameInterval[%d]", mFramerate, mFrameInterval);
93 }
94
InitLostPktList()95 void VideoJitterBuffer::InitLostPktList()
96 {
97 IMLOGD0("[InitLostPktList]");
98
99 while (!mLostPktList.empty())
100 {
101 LostPacket* entry = mLostPktList.front();
102
103 if (entry != nullptr)
104 {
105 delete entry;
106 }
107
108 mLostPktList.pop_front();
109 }
110
111 mLostPktList.clear();
112 }
113
Reset()114 void VideoJitterBuffer::Reset()
115 {
116 BaseJitterBuffer::Reset();
117 mSavedFrameNum = 0;
118 mMarkedFrameNum = 0;
119 mLastPlayedTime = 0;
120 mNumAddedPacket = 0;
121 mNumLossPacket = 0;
122 mAccumulatedPacketSize = 0;
123 mLastPlayedTimestamp = 0;
124 mLastAddedTimestamp = 0;
125 mLastAddedSeqNum = 0;
126 InitLostPktList();
127 mResponseWaitTime = 0;
128 }
129
StartTimer(uint32_t time,uint32_t rate)130 void VideoJitterBuffer::StartTimer(uint32_t time, uint32_t rate)
131 {
132 IMLOGD2("[StartTimer] time[%d], rate[%u]", time, rate);
133
134 if (mTimer == nullptr)
135 {
136 IMLOGD0("[StartTimer] timer start");
137 mTimer = ImsMediaTimer::TimerStart(1000, true, OnTimer, this);
138 }
139 }
140
StopTimer()141 void VideoJitterBuffer::StopTimer()
142 {
143 if (mTimer != nullptr)
144 {
145 IMLOGD0("[StopTimer] stop timer");
146
147 if (ImsMediaTimer::TimerStop(mTimer, nullptr) == false)
148 {
149 IMLOGE0("[StopTimer] stop timer error");
150 }
151
152 mTimer = nullptr;
153 }
154 }
155
SetResponseWaitTime(const uint32_t time)156 void VideoJitterBuffer::SetResponseWaitTime(const uint32_t time)
157 {
158 IMLOGD1("[SetResponseWaitTime] time[%u]", time);
159 mResponseWaitTime = time;
160 }
161
Add(ImsMediaSubType subtype,uint8_t * pbBuffer,uint32_t nBufferSize,uint32_t nTimestamp,bool bMark,uint32_t nSeqNum,ImsMediaSubType eDataType,uint32_t arrivalTime)162 void VideoJitterBuffer::Add(ImsMediaSubType subtype, uint8_t* pbBuffer, uint32_t nBufferSize,
163 uint32_t nTimestamp, bool bMark, uint32_t nSeqNum, ImsMediaSubType eDataType,
164 uint32_t arrivalTime)
165 {
166 if (subtype == MEDIASUBTYPE_REFRESHED)
167 {
168 // nBufferSize is ssrc value
169 mSsrc = nBufferSize;
170 IMLOGI1("[Add] ssrc[%u]", mSsrc);
171 return;
172 }
173
174 DataEntry currEntry = DataEntry();
175 currEntry.pbBuffer = pbBuffer;
176 currEntry.nBufferSize = nBufferSize;
177 currEntry.nTimestamp = nTimestamp;
178 currEntry.bMark = bMark;
179 currEntry.nSeqNum = nSeqNum;
180 currEntry.bHeader = CheckHeader(pbBuffer);
181 currEntry.bValid = false;
182 currEntry.subtype = subtype;
183 currEntry.eDataType = eDataType;
184 currEntry.arrivalTime = arrivalTime;
185
186 std::lock_guard<std::mutex> guard(mMutex);
187
188 if (currEntry.bHeader == true)
189 {
190 if (eDataType == MEDIASUBTYPE_VIDEO_CONFIGSTRING)
191 {
192 currEntry.bMark = true;
193 }
194
195 if (eDataType == MEDIASUBTYPE_VIDEO_SEI_FRAME)
196 {
197 IMLOGD1("[Add] ignore SEI frame seq[%d]", nSeqNum);
198 return;
199 }
200 }
201
202 IMLOGD_PACKET6(IM_PACKET_LOG_JITTER,
203 "[Add] eDataType[%u], Seq[%u], Mark[%u], Header[%u], TS[%u], Size[%u]",
204 currEntry.eDataType, nSeqNum, bMark, currEntry.bHeader, nTimestamp, nBufferSize);
205
206 // very old frame, don't add this frame, nothing to do
207 if ((!USHORT_SEQ_ROUND_COMPARE(nSeqNum, mLastPlayedSeqNum)) && (mLastPlayedTime != 0))
208 {
209 IMLOGE2("[Add] Receive very old frame!!! Drop Packet. Seq[%u], LastPlayedSeqNum[%u]",
210 nSeqNum, mLastPlayedSeqNum);
211 }
212 else if (mDataQueue.GetCount() == 0)
213 { // jitter buffer is empty
214 mDataQueue.Add(&currEntry);
215 mNumAddedPacket++;
216 mAccumulatedPacketSize += nBufferSize;
217 IMLOGD_PACKET4(IM_PACKET_LOG_JITTER,
218 "[Add] queue[%u] Seq[%u], LastPlayedSeqNum[%u], LastAddedTimestamp[%u]",
219 mDataQueue.GetCount(), nSeqNum, mLastPlayedSeqNum, mLastAddedTimestamp);
220 mLastAddedTimestamp = nTimestamp;
221 mLastAddedSeqNum = nSeqNum;
222 }
223 else
224 {
225 DataEntry* pEntry;
226 mDataQueue.GetLast(&pEntry);
227
228 if (pEntry == nullptr)
229 {
230 return;
231 }
232
233 if (USHORT_SEQ_ROUND_COMPARE(nSeqNum, pEntry->nSeqNum))
234 {
235 // current data is the latest data
236 if (nSeqNum == pEntry->nSeqNum && nBufferSize == pEntry->nBufferSize)
237 {
238 IMLOGD1("[Add] drop duplicate Seq[%u]", nSeqNum);
239 return;
240 }
241
242 mDataQueue.Add(&currEntry);
243 mNumAddedPacket++;
244 mAccumulatedPacketSize += nBufferSize;
245 IMLOGD_PACKET4(IM_PACKET_LOG_JITTER,
246 "[Add] queue[%u] Seq[%u], LastPlayedSeqNum[%u], LastAddedTimestamp[%u]",
247 mDataQueue.GetCount(), nSeqNum, mLastPlayedSeqNum, mLastAddedTimestamp);
248 }
249 else
250 {
251 // find the position of current data and insert current data to the correct position
252 uint32_t i;
253 mDataQueue.SetReadPosFirst();
254
255 for (i = 0; mDataQueue.GetNext(&pEntry); i++)
256 {
257 if (nSeqNum == pEntry->nSeqNum && nBufferSize == pEntry->nBufferSize)
258 {
259 IMLOGD1("[Add] drop duplicate Seq[%u]", nSeqNum);
260 return;
261 }
262
263 if (!USHORT_SEQ_ROUND_COMPARE(nSeqNum, pEntry->nSeqNum))
264 {
265 mDataQueue.InsertAt(i, &currEntry);
266 break;
267 }
268 }
269 }
270
271 // Remove mark of packets with same Timestamp and less SeqNum
272 mDataQueue.SetReadPosFirst();
273
274 while (mDataQueue.GetNext(&pEntry))
275 {
276 if (pEntry->nSeqNum != nSeqNum && pEntry->nTimestamp == nTimestamp &&
277 pEntry->bMark == true && pEntry->eDataType != MEDIASUBTYPE_VIDEO_CONFIGSTRING &&
278 USHORT_SEQ_ROUND_COMPARE(nSeqNum, pEntry->nSeqNum))
279 {
280 IMLOGD_PACKET6(IM_PACKET_LOG_JITTER,
281 "[Add] Remove marker of Seq/TS/Mark[%u/%u/%u], pEntry "
282 "Seq/TS/Mark[%u/%u/%u]",
283 pEntry->nSeqNum, pEntry->nTimestamp, pEntry->bMark, currEntry.nSeqNum,
284 currEntry.nTimestamp, currEntry.bMark);
285 pEntry->bMark = false;
286 continue;
287 }
288 else if (!USHORT_SEQ_ROUND_COMPARE(nSeqNum, pEntry->nSeqNum))
289 {
290 break;
291 }
292 }
293
294 mLastAddedTimestamp = nTimestamp;
295 mLastAddedSeqNum = nSeqNum;
296 }
297
298 mNewInputData = true;
299 }
300
Get(ImsMediaSubType * subtype,uint8_t ** ppData,uint32_t * pnDataSize,uint32_t * pnTimestamp,bool * pbMark,uint32_t * pnSeqNum,uint32_t currentTime,ImsMediaSubType *)301 bool VideoJitterBuffer::Get(ImsMediaSubType* subtype, uint8_t** ppData, uint32_t* pnDataSize,
302 uint32_t* pnTimestamp, bool* pbMark, uint32_t* pnSeqNum, uint32_t currentTime,
303 ImsMediaSubType* /*pDataType*/)
304 {
305 DataEntry* pEntry = nullptr;
306 bool bValidPacket = false;
307 std::lock_guard<std::mutex> guard(mMutex);
308
309 // check validation
310 if (mNewInputData)
311 {
312 mSavedFrameNum = 0;
313 mMarkedFrameNum = 0;
314 bool bFoundHeader = false;
315 uint16_t nLastRecvSeq = 0; // for NACK generation
316 mDataQueue.SetReadPosFirst();
317
318 uint32_t nIndex = 0;
319 uint32_t nHeaderIndex = 0;
320 uint16_t nHeaderSeq = 0;
321 uint32_t nHeaderTimestamp = 0;
322 uint32_t nLastTimeStamp = 0;
323 uint32_t nSavedIdrFrame = 0;
324 for (nIndex = 0; mDataQueue.GetNext(&pEntry); nIndex++)
325 {
326 IMLOGD_PACKET8(IM_PACKET_LOG_JITTER,
327 "[Get] queue[%u/%u] bValid[%u], Seq[%u], Mark[%u], Header[%u], TS[%u], "
328 "Size[%u]",
329 nIndex, mDataQueue.GetCount(), pEntry->bValid, pEntry->nSeqNum, pEntry->bMark,
330 pEntry->bHeader, pEntry->nTimestamp, pEntry->nBufferSize);
331
332 if (mResponseWaitTime > 0 && nLastTimeStamp != 0)
333 {
334 CheckPacketLoss(pEntry->nSeqNum, nLastRecvSeq);
335 }
336
337 nLastRecvSeq = pEntry->nSeqNum;
338
339 if (pEntry->nTimestamp != nLastTimeStamp || nLastTimeStamp == 0)
340 {
341 if (pEntry->eDataType != MEDIASUBTYPE_VIDEO_CONFIGSTRING)
342 {
343 if (pEntry->eDataType == MEDIASUBTYPE_VIDEO_IDR_FRAME)
344 {
345 nSavedIdrFrame++;
346 }
347
348 mSavedFrameNum++;
349 nLastTimeStamp = pEntry->nTimestamp;
350 }
351 }
352
353 if (pEntry->bMark)
354 {
355 mMarkedFrameNum++;
356 }
357
358 if (nSavedIdrFrame == mIDRCheckCnt && pEntry->eDataType == MEDIASUBTYPE_VIDEO_IDR_FRAME)
359 {
360 /** TODO: improve this logic later */
361 CheckValidIDR(pEntry);
362 }
363
364 IMLOGD_PACKET3(IM_PACKET_LOG_JITTER,
365 "[Get] SavedFrameNum[%u], mMarkedFrameNum[%u], nLastTimeStamp[%u]",
366 mSavedFrameNum, mMarkedFrameNum, nLastTimeStamp);
367
368 if (pEntry->bHeader)
369 {
370 if (pEntry->bMark)
371 {
372 pEntry->bValid = true;
373
374 if (pEntry->nTimestamp > nHeaderTimestamp)
375 {
376 bFoundHeader = false; // make Header false only for new frame
377 }
378 }
379 else
380 {
381 // Check Header with new timestamp. Otherwise(duplicated header with
382 // same timestamp), do not update Header Information
383 if (bFoundHeader == false || pEntry->nTimestamp < nHeaderTimestamp ||
384 nHeaderTimestamp == 0)
385 {
386 nHeaderTimestamp = pEntry->nTimestamp;
387 nHeaderIndex = nIndex;
388 nHeaderSeq = pEntry->nSeqNum;
389 bFoundHeader = true;
390
391 IMLOGD_PACKET3(IM_PACKET_LOG_JITTER,
392 "[Get] New Header Found at [%u] - Seq[%u], Timestamp[%u]",
393 nHeaderIndex, nHeaderSeq, nHeaderTimestamp);
394 }
395 }
396 }
397
398 if (bFoundHeader)
399 {
400 IMLOGD_PACKET4(IM_PACKET_LOG_JITTER,
401 "[Get] bFoundHeader[%u] - Check Valid of Seq[%u ~ %u], "
402 "nHeaderTimestamp[%u]",
403 bFoundHeader, nHeaderSeq, pEntry->nSeqNum, nHeaderTimestamp);
404
405 if (pEntry->bMark)
406 {
407 uint32_t nMarkIndex = nIndex;
408 uint16_t nMarkSeq = pEntry->nSeqNum;
409
410 // make sure type of 16bit unsigned int sequence number
411 if (nMarkIndex - nHeaderIndex == nMarkSeq - nHeaderSeq)
412 {
413 uint32_t i;
414
415 for (i = nHeaderIndex; i <= nMarkIndex; i++)
416 {
417 DataEntry* pValidEntry;
418 mDataQueue.GetAt(i, &pValidEntry);
419
420 if (pValidEntry == nullptr)
421 {
422 return false;
423 }
424
425 pValidEntry->bValid = true;
426
427 IMLOGD_PACKET3(IM_PACKET_LOG_JITTER,
428 "[Get] Validation Check for Seq[%u] true :: nHeaderIndex[%u] "
429 "to nMarkIndex[%u]",
430 pValidEntry->nSeqNum, nHeaderIndex, nMarkIndex);
431 }
432 }
433
434 bFoundHeader = false;
435 }
436 }
437 }
438
439 if (mSavedFrameNum > mMaxSaveFrameNum)
440 {
441 IMLOGD_PACKET2(IM_PACKET_LOG_JITTER,
442 "[Get] Delete - SavedFrameNum[%u], nMaxFrameNum[%u]", mSavedFrameNum,
443 mMaxSaveFrameNum);
444
445 mDataQueue.Get(&pEntry);
446
447 if (pEntry == nullptr)
448 return false;
449
450 if (pEntry->bValid == false)
451 {
452 uint32_t nDeleteTimeStamp = pEntry->nTimestamp;
453 uint32_t nDeleteSeqNum = pEntry->nSeqNum;
454
455 while (nDeleteTimeStamp == pEntry->nTimestamp)
456 {
457 IMLOGD_PACKET7(IM_PACKET_LOG_JITTER,
458 "[Get] Delete - Seq[%u], Count[%u], bValid[%u], eDataType[%u], "
459 "bHeader[%u], TimeStamp[%u], Size[%u]",
460 pEntry->nSeqNum, mDataQueue.GetCount(), pEntry->bValid,
461 pEntry->eDataType, pEntry->bHeader, pEntry->nTimestamp,
462 pEntry->nBufferSize);
463
464 nDeleteSeqNum = pEntry->nSeqNum;
465 mDataQueue.Delete();
466 mDataQueue.Get(&pEntry); // next packet
467
468 if (pEntry == nullptr)
469 {
470 break;
471 }
472 }
473
474 mSavedFrameNum--;
475
476 // remove the packets from NACK / PLI checkList
477 if (mLostPktList.size() > 0)
478 {
479 RemovePacketFromLostList(nDeleteSeqNum, true);
480 }
481 }
482 }
483
484 if (mSavedFrameNum >= mMaxSaveFrameNum)
485 {
486 mDataQueue.Get(&pEntry);
487
488 if (pEntry == nullptr)
489 {
490 return false;
491 }
492
493 mLastPlayedSeqNum = pEntry->nSeqNum - 1;
494 }
495
496 mNewInputData = false;
497 }
498
499 if (mSavedFrameNum >= (mMaxSaveFrameNum / 2) && mDataQueue.Get(&pEntry) == true &&
500 pEntry->bValid && (mLastPlayedSeqNum == 0 || pEntry->nSeqNum <= mLastPlayedSeqNum + 1))
501 {
502 IMLOGD_PACKET4(IM_PACKET_LOG_JITTER,
503 "[Get] bValid[%u], LastPlayedTS[%u], Seq[%u], LastPlayedSeq[%u]", pEntry->bValid,
504 mLastPlayedTimestamp, pEntry->nSeqNum, mLastPlayedSeqNum);
505
506 uint32_t nCurrTime = currentTime;
507
508 if (mLastPlayedTimestamp == 0 || mLastPlayedTime == 0 ||
509 pEntry->nTimestamp == mLastPlayedTimestamp)
510 {
511 bValidPacket = true;
512 }
513 else
514 {
515 uint32_t nTimeDiff = nCurrTime - mLastPlayedTime;
516 uint32_t nThreshold;
517
518 // this is optimized in 15fps
519 if (mSavedFrameNum <= 2 && mMarkedFrameNum <= 1)
520 {
521 nThreshold = nTimeDiff;
522 }
523 else if (mSavedFrameNum <= 3)
524 {
525 nThreshold = nTimeDiff / 2;
526 }
527 else
528 {
529 nThreshold = nTimeDiff / ((mSavedFrameNum + 2) / 2);
530 }
531
532 if (nThreshold > 66)
533 {
534 nThreshold = 66; // 15fps
535 }
536
537 if (nTimeDiff >= nThreshold || (mLastPlayedTimestamp > pEntry->nTimestamp) ||
538 (mLastPlayedTime > nCurrTime))
539 {
540 bValidPacket = true;
541 }
542 else
543 {
544 IMLOGD_PACKET3(IM_PACKET_LOG_JITTER,
545 "[Get] bValidPacket[%u], nTimeDiff[%u], nThreshold[%u]", bValidPacket,
546 nTimeDiff, nThreshold);
547
548 bValidPacket = false;
549 }
550 }
551
552 if (bValidPacket)
553 {
554 mLastPlayedTimestamp = pEntry->nTimestamp;
555 mLastPlayedTime = nCurrTime;
556 }
557
558 mNewInputData = true;
559 }
560 else
561 {
562 bValidPacket = false;
563 }
564
565 if (bValidPacket && pEntry != nullptr)
566 {
567 if (subtype)
568 *subtype = pEntry->subtype;
569 if (ppData)
570 *ppData = pEntry->pbBuffer;
571 if (pnDataSize)
572 *pnDataSize = pEntry->nBufferSize;
573 if (pnTimestamp)
574 *pnTimestamp = pEntry->nTimestamp;
575 if (pbMark)
576 *pbMark = pEntry->bMark;
577 if (pnSeqNum)
578 *pnSeqNum = pEntry->nSeqNum;
579
580 mLastPlayedSeqNum = pEntry->nSeqNum;
581
582 IMLOGD_PACKET7(IM_PACKET_LOG_JITTER,
583 "[Get] Seq[%u], Mark[%u], TS[%u], Size[%u], SavedFrame[%u], MarkedFrame[%u], "
584 "queue[%u]",
585 pEntry->nSeqNum, pEntry->bMark, pEntry->nTimestamp, pEntry->nBufferSize,
586 mSavedFrameNum, mMarkedFrameNum, mDataQueue.GetCount());
587 return true;
588 }
589 else
590 {
591 if (subtype)
592 *subtype = MEDIASUBTYPE_UNDEFINED;
593 if (ppData)
594 *ppData = nullptr;
595 if (pnDataSize)
596 *pnDataSize = 0;
597 if (pnTimestamp)
598 *pnTimestamp = 0;
599 if (pbMark)
600 *pbMark = false;
601 if (pnSeqNum)
602 *pnSeqNum = 0;
603 IMLOGD_PACKET3(IM_PACKET_LOG_JITTER,
604 "[Get] false - SavedFrame[%u], MarkedFrame[%u], queue[%u]", mSavedFrameNum,
605 mMarkedFrameNum, mDataQueue.GetCount());
606 return false;
607 }
608 }
609
CheckValidIDR(DataEntry * pIDREntry)610 void VideoJitterBuffer::CheckValidIDR(DataEntry* pIDREntry)
611 {
612 if (pIDREntry == nullptr)
613 {
614 return;
615 }
616
617 if (pIDREntry->bValid == false)
618 {
619 IMLOGD2("[CheckValidIDR] mFirTimeStamp[%u] -> nTimestamp[%u]", mFirTimeStamp,
620 pIDREntry->nTimestamp);
621
622 if (pIDREntry->nTimestamp == mFirTimeStamp)
623 {
624 return;
625 }
626
627 RequestToSendPictureLost(kPsfbFir);
628 mFirTimeStamp = pIDREntry->nTimestamp;
629 return;
630 }
631 }
632
Delete()633 void VideoJitterBuffer::Delete()
634 {
635 DataEntry* pEntry;
636 std::lock_guard<std::mutex> guard(mMutex);
637 mDataQueue.Get(&pEntry);
638
639 if (pEntry == nullptr)
640 {
641 return;
642 }
643
644 IMLOGD_PACKET2(IM_PACKET_LOG_JITTER, "[Delete] Seq[%u] / BufferCount[%u]", pEntry->nSeqNum,
645 mDataQueue.GetCount());
646 mLastPlayedSeqNum = pEntry->nSeqNum;
647 mDataQueue.Delete();
648 mNewInputData = true;
649
650 if (mLostPktList.size() > 0)
651 {
652 RemovePacketFromLostList(mLastPlayedSeqNum, true);
653 }
654 }
655
GetCount()656 uint32_t VideoJitterBuffer::GetCount()
657 {
658 return mDataQueue.GetCount();
659 }
660
CheckHeader(uint8_t * pbBuffer)661 bool VideoJitterBuffer::CheckHeader(uint8_t* pbBuffer)
662 {
663 if (pbBuffer == nullptr)
664 {
665 return false;
666 }
667
668 // check start code
669 if ((pbBuffer[0] == 0x00 && pbBuffer[1] == 0x00 && pbBuffer[2] == 0x00) ||
670 (pbBuffer[0] == 0x00 && pbBuffer[1] == 0x00 && pbBuffer[2] == 0x01))
671 {
672 return true;
673 }
674 else
675 {
676 return false;
677 }
678 }
679
RemovePacketFromLostList(uint16_t seqNum,bool bRemoveOldPacket)680 void VideoJitterBuffer::RemovePacketFromLostList(uint16_t seqNum, bool bRemoveOldPacket)
681 {
682 LostPacket* pEntry = nullptr;
683 std::list<LostPacket*>::iterator it = mLostPktList.begin();
684
685 while (it != mLostPktList.end())
686 {
687 pEntry = *it;
688
689 if (pEntry == nullptr)
690 {
691 mLostPktList.erase(it++);
692 continue;
693 }
694
695 if (bRemoveOldPacket && pEntry->seqNum < seqNum)
696 {
697 IMLOGD_PACKET3(IM_PACKET_LOG_JITTER,
698 "[RemovePacketFromLostList] delete lost seq[%u], target seq[%u], "
699 "bRemoveOldPacket[%u]",
700 pEntry->seqNum, seqNum, bRemoveOldPacket);
701
702 it = mLostPktList.erase(it);
703 delete pEntry;
704 }
705 else if (pEntry->seqNum == seqNum)
706 {
707 IMLOGD_PACKET1(IM_PACKET_LOG_JITTER, "[RemovePacketFromLostList] remove lost seq[%u]",
708 pEntry->seqNum);
709
710 it = mLostPktList.erase(it);
711 delete pEntry;
712 }
713 else
714 {
715 it++;
716 }
717 }
718 }
719
CheckPacketLoss(uint16_t seqNum,uint16_t nLastRecvPkt)720 void VideoJitterBuffer::CheckPacketLoss(uint16_t seqNum, uint16_t nLastRecvPkt)
721 {
722 if (mLostPktList.size() > 0)
723 {
724 RemovePacketFromLostList(seqNum);
725 }
726
727 // normal case : no packet loss
728 if (RTCPNACK_SEQ_INCREASE(nLastRecvPkt) == seqNum)
729 {
730 return;
731 }
732
733 if (nLastRecvPkt == seqNum)
734 {
735 return; // same packet should be removed in STAP-A type case.
736 }
737
738 // the first lost packet.
739 uint16_t PID = RTCPNACK_SEQ_INCREASE(nLastRecvPkt);
740
741 // the number of lost packets
742 uint16_t nLossGap = RTCPNACK_SEQ_ROUND_COMPARE(nLastRecvPkt, seqNum)
743 ? ((0xffff - nLastRecvPkt) + seqNum)
744 : (seqNum - PID);
745
746 if (nLossGap > 0x000f)
747 {
748 nLossGap = 0x000f;
749 }
750
751 uint16_t countNackPacketNum = 0;
752 uint16_t countSecondNack = 0;
753 uint16_t nPLIPkt = 0;
754 bool bPLIPkt = false;
755 bool bSentPLI = false;
756
757 for (int32_t index = 0; index < nLossGap; index++)
758 {
759 if (UpdateLostPacketList(PID + index, &countSecondNack, &nPLIPkt, &bPLIPkt))
760 {
761 countNackPacketNum++;
762 }
763
764 // request PLI Message
765 if (bPLIPkt && !bSentPLI)
766 {
767 IMLOGD1("[CheckPacketLoss] nPLI pkt[%u]", nPLIPkt);
768 RequestToSendPictureLost(kPsfbPli);
769 bSentPLI = true;
770 }
771 }
772
773 if (nPLIPkt > PID)
774 {
775 PID = nPLIPkt + 1;
776 }
777
778 // return if PID is same as the current seq.
779 if (PID == seqNum)
780 {
781 return;
782 }
783
784 // request NACK Message
785 if (countNackPacketNum > 0)
786 {
787 RequestSendNack(countNackPacketNum, PID, countSecondNack);
788 }
789 }
790
UpdateLostPacketList(uint16_t lostSeq,uint16_t * countSecondNack,uint16_t * nPLIPkt,bool * bPLIPkt)791 bool VideoJitterBuffer::UpdateLostPacketList(
792 uint16_t lostSeq, uint16_t* countSecondNack, uint16_t* nPLIPkt, bool* bPLIPkt)
793 {
794 LostPacket* foundLostPacket = nullptr;
795 auto result = std::find_if(mLostPktList.begin(), mLostPktList.end(),
796 [lostSeq, &foundLostPacket](LostPacket* entry)
797 {
798 foundLostPacket = entry;
799 return (entry->seqNum == lostSeq);
800 });
801
802 if (result != mLostPktList.end() && foundLostPacket != nullptr)
803 {
804 return UpdateNackStatus(foundLostPacket, lostSeq, countSecondNack, nPLIPkt, bPLIPkt);
805 }
806
807 IMLOGD_PACKET2(IM_PACKET_LOG_JITTER, "[UpdateLostPacketList] add lost seq[%u], queue size[%d]",
808 lostSeq, mLostPktList.size());
809
810 LostPacket* entry =
811 new LostPacket(lostSeq, ImsMediaTimer::GetTimeInMilliSeconds(), kRequestSendNackNone);
812 mLostPktList.push_back(entry);
813 mNumLossPacket++;
814 return false;
815 }
816
UpdateNackStatus(LostPacket * pEntry,uint16_t lostSeq,uint16_t * countSecondNack,uint16_t * nPLIPkt,bool * bPLIPkt)817 bool VideoJitterBuffer::UpdateNackStatus(LostPacket* pEntry, uint16_t lostSeq,
818 uint16_t* countSecondNack, uint16_t* nPLIPkt, bool* bPLIPkt)
819 {
820 /**
821 * Send initial NACK if there is error in decoding frame due to packet loss
822 */
823 if (pEntry->option == kRequestSendNackNone)
824 {
825 if ((ImsMediaTimer::GetTimeInMilliSeconds() - pEntry->markedTime) < mFrameInterval)
826 {
827 return false;
828 }
829
830 pEntry->markedTime = ImsMediaTimer::GetTimeInMilliSeconds();
831 pEntry->option = kRequestInitialNack;
832 IMLOGD_PACKET1(IM_PACKET_LOG_JITTER, "[UpdateNackStatus] initial NACK, seq[%u]", lostSeq);
833 return true;
834 }
835
836 if ((ImsMediaTimer::GetTimeInMilliSeconds() - pEntry->markedTime) < mResponseWaitTime)
837 {
838 return false;
839 }
840
841 /**
842 * Send Second NACK if there is first packet still not arrived within RWT duration
843 */
844 if (pEntry->option == kRequestInitialNack)
845 {
846 (*countSecondNack)++;
847 pEntry->markedTime = ImsMediaTimer::GetTimeInMilliSeconds();
848 pEntry->option = kRequestSecondNack;
849 IMLOGD_PACKET1(IM_PACKET_LOG_JITTER, "[UpdateNackStatus] second NACK, seq[%u]", lostSeq);
850 return true;
851 }
852 else if (pEntry->option == kRequestSecondNack)
853 {
854 /**
855 * Send PLI if the recovery picture does not arrived within two RWT duration
856 */
857 *nPLIPkt = lostSeq;
858 *bPLIPkt = true;
859 pEntry->markedTime = ImsMediaTimer::GetTimeInMilliSeconds();
860 pEntry->option = kRequestPli;
861 IMLOGD_PACKET1(IM_PACKET_LOG_JITTER, "[UpdateNackStatus] request PLI seq[%u]", lostSeq);
862 }
863 else if (pEntry->option == kRequestPli)
864 {
865 *nPLIPkt = lostSeq;
866 pEntry->markedTime = ImsMediaTimer::GetTimeInMilliSeconds();
867 }
868
869 return false;
870 }
871
RequestSendNack(uint16_t nLossGap,uint16_t PID,uint16_t countSecondNack,bool bNACK)872 void VideoJitterBuffer::RequestSendNack(
873 uint16_t nLossGap, uint16_t PID, uint16_t countSecondNack, bool bNACK)
874 {
875 uint32_t BLP = 0x0; // bitmask of following lost packets
876
877 if (nLossGap > 1)
878 {
879 BLP = (0x01 << (nLossGap - 1)) - 1;
880 }
881
882 InternalRequestEventParam* pParam = new InternalRequestEventParam(
883 kRequestVideoSendNack, NackParams(PID, BLP, countSecondNack, bNACK));
884
885 IMLOGD0("[RequestSendNack]");
886 mCallback->SendEvent(kRequestVideoSendNack, reinterpret_cast<uint64_t>(pParam));
887 }
888
RequestToSendPictureLost(uint32_t type)889 void VideoJitterBuffer::RequestToSendPictureLost(uint32_t type)
890 {
891 IMLOGD0("[RequestToSendPictureLost]");
892 InternalRequestEventParam* pParam =
893 new InternalRequestEventParam(kRequestVideoSendPictureLost, type);
894 mCallback->SendEvent(kRequestVideoSendPictureLost, reinterpret_cast<uint64_t>(pParam));
895 }
896
RequestToSendTmmbr(uint32_t bitrate)897 void VideoJitterBuffer::RequestToSendTmmbr(uint32_t bitrate)
898 {
899 IMLOGD2("[RequestToSendTmmbr] ssrc[%x], bitrate[%d]", mSsrc, bitrate);
900 uint32_t exp = 0;
901 uint32_t mantissa = 0;
902 ImsMediaVideoUtil::ConvertBitrateToPower(bitrate, exp, mantissa);
903
904 InternalRequestEventParam* pParam =
905 new InternalRequestEventParam(kRtpFbTmmbr, TmmbrParams(mSsrc, exp, mantissa, 40));
906 mCallback->SendEvent(kRequestVideoSendTmmbr, reinterpret_cast<uint64_t>(pParam));
907 }
908
OnTimer(hTimerHandler hTimer,void * pUserData)909 void VideoJitterBuffer::OnTimer(hTimerHandler hTimer, void* pUserData)
910 {
911 (void)hTimer;
912 VideoJitterBuffer* jitter = reinterpret_cast<VideoJitterBuffer*>(pUserData);
913
914 if (jitter != nullptr)
915 {
916 jitter->ProcessTimer();
917 }
918 }
919
ProcessTimer()920 void VideoJitterBuffer::ProcessTimer()
921 {
922 mCountTimerExpired++;
923
924 /** calculate bitrate */
925 mIncomingBitrate = mAccumulatedPacketSize * 8;
926
927 if (mIncomingBitrate > mMaxBitrate)
928 {
929 mMaxBitrate = mIncomingBitrate;
930 }
931
932 IMLOGD_PACKET2(IM_PACKET_LOG_JITTER, "[ProcessTimer] bitrate[%d] maxBitrate[%d]",
933 mIncomingBitrate, mMaxBitrate);
934
935 mAccumulatedPacketSize = 0;
936
937 /** calculate loss rate in every seconds */
938 double lossRate =
939 static_cast<double>(mNumLossPacket) * 100 / (mNumAddedPacket + mNumLossPacket);
940
941 IMLOGD3("[ProcessTimer] rate[%lf], lossPackets[%d], addedPackets[%d]", lossRate, mNumLossPacket,
942 mNumAddedPacket);
943
944 if (mIncomingBitrate > 0)
945 {
946 CheckBitrateAdaptation(lossRate);
947 }
948
949 /** compare loss rate with threshold */
950 if (mLossDuration != 0 && mCountTimerExpired >= mLossDuration)
951 {
952 if (lossRate >= mLossRateThreshold && mLossRateThreshold != 0)
953 {
954 /** TODO: request send loss indication event */
955 }
956
957 mNumLossPacket = 0;
958 mNumAddedPacket = 0;
959 mCountTimerExpired = 0;
960 }
961 }
962
CheckBitrateAdaptation(double lossRate)963 void VideoJitterBuffer::CheckBitrateAdaptation(double lossRate)
964 {
965 if (mCountTimerExpired % DEFAULT_PACKET_LOSS_MONITORING_TIME == 0)
966 {
967 if (lossRate >= PACKET_LOSS_RATIO)
968 {
969 RequestToSendTmmbr(mIncomingBitrate);
970 }
971 else if (lossRate == 0)
972 {
973 mRequestedBitrate = mIncomingBitrate + mIncomingBitrate * BITRATE_ADAPTIVE_RATIO;
974
975 if (mRequestedBitrate > mMaxBitrate)
976 {
977 mRequestedBitrate = mMaxBitrate;
978 }
979
980 RequestToSendTmmbr(mRequestedBitrate);
981 }
982 }
983 }