1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/frame_buffer.h"
12
13 #include <assert.h>
14 #include <string.h>
15
16 #include "api/video/encoded_image.h"
17 #include "api/video/video_timing.h"
18 #include "modules/video_coding/include/video_codec_interface.h"
19 #include "modules/video_coding/packet.h"
20 #include "rtc_base/checks.h"
21 #include "rtc_base/logging.h"
22 #include "rtc_base/trace_event.h"
23
24 namespace webrtc {
25
VCMFrameBuffer()26 VCMFrameBuffer::VCMFrameBuffer()
27 : _state(kStateEmpty), _nackCount(0), _latestPacketTimeMs(-1) {}
28
~VCMFrameBuffer()29 VCMFrameBuffer::~VCMFrameBuffer() {}
30
FrameType() const31 webrtc::VideoFrameType VCMFrameBuffer::FrameType() const {
32 return _sessionInfo.FrameType();
33 }
34
GetLowSeqNum() const35 int32_t VCMFrameBuffer::GetLowSeqNum() const {
36 return _sessionInfo.LowSequenceNumber();
37 }
38
GetHighSeqNum() const39 int32_t VCMFrameBuffer::GetHighSeqNum() const {
40 return _sessionInfo.HighSequenceNumber();
41 }
42
PictureId() const43 int VCMFrameBuffer::PictureId() const {
44 return _sessionInfo.PictureId();
45 }
46
TemporalId() const47 int VCMFrameBuffer::TemporalId() const {
48 return _sessionInfo.TemporalId();
49 }
50
LayerSync() const51 bool VCMFrameBuffer::LayerSync() const {
52 return _sessionInfo.LayerSync();
53 }
54
Tl0PicId() const55 int VCMFrameBuffer::Tl0PicId() const {
56 return _sessionInfo.Tl0PicId();
57 }
58
GetNaluInfos() const59 std::vector<NaluInfo> VCMFrameBuffer::GetNaluInfos() const {
60 return _sessionInfo.GetNaluInfos();
61 }
62
SetGofInfo(const GofInfoVP9 & gof_info,size_t idx)63 void VCMFrameBuffer::SetGofInfo(const GofInfoVP9& gof_info, size_t idx) {
64 TRACE_EVENT0("webrtc", "VCMFrameBuffer::SetGofInfo");
65 _sessionInfo.SetGofInfo(gof_info, idx);
66 // TODO(asapersson): Consider adding hdr->VP9.ref_picture_id for testing.
67 _codecSpecificInfo.codecSpecific.VP9.temporal_idx =
68 gof_info.temporal_idx[idx];
69 _codecSpecificInfo.codecSpecific.VP9.temporal_up_switch =
70 gof_info.temporal_up_switch[idx];
71 }
72
IsSessionComplete() const73 bool VCMFrameBuffer::IsSessionComplete() const {
74 TRACE_EVENT0("webrtc", "VCMFrameBuffer::IsSessionComplete");
75 return _sessionInfo.complete();
76 }
77
78 // Insert packet
InsertPacket(const VCMPacket & packet,int64_t timeInMs,const FrameData & frame_data)79 VCMFrameBufferEnum VCMFrameBuffer::InsertPacket(const VCMPacket& packet,
80 int64_t timeInMs,
81 const FrameData& frame_data) {
82 TRACE_EVENT0("webrtc", "VCMFrameBuffer::InsertPacket");
83 assert(!(NULL == packet.dataPtr && packet.sizeBytes > 0));
84 if (packet.dataPtr != NULL) {
85 _payloadType = packet.payloadType;
86 }
87
88 if (kStateEmpty == _state) {
89 // First packet (empty and/or media) inserted into this frame.
90 // store some info and set some initial values.
91 SetTimestamp(packet.timestamp);
92 // We only take the ntp timestamp of the first packet of a frame.
93 ntp_time_ms_ = packet.ntp_time_ms_;
94 _codec = packet.codec();
95 if (packet.video_header.frame_type != VideoFrameType::kEmptyFrame) {
96 // first media packet
97 SetState(kStateIncomplete);
98 }
99 }
100
101 uint32_t requiredSizeBytes =
102 size() + packet.sizeBytes +
103 (packet.insertStartCode ? kH264StartCodeLengthBytes : 0);
104 if (requiredSizeBytes > capacity()) {
105 const uint8_t* prevBuffer = data();
106 const uint32_t increments =
107 requiredSizeBytes / kBufferIncStepSizeBytes +
108 (requiredSizeBytes % kBufferIncStepSizeBytes > 0);
109 const uint32_t newSize = capacity() + increments * kBufferIncStepSizeBytes;
110 if (newSize > kMaxJBFrameSizeBytes) {
111 RTC_LOG(LS_ERROR) << "Failed to insert packet due to frame being too "
112 "big.";
113 return kSizeError;
114 }
115 if (data() == nullptr) {
116 encoded_image_buffer_ = EncodedImageBuffer::Create(newSize);
117 SetEncodedData(encoded_image_buffer_);
118 set_size(0);
119 } else {
120 RTC_CHECK(encoded_image_buffer_ != nullptr);
121 RTC_DCHECK_EQ(encoded_image_buffer_->data(), data());
122 encoded_image_buffer_->Realloc(newSize);
123 }
124 _sessionInfo.UpdateDataPointers(prevBuffer, data());
125 }
126
127 if (packet.width() > 0 && packet.height() > 0) {
128 _encodedWidth = packet.width();
129 _encodedHeight = packet.height();
130 }
131
132 // Don't copy payload specific data for empty packets (e.g padding packets).
133 if (packet.sizeBytes > 0)
134 CopyCodecSpecific(&packet.video_header);
135
136 int retVal = _sessionInfo.InsertPacket(packet, data(), frame_data);
137 if (retVal == -1) {
138 return kSizeError;
139 } else if (retVal == -2) {
140 return kDuplicatePacket;
141 } else if (retVal == -3) {
142 return kOutOfBoundsPacket;
143 }
144 // update size
145 set_size(size() + static_cast<uint32_t>(retVal));
146
147 _latestPacketTimeMs = timeInMs;
148
149 // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
150 // ts_126114v120700p.pdf Section 7.4.5.
151 // The MTSI client shall add the payload bytes as defined in this clause
152 // onto the last RTP packet in each group of packets which make up a key
153 // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
154 // (HEVC)).
155 if (packet.markerBit) {
156 rotation_ = packet.video_header.rotation;
157 content_type_ = packet.video_header.content_type;
158 if (packet.video_header.video_timing.flags != VideoSendTiming::kInvalid) {
159 timing_.encode_start_ms =
160 ntp_time_ms_ + packet.video_header.video_timing.encode_start_delta_ms;
161 timing_.encode_finish_ms =
162 ntp_time_ms_ +
163 packet.video_header.video_timing.encode_finish_delta_ms;
164 timing_.packetization_finish_ms =
165 ntp_time_ms_ +
166 packet.video_header.video_timing.packetization_finish_delta_ms;
167 timing_.pacer_exit_ms =
168 ntp_time_ms_ + packet.video_header.video_timing.pacer_exit_delta_ms;
169 timing_.network_timestamp_ms =
170 ntp_time_ms_ +
171 packet.video_header.video_timing.network_timestamp_delta_ms;
172 timing_.network2_timestamp_ms =
173 ntp_time_ms_ +
174 packet.video_header.video_timing.network2_timestamp_delta_ms;
175 }
176 timing_.flags = packet.video_header.video_timing.flags;
177 }
178
179 if (packet.is_first_packet_in_frame()) {
180 playout_delay_ = packet.video_header.playout_delay;
181 }
182
183 if (_sessionInfo.complete()) {
184 SetState(kStateComplete);
185 return kCompleteSession;
186 }
187 return kIncomplete;
188 }
189
LatestPacketTimeMs() const190 int64_t VCMFrameBuffer::LatestPacketTimeMs() const {
191 TRACE_EVENT0("webrtc", "VCMFrameBuffer::LatestPacketTimeMs");
192 return _latestPacketTimeMs;
193 }
194
IncrementNackCount()195 void VCMFrameBuffer::IncrementNackCount() {
196 TRACE_EVENT0("webrtc", "VCMFrameBuffer::IncrementNackCount");
197 _nackCount++;
198 }
199
GetNackCount() const200 int16_t VCMFrameBuffer::GetNackCount() const {
201 TRACE_EVENT0("webrtc", "VCMFrameBuffer::GetNackCount");
202 return _nackCount;
203 }
204
HaveFirstPacket() const205 bool VCMFrameBuffer::HaveFirstPacket() const {
206 TRACE_EVENT0("webrtc", "VCMFrameBuffer::HaveFirstPacket");
207 return _sessionInfo.HaveFirstPacket();
208 }
209
NumPackets() const210 int VCMFrameBuffer::NumPackets() const {
211 TRACE_EVENT0("webrtc", "VCMFrameBuffer::NumPackets");
212 return _sessionInfo.NumPackets();
213 }
214
Reset()215 void VCMFrameBuffer::Reset() {
216 TRACE_EVENT0("webrtc", "VCMFrameBuffer::Reset");
217 set_size(0);
218 _sessionInfo.Reset();
219 _payloadType = 0;
220 _nackCount = 0;
221 _latestPacketTimeMs = -1;
222 _state = kStateEmpty;
223 VCMEncodedFrame::Reset();
224 }
225
226 // Set state of frame
SetState(VCMFrameBufferStateEnum state)227 void VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state) {
228 TRACE_EVENT0("webrtc", "VCMFrameBuffer::SetState");
229 if (_state == state) {
230 return;
231 }
232 switch (state) {
233 case kStateIncomplete:
234 // we can go to this state from state kStateEmpty
235 assert(_state == kStateEmpty);
236
237 // Do nothing, we received a packet
238 break;
239
240 case kStateComplete:
241 assert(_state == kStateEmpty || _state == kStateIncomplete);
242
243 break;
244
245 case kStateEmpty:
246 // Should only be set to empty through Reset().
247 assert(false);
248 break;
249 }
250 _state = state;
251 }
252
253 // Get current state of frame
GetState() const254 VCMFrameBufferStateEnum VCMFrameBuffer::GetState() const {
255 return _state;
256 }
257
PrepareForDecode(bool continuous)258 void VCMFrameBuffer::PrepareForDecode(bool continuous) {
259 TRACE_EVENT0("webrtc", "VCMFrameBuffer::PrepareForDecode");
260 size_t bytes_removed = _sessionInfo.MakeDecodable();
261 set_size(size() - bytes_removed);
262 // Transfer frame information to EncodedFrame and create any codec
263 // specific information.
264 _frameType = _sessionInfo.FrameType();
265 _completeFrame = _sessionInfo.complete();
266 _missingFrame = !continuous;
267 }
268
269 } // namespace webrtc
270