1 /**
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <ImsMediaDefine.h>
18 #include <VideoRtpPayloadDecoderNode.h>
19 #include <ImsMediaVideoUtil.h>
20 #include <ImsMediaTrace.h>
21 #include <VideoConfig.h>
22 #include <stdio.h>
23 #include <stdlib.h>
24 #include <string.h>
25
26 using namespace android::telephony::imsmedia;
27
VideoRtpPayloadDecoderNode(BaseSessionCallback * callback)28 VideoRtpPayloadDecoderNode::VideoRtpPayloadDecoderNode(BaseSessionCallback* callback) :
29 BaseNode(callback)
30 {
31 mCodecType = 0;
32 mPayloadMode = 0;
33 mBuffer = nullptr;
34 mSbitfirstByte = 0;
35 }
36
~VideoRtpPayloadDecoderNode()37 VideoRtpPayloadDecoderNode::~VideoRtpPayloadDecoderNode() {}
38
GetNodeId()39 kBaseNodeId VideoRtpPayloadDecoderNode::GetNodeId()
40 {
41 return kNodeIdVideoPayloadDecoder;
42 }
43
Start()44 ImsMediaResult VideoRtpPayloadDecoderNode::Start()
45 {
46 IMLOGD2("[Start] Codec[%d], PayloadMode[%d]", mCodecType, mPayloadMode);
47
48 mBuffer = reinterpret_cast<uint8_t*>(malloc(MAX_RTP_PAYLOAD_BUFFER_SIZE * sizeof(uint8_t)));
49
50 if (mBuffer == nullptr)
51 {
52 return RESULT_NO_MEMORY;
53 }
54
55 mNodeState = kNodeStateRunning;
56 return RESULT_SUCCESS;
57 }
58
Stop()59 void VideoRtpPayloadDecoderNode::Stop()
60 {
61 if (mBuffer != nullptr)
62 {
63 free(mBuffer);
64 mBuffer = nullptr;
65 }
66
67 mNodeState = kNodeStateStopped;
68 }
69
IsRunTime()70 bool VideoRtpPayloadDecoderNode::IsRunTime()
71 {
72 return true;
73 }
74
IsSourceNode()75 bool VideoRtpPayloadDecoderNode::IsSourceNode()
76 {
77 return false;
78 }
79
SetConfig(void * config)80 void VideoRtpPayloadDecoderNode::SetConfig(void* config)
81 {
82 if (config == nullptr)
83 {
84 return;
85 }
86
87 VideoConfig* pConfig = reinterpret_cast<VideoConfig*>(config);
88 mCodecType = pConfig->getCodecType();
89 mPayloadMode = pConfig->getPacketizationMode();
90 }
91
IsSameConfig(void * config)92 bool VideoRtpPayloadDecoderNode::IsSameConfig(void* config)
93 {
94 if (config == nullptr)
95 {
96 return false;
97 }
98
99 VideoConfig* pConfig = reinterpret_cast<VideoConfig*>(config);
100 return (mCodecType == pConfig->getCodecType() &&
101 mPayloadMode == pConfig->getPacketizationMode());
102 }
103
OnDataFromFrontNode(ImsMediaSubType subtype,uint8_t * pData,uint32_t nDataSize,uint32_t nTimeStamp,bool bMark,uint32_t nSeqNum,ImsMediaSubType nDataType,uint32_t arrivalTime)104 void VideoRtpPayloadDecoderNode::OnDataFromFrontNode(ImsMediaSubType subtype, uint8_t* pData,
105 uint32_t nDataSize, uint32_t nTimeStamp, bool bMark, uint32_t nSeqNum,
106 ImsMediaSubType nDataType, uint32_t arrivalTime)
107 {
108 if (subtype == MEDIASUBTYPE_REFRESHED)
109 {
110 SendDataToRearNode(subtype, nullptr, nDataSize, 0, 0, 0, MEDIASUBTYPE_UNDEFINED);
111 return;
112 }
113
114 switch (mCodecType)
115 {
116 case VideoConfig::CODEC_AVC:
117 DecodeAvc(subtype, pData, nDataSize, nTimeStamp, bMark, nSeqNum);
118 break;
119 case VideoConfig::CODEC_HEVC:
120 DecodeHevc(subtype, pData, nDataSize, nTimeStamp, bMark, nSeqNum);
121 break;
122 default:
123 IMLOGE1("[OnDataFromFrontNode] invalid codec type[%d]", mCodecType);
124 SendDataToRearNode(MEDIASUBTYPE_UNDEFINED, pData, nDataSize, nTimeStamp, bMark, nSeqNum,
125 nDataType, arrivalTime);
126 break;
127 }
128 }
129
DecodeAvc(ImsMediaSubType subtype,uint8_t * pData,uint32_t nDataSize,uint32_t nTimeStamp,bool bMark,uint32_t nSeqNum)130 void VideoRtpPayloadDecoderNode::DecodeAvc(ImsMediaSubType subtype, uint8_t* pData,
131 uint32_t nDataSize, uint32_t nTimeStamp, bool bMark, uint32_t nSeqNum)
132 {
133 if (pData == nullptr || nDataSize == 0 || mBuffer == nullptr)
134 {
135 return;
136 }
137
138 // check packet type
139 uint8_t bPacketType = pData[0] & 0x1F;
140 ImsMediaSubType eDataType = MEDIASUBTYPE_UNDEFINED;
141
142 // make start code prefix
143 mBuffer[0] = 0x00;
144 mBuffer[1] = 0x00;
145 mBuffer[2] = 0x00;
146 mBuffer[3] = 0x01;
147
148 IMLOGD_PACKET2(IM_PACKET_LOG_PH, "[DecodeAvc] [%02X] nDataSize[%d]", pData[0], nDataSize);
149
150 if (bPacketType >= 1 && bPacketType <= 23)
151 { // Single NAL unit packet
152 memcpy(mBuffer + 4, pData, nDataSize);
153
154 if ((bPacketType & 0x1F) == 7 || (bPacketType & 0x1F) == 8)
155 {
156 eDataType = MEDIASUBTYPE_VIDEO_CONFIGSTRING;
157 }
158 else if ((bPacketType & 0x1F) == 5)
159 {
160 eDataType = MEDIASUBTYPE_VIDEO_IDR_FRAME;
161 }
162 else if ((bPacketType & 0x1F) == 6)
163 {
164 eDataType = MEDIASUBTYPE_VIDEO_SEI_FRAME;
165 }
166 else
167 {
168 eDataType = MEDIASUBTYPE_VIDEO_NON_IDR_FRAME;
169 }
170
171 SendDataToRearNode(subtype, mBuffer, nDataSize + 4, nTimeStamp, bMark, nSeqNum, eDataType);
172 }
173 else if (bPacketType == 24)
174 { // STAP-A
175 uint8_t* pCurrData = pData + 1;
176 int32_t nRemainSize = (int32_t)(nDataSize - 1);
177
178 if (mPayloadMode == kRtpPyaloadHeaderModeSingleNalUnit)
179 {
180 IMLOGD_PACKET0(IM_PACKET_LOG_PH, "[DecodeAvc] Warning - single nal unit mode");
181 }
182
183 bPacketType = pCurrData[2] & 0x1F;
184
185 if ((bPacketType & 0x1F) == 7 || (bPacketType & 0x1F) == 8)
186 {
187 eDataType = MEDIASUBTYPE_VIDEO_CONFIGSTRING;
188 }
189 else if ((bPacketType & 0x1F) == 5)
190 { // check idr frame
191 eDataType = MEDIASUBTYPE_VIDEO_IDR_FRAME;
192 }
193 else
194 {
195 eDataType = MEDIASUBTYPE_VIDEO_NON_IDR_FRAME;
196 }
197
198 IMLOGD2("[DecodeAvc] eDataType[%u], nRemainSize[%u]", eDataType, nRemainSize);
199
200 while (nRemainSize > 2)
201 {
202 // read NAL unit size
203 uint32_t nNALUnitsize = pCurrData[0];
204 nNALUnitsize = (nNALUnitsize << 8) + pCurrData[1];
205 IMLOGD_PACKET1(IM_PACKET_LOG_PH, "[DecodeAvc] STAP-A nNALUnitsize[%d]", nNALUnitsize);
206 pCurrData += 2;
207 nRemainSize -= 2;
208 // Read and Send NAL
209 if (nRemainSize >= (int32_t)nNALUnitsize)
210 {
211 IMLOGD_PACKET1(IM_PACKET_LOG_PH, "[DecodeAvc] STAP-A [%02X] nNALUnitsize[%d]",
212 nNALUnitsize);
213 memcpy(mBuffer + 4, pCurrData, nNALUnitsize);
214 SendDataToRearNode(
215 subtype, mBuffer, nNALUnitsize + 4, nTimeStamp, bMark, nSeqNum, eDataType);
216 }
217 pCurrData += nNALUnitsize;
218 nRemainSize -= nNALUnitsize;
219 }
220 }
221 else if (bPacketType == 28)
222 { // FU-A
223 uint8_t bFUIndicator;
224 uint8_t bFUHeader;
225 uint8_t bNALUnitType;
226 uint8_t bStartBit;
227 uint8_t bEndBit;
228
229 if (mPayloadMode == kRtpPyaloadHeaderModeSingleNalUnit)
230 {
231 IMLOGW0("[DecodeAvc] Warning - (FU-A, 28) for single nal unit mode");
232 }
233
234 bFUIndicator = pData[0];
235 bFUHeader = pData[1];
236 bNALUnitType = (bFUIndicator & 0xE0) | (bFUHeader & 0x1F);
237 bStartBit = (bFUHeader >> 7) & 0x01;
238 bEndBit = (bFUHeader >> 6) & 0x01;
239
240 if ((bNALUnitType & 0x1F) == 7 || (bNALUnitType & 0x1F) == 8)
241 {
242 eDataType = MEDIASUBTYPE_VIDEO_CONFIGSTRING;
243 }
244 else if ((bNALUnitType & 0x1F) == 5)
245 { // check idr frame
246 eDataType = MEDIASUBTYPE_VIDEO_IDR_FRAME;
247 }
248 else
249 {
250 eDataType = MEDIASUBTYPE_VIDEO_NON_IDR_FRAME;
251 }
252
253 if (bStartBit)
254 {
255 mBuffer[4] = bNALUnitType; // for video decoder
256 memcpy(mBuffer + 5, pData + 2, nDataSize - 2);
257
258 if (bEndBit)
259 {
260 SendDataToRearNode(
261 subtype, mBuffer, nDataSize + 3, nTimeStamp, bMark, nSeqNum, eDataType);
262 }
263 else
264 {
265 SendDataToRearNode(
266 subtype, mBuffer, nDataSize + 3, nTimeStamp, bEndBit, nSeqNum, eDataType);
267 }
268 }
269 else
270 {
271 SendDataToRearNode(
272 subtype, pData + 2, nDataSize - 2, nTimeStamp, bEndBit, nSeqNum, eDataType);
273 }
274 }
275 else
276 {
277 IMLOGE1("[DecodeAvc] Unsupported payload type[%d]", bPacketType);
278 }
279 }
280
DecodeHevc(ImsMediaSubType subtype,uint8_t * pData,uint32_t nDataSize,uint32_t nTimeStamp,bool bMark,uint32_t nSeqNum)281 void VideoRtpPayloadDecoderNode::DecodeHevc(ImsMediaSubType subtype, uint8_t* pData,
282 uint32_t nDataSize, uint32_t nTimeStamp, bool bMark, uint32_t nSeqNum)
283 {
284 if (subtype == MEDIASUBTYPE_REFRESHED)
285 {
286 IMLOGD0("[DecodeHevc] REFRESHED");
287 SendDataToRearNode(subtype, 0, 0, 0, 0, 0, MEDIASUBTYPE_UNDEFINED);
288 return;
289 }
290
291 if (pData == nullptr || nDataSize == 0)
292 {
293 IMLOGE1("[DecodeHevc] INVALID Data, Size[%d]", nDataSize);
294 return;
295 }
296
297 if (mBuffer == nullptr)
298 {
299 return;
300 }
301
302 // check packet type
303 uint8_t bPacketType = (pData[0] & 0x7E) >> 1;
304 ImsMediaSubType eDataType = MEDIASUBTYPE_UNDEFINED;
305
306 // Please check Decoder Start Code...
307 // make start code prefix
308 mBuffer[0] = 0x00;
309 mBuffer[1] = 0x00;
310 mBuffer[2] = 0x00;
311 mBuffer[3] = 0x01;
312
313 IMLOGD_PACKET3(IM_PACKET_LOG_PH, "[DecodeHevc] [%02X %02X] nDataSize[%d]", pData[0], pData[1],
314 nDataSize);
315
316 if (bPacketType <= 40)
317 { // Single NAL unit packet
318 memcpy(mBuffer + 4, pData, nDataSize);
319
320 if (bPacketType >= 32 && bPacketType <= 34)
321 { // 32: VPS, 33: SPS, 34: PPS
322 eDataType = MEDIASUBTYPE_VIDEO_CONFIGSTRING;
323 }
324 else if (bPacketType == 19 || bPacketType == 20)
325 { // IDR
326 eDataType = MEDIASUBTYPE_VIDEO_IDR_FRAME;
327 }
328 else
329 {
330 eDataType = MEDIASUBTYPE_VIDEO_NON_IDR_FRAME;
331 }
332
333 SendDataToRearNode(subtype, mBuffer, nDataSize + 4, nTimeStamp, bMark, nSeqNum, eDataType);
334 }
335 else if (bPacketType == 48)
336 { // Aggregation packet(AP)
337 // need to implement
338 }
339 else if (bPacketType == 49)
340 { // FU-A
341 uint8_t bFUIndicator1;
342 uint8_t bFUIndicator2;
343 uint8_t bFUHeader;
344 uint8_t bNALUnitType;
345 uint8_t bStartBit;
346 uint8_t bEndBit;
347
348 if (mPayloadMode == kRtpPyaloadHeaderModeSingleNalUnit)
349 {
350 IMLOGW0("[DecodeHevc] Warning - invalid packet type(FU, 49) for single nal unit mode");
351 }
352
353 bFUIndicator1 = pData[0];
354 bFUIndicator2 = pData[1];
355 bFUHeader = pData[2];
356 bNALUnitType = (bFUIndicator1 & 0x81) | ((bFUHeader & 0x3F) << 1);
357 bStartBit = (bFUHeader >> 7) & 0x01;
358 bEndBit = (bFUHeader >> 6) & 0x01;
359
360 uint8_t frameType = (bNALUnitType & 0x7E) >> 1;
361
362 if (frameType >= 32 && frameType <= 34)
363 { // 32: VPS, 33: SPS, 34: PPS
364 eDataType = MEDIASUBTYPE_VIDEO_CONFIGSTRING;
365 }
366 else if (frameType == 19 || frameType == 20)
367 { // IDR
368 eDataType = MEDIASUBTYPE_VIDEO_IDR_FRAME;
369 }
370 else
371 {
372 eDataType = MEDIASUBTYPE_VIDEO_NON_IDR_FRAME;
373 }
374
375 if (bStartBit)
376 {
377 mBuffer[4] = bNALUnitType; // for decoder
378 mBuffer[5] = bFUIndicator2;
379 memcpy(mBuffer + 6, pData + 3, nDataSize - 3);
380 // exclude FU header
381 SendDataToRearNode(
382 subtype, mBuffer, nDataSize + 3, nTimeStamp, bEndBit, nSeqNum, eDataType);
383 }
384 else
385 { // exclude start code
386 SendDataToRearNode(
387 subtype, pData + 3, nDataSize - 3, nTimeStamp, bEndBit, nSeqNum, eDataType);
388 }
389 }
390 else
391 {
392 IMLOGE1("[DecodeHevc] Unsupported payload type[%d]", bPacketType);
393 }
394 }
395