1 /*
2 * Copyright (c) 2025-2025 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "sei_parser_helper.h"
17
18 #include <unordered_set>
19
20 #include "avcodec_trace.h"
21 #include "common/event.h"
22 #include "common/log.h"
23 #include "scope_guard.h"
24 #include "media_core.h"
25 #include "meta/format.h"
26 #include "plugin/plugin_time.h"
27
28 namespace {
29 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_SYSTEM_PLAYER, "SeiParserHelper" };
30
31 constexpr uint16_t ANNEX_B_PREFIX_LEN = 4;
32 constexpr uint8_t SEI_UUID_LEN = 16;
33 constexpr int32_t SEI_PAYLOAD_SIZE_MAX = 1024 * 1024 - SEI_UUID_LEN;
34 constexpr int32_t KILO_BYTE = 1024;
35
36 const std::string TYPE_AVC = "video/avc";
37 const std::string TYPE_HEVC = "video/hevc";
38
39 constexpr uint16_t HEVC_SEI_TYPE_ONE = 0x4E; // second bit to 7th bit at first Byte of nalu header is 39 0100 1110
40 constexpr uint16_t HEVC_SEI_TYPE_TWO = 0x50; // second bit to 7th bit at first Byte of nalu header is 40 0101 0000
41 constexpr uint16_t HEVC_NAL_UNIT_TYPE_FLAG = (0x80 | 0x7E); // 0x80 for forbidden bit, 0x7E for nalu type 1111 1110
42 constexpr uint16_t HEVC_SEI_HEAD_LEN = 2;
43
44 constexpr uint16_t AVC_SEI_TYPE = 0x06; // 4th bit to 8th bit at nalu header is 6
45 constexpr uint16_t AVC_NAL_UNIT_TYPE_FLAG = (0x80 | 0x1F); // 0x80 for forbidden bit, 0x1F for nalu type 1001 1111
46 constexpr uint16_t AVC_SEI_HEAD_LEN = 1;
47
48 constexpr uint8_t EMULATION_GUIDE_0_LEN = 2;
49 constexpr uint8_t EMULATION_PREVENTION_CODE = 0X03;
50 constexpr uint8_t SEI_ASSEMBLE_BYTE = 0xFF;
51 constexpr uint8_t SEI_BYTE_MASK_HIGH_7BITS = 0xFE;
52 constexpr uint8_t SEI_SHIFT_FORWARD_BYTES = 0x04;
53 constexpr uint8_t SEI_SHIFT_BACKWARD_BYTES = 0x03;
54
55 constexpr uint32_t NALU_START_BIG_ENDIAN = 0x00000001;
56 constexpr uint32_t NALU_START_LITTLE_ENDIAN = 0x01000000;
57
58 constexpr int64_t SHIFT_THREE_BYTES = 0x03;
59 } // namespace
60
61 namespace OHOS {
62 namespace Media {
63 const std::map<std::string, HelperConstructFunc> SeiParserHelperFactory::HELPER_CONSTRUCTOR_MAP = {
64 { TYPE_AVC,
__anonedb7506a0202() 65 []() {
66 return std::make_shared<AvcSeiParserHelper>();
67 } },
68 { TYPE_HEVC,
__anonedb7506a0302() 69 []() {
70 return std::make_shared<HevcSeiParserHelper>();
71 } }
72 };
73
ParseSeiPayload(const std::shared_ptr<AVBuffer> & buffer,std::shared_ptr<SeiPayloadInfoGroup> & group)74 Status SeiParserHelper::ParseSeiPayload(
75 const std::shared_ptr<AVBuffer> &buffer, std::shared_ptr<SeiPayloadInfoGroup> &group)
76 {
77 FALSE_RETURN_V_MSG(!payloadTypeVec_.empty(), Status::ERROR_INVALID_DATA, "no listener type");
78 FALSE_RETURN_V_MSG(buffer != nullptr, Status::ERROR_INVALID_DATA, "buffer is nullptr");
79 FALSE_RETURN_V_MSG(buffer->memory_ != nullptr, Status::ERROR_INVALID_DATA, "memory is nullptr");
80 MediaAVCodec::AVCodecTrace trace("ParseSeiPayload " + std::to_string(buffer->pts_) + " size " +
81 std::to_string(buffer->memory_->GetSize() / KILO_BYTE));
82
83 auto bufferParseRes = Status::ERROR_UNSUPPORTED_FORMAT;
84 uint8_t seiNaluPrefixLen = ANNEX_B_PREFIX_LEN + 1 + 1 + SEI_UUID_LEN;
85 uint8_t *naluStartPtr = buffer->memory_->GetAddr() + SHIFT_THREE_BYTES;
86 uint8_t *maxPointer = naluStartPtr + buffer->memory_->GetSize() - SHIFT_THREE_BYTES;
87 uint8_t *maxSeiPointer = maxPointer - seiNaluPrefixLen - 1;
88 while (FindNextSeiNaluPos(naluStartPtr, maxSeiPointer)) {
89 if (!group) {
90 group = std::make_shared<SeiPayloadInfoGroup>();
91 }
92 auto naluParseRes = ParseSeiRbsp(naluStartPtr, maxPointer, group);
93 bufferParseRes = (bufferParseRes == Status::OK ? bufferParseRes : naluParseRes);
94 }
95 if (group != nullptr && bufferParseRes == Status::OK) {
96 group->playbackPosition = Plugins::Us2Ms(buffer->pts_);
97 }
98 return bufferParseRes;
99 }
100
SetPayloadTypeVec(const std::vector<int32_t> & vector)101 void SeiParserHelper::SetPayloadTypeVec(const std::vector<int32_t> &vector)
102 {
103 AutoSpinLock lock(spinLock_);
104 payloadTypeVec_ = vector;
105 }
106
FindNextSeiNaluPos(uint8_t * & startPtr,const uint8_t * const maxPtr)107 bool SeiParserHelper::FindNextSeiNaluPos(uint8_t *&startPtr, const uint8_t *const maxPtr)
108 {
109 while (startPtr < maxPtr) {
110 // *startPtr != 0 && *startPtr != 1
111 if (*startPtr & SEI_BYTE_MASK_HIGH_7BITS) {
112 startPtr += SEI_SHIFT_FORWARD_BYTES ;
113 continue;
114 }
115
116 // find the first '1' after '0'
117 if (*startPtr == 0) {
118 startPtr++;
119 continue;
120 }
121
122 // check if '1' after '000'
123 static const uint32_t NALU_START_SEQ = GetNaluStartSeq();
124 if (*(reinterpret_cast<uint32_t *>(startPtr - SEI_SHIFT_BACKWARD_BYTES)) != NALU_START_SEQ) {
125 startPtr += SEI_SHIFT_FORWARD_BYTES;
126 continue;
127 }
128 FALSE_CONTINUE_NOLOG(IsSeiNalu(++startPtr));
129 return true;
130 }
131 return false;
132 }
133
GetNaluStartSeq()134 uint32_t SeiParserHelper::GetNaluStartSeq()
135 {
136 // Big Endian: high byte at low address; Little Endian: low byte at low address
137 uint32_t temp = 0x00000001;
138 return *reinterpret_cast<uint8_t *>(&temp) == 0 ? NALU_START_BIG_ENDIAN : NALU_START_LITTLE_ENDIAN;
139 }
140
IsSeiNalu(uint8_t * & headerPtr)141 bool AvcSeiParserHelper::IsSeiNalu(uint8_t *&headerPtr)
142 {
143 uint8_t header = *headerPtr;
144 auto naluType = header & AVC_NAL_UNIT_TYPE_FLAG;
145 headerPtr += AVC_SEI_HEAD_LEN;
146 if (naluType == AVC_SEI_TYPE) {
147 return true;
148 }
149 return false;
150 }
151
IsSeiNalu(uint8_t * & headerPtr)152 bool HevcSeiParserHelper::IsSeiNalu(uint8_t *&headerPtr)
153 {
154 uint8_t header = *headerPtr;
155 auto naluType = header & HEVC_NAL_UNIT_TYPE_FLAG;
156 headerPtr += HEVC_SEI_HEAD_LEN;
157 if (naluType == HEVC_SEI_TYPE_ONE || naluType == HEVC_SEI_TYPE_TWO) {
158 return true;
159 }
160 return false;
161 }
162
ParseSeiRbsp(uint8_t * & bodyPtr,const uint8_t * const maxPtr,const std::shared_ptr<SeiPayloadInfoGroup> & group)163 Status SeiParserHelper::ParseSeiRbsp(
164 uint8_t *&bodyPtr, const uint8_t *const maxPtr, const std::shared_ptr<SeiPayloadInfoGroup> &group)
165 {
166 FALSE_RETURN_V(group != nullptr, Status::ERROR_NO_MEMORY);
167 Status unSupRetCode = Status::ERROR_UNSUPPORTED_FORMAT;
168 AutoSpinLock lock(spinLock_);
169 std::vector<int32_t> payloadTypeVec = payloadTypeVec_;
170
171 // one sei nalu may has several sei message parts
172 while (bodyPtr + SEI_UUID_LEN < maxPtr) {
173 int32_t payloadType = GetSeiTypeOrSize(bodyPtr, maxPtr);
174 int32_t payloadSize = GetSeiTypeOrSize(bodyPtr, maxPtr);
175 FALSE_RETURN_V_NOLOG(
176 payloadSize > 0 && payloadSize <= SEI_PAYLOAD_SIZE_MAX && bodyPtr + payloadSize < maxPtr, unSupRetCode);
177
178 // if sei payload type is not 5, don't report, jump to next sei message
179 if (std::find(payloadTypeVec.begin(), payloadTypeVec.end(), payloadType) == payloadTypeVec.end()) {
180 auto res = FillTargetBuffer(nullptr, bodyPtr, maxPtr, payloadSize);
181 FALSE_RETURN_V_NOLOG(res == Status::OK, res);
182 continue;
183 }
184
185 AVBufferConfig config;
186 config.size = payloadSize;
187 config.memoryType = MemoryType::SHARED_MEMORY;
188 auto avBuffer = AVBuffer::CreateAVBuffer(config);
189 bool validMem = avBuffer != nullptr && avBuffer->memory_ != nullptr && avBuffer->memory_->GetAddr() != nullptr;
190 FALSE_RETURN_V(validMem, Status::ERROR_NO_MEMORY);
191 auto copyRes = FillTargetBuffer(avBuffer, bodyPtr, maxPtr, payloadSize);
192 FALSE_RETURN_V_NOLOG(copyRes == Status::OK, copyRes);
193
194 group->vec.push_back({ payloadType, avBuffer });
195 unSupRetCode = Status::OK;
196 }
197 return unSupRetCode;
198 }
199
GetSeiTypeOrSize(uint8_t * & bodyPtr,const uint8_t * const maxPtr)200 int32_t SeiParserHelper::GetSeiTypeOrSize(uint8_t *&bodyPtr, const uint8_t *const maxPtr)
201 {
202 int32_t res = 0;
203 const uint8_t *const upperPtr = maxPtr - SEI_UUID_LEN;
204 while (*bodyPtr == SEI_ASSEMBLE_BYTE && bodyPtr < upperPtr) {
205 res += SEI_ASSEMBLE_BYTE;
206 bodyPtr++;
207 }
208 res += *bodyPtr++;
209 return res;
210 }
211
FillTargetBuffer(const std::shared_ptr<AVBuffer> buffer,uint8_t * & payloadPtr,const uint8_t * const maxPtr,const int32_t payloadSize)212 Status SeiParserHelper::FillTargetBuffer(const std::shared_ptr<AVBuffer> buffer,
213 uint8_t *&payloadPtr, const uint8_t *const maxPtr, const int32_t payloadSize)
214 {
215 int32_t writtenSize = 0;
216 uint8_t *targetPtr = (buffer == nullptr ? nullptr : buffer->memory_->GetAddr());
217 for (int32_t zeroNum = 0; writtenSize < payloadSize && payloadPtr < maxPtr; payloadPtr++) {
218 // in H.264 and H.265, 0x000000, 0x000001, 0x000002, 0x000003 will be replaced while encoding
219 if (*payloadPtr == EMULATION_PREVENTION_CODE && zeroNum == EMULATION_GUIDE_0_LEN) {
220 zeroNum = 0;
221 continue;
222 }
223 zeroNum = *payloadPtr == 0 ? zeroNum + 1 : 0;
224 if (targetPtr != nullptr) {
225 targetPtr[writtenSize] = *payloadPtr;
226 }
227 writtenSize++;
228 }
229 FALSE_RETURN_V_MSG(
230 writtenSize == payloadSize, Status::ERROR_UNSUPPORTED_FORMAT, "avalid data less than payloadSize");
231 FALSE_RETURN_V_NOLOG(buffer != nullptr, Status::OK);
232 buffer->memory_->SetSize(writtenSize);
233 return Status::OK;
234 }
235
CreateHelper(const std::string & mimeType)236 std::shared_ptr<SeiParserHelper> SeiParserHelperFactory::CreateHelper(const std::string &mimeType)
237 {
238 auto constructor = HELPER_CONSTRUCTOR_MAP.find(mimeType);
239 FALSE_RETURN_V_MSG(
240 constructor != HELPER_CONSTRUCTOR_MAP.end(), nullptr, "unsupported mime %{public}s", mimeType.c_str());
241 return (constructor->second)();
242 }
243
SeiParserListener(const std::string & mimeType,sptr<AVBufferQueueProducer> producer,std::shared_ptr<Pipeline::EventReceiver> eventReceiver,bool isFlowLimited)244 SeiParserListener::SeiParserListener(const std::string &mimeType, sptr<AVBufferQueueProducer> producer,
245 std::shared_ptr<Pipeline::EventReceiver> eventReceiver, bool isFlowLimited)
246 : producer_(producer),
247 eventReceiver_(eventReceiver),
248 isFlowLimited_(isFlowLimited)
249 {
250 seiParserHelper_ = SeiParserHelperFactory::CreateHelper(mimeType);
251 FALSE_RETURN_MSG(seiParserHelper_ != nullptr, "Create SeiParserHelper failed for %{public}s", mimeType.c_str());
252 FALSE_RETURN_MSG(producer_ != nullptr, "buffer queue producer is nullptr");
253 sptr<IBrokerListener> tmpListener = this;
254 producer_->SetBufferFilledListener(tmpListener);
255 }
256
OnBufferFilled(std::shared_ptr<AVBuffer> & avBuffer)257 void SeiParserListener::OnBufferFilled(std::shared_ptr<AVBuffer> &avBuffer)
258 {
259 FALSE_RETURN_MSG(avBuffer != nullptr, "avbuffer is nullptr");
260 FALSE_RETURN_MSG(producer_ != nullptr, "report sei failed, buffer queue producer is nullptr");
261 ON_SCOPE_EXIT(0)
262 {
263 producer_->ReturnBuffer(avBuffer, true);
264 };
265 FALSE_RETURN_NOLOG(seiParserHelper_ != nullptr);
266 FALSE_RETURN_NOLOG(eventReceiver_ != nullptr);
267
268 FlowLimit(avBuffer);
269 std::shared_ptr<SeiPayloadInfoGroup> group = nullptr;
270 auto res = seiParserHelper_->ParseSeiPayload(avBuffer, group);
271 FALSE_RETURN_NOLOG(res == Status::OK);
272
273 Format seiInfoFormat;
274 bool fmtRes = false;
275 fmtRes = seiInfoFormat.PutIntValue(Tag::AV_PLAYER_SEI_PLAYBACK_POSITION, group->playbackPosition);
276 FALSE_RETURN_MSG(fmtRes, "sei fill format failed");
277
278 std::vector<Format> vec;
279 for (SeiPayloadInfo &payloadInfo : group->vec) {
280 FALSE_RETURN(payloadInfo.payload != nullptr && payloadInfo.payload->memory_ != nullptr);
281
282 Format tmpFormat;
283 fmtRes = tmpFormat.PutBuffer(Tag::AV_PLAYER_SEI_PAYLOAD, payloadInfo.payload->memory_->GetAddr(),
284 payloadInfo.payload->memory_->GetSize());
285 FALSE_RETURN_MSG(fmtRes, "put payload buffer failed");
286 fmtRes = tmpFormat.PutIntValue(Tag::AV_PLAYER_SEI_PAYLOAD_TYPE, payloadInfo.payloadType);
287 FALSE_RETURN_MSG(fmtRes, "put payload type failed");
288 vec.push_back(tmpFormat);
289 }
290 seiInfoFormat.PutFormatVector(Tag::AV_PLAYER_SEI_PLAYBACK_GROUP, vec);
291 eventReceiver_->OnEvent({ "SeiParserHelper", EventType::EVENT_SEI_INFO, seiInfoFormat });
292 }
293
FlowLimit(const std::shared_ptr<AVBuffer> & avBuffer)294 void SeiParserListener::FlowLimit(const std::shared_ptr<AVBuffer> &avBuffer)
295 {
296 FALSE_RETURN_NOLOG(isFlowLimited_ && syncCenter_ != nullptr);
297 MediaAVCodec::AVCodecTrace trace("ParseSeiPayload FlowLimit");
298
299 if (startPts_ == 0) {
300 startPts_ = avBuffer->pts_;
301 }
302
303 auto mediaTimeUs = syncCenter_->GetMediaTimeNow();
304 auto diff = avBuffer->pts_ - startPts_ - mediaTimeUs;
305 FALSE_RETURN_NOLOG(diff > 0);
306
307 std::unique_lock<std::mutex> lock(mutex_);
308 cond_.wait_for(lock, std::chrono::microseconds(diff), [this] () { return isInterruptNeeded_.load(); });
309 }
310
SetPayloadTypeVec(const std::vector<int32_t> & vector)311 void SeiParserListener::SetPayloadTypeVec(const std::vector<int32_t> &vector)
312 {
313 FALSE_RETURN_MSG(seiParserHelper_ != nullptr, "seiParserHelper is nullptr");
314 seiParserHelper_->SetPayloadTypeVec(vector);
315 }
316
OnInterrupted(bool isInterruptNeeded)317 void SeiParserListener::OnInterrupted(bool isInterruptNeeded)
318 {
319 {
320 std::unique_lock<std::mutex> lock(mutex_);
321 isInterruptNeeded_ = isInterruptNeeded;
322 }
323 cond_.notify_all();
324 }
325
SetSeiMessageCbStatus(bool status,const std::vector<int32_t> & payloadTypes)326 Status SeiParserListener::SetSeiMessageCbStatus(
327 bool status, const std::vector<int32_t> &payloadTypes)
328 {
329 MEDIA_LOG_I("seiMessageCbStatus_ = " PUBLIC_LOG_D32, status);
330 if (status) {
331 payloadTypes_ = payloadTypes;
332 SetPayloadTypeVec(payloadTypes_);
333 return Status::OK;
334 }
335 if (payloadTypes.empty()) {
336 payloadTypes_ = {};
337 SetPayloadTypeVec(payloadTypes_);
338 return Status::OK;
339 }
340 payloadTypes_.erase(
341 std::remove_if(payloadTypes_.begin(), payloadTypes_.end(), [&payloadTypes](int value) {
342 return std::find(payloadTypes.begin(), payloadTypes.end(), value) != payloadTypes.end();
343 }), payloadTypes_.end());
344 SetPayloadTypeVec(payloadTypes_);
345 return Status::OK;
346 }
347 } // namespace Media
348 } // namespace OHOS