1 /*
2 * Copyright 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "NuPlayerCCDecoder"
19 #include <utils/Log.h>
20 #include <inttypes.h>
21
22 #include "avc_utils.h"
23 #include "NuPlayerCCDecoder.h"
24
25 #include <media/stagefright/foundation/ABitReader.h>
26 #include <media/stagefright/foundation/ABuffer.h>
27 #include <media/stagefright/foundation/ADebug.h>
28 #include <media/stagefright/foundation/AMessage.h>
29 #include <media/stagefright/MediaDefs.h>
30
31 namespace android {
32
33 // In CEA-708B, the maximum bandwidth of CC is set to 9600bps.
34 static const size_t kMaxBandwithSizeBytes = 9600 / 8;
35
36 struct CCData {
CCDataandroid::CCData37 CCData(uint8_t type, uint8_t data1, uint8_t data2)
38 : mType(type), mData1(data1), mData2(data2) {
39 }
getChannelandroid::CCData40 bool getChannel(size_t *channel) const {
41 if (mData1 >= 0x10 && mData1 <= 0x1f) {
42 *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0);
43 return true;
44 }
45 return false;
46 }
47
48 uint8_t mType;
49 uint8_t mData1;
50 uint8_t mData2;
51 };
52
isNullPad(CCData * cc)53 static bool isNullPad(CCData *cc) {
54 return cc->mData1 < 0x10 && cc->mData2 < 0x10;
55 }
56
57 static void dumpBytePair(const sp<ABuffer> &ccBuf) __attribute__ ((unused));
dumpBytePair(const sp<ABuffer> & ccBuf)58 static void dumpBytePair(const sp<ABuffer> &ccBuf) {
59 size_t offset = 0;
60 AString out;
61
62 while (offset < ccBuf->size()) {
63 char tmp[128];
64
65 CCData *cc = (CCData *) (ccBuf->data() + offset);
66
67 if (isNullPad(cc)) {
68 // 1 null pad or XDS metadata, ignore
69 offset += sizeof(CCData);
70 continue;
71 }
72
73 if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) {
74 // 2 basic chars
75 snprintf(tmp, sizeof(tmp), "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2);
76 } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
77 && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) {
78 // 1 special char
79 snprintf(tmp, sizeof(tmp), "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2);
80 } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A)
81 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
82 // 1 Spanish/French char
83 snprintf(tmp, sizeof(tmp), "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2);
84 } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B)
85 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
86 // 1 Portuguese/German/Danish char
87 snprintf(tmp, sizeof(tmp), "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2);
88 } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
89 && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){
90 // Mid-Row Codes (Table 69)
91 snprintf(tmp, sizeof(tmp), "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2);
92 } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c)
93 && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f)
94 ||
95 ((cc->mData1 == 0x17 || cc->mData1 == 0x1f)
96 && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){
97 // Misc Control Codes (Table 70)
98 snprintf(tmp, sizeof(tmp), "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2);
99 } else if ((cc->mData1 & 0x70) == 0x10
100 && (cc->mData2 & 0x40) == 0x40
101 && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) {
102 // Preamble Address Codes (Table 71)
103 snprintf(tmp, sizeof(tmp), "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2);
104 } else {
105 snprintf(tmp, sizeof(tmp), "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2);
106 }
107
108 if (out.size() > 0) {
109 out.append(", ");
110 }
111
112 out.append(tmp);
113
114 offset += sizeof(CCData);
115 }
116
117 ALOGI("%s", out.c_str());
118 }
119
CCDecoder(const sp<AMessage> & notify)120 NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> ¬ify)
121 : mNotify(notify),
122 mSelectedTrack(-1),
123 mDTVCCPacket(new ABuffer(kMaxBandwithSizeBytes)) {
124 mDTVCCPacket->setRange(0, 0);
125
126 // In CEA-608, streams from packets which have the value 0 of cc_type contain CC1 and CC2, and
127 // streams from packets which have the value 1 of cc_type contain CC3 and CC4.
128 // The following array indicates the current transmitting channels for each value of cc_type.
129 mLine21Channels[0] = 0; // CC1
130 mLine21Channels[1] = 2; // CC3
131 }
132
getTrackCount() const133 size_t NuPlayer::CCDecoder::getTrackCount() const {
134 return mTracks.size();
135 }
136
getTrackInfo(size_t index) const137 sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const {
138 if (!isTrackValid(index)) {
139 return NULL;
140 }
141
142 sp<AMessage> format = new AMessage();
143
144 CCTrack track = mTracks[index];
145
146 format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE);
147 format->setString("language", "und");
148
149 switch (track.mTrackType) {
150 case kTrackTypeCEA608:
151 format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608);
152 break;
153 case kTrackTypeCEA708:
154 format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_708);
155 break;
156 default:
157 ALOGE("Unknown track type: %d", track.mTrackType);
158 return NULL;
159 }
160
161 // For CEA-608 CC1, field 0 channel 0
162 bool isDefaultAuto = track.mTrackType == kTrackTypeCEA608
163 && track.mTrackChannel == 0;
164 // For CEA-708, Primary Caption Service.
165 bool isDefaultOnly = track.mTrackType == kTrackTypeCEA708
166 && track.mTrackChannel == 1;
167 format->setInt32("auto", isDefaultAuto);
168 format->setInt32("default", isDefaultAuto || isDefaultOnly);
169 format->setInt32("forced", 0);
170
171 return format;
172 }
173
selectTrack(size_t index,bool select)174 status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) {
175 if (!isTrackValid(index)) {
176 return BAD_VALUE;
177 }
178
179 if (select) {
180 if (mSelectedTrack == (ssize_t)index) {
181 ALOGE("track %zu already selected", index);
182 return BAD_VALUE;
183 }
184 ALOGV("selected track %zu", index);
185 mSelectedTrack = index;
186 } else {
187 if (mSelectedTrack != (ssize_t)index) {
188 ALOGE("track %zu is not selected", index);
189 return BAD_VALUE;
190 }
191 ALOGV("unselected track %zu", index);
192 mSelectedTrack = -1;
193 }
194
195 // Clear the previous track payloads
196 mCCMap.clear();
197
198 return OK;
199 }
200
isSelected() const201 bool NuPlayer::CCDecoder::isSelected() const {
202 return mSelectedTrack >= 0 && mSelectedTrack < (int32_t)getTrackCount();
203 }
204
isTrackValid(size_t index) const205 bool NuPlayer::CCDecoder::isTrackValid(size_t index) const {
206 return index < getTrackCount();
207 }
208
209 // returns true if a new CC track is found
extractFromSEI(const sp<ABuffer> & accessUnit)210 bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) {
211 sp<ABuffer> sei;
212 if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) {
213 return false;
214 }
215
216 int64_t timeUs;
217 CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
218
219 bool trackAdded = false;
220
221 const NALPosition *nal = (NALPosition *)sei->data();
222
223 for (size_t i = 0; i < sei->size() / sizeof(NALPosition); ++i, ++nal) {
224 trackAdded |= parseSEINalUnit(
225 timeUs, accessUnit->data() + nal->nalOffset, nal->nalSize);
226 }
227
228 return trackAdded;
229 }
230
231 // returns true if a new CC track is found
parseSEINalUnit(int64_t timeUs,const uint8_t * data,size_t size)232 bool NuPlayer::CCDecoder::parseSEINalUnit(int64_t timeUs, const uint8_t *data, size_t size) {
233 unsigned nalType = data[0] & 0x1f;
234
235 // the buffer should only have SEI in it
236 if (nalType != 6) {
237 return false;
238 }
239
240 bool trackAdded = false;
241 NALBitReader br(data + 1, size - 1);
242
243 // sei_message()
244 while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message()
245 uint32_t payload_type = 0;
246 size_t payload_size = 0;
247 uint8_t last_byte;
248
249 do {
250 last_byte = br.getBits(8);
251 payload_type += last_byte;
252 } while (last_byte == 0xFF);
253
254 do {
255 last_byte = br.getBits(8);
256 payload_size += last_byte;
257 } while (last_byte == 0xFF);
258
259 if (payload_size > SIZE_MAX / 8
260 || !br.atLeastNumBitsLeft(payload_size * 8)) {
261 ALOGV("Malformed SEI payload");
262 break;
263 }
264
265 // sei_payload()
266 if (payload_type == 4) {
267 bool isCC = false;
268 if (payload_size > 1 + 2 + 4 + 1) {
269 // user_data_registered_itu_t_t35()
270
271 // ATSC A/72: 6.4.2
272 uint8_t itu_t_t35_country_code = br.getBits(8);
273 uint16_t itu_t_t35_provider_code = br.getBits(16);
274 uint32_t user_identifier = br.getBits(32);
275 uint8_t user_data_type_code = br.getBits(8);
276
277 payload_size -= 1 + 2 + 4 + 1;
278
279 isCC = itu_t_t35_country_code == 0xB5
280 && itu_t_t35_provider_code == 0x0031
281 && user_identifier == 'GA94'
282 && user_data_type_code == 0x3;
283 }
284
285 if (isCC && payload_size > 2) {
286 trackAdded |= parseMPEGCCData(timeUs, br.data(), br.numBitsLeft() / 8);
287 } else {
288 ALOGV("Malformed SEI payload type 4");
289 }
290 } else {
291 ALOGV("Unsupported SEI payload type %d", payload_type);
292 }
293
294 // skipping remaining bits of this payload
295 br.skipBits(payload_size * 8);
296 }
297
298 return trackAdded;
299 }
300
301 // returns true if a new CC track is found
extractFromMPEGUserData(const sp<ABuffer> & accessUnit)302 bool NuPlayer::CCDecoder::extractFromMPEGUserData(const sp<ABuffer> &accessUnit) {
303 sp<ABuffer> mpegUserData;
304 if (!accessUnit->meta()->findBuffer("mpegUserData", &mpegUserData)
305 || mpegUserData == NULL) {
306 return false;
307 }
308
309 int64_t timeUs;
310 CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
311
312 bool trackAdded = false;
313
314 const size_t *userData = (size_t *)mpegUserData->data();
315
316 for (size_t i = 0; i < mpegUserData->size() / sizeof(size_t); ++i) {
317 if (accessUnit->size() < userData[i]) {
318 ALOGW("b/129068792, skip invalid offset for user data");
319 android_errorWriteLog(0x534e4554, "129068792");
320 continue;
321 }
322 trackAdded |= parseMPEGUserDataUnit(
323 timeUs, accessUnit->data() + userData[i], accessUnit->size() - userData[i]);
324 }
325
326 return trackAdded;
327 }
328
329 // returns true if a new CC track is found
parseMPEGUserDataUnit(int64_t timeUs,const uint8_t * data,size_t size)330 bool NuPlayer::CCDecoder::parseMPEGUserDataUnit(int64_t timeUs, const uint8_t *data, size_t size) {
331 if (size < 9) {
332 ALOGW("b/129068792, MPEG user data size too small %zu", size);
333 android_errorWriteLog(0x534e4554, "129068792");
334 return false;
335 }
336
337 ABitReader br(data + 4, 5);
338
339 uint32_t user_identifier = br.getBits(32);
340 uint8_t user_data_type = br.getBits(8);
341
342 if (user_identifier == 'GA94' && user_data_type == 0x3) {
343 return parseMPEGCCData(timeUs, data + 9, size - 9);
344 }
345
346 return false;
347 }
348
349 // returns true if a new CC track is found
parseMPEGCCData(int64_t timeUs,const uint8_t * data,size_t size)350 bool NuPlayer::CCDecoder::parseMPEGCCData(int64_t timeUs, const uint8_t *data, size_t size) {
351 bool trackAdded = false;
352
353 // MPEG_cc_data()
354 // ATSC A/53 Part 4: 6.2.3.1
355 ABitReader br(data, size);
356
357 if (br.numBitsLeft() <= 16) {
358 return false;
359 }
360
361 br.skipBits(1);
362 bool process_cc_data_flag = br.getBits(1);
363 br.skipBits(1);
364 size_t cc_count = br.getBits(5);
365 br.skipBits(8);
366
367 if (!process_cc_data_flag || 3 * 8 * cc_count >= br.numBitsLeft()) {
368 return false;
369 }
370
371 sp<ABuffer> line21CCBuf = NULL;
372
373 for (size_t i = 0; i < cc_count; ++i) {
374 br.skipBits(5);
375 bool cc_valid = br.getBits(1);
376 uint8_t cc_type = br.getBits(2);
377
378 if (cc_valid) {
379 if (cc_type == 3) {
380 if (mDTVCCPacket->size() > 0) {
381 trackAdded |= parseDTVCCPacket(
382 timeUs, mDTVCCPacket->data(), mDTVCCPacket->size());
383 mDTVCCPacket->setRange(0, 0);
384 }
385 memcpy(mDTVCCPacket->data() + mDTVCCPacket->size(), br.data(), 2);
386 mDTVCCPacket->setRange(0, mDTVCCPacket->size() + 2);
387 br.skipBits(16);
388 } else if (mDTVCCPacket->size() > 0 && cc_type == 2) {
389 if (mDTVCCPacket->capacity() - mDTVCCPacket->size() >= 2) {
390 memcpy(mDTVCCPacket->data() + mDTVCCPacket->size(), br.data(), 2);
391 mDTVCCPacket->setRange(0, mDTVCCPacket->size() + 2);
392 } else {
393 ALOGW("b/129068792, skip CC due to too much data(%zu, %zu)",
394 mDTVCCPacket->capacity(), mDTVCCPacket->size());
395 android_errorWriteLog(0x534e4554, "129068792");
396 }
397 br.skipBits(16);
398 } else if (cc_type == 0 || cc_type == 1) {
399 uint8_t cc_data_1 = br.getBits(8) & 0x7f;
400 uint8_t cc_data_2 = br.getBits(8) & 0x7f;
401
402 CCData cc(cc_type, cc_data_1, cc_data_2);
403
404 if (isNullPad(&cc)) {
405 continue;
406 }
407
408 size_t channel;
409 if (cc.getChannel(&channel)) {
410 mLine21Channels[cc_type] = channel;
411
412 // create a new track if it does not exist.
413 getTrackIndex(kTrackTypeCEA608, channel, &trackAdded);
414 }
415
416 if (isSelected() && mTracks[mSelectedTrack].mTrackType == kTrackTypeCEA608
417 && mTracks[mSelectedTrack].mTrackChannel == mLine21Channels[cc_type]) {
418 if (line21CCBuf == NULL) {
419 line21CCBuf = new ABuffer((cc_count - i) * sizeof(CCData));
420 line21CCBuf->setRange(0, 0);
421 }
422 memcpy(line21CCBuf->data() + line21CCBuf->size(), &cc, sizeof(cc));
423 line21CCBuf->setRange(0, line21CCBuf->size() + sizeof(CCData));
424 }
425 } else {
426 br.skipBits(16);
427 }
428 } else {
429 if ((cc_type == 3 || cc_type == 2) && mDTVCCPacket->size() > 0) {
430 trackAdded |= parseDTVCCPacket(timeUs, mDTVCCPacket->data(), mDTVCCPacket->size());
431 mDTVCCPacket->setRange(0, 0);
432 }
433 br.skipBits(16);
434 }
435 }
436
437 if (isSelected() && mTracks[mSelectedTrack].mTrackType == kTrackTypeCEA608
438 && line21CCBuf != NULL && line21CCBuf->size() > 0) {
439 mCCMap.add(timeUs, line21CCBuf);
440 }
441
442 return trackAdded;
443 }
444
445 // returns true if a new CC track is found
parseDTVCCPacket(int64_t timeUs,const uint8_t * data,size_t size)446 bool NuPlayer::CCDecoder::parseDTVCCPacket(int64_t timeUs, const uint8_t *data, size_t size) {
447 // CEA-708B 5 DTVCC Packet Layer.
448 ABitReader br(data, size);
449 br.skipBits(2);
450
451 size_t packet_size = br.getBits(6);
452 if (packet_size == 0) packet_size = 64;
453 packet_size *= 2;
454
455 if (size != packet_size) {
456 return false;
457 }
458
459 bool trackAdded = false;
460
461 while (br.numBitsLeft() >= 16) {
462 // CEA-708B Figure 5 and 6.
463 uint8_t service_number = br.getBits(3);
464 size_t block_size = br.getBits(5);
465
466 if (service_number == 64) {
467 br.skipBits(2);
468 service_number = br.getBits(6);
469
470 if (service_number < 64) {
471 return trackAdded;
472 }
473 }
474
475 if (br.numBitsLeft() < block_size * 8) {
476 return trackAdded;
477 }
478
479 if (block_size > 0) {
480 size_t trackIndex = getTrackIndex(kTrackTypeCEA708, service_number, &trackAdded);
481 if (mSelectedTrack == (ssize_t)trackIndex) {
482 sp<ABuffer> ccPacket = new ABuffer(block_size);
483 if (ccPacket->capacity() == 0) {
484 ALOGW("b/129068792, no memory available, %zu", block_size);
485 android_errorWriteLog(0x534e4554, "129068792");
486 return false;
487 }
488 memcpy(ccPacket->data(), br.data(), block_size);
489 mCCMap.add(timeUs, ccPacket);
490 }
491 }
492 br.skipBits(block_size * 8);
493 }
494
495 return trackAdded;
496 }
497
498 // return the track index for a given type and channel.
499 // if the track does not exist, creates a new one.
getTrackIndex(int32_t trackType,size_t channel,bool * trackAdded)500 size_t NuPlayer::CCDecoder::getTrackIndex(
501 int32_t trackType, size_t channel, bool *trackAdded) {
502 CCTrack track(trackType, channel);
503 ssize_t index = mTrackIndices.indexOfKey(track);
504
505 if (index < 0) {
506 // A new track is added.
507 index = mTracks.size();
508 mTrackIndices.add(track, index);
509 mTracks.add(track);
510 *trackAdded = true;
511 return index;
512 }
513
514 return mTrackIndices.valueAt(index);
515 }
516
decode(const sp<ABuffer> & accessUnit)517 void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) {
518 if (extractFromMPEGUserData(accessUnit) || extractFromSEI(accessUnit)) {
519 sp<AMessage> msg = mNotify->dup();
520 msg->setInt32("what", kWhatTrackAdded);
521 msg->post();
522 }
523 // TODO: extract CC from other sources
524 }
525
display(int64_t timeUs)526 void NuPlayer::CCDecoder::display(int64_t timeUs) {
527 if (!isSelected()) {
528 return;
529 }
530
531 ssize_t index = mCCMap.indexOfKey(timeUs);
532 if (index < 0) {
533 ALOGV("cc for timestamp %" PRId64 " not found", timeUs);
534 return;
535 }
536
537 sp<ABuffer> ccBuf;
538
539 if (index == 0) {
540 ccBuf = mCCMap.valueAt(index);
541 } else {
542 size_t size = 0;
543
544 for (ssize_t i = 0; i <= index; ++i) {
545 size += mCCMap.valueAt(i)->size();
546 }
547
548 ccBuf = new ABuffer(size);
549 ccBuf->setRange(0, 0);
550
551 for (ssize_t i = 0; i <= index; ++i) {
552 sp<ABuffer> buf = mCCMap.valueAt(i);
553 memcpy(ccBuf->data() + ccBuf->size(), buf->data(), buf->size());
554 ccBuf->setRange(0, ccBuf->size() + buf->size());
555 }
556 }
557
558 if (ccBuf->size() > 0) {
559 #if 0
560 dumpBytePair(ccBuf);
561 #endif
562
563 ccBuf->meta()->setInt32("trackIndex", mSelectedTrack);
564 ccBuf->meta()->setInt64("timeUs", timeUs);
565 ccBuf->meta()->setInt64("durationUs", 0ll);
566
567 sp<AMessage> msg = mNotify->dup();
568 msg->setInt32("what", kWhatClosedCaptionData);
569 msg->setBuffer("buffer", ccBuf);
570 msg->post();
571 }
572
573 // remove all entries before timeUs
574 mCCMap.removeItemsAt(0, index + 1);
575 }
576
flush()577 void NuPlayer::CCDecoder::flush() {
578 mCCMap.clear();
579 mDTVCCPacket->setRange(0, 0);
580 }
581
compare(const NuPlayer::CCDecoder::CCTrack & rhs) const582 int32_t NuPlayer::CCDecoder::CCTrack::compare(const NuPlayer::CCDecoder::CCTrack& rhs) const {
583 int32_t cmp = mTrackType - rhs.mTrackType;
584 if (cmp != 0) return cmp;
585 return mTrackChannel - rhs.mTrackChannel;
586 }
587
operator <(const NuPlayer::CCDecoder::CCTrack & rhs) const588 bool NuPlayer::CCDecoder::CCTrack::operator<(const NuPlayer::CCDecoder::CCTrack& rhs) const {
589 return compare(rhs) < 0;
590 }
591
operator ==(const NuPlayer::CCDecoder::CCTrack & rhs) const592 bool NuPlayer::CCDecoder::CCTrack::operator==(const NuPlayer::CCDecoder::CCTrack& rhs) const {
593 return compare(rhs) == 0;
594 }
595
operator !=(const NuPlayer::CCDecoder::CCTrack & rhs) const596 bool NuPlayer::CCDecoder::CCTrack::operator!=(const NuPlayer::CCDecoder::CCTrack& rhs) const {
597 return compare(rhs) != 0;
598 }
599
600 } // namespace android
601
602