1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "common_video/h264/sps_vui_rewriter.h"
12
13 #include <cstdint>
14 #include <vector>
15
16 #include "api/video/color_space.h"
17 #include "common_video/h264/h264_common.h"
18 #include "rtc_base/bit_buffer.h"
19 #include "rtc_base/buffer.h"
20 #include "rtc_base/logging.h"
21 #include "test/gmock.h"
22 #include "test/gtest.h"
23
24 namespace webrtc {
25
26 namespace {
27 enum SpsMode {
28 kNoRewriteRequired_VuiOptimal,
29 kRewriteRequired_NoVui,
30 kRewriteRequired_NoBitstreamRestriction,
31 kRewriteRequired_VuiSuboptimal,
32 };
33
34 static const size_t kSpsBufferMaxSize = 256;
35 static const size_t kWidth = 640;
36 static const size_t kHeight = 480;
37
38 static const uint8_t kStartSequence[] = {0x00, 0x00, 0x00, 0x01};
39 static const uint8_t kSpsNaluType[] = {H264::NaluType::kSps};
40 static const uint8_t kIdr1[] = {H264::NaluType::kIdr, 0xFF, 0x00, 0x00, 0x04};
41 static const uint8_t kIdr2[] = {H264::NaluType::kIdr, 0xFF, 0x00, 0x11};
42
43 struct VuiHeader {
44 uint32_t vui_parameters_present_flag;
45 uint32_t bitstream_restriction_flag;
46 uint32_t max_num_reorder_frames;
47 uint32_t max_dec_frame_buffering;
48 uint32_t video_signal_type_present_flag;
49 uint32_t video_full_range_flag;
50 uint32_t colour_description_present_flag;
51 uint8_t colour_primaries;
52 uint8_t transfer_characteristics;
53 uint8_t matrix_coefficients;
54 };
55
56 static const VuiHeader kVuiNotPresent = {
57 /* vui_parameters_present_flag= */ 0,
58 /* bitstream_restriction_flag= */ 0,
59 /* max_num_reorder_frames= */ 0,
60 /* max_dec_frame_buffering= */ 0,
61 /* video_signal_type_present_flag= */ 0,
62 /* video_full_range_flag= */ 0,
63 /* colour_description_present_flag= */ 0,
64 /* colour_primaries= */ 0,
65 /* transfer_characteristics= */ 0,
66 /* matrix_coefficients= */ 0};
67
68 static const VuiHeader kVuiNoBitstreamRestriction = {
69 /* vui_parameters_present_flag= */ 1,
70 /* bitstream_restriction_flag= */ 0,
71 /* max_num_reorder_frames= */ 0,
72 /* max_dec_frame_buffering= */ 0,
73 /* video_signal_type_present_flag= */ 0,
74 /* video_full_range_flag= */ 0,
75 /* colour_description_present_flag= */ 0,
76 /* colour_primaries= */ 0,
77 /* transfer_characteristics= */ 0,
78 /* matrix_coefficients= */ 0};
79
80 static const VuiHeader kVuiNoFrameBuffering = {
81 /* vui_parameters_present_flag= */ 1,
82 /* bitstream_restriction_flag= */ 1,
83 /* max_num_reorder_frames= */ 0,
84 /* max_dec_frame_buffering= */ 1,
85 /* video_signal_type_present_flag= */ 0,
86 /* video_full_range_flag= */ 0,
87 /* colour_description_present_flag= */ 0,
88 /* colour_primaries= */ 0,
89 /* transfer_characteristics= */ 0,
90 /* matrix_coefficients= */ 0};
91
92 static const VuiHeader kVuiFrameBuffering = {
93 /* vui_parameters_present_flag= */ 1,
94 /* bitstream_restriction_flag= */ 1,
95 /* max_num_reorder_frames= */ 3,
96 /* max_dec_frame_buffering= */ 3,
97 /* video_signal_type_present_flag= */ 0,
98 /* video_full_range_flag= */ 0,
99 /* colour_description_present_flag= */ 0,
100 /* colour_primaries= */ 0,
101 /* transfer_characteristics= */ 0,
102 /* matrix_coefficients= */ 0};
103
104 static const VuiHeader kVuiNoVideoSignalType = {
105 /* vui_parameters_present_flag= */ 1,
106 /* bitstream_restriction_flag= */ 1,
107 /* max_num_reorder_frames= */ 0,
108 /* max_dec_frame_buffering= */ 1,
109 /* video_signal_type_present_flag= */ 0,
110 /* video_full_range_flag= */ 0,
111 /* colour_description_present_flag= */ 0,
112 /* colour_primaries= */ 0,
113 /* transfer_characteristics= */ 0,
114 /* matrix_coefficients= */ 0};
115
116 static const VuiHeader kVuiLimitedRangeNoColourDescription = {
117 /* vui_parameters_present_flag= */ 1,
118 /* bitstream_restriction_flag= */ 1,
119 /* max_num_reorder_frames= */ 0,
120 /* max_dec_frame_buffering= */ 1,
121 /* video_signal_type_present_flag= */ 1,
122 /* video_full_range_flag= */ 0,
123 /* colour_description_present_flag= */ 0,
124 /* colour_primaries= */ 0,
125 /* transfer_characteristics= */ 0,
126 /* matrix_coefficients= */ 0};
127
128 static const VuiHeader kVuiFullRangeNoColourDescription = {
129 /* vui_parameters_present_flag= */ 1,
130 /* bitstream_restriction_flag= */ 1,
131 /* max_num_reorder_frames= */ 0,
132 /* max_dec_frame_buffering= */ 1,
133 /* video_signal_type_present_flag= */ 1,
134 /* video_full_range_flag= */ 1,
135 /* colour_description_present_flag= */ 0,
136 /* colour_primaries= */ 0,
137 /* transfer_characteristics= */ 0,
138 /* matrix_coefficients= */ 0};
139
140 static const VuiHeader kVuiLimitedRangeBt709Color = {
141 /* vui_parameters_present_flag= */ 1,
142 /* bitstream_restriction_flag= */ 1,
143 /* max_num_reorder_frames= */ 0,
144 /* max_dec_frame_buffering= */ 1,
145 /* video_signal_type_present_flag= */ 1,
146 /* video_full_range_flag= */ 0,
147 /* colour_description_present_flag= */ 1,
148 /* colour_primaries= */ 1,
149 /* transfer_characteristics= */ 1,
150 /* matrix_coefficients= */ 1};
151
152 static const webrtc::ColorSpace kColorSpaceH264Default(
153 ColorSpace::PrimaryID::kUnspecified,
154 ColorSpace::TransferID::kUnspecified,
155 ColorSpace::MatrixID::kUnspecified,
156 ColorSpace::RangeID::kLimited);
157
158 static const webrtc::ColorSpace kColorSpacePrimariesBt709(
159 ColorSpace::PrimaryID::kBT709,
160 ColorSpace::TransferID::kUnspecified,
161 ColorSpace::MatrixID::kUnspecified,
162 ColorSpace::RangeID::kLimited);
163
164 static const webrtc::ColorSpace kColorSpaceTransferBt709(
165 ColorSpace::PrimaryID::kUnspecified,
166 ColorSpace::TransferID::kBT709,
167 ColorSpace::MatrixID::kUnspecified,
168 ColorSpace::RangeID::kLimited);
169
170 static const webrtc::ColorSpace kColorSpaceMatrixBt709(
171 ColorSpace::PrimaryID::kUnspecified,
172 ColorSpace::TransferID::kUnspecified,
173 ColorSpace::MatrixID::kBT709,
174 ColorSpace::RangeID::kLimited);
175
176 static const webrtc::ColorSpace kColorSpaceFullRange(
177 ColorSpace::PrimaryID::kBT709,
178 ColorSpace::TransferID::kUnspecified,
179 ColorSpace::MatrixID::kUnspecified,
180 ColorSpace::RangeID::kFull);
181
182 static const webrtc::ColorSpace kColorSpaceBt709LimitedRange(
183 ColorSpace::PrimaryID::kBT709,
184 ColorSpace::TransferID::kBT709,
185 ColorSpace::MatrixID::kBT709,
186 ColorSpace::RangeID::kLimited);
187 } // namespace
188
189 // Generates a fake SPS with basically everything empty and with characteristics
190 // based off SpsMode.
191 // Pass in a buffer of at least kSpsBufferMaxSize.
192 // The fake SPS that this generates also always has at least one emulation byte
193 // at offset 2, since the first two bytes are always 0, and has a 0x3 as the
194 // level_idc, to make sure the parser doesn't eat all 0x3 bytes.
GenerateFakeSps(const VuiHeader & vui,rtc::Buffer * out_buffer)195 void GenerateFakeSps(const VuiHeader& vui, rtc::Buffer* out_buffer) {
196 uint8_t rbsp[kSpsBufferMaxSize] = {0};
197 rtc::BitBufferWriter writer(rbsp, kSpsBufferMaxSize);
198 // Profile byte.
199 writer.WriteUInt8(0);
200 // Constraint sets and reserved zero bits.
201 writer.WriteUInt8(0);
202 // level_idc.
203 writer.WriteUInt8(3);
204 // seq_paramter_set_id.
205 writer.WriteExponentialGolomb(0);
206 // Profile is not special, so we skip all the chroma format settings.
207
208 // Now some bit magic.
209 // log2_max_frame_num_minus4: ue(v). 0 is fine.
210 writer.WriteExponentialGolomb(0);
211 // pic_order_cnt_type: ue(v).
212 writer.WriteExponentialGolomb(0);
213 // log2_max_pic_order_cnt_lsb_minus4: ue(v). 0 is fine.
214 writer.WriteExponentialGolomb(0);
215
216 // max_num_ref_frames: ue(v). Use 1, to make optimal/suboptimal more obvious.
217 writer.WriteExponentialGolomb(1);
218 // gaps_in_frame_num_value_allowed_flag: u(1).
219 writer.WriteBits(0, 1);
220 // Next are width/height. First, calculate the mbs/map_units versions.
221 uint16_t width_in_mbs_minus1 = (kWidth + 15) / 16 - 1;
222
223 // For the height, we're going to define frame_mbs_only_flag, so we need to
224 // divide by 2. See the parser for the full calculation.
225 uint16_t height_in_map_units_minus1 = ((kHeight + 15) / 16 - 1) / 2;
226 // Write each as ue(v).
227 writer.WriteExponentialGolomb(width_in_mbs_minus1);
228 writer.WriteExponentialGolomb(height_in_map_units_minus1);
229 // frame_mbs_only_flag: u(1). Needs to be false.
230 writer.WriteBits(0, 1);
231 // mb_adaptive_frame_field_flag: u(1).
232 writer.WriteBits(0, 1);
233 // direct_8x8_inferene_flag: u(1).
234 writer.WriteBits(0, 1);
235 // frame_cropping_flag: u(1). 1, so we can supply crop.
236 writer.WriteBits(1, 1);
237 // Now we write the left/right/top/bottom crop. For simplicity, we'll put all
238 // the crop at the left/top.
239 // We picked a 4:2:0 format, so the crops are 1/2 the pixel crop values.
240 // Left/right.
241 writer.WriteExponentialGolomb(((16 - (kWidth % 16)) % 16) / 2);
242 writer.WriteExponentialGolomb(0);
243 // Top/bottom.
244 writer.WriteExponentialGolomb(((16 - (kHeight % 16)) % 16) / 2);
245 writer.WriteExponentialGolomb(0);
246
247 // Finally! The VUI.
248 // vui_parameters_present_flag: u(1)
249 writer.WriteBits(vui.vui_parameters_present_flag, 1);
250 if (vui.vui_parameters_present_flag) {
251 // aspect_ratio_info_present_flag, overscan_info_present_flag. Both u(1).
252 writer.WriteBits(0, 2);
253
254 writer.WriteBits(vui.video_signal_type_present_flag, 1);
255 if (vui.video_signal_type_present_flag) {
256 // video_format: u(3). 5 = Unspecified
257 writer.WriteBits(5, 3);
258 writer.WriteBits(vui.video_full_range_flag, 1);
259 writer.WriteBits(vui.colour_description_present_flag, 1);
260 if (vui.colour_description_present_flag) {
261 writer.WriteUInt8(vui.colour_primaries);
262 writer.WriteUInt8(vui.transfer_characteristics);
263 writer.WriteUInt8(vui.matrix_coefficients);
264 }
265 }
266
267 // chroma_loc_info_present_flag, timing_info_present_flag,
268 // nal_hrd_parameters_present_flag, vcl_hrd_parameters_present_flag,
269 // pic_struct_present_flag, All u(1)
270 writer.WriteBits(0, 5);
271
272 writer.WriteBits(vui.bitstream_restriction_flag, 1);
273 if (vui.bitstream_restriction_flag) {
274 // Write some defaults. Shouldn't matter for parsing, though.
275 // motion_vectors_over_pic_boundaries_flag: u(1)
276 writer.WriteBits(1, 1);
277 // max_bytes_per_pic_denom: ue(v)
278 writer.WriteExponentialGolomb(2);
279 // max_bits_per_mb_denom: ue(v)
280 writer.WriteExponentialGolomb(1);
281 // log2_max_mv_length_horizontal: ue(v)
282 // log2_max_mv_length_vertical: ue(v)
283 writer.WriteExponentialGolomb(16);
284 writer.WriteExponentialGolomb(16);
285
286 // Next are the limits we care about.
287 writer.WriteExponentialGolomb(vui.max_num_reorder_frames);
288 writer.WriteExponentialGolomb(vui.max_dec_frame_buffering);
289 }
290 }
291
292 // Get the number of bytes written (including the last partial byte).
293 size_t byte_count, bit_offset;
294 writer.GetCurrentOffset(&byte_count, &bit_offset);
295 if (bit_offset > 0) {
296 byte_count++;
297 }
298
299 H264::WriteRbsp(rbsp, byte_count, out_buffer);
300 }
301
TestSps(const VuiHeader & vui,const ColorSpace * color_space,SpsVuiRewriter::ParseResult expected_parse_result)302 void TestSps(const VuiHeader& vui,
303 const ColorSpace* color_space,
304 SpsVuiRewriter::ParseResult expected_parse_result) {
305 rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
306 rtc::Buffer original_sps;
307 GenerateFakeSps(vui, &original_sps);
308
309 absl::optional<SpsParser::SpsState> sps;
310 rtc::Buffer rewritten_sps;
311 SpsVuiRewriter::ParseResult result = SpsVuiRewriter::ParseAndRewriteSps(
312 original_sps.data(), original_sps.size(), &sps, color_space,
313 &rewritten_sps, SpsVuiRewriter::Direction::kIncoming);
314 EXPECT_EQ(expected_parse_result, result);
315 ASSERT_TRUE(sps);
316 EXPECT_EQ(sps->width, kWidth);
317 EXPECT_EQ(sps->height, kHeight);
318 if (vui.vui_parameters_present_flag) {
319 EXPECT_EQ(sps->vui_params_present, 1u);
320 }
321
322 if (result == SpsVuiRewriter::ParseResult::kVuiRewritten) {
323 // Ensure that added/rewritten SPS is parsable.
324 rtc::Buffer tmp;
325 result = SpsVuiRewriter::ParseAndRewriteSps(
326 rewritten_sps.data(), rewritten_sps.size(), &sps, nullptr, &tmp,
327 SpsVuiRewriter::Direction::kIncoming);
328 EXPECT_EQ(SpsVuiRewriter::ParseResult::kVuiOk, result);
329 ASSERT_TRUE(sps);
330 EXPECT_EQ(sps->width, kWidth);
331 EXPECT_EQ(sps->height, kHeight);
332 EXPECT_EQ(sps->vui_params_present, 1u);
333 }
334 }
335
336 class SpsVuiRewriterTest : public ::testing::Test,
337 public ::testing::WithParamInterface<
338 ::testing::tuple<VuiHeader,
339 const ColorSpace*,
340 SpsVuiRewriter::ParseResult>> {
341 };
342
TEST_P(SpsVuiRewriterTest,RewriteVui)343 TEST_P(SpsVuiRewriterTest, RewriteVui) {
344 VuiHeader vui = ::testing::get<0>(GetParam());
345 const ColorSpace* color_space = ::testing::get<1>(GetParam());
346 SpsVuiRewriter::ParseResult expected_parse_result =
347 ::testing::get<2>(GetParam());
348 TestSps(vui, color_space, expected_parse_result);
349 }
350
351 INSTANTIATE_TEST_SUITE_P(
352 All,
353 SpsVuiRewriterTest,
354 ::testing::Values(
355 std::make_tuple(kVuiNoFrameBuffering,
356 nullptr,
357 SpsVuiRewriter::ParseResult::kVuiOk),
358 std::make_tuple(kVuiNoVideoSignalType,
359 &kColorSpaceH264Default,
360 SpsVuiRewriter::ParseResult::kVuiOk),
361 std::make_tuple(kVuiLimitedRangeBt709Color,
362 &kColorSpaceBt709LimitedRange,
363 SpsVuiRewriter::ParseResult::kVuiOk),
364 std::make_tuple(kVuiNotPresent,
365 nullptr,
366 SpsVuiRewriter::ParseResult::kVuiRewritten),
367 std::make_tuple(kVuiNoBitstreamRestriction,
368 nullptr,
369 SpsVuiRewriter::ParseResult::kVuiRewritten),
370 std::make_tuple(kVuiFrameBuffering,
371 nullptr,
372 SpsVuiRewriter::ParseResult::kVuiRewritten),
373 std::make_tuple(kVuiLimitedRangeNoColourDescription,
374 &kColorSpaceFullRange,
375 SpsVuiRewriter::ParseResult::kVuiRewritten),
376 std::make_tuple(kVuiNoVideoSignalType,
377 &kColorSpacePrimariesBt709,
378 SpsVuiRewriter::ParseResult::kVuiRewritten),
379 std::make_tuple(kVuiNoVideoSignalType,
380 &kColorSpaceTransferBt709,
381 SpsVuiRewriter::ParseResult::kVuiRewritten),
382 std::make_tuple(kVuiNoVideoSignalType,
383 &kColorSpaceMatrixBt709,
384 SpsVuiRewriter::ParseResult::kVuiRewritten),
385 std::make_tuple(kVuiFullRangeNoColourDescription,
386 &kColorSpaceH264Default,
387 SpsVuiRewriter::ParseResult::kVuiRewritten),
388 std::make_tuple(kVuiLimitedRangeBt709Color,
389 &kColorSpaceH264Default,
390 SpsVuiRewriter::ParseResult::kVuiRewritten)));
391
TEST(SpsVuiRewriterOutgoingVuiTest,ParseOutgoingBitstreamOptimalVui)392 TEST(SpsVuiRewriterOutgoingVuiTest, ParseOutgoingBitstreamOptimalVui) {
393 rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
394
395 rtc::Buffer optimal_sps;
396 GenerateFakeSps(kVuiNoFrameBuffering, &optimal_sps);
397
398 rtc::Buffer buffer;
399 const size_t kNumNalus = 2;
400 size_t nalu_offsets[kNumNalus];
401 size_t nalu_lengths[kNumNalus];
402 buffer.AppendData(kStartSequence);
403 nalu_offsets[0] = buffer.size();
404 nalu_lengths[0] = optimal_sps.size();
405 buffer.AppendData(optimal_sps);
406 buffer.AppendData(kStartSequence);
407 nalu_offsets[1] = buffer.size();
408 nalu_lengths[1] = sizeof(kIdr1);
409 buffer.AppendData(kIdr1);
410
411 rtc::Buffer modified_buffer;
412 size_t modified_nalu_offsets[kNumNalus];
413 size_t modified_nalu_lengths[kNumNalus];
414
415 SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps(
416 buffer, kNumNalus, nalu_offsets, nalu_lengths, nullptr, &modified_buffer,
417 modified_nalu_offsets, modified_nalu_lengths);
418
419 EXPECT_THAT(
420 std::vector<uint8_t>(modified_buffer.data(),
421 modified_buffer.data() + modified_buffer.size()),
422 ::testing::ElementsAreArray(buffer.data(), buffer.size()));
423 EXPECT_THAT(std::vector<size_t>(modified_nalu_offsets,
424 modified_nalu_offsets + kNumNalus),
425 ::testing::ElementsAreArray(nalu_offsets, kNumNalus));
426 EXPECT_THAT(std::vector<size_t>(modified_nalu_lengths,
427 modified_nalu_lengths + kNumNalus),
428 ::testing::ElementsAreArray(nalu_lengths, kNumNalus));
429 }
430
TEST(SpsVuiRewriterOutgoingVuiTest,ParseOutgoingBitstreamNoVui)431 TEST(SpsVuiRewriterOutgoingVuiTest, ParseOutgoingBitstreamNoVui) {
432 rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
433
434 rtc::Buffer sps;
435 GenerateFakeSps(kVuiNotPresent, &sps);
436
437 rtc::Buffer buffer;
438 const size_t kNumNalus = 3;
439 size_t nalu_offsets[kNumNalus];
440 size_t nalu_lengths[kNumNalus];
441 buffer.AppendData(kStartSequence);
442 nalu_offsets[0] = buffer.size();
443 nalu_lengths[0] = sizeof(kIdr1);
444 buffer.AppendData(kIdr1);
445 buffer.AppendData(kStartSequence);
446 nalu_offsets[1] = buffer.size();
447 nalu_lengths[1] = sizeof(kSpsNaluType) + sps.size();
448 buffer.AppendData(kSpsNaluType);
449 buffer.AppendData(sps);
450 buffer.AppendData(kStartSequence);
451 nalu_offsets[2] = buffer.size();
452 nalu_lengths[2] = sizeof(kIdr2);
453 buffer.AppendData(kIdr2);
454
455 rtc::Buffer optimal_sps;
456 GenerateFakeSps(kVuiNoFrameBuffering, &optimal_sps);
457
458 rtc::Buffer expected_buffer;
459 size_t expected_nalu_offsets[kNumNalus];
460 size_t expected_nalu_lengths[kNumNalus];
461 expected_buffer.AppendData(kStartSequence);
462 expected_nalu_offsets[0] = expected_buffer.size();
463 expected_nalu_lengths[0] = sizeof(kIdr1);
464 expected_buffer.AppendData(kIdr1);
465 expected_buffer.AppendData(kStartSequence);
466 expected_nalu_offsets[1] = expected_buffer.size();
467 expected_nalu_lengths[1] = sizeof(kSpsNaluType) + optimal_sps.size();
468 expected_buffer.AppendData(kSpsNaluType);
469 expected_buffer.AppendData(optimal_sps);
470 expected_buffer.AppendData(kStartSequence);
471 expected_nalu_offsets[2] = expected_buffer.size();
472 expected_nalu_lengths[2] = sizeof(kIdr2);
473 expected_buffer.AppendData(kIdr2);
474
475 rtc::Buffer modified_buffer;
476 size_t modified_nalu_offsets[kNumNalus];
477 size_t modified_nalu_lengths[kNumNalus];
478
479 SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps(
480 buffer, kNumNalus, nalu_offsets, nalu_lengths, nullptr, &modified_buffer,
481 modified_nalu_offsets, modified_nalu_lengths);
482
483 EXPECT_THAT(
484 std::vector<uint8_t>(modified_buffer.data(),
485 modified_buffer.data() + modified_buffer.size()),
486 ::testing::ElementsAreArray(expected_buffer.data(),
487 expected_buffer.size()));
488 EXPECT_THAT(std::vector<size_t>(modified_nalu_offsets,
489 modified_nalu_offsets + kNumNalus),
490 ::testing::ElementsAreArray(expected_nalu_offsets, kNumNalus));
491 EXPECT_THAT(std::vector<size_t>(modified_nalu_lengths,
492 modified_nalu_lengths + kNumNalus),
493 ::testing::ElementsAreArray(expected_nalu_lengths, kNumNalus));
494 }
495 } // namespace webrtc
496