1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.h"
12
13 #include <cstring>
14 #include <utility>
15
16 #include "modules/rtp_rtcp/source/byte_io.h"
17 #include "rtc_base/checks.h"
18
19 namespace webrtc {
PackHeader(uint8_t * buffer,MultiplexImageHeader header)20 int PackHeader(uint8_t* buffer, MultiplexImageHeader header) {
21 int offset = 0;
22 ByteWriter<uint8_t>::WriteBigEndian(buffer + offset, header.component_count);
23 offset += sizeof(uint8_t);
24
25 ByteWriter<uint16_t>::WriteBigEndian(buffer + offset, header.image_index);
26 offset += sizeof(uint16_t);
27
28 ByteWriter<uint16_t>::WriteBigEndian(buffer + offset,
29 header.augmenting_data_size);
30 offset += sizeof(uint16_t);
31
32 ByteWriter<uint32_t>::WriteBigEndian(buffer + offset,
33 header.augmenting_data_offset);
34 offset += sizeof(uint32_t);
35
36 ByteWriter<uint32_t>::WriteBigEndian(buffer + offset,
37 header.first_component_header_offset);
38 offset += sizeof(uint32_t);
39
40 RTC_DCHECK_EQ(offset, kMultiplexImageHeaderSize);
41 return offset;
42 }
43
UnpackHeader(const uint8_t * buffer)44 MultiplexImageHeader UnpackHeader(const uint8_t* buffer) {
45 MultiplexImageHeader header;
46 int offset = 0;
47 header.component_count = ByteReader<uint8_t>::ReadBigEndian(buffer + offset);
48 offset += sizeof(uint8_t);
49
50 header.image_index = ByteReader<uint16_t>::ReadBigEndian(buffer + offset);
51 offset += sizeof(uint16_t);
52
53 header.augmenting_data_size =
54 ByteReader<uint16_t>::ReadBigEndian(buffer + offset);
55 offset += sizeof(uint16_t);
56
57 header.augmenting_data_offset =
58 ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
59 offset += sizeof(uint32_t);
60
61 header.first_component_header_offset =
62 ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
63 offset += sizeof(uint32_t);
64
65 RTC_DCHECK_EQ(offset, kMultiplexImageHeaderSize);
66 return header;
67 }
68
PackFrameHeader(uint8_t * buffer,MultiplexImageComponentHeader frame_header)69 int PackFrameHeader(uint8_t* buffer,
70 MultiplexImageComponentHeader frame_header) {
71 int offset = 0;
72 ByteWriter<uint32_t>::WriteBigEndian(
73 buffer + offset, frame_header.next_component_header_offset);
74 offset += sizeof(uint32_t);
75
76 ByteWriter<uint8_t>::WriteBigEndian(buffer + offset,
77 frame_header.component_index);
78 offset += sizeof(uint8_t);
79
80 ByteWriter<uint32_t>::WriteBigEndian(buffer + offset,
81 frame_header.bitstream_offset);
82 offset += sizeof(uint32_t);
83
84 ByteWriter<uint32_t>::WriteBigEndian(buffer + offset,
85 frame_header.bitstream_length);
86 offset += sizeof(uint32_t);
87
88 ByteWriter<uint8_t>::WriteBigEndian(buffer + offset, frame_header.codec_type);
89 offset += sizeof(uint8_t);
90
91 ByteWriter<uint8_t>::WriteBigEndian(
92 buffer + offset, static_cast<uint8_t>(frame_header.frame_type));
93 offset += sizeof(uint8_t);
94
95 RTC_DCHECK_EQ(offset, kMultiplexImageComponentHeaderSize);
96 return offset;
97 }
98
UnpackFrameHeader(const uint8_t * buffer)99 MultiplexImageComponentHeader UnpackFrameHeader(const uint8_t* buffer) {
100 MultiplexImageComponentHeader frame_header;
101 int offset = 0;
102
103 frame_header.next_component_header_offset =
104 ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
105 offset += sizeof(uint32_t);
106
107 frame_header.component_index =
108 ByteReader<uint8_t>::ReadBigEndian(buffer + offset);
109 offset += sizeof(uint8_t);
110
111 frame_header.bitstream_offset =
112 ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
113 offset += sizeof(uint32_t);
114
115 frame_header.bitstream_length =
116 ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
117 offset += sizeof(uint32_t);
118
119 // TODO(nisse): This makes the wire format depend on the numeric values of the
120 // VideoCodecType and VideoFrameType enum constants.
121 frame_header.codec_type = static_cast<VideoCodecType>(
122 ByteReader<uint8_t>::ReadBigEndian(buffer + offset));
123 offset += sizeof(uint8_t);
124
125 frame_header.frame_type = static_cast<VideoFrameType>(
126 ByteReader<uint8_t>::ReadBigEndian(buffer + offset));
127 offset += sizeof(uint8_t);
128
129 RTC_DCHECK_EQ(offset, kMultiplexImageComponentHeaderSize);
130 return frame_header;
131 }
132
PackBitstream(uint8_t * buffer,MultiplexImageComponent image)133 void PackBitstream(uint8_t* buffer, MultiplexImageComponent image) {
134 memcpy(buffer, image.encoded_image.data(), image.encoded_image.size());
135 }
136
MultiplexImage(uint16_t picture_index,uint8_t frame_count,std::unique_ptr<uint8_t[]> augmenting_data,uint16_t augmenting_data_size)137 MultiplexImage::MultiplexImage(uint16_t picture_index,
138 uint8_t frame_count,
139 std::unique_ptr<uint8_t[]> augmenting_data,
140 uint16_t augmenting_data_size)
141 : image_index(picture_index),
142 component_count(frame_count),
143 augmenting_data_size(augmenting_data_size),
144 augmenting_data(std::move(augmenting_data)) {}
145
PackAndRelease(const MultiplexImage & multiplex_image)146 EncodedImage MultiplexEncodedImagePacker::PackAndRelease(
147 const MultiplexImage& multiplex_image) {
148 MultiplexImageHeader header;
149 std::vector<MultiplexImageComponentHeader> frame_headers;
150
151 header.component_count = multiplex_image.component_count;
152 header.image_index = multiplex_image.image_index;
153 int header_offset = kMultiplexImageHeaderSize;
154 header.first_component_header_offset = header_offset;
155 header.augmenting_data_offset =
156 header_offset +
157 kMultiplexImageComponentHeaderSize * header.component_count;
158 header.augmenting_data_size = multiplex_image.augmenting_data_size;
159 int bitstream_offset =
160 header.augmenting_data_offset + header.augmenting_data_size;
161
162 const std::vector<MultiplexImageComponent>& images =
163 multiplex_image.image_components;
164 EncodedImage combined_image = images[0].encoded_image;
165 for (size_t i = 0; i < images.size(); i++) {
166 MultiplexImageComponentHeader frame_header;
167 header_offset += kMultiplexImageComponentHeaderSize;
168 frame_header.next_component_header_offset =
169 (i == images.size() - 1) ? 0 : header_offset;
170 frame_header.component_index = images[i].component_index;
171
172 frame_header.bitstream_offset = bitstream_offset;
173 frame_header.bitstream_length =
174 static_cast<uint32_t>(images[i].encoded_image.size());
175 bitstream_offset += frame_header.bitstream_length;
176
177 frame_header.codec_type = images[i].codec_type;
178 frame_header.frame_type = images[i].encoded_image._frameType;
179
180 // As long as one component is delta frame, we have to mark the combined
181 // frame as delta frame, because it is necessary for all components to be
182 // key frame so as to decode the whole image without previous frame data.
183 // Thus only when all components are key frames, we can mark the combined
184 // frame as key frame.
185 if (frame_header.frame_type == VideoFrameType::kVideoFrameDelta) {
186 combined_image._frameType = VideoFrameType::kVideoFrameDelta;
187 }
188
189 frame_headers.push_back(frame_header);
190 }
191
192 auto buffer = EncodedImageBuffer::Create(bitstream_offset);
193 combined_image.SetEncodedData(buffer);
194
195 // header
196 header_offset = PackHeader(buffer->data(), header);
197 RTC_DCHECK_EQ(header.first_component_header_offset,
198 kMultiplexImageHeaderSize);
199
200 // Frame Header
201 for (size_t i = 0; i < images.size(); i++) {
202 int relative_offset =
203 PackFrameHeader(buffer->data() + header_offset, frame_headers[i]);
204 RTC_DCHECK_EQ(relative_offset, kMultiplexImageComponentHeaderSize);
205
206 header_offset = frame_headers[i].next_component_header_offset;
207 RTC_DCHECK_EQ(header_offset,
208 (i == images.size() - 1)
209 ? 0
210 : (kMultiplexImageHeaderSize +
211 kMultiplexImageComponentHeaderSize * (i + 1)));
212 }
213
214 // Augmenting Data
215 if (multiplex_image.augmenting_data_size != 0) {
216 memcpy(buffer->data() + header.augmenting_data_offset,
217 multiplex_image.augmenting_data.get(),
218 multiplex_image.augmenting_data_size);
219 }
220
221 // Bitstreams
222 for (size_t i = 0; i < images.size(); i++) {
223 PackBitstream(buffer->data() + frame_headers[i].bitstream_offset,
224 images[i]);
225 }
226
227 return combined_image;
228 }
229
Unpack(const EncodedImage & combined_image)230 MultiplexImage MultiplexEncodedImagePacker::Unpack(
231 const EncodedImage& combined_image) {
232 const MultiplexImageHeader& header = UnpackHeader(combined_image.data());
233
234 std::vector<MultiplexImageComponentHeader> frame_headers;
235 int header_offset = header.first_component_header_offset;
236
237 while (header_offset > 0) {
238 frame_headers.push_back(
239 UnpackFrameHeader(combined_image.data() + header_offset));
240 header_offset = frame_headers.back().next_component_header_offset;
241 }
242
243 RTC_DCHECK_LE(frame_headers.size(), header.component_count);
244 std::unique_ptr<uint8_t[]> augmenting_data = nullptr;
245 if (header.augmenting_data_size != 0) {
246 augmenting_data =
247 std::unique_ptr<uint8_t[]>(new uint8_t[header.augmenting_data_size]);
248 memcpy(augmenting_data.get(),
249 combined_image.data() + header.augmenting_data_offset,
250 header.augmenting_data_size);
251 }
252
253 MultiplexImage multiplex_image(header.image_index, header.component_count,
254 std::move(augmenting_data),
255 header.augmenting_data_size);
256
257 for (size_t i = 0; i < frame_headers.size(); i++) {
258 MultiplexImageComponent image_component;
259 image_component.component_index = frame_headers[i].component_index;
260 image_component.codec_type = frame_headers[i].codec_type;
261
262 EncodedImage encoded_image = combined_image;
263 encoded_image.SetTimestamp(combined_image.Timestamp());
264 encoded_image._frameType = frame_headers[i].frame_type;
265 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
266 combined_image.data() + frame_headers[i].bitstream_offset,
267 frame_headers[i].bitstream_length));
268
269 image_component.encoded_image = encoded_image;
270
271 multiplex_image.image_components.push_back(image_component);
272 }
273
274 return multiplex_image;
275 }
276
277 } // namespace webrtc
278