1 /* GStreamer
2 * Copyright (C) 2009 Wim Taymans <wim.taymans@gmail.com>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
13 *
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
20 /**
21 * SECTION:element-rtpj2kpay
22 * @title: rtpj2kpay
23 *
24 * Payload encode JPEG 2000 images into RTP packets according to RFC 5371
25 * and RFC 5372.
26 * For detailed information see: https://datatracker.ietf.org/doc/rfc5371/
27 * and https://datatracker.ietf.org/doc/rfc5372/
28 *
29 * The payloader takes a JPEG 2000 image, scans it for "packetization
30 * units" and constructs the RTP packet header followed by the JPEG 2000
31 * codestream. A "packetization unit" is defined as either a JPEG 2000 main header,
32 * a JPEG 2000 tile-part header, or a JPEG 2000 packet.
33 *
34 */
35
36 #ifdef HAVE_CONFIG_H
37 # include "config.h"
38 #endif
39
40 #include <string.h>
41 #include <gst/rtp/gstrtpbuffer.h>
42 #include <gst/video/video.h>
43 #include "gstrtpelements.h"
44 #include "gstrtpj2kcommon.h"
45 #include "gstrtpj2kpay.h"
46 #include "gstrtputils.h"
47
48 static GstStaticPadTemplate gst_rtp_j2k_pay_sink_template =
49 GST_STATIC_PAD_TEMPLATE ("sink",
50 GST_PAD_SINK,
51 GST_PAD_ALWAYS,
52 GST_STATIC_CAPS ("image/x-jpc, " GST_RTP_J2K_SAMPLING_LIST)
53 );
54
55
56 static GstStaticPadTemplate gst_rtp_j2k_pay_src_template =
57 GST_STATIC_PAD_TEMPLATE ("src",
58 GST_PAD_SRC,
59 GST_PAD_ALWAYS,
60 GST_STATIC_CAPS ("application/x-rtp, "
61 " media = (string) \"video\", "
62 " payload = (int) " GST_RTP_PAYLOAD_DYNAMIC_STRING ", "
63 " clock-rate = (int) 90000, "
64 GST_RTP_J2K_SAMPLING_LIST "," " encoding-name = (string) \"JPEG2000\"")
65 );
66
67 GST_DEBUG_CATEGORY_STATIC (rtpj2kpay_debug);
68 #define GST_CAT_DEFAULT (rtpj2kpay_debug)
69
70
71 enum
72 {
73 PROP_0,
74 PROP_LAST
75 };
76
77 typedef struct
78 {
79 guint tp:2;
80 guint MHF:2;
81 guint mh_id:3;
82 guint T:1;
83 guint priority:8;
84 guint tile:16;
85 guint offset:24;
86 } RtpJ2KHeader;
87
88 static void gst_rtp_j2k_pay_set_property (GObject * object, guint prop_id,
89 const GValue * value, GParamSpec * pspec);
90 static void gst_rtp_j2k_pay_get_property (GObject * object, guint prop_id,
91 GValue * value, GParamSpec * pspec);
92
93 static gboolean gst_rtp_j2k_pay_setcaps (GstRTPBasePayload * basepayload,
94 GstCaps * caps);
95
96 static GstFlowReturn gst_rtp_j2k_pay_handle_buffer (GstRTPBasePayload * pad,
97 GstBuffer * buffer);
98
99 #define gst_rtp_j2k_pay_parent_class parent_class
100 G_DEFINE_TYPE (GstRtpJ2KPay, gst_rtp_j2k_pay, GST_TYPE_RTP_BASE_PAYLOAD);
101 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (rtpj2kpay, "rtpj2kpay",
102 GST_RANK_SECONDARY, GST_TYPE_RTP_J2K_PAY, rtp_element_init (plugin));
103
104 static void
gst_rtp_j2k_pay_class_init(GstRtpJ2KPayClass * klass)105 gst_rtp_j2k_pay_class_init (GstRtpJ2KPayClass * klass)
106 {
107 GObjectClass *gobject_class;
108 GstElementClass *gstelement_class;
109 GstRTPBasePayloadClass *gstrtpbasepayload_class;
110
111 gobject_class = (GObjectClass *) klass;
112 gstelement_class = (GstElementClass *) klass;
113 gstrtpbasepayload_class = (GstRTPBasePayloadClass *) klass;
114
115 gobject_class->set_property = gst_rtp_j2k_pay_set_property;
116 gobject_class->get_property = gst_rtp_j2k_pay_get_property;
117
118 gst_element_class_add_static_pad_template (gstelement_class,
119 &gst_rtp_j2k_pay_src_template);
120 gst_element_class_add_static_pad_template (gstelement_class,
121 &gst_rtp_j2k_pay_sink_template);
122
123 gst_element_class_set_static_metadata (gstelement_class,
124 "RTP JPEG 2000 payloader", "Codec/Payloader/Network/RTP",
125 "Payload-encodes JPEG 2000 pictures into RTP packets (RFC 5371)",
126 "Wim Taymans <wim.taymans@gmail.com>");
127
128 gstrtpbasepayload_class->set_caps = gst_rtp_j2k_pay_setcaps;
129 gstrtpbasepayload_class->handle_buffer = gst_rtp_j2k_pay_handle_buffer;
130
131 GST_DEBUG_CATEGORY_INIT (rtpj2kpay_debug, "rtpj2kpay", 0,
132 "JPEG 2000 RTP Payloader");
133 }
134
135 static void
gst_rtp_j2k_pay_init(GstRtpJ2KPay * pay)136 gst_rtp_j2k_pay_init (GstRtpJ2KPay * pay)
137 {
138 }
139
140 static gboolean
gst_rtp_j2k_pay_setcaps(GstRTPBasePayload * basepayload,GstCaps * caps)141 gst_rtp_j2k_pay_setcaps (GstRTPBasePayload * basepayload, GstCaps * caps)
142 {
143 GstStructure *caps_structure = gst_caps_get_structure (caps, 0);
144 gboolean res;
145 gint width = 0, height = 0;
146 const gchar *sampling = NULL;
147
148 gboolean has_width = gst_structure_get_int (caps_structure, "width", &width);
149 gboolean has_height =
150 gst_structure_get_int (caps_structure, "height", &height);
151
152
153 /* sampling is a required field */
154 sampling = gst_structure_get_string (caps_structure, "sampling");
155
156 gst_rtp_base_payload_set_options (basepayload, "video", TRUE, "JPEG2000",
157 90000);
158
159 if (has_width && has_height)
160 res = gst_rtp_base_payload_set_outcaps (basepayload,
161 "sampling", G_TYPE_STRING, sampling, "width", G_TYPE_INT, width,
162 "height", G_TYPE_INT, height, NULL);
163 else
164 res =
165 gst_rtp_base_payload_set_outcaps (basepayload, "sampling",
166 G_TYPE_STRING, sampling, NULL);
167 return res;
168 }
169
170
171 static guint
gst_rtp_j2k_pay_header_size(const guint8 * data,guint offset)172 gst_rtp_j2k_pay_header_size (const guint8 * data, guint offset)
173 {
174 return data[offset] << 8 | data[offset + 1];
175 }
176
177
178 static GstRtpJ2KMarker
gst_rtp_j2k_pay_scan_marker(const guint8 * data,guint size,guint * offset)179 gst_rtp_j2k_pay_scan_marker (const guint8 * data, guint size, guint * offset)
180 {
181 while ((data[(*offset)++] != GST_J2K_MARKER) && ((*offset) < size));
182
183 if (G_UNLIKELY ((*offset) >= size)) {
184 return GST_J2K_MARKER_EOC;
185 } else {
186 guint8 marker = data[(*offset)++];
187 return (GstRtpJ2KMarker) marker;
188 }
189 }
190
191 typedef struct
192 {
193 RtpJ2KHeader header;
194 gboolean multi_tile;
195 gboolean bitstream;
196 guint next_sot;
197 gboolean force_packet;
198 } RtpJ2KState;
199
200
201 /* Note: The standard recommends that headers be put in their own RTP packets, so we follow
202 * this recommendation in the code. Also, this method groups together all J2K packets
203 * for a tile part and treats this group as a packetization unit. According to the RFC,
204 * only an individual J2K packet is considered a packetization unit.
205 */
206
207 static guint
find_pu_end(GstRtpJ2KPay * pay,const guint8 * data,guint size,guint offset,RtpJ2KState * state)208 find_pu_end (GstRtpJ2KPay * pay, const guint8 * data, guint size,
209 guint offset, RtpJ2KState * state)
210 {
211 gboolean cut_sop = FALSE;
212 GstRtpJ2KMarker marker;
213
214 /* parse the j2k header for 'start of codestream' */
215 GST_LOG_OBJECT (pay, "checking from offset %u", offset);
216 while (offset < size) {
217 marker = gst_rtp_j2k_pay_scan_marker (data, size, &offset);
218
219 if (state->bitstream) {
220 /* parsing bitstream, only look for SOP */
221 switch (marker) {
222 case GST_J2K_MARKER_SOP:
223 GST_LOG_OBJECT (pay, "found SOP at %u", offset);
224 if (cut_sop)
225 return offset - 2;
226 cut_sop = TRUE;
227 break;
228 case GST_J2K_MARKER_EPH:
229 /* just skip over EPH */
230 GST_LOG_OBJECT (pay, "found EPH at %u", offset);
231 break;
232 default:
233 if (offset >= state->next_sot) {
234 GST_LOG_OBJECT (pay, "reached next SOT at %u", offset);
235 state->bitstream = FALSE;
236 state->force_packet = TRUE;
237 if (marker == GST_J2K_MARKER_EOC && state->next_sot + 2 <= size)
238 /* include EOC but never go past the max size */
239 return state->next_sot + 2;
240 else
241 return state->next_sot;
242 }
243 break;
244 }
245 } else {
246 switch (marker) {
247 case GST_J2K_MARKER_SOC:
248 GST_LOG_OBJECT (pay, "found SOC at %u", offset);
249 /* start off by assuming that we will fit the entire header
250 into the RTP payload */
251 state->header.MHF = 3;
252 break;
253 case GST_J2K_MARKER_SOT:
254 {
255 guint len, Psot, tile;
256
257 GST_LOG_OBJECT (pay, "found SOT at %u", offset);
258 /* SOT for first tile part in code stream:
259 force close of current RTP packet, so that it
260 only contains main header */
261 if (state->header.MHF) {
262 state->force_packet = TRUE;
263 return offset - 2;
264 }
265
266 /* parse SOT but do some sanity checks first */
267 len = gst_rtp_j2k_pay_header_size (data, offset);
268 GST_LOG_OBJECT (pay, "SOT length %u", len);
269 if (len < 8)
270 return size;
271 if (offset + len >= size)
272 return size;
273
274 /* Isot */
275 tile = GST_READ_UINT16_BE (&data[offset + 2]);
276
277 if (!state->multi_tile) {
278 /* we have detected multiple tiles in this rtp packet : tile bit is now invalid */
279 if (state->header.T == 0 && state->header.tile != tile) {
280 state->header.T = 1;
281 state->multi_tile = TRUE;
282 } else {
283 state->header.T = 0;
284 }
285 }
286 state->header.tile = tile;
287
288 /* Note: Tile parts from multiple tiles in single RTP packet
289 will make T invalid.
290 This cannot happen in our case since we always
291 send tile headers in their own RTP packets, so we cannot mix
292 tile parts in a single RTP packet */
293
294 /* Psot: offset of next tile. If it's 0, next tile goes all the way
295 to the end of the data */
296 Psot = GST_READ_UINT32_BE (&data[offset + 4]);
297 if (Psot == 0)
298 state->next_sot = size;
299 else
300 state->next_sot = offset - 2 + Psot;
301
302 offset += len;
303 GST_LOG_OBJECT (pay, "Isot %u, Psot %u, next %u", state->header.tile,
304 Psot, state->next_sot);
305 break;
306 }
307 case GST_J2K_MARKER_SOD:
308 GST_LOG_OBJECT (pay, "found SOD at %u", offset);
309 /* go to bitstream parsing */
310 state->bitstream = TRUE;
311 /* cut at the next SOP or else include all data */
312 cut_sop = TRUE;
313 /* force a new packet when we see SOP, this can be optional but the
314 * spec recommends packing headers separately */
315 state->force_packet = TRUE;
316 break;
317 case GST_J2K_MARKER_EOC:
318 GST_LOG_OBJECT (pay, "found EOC at %u", offset);
319 return offset;
320 default:
321 {
322 guint len = gst_rtp_j2k_pay_header_size (data, offset);
323 GST_LOG_OBJECT (pay, "skip 0x%02x len %u", marker, len);
324 offset += len;
325 break;
326 }
327 }
328 }
329 }
330 GST_DEBUG_OBJECT (pay, "reached end of data");
331 return size;
332 }
333
334 static GstFlowReturn
gst_rtp_j2k_pay_handle_buffer(GstRTPBasePayload * basepayload,GstBuffer * buffer)335 gst_rtp_j2k_pay_handle_buffer (GstRTPBasePayload * basepayload,
336 GstBuffer * buffer)
337 {
338 GstRtpJ2KPay *pay;
339 GstClockTime timestamp;
340 GstFlowReturn ret = GST_FLOW_ERROR;
341 RtpJ2KState state;
342 GstBufferList *list = NULL;
343 GstMapInfo map;
344 guint mtu, max_size;
345 guint offset;
346 guint end, pos;
347
348 pay = GST_RTP_J2K_PAY (basepayload);
349 mtu = GST_RTP_BASE_PAYLOAD_MTU (pay);
350
351 gst_buffer_map (buffer, &map, GST_MAP_READ);
352 timestamp = GST_BUFFER_PTS (buffer);
353 offset = pos = end = 0;
354
355 GST_LOG_OBJECT (pay,
356 "got buffer size %" G_GSIZE_FORMAT ", timestamp %" GST_TIME_FORMAT,
357 map.size, GST_TIME_ARGS (timestamp));
358
359 /* do some header defaults first */
360 state.header.tp = 0; /* only progressive scan */
361 state.header.MHF = 0; /* no header */
362 state.header.mh_id = 0; /* always 0 for now */
363 state.header.T = 1; /* invalid tile, because we always begin with the main header */
364 state.header.priority = 255; /* always 255 for now */
365 state.header.tile = 0xffff; /* no tile number */
366 state.header.offset = 0; /* offset of 0 */
367 state.multi_tile = FALSE;
368 state.bitstream = FALSE;
369 state.next_sot = 0;
370 state.force_packet = FALSE;
371
372 /* get max packet length */
373 max_size =
374 gst_rtp_buffer_calc_payload_len (mtu - GST_RTP_J2K_HEADER_SIZE, 0, 0);
375
376 list = gst_buffer_list_new_sized ((mtu / max_size) + 1);
377
378 do {
379 GstBuffer *outbuf;
380 guint8 *header;
381 guint payload_size;
382 guint pu_size;
383 GstRTPBuffer rtp = { NULL };
384
385 /* try to pack as much as we can */
386 do {
387 /* see how much we have scanned already */
388 pu_size = end - offset;
389 GST_DEBUG_OBJECT (pay, "scanned pu size %u", pu_size);
390
391 /* we need to make a new packet */
392 if (state.force_packet) {
393 GST_DEBUG_OBJECT (pay, "need to force a new packet");
394 state.force_packet = FALSE;
395 pos = end;
396 break;
397 }
398
399 /* else see if we have enough */
400 if (pu_size > max_size) {
401 if (pos != offset)
402 /* the packet became too large, use previous scanpos */
403 pu_size = pos - offset;
404 else
405 /* the already scanned data was already too big, make sure we start
406 * scanning from the last searched position */
407 pos = end;
408
409 GST_DEBUG_OBJECT (pay, "max size exceeded pu_size %u", pu_size);
410 break;
411 }
412
413 pos = end;
414
415 /* exit when finished */
416 if (pos == map.size)
417 break;
418
419 /* scan next packetization unit and fill in the header */
420 end = find_pu_end (pay, map.data, map.size, pos, &state);
421 } while (TRUE);
422
423 while (pu_size > 0) {
424 guint packet_size, data_size;
425 GstBuffer *paybuf;
426
427 /* calculate the packet size */
428 packet_size =
429 gst_rtp_buffer_calc_packet_len (pu_size + GST_RTP_J2K_HEADER_SIZE, 0,
430 0);
431
432 if (packet_size > mtu) {
433 GST_DEBUG_OBJECT (pay, "needed packet size %u clamped to MTU %u",
434 packet_size, mtu);
435 packet_size = mtu;
436 } else {
437 GST_DEBUG_OBJECT (pay, "needed packet size %u fits in MTU %u",
438 packet_size, mtu);
439 }
440
441 /* get total payload size and data size */
442 payload_size = gst_rtp_buffer_calc_payload_len (packet_size, 0, 0);
443 data_size = payload_size - GST_RTP_J2K_HEADER_SIZE;
444
445 /* make buffer for header */
446 outbuf =
447 gst_rtp_base_payload_allocate_output_buffer (basepayload,
448 GST_RTP_J2K_HEADER_SIZE, 0, 0);
449
450 GST_BUFFER_PTS (outbuf) = timestamp;
451
452 gst_rtp_buffer_map (outbuf, GST_MAP_WRITE, &rtp);
453
454 /* get pointer to header */
455 header = gst_rtp_buffer_get_payload (&rtp);
456
457 pu_size -= data_size;
458
459 /* reached the end of a packetization unit */
460 if (pu_size == 0 && end >= map.size) {
461 gst_rtp_buffer_set_marker (&rtp, TRUE);
462 }
463 /* If we were processing a header, see if all fits in one RTP packet
464 or if we have to fragment it */
465 if (state.header.MHF) {
466 switch (state.header.MHF) {
467 case 3:
468 if (pu_size > 0)
469 state.header.MHF = 1;
470 break;
471 case 1:
472 if (pu_size == 0)
473 state.header.MHF = 2;
474 break;
475 default:
476 break;
477 }
478 }
479
480 /*
481 * RtpJ2KHeader:
482 * @tp: type (0 progressive, 1 odd field, 2 even field)
483 * @MHF: Main Header Flag
484 * @mh_id: Main Header Identification
485 * @T: Tile field invalidation flag
486 * @priority: priority
487 * @tile number: the tile number of the payload
488 * @reserved: set to 0
489 * @fragment offset: the byte offset of the current payload
490 *
491 * 0 1 2 3
492 * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
493 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
494 * |tp |MHF|mh_id|T| priority | tile number |
495 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
496 * |reserved | fragment offset |
497 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
498 */
499 header[0] = (state.header.tp << 6) | (state.header.MHF << 4) |
500 (state.header.mh_id << 1) | state.header.T;
501 header[1] = state.header.priority;
502 header[2] = state.header.tile >> 8;
503 header[3] = state.header.tile & 0xff;
504 header[4] = 0;
505 header[5] = state.header.offset >> 16;
506 header[6] = (state.header.offset >> 8) & 0xff;
507 header[7] = state.header.offset & 0xff;
508
509 gst_rtp_buffer_unmap (&rtp);
510
511 /* make subbuffer of j2k data */
512 paybuf = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_ALL,
513 offset, data_size);
514 gst_rtp_copy_video_meta (basepayload, outbuf, paybuf);
515 outbuf = gst_buffer_append (outbuf, paybuf);
516
517 gst_buffer_list_add (list, outbuf);
518
519 /* reset multi_tile */
520 state.multi_tile = FALSE;
521
522
523 /* set MHF to zero if there is no more main header to process */
524 if (state.header.MHF & 2)
525 state.header.MHF = 0;
526
527 /* tile is valid, if there is no more header to process */
528 if (!state.header.MHF)
529 state.header.T = 0;
530
531
532 offset += data_size;
533 state.header.offset = offset;
534 }
535 offset = pos;
536 } while (offset < map.size);
537
538 gst_buffer_unmap (buffer, &map);
539 gst_buffer_unref (buffer);
540
541 /* push the whole buffer list at once */
542 ret = gst_rtp_base_payload_push_list (basepayload, list);
543
544 return ret;
545 }
546
547 static void
gst_rtp_j2k_pay_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)548 gst_rtp_j2k_pay_set_property (GObject * object, guint prop_id,
549 const GValue * value, GParamSpec * pspec)
550 {
551 switch (prop_id) {
552 default:
553 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
554 break;
555 }
556 }
557
558 static void
gst_rtp_j2k_pay_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)559 gst_rtp_j2k_pay_get_property (GObject * object, guint prop_id,
560 GValue * value, GParamSpec * pspec)
561 {
562 switch (prop_id) {
563 default:
564 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
565 break;
566 }
567 }
568