2 * Copyright (C) 2009 Wim Taymans <wim.taymans@gmail.com>
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
21 * SECTION:element-rtpj2kpay
23 * Payload encode JPEG 2000 images into RTP packets according to RFC 5371
25 * For detailed information see: https://datatracker.ietf.org/doc/rfc5371/
26 * and https://datatracker.ietf.org/doc/rfc5372/
28 * The payloader takes a JPEG 2000 image, scans it for "packetization
29 * units" and constructs the RTP packet header followed by the JPEG 2000
30 * codestream. A "packetization unit" is defined as either a JPEG 2000 main header,
31 * a JPEG 2000 tile-part header, or a JPEG 2000 packet.
41 #include <gst/rtp/gstrtpbuffer.h>
42 #include <gst/video/video.h>
43 #include "gstrtpj2kcommon.h"
44 #include "gstrtpj2kpay.h"
45 #include "gstrtputils.h"
47 static GstStaticPadTemplate gst_rtp_j2k_pay_sink_template =
48 GST_STATIC_PAD_TEMPLATE ("sink",
51 GST_STATIC_CAPS ("image/x-jpc, " GST_RTP_J2K_SAMPLING_LIST)
55 static GstStaticPadTemplate gst_rtp_j2k_pay_src_template =
56 GST_STATIC_PAD_TEMPLATE ("src",
59 GST_STATIC_CAPS ("application/x-rtp, "
60 " media = (string) \"video\", "
61 " payload = (int) " GST_RTP_PAYLOAD_DYNAMIC_STRING ", "
62 " clock-rate = (int) 90000, "
63 GST_RTP_J2K_SAMPLING_LIST "," " encoding-name = (string) \"JPEG2000\"")
66 GST_DEBUG_CATEGORY_STATIC (rtpj2kpay_debug);
67 #define GST_CAT_DEFAULT (rtpj2kpay_debug)
87 static void gst_rtp_j2k_pay_set_property (GObject * object, guint prop_id,
88 const GValue * value, GParamSpec * pspec);
89 static void gst_rtp_j2k_pay_get_property (GObject * object, guint prop_id,
90 GValue * value, GParamSpec * pspec);
92 static gboolean gst_rtp_j2k_pay_setcaps (GstRTPBasePayload * basepayload,
95 static GstFlowReturn gst_rtp_j2k_pay_handle_buffer (GstRTPBasePayload * pad,
98 #define gst_rtp_j2k_pay_parent_class parent_class
99 G_DEFINE_TYPE (GstRtpJ2KPay, gst_rtp_j2k_pay, GST_TYPE_RTP_BASE_PAYLOAD);
102 gst_rtp_j2k_pay_class_init (GstRtpJ2KPayClass * klass)
104 GObjectClass *gobject_class;
105 GstElementClass *gstelement_class;
106 GstRTPBasePayloadClass *gstrtpbasepayload_class;
108 gobject_class = (GObjectClass *) klass;
109 gstelement_class = (GstElementClass *) klass;
110 gstrtpbasepayload_class = (GstRTPBasePayloadClass *) klass;
112 gobject_class->set_property = gst_rtp_j2k_pay_set_property;
113 gobject_class->get_property = gst_rtp_j2k_pay_get_property;
115 gst_element_class_add_static_pad_template (gstelement_class,
116 &gst_rtp_j2k_pay_src_template);
117 gst_element_class_add_static_pad_template (gstelement_class,
118 &gst_rtp_j2k_pay_sink_template);
120 gst_element_class_set_static_metadata (gstelement_class,
121 "RTP JPEG 2000 payloader", "Codec/Payloader/Network/RTP",
122 "Payload-encodes JPEG 2000 pictures into RTP packets (RFC 5371)",
123 "Wim Taymans <wim.taymans@gmail.com>");
125 gstrtpbasepayload_class->set_caps = gst_rtp_j2k_pay_setcaps;
126 gstrtpbasepayload_class->handle_buffer = gst_rtp_j2k_pay_handle_buffer;
128 GST_DEBUG_CATEGORY_INIT (rtpj2kpay_debug, "rtpj2kpay", 0,
129 "JPEG 2000 RTP Payloader");
133 gst_rtp_j2k_pay_init (GstRtpJ2KPay * pay)
138 gst_rtp_j2k_pay_setcaps (GstRTPBasePayload * basepayload, GstCaps * caps)
140 GstStructure *caps_structure = gst_caps_get_structure (caps, 0);
142 gint width = 0, height = 0;
143 const gchar *sampling = NULL;
145 gboolean has_width = gst_structure_get_int (caps_structure, "width", &width);
146 gboolean has_height =
147 gst_structure_get_int (caps_structure, "height", &height);
150 /* sampling is a required field */
151 sampling = gst_structure_get_string (caps_structure, "sampling");
153 gst_rtp_base_payload_set_options (basepayload, "video", TRUE, "JPEG2000",
156 if (has_width && has_height)
157 res = gst_rtp_base_payload_set_outcaps (basepayload,
158 "sampling", G_TYPE_STRING, sampling, "width", G_TYPE_INT, width,
159 "height", G_TYPE_INT, height, NULL);
162 gst_rtp_base_payload_set_outcaps (basepayload, "sampling",
163 G_TYPE_STRING, sampling, NULL);
169 gst_rtp_j2k_pay_header_size (const guint8 * data, guint offset)
171 return data[offset] << 8 | data[offset + 1];
175 static GstRtpJ2KMarker
176 gst_rtp_j2k_pay_scan_marker (const guint8 * data, guint size, guint * offset)
178 while ((data[(*offset)++] != GST_J2K_MARKER) && ((*offset) < size));
180 if (G_UNLIKELY ((*offset) >= size)) {
181 return GST_J2K_MARKER_EOC;
183 guint8 marker = data[(*offset)++];
184 return (GstRtpJ2KMarker) marker;
194 gboolean force_packet;
198 /* Note: The standard recommends that headers be put in their own RTP packets, so we follow
199 * this recommendation in the code. Also, this method groups together all J2K packets
200 * for a tile part and treats this group as a packetization unit. According to the RFC,
201 * only an individual J2K packet is considered a packetization unit.
205 find_pu_end (GstRtpJ2KPay * pay, const guint8 * data, guint size,
206 guint offset, RtpJ2KState * state)
208 gboolean cut_sop = FALSE;
209 GstRtpJ2KMarker marker;
211 /* parse the j2k header for 'start of codestream' */
212 GST_LOG_OBJECT (pay, "checking from offset %u", offset);
213 while (offset < size) {
214 marker = gst_rtp_j2k_pay_scan_marker (data, size, &offset);
216 if (state->bitstream) {
217 /* parsing bitstream, only look for SOP */
219 case GST_J2K_MARKER_SOP:
220 GST_LOG_OBJECT (pay, "found SOP at %u", offset);
225 case GST_J2K_MARKER_EPH:
226 /* just skip over EPH */
227 GST_LOG_OBJECT (pay, "found EPH at %u", offset);
230 if (offset >= state->next_sot) {
231 GST_LOG_OBJECT (pay, "reached next SOT at %u", offset);
232 state->bitstream = FALSE;
233 state->force_packet = TRUE;
234 if (marker == GST_J2K_MARKER_EOC && state->next_sot + 2 <= size)
235 /* include EOC but never go past the max size */
236 return state->next_sot + 2;
238 return state->next_sot;
244 case GST_J2K_MARKER_SOC:
245 GST_LOG_OBJECT (pay, "found SOC at %u", offset);
246 /* start off by assuming that we will fit the entire header
247 into the RTP payload */
248 state->header.MHF = 3;
250 case GST_J2K_MARKER_SOT:
252 guint len, Psot, tile;
254 GST_LOG_OBJECT (pay, "found SOT at %u", offset);
255 /* SOT for first tile part in code stream:
256 force close of current RTP packet, so that it
257 only contains main header */
258 if (state->header.MHF) {
259 state->force_packet = TRUE;
263 /* parse SOT but do some sanity checks first */
264 len = gst_rtp_j2k_pay_header_size (data, offset);
265 GST_LOG_OBJECT (pay, "SOT length %u", len);
268 if (offset + len >= size)
272 tile = GST_READ_UINT16_BE (&data[offset + 2]);
274 if (!state->multi_tile) {
275 /* we have detected multiple tiles in this rtp packet : tile bit is now invalid */
276 if (state->header.T == 0 && state->header.tile != tile) {
278 state->multi_tile = TRUE;
283 state->header.tile = tile;
285 /* Note: Tile parts from multiple tiles in single RTP packet
287 This cannot happen in our case since we always
288 send tile headers in their own RTP packets, so we cannot mix
289 tile parts in a single RTP packet */
291 /* Psot: offset of next tile. If it's 0, next tile goes all the way
292 to the end of the data */
293 Psot = GST_READ_UINT32_BE (&data[offset + 4]);
295 state->next_sot = size;
297 state->next_sot = offset - 2 + Psot;
300 GST_LOG_OBJECT (pay, "Isot %u, Psot %u, next %u", state->header.tile,
301 Psot, state->next_sot);
304 case GST_J2K_MARKER_SOD:
305 GST_LOG_OBJECT (pay, "found SOD at %u", offset);
306 /* go to bitstream parsing */
307 state->bitstream = TRUE;
308 /* cut at the next SOP or else include all data */
310 /* force a new packet when we see SOP, this can be optional but the
311 * spec recommends packing headers separately */
312 state->force_packet = TRUE;
314 case GST_J2K_MARKER_EOC:
315 GST_LOG_OBJECT (pay, "found EOC at %u", offset);
319 guint len = gst_rtp_j2k_pay_header_size (data, offset);
320 GST_LOG_OBJECT (pay, "skip 0x%02x len %u", marker, len);
327 GST_DEBUG_OBJECT (pay, "reached end of data");
332 gst_rtp_j2k_pay_handle_buffer (GstRTPBasePayload * basepayload,
336 GstClockTime timestamp;
337 GstFlowReturn ret = GST_FLOW_ERROR;
339 GstBufferList *list = NULL;
345 pay = GST_RTP_J2K_PAY (basepayload);
346 mtu = GST_RTP_BASE_PAYLOAD_MTU (pay);
348 gst_buffer_map (buffer, &map, GST_MAP_READ);
349 timestamp = GST_BUFFER_PTS (buffer);
350 offset = pos = end = 0;
353 "got buffer size %" G_GSIZE_FORMAT ", timestamp %" GST_TIME_FORMAT,
354 map.size, GST_TIME_ARGS (timestamp));
356 /* do some header defaults first */
357 state.header.tp = 0; /* only progressive scan */
358 state.header.MHF = 0; /* no header */
359 state.header.mh_id = 0; /* always 0 for now */
360 state.header.T = 1; /* invalid tile, because we always begin with the main header */
361 state.header.priority = 255; /* always 255 for now */
362 state.header.tile = 0xffff; /* no tile number */
363 state.header.offset = 0; /* offset of 0 */
364 state.multi_tile = FALSE;
365 state.bitstream = FALSE;
367 state.force_packet = FALSE;
369 /* get max packet length */
371 gst_rtp_buffer_calc_payload_len (mtu - GST_RTP_J2K_HEADER_SIZE, 0, 0);
373 list = gst_buffer_list_new_sized ((mtu / max_size) + 1);
380 GstRTPBuffer rtp = { NULL };
382 /* try to pack as much as we can */
384 /* see how much we have scanned already */
385 pu_size = end - offset;
386 GST_DEBUG_OBJECT (pay, "scanned pu size %u", pu_size);
388 /* we need to make a new packet */
389 if (state.force_packet) {
390 GST_DEBUG_OBJECT (pay, "need to force a new packet");
391 state.force_packet = FALSE;
396 /* else see if we have enough */
397 if (pu_size > max_size) {
399 /* the packet became too large, use previous scanpos */
400 pu_size = pos - offset;
402 /* the already scanned data was already too big, make sure we start
403 * scanning from the last searched position */
406 GST_DEBUG_OBJECT (pay, "max size exceeded pu_size %u", pu_size);
412 /* exit when finished */
416 /* scan next packetization unit and fill in the header */
417 end = find_pu_end (pay, map.data, map.size, pos, &state);
420 while (pu_size > 0) {
421 guint packet_size, data_size;
424 /* calculate the packet size */
426 gst_rtp_buffer_calc_packet_len (pu_size + GST_RTP_J2K_HEADER_SIZE, 0,
429 if (packet_size > mtu) {
430 GST_DEBUG_OBJECT (pay, "needed packet size %u clamped to MTU %u",
434 GST_DEBUG_OBJECT (pay, "needed packet size %u fits in MTU %u",
438 /* get total payload size and data size */
439 payload_size = gst_rtp_buffer_calc_payload_len (packet_size, 0, 0);
440 data_size = payload_size - GST_RTP_J2K_HEADER_SIZE;
442 /* make buffer for header */
443 outbuf = gst_rtp_buffer_new_allocate (GST_RTP_J2K_HEADER_SIZE, 0, 0);
445 GST_BUFFER_PTS (outbuf) = timestamp;
447 gst_rtp_buffer_map (outbuf, GST_MAP_WRITE, &rtp);
449 /* get pointer to header */
450 header = gst_rtp_buffer_get_payload (&rtp);
452 pu_size -= data_size;
454 /* reached the end of a packetization unit */
455 if (pu_size == 0 && end >= map.size) {
456 gst_rtp_buffer_set_marker (&rtp, TRUE);
458 /* If we were processing a header, see if all fits in one RTP packet
459 or if we have to fragment it */
460 if (state.header.MHF) {
461 switch (state.header.MHF) {
464 state.header.MHF = 1;
468 state.header.MHF = 2;
477 * @tp: type (0 progressive, 1 odd field, 2 even field)
478 * @MHF: Main Header Flag
479 * @mh_id: Main Header Identification
480 * @T: Tile field invalidation flag
481 * @priority: priority
482 * @tile number: the tile number of the payload
483 * @reserved: set to 0
484 * @fragment offset: the byte offset of the current payload
487 * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
488 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
489 * |tp |MHF|mh_id|T| priority | tile number |
490 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
491 * |reserved | fragment offset |
492 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
494 header[0] = (state.header.tp << 6) | (state.header.MHF << 4) |
495 (state.header.mh_id << 1) | state.header.T;
496 header[1] = state.header.priority;
497 header[2] = state.header.tile >> 8;
498 header[3] = state.header.tile & 0xff;
500 header[5] = state.header.offset >> 16;
501 header[6] = (state.header.offset >> 8) & 0xff;
502 header[7] = state.header.offset & 0xff;
504 gst_rtp_buffer_unmap (&rtp);
506 /* make subbuffer of j2k data */
507 paybuf = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_ALL,
509 gst_rtp_copy_meta (GST_ELEMENT_CAST (basepayload), outbuf, paybuf,
510 g_quark_from_static_string (GST_META_TAG_VIDEO_STR));
511 outbuf = gst_buffer_append (outbuf, paybuf);
513 gst_buffer_list_add (list, outbuf);
515 /* reset multi_tile */
516 state.multi_tile = FALSE;
519 /* set MHF to zero if there is no more main header to process */
520 if (state.header.MHF & 2)
521 state.header.MHF = 0;
523 /* tile is valid, if there is no more header to process */
524 if (!state.header.MHF)
529 state.header.offset = offset;
532 } while (offset < map.size);
534 gst_buffer_unmap (buffer, &map);
535 gst_buffer_unref (buffer);
537 /* push the whole buffer list at once */
538 ret = gst_rtp_base_payload_push_list (basepayload, list);
544 gst_rtp_j2k_pay_set_property (GObject * object, guint prop_id,
545 const GValue * value, GParamSpec * pspec)
549 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
555 gst_rtp_j2k_pay_get_property (GObject * object, guint prop_id,
556 GValue * value, GParamSpec * pspec)
560 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
566 gst_rtp_j2k_pay_plugin_init (GstPlugin * plugin)
568 return gst_element_register (plugin, "rtpj2kpay", GST_RANK_SECONDARY,
569 GST_TYPE_RTP_J2K_PAY);