3 * Copyright (C) 2007 David Schleef <ds@schleef.org>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 * Boston, MA 02111-1307, USA.
21 * Logic enhanced by William Brack <wbrack@mmm.com.hk>
25 * SECTION:element-bayer2rgb
27 * Decodes raw camera bayer (fourcc BA81) to RGB.
31 * In order to guard against my advancing maturity, some extra detailed
32 * information about the logic of the decode is included here. Much of
33 * this was inspired by a technical paper from siliconimaging.com, which
34 * in turn was based upon an article from IEEE,
35 * T. Sakamoto, C. Nakanishi and T. Hase,
36 * “Software pixel interpolation for digital still cameras suitable for
38 * IEEE Trans. Consumer Electronics, vol. 44, no. 4, November 1998.
40 * The code assumes a Bayer matrix of the type produced by the fourcc
41 * BA81 (v4l2 format SBGGR8) of width w and height h which looks like:
51 * We expand this matrix, producing a separate {r, g, b} triple for each
52 * of the individual elements. The algorithm for doing this expansion is
55 * We are designing for speed of transformation, at a slight expense of code.
56 * First, we calculate the appropriate triples for the four corners, the
57 * remainder of the top and bottom rows, and the left and right columns.
58 * The reason for this is that those elements are transformed slightly
59 * differently than all of the remainder of the matrix. Finally, we transform
60 * all of the remainder.
62 * The transformation into the "appropriate triples" is based upon the
63 * "nearest neighbor" principal, with some additional complexity for the
64 * calculation of the "green" element, where an "adaptive" pairing is used.
66 * For purposes of documentation and indentification, each element of the
67 * original array can be put into one of four classes:
70 * GR A green element which is followed by a red one
71 * GB A green element which is followed by a blue one
79 #include <gst/base/gstbasetransform.h>
80 #include <gst/video/video.h>
85 #define GST_CAT_DEFAULT gst_bayer2rgb_debug
86 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
90 GST_BAYER_2_RGB_FORMAT_BGGR = 0,
91 GST_BAYER_2_RGB_FORMAT_GBRG,
92 GST_BAYER_2_RGB_FORMAT_GRBG,
93 GST_BAYER_2_RGB_FORMAT_RGGB
97 #define GST_TYPE_BAYER2RGB (gst_bayer2rgb_get_type())
98 #define GST_BAYER2RGB(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_BAYER2RGB,GstBayer2RGB))
99 #define GST_IS_BAYER2RGB(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_BAYER2RGB))
100 #define GST_BAYER2RGB_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass) ,GST_TYPE_BAYER2RGB,GstBayer2RGBClass))
101 #define GST_IS_BAYER2RGB_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass) ,GST_TYPE_BAYER2RGB))
102 #define GST_BAYER2RGB_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj) ,GST_TYPE_BAYER2RGB,GstBayer2RGBClass))
103 typedef struct _GstBayer2RGB GstBayer2RGB;
104 typedef struct _GstBayer2RGBClass GstBayer2RGBClass;
106 typedef void (*GstBayer2RGBProcessFunc) (GstBayer2RGB *, guint8 *, guint);
110 GstBaseTransform basetransform;
116 int pixsize; /* bytes per pixel */
117 int r_off; /* offset for red */
118 int g_off; /* offset for green */
119 int b_off; /* offset for blue */
123 struct _GstBayer2RGBClass
125 GstBaseTransformClass parent;
128 //#define SRC_CAPS GST_VIDEO_CAPS_RGBx
130 GST_VIDEO_CAPS_RGBx ";" \
131 GST_VIDEO_CAPS_xRGB ";" \
132 GST_VIDEO_CAPS_BGRx ";" \
133 GST_VIDEO_CAPS_xBGR ";" \
134 GST_VIDEO_CAPS_RGBA ";" \
135 GST_VIDEO_CAPS_ARGB ";" \
136 GST_VIDEO_CAPS_BGRA ";" \
137 GST_VIDEO_CAPS_ABGR ";" \
138 GST_VIDEO_CAPS_RGB ";" \
141 #define SINK_CAPS "video/x-raw-bayer,width=(int)[1,MAX],height=(int)[1,MAX]"
148 #define DEBUG_INIT(bla) \
149 GST_DEBUG_CATEGORY_INIT (gst_bayer2rgb_debug, "bayer2rgb", 0, "bayer2rgb element");
151 GType gst_bayer2rgb_get_type (void);
152 GST_BOILERPLATE_FULL (GstBayer2RGB, gst_bayer2rgb, GstBaseTransform,
153 GST_TYPE_BASE_TRANSFORM, DEBUG_INIT);
155 static void gst_bayer2rgb_set_property (GObject * object, guint prop_id,
156 const GValue * value, GParamSpec * pspec);
157 static void gst_bayer2rgb_get_property (GObject * object, guint prop_id,
158 GValue * value, GParamSpec * pspec);
160 static gboolean gst_bayer2rgb_set_caps (GstBaseTransform * filter,
161 GstCaps * incaps, GstCaps * outcaps);
162 static GstFlowReturn gst_bayer2rgb_transform (GstBaseTransform * base,
163 GstBuffer * inbuf, GstBuffer * outbuf);
164 static void gst_bayer2rgb_reset (GstBayer2RGB * filter);
165 static GstCaps *gst_bayer2rgb_transform_caps (GstBaseTransform * base,
166 GstPadDirection direction, GstCaps * caps);
167 static gboolean gst_bayer2rgb_get_unit_size (GstBaseTransform * base,
168 GstCaps * caps, guint * size);
172 gst_bayer2rgb_base_init (gpointer klass)
174 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
176 gst_element_class_set_details_simple (element_class,
177 "Bayer to RGB decoder for cameras", "Filter/Converter/Video",
178 "Converts video/x-raw-bayer to video/x-raw-rgb",
179 "William Brack <wbrack@mmm.com.hk>");
181 gst_element_class_add_pad_template (element_class,
182 gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
183 gst_caps_from_string (SRC_CAPS)));
184 gst_element_class_add_pad_template (element_class,
185 gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
186 gst_caps_from_string (SINK_CAPS)));
190 gst_bayer2rgb_class_init (GstBayer2RGBClass * klass)
192 GObjectClass *gobject_class;
194 gobject_class = (GObjectClass *) klass;
195 gobject_class->set_property = gst_bayer2rgb_set_property;
196 gobject_class->get_property = gst_bayer2rgb_get_property;
198 GST_BASE_TRANSFORM_CLASS (klass)->transform_caps =
199 GST_DEBUG_FUNCPTR (gst_bayer2rgb_transform_caps);
200 GST_BASE_TRANSFORM_CLASS (klass)->get_unit_size =
201 GST_DEBUG_FUNCPTR (gst_bayer2rgb_get_unit_size);
202 GST_BASE_TRANSFORM_CLASS (klass)->set_caps =
203 GST_DEBUG_FUNCPTR (gst_bayer2rgb_set_caps);
204 GST_BASE_TRANSFORM_CLASS (klass)->transform =
205 GST_DEBUG_FUNCPTR (gst_bayer2rgb_transform);
209 gst_bayer2rgb_init (GstBayer2RGB * filter, GstBayer2RGBClass * klass)
211 gst_bayer2rgb_reset (filter);
212 gst_base_transform_set_in_place (GST_BASE_TRANSFORM (filter), TRUE);
215 /* No properties are implemented, so only a warning is produced */
217 gst_bayer2rgb_set_property (GObject * object, guint prop_id,
218 const GValue * value, GParamSpec * pspec)
223 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
229 gst_bayer2rgb_get_property (GObject * object, guint prop_id,
230 GValue * value, GParamSpec * pspec)
235 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
240 /* Routine to convert colormask value into relative byte offset */
242 get_pix_offset (int mask, int bpp)
244 int bpp32 = (bpp / 8) - 3;
256 GST_ERROR ("Invalid color mask 0x%08x", mask);
262 gst_bayer2rgb_set_caps (GstBaseTransform * base, GstCaps * incaps,
265 GstBayer2RGB *bayer2rgb = GST_BAYER2RGB (base);
266 GstStructure *structure;
270 GST_DEBUG ("in caps %" GST_PTR_FORMAT " out caps %" GST_PTR_FORMAT, incaps,
273 structure = gst_caps_get_structure (incaps, 0);
275 gst_structure_get_int (structure, "width", &bayer2rgb->width);
276 gst_structure_get_int (structure, "height", &bayer2rgb->height);
277 bayer2rgb->stride = GST_ROUND_UP_4 (bayer2rgb->width);
279 format = gst_structure_get_string (structure, "format");
280 if (g_str_equal (format, "bggr")) {
281 bayer2rgb->format = GST_BAYER_2_RGB_FORMAT_BGGR;
282 } else if (g_str_equal (format, "gbrg")) {
283 bayer2rgb->format = GST_BAYER_2_RGB_FORMAT_GBRG;
284 } else if (g_str_equal (format, "grbg")) {
285 bayer2rgb->format = GST_BAYER_2_RGB_FORMAT_GRBG;
286 } else if (g_str_equal (format, "rggb")) {
287 bayer2rgb->format = GST_BAYER_2_RGB_FORMAT_RGGB;
292 /* To cater for different RGB formats, we need to set params for later */
293 structure = gst_caps_get_structure (outcaps, 0);
294 gst_structure_get_int (structure, "bpp", &bpp);
295 bayer2rgb->pixsize = bpp / 8;
296 gst_structure_get_int (structure, "red_mask", &val);
297 bayer2rgb->r_off = get_pix_offset (val, bpp);
298 gst_structure_get_int (structure, "green_mask", &val);
299 bayer2rgb->g_off = get_pix_offset (val, bpp);
300 gst_structure_get_int (structure, "blue_mask", &val);
301 bayer2rgb->b_off = get_pix_offset (val, bpp);
307 gst_bayer2rgb_reset (GstBayer2RGB * filter)
319 gst_bayer2rgb_transform_caps (GstBaseTransform * base,
320 GstPadDirection direction, GstCaps * caps)
322 GstStructure *structure;
324 GstStructure *newstruct;
326 GST_DEBUG_OBJECT (caps, "transforming caps (from)");
328 structure = gst_caps_get_structure (caps, 0);
330 if (direction == GST_PAD_SRC) {
331 newcaps = gst_caps_new_simple ("video/x-raw-bayer", NULL);
333 newcaps = gst_caps_new_simple ("video/x-raw-rgb", NULL);
335 newstruct = gst_caps_get_structure (newcaps, 0);
337 gst_structure_set_value (newstruct, "width",
338 gst_structure_get_value (structure, "width"));
339 gst_structure_set_value (newstruct, "height",
340 gst_structure_get_value (structure, "height"));
341 gst_structure_set_value (newstruct, "framerate",
342 gst_structure_get_value (structure, "framerate"));
344 GST_DEBUG_OBJECT (newcaps, "transforming caps (into)");
350 gst_bayer2rgb_get_unit_size (GstBaseTransform * base, GstCaps * caps,
353 GstStructure *structure;
359 structure = gst_caps_get_structure (caps, 0);
361 if (gst_structure_get_int (structure, "width", &width) &&
362 gst_structure_get_int (structure, "height", &height)) {
363 name = gst_structure_get_name (structure);
364 /* Our name must be either video/x-raw-bayer video/x-raw-rgb */
365 if (strcmp (name, "video/x-raw-rgb")) {
366 /* For bayer, we handle only BA81 (BGGR), which is BPP=24 */
367 *size = GST_ROUND_UP_4 (width) * height;
370 /* For output, calculate according to format */
371 if (gst_structure_get_int (structure, "bpp", &pixsize)) {
372 *size = width * height * (pixsize / 8);
378 GST_ELEMENT_ERROR (base, CORE, NEGOTIATION, (NULL),
379 ("Incomplete caps, some required field missing"));
384 * We define values for the colors, just to make the code more readable.
386 #define RED 0 /* Pure red element */
387 #define GREENB 1 /* Green element which is on a blue line */
388 #define BLUE 2 /* Pure blue element */
389 #define GREENR 3 /* Green element which is on a red line */
392 get_pixel_type (GstBayer2RGB * filter, int x, int y)
396 if (((x ^ filter->format) & 1)) {
397 if ((y ^ (filter->format >> 1)) & 1)
402 if ((y ^ (filter->format >> 1)) & 1)
410 /* Routine to generate the top and bottom edges (not including corners) */
412 hborder (uint8_t * input, uint8_t * output, int bot_top,
413 int typ, GstBayer2RGB * filter)
415 uint8_t *op; /* output pointer */
416 uint8_t *ip; /* input pointer */
417 uint8_t *nx; /* next line pointer */
418 int ix; /* loop index */
420 op = output + (bot_top * filter->width * (filter->height - 1) + 1) *
422 ip = input + bot_top * filter->stride * (filter->height - 1);
423 /* calculate minus or plus one line, depending upon bot_top flag */
424 nx = ip + (1 - 2 * bot_top) * filter->stride;
425 /* Stepping horizontally */
426 for (ix = 1; ix < filter->width - 1; ix++, op += filter->pixsize) {
429 op[filter->r_off] = ip[ix];
430 op[filter->g_off] = (ip[ix + 1] + ip[ix - 1] + nx[ix] + 1) / 3;
431 op[filter->b_off] = (nx[ix + 1] + nx[ix - 1] + 1) / 2;
435 op[filter->r_off] = (ip[ix + 1] + ip[ix - 1] + 1) / 2;
436 op[filter->g_off] = ip[ix];
437 op[filter->b_off] = nx[ix];
441 op[filter->r_off] = nx[ix];
442 op[filter->g_off] = ip[ix];
443 op[filter->b_off] = (ip[ix + 1] + ip[ix - 1] + 1) / 2;
447 op[filter->r_off] = (nx[ix + 1] + nx[ix - 1] + 1) / 2;
448 op[filter->g_off] = (ip[ix + 1] + ip[ix - 1] + nx[ix] + 1) / 3;
449 op[filter->b_off] = ip[ix];
456 /* Routine to generate the left and right edges, not including corners */
458 vborder (uint8_t * input, uint8_t * output, int right_left,
459 int typ, GstBayer2RGB * filter)
461 uint8_t *op; /* output pointer */
462 uint8_t *ip; /* input pointer */
463 uint8_t *la; /* line above pointer */
464 uint8_t *lb; /* line below pointer */
465 int ix; /* loop index */
466 int lr; /* 'left-right' flag - +1 is right, -1 is left */
468 lr = (1 - 2 * right_left);
469 /* stepping vertically */
470 for (ix = 1; ix < filter->height - 1; ix++) {
471 ip = input + right_left * (filter->width - 1) + ix * filter->stride;
472 op = output + (right_left * (filter->width - 1) + ix * filter->width) *
474 la = ip + filter->stride;
475 lb = ip - filter->stride;
478 op[filter->r_off] = ip[0];
479 op[filter->g_off] = (la[0] + ip[lr] + lb[0] + 1) / 3;
480 op[filter->b_off] = (la[lr] + lb[lr] + 1) / 2;
484 op[filter->r_off] = ip[lr];
485 op[filter->g_off] = ip[0];
486 op[filter->b_off] = (la[lr] + lb[lr] + 1) / 2;
490 op[filter->r_off] = (la[lr] + lb[lr] + 1) / 2;
491 op[filter->g_off] = ip[0];
492 op[filter->b_off] = ip[lr];
496 op[filter->r_off] = (la[lr] + lb[lr] + 1) / 2;
497 op[filter->g_off] = (la[0] + ip[lr] + lb[0] + 1) / 3;
498 op[filter->b_off] = ip[0];
505 /* Produce the four (top, bottom, left, right) edges */
507 do_row0_col0 (uint8_t * input, uint8_t * output, GstBayer2RGB * filter)
509 /* Horizontal edges */
510 hborder (input, output, 0, get_pixel_type (filter, 1, 0), filter);
511 hborder (input, output, 1, get_pixel_type (filter, 1, filter->height - 1),
515 vborder (input, output, 0, get_pixel_type (filter, 0, 1), filter);
516 vborder (input, output, 1, get_pixel_type (filter, filter->width - 1, 1),
521 corner (uint8_t * input, uint8_t * output, int x, int y,
522 int xd, int yd, int typ, GstBayer2RGB * filter)
524 uint8_t *ip; /* input pointer */
525 uint8_t *op; /* output pointer */
526 uint8_t *nx; /* adjacent line */
528 op = output + y * filter->width * filter->pixsize + x * filter->pixsize;
529 ip = input + y * filter->stride + x;
530 nx = ip + yd * filter->stride;
533 op[filter->r_off] = ip[0];
534 op[filter->g_off] = (nx[0] + ip[xd] + 1) / 2;
535 op[filter->b_off] = nx[xd];
538 op[filter->r_off] = ip[xd];
539 op[filter->g_off] = ip[0];
540 op[filter->b_off] = nx[0];
543 op[filter->r_off] = nx[0];
544 op[filter->g_off] = ip[0];
545 op[filter->b_off] = ip[xd];
548 op[filter->r_off] = nx[xd];
549 op[filter->g_off] = (nx[0] + ip[xd] + 1) / 2;
550 op[filter->b_off] = ip[0];
556 do_corners (uint8_t * input, uint8_t * output, GstBayer2RGB * filter)
559 corner (input, output, 0, 0, 1, 1, get_pixel_type (filter, 0, 0), filter);
561 corner (input, output, 0, filter->height - 1, 1, -1,
562 get_pixel_type (filter, 0, filter->height - 1), filter);
564 corner (input, output, filter->width - 1, 0, -1, 0,
565 get_pixel_type (filter, filter->width - 1, 0), filter);
567 corner (input, output, filter->width - 1, filter->height - 1, -1, -1,
568 get_pixel_type (filter, filter->width - 1, filter->height - 1), filter);
572 do_body (uint8_t * input, uint8_t * output, GstBayer2RGB * filter)
574 int ip, op; /* input and output pointers */
575 int w, h; /* loop indices */
576 int type; /* calculated colour of current element */
581 * We are processing row (line) by row, starting with the second
582 * row and continuing through the next to last. Each row is processed
583 * column by column, starting with the second and continuing through
584 * to the next to last.
586 for (h = 1; h < filter->height - 1; h++) {
588 * Remember we are processing "row by row". For each row, we need
589 * to set the type of the first element to be processed. Since we
590 * have already processed the edges, the "first element" will be
591 * the pixel at position (1,1). Assuming BG format, this should
592 * be RED for odd-numbered rows and GREENB for even rows.
594 type = get_pixel_type (filter, 1, h);
595 /* Calculate the starting position for the row */
596 op = h * filter->width * filter->pixsize; /* output (converted) pos */
597 ip = h * filter->stride; /* input (bayer data) pos */
598 for (w = 1; w < filter->width - 1; w++) {
599 op += filter->pixsize; /* we are processing "horizontally" */
603 output[op + filter->r_off] = input[ip];
604 output[op + filter->b_off] = (input[ip - filter->stride - 1] +
605 input[ip - filter->stride + 1] +
606 input[ip + filter->stride - 1] +
607 input[ip + filter->stride + 1] + 2) / 4;
608 v1 = input[ip + filter->stride];
609 v2 = input[ip - filter->stride];
615 output[op + filter->g_off] = (v1 + v2 + 1) / 2;
617 output[op + filter->g_off] = (h1 + h2 + 1) / 2;
619 output[op + filter->g_off] = (v1 + h1 + v2 + h2 + 2) / 4;
623 output[op + filter->r_off] = (input[ip + 1] + input[ip - 1] + 1) / 2;
624 output[op + filter->g_off] = input[ip];
625 output[op + filter->b_off] = (input[ip - filter->stride] +
626 input[ip + filter->stride] + 1) / 2;
630 output[op + filter->r_off] = (input[ip - filter->stride] +
631 input[ip + filter->stride] + 1) / 2;
632 output[op + filter->g_off] = input[ip];
633 output[op + filter->b_off] = (input[ip + 1] + input[ip - 1] + 1) / 2;
637 output[op + filter->r_off] = (input[ip - filter->stride - 1] +
638 input[ip - filter->stride + 1] +
639 input[ip + filter->stride - 1] +
640 input[ip + filter->stride + 1] + 2) / 4;
641 output[op + filter->b_off] = input[ip];
642 v1 = input[ip + filter->stride];
643 v2 = input[ip - filter->stride];
649 output[op + filter->g_off] = (v1 + v2 + 1) / 2;
651 output[op + filter->g_off] = (h1 + h2 + 1) / 2;
653 output[op + filter->g_off] = (v1 + h1 + v2 + h2 + 2) / 4;
662 gst_bayer2rgb_transform (GstBaseTransform * base, GstBuffer * inbuf,
665 GstBayer2RGB *filter = GST_BAYER2RGB (base);
666 uint8_t *input, *output;
669 * We need to lock our filter params to prevent changing
670 * caps in the middle of a transformation (nice way to get
673 GST_OBJECT_LOCK (filter);
675 GST_DEBUG ("transforming buffer");
676 input = (uint8_t *) GST_BUFFER_DATA (inbuf);
677 output = (uint8_t *) GST_BUFFER_DATA (outbuf);
678 do_corners (input, output, filter);
679 do_row0_col0 (input, output, filter);
680 do_body (input, output, filter);
682 GST_OBJECT_UNLOCK (filter);