3 * Copyright (C) 2007 David Schleef <ds@schleef.org>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 * Boston, MA 02111-1307, USA.
21 * Logic enhanced by William Brack <wbrack@mmm.com.hk>
25 * SECTION:element-bayer2rgb
27 * Decodes raw camera bayer (fourcc BA81) to RGB.
31 * In order to guard against my advancing maturity, some extra detailed
32 * information about the logic of the decode is included here. Much of
33 * this was inspired by a technical paper from siliconimaging.com, which
34 * in turn was based upon an article from IEEE,
35 * T. Sakamoto, C. Nakanishi and T. Hase,
36 * “Software pixel interpolation for digital still cameras suitable for
38 * IEEE Trans. Consumer Electronics, vol. 44, no. 4, November 1998.
40 * The code assumes a Bayer matrix of the type produced by the fourcc
41 * BA81 (v4l2 format SBGGR8) of width w and height h which looks like:
51 * We expand this matrix, producing a separate {r, g, b} triple for each
52 * of the individual elements. The algorithm for doing this expansion is
55 * We are designing for speed of transformation, at a slight expense of code.
56 * First, we calculate the appropriate triples for the four corners, the
57 * remainder of the top and bottom rows, and the left and right columns.
58 * The reason for this is that those elements are transformed slightly
59 * differently than all of the remainder of the matrix. Finally, we transform
60 * all of the remainder.
62 * The transformation into the "appropriate triples" is based upon the
63 * "nearest neighbor" principal, with some additional complexity for the
64 * calculation of the "green" element, where an "adaptive" pairing is used.
66 * For purposes of documentation and indentification, each element of the
67 * original array can be put into one of four classes:
70 * GR A green element which is followed by a red one
71 * GB A green element which is followed by a blue one
79 #include <gst/base/gstbasetransform.h>
80 #include <gst/video/video.h>
85 #define GST_CAT_DEFAULT gst_bayer2rgb_debug
86 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
90 GST_BAYER_2_RGB_FORMAT_BGGR = 0,
91 GST_BAYER_2_RGB_FORMAT_GBRG,
92 GST_BAYER_2_RGB_FORMAT_GRBG,
93 GST_BAYER_2_RGB_FORMAT_RGGB
97 #define GST_TYPE_BAYER2RGB (gst_bayer2rgb_get_type())
98 #define GST_BAYER2RGB(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_BAYER2RGB,GstBayer2RGB))
99 #define GST_IS_BAYER2RGB(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_BAYER2RGB))
100 #define GST_BAYER2RGB_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass) ,GST_TYPE_BAYER2RGB,GstBayer2RGBClass))
101 #define GST_IS_BAYER2RGB_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass) ,GST_TYPE_BAYER2RGB))
102 #define GST_BAYER2RGB_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj) ,GST_TYPE_BAYER2RGB,GstBayer2RGBClass))
103 typedef struct _GstBayer2RGB GstBayer2RGB;
104 typedef struct _GstBayer2RGBClass GstBayer2RGBClass;
106 typedef void (*GstBayer2RGBProcessFunc) (GstBayer2RGB *, guint8 *, guint);
110 GstBaseTransform basetransform;
116 int pixsize; /* bytes per pixel */
117 int r_off; /* offset for red */
118 int g_off; /* offset for green */
119 int b_off; /* offset for blue */
123 struct _GstBayer2RGBClass
125 GstBaseTransformClass parent;
128 //#define SRC_CAPS GST_VIDEO_CAPS_RGBx
130 GST_VIDEO_CAPS_RGBx ";" \
131 GST_VIDEO_CAPS_xRGB ";" \
132 GST_VIDEO_CAPS_BGRx ";" \
133 GST_VIDEO_CAPS_xBGR ";" \
134 GST_VIDEO_CAPS_RGBA ";" \
135 GST_VIDEO_CAPS_ARGB ";" \
136 GST_VIDEO_CAPS_BGRA ";" \
137 GST_VIDEO_CAPS_ABGR ";" \
138 GST_VIDEO_CAPS_RGB ";" \
141 #define SINK_CAPS "video/x-raw-bayer,format=(string){bggr,grbg,gbrg,rggb}," \
142 "width=(int)[1,MAX],height=(int)[1,MAX],framerate=(fraction)[0/1,MAX]"
149 #define DEBUG_INIT(bla) \
150 GST_DEBUG_CATEGORY_INIT (gst_bayer2rgb_debug, "bayer2rgb", 0, "bayer2rgb element");
152 GType gst_bayer2rgb_get_type (void);
153 GST_BOILERPLATE_FULL (GstBayer2RGB, gst_bayer2rgb, GstBaseTransform,
154 GST_TYPE_BASE_TRANSFORM, DEBUG_INIT);
156 static void gst_bayer2rgb_set_property (GObject * object, guint prop_id,
157 const GValue * value, GParamSpec * pspec);
158 static void gst_bayer2rgb_get_property (GObject * object, guint prop_id,
159 GValue * value, GParamSpec * pspec);
161 static gboolean gst_bayer2rgb_set_caps (GstBaseTransform * filter,
162 GstCaps * incaps, GstCaps * outcaps);
163 static GstFlowReturn gst_bayer2rgb_transform (GstBaseTransform * base,
164 GstBuffer * inbuf, GstBuffer * outbuf);
165 static void gst_bayer2rgb_reset (GstBayer2RGB * filter);
166 static GstCaps *gst_bayer2rgb_transform_caps (GstBaseTransform * base,
167 GstPadDirection direction, GstCaps * caps);
168 static gboolean gst_bayer2rgb_get_unit_size (GstBaseTransform * base,
169 GstCaps * caps, guint * size);
173 gst_bayer2rgb_base_init (gpointer klass)
175 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
177 gst_element_class_set_details_simple (element_class,
178 "Bayer to RGB decoder for cameras", "Filter/Converter/Video",
179 "Converts video/x-raw-bayer to video/x-raw-rgb",
180 "William Brack <wbrack@mmm.com.hk>");
182 gst_element_class_add_pad_template (element_class,
183 gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
184 gst_caps_from_string (SRC_CAPS)));
185 gst_element_class_add_pad_template (element_class,
186 gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
187 gst_caps_from_string (SINK_CAPS)));
191 gst_bayer2rgb_class_init (GstBayer2RGBClass * klass)
193 GObjectClass *gobject_class;
195 gobject_class = (GObjectClass *) klass;
196 gobject_class->set_property = gst_bayer2rgb_set_property;
197 gobject_class->get_property = gst_bayer2rgb_get_property;
199 GST_BASE_TRANSFORM_CLASS (klass)->transform_caps =
200 GST_DEBUG_FUNCPTR (gst_bayer2rgb_transform_caps);
201 GST_BASE_TRANSFORM_CLASS (klass)->get_unit_size =
202 GST_DEBUG_FUNCPTR (gst_bayer2rgb_get_unit_size);
203 GST_BASE_TRANSFORM_CLASS (klass)->set_caps =
204 GST_DEBUG_FUNCPTR (gst_bayer2rgb_set_caps);
205 GST_BASE_TRANSFORM_CLASS (klass)->transform =
206 GST_DEBUG_FUNCPTR (gst_bayer2rgb_transform);
210 gst_bayer2rgb_init (GstBayer2RGB * filter, GstBayer2RGBClass * klass)
212 gst_bayer2rgb_reset (filter);
213 gst_base_transform_set_in_place (GST_BASE_TRANSFORM (filter), TRUE);
216 /* No properties are implemented, so only a warning is produced */
218 gst_bayer2rgb_set_property (GObject * object, guint prop_id,
219 const GValue * value, GParamSpec * pspec)
224 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
230 gst_bayer2rgb_get_property (GObject * object, guint prop_id,
231 GValue * value, GParamSpec * pspec)
236 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
241 /* Routine to convert colormask value into relative byte offset */
243 get_pix_offset (int mask, int bpp)
245 int bpp32 = (bpp / 8) - 3;
257 GST_ERROR ("Invalid color mask 0x%08x", mask);
263 gst_bayer2rgb_set_caps (GstBaseTransform * base, GstCaps * incaps,
266 GstBayer2RGB *bayer2rgb = GST_BAYER2RGB (base);
267 GstStructure *structure;
271 GST_DEBUG ("in caps %" GST_PTR_FORMAT " out caps %" GST_PTR_FORMAT, incaps,
274 structure = gst_caps_get_structure (incaps, 0);
276 gst_structure_get_int (structure, "width", &bayer2rgb->width);
277 gst_structure_get_int (structure, "height", &bayer2rgb->height);
278 bayer2rgb->stride = GST_ROUND_UP_4 (bayer2rgb->width);
280 format = gst_structure_get_string (structure, "format");
281 if (g_str_equal (format, "bggr")) {
282 bayer2rgb->format = GST_BAYER_2_RGB_FORMAT_BGGR;
283 } else if (g_str_equal (format, "gbrg")) {
284 bayer2rgb->format = GST_BAYER_2_RGB_FORMAT_GBRG;
285 } else if (g_str_equal (format, "grbg")) {
286 bayer2rgb->format = GST_BAYER_2_RGB_FORMAT_GRBG;
287 } else if (g_str_equal (format, "rggb")) {
288 bayer2rgb->format = GST_BAYER_2_RGB_FORMAT_RGGB;
293 /* To cater for different RGB formats, we need to set params for later */
294 structure = gst_caps_get_structure (outcaps, 0);
295 gst_structure_get_int (structure, "bpp", &bpp);
296 bayer2rgb->pixsize = bpp / 8;
297 gst_structure_get_int (structure, "red_mask", &val);
298 bayer2rgb->r_off = get_pix_offset (val, bpp);
299 gst_structure_get_int (structure, "green_mask", &val);
300 bayer2rgb->g_off = get_pix_offset (val, bpp);
301 gst_structure_get_int (structure, "blue_mask", &val);
302 bayer2rgb->b_off = get_pix_offset (val, bpp);
308 gst_bayer2rgb_reset (GstBayer2RGB * filter)
320 gst_bayer2rgb_transform_caps (GstBaseTransform * base,
321 GstPadDirection direction, GstCaps * caps)
323 GstStructure *structure;
325 GstStructure *newstruct;
327 GST_DEBUG_OBJECT (caps, "transforming caps (from)");
329 structure = gst_caps_get_structure (caps, 0);
331 if (direction == GST_PAD_SRC) {
332 newcaps = gst_caps_from_string ("video/x-raw-bayer,"
333 "format=(string){bggr,grbg,gbrg,rggb}");
335 newcaps = gst_caps_new_simple ("video/x-raw-rgb", NULL);
337 newstruct = gst_caps_get_structure (newcaps, 0);
339 gst_structure_set_value (newstruct, "width",
340 gst_structure_get_value (structure, "width"));
341 gst_structure_set_value (newstruct, "height",
342 gst_structure_get_value (structure, "height"));
343 gst_structure_set_value (newstruct, "framerate",
344 gst_structure_get_value (structure, "framerate"));
346 GST_DEBUG_OBJECT (newcaps, "transforming caps (into)");
352 gst_bayer2rgb_get_unit_size (GstBaseTransform * base, GstCaps * caps,
355 GstStructure *structure;
361 structure = gst_caps_get_structure (caps, 0);
363 if (gst_structure_get_int (structure, "width", &width) &&
364 gst_structure_get_int (structure, "height", &height)) {
365 name = gst_structure_get_name (structure);
366 /* Our name must be either video/x-raw-bayer video/x-raw-rgb */
367 if (strcmp (name, "video/x-raw-rgb")) {
368 /* For bayer, we handle only BA81 (BGGR), which is BPP=24 */
369 *size = GST_ROUND_UP_4 (width) * height;
372 /* For output, calculate according to format */
373 if (gst_structure_get_int (structure, "bpp", &pixsize)) {
374 *size = width * height * (pixsize / 8);
380 GST_ELEMENT_ERROR (base, CORE, NEGOTIATION, (NULL),
381 ("Incomplete caps, some required field missing"));
385 #define RECONSTRUCT_SQUARE(x) \
387 int _h1 = next[i-1]; \
388 int _h2 = prev[i+1]; \
389 int _v1 = next[i+1]; \
390 int _v2 = prev[i-1]; \
391 (x) = (_h1+_h2+_v1+_v2+2)>>2; \
394 #define RECONSTRUCT_DIAMOND(x) \
396 int _h1 = src[i-1]; \
397 int _h2 = src[i+1]; \
400 (x) = (_h1+_h2+_v1+_v2+2)>>2; \
403 #define RECONSTRUCT_HORIZ(x) \
405 (x) = (src[i-1] + src[i+1] + 1) >> 1; \
408 #define RECONSTRUCT_VERT(x) \
410 (x) = (next[i] + prev[i] + 1) >> 1; \
415 reconstruct_blue_green (GstBayer2RGB * bayer2rgb, uint8_t * dest,
416 uint8_t * src, int src_stride, int blue_loc)
422 int width = bayer2rgb->width;
424 prev = src - src_stride;
425 next = src + src_stride;
428 if ((i & 1) == blue_loc) {
430 r = (next[i + 1] + prev[i + 1] + 1) >> 1;
431 g = (next[i] + prev[i] + 1) >> 1;
434 r = (next[i] + prev[i] + 1) >> 1;
437 dest[i * 4 + bayer2rgb->r_off] = r;
438 dest[i * 4 + bayer2rgb->g_off] = g;
439 dest[i * 4 + bayer2rgb->b_off] = b;
440 for (i = 1; i < width - 1; i++) {
441 if ((i & 1) == blue_loc) {
443 RECONSTRUCT_SQUARE (r);
444 RECONSTRUCT_DIAMOND (g);
446 RECONSTRUCT_HORIZ (b);
447 RECONSTRUCT_VERT (r);
450 dest[i * 4 + bayer2rgb->r_off] = r;
451 dest[i * 4 + bayer2rgb->g_off] = g;
452 dest[i * 4 + bayer2rgb->b_off] = b;
454 if ((i & 1) == blue_loc) {
456 r = (next[i - 1] + prev[i - 1] + 1) >> 1;
457 g = (next[i] + prev[i] + 1) >> 1;
460 r = (next[i] + prev[i] + 1) >> 1;
463 dest[i * 4 + bayer2rgb->r_off] = r;
464 dest[i * 4 + bayer2rgb->g_off] = g;
465 dest[i * 4 + bayer2rgb->b_off] = b;
469 reconstruct_green_red (GstBayer2RGB * bayer2rgb, uint8_t * dest,
470 uint8_t * src, int src_stride, int red_loc)
476 int width = bayer2rgb->width;
478 prev = src - src_stride;
479 next = src + src_stride;
482 if ((i & 1) == red_loc) {
484 b = (next[i + 1] + prev[i + 1] + 1) >> 1;
485 g = (next[i] + prev[i] + 1) >> 1;
488 b = (next[i] + prev[i] + 1) >> 1;
491 dest[i * 4 + bayer2rgb->r_off] = r;
492 dest[i * 4 + bayer2rgb->g_off] = g;
493 dest[i * 4 + bayer2rgb->b_off] = b;
494 for (i = 1; i < width - 1; i++) {
495 if ((i & 1) == red_loc) {
497 RECONSTRUCT_SQUARE (b);
498 RECONSTRUCT_DIAMOND (g);
500 RECONSTRUCT_HORIZ (r);
501 RECONSTRUCT_VERT (b);
504 dest[i * 4 + bayer2rgb->r_off] = r;
505 dest[i * 4 + bayer2rgb->g_off] = g;
506 dest[i * 4 + bayer2rgb->b_off] = b;
508 if ((i & 1) == red_loc) {
510 b = (next[i - 1] + prev[i - 1] + 1) >> 1;
511 g = (next[i] + prev[i] + 1) >> 1;
514 b = (next[i] + prev[i] + 1) >> 1;
517 dest[i * 4 + bayer2rgb->r_off] = r;
518 dest[i * 4 + bayer2rgb->g_off] = g;
519 dest[i * 4 + bayer2rgb->b_off] = b;
523 reconstruct_blue_green_edge (GstBayer2RGB * bayer2rgb, uint8_t * dest,
524 uint8_t * src, int src_stride, int blue_loc, int offset)
529 int width = bayer2rgb->width;
531 next = src + offset * src_stride;
534 if ((i & 1) == blue_loc) {
543 dest[i * 4 + bayer2rgb->r_off] = r;
544 dest[i * 4 + bayer2rgb->g_off] = g;
545 dest[i * 4 + bayer2rgb->b_off] = b;
546 for (i = 1; i < width - 1; i++) {
547 if ((i & 1) == blue_loc) {
549 r = (next[i - 1] + next[i + 1] + 1) >> 1;
550 g = (src[i - 1] + src[i + 1] + 1) >> 1;
552 b = (src[i - 1] + src[i + 1] + 1) >> 1;
556 dest[i * 4 + bayer2rgb->r_off] = r;
557 dest[i * 4 + bayer2rgb->g_off] = g;
558 dest[i * 4 + bayer2rgb->b_off] = b;
560 if ((i & 1) == blue_loc) {
569 dest[i * 4 + bayer2rgb->r_off] = r;
570 dest[i * 4 + bayer2rgb->g_off] = g;
571 dest[i * 4 + bayer2rgb->b_off] = b;
575 reconstruct_green_red_edge (GstBayer2RGB * bayer2rgb, uint8_t * dest,
576 uint8_t * src, int src_stride, int red_loc, int offset)
581 int width = bayer2rgb->width;
583 next = src + offset * src_stride;
586 if ((i & 1) == red_loc) {
595 dest[i * 4 + bayer2rgb->r_off] = r;
596 dest[i * 4 + bayer2rgb->g_off] = g;
597 dest[i * 4 + bayer2rgb->b_off] = b;
598 for (i = 1; i < width - 1; i++) {
599 if ((i & 1) == red_loc) {
601 b = (next[i - 1] + next[i + 1] + 1) >> 1;
602 g = (src[i - 1] + src[i + 1] + 1) >> 1;
604 r = (src[i - 1] + src[i + 1] + 1) >> 1;
608 dest[i * 4 + bayer2rgb->r_off] = r;
609 dest[i * 4 + bayer2rgb->g_off] = g;
610 dest[i * 4 + bayer2rgb->b_off] = b;
612 if ((i & 1) == red_loc) {
621 dest[i * 4 + bayer2rgb->r_off] = r;
622 dest[i * 4 + bayer2rgb->g_off] = g;
623 dest[i * 4 + bayer2rgb->b_off] = b;
627 gst_bayer2rgb_process_ref (GstBayer2RGB * bayer2rgb, uint8_t * dest,
628 int dest_stride, uint8_t * src, int src_stride)
631 int format = bayer2rgb->format;
634 if ((j & 1) == (format & 2) >> 1) {
635 reconstruct_blue_green_edge (bayer2rgb, dest + j * dest_stride,
636 src + j * src_stride, src_stride, format & 1, 1);
638 reconstruct_green_red_edge (bayer2rgb, dest + j * dest_stride,
639 src + j * src_stride, src_stride, (format & 1) ^ 1, 1);
641 for (j = 1; j < bayer2rgb->height - 1; j++) {
642 if ((j & 1) == (format & 2) >> 1) {
643 reconstruct_blue_green (bayer2rgb, dest + j * dest_stride,
644 src + j * src_stride, src_stride, format & 1);
646 reconstruct_green_red (bayer2rgb, dest + j * dest_stride,
647 src + j * src_stride, src_stride, (format & 1) ^ 1);
650 if ((j & 1) == (format & 2) >> 1) {
651 reconstruct_blue_green_edge (bayer2rgb, dest + j * dest_stride,
652 src + j * src_stride, src_stride, format & 1, -1);
654 reconstruct_green_red_edge (bayer2rgb, dest + j * dest_stride,
655 src + j * src_stride, src_stride, (format & 1) ^ 1, -1);
662 gst_bayer2rgb_transform (GstBaseTransform * base, GstBuffer * inbuf,
665 GstBayer2RGB *filter = GST_BAYER2RGB (base);
666 uint8_t *input, *output;
669 * We need to lock our filter params to prevent changing
670 * caps in the middle of a transformation (nice way to get
673 GST_OBJECT_LOCK (filter);
675 GST_DEBUG ("transforming buffer");
676 input = (uint8_t *) GST_BUFFER_DATA (inbuf);
677 output = (uint8_t *) GST_BUFFER_DATA (outbuf);
678 gst_bayer2rgb_process_ref (filter, output, filter->width * 4,
679 input, filter->width);
681 GST_OBJECT_UNLOCK (filter);