From 80b032ef17c89f270cac4c64a066c307b6821044 Mon Sep 17 00:00:00 2001 From: "William M. Brack" Date: Fri, 14 Mar 2008 17:38:10 +0000 Subject: [PATCH] gst/bayer/gstbayer2rgb.c: Significant improvements. Fixes #521392. Original commit message from CVS: Patch by: William M. Brack * gst/bayer/gstbayer2rgb.c: Significant improvements. Fixes #521392. --- ChangeLog | 6 + gst/bayer/gstbayer2rgb.c | 519 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 404 insertions(+), 121 deletions(-) diff --git a/ChangeLog b/ChangeLog index ce5eb42..9699975 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +2008-03-14 David Schleef + + Patch by: William M. Brack + + * gst/bayer/gstbayer2rgb.c: Significant improvements. Fixes #521392. + 2008-03-14 Wim Taymans * gst/selector/gstinputselector.c: (gst_selector_pad_event), diff --git a/gst/bayer/gstbayer2rgb.c b/gst/bayer/gstbayer2rgb.c index f1f3f7f..f8d569e 100644 --- a/gst/bayer/gstbayer2rgb.c +++ b/gst/bayer/gstbayer2rgb.c @@ -16,12 +16,59 @@ * License along with this library; if not, write to the * Free Software Foundation, Inc., 59 Temple Place - Suite 330, * Boston, MA 02111-1307, USA. + * + * March 2008 + * Logic enhanced by William Brack */ /* * SECTION:element-bayer2rgb * - * Decodes raw camera sensor images. + * Decodes raw camera bayer (fourcc BA81) to RGB. + */ + +/* + * In order to guard against my advancing maturity, some extra detailed + * information about the logic of the decode is included here. Much of + * this was inspired by a technical paper from siliconimaging.com, which + * in turn was based upon an article from IEEE, + * T. Sakamoto, C. Nakanishi and T. Hase, + * “Software pixel interpolation for digital still cameras suitable for + * a 32-bit MCU,” + * IEEE Trans. Consumer Electronics, vol. 44, no. 4, November 1998. + * + * The code assumes a Bayer matrix of the type produced by the fourcc + * BA81 (v4l2 format SBGGR8) of width w and height h which looks like: + * 0 1 2 3 w-2 w-1 + * + * 0 B G B G ....B G + * 1 G R G R ....G R + * 2 B G B G ....B G + * ............... + * h-2 B G B G ....B G + * h-1 G R G R ....G R + * + * We expand this matrix, producing a separate {r, g, b} triple for each + * of the individual elements. The algorithm for doing this expansion is + * as follows. + * + * We are designing for speed of transformation, at a slight expense of code. + * First, we calculate the appropriate triples for the four corners, the + * remainder of the top and bottom rows, and the left and right columns. + * The reason for this is that those elements are transformed slightly + * differently than all of the remainder of the matrix. Finally, we transform + * all of the remainder. + * + * The transformation into the "appropriate triples" is based upon the + * "nearest neighbor" principal, with some additional complexity for the + * calculation of the "green" element, where an "adaptive" pairing is used. + * + * For purposes of documentation and indentification, each element of the + * original array can be put into one of four classes: + * R A red element + * B A blue element + * GR A green element which is followed by a red one + * GB A green element which is followed by a blue one */ #ifdef HAVE_CONFIG_H @@ -56,8 +103,10 @@ struct _GstBayer2RGB int width; int height; int stride; - - uint8_t *tmpdata; + int pixsize; /* bytes per pixel */ + int r_off; /* offset for red */ + int g_off; /* offset for green */ + int b_off; /* offset for blue */ }; struct _GstBayer2RGBClass @@ -66,12 +115,24 @@ struct _GstBayer2RGBClass }; static const GstElementDetails element_details = -GST_ELEMENT_DETAILS ("RAW Camera sensor decoder", - "Filter/Effect", - "FIXME example filter", - "FIXME "); +GST_ELEMENT_DETAILS ("Bayer to RGB decoder for cameras", + "Filter/Converter/Video", + "Converts video/x-raw-bayer to video/x-raw-rgb", + "William Brack "); + +//#define SRC_CAPS GST_VIDEO_CAPS_RGBx +#define SRC_CAPS \ + GST_VIDEO_CAPS_RGBx ";" \ + GST_VIDEO_CAPS_xRGB ";" \ + GST_VIDEO_CAPS_BGRx ";" \ + GST_VIDEO_CAPS_xBGR ";" \ + GST_VIDEO_CAPS_RGBA ";" \ + GST_VIDEO_CAPS_ARGB ";" \ + GST_VIDEO_CAPS_BGRA ";" \ + GST_VIDEO_CAPS_ABGR ";" \ + GST_VIDEO_CAPS_RGB ";" \ + GST_VIDEO_CAPS_BGR -#define SRC_CAPS GST_VIDEO_CAPS_ARGB #define SINK_CAPS "video/x-raw-bayer,width=(int)[1,MAX],height=(int)[1,MAX]" enum @@ -142,11 +203,11 @@ gst_bayer2rgb_init (GstBayer2RGB * filter, GstBayer2RGBClass * klass) gst_base_transform_set_in_place (GST_BASE_TRANSFORM (filter), TRUE); } +/* No properties are implemented, so only a warning is produced */ static void gst_bayer2rgb_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { - //GstBayer2RGB *filter = GST_BAYER2RGB (object); switch (prop_id) { default: @@ -159,7 +220,6 @@ static void gst_bayer2rgb_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { - //GstBayer2RGB *filter = GST_BAYER2RGB (object); switch (prop_id) { default: @@ -168,14 +228,34 @@ gst_bayer2rgb_get_property (GObject * object, guint prop_id, } } +/* Routine to convert colormask value into relative byte offset */ +static int +get_pix_offset (int offset) +{ + switch (offset) { + case 255: + return 3; + case 65280: + return 2; + case 16711680: + return 1; + case -16777216: + return 0; + default: + GST_ERROR ("Invalid color mask 0x%08x", offset); + return -1; + } +} + static gboolean gst_bayer2rgb_set_caps (GstBaseTransform * base, GstCaps * incaps, GstCaps * outcaps) { GstBayer2RGB *filter = GST_BAYER2RGB (base); GstStructure *structure; + int val; - GST_ERROR ("in caps %" GST_PTR_FORMAT " out caps %" GST_PTR_FORMAT, incaps, + GST_DEBUG ("in caps %" GST_PTR_FORMAT " out caps %" GST_PTR_FORMAT, incaps, outcaps); structure = gst_caps_get_structure (incaps, 0); @@ -184,10 +264,16 @@ gst_bayer2rgb_set_caps (GstBaseTransform * base, GstCaps * incaps, gst_structure_get_int (structure, "height", &filter->height); filter->stride = GST_ROUND_UP_4 (filter->width); - if (filter->tmpdata) { - g_free (filter->tmpdata); - } - filter->tmpdata = g_malloc (filter->stride * (4 * 3 + 1)); + /* To cater for different RGB formats, we need to set params for later */ + structure = gst_caps_get_structure (outcaps, 0); + gst_structure_get_int (structure, "bpp", &val); + filter->pixsize = val / 8; + gst_structure_get_int (structure, "red_mask", &val); + filter->r_off = get_pix_offset (val); + gst_structure_get_int (structure, "green_mask", &val); + filter->g_off = get_pix_offset (val); + gst_structure_get_int (structure, "blue_mask", &val); + filter->b_off = get_pix_offset (val); return TRUE; } @@ -198,10 +284,10 @@ gst_bayer2rgb_reset (GstBayer2RGB * filter) filter->width = 0; filter->height = 0; filter->stride = 0; - if (filter->tmpdata) { - g_free (filter->tmpdata); - filter->tmpdata = NULL; - } + filter->pixsize = 0; + filter->r_off = 0; + filter->g_off = 0; + filter->b_off = 0; } static GstCaps * @@ -212,7 +298,7 @@ gst_bayer2rgb_transform_caps (GstBaseTransform * base, GstCaps *newcaps; GstStructure *newstruct; - GST_ERROR ("transforming caps %" GST_PTR_FORMAT, caps); + GST_DEBUG_OBJECT (caps, "transforming caps (from)"); structure = gst_caps_get_structure (caps, 0); @@ -229,10 +315,8 @@ gst_bayer2rgb_transform_caps (GstBaseTransform * base, gst_structure_get_value (structure, "height")); gst_structure_set_value (newstruct, "framerate", gst_structure_get_value (structure, "framerate")); - gst_structure_set_value (newstruct, "pixel-aspect-ratio", - gst_structure_get_value (structure, "pixel-aspect-ratio")); - GST_ERROR ("into %" GST_PTR_FORMAT, newcaps); + GST_DEBUG_OBJECT (newcaps, "transforming caps (into)"); return newcaps; } @@ -244,72 +328,306 @@ gst_bayer2rgb_get_unit_size (GstBaseTransform * base, GstCaps * caps, GstStructure *structure; int width; int height; + int pixsize; const char *name; structure = gst_caps_get_structure (caps, 0); - gst_structure_get_int (structure, "width", &width); - gst_structure_get_int (structure, "height", &height); - name = gst_structure_get_name (structure); - if (strcmp (name, "video/x-raw-rgb")) { - *size = GST_ROUND_UP_4 (width) * height; - } else { - *size = 4 * width * height; - } + if (gst_structure_get_int (structure, "width", &width) && + gst_structure_get_int (structure, "height", &height)) { + name = gst_structure_get_name (structure); + /* Our name must be either video/x-raw-bayer video/x-raw-rgb */ + if (strcmp (name, "video/x-raw-rgb")) { + /* For bayer, we handle only BA81 (BGGR), which is BPP=24 */ + *size = GST_ROUND_UP_4 (width) * height; + return TRUE; + } else { + /* For output, calculate according to format */ + if (gst_structure_get_int (structure, "bpp", &pixsize)) { + *size = width * height * (pixsize / 8); + return TRUE; + } + } - return TRUE; + } + GST_ELEMENT_ERROR (base, CORE, NEGOTIATION, (NULL), + ("Incomplete caps, some required field missing")); + return FALSE; } -#define ARGB(a,r,g,b) ((b)<<24 | (g)<<16 | (r)<<8 | a) +/* + * We define values for the colors, just to make the code more readable. + */ +#define RED 0 /* Pure red element */ +#define GREENB 1 /* Green element which is on a blue line */ +#define BLUE 2 /* Pure blue element */ +#define GREENR 3 /* Green element which is on a red line */ +/* Routine to generate the top and bottom edges (not including corners) */ static void -upsample_even (uint8_t * dest, uint8_t * src, int width) +hborder (uint8_t * input, uint8_t * output, int bot_top, + int typ, GstBayer2RGB * filter) { - int i; - - for (i = 0; i < width - 2; i += 2) { - dest[i] = src[i]; - dest[i + 1] = (src[i] + src[i + 2] + 1) / 2; - } - dest[i] = src[i]; - if (i + 1 < width) { - dest[i + 1] = src[i]; + uint8_t *op; /* output pointer */ + uint8_t *ip; /* input pointer */ + uint8_t *nx; /* next line pointer */ + int ix; /* loop index */ + + op = output + (bot_top * filter->width * (filter->height - 1) + 1) * + filter->pixsize; + ip = input + bot_top * filter->stride * (filter->height - 1); + /* calculate minus or plus one line, depending upon bot_top flag */ + nx = ip + (1 - 2 * bot_top) * filter->stride; + /* Stepping horizontally */ + for (ix = 1; ix < filter->width - 1; ix++, op += filter->pixsize) { + switch (typ) { + case RED: + op[filter->r_off] = ip[ix]; + op[filter->g_off] = (ip[ix + 1] + ip[ix - 1] + nx[ix] + 1) / 3; + op[filter->b_off] = (nx[ix + 1] + nx[ix - 1] + 1) / 2; + typ = GREENR; + break; + case GREENR: + op[filter->r_off] = (ip[ix + 1] + ip[ix - 1] + 1) / 2; + op[filter->g_off] = ip[ix]; + op[filter->b_off] = nx[ix]; + typ = RED; + break; + case GREENB: + op[filter->r_off] = nx[ix]; + op[filter->g_off] = ip[ix]; + op[filter->b_off] = (ip[ix + 1] + ip[ix - 1] + 1) / 2; + typ = BLUE; + break; + case BLUE: + op[filter->r_off] = (nx[ix + 1] + nx[ix - 1] + 1) / 2; + op[filter->g_off] = (ip[ix + 1] + ip[ix - 1] + nx[ix] + 1) / 3; + op[filter->b_off] = ip[ix]; + typ = GREENB; + break; + } } } +/* Routine to generate the left and right edges, not including corners */ static void -upsample_odd (uint8_t * dest, uint8_t * src, int width) +vborder (uint8_t * input, uint8_t * output, int right_left, + int typ, GstBayer2RGB * filter) { - int i; - - dest[0] = src[1]; - for (i = 1; i < width - 2; i += 2) { - dest[i] = src[i]; - dest[i + 1] = (src[i] + src[i + 2] + 1) / 2; - } - dest[i] = src[i]; - if (i + 1 < width) { - dest[i + 1] = src[i]; + uint8_t *op; /* output pointer */ + uint8_t *ip; /* input pointer */ + uint8_t *la; /* line above pointer */ + uint8_t *lb; /* line below pointer */ + int ix; /* loop index */ + int lr; /* 'left-right' flag - +1 is right, -1 is left */ + + lr = (1 - 2 * right_left); + /* stepping vertically */ + for (ix = 1; ix < filter->height - 1; ix++) { + ip = input + right_left * (filter->width - 1) + ix * filter->stride; + op = output + (right_left * (filter->width - 1) + ix * filter->width) * + filter->pixsize; + la = ip + filter->stride; + lb = ip - filter->stride; + switch (typ) { + case RED: + op[filter->r_off] = ip[0]; + op[filter->g_off] = (la[0] + ip[lr] + lb[0] + 1) / 3; + op[filter->b_off] = (la[lr] + lb[lr] + 1) / 2; + typ = GREENB; + break; + case GREENR: + op[filter->r_off] = ip[lr]; + op[filter->g_off] = ip[0]; + op[filter->b_off] = (la[lr] + lb[lr] + 1) / 2; + typ = BLUE; + break; + case GREENB: + op[filter->r_off] = (la[lr] + lb[lr] + 1) / 2; + op[filter->g_off] = ip[0]; + op[filter->b_off] = ip[lr]; + typ = RED; + break; + case BLUE: + op[filter->r_off] = (la[lr] + lb[lr] + 1) / 2; + op[filter->g_off] = (la[0] + ip[lr] + lb[0] + 1) / 3; + op[filter->b_off] = ip[0]; + typ = GREENR; + break; + } } } +/* Produce the four (top, bottom, left, right) edges */ static void -interpolate (uint8_t * dest, uint8_t * src1, uint8_t * src2, int width) +do_row0_col0 (uint8_t * input, uint8_t * output, GstBayer2RGB * filter) { - int i; + int type; + + /* Horizontal edges */ + hborder (input, output, 0, GREENB, filter); + if (filter->height & 1) + type = GREENB; /* odd # rows, "bottom" edge same as top */ + else + type = RED; /* even #, bottom side different */ + hborder (input, output, 1, type, filter); + + /* Vertical edges */ + vborder (input, output, 0, GREENR, filter); + if (filter->width & 1) + type = GREENR; /* odd # cols, "right" edge same as left */ + else + type = RED; /* even #, right side different */ + vborder (input, output, 1, type, filter); +} - for (i = 0; i < width; i++) { - dest[i] = (src1[i] + src2[i] + 1) / 2; +static void +corner (uint8_t * input, uint8_t * output, int x, int y, + int xd, int yd, int typ, GstBayer2RGB * filter) +{ + uint8_t *ip; /* input pointer */ + uint8_t *op; /* output pointer */ + uint8_t *nx; /* adjacent line */ + + op = output + y * filter->width * filter->pixsize + x * filter->pixsize; + ip = input + y * filter->stride + x; + nx = ip + yd * filter->stride; + switch (typ) { + case RED: + op[filter->r_off] = ip[0]; + op[filter->g_off] = (nx[0] + ip[xd] + 1) / 2; + op[filter->b_off] = nx[xd]; + break; + case GREENR: + op[filter->r_off] = ip[xd]; + op[filter->g_off] = ip[0]; + op[filter->b_off] = nx[0]; + break; + case GREENB: + op[filter->r_off] = nx[0]; + op[filter->g_off] = ip[0]; + op[filter->b_off] = ip[xd]; + break; + case BLUE: + op[filter->r_off] = nx[xd]; + op[filter->g_off] = (nx[0] + ip[xd] + 1) / 2; + op[filter->b_off] = ip[0]; + break; } } - static void -merge (uint32_t * dest, uint8_t * r, uint8_t * g, uint8_t * b, int width) +do_corners (uint8_t * input, uint8_t * output, GstBayer2RGB * filter) { - int i; + int typ; + + /* Top left */ + corner (input, output, 0, 0, 1, 1, BLUE, filter); + /* Bottom left */ + corner (input, output, 0, filter->height - 1, 1, -1, + (filter->height & 1) ? BLUE : GREENR, filter); + /* Top right */ + corner (input, output, filter->width - 1, 0, -1, 0, + (filter->width & 1) ? BLUE : GREENB, filter); + /* Bottom right */ + if (filter->width & 1) /* if odd # cols, B or GB */ + typ = BLUE; + else + typ = GREENB; /* if even # cols, B or GR */ + typ |= (filter->height & 1); /* if odd # rows, GB or GR */ + corner (input, output, filter->width - 1, filter->height - 1, -1, -1, + typ, filter); +} - for (i = 0; i < width; i++) { - dest[i] = ARGB (0xff, r[i], g[i], b[i]); +static void +do_body (uint8_t * input, uint8_t * output, GstBayer2RGB * filter) +{ + int ip, op; /* input and output pointers */ + int w, h; /* loop indices */ + int type; /* calculated colour of current element */ + int a1, a2; + int v1, v2, h1, h2; + + /* + * We are processing row (line) by row, starting with the second + * row and continuing through the next to last. Each row is processed + * column by column, starting with the second and continuing through + * to the next to last. + */ + for (h = 1; h < filter->height - 1; h++) { + /* + * Remember we are processing "row by row". For each row, we need + * to set the type of the first element to be processed. Since we + * have already processed the edges, the "first element" will be + * the pixel at position (1,1). Assuming BG format, this should + * be RED for odd-numbered rows and GREENB for even rows. + */ + if (h & 1) + type = RED; + else + type = GREENB; + /* Calculate the starting position for the row */ + op = h * filter->width * filter->pixsize; /* output (converted) pos */ + ip = h * filter->stride; /* input (bayer data) pos */ + for (w = 1; w < filter->width - 1; w++) { + op += filter->pixsize; /* we are processing "horizontally" */ + ip++; + switch (type) { + case RED: + output[op + filter->r_off] = input[ip]; + output[op + filter->b_off] = (input[ip - filter->stride - 1] + + input[ip - filter->stride + 1] + + input[ip + filter->stride - 1] + + input[ip + filter->stride + 1] + 2) / 4; + v1 = input[ip + filter->stride]; + v2 = input[ip - filter->stride]; + h1 = input[ip + 1]; + h2 = input[ip - 1]; + a1 = abs (v1 - v2); + a2 = abs (h1 - h2); + if (a1 < a2) + output[op + filter->g_off] = (v1 + v2 + 1) / 2; + else if (a1 > a2) + output[op + filter->g_off] = (h1 + h2 + 1) / 2; + else + output[op + filter->g_off] = (v1 + h1 + v2 + h2 + 2) / 4; + type = GREENR; + break; + case GREENR: + output[op + filter->r_off] = (input[ip + 1] + input[ip - 1] + 1) / 2; + output[op + filter->g_off] = input[ip]; + output[op + filter->b_off] = (input[ip - filter->stride] + + input[ip + filter->stride] + 1) / 2; + type = RED; + break; + case GREENB: + output[op + filter->r_off] = (input[ip - filter->stride] + + input[ip + filter->stride] + 1) / 2; + output[op + filter->g_off] = input[ip]; + output[op + filter->b_off] = (input[ip + 1] + input[ip - 1] + 1) / 2; + type = BLUE; + break; + case BLUE: + output[op + filter->r_off] = (input[ip - filter->stride - 1] + + input[ip - filter->stride + 1] + + input[ip + filter->stride - 1] + + input[ip + filter->stride + 1] + 2) / 4; + output[op + filter->b_off] = input[ip]; + v1 = input[ip + filter->stride]; + v2 = input[ip - filter->stride]; + h1 = input[ip + 1]; + h2 = input[ip - 1]; + a1 = abs (v1 - v2); + a2 = abs (h1 - h2); + if (a1 < a2) + output[op + filter->g_off] = (v1 + v2 + 1) / 2; + else if (a1 > a2) + output[op + filter->g_off] = (h1 + h2 + 1) / 2; + else + output[op + filter->g_off] = (v1 + h1 + v2 + h2 + 2) / 4; + type = GREENB; + break; + } + } } } @@ -318,63 +636,22 @@ gst_bayer2rgb_transform (GstBaseTransform * base, GstBuffer * inbuf, GstBuffer * outbuf) { GstBayer2RGB *filter = GST_BAYER2RGB (base); - uint8_t *tmpdata; - int j; - - GST_DEBUG ("got here"); - tmpdata = filter->tmpdata; - - /* This is a pretty lousy algorithm. In particular, most higher - * quality algorithms will apply some non-linear weighting factors - * in red/blue interpolation based on the green components. This - * just does a linear interpolation between surrounding pixels. - * For green, we only interpolate horizontally. */ - - for (j = 0; j < filter->height + 1; j++) { - if (j < filter->height) { - /* upsample horizontally */ - if ((j & 1) == 0) { - upsample_even (tmpdata + (1 * 4 + (j & 3)) * filter->stride, - (uint8_t *) GST_BUFFER_DATA (inbuf) + filter->stride * j, - filter->width); - upsample_odd (tmpdata + (0 * 4 + (j & 3)) * filter->stride, - (uint8_t *) GST_BUFFER_DATA (inbuf) + filter->stride * j, - filter->width); - } else { - upsample_even (tmpdata + (2 * 4 + (j & 3)) * filter->stride, - (uint8_t *) GST_BUFFER_DATA (inbuf) + filter->stride * j, - filter->width); - upsample_odd (tmpdata + (1 * 4 + (j & 3)) * filter->stride, - (uint8_t *) GST_BUFFER_DATA (inbuf) + filter->stride * j, - filter->width); - } - } - if (j - 1 >= 0 && j - 1 < filter->height) { - int comp, j1, j2; - - if (((j - 1) & 1) == 0) { - comp = 2; - } else { - comp = 0; - } - j1 = j - 2; - if (j1 < 0) - j1 += 2; - j2 = j; - if (j2 > filter->height - 1) - j2 -= 2; - interpolate (tmpdata + (comp * 4 + ((j - 1) & 3)) * filter->stride, - tmpdata + (comp * 4 + (j1 & 3)) * filter->stride, - tmpdata + (comp * 4 + (j2 & 3)) * filter->stride, filter->width); - - merge ( - (uint32_t *) ((uint8_t *) GST_BUFFER_DATA (outbuf) + - 4 * filter->width * (j - 1)), - tmpdata + (0 * 4 + ((j - 1) & 3)) * filter->stride, - tmpdata + (1 * 4 + ((j - 1) & 3)) * filter->stride, - tmpdata + (2 * 4 + ((j - 1) & 3)) * filter->stride, filter->width); - } - } - + uint8_t *input, *output; + + /* + * We need to lock our filter params to prevent changing + * caps in the middle of a transformation (nice way to get + * segfaults) + */ + GST_OBJECT_LOCK (filter); + + GST_DEBUG ("transforming buffer"); + input = (uint8_t *) GST_BUFFER_DATA (inbuf); + output = (uint8_t *) GST_BUFFER_DATA (outbuf); + do_corners (input, output, filter); + do_row0_col0 (input, output, filter); + do_body (input, output, filter); + + GST_OBJECT_UNLOCK (filter); return GST_FLOW_OK; } -- 2.7.4