3 * Copyright (C) 2013 Miguel Casas-Sanchez <miguelecasassanchez@gmail.com>
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
23 * Alternatively, the contents of this file may be used under the
24 * GNU Lesser General Public License Version 2.1 (the "LGPL"), in
25 * which case the following provisions apply instead of the ones
28 * This library is free software; you can redistribute it and/or
29 * modify it under the terms of the GNU Library General Public
30 * License as published by the Free Software Foundation; either
31 * version 2 of the License, or (at your option) any later version.
33 * This library is distributed in the hope that it will be useful,
34 * but WITHOUT ANY WARRANTY; without even the implied warranty of
35 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
36 * Library General Public License for more details.
38 * You should have received a copy of the GNU Library General Public
39 * License along with this library; if not, write to the
40 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
41 * Boston, MA 02110-1301, USA.
45 * SECTION:element-grabcut
48 * This element is a wrapper around OpenCV grabcut implementation. GrabCut is an
49 * image segmentation method based on graph cuts technique. It can be seen as a
50 * way of fine-grain segmenting the image from some FG and BG "seed" areas. The
51 * OpenCV implementation follows the article [1].
52 * The "seed" areas are taken in this element from either an input bounding box
53 * coming from a face detection, or from alpha channel values. The input box is
54 * taken from a "face" event such as the one generated from the 'facedetect'
55 * element. The Alpha channel values should be one of the following (cv.hpp):
57 * GC_BGD = 0, //!< background
58 * GC_FGD = 1, //!< foreground
59 * GC_PR_BGD = 2, //!< most probably background
60 * GC_PR_FGD = 3 //!< most probably foreground
62 * with values over GC_PR_FGD interpreted as GC_PR_FGD. IN CASE OF no alpha mask
63 * input (all 0's or all 1's), the 'GstOpenCvFaceDetect-face' downstream event
64 * is used to create a bbox of PR_FG elements. If both foreground alpha
65 * is not specified and there is no face detection, nothing is done.
67 * [1] C. Rother, V. Kolmogorov, and A. Blake, "GrabCut: Interactive foreground
68 * extraction using iterated graph cuts, ACM Trans. Graph., vol. 23, pp. 309–314,
71 * ## Example launch line
74 * gst-launch-1.0 --gst-debug=grabcut=4 v4l2src device=/dev/video0 ! videoconvert ! grabcut ! videoconvert ! video/x-raw,width=320,height=240 ! ximagesink
76 * Another example launch line
78 * gst-launch-1.0 --gst-debug=grabcut=4 v4l2src device=/dev/video0 ! videoconvert ! facedetect display=0 ! videoconvert ! grabcut test-mode=true ! videoconvert ! video/x-raw,width=320,height=240 ! ximagesink
86 #include "gstgrabcut.h"
87 #include <opencv2/imgproc.hpp>
89 GST_DEBUG_CATEGORY_STATIC (gst_grabcut_debug);
90 #define GST_CAT_DEFAULT gst_grabcut_debug
93 /* Filter signals and args */
107 #define DEFAULT_TEST_MODE FALSE
108 #define DEFAULT_SCALE 1.6
110 G_DEFINE_TYPE_WITH_CODE (GstGrabcut, gst_grabcut, GST_TYPE_OPENCV_VIDEO_FILTER,
111 GST_DEBUG_CATEGORY_INIT (gst_grabcut_debug, "grabcut", 0,
112 "Grabcut image segmentation on either input alpha or input bounding box"););
113 GST_ELEMENT_REGISTER_DEFINE (grabcut, "grabcut", GST_RANK_NONE,
116 static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
119 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("RGBA")));
121 static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
124 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("RGBA")));
127 static void gst_grabcut_set_property (GObject * object, guint prop_id,
128 const GValue * value, GParamSpec * pspec);
129 static void gst_grabcut_get_property (GObject * object, guint prop_id,
130 GValue * value, GParamSpec * pspec);
132 static GstFlowReturn gst_grabcut_transform_ip (GstOpencvVideoFilter * filter,
133 GstBuffer * buf, Mat img);
134 static gboolean gst_grabcut_set_caps (GstOpencvVideoFilter * filter,
135 gint in_width, gint in_height, int in_cv_type,
136 gint out_width, gint out_height, int out_cv_type);
138 //static void gst_grabcut_release_all_pointers (GstGrabcut * filter);
140 static void compose_matrix_from_image (Mat output, Mat input);
142 static int run_grabcut_iteration (Mat image_c, Mat mask_c, Mat bgdModel,
144 static int run_grabcut_iteration2 (Mat image_c, Mat mask_c, Mat bgdModel,
145 Mat fgdModel, Rect bbox);
149 gst_grabcut_finalize (GObject * obj)
151 GstGrabcut *filter = GST_GRABCUT (obj);
153 filter->cvRGBin.release ();
154 filter->cvA.release ();
155 filter->cvB.release ();
156 filter->cvC.release ();
157 filter->cvD.release ();
158 filter->grabcut_mask.release ();
159 filter->bgdModel.release ();
160 filter->fgdModel.release ();
162 G_OBJECT_CLASS (gst_grabcut_parent_class)->finalize (obj);
165 /* initialize the grabcut's class */
167 gst_grabcut_class_init (GstGrabcutClass * klass)
169 GObjectClass *gobject_class = (GObjectClass *) klass;
170 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
171 GstOpencvVideoFilterClass *cvbasefilter_class =
172 (GstOpencvVideoFilterClass *) klass;
173 GstBaseTransformClass *btrans_class = (GstBaseTransformClass *) klass;
175 gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_grabcut_finalize);
176 gobject_class->set_property = gst_grabcut_set_property;
177 gobject_class->get_property = gst_grabcut_get_property;
179 btrans_class->passthrough_on_same_caps = TRUE;
181 cvbasefilter_class->cv_trans_ip_func = gst_grabcut_transform_ip;
182 cvbasefilter_class->cv_set_caps = gst_grabcut_set_caps;
184 g_object_class_install_property (gobject_class, PROP_TEST_MODE,
185 g_param_spec_boolean ("test-mode", "test-mode",
186 "If true, the output RGB is overwritten with the segmented foreground. Alpha channel same as normal case ",
187 DEFAULT_TEST_MODE, (GParamFlags)
188 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
190 g_object_class_install_property (gobject_class, PROP_SCALE,
191 g_param_spec_float ("scale", "scale",
192 "Grow factor for the face bounding box, if present", 1.0,
194 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
196 gst_element_class_set_static_metadata (element_class,
197 "Grabcut-based image FG/BG segmentation", "Filter/Effect/Video",
198 "Runs Grabcut algorithm on input alpha. Values: BG=0, FG=1, PR_BG=2, PR_FGD=3; \
199 NOTE: larger values of alpha (notably 255) are interpreted as PR_FGD too. \n\
200 IN CASE OF no alpha mask input (all 0's or all 1's), the 'face' \
201 downstream event is used to create a bbox of PR_FG elements.\n\
202 IF nothing is present, then nothing is done.", "Miguel Casas-Sanchez <miguelecasassanchez@gmail.com>");
204 gst_element_class_add_static_pad_template (element_class, &src_factory);
205 gst_element_class_add_static_pad_template (element_class, &sink_factory);
209 /* initialize the new element
210 * instantiate pads and add them to element
211 * set pad callback functions
212 * initialize instance structure
215 gst_grabcut_init (GstGrabcut * filter)
217 filter->test_mode = DEFAULT_TEST_MODE;
218 filter->scale = DEFAULT_SCALE;
219 gst_opencv_video_filter_set_in_place (GST_OPENCV_VIDEO_FILTER (filter), TRUE);
224 gst_grabcut_set_property (GObject * object, guint prop_id,
225 const GValue * value, GParamSpec * pspec)
227 GstGrabcut *grabcut = GST_GRABCUT (object);
231 grabcut->test_mode = g_value_get_boolean (value);
234 grabcut->scale = g_value_get_float (value);
237 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
243 gst_grabcut_get_property (GObject * object, guint prop_id,
244 GValue * value, GParamSpec * pspec)
246 GstGrabcut *filter = GST_GRABCUT (object);
250 g_value_set_boolean (value, filter->test_mode);
253 g_value_set_float (value, filter->scale);
256 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
261 /* GstElement vmethod implementations */
262 /* this function handles the link with other elements */
264 gst_grabcut_set_caps (GstOpencvVideoFilter * filter, gint in_width,
265 gint in_height, int in_cv_type, gint out_width,
266 gint out_height, int out_cv_type)
268 GstGrabcut *grabcut = GST_GRABCUT (filter);
271 size = Size (in_width, in_height);
274 grabcut->cvRGBin.create (size, CV_8UC3);
276 grabcut->cvA.create (size, CV_8UC1);
277 grabcut->cvB.create (size, CV_8UC1);
278 grabcut->cvC.create (size, CV_8UC1);
279 grabcut->cvD.create (size, CV_8UC1);
281 grabcut->grabcut_mask = Mat::zeros (size, CV_8UC1);
282 grabcut->bgdModel = Mat ();
283 grabcut->fgdModel = Mat ();
284 //initialise_grabcut (&(grabcut->GC), grabcut->cvRGBin, grabcut->grabcut_mask);
290 gst_grabcut_transform_ip (GstOpencvVideoFilter * filter, GstBuffer * buffer,
293 GstGrabcut *gc = GST_GRABCUT (filter);
295 std::vector < Mat > channels (4);
297 GstVideoRegionOfInterestMeta *meta;
298 meta = gst_buffer_get_video_region_of_interest_meta (buffer);
300 gc->facepos.x = (meta->x) - ((gc->scale - 1) * meta->w / 2);
301 gc->facepos.y = (meta->y) - ((gc->scale - 1) * meta->h / 2);
302 gc->facepos.width = meta->w * gc->scale * 0.9;
303 gc->facepos.height = meta->h * gc->scale * 1.1;
305 memset (static_cast < void *>(&(gc->facepos)), 0, sizeof (gc->facepos));
308 /* normally input should be RGBA */
309 split (img, channels);
310 gc->cvA = channels.at (0);
311 gc->cvB = channels.at (1);
312 gc->cvC = channels.at (2);
313 gc->cvD = channels.at (3);
314 cvtColor (img, gc->cvRGBin, COLOR_BGRA2BGR);
315 compose_matrix_from_image (gc->grabcut_mask, gc->cvD);
317 /* Pass cvD to grabcut_mask for the graphcut stuff but that only if
318 really there is something in the mask! otherwise -->input bbox is
320 alphapixels = countNonZero (gc->cvD);
321 if ((0 < alphapixels) && (alphapixels < (gc->width * gc->height))) {
322 GST_INFO ("running on mask");
323 run_grabcut_iteration (gc->cvRGBin, gc->grabcut_mask, gc->bgdModel,
326 if ((abs (gc->facepos.width) > 2) && (abs (gc->facepos.height) > 2)) {
327 GST_INFO ("running on bbox (%d,%d),(%d,%d)", gc->facepos.x, gc->facepos.y,
328 gc->facepos.width, gc->facepos.height);
329 run_grabcut_iteration2 (gc->cvRGBin, gc->grabcut_mask, gc->bgdModel,
330 gc->fgdModel, gc->facepos);
332 GST_WARNING ("No face info present, skipping frame.");
337 /* if we want to display, just overwrite the output */
339 /* get only FG, PR_FG */
340 bitwise_and (gc->grabcut_mask, Scalar (1), gc->grabcut_mask);
341 /* (saturated) FG, PR_FG --> 255 */
342 gc->grabcut_mask.convertTo (gc->grabcut_mask, -1, 255.0, 0.0);
344 bitwise_and (gc->grabcut_mask, gc->cvA, gc->cvA);
345 bitwise_and (gc->grabcut_mask, gc->cvB, gc->cvB);
346 bitwise_and (gc->grabcut_mask, gc->cvC, gc->cvC);
349 merge (channels, img);
353 Point (gc->facepos.x, gc->facepos.y),
354 Point (gc->facepos.x + gc->facepos.width,
355 gc->facepos.y + gc->facepos.height), CV_RGB (255, 0, 255), 1, 8, 0);
363 compose_matrix_from_image (Mat output, Mat input)
367 for (x = 0; x < output.cols; x++) {
368 for (y = 0; y < output.rows; y++) {
369 output.data[output.step[0] * y + x] =
370 (input.data[input.step[0] * y + x] <=
371 GC_PR_FGD) ? input.data[input.step[0] * y + x] : GC_PR_FGD;
377 run_grabcut_iteration (Mat image_c, Mat mask_c, Mat bgdModel, Mat fgdModel)
379 if (countNonZero (mask_c))
380 grabCut (image_c, mask_c, Rect (),
381 bgdModel, bgdModel, 1, GC_INIT_WITH_MASK);
387 run_grabcut_iteration2 (Mat image_c, Mat mask_c, Mat bgdModel, Mat fgdModel,
390 grabCut (image_c, mask_c, bbox, bgdModel, fgdModel, 1, GC_INIT_WITH_RECT);