1 /* Image Quality Assessment plugin
2 * Copyright (C) 2015 Mathieu Duponchelle <mathieu.duponchelle@collabora.co.uk>
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
23 * @short_description: Image Quality Assessment plugin.
25 * IQA will perform full reference image quality assessment, with the
26 * first added pad being the reference.
28 * It will perform comparisons on video streams with the same geometry.
30 * The image output will be the heat map of differences, between
31 * the two pads with the highest measured difference.
33 * For each reference frame, IQA will post a message containing
34 * a structure named IQA.
36 * The only metric supported for now is "dssim", which will be available
37 * if https://github.com/pornel/dssim was installed on the system
38 * at the time that plugin was compiled.
40 * For each metric activated, this structure will contain another
41 * structure, named after the metric.
43 * The message will also contain a "time" field.
45 * For example, if do-dssim is set to true, and there are
46 * two compared streams, the emitted structure will look like this:
48 * IQA, dssim=(structure)"dssim\,\ sink_1\=\(double\)0.053621271267184856\,\
49 * sink_2\=\(double\)0.0082939683976297474\;",
52 * ## Example launch line
54 * gst-launch-1.0 -m uridecodebin uri=file:///test/file/1 ! iqa name=iqa do-dssim=true \
55 * ! videoconvert ! autovideosink uridecodebin uri=file:///test/file/2 ! iqa.
56 * ]| This pipeline will output messages to the console for each set of compared frames.
70 GST_DEBUG_CATEGORY_STATIC (gst_iqa_debug);
71 #define GST_CAT_DEFAULT gst_iqa_debug
73 #define SINK_FORMATS " { AYUV, BGRA, ARGB, RGBA, ABGR, Y444, Y42B, YUY2, UYVY, "\
74 " YVYU, I420, YV12, NV12, NV21, Y41B, RGB, BGR, xRGB, xBGR, "\
77 #define SRC_FORMAT " { RGBA } "
78 #define DEFAULT_DSSIM_ERROR_THRESHOLD -1.0
80 static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
83 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (SRC_FORMAT))
90 PROP_SSIM_ERROR_THRESHOLD,
95 static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u",
98 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (SINK_FORMATS))
101 /* Child proxy implementation */
103 gst_iqa_child_proxy_get_child_by_index (GstChildProxy * child_proxy,
106 GstIqa *iqa = GST_IQA (child_proxy);
109 GST_OBJECT_LOCK (iqa);
110 obj = g_list_nth_data (GST_ELEMENT_CAST (iqa)->sinkpads, index);
112 gst_object_ref (obj);
113 GST_OBJECT_UNLOCK (iqa);
119 gst_iqa_child_proxy_get_children_count (GstChildProxy * child_proxy)
122 GstIqa *iqa = GST_IQA (child_proxy);
124 GST_OBJECT_LOCK (iqa);
125 count = GST_ELEMENT_CAST (iqa)->numsinkpads;
126 GST_OBJECT_UNLOCK (iqa);
127 GST_INFO_OBJECT (iqa, "Children Count: %d", count);
133 gst_iqa_child_proxy_init (gpointer g_iface, gpointer iface_data)
135 GstChildProxyInterface *iface = g_iface;
137 iface->get_child_by_index = gst_iqa_child_proxy_get_child_by_index;
138 iface->get_children_count = gst_iqa_child_proxy_get_children_count;
143 * @GST_IQA_MODE_STRICT: Strict checks of the frames is enabled, this for
144 * example implies that an error will be posted in case all the streams don't
145 * have the exact same number of frames.
151 GST_IQA_MODE_STRICT = (1 << 1),
154 #define GST_TYPE_IQA_MODE (gst_iqa_mode_flags_get_type())
156 gst_iqa_mode_flags_get_type (void)
158 static const GFlagsValue values[] = {
159 {GST_IQA_MODE_STRICT, "Strict comparison of frames.", "strict"},
164 if (g_once_init_enter ((gsize *) & id)) {
167 _id = g_flags_register_static ("GstIqaMode", values);
169 g_once_init_leave ((gsize *) & id, _id);
176 #define gst_iqa_parent_class parent_class
177 G_DEFINE_TYPE_WITH_CODE (GstIqa, gst_iqa, GST_TYPE_VIDEO_AGGREGATOR,
178 G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, gst_iqa_child_proxy_init));
181 inline static unsigned char
186 if (in >= 255.f / 256.f)
192 do_dssim (GstIqa * self, GstVideoFrame * ref, GstVideoFrame * cmp,
193 GstBuffer * outbuf, GstStructure * msg_structure, gchar * padname)
197 unsigned char **ptrs, **ptrs2;
201 dssim_image *ref_image;
202 dssim_image *cmp_image;
204 dssim_ssim_map map_meta;
208 GstStructure *dssim_structure;
211 if (ref->info.width != cmp->info.width ||
212 ref->info.height != cmp->info.height) {
213 GST_OBJECT_UNLOCK (self);
215 GST_ELEMENT_ERROR (self, STREAM, FAILED,
216 ("Video streams do not have the same sizes (add videoscale"
217 " and force the sizes to be equal on all sink pads.)"),
218 ("Reference width %d - compared width: %d. "
219 "Reference height %d - compared height: %d",
220 ref->info.width, cmp->info.width, ref->info.height,
223 GST_OBJECT_LOCK (self);
227 gst_structure_get (msg_structure, "dssim", GST_TYPE_STRUCTURE,
228 &dssim_structure, NULL);
230 attr = dssim_create_attr ();
231 dssim_set_save_ssim_maps (attr, 1, 1);
233 gst_buffer_map (ref->buffer, &ref_info, GST_MAP_READ);
234 gst_buffer_map (cmp->buffer, &cmp_info, GST_MAP_READ);
235 gst_buffer_map (outbuf, &out_info, GST_MAP_WRITE);
236 out = (dssim_rgba *) out_info.data;
238 ptrs = g_malloc (sizeof (char **) * ref->info.height);
240 for (y = 0; y < ref->info.height; y++) {
241 ptrs[y] = ref_info.data + (ref->info.width * 4 * y);
245 dssim_create_image (attr, ptrs, DSSIM_RGBA, ref->info.width,
246 ref->info.height, 0.45455);
248 ptrs2 = g_malloc (sizeof (char **) * cmp->info.height);
250 for (y = 0; y < cmp->info.height; y++) {
251 ptrs2[y] = cmp_info.data + (cmp->info.width * 4 * y);
255 dssim_create_image (attr, ptrs2, DSSIM_RGBA, cmp->info.width,
256 cmp->info.height, 0.45455);
257 dssim = dssim_compare (attr, ref_image, cmp_image);
259 map_meta = dssim_pop_ssim_map (attr, 0, 0);
261 /* Comparing floats... should not be a big deal anyway */
262 if (self->ssim_threshold > 0 && dssim > self->ssim_threshold) {
263 /* We do not really care about our state... we are going to error ou
265 GST_OBJECT_UNLOCK (self);
267 GST_ELEMENT_ERROR (self, STREAM, FAILED,
268 ("Dssim check failed on %s at %"
269 GST_TIME_FORMAT " with dssim %f > %f",
271 GST_TIME_ARGS (GST_AGGREGATOR_PAD (GST_AGGREGATOR (self)->
272 srcpad)->segment.position), dssim, self->ssim_threshold),
275 GST_OBJECT_LOCK (self);
281 if (dssim > self->max_dssim) {
284 for (i = 0; i < map_meta.width * map_meta.height; i++) {
285 const float max = 1.0 - map[i];
286 const float maxsq = max * max;
287 out[i] = (dssim_rgba) {
288 .r = to_byte (max * 3.0),.g = to_byte (maxsq * 6.0),.b =
289 to_byte (max / ((1.0 - map_meta.dssim) * 4.0)),.a = 255,};
291 self->max_dssim = dssim;
294 gst_structure_set (dssim_structure, padname, G_TYPE_DOUBLE, dssim, NULL);
295 gst_structure_set (msg_structure, "dssim", GST_TYPE_STRUCTURE,
296 dssim_structure, NULL);
302 gst_structure_free (dssim_structure);
304 free (map_meta.data);
307 gst_buffer_unmap (ref->buffer, &ref_info);
308 gst_buffer_unmap (cmp->buffer, &cmp_info);
309 gst_buffer_unmap (outbuf, &out_info);
310 dssim_dealloc_image (ref_image);
311 dssim_dealloc_image (cmp_image);
312 dssim_dealloc_attr (attr);
319 compare_frames (GstIqa * self, GstVideoFrame * ref, GstVideoFrame * cmp,
320 GstBuffer * outbuf, GstStructure * msg_structure, gchar * padname)
323 if (self->do_dssim) {
324 if (!do_dssim (self, ref, cmp, outbuf, msg_structure, padname))
333 gst_iqa_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf)
336 GstVideoFrame *ref_frame = NULL;
337 GstIqa *self = GST_IQA (vagg);
338 GstStructure *msg_structure = gst_structure_new_empty ("IQA");
339 GstMessage *m = gst_message_new_element (GST_OBJECT (self), msg_structure);
340 GstAggregator *agg = GST_AGGREGATOR (vagg);
342 if (self->do_dssim) {
343 gst_structure_set (msg_structure, "dssim", GST_TYPE_STRUCTURE,
344 gst_structure_new_empty ("dssim"), NULL);
345 self->max_dssim = 0.0;
348 GST_OBJECT_LOCK (vagg);
349 for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
350 GstVideoAggregatorPad *pad = l->data;
351 GstVideoFrame *prepared_frame =
352 gst_video_aggregator_pad_get_prepared_frame (pad);
354 if (prepared_frame != NULL) {
356 ref_frame = prepared_frame;
359 gchar *padname = gst_pad_get_name (pad);
360 GstVideoFrame *cmp_frame = prepared_frame;
362 res = compare_frames (self, ref_frame, cmp_frame, outbuf, msg_structure,
369 } else if ((self->mode & GST_IQA_MODE_STRICT) && ref_frame) {
370 GST_OBJECT_UNLOCK (vagg);
372 GST_ELEMENT_ERROR (self, STREAM, FAILED,
373 ("All sources are supposed to have the same number of buffers"
374 " but got no buffer matching %" GST_PTR_FORMAT " on pad: %"
375 GST_PTR_FORMAT, outbuf, pad), (NULL));
377 GST_OBJECT_LOCK (vagg);
382 GST_OBJECT_UNLOCK (vagg);
384 /* We only post the message here, because we can't post it while the object
387 gst_structure_set (msg_structure, "time", GST_TYPE_CLOCK_TIME,
388 GST_AGGREGATOR_PAD (agg->srcpad)->segment.position, NULL);
389 gst_element_post_message (GST_ELEMENT (self), m);
393 GST_OBJECT_UNLOCK (vagg);
395 return GST_FLOW_ERROR;
399 _set_property (GObject * object, guint prop_id, const GValue * value,
402 GstIqa *self = GST_IQA (object);
406 GST_OBJECT_LOCK (self);
407 self->do_dssim = g_value_get_boolean (value);
408 GST_OBJECT_UNLOCK (self);
410 case PROP_SSIM_ERROR_THRESHOLD:
411 GST_OBJECT_LOCK (self);
412 self->ssim_threshold = g_value_get_double (value);
413 GST_OBJECT_UNLOCK (self);
416 GST_OBJECT_LOCK (self);
417 self->mode = g_value_get_flags (value);
418 GST_OBJECT_UNLOCK (self);
421 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
427 _get_property (GObject * object,
428 guint prop_id, GValue * value, GParamSpec * pspec)
430 GstIqa *self = GST_IQA (object);
434 GST_OBJECT_LOCK (self);
435 g_value_set_boolean (value, self->do_dssim);
436 GST_OBJECT_UNLOCK (self);
438 case PROP_SSIM_ERROR_THRESHOLD:
439 GST_OBJECT_LOCK (self);
440 g_value_set_double (value, self->ssim_threshold);
441 GST_OBJECT_UNLOCK (self);
444 GST_OBJECT_LOCK (self);
445 g_value_set_flags (value, self->mode);
446 GST_OBJECT_UNLOCK (self);
449 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
454 /* GObject boilerplate */
456 gst_iqa_class_init (GstIqaClass * klass)
458 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
459 GstElementClass *gstelement_class = (GstElementClass *) klass;
460 GstVideoAggregatorClass *videoaggregator_class =
461 (GstVideoAggregatorClass *) klass;
463 videoaggregator_class->aggregate_frames = gst_iqa_aggregate_frames;
465 gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
466 &src_factory, GST_TYPE_AGGREGATOR_PAD);
467 gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
468 &sink_factory, GST_TYPE_VIDEO_AGGREGATOR_CONVERT_PAD);
470 gobject_class->set_property = _set_property;
471 gobject_class->get_property = _get_property;
474 g_object_class_install_property (gobject_class, PROP_DO_SSIM,
475 g_param_spec_boolean ("do-dssim", "do-dssim",
476 "Run structural similarity checks", FALSE, G_PARAM_READWRITE));
478 g_object_class_install_property (gobject_class, PROP_SSIM_ERROR_THRESHOLD,
479 g_param_spec_double ("dssim-error-threshold", "dssim error threshold",
480 "dssim value over which the element will post an error message on the bus."
481 " A value < 0.0 means 'disabled'.",
482 -1.0, G_MAXDOUBLE, DEFAULT_DSSIM_ERROR_THRESHOLD, G_PARAM_READWRITE));
488 * Controls the frame comparison mode.
492 g_object_class_install_property (gobject_class, PROP_MODE,
493 g_param_spec_flags ("mode", "IQA mode",
494 "Controls the frame comparison mode.", GST_TYPE_IQA_MODE,
495 0, G_PARAM_READWRITE));
497 gst_type_mark_as_plugin_api (GST_TYPE_IQA_MODE, 0);
499 gst_element_class_set_static_metadata (gstelement_class, "Iqa",
500 "Filter/Analyzer/Video",
501 "Provides various Image Quality Assessment metrics",
502 "Mathieu Duponchelle <mathieu.duponchelle@collabora.co.uk>");
506 gst_iqa_init (GstIqa * self)
511 plugin_init (GstPlugin * plugin)
513 GST_DEBUG_CATEGORY_INIT (gst_iqa_debug, "iqa", 0, "iqa");
515 return gst_element_register (plugin, "iqa", GST_RANK_PRIMARY, GST_TYPE_IQA);
518 GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
521 "Iqa", plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME,