From 38c6f6366c6ee089bde86f171050b6bef6452fda Mon Sep 17 00:00:00 2001 From: David Schleef Date: Fri, 8 Apr 2011 12:11:07 -0700 Subject: [PATCH] scenechange: new scene change detection element --- gst/videofilters/Makefile.am | 4 +- gst/videofilters/gstscenechange.c | 454 ++++++++++++++++++++++++++++++++++ gst/videofilters/gstscenechange.h | 58 +++++ gst/videofilters/gstvideofilter2.c | 2 +- gst/videofilters/gstvideofiltersbad.c | 3 + 5 files changed, 519 insertions(+), 2 deletions(-) create mode 100644 gst/videofilters/gstscenechange.c create mode 100644 gst/videofilters/gstscenechange.h diff --git a/gst/videofilters/Makefile.am b/gst/videofilters/Makefile.am index 262ce7d..5986ce7 100644 --- a/gst/videofilters/Makefile.am +++ b/gst/videofilters/Makefile.am @@ -7,6 +7,7 @@ libgstvideofiltersbad_la_SOURCES = \ gstvideofilter2.c \ gstvideofilter2.h \ gstzebrastripe.c \ + gstscenechange.c \ gstvideofiltersbad.c #nodist_libgstvideofiltersbad_la_SOURCES = $(ORC_NODIST_SOURCES) libgstvideofiltersbad_la_CFLAGS = \ @@ -23,6 +24,7 @@ libgstvideofiltersbad_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) libgstvideofiltersbad_la_LIBTOOLFLAGS = --tag=disable-static noinst_HEADERS = \ - gstzebrastripe.h + gstzebrastripe.h \ + gstscenechange.h diff --git a/gst/videofilters/gstscenechange.c b/gst/videofilters/gstscenechange.c new file mode 100644 index 0000000..385ed0b --- /dev/null +++ b/gst/videofilters/gstscenechange.c @@ -0,0 +1,454 @@ +/* GStreamer + * Copyright (C) 2011 Entropy Wave Inc + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin Street, Suite 500, + * Boston, MA 02110-1335, USA. + */ +/** + * SECTION:element-gstscenechange + * + * The scenechange element detects scene changes (also known as shot + * changes) in a video stream, and sends a signal when this occurs. + * Applications can listen to this signal and make changes to the + * pipeline such as cutting the stream. In addition, whenever a + * scene change is detected, a custom downstream "GstForceKeyUnit" + * event is sent to downstream elements. Most video encoder elements + * will insert synchronization points into the stream when this event + * is received. When used with a tee element, the scenechange element + * can be used to align the synchronization points among multiple + * video encoders, which is useful for segmented streaming. + * + * The scenechange element does not work with compressed video. + * + * + * Example launch line + * |[ + * gst-launch -v filesrc location=some_file.ogv ! decodebin ! + * scenechange ! theoraenc ! fakesink + * ]| + * + */ +/* + * The algorithm used for scene change detection is a modification + * of Jim Easterbrook's shot change detector. I'm not aware of a + * research paper, but the code I got the idea from is here: + * http://sourceforge.net/projects/shot-change/ + * + * The method is relatively simple. Calculate the sum of absolute + * differences of a picture and the previous picture, and compare this + * picture difference value with neighboring pictures. In the original + * algorithm, the value is compared to a configurable number of past + * and future pictures. However, comparing to future frames requires + * introducing latency into the stream, which I did not want. So this + * implementation only compared to previous frames. + * + * This code is more directly derived from the scene change detection + * implementation in Schroedinger. Schro's implementation is closer + * to the Easterbrook algorithm, comparing to future pictures. In + * terms of accuracy, schro's implementation has about 2-3 false positives + * or false negatives per 100 scene changes. This implementation has + * about 5 per 100. The threshold is tuned for minimum total false + * positives or negatives, on the assumption that the badness of a + * false negative is the same as a false positive. + * + * This algorithm is pretty much at its limit for error rate. I + * recommend any future work in this area to increase the complexity + * of detection, and then write an automatic tuning system as opposed + * to the manual tuning I did here. + * + * Inside the TESTING define are some hard-coded (mostly hand-written) + * scene change frame numbers for some easily available sequences. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include "gstvideofilter2.h" +#include "gstscenechange.h" +#include + +GST_DEBUG_CATEGORY_STATIC (gst_scene_change_debug_category); +#define GST_CAT_DEFAULT gst_scene_change_debug_category + +/* prototypes */ + + +static void gst_scene_change_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_scene_change_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); +static void gst_scene_change_dispose (GObject * object); +static void gst_scene_change_finalize (GObject * object); + +static gboolean gst_scene_change_start (GstBaseTransform * trans); +static gboolean gst_scene_change_stop (GstBaseTransform * trans); +static GstFlowReturn +gst_scene_change_prefilter (GstVideoFilter2 * videofilter2, GstBuffer * buf); + +static GstVideoFilter2Functions gst_scene_change_filter_functions[]; + +#undef TESTING +#ifdef TESTING +static gboolean is_shot_change (int frame_number); +#endif + +enum +{ + PROP_0 +}; + +/* pad templates */ + + +/* class initialization */ + +#define DEBUG_INIT(bla) \ + GST_DEBUG_CATEGORY_INIT (gst_scene_change_debug_category, "scenechange", 0, \ + "debug category for scenechange element"); + +GST_BOILERPLATE_FULL (GstSceneChange, gst_scene_change, GstVideoFilter2, + GST_TYPE_VIDEO_FILTER2, DEBUG_INIT); + +static void +gst_scene_change_base_init (gpointer g_class) +{ + GstElementClass *element_class = GST_ELEMENT_CLASS (g_class); + + + gst_element_class_set_details_simple (element_class, "Scene change detector", + "Video/Filter", "Detects scene changes in video", + "David Schleef "); +} + +static void +gst_scene_change_class_init (GstSceneChangeClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstBaseTransformClass *base_transform_class = + GST_BASE_TRANSFORM_CLASS (klass); + GstVideoFilter2Class *video_filter2_class = GST_VIDEO_FILTER2_CLASS (klass); + + gobject_class->set_property = gst_scene_change_set_property; + gobject_class->get_property = gst_scene_change_get_property; + gobject_class->dispose = gst_scene_change_dispose; + gobject_class->finalize = gst_scene_change_finalize; + base_transform_class->start = GST_DEBUG_FUNCPTR (gst_scene_change_start); + base_transform_class->stop = GST_DEBUG_FUNCPTR (gst_scene_change_stop); + video_filter2_class->prefilter = + GST_DEBUG_FUNCPTR (gst_scene_change_prefilter); + + gst_video_filter2_class_add_functions (video_filter2_class, + gst_scene_change_filter_functions); + +} + +static void +gst_scene_change_init (GstSceneChange * scenechange, + GstSceneChangeClass * scenechange_class) +{ +} + +void +gst_scene_change_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec) +{ + GstSceneChange *scenechange; + + g_return_if_fail (GST_IS_SCENE_CHANGE (object)); + scenechange = GST_SCENE_CHANGE (object); + + switch (property_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_scene_change_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec) +{ + GstSceneChange *scenechange; + + g_return_if_fail (GST_IS_SCENE_CHANGE (object)); + scenechange = GST_SCENE_CHANGE (object); + + switch (property_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_scene_change_dispose (GObject * object) +{ + GstSceneChange *scenechange; + + g_return_if_fail (GST_IS_SCENE_CHANGE (object)); + scenechange = GST_SCENE_CHANGE (object); + + /* clean up as possible. may be called multiple times */ + + G_OBJECT_CLASS (parent_class)->dispose (object); +} + +void +gst_scene_change_finalize (GObject * object) +{ + GstSceneChange *scenechange; + + g_return_if_fail (GST_IS_SCENE_CHANGE (object)); + scenechange = GST_SCENE_CHANGE (object); + + if (scenechange->oldbuf) + gst_buffer_unref (scenechange->oldbuf); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + + +static gboolean +gst_scene_change_start (GstBaseTransform * trans) +{ + + return TRUE; +} + +static gboolean +gst_scene_change_stop (GstBaseTransform * trans) +{ + + return TRUE; +} + +static GstFlowReturn +gst_scene_change_prefilter (GstVideoFilter2 * video_filter2, GstBuffer * buf) +{ + + return GST_FLOW_OK; +} + +static double +get_frame_score (guint8 * s1, guint8 * s2, int width, int height) +{ + int i; + int j; + int score = 0; + + for (j = 0; j < height; j++) { + for (i = 0; i < width; i++) { + score += ABS (s1[i] - s2[i]); + } + s1 += width; + s2 += width; + } + + return ((double) score) / (width * height); +} + +static GstFlowReturn +gst_scene_change_filter_ip_I420 (GstVideoFilter2 * videofilter2, + GstBuffer * buf, int start, int end) +{ + GstSceneChange *scenechange; + double score_min; + double score_max; + double threshold; + double score; + gboolean change; + int i; + int width; + int height; + + g_return_val_if_fail (GST_IS_SCENE_CHANGE (videofilter2), GST_FLOW_ERROR); + scenechange = GST_SCENE_CHANGE (videofilter2); + + width = GST_VIDEO_FILTER2_WIDTH (videofilter2); + height = GST_VIDEO_FILTER2_HEIGHT (videofilter2); + + if (!scenechange->oldbuf) { + scenechange->n_diffs = 0; + memset (scenechange->diffs, 0, sizeof (double) * SC_N_DIFFS); + scenechange->oldbuf = gst_buffer_ref (buf); + return GST_FLOW_OK; + } + + score = get_frame_score (GST_BUFFER_DATA (scenechange->oldbuf), + GST_BUFFER_DATA (buf), width, height); + + memmove (scenechange->diffs, scenechange->diffs + 1, + sizeof (double) * (SC_N_DIFFS - 1)); + scenechange->diffs[SC_N_DIFFS - 1] = score; + scenechange->n_diffs++; + + gst_buffer_unref (scenechange->oldbuf); + scenechange->oldbuf = gst_buffer_ref (buf); + + score_min = scenechange->diffs[0]; + score_max = scenechange->diffs[0]; + for (i = 1; i < SC_N_DIFFS - 1; i++) { + score_min = MIN (score_min, scenechange->diffs[i]); + score_max = MAX (score_max, scenechange->diffs[i]); + } + + threshold = 1.8 * score_max - 0.8 * score_min; + + if (scenechange->n_diffs > 2) { + if (score < 5) { + change = FALSE; + } else if (score / threshold < 1.0) { + change = FALSE; + } else if (score / threshold > 2.5) { + change = TRUE; + } else if (score > 50) { + change = TRUE; + } else { + change = FALSE; + } + } else { + change = FALSE; + } + +#ifdef TESTING + if (change != is_shot_change (scenechange->n_diffs)) { + g_print ("%d %g %g %g %d\n", scenechange->n_diffs, score / threshold, + score, threshold, change); + } +#endif + + if (change) { + GstEvent *event; + + GST_DEBUG_OBJECT (scenechange, "%d %g %g %g %d", + scenechange->n_diffs, score / threshold, score, threshold, change); + + event = gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM, + gst_structure_new ("GstForceKeyUnit", NULL)); + + gst_pad_push_event (GST_BASE_TRANSFORM_SRC_PAD (scenechange), event); + } + + return GST_FLOW_OK; +} + +static GstVideoFilter2Functions gst_scene_change_filter_functions[] = { + {GST_VIDEO_FORMAT_I420, NULL, gst_scene_change_filter_ip_I420}, + {GST_VIDEO_FORMAT_UNKNOWN} +}; + + + + + + +#ifdef TESTING +/* This is from ds's personal collection. No, you can't have it. */ +int showreel_changes[] = { + 242, 483, 510, 550, 579, 603, 609, 1056, 1067, 1074, 1079, 1096, + 1106, 1113, 1127, 1145, 1156, 1170, 1212, 1228, 1243, 1269, 1274, + 1322, 1349, 1370, 1378, 1423, 1456, 1458, 1508, 1519, 1542, 1679, + 1767, 1837, 1895, 1962, 2006, 2035, 2102, 2139, 2196, 2561, 2664, + 2837, 2895, 2985, 3035, 3077, 3128, 3176, 3218, 3306, 3351, 3388, + 3421, 3470, 3711, 3832, 4029, 4184, 4444, 4686, 4719, 4825, 4941, + 5009, 5091, 5194, 5254, 5286, 5287, 5343, 5431, 5501, 5634, 5695, 5788, + 5839, 5861, 5930, 6030, 6168, 6193, 6237, 6336, 6376, 6421, 6495, + 6550, 6611, 6669, 6733, 6819, 6852, 6944, 7087, 7148, 7189, 7431, + 7540, 7599, 7632, 7661, 7693, 7930, 7963, 8003, 8076, 8109, 8147, + 8177, 8192, 8219, 8278, 8322, 8370, 8409, 8566, 8603, 8747, 8775, + 8873, 8907, 8955, 8969, 8983, 8997, 9026, 9079, 9140, 9165, 9206, + 9276, 9378, 9449, 9523, 9647, 9703, 9749, 9790, 9929, 10056, 10216, + 10307, 10411, 10487, 10557, 10695, 10770, 10854, 11095, 11265, 11517, 11589, + 11686, 11825, 11940, 12004, 12047, 12113, 12179, 12233, 12532, 12586, 12708, + 12793, 12877, 12954, 13030, 13105, 13177, 13279, 13396, 13486, 13538, 13561, + 13591, 13627, 13656, 13709, 13763, 13815, 13842, 13876, 13906, 13929, 13955, + 14003, 14070, 14097, 14127, 14153, 14198, 14269, 14348, 14367, 14440, 14488, + 14548, 14573, 14599, 14630, 14665, 14907, 14962, 15013, 15089, 15148, 15227, + 15314, 15355, 15369, 15451, 15470, 15542, 15570, 15640, 15684, 15781, 15869, + 15938, 16172, 16266, 16429, 16479, 16521, 16563, 16612, 16671, 16692, 16704, + 16720, 16756, 16789, 16802, 16815, 16867, 16908, 16939, 16953, 16977, 17006, + 17014, 17026, 17040, 17062, 17121, 17176, 17226, 17322, 17444, 17496, 17641, + 17698, 17744, 17826, 17913, 17993, 18073, 18219, 18279, 18359, 18475, 18544, + 18587, 18649, 18698, 18756, 18826, 18853, 18866, 19108, 19336, 19481, 19544, + 19720, 19816, 19908, 19982, 20069, 20310, 20355, 20374, 20409, 20469, 20599, + 20607, 20652, 20805, 20822, 20882, 20982, 21029, 21433, 21468, 21561, 21602, + 21661, 21720, 21909, 22045, 22166, 22225, 22323, 22362, 22433, 22477, 22529, + 22571, 22617, 22642, 22676, 22918, 22978, 23084, 23161, 23288, 23409, 23490, + 23613, 23721, 23815, 24131, 24372, 24468, 24507, 24555, 24568, 24616, 24634, + 24829, 24843, 24919, 24992, 25040, 25160, 25288, 25607, 25684, 25717, 25764, + 25821, 25866, 25901, 25925, 25941, 25978, 25998, 26011, 26030, 26055, 26118, + 26133, 26145, 26159, 26175, 26182, 26195, 26205, 26238, 26258, 26316, 26340, + 26581, 26725, 26834, 26874, 26995, 27065, 27178, 27238, 27365, 27607, 27669, + 27694, + 27774, 27800, 27841, 27930, 27985, 28057, 28091, 28132, 28189, 28270, 28545, + 28653, 28711, 28770, 28886, 28966, 29139, 29241, 29356, 29415, 29490, 29576, + 29659, 29776, 29842, 29910, 30029, 30056, 30100, 30129, 30175, 30316, 30376, + 30441, 30551, 30666, 30784, 30843, 30948, 31045, 31286, 31315, 31534, 31607, + 31742, + 31817, 31853, 31984, 32009, 32112, 32162, 32210, 32264 +}; + +/* Sintel */ +int sintel_changes[] = { + 752, 1018, 1036, 1056, 1078, 1100, 1169, 1319, 1339, 1370, + 1425, 1455, 1494, 1552, 1572, 1637, 1663, 1777, 1955, 2060, + 2125, 2429, 2624, 2780, 2835, 2881, 2955, 3032, 3144, 3217, + 3315, 3384, 3740, 3890, 4234, 4261, 4322, 4368, 4425, 4481, + 4555, 4605, 4671, 4714, 4743, 4875, 4920, 5082, 5158, 5267, + 5379, 5956, 6021, 6071, 6112, 6139, 6221, 6318, 6374, 6519, + 6558, 6615, 6691, 6803, 6900, 6944, 7134, 7266, 7351, 7414, + 7467, 7503, 7559, 7573, 7656, 7733, 7876, 7929, 7971, 7985, + 8047, 8099, 8144, 8215, 8394, 8435, 8480, 9133, 9190, 9525, + 9962, +}; + +/* Breathe Out video, http://media.xiph.org/video/misc/ */ +int breatheout_changes[] = { + 143, 263, 334, 426, 462, 563, 583, 618, 655, 707, + 818, 823, 858, 913, 956, 977, 999, 1073, 1124, 1144, + 1166, 1187, 1206, 1227, 1240, 1264, 1289, 1312, 1477, 1535, + 1646, 1692, 1739, 1757, 1798, 1855, 1974, 2048, 2129, 2212, + 2369, 2412, 2463, 2578, 2649, 2699, 2778, 2857, 2923, 3014, + 3107, 3246, 3321, 3350, 3459, 3498, 3541, 3567, 3613, 3636, + 3673, 3709, 3747, 3834, 3862, 3902, 3922, 4022, 4117, 4262, + 4303, 4357, 4556, 4578, 4617, 4716, 4792, 4873, 4895, 4917, + 4932, 4972, 5015, 5034, 5058, 5090, 5162, 5180, 5202, 5222, + 5239, 5258, 5281, 5298, 5397, 5430, + 485, 507, 534, 665, 685, 755, 1023, 1379, 1441, 1503, + 1584, 1621, 1903, 2081, 2281, 2511, 2958, 3071, 3185, 3214, + 3271, 3424, 3479, 3588, 3879, 3979, 4043, 4062, 4143, 4207, + 4237, 4336, 4461, 4476, 4533, 4647, 4815, 4853, 4949, 5075, + 5142, 5316, 5376, + 3514, 3952, 4384, 5337 +}; + +#define changes showreel_changes + +static gboolean +is_shot_change (int frame_number) +{ + int i; + for (i = 0; i < sizeof (changes) / sizeof (changes[0]); i++) { + if (changes[i] == frame_number) + return TRUE; + } + return FALSE; +} +#endif diff --git a/gst/videofilters/gstscenechange.h b/gst/videofilters/gstscenechange.h new file mode 100644 index 0000000..82ad3e4 --- /dev/null +++ b/gst/videofilters/gstscenechange.h @@ -0,0 +1,58 @@ +/* GStreamer + * Copyright (C) 2011 FIXME + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#ifndef _GST_SCENE_CHANGE_H_ +#define _GST_SCENE_CHANGE_H_ + +#include +#include +#include "gstvideofilter2.h" + +G_BEGIN_DECLS + +#define GST_TYPE_SCENE_CHANGE (gst_scene_change_get_type()) +#define GST_SCENE_CHANGE(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_SCENE_CHANGE,GstSceneChange)) +#define GST_SCENE_CHANGE_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_SCENE_CHANGE,GstSceneChangeClass)) +#define GST_IS_SCENE_CHANGE(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_SCENE_CHANGE)) +#define GST_IS_SCENE_CHANGE_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_SCENE_CHANGE)) + +typedef struct _GstSceneChange GstSceneChange; +typedef struct _GstSceneChangeClass GstSceneChangeClass; + +#define SC_N_DIFFS 5 + +struct _GstSceneChange +{ + GstVideoFilter2 base_scenechange; + + int n_diffs; + double diffs[SC_N_DIFFS]; + GstBuffer *oldbuf; +}; + +struct _GstSceneChangeClass +{ + GstVideoFilter2Class base_scenechange_class; +}; + +GType gst_scene_change_get_type (void); + +G_END_DECLS + +#endif diff --git a/gst/videofilters/gstvideofilter2.c b/gst/videofilters/gstvideofilter2.c index 1cba5db..fdffb65 100644 --- a/gst/videofilters/gstvideofilter2.c +++ b/gst/videofilters/gstvideofilter2.c @@ -90,7 +90,7 @@ gst_video_filter2_base_init (gpointer g_class) GstCaps *caps = NULL; caps = gst_caps_new_empty (); - for (i = GST_VIDEO_FORMAT_I420; i <= GST_VIDEO_FORMAT_AYUV; i++) { + for (i = GST_VIDEO_FORMAT_I420; i <= GST_VIDEO_FORMAT_I420; i++) { gst_caps_append (caps, gst_video_format_new_template_caps (i)); } diff --git a/gst/videofilters/gstvideofiltersbad.c b/gst/videofilters/gstvideofiltersbad.c index f05ace5..9f15cea 100644 --- a/gst/videofilters/gstvideofiltersbad.c +++ b/gst/videofilters/gstvideofiltersbad.c @@ -23,6 +23,7 @@ #include #include +#include "gstscenechange.h" #include "gstzebrastripe.h" @@ -30,6 +31,8 @@ static gboolean plugin_init (GstPlugin * plugin) { + gst_element_register (plugin, "scenechange", GST_RANK_NONE, + gst_scene_change_get_type ()); gst_element_register (plugin, "zebrastripe", GST_RANK_NONE, gst_zebra_stripe_get_type ()); -- 2.7.4