From 2e56032031e7dda0d654bddd59efad22bb8b1183 Mon Sep 17 00:00:00 2001 From: Stefan Sauer Date: Wed, 3 Apr 2013 19:05:38 +0200 Subject: [PATCH] level: subdivide buffers for sample accurate interval handling Previously we would skip level message when processing buffers > the requested interval. Also the message frequency would contain quite some jitter due to only considering them at the end of buffers. Cleanup the tests while we're at it. --- gst/level/gstlevel.c | 154 ++++++++++++++++++++++++------------------- tests/check/elements/level.c | 144 ++++++++++++++++++++++++---------------- 2 files changed, 172 insertions(+), 126 deletions(-) diff --git a/gst/level/gstlevel.c b/gst/level/gstlevel.c index a0edd13..bb21a8b 100644 --- a/gst/level/gstlevel.c +++ b/gst/level/gstlevel.c @@ -244,6 +244,10 @@ gst_level_init (GstLevel * filter) { filter->CS = NULL; filter->peak = NULL; + filter->last_peak = NULL; + filter->decay_peak = NULL; + filter->decay_peak_base = NULL; + filter->decay_peak_age = NULL; gst_audio_info_init (&filter->info); @@ -367,20 +371,18 @@ gst_level_calculate_##TYPE (gpointer data, guint num, guint channels, \ gdouble squaresum = 0.0; /* square sum of the integer samples */ \ register gdouble square = 0.0; /* Square */ \ register gdouble peaksquare = 0.0; /* Peak Square Sample */ \ - gdouble normalizer; /* divisor to get a [-1.0, 1.0] range */ \ + gdouble normalizer; /* divisor to get a [-1.0, 1.0] range */ \ \ /* *NCS = 0.0; Normalized Cumulative Square */ \ /* *NPS = 0.0; Normalized Peask Square */ \ \ - normalizer = (gdouble) (G_GINT64_CONSTANT(1) << (RESOLUTION * 2)); \ - \ - for (j = 0; j < num; j += channels) \ - { \ + for (j = 0; j < num; j += channels) { \ square = ((gdouble) in[j]) * in[j]; \ if (square > peaksquare) peaksquare = square; \ squaresum += square; \ } \ \ + normalizer = (gdouble) (G_GINT64_CONSTANT(1) << (RESOLUTION * 2)); \ *NCS = squaresum / normalizer; \ *NPS = peaksquare / normalizer; \ } @@ -405,8 +407,7 @@ gst_level_calculate_##TYPE (gpointer data, guint num, guint channels, \ /* *NPS = 0.0; Normalized Peask Square */ \ \ /* orc_level_squaresum_f64(&squaresum,in,num); */ \ - for (j = 0; j < num; j += channels) \ - { \ + for (j = 0; j < num; j += channels) { \ square = ((gdouble) in[j]) * in[j]; \ if (square > peaksquare) peaksquare = square; \ squaresum += square; \ @@ -498,6 +499,7 @@ gst_level_start (GstBaseTransform * trans) GstLevel *filter = GST_LEVEL (trans); filter->num_frames = 0; + filter->message_ts = GST_CLOCK_TIME_NONE; return TRUE; } @@ -580,9 +582,11 @@ gst_level_transform_ip (GstBaseTransform * trans, GstBuffer * in) gsize in_size; gdouble CS; guint i; - guint num_frames = 0; + guint num_frames; guint num_int_samples = 0; /* number of interleaved samples * ie. total count for all channels combined */ + guint block_size, block_int_size; /* we subdivide buffers to not skip message + * intervals */ GstClockTimeDiff falloff_time; gint channels, rate, bps; @@ -603,75 +607,87 @@ gst_level_transform_ip (GstBaseTransform * trans, GstBuffer * in) g_return_val_if_fail (num_int_samples % channels == 0, GST_FLOW_ERROR); + if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (filter->message_ts))) { + filter->message_ts = GST_BUFFER_TIMESTAMP (in); + } + num_frames = num_int_samples / channels; + while (num_frames > 0) { + block_size = filter->interval_frames - filter->num_frames; + block_size = MIN (block_size, num_frames); + block_int_size = block_size * channels; - for (i = 0; i < channels; ++i) { - if (!GST_BUFFER_FLAG_IS_SET (in, GST_BUFFER_FLAG_GAP)) { - filter->process (in_data, num_int_samples, channels, &CS, - &filter->peak[i]); - GST_LOG_OBJECT (filter, - "channel %d, cumulative sum %f, peak %f, over %d samples/%d channels", - i, CS, filter->peak[i], num_int_samples, channels); - filter->CS[i] += CS; - } else { - filter->peak[i] = 0.0; - } - in_data += bps; - - filter->decay_peak_age[i] += GST_FRAMES_TO_CLOCK_TIME (num_frames, rate); - GST_LOG_OBJECT (filter, "filter peak info [%d]: decay peak %f, age %" - GST_TIME_FORMAT, i, - filter->decay_peak[i], GST_TIME_ARGS (filter->decay_peak_age[i])); - - /* update running peak */ - if (filter->peak[i] > filter->last_peak[i]) - filter->last_peak[i] = filter->peak[i]; - - /* make decay peak fall off if too old */ - falloff_time = - GST_CLOCK_DIFF (gst_gdouble_to_guint64 (filter->decay_peak_ttl), - filter->decay_peak_age[i]); - if (falloff_time > 0) { - gdouble falloff_dB; - gdouble falloff; - gdouble length; /* length of falloff time in seconds */ - - length = (gdouble) falloff_time / (gdouble) GST_SECOND; - falloff_dB = filter->decay_peak_falloff * length; - falloff = pow (10, falloff_dB / -20.0); + GST_LOG_OBJECT (filter, "run inner loop for %u sample frames", + block_int_size); + for (i = 0; i < channels; ++i) { + if (!GST_BUFFER_FLAG_IS_SET (in, GST_BUFFER_FLAG_GAP)) { + filter->process (in_data, block_int_size, channels, &CS, + &filter->peak[i]); + GST_LOG_OBJECT (filter, + "channel %d, cumulative sum %f, over %d samples/%d channels", i, CS, + block_int_size, channels); + filter->CS[i] += CS; + } else { + filter->peak[i] = 0.0; + } + in_data += bps; + + filter->decay_peak_age[i] += GST_FRAMES_TO_CLOCK_TIME (num_frames, rate); GST_LOG_OBJECT (filter, - "falloff: current %f, base %f, interval %" GST_TIME_FORMAT - ", dB falloff %f, factor %e", - filter->decay_peak[i], filter->decay_peak_base[i], - GST_TIME_ARGS (falloff_time), falloff_dB, falloff); - filter->decay_peak[i] = filter->decay_peak_base[i] * falloff; - GST_LOG_OBJECT (filter, - "peak is %" GST_TIME_FORMAT " old, decayed with factor %e to %f", - GST_TIME_ARGS (filter->decay_peak_age[i]), falloff, - filter->decay_peak[i]); - } else { - GST_LOG_OBJECT (filter, "peak not old enough, not decaying"); - } + "[%d]: peak %f, last peak %f, decay peak %f, age %" GST_TIME_FORMAT, + i, filter->peak[i], filter->last_peak[i], filter->decay_peak[i], + GST_TIME_ARGS (filter->decay_peak_age[i])); + + /* update running peak */ + if (filter->peak[i] > filter->last_peak[i]) + filter->last_peak[i] = filter->peak[i]; + + /* make decay peak fall off if too old */ + falloff_time = + GST_CLOCK_DIFF (gst_gdouble_to_guint64 (filter->decay_peak_ttl), + filter->decay_peak_age[i]); + if (falloff_time > 0) { + gdouble falloff_dB; + gdouble falloff; + gdouble length; /* length of falloff time in seconds */ + + length = (gdouble) falloff_time / (gdouble) GST_SECOND; + falloff_dB = filter->decay_peak_falloff * length; + falloff = pow (10, falloff_dB / -20.0); + + GST_LOG_OBJECT (filter, + "falloff: current %f, base %f, interval %" GST_TIME_FORMAT + ", dB falloff %f, factor %e", + filter->decay_peak[i], filter->decay_peak_base[i], + GST_TIME_ARGS (falloff_time), falloff_dB, falloff); + filter->decay_peak[i] = filter->decay_peak_base[i] * falloff; + GST_LOG_OBJECT (filter, + "peak is %" GST_TIME_FORMAT " old, decayed with factor %e to %f", + GST_TIME_ARGS (filter->decay_peak_age[i]), falloff, + filter->decay_peak[i]); + } else { + GST_LOG_OBJECT (filter, "peak not old enough, not decaying"); + } - /* if the peak of this run is higher, the decay peak gets reset */ - if (filter->peak[i] >= filter->decay_peak[i]) { - GST_LOG_OBJECT (filter, "new peak, %f", filter->peak[i]); - filter->decay_peak[i] = filter->peak[i]; - filter->decay_peak_base[i] = filter->peak[i]; - filter->decay_peak_age[i] = G_GINT64_CONSTANT (0); + /* if the peak of this run is higher, the decay peak gets reset */ + if (filter->peak[i] >= filter->decay_peak[i]) { + GST_LOG_OBJECT (filter, "new peak, %f", filter->peak[i]); + filter->decay_peak[i] = filter->peak[i]; + filter->decay_peak_base[i] = filter->peak[i]; + filter->decay_peak_age[i] = G_GINT64_CONSTANT (0); + } } - } + in_data += ((block_int_size * bps) - bps); - if (G_UNLIKELY (!filter->num_frames)) { - /* remember start timestamp for message */ - filter->message_ts = GST_BUFFER_TIMESTAMP (in); - } - filter->num_frames += num_frames; + filter->num_frames += block_size; + num_frames -= block_size; - /* do we need to message ? */ - if (filter->num_frames >= filter->interval_frames) { - gst_level_post_message (filter); + /* do we need to message ? */ + if (filter->num_frames >= filter->interval_frames) { + gst_level_post_message (filter); + filter->message_ts += GST_FRAMES_TO_CLOCK_TIME (block_size, rate); + } } gst_buffer_unmap (in, &map); diff --git a/tests/check/elements/level.c b/tests/check/elements/level.c index ef1d9fe..4a3cb09 100644 --- a/tests/check/elements/level.c +++ b/tests/check/elements/level.c @@ -42,6 +42,7 @@ GstPad *mysrcpad, *mysinkpad; "rate = (int) [ 1, MAX ], " \ "channels = (int) [ 1, 8 ]" +/* we use rate = 1000 here for easy buffer size calculations */ #define LEVEL_CAPS_STRING \ "audio/x-raw, " \ "format = (string) "GST_AUDIO_NE(S16)", " \ @@ -66,6 +67,8 @@ static GstElement * setup_level (void) { GstElement *level; + GstCaps *caps; + GstSegment segment; GST_DEBUG ("setup_level"); level = gst_check_setup_element ("level"); @@ -74,6 +77,13 @@ setup_level (void) gst_pad_set_active (mysrcpad, TRUE); gst_pad_set_active (mysinkpad, TRUE); + caps = gst_caps_from_string (LEVEL_CAPS_STRING); + gst_pad_set_caps (mysrcpad, caps); + gst_caps_unref (caps); + + gst_segment_init (&segment, GST_FORMAT_TIME); + gst_pad_push_event (mysrcpad, gst_event_new_segment (&segment)); + return level; } @@ -89,21 +99,39 @@ cleanup_level (GstElement * level) gst_check_teardown_element (level); } +/* create a 0.1 sec buffer */ +static GstBuffer * +create_buffer (gint16 val_l, gint16 val_r) +{ + GstBuffer *buf = gst_buffer_new_and_alloc (400); + GstMapInfo map; + gint j; + gint16 *data; + + gst_buffer_map (buf, &map, GST_MAP_WRITE); + data = (gint16 *) map.data; + for (j = 0; j < 100; ++j) { + *(data++) = val_l; + *(data++) = val_r; + } + gst_buffer_unmap (buf, &map); + GST_BUFFER_TIMESTAMP (buf) = G_GUINT64_CONSTANT (0); + return buf; +} + GST_START_TEST (test_int16) { GstElement *level; GstBuffer *inbuffer, *outbuffer; GstBus *bus; - GstCaps *caps; GstMessage *message; const GstStructure *structure; - int i, j; - GstMapInfo map; - gint16 *data; + gint i, j; const GValue *list, *value; GstClockTime endtime; gdouble dB; + const gchar *fields[3] = { "rms", "peak", "decay" }; level = setup_level (); g_object_set (level, "message", TRUE, "interval", GST_SECOND / 10, NULL); @@ -113,17 +141,7 @@ GST_START_TEST (test_int16) "could not set to playing"); /* create a fake 0.1 sec buffer with a half-amplitude block signal */ - inbuffer = gst_buffer_new_and_alloc (400); - gst_buffer_map (inbuffer, &map, GST_MAP_WRITE); - data = (gint16 *) map.data; - for (j = 0; j < 200; ++j) { - *data = 16536; - ++data; - } - gst_buffer_unmap (inbuffer, &map); - caps = gst_caps_from_string (LEVEL_CAPS_STRING); - gst_pad_set_caps (mysrcpad, caps); - gst_caps_unref (caps); + inbuffer = create_buffer (16536, 16536); ASSERT_BUFFER_REFCOUNT (inbuffer, "inbuffer", 1); /* create a bus to get the level message on */ @@ -154,7 +172,6 @@ GST_START_TEST (test_int16) /* block wave of half amplitude has -5.94 dB for rms, peak and decay */ for (i = 0; i < 2; ++i) { - const gchar *fields[3] = { "rms", "peak", "decay" }; for (j = 0; j < 3; ++j) { GValueArray *arr; @@ -167,9 +184,6 @@ GST_START_TEST (test_int16) fail_if (dB > -5.9); } } - fail_unless_equals_int (g_list_length (buffers), 1); - fail_if ((outbuffer = (GstBuffer *) buffers->data) == NULL); - fail_unless (inbuffer == outbuffer); /* clean up */ /* flush current messages,and future state change messages */ @@ -197,12 +211,9 @@ GST_START_TEST (test_int16_panned) GstElement *level; GstBuffer *inbuffer, *outbuffer; GstBus *bus; - GstCaps *caps; GstMessage *message; const GstStructure *structure; - int j; - gint16 *data; - GstMapInfo map; + gint j; const GValue *list, *value; GstClockTime endtime; gdouble dB; @@ -216,19 +227,7 @@ GST_START_TEST (test_int16_panned) "could not set to playing"); /* create a fake 0.1 sec buffer with a half-amplitude block signal */ - inbuffer = gst_buffer_new_and_alloc (400); - gst_buffer_map (inbuffer, &map, GST_MAP_WRITE); - data = (gint16 *) map.data; - for (j = 0; j < 100; ++j) { - *data = 0; - ++data; - *data = 16536; - ++data; - } - gst_buffer_unmap (inbuffer, &map); - caps = gst_caps_from_string (LEVEL_CAPS_STRING); - gst_pad_set_caps (mysrcpad, caps); - gst_caps_unref (caps); + inbuffer = create_buffer (0, 16536); ASSERT_BUFFER_REFCOUNT (inbuffer, "inbuffer", 1); /* create a bus to get the level message on */ @@ -284,9 +283,6 @@ GST_START_TEST (test_int16_panned) fail_if (dB < -6.0); fail_if (dB > -5.9); } - fail_unless_equals_int (g_list_length (buffers), 1); - fail_if ((outbuffer = (GstBuffer *) buffers->data) == NULL); - fail_unless (inbuffer == outbuffer); /* clean up */ /* flush current messages,and future state change messages */ @@ -300,6 +296,7 @@ GST_START_TEST (test_int16_panned) gst_element_set_bus (level, NULL); ASSERT_OBJECT_REFCOUNT (bus, "bus", 1); gst_object_unref (bus); + ASSERT_BUFFER_REFCOUNT (outbuffer, "outbuffer", 1); gst_buffer_unref (outbuffer); fail_unless (gst_element_set_state (level, GST_STATE_NULL) == GST_STATE_CHANGE_SUCCESS, "could not set to null"); @@ -315,12 +312,9 @@ GST_START_TEST (test_message_on_eos) GstBuffer *inbuffer, *outbuffer; GstEvent *event; GstBus *bus; - GstCaps *caps; GstMessage *message; const GstStructure *structure; - int i, j; - GstMapInfo map; - gint16 *data; + gint i, j; const GValue *list, *value; GstClockTime endtime; gdouble dB; @@ -333,17 +327,7 @@ GST_START_TEST (test_message_on_eos) "could not set to playing"); /* create a fake 0.1 sec buffer with a half-amplitude block signal */ - inbuffer = gst_buffer_new_and_alloc (400); - gst_buffer_map (inbuffer, &map, GST_MAP_WRITE); - data = (gint16 *) map.data; - for (j = 0; j < 200; ++j) { - *data = 16536; - ++data; - } - gst_buffer_unmap (inbuffer, &map); - caps = gst_caps_from_string (LEVEL_CAPS_STRING); - gst_pad_set_caps (mysrcpad, caps); - gst_caps_unref (caps); + inbuffer = create_buffer (16536, 16536); ASSERT_BUFFER_REFCOUNT (inbuffer, "inbuffer", 1); /* create a bus to get the level message on */ @@ -395,9 +379,6 @@ GST_START_TEST (test_message_on_eos) fail_if (dB > -5.9); } } - fail_unless_equals_int (g_list_length (buffers), 1); - fail_if ((outbuffer = (GstBuffer *) buffers->data) == NULL); - fail_unless (inbuffer == outbuffer); /* clean up */ /* flush current messages,and future state change messages */ @@ -420,6 +401,54 @@ GST_START_TEST (test_message_on_eos) GST_END_TEST; +GST_START_TEST (test_message_interval) +{ + GstElement *level; + GstBuffer *inbuffer, *outbuffer; + GstBus *bus; + GstMessage *message; + + level = setup_level (); + g_object_set (level, "message", TRUE, "interval", GST_SECOND / 20, NULL); + + fail_unless (gst_element_set_state (level, + GST_STATE_PLAYING) == GST_STATE_CHANGE_SUCCESS, + "could not set to playing"); + + /* create a fake 0.1 sec buffer with a half-amplitude block signal */ + inbuffer = create_buffer (16536, 16536); + + /* create a bus to get the level message on */ + bus = gst_bus_new (); + gst_element_set_bus (level, bus); + + /* pushing gives away my reference ... */ + fail_unless (gst_pad_push (mysrcpad, inbuffer) == GST_FLOW_OK); + /* ... but it ends up being collected on the global buffer list */ + fail_unless_equals_int (g_list_length (buffers), 1); + fail_if ((outbuffer = (GstBuffer *) buffers->data) == NULL); + fail_unless (inbuffer == outbuffer); + + /* we should get two messages per buffer */ + message = gst_bus_poll (bus, GST_MESSAGE_ELEMENT, -1); + fail_unless (message != NULL); + gst_message_unref (message); + message = gst_bus_poll (bus, GST_MESSAGE_ELEMENT, -1); + fail_unless (message != NULL); + gst_message_unref (message); + + gst_element_set_bus (level, NULL); + ASSERT_OBJECT_REFCOUNT (bus, "bus", 1); + gst_object_unref (bus); + gst_buffer_unref (outbuffer); + fail_unless (gst_element_set_state (level, + GST_STATE_NULL) == GST_STATE_CHANGE_SUCCESS, "could not set to null"); + ASSERT_OBJECT_REFCOUNT (level, "level", 1); + cleanup_level (level); +} + +GST_END_TEST; + static Suite * level_suite (void) @@ -431,6 +460,7 @@ level_suite (void) tcase_add_test (tc_chain, test_int16); tcase_add_test (tc_chain, test_int16_panned); tcase_add_test (tc_chain, test_message_on_eos); + tcase_add_test (tc_chain, test_message_interval); return s; } -- 2.7.4