2 * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
11 #include "webrtc/modules/video_processing/main/source/deflickering.h"
16 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
17 #include "webrtc/system_wrappers/interface/logging.h"
18 #include "webrtc/system_wrappers/interface/sort.h"
22 // Detection constants
23 // (Q4) Maximum allowed deviation for detection.
24 enum { kFrequencyDeviation = 39 };
25 // (Q4) Minimum frequency that can be detected.
26 enum { kMinFrequencyToDetect = 32 };
27 // Number of flickers before we accept detection
28 enum { kNumFlickerBeforeDetect = 2 };
29 enum { kmean_valueScaling = 4 }; // (Q4) In power of 2
30 // Dead-zone region in terms of pixel values
31 enum { kZeroCrossingDeadzone = 10 };
32 // Deflickering constants.
33 // Compute the quantiles over 1 / DownsamplingFactor of the image.
34 enum { kDownsamplingFactor = 8 };
35 enum { kLog2OfDownsamplingFactor = 3 };
37 // To generate in Matlab:
38 // >> probUW16 = round(2^11 *
39 // [0.05,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.95,0.97]);
40 // >> fprintf('%d, ', probUW16)
41 // Resolution reduced to avoid overflow when multiplying with the
42 // (potentially) large number of pixels.
43 const uint16_t VPMDeflickering::prob_uw16_[kNumProbs] = {102, 205, 410, 614,
44 819, 1024, 1229, 1434, 1638, 1843, 1946, 1987}; // <Q11>
46 // To generate in Matlab:
47 // >> numQuants = 14; maxOnlyLength = 5;
48 // >> weightUW16 = round(2^15 *
49 // [linspace(0.5, 1.0, numQuants - maxOnlyLength)]);
50 // >> fprintf('%d, %d,\n ', weightUW16);
51 const uint16_t VPMDeflickering::weight_uw16_[kNumQuants - kMaxOnlyLength] =
52 {16384, 18432, 20480, 22528, 24576, 26624, 28672, 30720, 32768}; // <Q15>
54 VPMDeflickering::VPMDeflickering()
59 VPMDeflickering::~VPMDeflickering() {}
61 int32_t VPMDeflickering::ChangeUniqueId(const int32_t id) {
66 void VPMDeflickering::Reset() {
67 mean_buffer_length_ = 0;
71 memset(mean_buffer_, 0, sizeof(int32_t) * kMeanBufferLength);
72 memset(timestamp_buffer_, 0, sizeof(int32_t) * kMeanBufferLength);
74 // Initialize the history with a uniformly distributed histogram.
75 quant_hist_uw8_[0][0] = 0;
76 quant_hist_uw8_[0][kNumQuants - 1] = 255;
77 for (int32_t i = 0; i < kNumProbs; i++) {
78 quant_hist_uw8_[0][i + 1] = static_cast<uint8_t>((WEBRTC_SPL_UMUL_16_16(
79 prob_uw16_[i], 255) + (1 << 10)) >> 11); // Unsigned round. <Q0>
82 for (int32_t i = 1; i < kFrameHistory_size; i++) {
83 memcpy(quant_hist_uw8_[i], quant_hist_uw8_[0],
84 sizeof(uint8_t) * kNumQuants);
88 int32_t VPMDeflickering::ProcessFrame(I420VideoFrame* frame,
89 VideoProcessingModule::FrameStats* stats) {
91 uint32_t frame_memory;
92 uint8_t quant_uw8[kNumQuants];
93 uint8_t maxquant_uw8[kNumQuants];
94 uint8_t minquant_uw8[kNumQuants];
95 uint16_t target_quant_uw16[kNumQuants];
96 uint16_t increment_uw16;
101 int width = frame->width();
102 int height = frame->height();
104 if (frame->IsZeroSize()) {
105 return VPM_GENERAL_ERROR;
108 // Stricter height check due to subsampling size calculation below.
110 LOG(LS_ERROR) << "Invalid frame size.";
111 return VPM_GENERAL_ERROR;
114 if (!VideoProcessingModule::ValidFrameStats(*stats)) {
115 return VPM_GENERAL_ERROR;
118 if (PreDetection(frame->timestamp(), *stats) == -1) return VPM_GENERAL_ERROR;
121 int32_t det_flicker = DetectFlicker();
122 if (det_flicker < 0) {
123 return VPM_GENERAL_ERROR;
124 } else if (det_flicker != 1) {
128 // Size of luminance component.
129 const uint32_t y_size = height * width;
131 const uint32_t y_sub_size = width * (((height - 1) >>
132 kLog2OfDownsamplingFactor) + 1);
133 uint8_t* y_sorted = new uint8_t[y_sub_size];
134 uint32_t sort_row_idx = 0;
135 for (int i = 0; i < height; i += kDownsamplingFactor) {
136 memcpy(y_sorted + sort_row_idx * width,
137 frame->buffer(kYPlane) + i * width, width);
141 webrtc::Sort(y_sorted, y_sub_size, webrtc::TYPE_UWord8);
143 uint32_t prob_idx_uw32 = 0;
145 quant_uw8[kNumQuants - 1] = 255;
147 // Ensure we won't get an overflow below.
148 // In practice, the number of subsampled pixels will not become this large.
149 if (y_sub_size > (1 << 21) - 1) {
150 LOG(LS_ERROR) << "Subsampled number of pixels too large.";
154 for (int32_t i = 0; i < kNumProbs; i++) {
156 prob_idx_uw32 = WEBRTC_SPL_UMUL_32_16(y_sub_size, prob_uw16_[i]) >> 11;
157 quant_uw8[i + 1] = y_sorted[prob_idx_uw32];
163 // Shift history for new frame.
164 memmove(quant_hist_uw8_[1], quant_hist_uw8_[0],
165 (kFrameHistory_size - 1) * kNumQuants * sizeof(uint8_t));
166 // Store current frame in history.
167 memcpy(quant_hist_uw8_[0], quant_uw8, kNumQuants * sizeof(uint8_t));
169 // We use a frame memory equal to the ceiling of half the frame rate to
170 // ensure we capture an entire period of flicker.
171 frame_memory = (frame_rate_ + (1 << 5)) >> 5; // Unsigned ceiling. <Q0>
172 // frame_rate_ in Q4.
173 if (frame_memory > kFrameHistory_size) {
174 frame_memory = kFrameHistory_size;
177 // Get maximum and minimum.
178 for (int32_t i = 0; i < kNumQuants; i++) {
180 minquant_uw8[i] = 255;
181 for (uint32_t j = 0; j < frame_memory; j++) {
182 if (quant_hist_uw8_[j][i] > maxquant_uw8[i]) {
183 maxquant_uw8[i] = quant_hist_uw8_[j][i];
186 if (quant_hist_uw8_[j][i] < minquant_uw8[i]) {
187 minquant_uw8[i] = quant_hist_uw8_[j][i];
192 // Get target quantiles.
193 for (int32_t i = 0; i < kNumQuants - kMaxOnlyLength; i++) {
194 target_quant_uw16[i] = static_cast<uint16_t>((WEBRTC_SPL_UMUL_16_16(
195 weight_uw16_[i], maxquant_uw8[i]) + WEBRTC_SPL_UMUL_16_16((1 << 15) -
196 weight_uw16_[i], minquant_uw8[i])) >> 8); // <Q7>
199 for (int32_t i = kNumQuants - kMaxOnlyLength; i < kNumQuants; i++) {
200 target_quant_uw16[i] = ((uint16_t)maxquant_uw8[i]) << 7;
203 // Compute the map from input to output pixels.
204 uint16_t mapUW16; // <Q7>
205 for (int32_t i = 1; i < kNumQuants; i++) {
206 // As quant and targetQuant are limited to UWord8, it's safe to use Q7 here.
207 tmp_uw32 = static_cast<uint32_t>(target_quant_uw16[i] -
208 target_quant_uw16[i - 1]);
209 tmp_uw16 = static_cast<uint16_t>(quant_uw8[i] - quant_uw8[i - 1]); // <Q0>
212 increment_uw16 = static_cast<uint16_t>(WebRtcSpl_DivU32U16(tmp_uw32,
215 // The value is irrelevant; the loop below will only iterate once.
219 mapUW16 = target_quant_uw16[i - 1];
220 for (uint32_t j = quant_uw8[i - 1]; j < (uint32_t)(quant_uw8[i] + 1); j++) {
221 // Unsigned round. <Q0>
222 map_uw8[j] = (uint8_t)((mapUW16 + (1 << 6)) >> 7);
223 mapUW16 += increment_uw16;
227 // Map to the output frame.
228 uint8_t* buffer = frame->buffer(kYPlane);
229 for (uint32_t i = 0; i < y_size; i++) {
230 buffer[i] = map_uw8[buffer[i]];
233 // Frame was altered, so reset stats.
234 VideoProcessingModule::ClearFrameStats(stats);
240 Performs some pre-detection operations. Must be called before
243 \param[in] timestamp Timestamp of the current frame.
244 \param[in] stats Statistics of the current frame.
247 2: Detection not possible due to flickering frequency too close to
251 int32_t VPMDeflickering::PreDetection(const uint32_t timestamp,
252 const VideoProcessingModule::FrameStats& stats) {
253 int32_t mean_val; // Mean value of frame (Q4)
254 uint32_t frame_rate = 0;
255 int32_t meanBufferLength; // Temp variable.
257 mean_val = ((stats.sum << kmean_valueScaling) / stats.num_pixels);
258 // Update mean value buffer.
259 // This should be done even though we might end up in an unreliable detection.
260 memmove(mean_buffer_ + 1, mean_buffer_,
261 (kMeanBufferLength - 1) * sizeof(int32_t));
262 mean_buffer_[0] = mean_val;
264 // Update timestamp buffer.
265 // This should be done even though we might end up in an unreliable detection.
266 memmove(timestamp_buffer_ + 1, timestamp_buffer_, (kMeanBufferLength - 1) *
268 timestamp_buffer_[0] = timestamp;
270 /* Compute current frame rate (Q4) */
271 if (timestamp_buffer_[kMeanBufferLength - 1] != 0) {
272 frame_rate = ((90000 << 4) * (kMeanBufferLength - 1));
274 (timestamp_buffer_[0] - timestamp_buffer_[kMeanBufferLength - 1]);
275 } else if (timestamp_buffer_[1] != 0) {
276 frame_rate = (90000 << 4) / (timestamp_buffer_[0] - timestamp_buffer_[1]);
279 /* Determine required size of mean value buffer (mean_buffer_length_) */
280 if (frame_rate == 0) {
281 meanBufferLength = 1;
284 (kNumFlickerBeforeDetect * frame_rate) / kMinFrequencyToDetect;
286 /* Sanity check of buffer length */
287 if (meanBufferLength >= kMeanBufferLength) {
288 /* Too long buffer. The flickering frequency is too close to zero, which
289 * makes the estimation unreliable.
291 mean_buffer_length_ = 0;
294 mean_buffer_length_ = meanBufferLength;
296 if ((timestamp_buffer_[mean_buffer_length_ - 1] != 0) &&
297 (mean_buffer_length_ != 1)) {
298 frame_rate = ((90000 << 4) * (mean_buffer_length_ - 1));
300 (timestamp_buffer_[0] - timestamp_buffer_[mean_buffer_length_ - 1]);
301 } else if (timestamp_buffer_[1] != 0) {
302 frame_rate = (90000 << 4) / (timestamp_buffer_[0] - timestamp_buffer_[1]);
304 frame_rate_ = frame_rate;
310 This function detects flicker in the video stream. As a side effect the
311 mean value buffer is updated with the new mean value.
313 \return 0: No flickering detected\n
314 1: Flickering detected\n
315 2: Detection not possible due to unreliable frequency interval
318 int32_t VPMDeflickering::DetectFlicker() {
320 int32_t freqEst; // (Q4) Frequency estimate to base detection upon
321 int32_t ret_val = -1;
323 /* Sanity check for mean_buffer_length_ */
324 if (mean_buffer_length_ < 2) {
325 /* Not possible to estimate frequency */
328 // Count zero crossings with a dead zone to be robust against noise. If the
329 // noise std is 2 pixel this corresponds to about 95% confidence interval.
330 int32_t deadzone = (kZeroCrossingDeadzone << kmean_valueScaling); // Q4
331 int32_t meanOfBuffer = 0; // Mean value of mean value buffer.
332 int32_t numZeros = 0; // Number of zeros that cross the dead-zone.
333 int32_t cntState = 0; // State variable for zero crossing regions.
334 int32_t cntStateOld = 0; // Previous state for zero crossing regions.
336 for (i = 0; i < mean_buffer_length_; i++) {
337 meanOfBuffer += mean_buffer_[i];
339 meanOfBuffer += (mean_buffer_length_ >> 1); // Rounding, not truncation.
340 meanOfBuffer /= mean_buffer_length_;
342 // Count zero crossings.
343 cntStateOld = (mean_buffer_[0] >= (meanOfBuffer + deadzone));
344 cntStateOld -= (mean_buffer_[0] <= (meanOfBuffer - deadzone));
345 for (i = 1; i < mean_buffer_length_; i++) {
346 cntState = (mean_buffer_[i] >= (meanOfBuffer + deadzone));
347 cntState -= (mean_buffer_[i] <= (meanOfBuffer - deadzone));
348 if (cntStateOld == 0) {
349 cntStateOld = -cntState;
351 if (((cntState + cntStateOld) == 0) && (cntState != 0)) {
353 cntStateOld = cntState;
356 // END count zero crossings.
358 /* Frequency estimation according to:
359 * freqEst = numZeros * frame_rate / 2 / mean_buffer_length_;
361 * Resolution is set to Q4
363 freqEst = ((numZeros * 90000) << 3);
365 (timestamp_buffer_[0] - timestamp_buffer_[mean_buffer_length_ - 1]);
367 /* Translate frequency estimate to regions close to 100 and 120 Hz */
368 uint8_t freqState = 0; // Current translation state;
369 // (0) Not in interval,
370 // (1) Within valid interval,
372 int32_t freqAlias = freqEst;
373 if (freqEst > kMinFrequencyToDetect) {
374 uint8_t aliasState = 1;
375 while(freqState == 0) {
376 /* Increase frequency */
377 freqAlias += (aliasState * frame_rate_);
378 freqAlias += ((freqEst << 1) * (1 - (aliasState << 1)));
380 freqState = (abs(freqAlias - (100 << 4)) <= kFrequencyDeviation);
381 freqState += (abs(freqAlias - (120 << 4)) <= kFrequencyDeviation);
382 freqState += 2 * (freqAlias > ((120 << 4) + kFrequencyDeviation));
383 /* Switch alias state */
388 /* Is frequency estimate within detection region? */
389 if (freqState == 1) {
391 } else if (freqState == 0) {
399 } // namespace webrtc