src/third_party/webrtc/video_engine/overuse_frame_detector.cc

   1 /*
   2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include "webrtc/video_engine/overuse_frame_detector.h"
  12
  13 #include <assert.h>
  14 #include <math.h>
  15
  16 #include <algorithm>
  17 #include <list>
  18
  19 #include "webrtc/modules/video_coding/utility/include/exp_filter.h"
  20 #include "webrtc/system_wrappers/interface/clock.h"
  21 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
  22 #include "webrtc/system_wrappers/interface/logging.h"
  23
  24 namespace webrtc {
  25
  26 // TODO(mflodman) Test different values for all of these to trigger correctly,
  27 // avoid fluctuations etc.
  28 namespace {
  29 const int64_t kProcessIntervalMs = 5000;
  30
  31 // Weight factor to apply to the standard deviation.
  32 const float kWeightFactor = 0.997f;
  33 // Weight factor to apply to the average.
  34 const float kWeightFactorMean = 0.98f;
  35
  36 // Delay between consecutive rampups. (Used for quick recovery.)
  37 const int kQuickRampUpDelayMs = 10 * 1000;
  38 // Delay between rampup attempts. Initially uses standard, scales up to max.
  39 const int kStandardRampUpDelayMs = 30 * 1000;
  40 const int kMaxRampUpDelayMs = 120 * 1000;
  41 // Expontential back-off factor, to prevent annoying up-down behaviour.
  42 const double kRampUpBackoffFactor = 2.0;
  43
  44 // The initial average encode time (set to a fairly small value).
  45 const float kInitialAvgEncodeTimeMs = 5.0f;
  46
  47 // The maximum exponent to use in VCMExpFilter.
  48 const float kSampleDiffMs = 33.0f;
  49 const float kMaxExp = 7.0f;
  50
  51 }  // namespace
  52
  53 Statistics::Statistics() :
  54     sum_(0.0),
  55     count_(0),
  56     filtered_samples_(new VCMExpFilter(kWeightFactorMean)),
  57     filtered_variance_(new VCMExpFilter(kWeightFactor)) {
  58   Reset();
  59 }
  60
  61 void Statistics::SetOptions(const CpuOveruseOptions& options) {
  62   options_ = options;
  63 }
  64
  65 void Statistics::Reset() {
  66   sum_ =  0.0;
  67   count_ = 0;
  68   filtered_variance_->Reset(kWeightFactor);
  69   filtered_variance_->Apply(1.0f, InitialVariance());
  70 }
  71
  72 void Statistics::AddSample(float sample_ms) {
  73   sum_ += sample_ms;
  74   ++count_;
  75
  76   if (count_ < static_cast<uint32_t>(options_.min_frame_samples)) {
  77     // Initialize filtered samples.
  78     filtered_samples_->Reset(kWeightFactorMean);
  79     filtered_samples_->Apply(1.0f, InitialMean());
  80     return;
  81   }
  82
  83   float exp = sample_ms / kSampleDiffMs;
  84   exp = std::min(exp, kMaxExp);
  85   filtered_samples_->Apply(exp, sample_ms);
  86   filtered_variance_->Apply(exp, (sample_ms - filtered_samples_->Value()) *
  87                                  (sample_ms - filtered_samples_->Value()));
  88 }
  89
  90 float Statistics::InitialMean() const {
  91   if (count_ == 0)
  92     return 0.0;
  93   return sum_ / count_;
  94 }
  95
  96 float Statistics::InitialVariance() const {
  97   // Start in between the underuse and overuse threshold.
  98   float average_stddev = (options_.low_capture_jitter_threshold_ms +
  99                           options_.high_capture_jitter_threshold_ms) / 2.0f;
 100   return average_stddev * average_stddev;
 101 }
 102
 103 float Statistics::Mean() const { return filtered_samples_->Value(); }
 104
 105 float Statistics::StdDev() const {
 106   return sqrt(std::max(filtered_variance_->Value(), 0.0f));
 107 }
 108
 109 uint64_t Statistics::Count() const { return count_; }
 110
 111
 112 // Class for calculating the average encode time.
 113 class OveruseFrameDetector::EncodeTimeAvg {
 114  public:
 115   EncodeTimeAvg()
 116       : kWeightFactor(0.5f),
 117         filtered_encode_time_ms_(new VCMExpFilter(kWeightFactor)) {
 118     filtered_encode_time_ms_->Apply(1.0f, kInitialAvgEncodeTimeMs);
 119   }
 120   ~EncodeTimeAvg() {}
 121
 122   void AddEncodeSample(float encode_time_ms, int64_t diff_last_sample_ms) {
 123     float exp =  diff_last_sample_ms / kSampleDiffMs;
 124     exp = std::min(exp, kMaxExp);
 125     filtered_encode_time_ms_->Apply(exp, encode_time_ms);
 126   }
 127
 128   int filtered_encode_time_ms() const {
 129     return static_cast<int>(filtered_encode_time_ms_->Value() + 0.5);
 130   }
 131
 132  private:
 133   const float kWeightFactor;
 134   scoped_ptr<VCMExpFilter> filtered_encode_time_ms_;
 135 };
 136
 137 // Class for calculating the encode usage.
 138 class OveruseFrameDetector::EncodeUsage {
 139  public:
 140   EncodeUsage()
 141       : kWeightFactorFrameDiff(0.998f),
 142         kWeightFactorEncodeTime(0.995f),
 143         kInitialSampleDiffMs(50.0f),
 144         kMaxSampleDiffMs(66.0f),
 145         count_(0),
 146         filtered_encode_time_ms_(new VCMExpFilter(kWeightFactorEncodeTime)),
 147         filtered_frame_diff_ms_(new VCMExpFilter(kWeightFactorFrameDiff)) {
 148     Reset();
 149   }
 150   ~EncodeUsage() {}
 151
 152   void SetOptions(const CpuOveruseOptions& options) {
 153     options_ = options;
 154   }
 155
 156   void Reset() {
 157     count_ = 0;
 158     filtered_frame_diff_ms_->Reset(kWeightFactorFrameDiff);
 159     filtered_frame_diff_ms_->Apply(1.0f, kInitialSampleDiffMs);
 160     filtered_encode_time_ms_->Reset(kWeightFactorEncodeTime);
 161     filtered_encode_time_ms_->Apply(1.0f, InitialEncodeTimeMs());
 162   }
 163
 164   void AddSample(float sample_ms) {
 165     float exp = sample_ms / kSampleDiffMs;
 166     exp = std::min(exp, kMaxExp);
 167     filtered_frame_diff_ms_->Apply(exp, sample_ms);
 168   }
 169
 170   void AddEncodeSample(float encode_time_ms, int64_t diff_last_sample_ms) {
 171     ++count_;
 172     float exp = diff_last_sample_ms / kSampleDiffMs;
 173     exp = std::min(exp, kMaxExp);
 174     filtered_encode_time_ms_->Apply(exp, encode_time_ms);
 175   }
 176
 177   int UsageInPercent() const {
 178     if (count_ < static_cast<uint32_t>(options_.min_frame_samples)) {
 179       return static_cast<int>(InitialUsageInPercent() + 0.5f);
 180     }
 181     float frame_diff_ms = std::max(filtered_frame_diff_ms_->Value(), 1.0f);
 182     frame_diff_ms = std::min(frame_diff_ms, kMaxSampleDiffMs);
 183     float encode_usage_percent =
 184         100.0f * filtered_encode_time_ms_->Value() / frame_diff_ms;
 185     return static_cast<int>(encode_usage_percent + 0.5);
 186   }
 187
 188   float InitialUsageInPercent() const {
 189     // Start in between the underuse and overuse threshold.
 190     return (options_.low_encode_usage_threshold_percent +
 191             options_.high_encode_usage_threshold_percent) / 2.0f;
 192   }
 193
 194   float InitialEncodeTimeMs() const {
 195     return InitialUsageInPercent() * kInitialSampleDiffMs / 100;
 196   }
 197
 198  private:
 199   const float kWeightFactorFrameDiff;
 200   const float kWeightFactorEncodeTime;
 201   const float kInitialSampleDiffMs;
 202   const float kMaxSampleDiffMs;
 203   uint64_t count_;
 204   CpuOveruseOptions options_;
 205   scoped_ptr<VCMExpFilter> filtered_encode_time_ms_;
 206   scoped_ptr<VCMExpFilter> filtered_frame_diff_ms_;
 207 };
 208
 209 // Class for calculating the capture queue delay change.
 210 class OveruseFrameDetector::CaptureQueueDelay {
 211  public:
 212   CaptureQueueDelay()
 213       : kWeightFactor(0.5f),
 214         delay_ms_(0),
 215         filtered_delay_ms_per_s_(new VCMExpFilter(kWeightFactor)) {
 216     filtered_delay_ms_per_s_->Apply(1.0f, 0.0f);
 217   }
 218   ~CaptureQueueDelay() {}
 219
 220   void FrameCaptured(int64_t now) {
 221     const size_t kMaxSize = 200;
 222     if (frames_.size() > kMaxSize) {
 223       frames_.pop_front();
 224     }
 225     frames_.push_back(now);
 226   }
 227
 228   void FrameProcessingStarted(int64_t now) {
 229     if (frames_.empty()) {
 230       return;
 231     }
 232     delay_ms_ = now - frames_.front();
 233     frames_.pop_front();
 234   }
 235
 236   void CalculateDelayChange(int64_t diff_last_sample_ms) {
 237     if (diff_last_sample_ms <= 0) {
 238       return;
 239     }
 240     float exp = static_cast<float>(diff_last_sample_ms) / kProcessIntervalMs;
 241     exp = std::min(exp, kMaxExp);
 242     filtered_delay_ms_per_s_->Apply(exp,
 243                                     delay_ms_ * 1000.0f / diff_last_sample_ms);
 244     ClearFrames();
 245   }
 246
 247   void ClearFrames() {
 248     frames_.clear();
 249   }
 250
 251   int delay_ms() const {
 252     return delay_ms_;
 253   }
 254
 255   int filtered_delay_ms_per_s() const {
 256     return static_cast<int>(filtered_delay_ms_per_s_->Value() + 0.5);
 257   }
 258
 259  private:
 260   const float kWeightFactor;
 261   std::list<int64_t> frames_;
 262   int delay_ms_;
 263   scoped_ptr<VCMExpFilter> filtered_delay_ms_per_s_;
 264 };
 265
 266 OveruseFrameDetector::OveruseFrameDetector(Clock* clock)
 267     : crit_(CriticalSectionWrapper::CreateCriticalSection()),
 268       observer_(NULL),
 269       clock_(clock),
 270       next_process_time_(clock_->TimeInMilliseconds()),
 271       num_process_times_(0),
 272       last_capture_time_(0),
 273       last_overuse_time_(0),
 274       checks_above_threshold_(0),
 275       last_rampup_time_(0),
 276       in_quick_rampup_(false),
 277       current_rampup_delay_ms_(kStandardRampUpDelayMs),
 278       num_pixels_(0),
 279       last_encode_sample_ms_(0),
 280       encode_time_(new EncodeTimeAvg()),
 281       encode_usage_(new EncodeUsage()),
 282       capture_queue_delay_(new CaptureQueueDelay()) {
 283 }
 284
 285 OveruseFrameDetector::~OveruseFrameDetector() {
 286 }
 287
 288 void OveruseFrameDetector::SetObserver(CpuOveruseObserver* observer) {
 289   CriticalSectionScoped cs(crit_.get());
 290   observer_ = observer;
 291 }
 292
 293 void OveruseFrameDetector::SetOptions(const CpuOveruseOptions& options) {
 294   assert(options.min_frame_samples > 0);
 295   CriticalSectionScoped cs(crit_.get());
 296   if (options_.Equals(options)) {
 297     return;
 298   }
 299   options_ = options;
 300   capture_deltas_.SetOptions(options);
 301   encode_usage_->SetOptions(options);
 302   ResetAll(num_pixels_);
 303 }
 304
 305 int OveruseFrameDetector::CaptureJitterMs() const {
 306   CriticalSectionScoped cs(crit_.get());
 307   return static_cast<int>(capture_deltas_.StdDev() + 0.5);
 308 }
 309
 310 int OveruseFrameDetector::AvgEncodeTimeMs() const {
 311   CriticalSectionScoped cs(crit_.get());
 312   return encode_time_->filtered_encode_time_ms();
 313 }
 314
 315 int OveruseFrameDetector::EncodeUsagePercent() const {
 316   CriticalSectionScoped cs(crit_.get());
 317   return encode_usage_->UsageInPercent();
 318 }
 319
 320 int OveruseFrameDetector::AvgCaptureQueueDelayMsPerS() const {
 321   CriticalSectionScoped cs(crit_.get());
 322   return capture_queue_delay_->filtered_delay_ms_per_s();
 323 }
 324
 325 int OveruseFrameDetector::CaptureQueueDelayMsPerS() const {
 326   CriticalSectionScoped cs(crit_.get());
 327   return capture_queue_delay_->delay_ms();
 328 }
 329
 330 int32_t OveruseFrameDetector::TimeUntilNextProcess() {
 331   CriticalSectionScoped cs(crit_.get());
 332   return next_process_time_ - clock_->TimeInMilliseconds();
 333 }
 334
 335 bool OveruseFrameDetector::FrameSizeChanged(int num_pixels) const {
 336   if (num_pixels != num_pixels_) {
 337     return true;
 338   }
 339   return false;
 340 }
 341
 342 bool OveruseFrameDetector::FrameTimeoutDetected(int64_t now) const {
 343   if (last_capture_time_ == 0) {
 344     return false;
 345   }
 346   return (now - last_capture_time_) > options_.frame_timeout_interval_ms;
 347 }
 348
 349 void OveruseFrameDetector::ResetAll(int num_pixels) {
 350   num_pixels_ = num_pixels;
 351   capture_deltas_.Reset();
 352   encode_usage_->Reset();
 353   capture_queue_delay_->ClearFrames();
 354   last_capture_time_ = 0;
 355   num_process_times_ = 0;
 356 }
 357
 358 void OveruseFrameDetector::FrameCaptured(int width, int height) {
 359   CriticalSectionScoped cs(crit_.get());
 360
 361   int64_t now = clock_->TimeInMilliseconds();
 362   if (FrameSizeChanged(width * height) || FrameTimeoutDetected(now)) {
 363     ResetAll(width * height);
 364   }
 365
 366   if (last_capture_time_ != 0) {
 367     capture_deltas_.AddSample(now - last_capture_time_);
 368     encode_usage_->AddSample(now - last_capture_time_);
 369   }
 370   last_capture_time_ = now;
 371
 372   capture_queue_delay_->FrameCaptured(now);
 373 }
 374
 375 void OveruseFrameDetector::FrameProcessingStarted() {
 376   CriticalSectionScoped cs(crit_.get());
 377   capture_queue_delay_->FrameProcessingStarted(clock_->TimeInMilliseconds());
 378 }
 379
 380 void OveruseFrameDetector::FrameEncoded(int encode_time_ms) {
 381   CriticalSectionScoped cs(crit_.get());
 382   int64_t time = clock_->TimeInMilliseconds();
 383   if (last_encode_sample_ms_ != 0) {
 384     int64_t diff_ms = time - last_encode_sample_ms_;
 385     encode_time_->AddEncodeSample(encode_time_ms, diff_ms);
 386     encode_usage_->AddEncodeSample(encode_time_ms, diff_ms);
 387   }
 388   last_encode_sample_ms_ = time;
 389 }
 390
 391 int32_t OveruseFrameDetector::Process() {
 392   CriticalSectionScoped cs(crit_.get());
 393
 394   int64_t now = clock_->TimeInMilliseconds();
 395
 396   // Used to protect against Process() being called too often.
 397   if (now < next_process_time_)
 398     return 0;
 399
 400   int64_t diff_ms = now - next_process_time_ + kProcessIntervalMs;
 401   next_process_time_ = now + kProcessIntervalMs;
 402   ++num_process_times_;
 403
 404   capture_queue_delay_->CalculateDelayChange(diff_ms);
 405
 406   if (num_process_times_ <= options_.min_process_count) {
 407     return 0;
 408   }
 409
 410   if (IsOverusing()) {
 411     // If the last thing we did was going up, and now have to back down, we need
 412     // to check if this peak was short. If so we should back off to avoid going
 413     // back and forth between this load, the system doesn't seem to handle it.
 414     bool check_for_backoff = last_rampup_time_ > last_overuse_time_;
 415     if (check_for_backoff) {
 416       if (now - last_rampup_time_ < kStandardRampUpDelayMs) {
 417         // Going up was not ok for very long, back off.
 418         current_rampup_delay_ms_ *= kRampUpBackoffFactor;
 419         if (current_rampup_delay_ms_ > kMaxRampUpDelayMs)
 420           current_rampup_delay_ms_ = kMaxRampUpDelayMs;
 421       } else {
 422         // Not currently backing off, reset rampup delay.
 423         current_rampup_delay_ms_ = kStandardRampUpDelayMs;
 424       }
 425     }
 426
 427     last_overuse_time_ = now;
 428     in_quick_rampup_ = false;
 429     checks_above_threshold_ = 0;
 430
 431     if (observer_ != NULL)
 432       observer_->OveruseDetected();
 433   } else if (IsUnderusing(now)) {
 434     last_rampup_time_ = now;
 435     in_quick_rampup_ = true;
 436
 437     if (observer_ != NULL)
 438       observer_->NormalUsage();
 439   }
 440
 441   int rampup_delay =
 442       in_quick_rampup_ ? kQuickRampUpDelayMs : current_rampup_delay_ms_;
 443   LOG(LS_VERBOSE) << "Capture input stats: avg: " << capture_deltas_.Mean()
 444                   << " std_dev " << capture_deltas_.StdDev()
 445                   << " rampup delay " << rampup_delay
 446                   << " overuse >= " << options_.high_capture_jitter_threshold_ms
 447                   << " underuse < " << options_.low_capture_jitter_threshold_ms;
 448   return 0;
 449 }
 450
 451 bool OveruseFrameDetector::IsOverusing() {
 452   bool overusing = false;
 453   if (options_.enable_capture_jitter_method) {
 454     overusing = capture_deltas_.StdDev() >=
 455         options_.high_capture_jitter_threshold_ms;
 456   } else if (options_.enable_encode_usage_method) {
 457     overusing = encode_usage_->UsageInPercent() >=
 458         options_.high_encode_usage_threshold_percent;
 459   }
 460
 461   if (overusing) {
 462     ++checks_above_threshold_;
 463   } else {
 464     checks_above_threshold_ = 0;
 465   }
 466   return checks_above_threshold_ >= options_.high_threshold_consecutive_count;
 467 }
 468
 469 bool OveruseFrameDetector::IsUnderusing(int64_t time_now) {
 470   int delay = in_quick_rampup_ ? kQuickRampUpDelayMs : current_rampup_delay_ms_;
 471   if (time_now < last_rampup_time_ + delay)
 472     return false;
 473
 474   bool underusing = false;
 475   if (options_.enable_capture_jitter_method) {
 476     underusing = capture_deltas_.StdDev() <
 477         options_.low_capture_jitter_threshold_ms;
 478   } else if (options_.enable_encode_usage_method) {
 479     underusing = encode_usage_->UsageInPercent() <
 480         options_.low_encode_usage_threshold_percent;
 481   }
 482   return underusing;
 483 }
 484 }  // namespace webrtc