src/third_party/webrtc/modules/video_coding/main/source/media_optimization.cc

   1 /*
   2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include "webrtc/modules/video_coding/main/source/media_optimization.h"
  12
  13 #include "webrtc/modules/video_coding/main/source/content_metrics_processing.h"
  14 #include "webrtc/modules/video_coding/main/source/qm_select.h"
  15 #include "webrtc/modules/video_coding/utility/include/frame_dropper.h"
  16 #include "webrtc/system_wrappers/interface/clock.h"
  17 #include "webrtc/system_wrappers/interface/logging.h"
  18
  19 namespace webrtc {
  20 namespace media_optimization {
  21 namespace {
  22 void UpdateProtectionCallback(
  23     VCMProtectionMethod* selected_method,
  24     uint32_t* video_rate_bps,
  25     uint32_t* nack_overhead_rate_bps,
  26     uint32_t* fec_overhead_rate_bps,
  27     VCMProtectionCallback* video_protection_callback) {
  28   FecProtectionParams delta_fec_params;
  29   FecProtectionParams key_fec_params;
  30   // Get the FEC code rate for Key frames (set to 0 when NA).
  31   key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK();
  32
  33   // Get the FEC code rate for Delta frames (set to 0 when NA).
  34   delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD();
  35
  36   // Get the FEC-UEP protection status for Key frames: UEP on/off.
  37   key_fec_params.use_uep_protection = selected_method->RequiredUepProtectionK();
  38
  39   // Get the FEC-UEP protection status for Delta frames: UEP on/off.
  40   delta_fec_params.use_uep_protection =
  41       selected_method->RequiredUepProtectionD();
  42
  43   // The RTP module currently requires the same |max_fec_frames| for both
  44   // key and delta frames.
  45   delta_fec_params.max_fec_frames = selected_method->MaxFramesFec();
  46   key_fec_params.max_fec_frames = selected_method->MaxFramesFec();
  47
  48   // Set the FEC packet mask type. |kFecMaskBursty| is more effective for
  49   // consecutive losses and little/no packet re-ordering. As we currently
  50   // do not have feedback data on the degree of correlated losses and packet
  51   // re-ordering, we keep default setting to |kFecMaskRandom| for now.
  52   delta_fec_params.fec_mask_type = kFecMaskRandom;
  53   key_fec_params.fec_mask_type = kFecMaskRandom;
  54
  55   // TODO(Marco): Pass FEC protection values per layer.
  56   video_protection_callback->ProtectionRequest(&delta_fec_params,
  57                                                &key_fec_params,
  58                                                video_rate_bps,
  59                                                nack_overhead_rate_bps,
  60                                                fec_overhead_rate_bps);
  61 }
  62 }  // namespace
  63
  64 struct MediaOptimization::EncodedFrameSample {
  65   EncodedFrameSample(int size_bytes,
  66                      uint32_t timestamp,
  67                      int64_t time_complete_ms)
  68       : size_bytes(size_bytes),
  69         timestamp(timestamp),
  70         time_complete_ms(time_complete_ms) {}
  71
  72   uint32_t size_bytes;
  73   uint32_t timestamp;
  74   int64_t time_complete_ms;
  75 };
  76
  77 MediaOptimization::MediaOptimization(Clock* clock)
  78     : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
  79       clock_(clock),
  80       max_bit_rate_(0),
  81       send_codec_type_(kVideoCodecUnknown),
  82       codec_width_(0),
  83       codec_height_(0),
  84       user_frame_rate_(0),
  85       frame_dropper_(new FrameDropper),
  86       loss_prot_logic_(
  87           new VCMLossProtectionLogic(clock_->TimeInMilliseconds())),
  88       fraction_lost_(0),
  89       send_statistics_zero_encode_(0),
  90       max_payload_size_(1460),
  91       target_bit_rate_(0),
  92       incoming_frame_rate_(0),
  93       enable_qm_(false),
  94       encoded_frame_samples_(),
  95       avg_sent_bit_rate_bps_(0),
  96       avg_sent_framerate_(0),
  97       key_frame_cnt_(0),
  98       delta_frame_cnt_(0),
  99       content_(new VCMContentMetricsProcessing()),
 100       qm_resolution_(new VCMQmResolution()),
 101       last_qm_update_time_(0),
 102       last_change_time_(0),
 103       num_layers_(0),
 104       suspension_enabled_(false),
 105       video_suspended_(false),
 106       suspension_threshold_bps_(0),
 107       suspension_window_bps_(0) {
 108   memset(send_statistics_, 0, sizeof(send_statistics_));
 109   memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
 110 }
 111
 112 MediaOptimization::~MediaOptimization(void) {
 113   loss_prot_logic_->Release();
 114 }
 115
 116 void MediaOptimization::Reset() {
 117   CriticalSectionScoped lock(crit_sect_.get());
 118   SetEncodingDataInternal(
 119       kVideoCodecUnknown, 0, 0, 0, 0, 0, 0, max_payload_size_);
 120   memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
 121   incoming_frame_rate_ = 0.0;
 122   frame_dropper_->Reset();
 123   loss_prot_logic_->Reset(clock_->TimeInMilliseconds());
 124   frame_dropper_->SetRates(0, 0);
 125   content_->Reset();
 126   qm_resolution_->Reset();
 127   loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_);
 128   loss_prot_logic_->Reset(clock_->TimeInMilliseconds());
 129   send_statistics_zero_encode_ = 0;
 130   target_bit_rate_ = 0;
 131   codec_width_ = 0;
 132   codec_height_ = 0;
 133   user_frame_rate_ = 0;
 134   key_frame_cnt_ = 0;
 135   delta_frame_cnt_ = 0;
 136   last_qm_update_time_ = 0;
 137   last_change_time_ = 0;
 138   encoded_frame_samples_.clear();
 139   avg_sent_bit_rate_bps_ = 0;
 140   num_layers_ = 1;
 141 }
 142
 143 void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type,
 144                                         int32_t max_bit_rate,
 145                                         uint32_t frame_rate,
 146                                         uint32_t target_bitrate,
 147                                         uint16_t width,
 148                                         uint16_t height,
 149                                         int num_layers,
 150                                         int32_t mtu) {
 151   CriticalSectionScoped lock(crit_sect_.get());
 152   SetEncodingDataInternal(send_codec_type,
 153                           max_bit_rate,
 154                           frame_rate,
 155                           target_bitrate,
 156                           width,
 157                           height,
 158                           num_layers,
 159                           mtu);
 160 }
 161
 162 void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type,
 163                                                 int32_t max_bit_rate,
 164                                                 uint32_t frame_rate,
 165                                                 uint32_t target_bitrate,
 166                                                 uint16_t width,
 167                                                 uint16_t height,
 168                                                 int num_layers,
 169                                                 int32_t mtu) {
 170   // Everything codec specific should be reset here since this means the codec
 171   // has changed. If native dimension values have changed, then either user
 172   // initiated change, or QM initiated change. Will be able to determine only
 173   // after the processing of the first frame.
 174   last_change_time_ = clock_->TimeInMilliseconds();
 175   content_->Reset();
 176   content_->UpdateFrameRate(frame_rate);
 177
 178   max_bit_rate_ = max_bit_rate;
 179   send_codec_type_ = send_codec_type;
 180   target_bit_rate_ = target_bitrate;
 181   float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
 182   loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
 183   loss_prot_logic_->UpdateFrameRate(static_cast<float>(frame_rate));
 184   loss_prot_logic_->UpdateFrameSize(width, height);
 185   loss_prot_logic_->UpdateNumLayers(num_layers);
 186   frame_dropper_->Reset();
 187   frame_dropper_->SetRates(target_bitrate_kbps, static_cast<float>(frame_rate));
 188   user_frame_rate_ = static_cast<float>(frame_rate);
 189   codec_width_ = width;
 190   codec_height_ = height;
 191   num_layers_ = (num_layers <= 1) ? 1 : num_layers;  // Can also be zero.
 192   max_payload_size_ = mtu;
 193   qm_resolution_->Initialize(target_bitrate_kbps,
 194                              user_frame_rate_,
 195                              codec_width_,
 196                              codec_height_,
 197                              num_layers_);
 198 }
 199
 200 uint32_t MediaOptimization::SetTargetRates(
 201     uint32_t target_bitrate,
 202     uint8_t fraction_lost,
 203     uint32_t round_trip_time_ms,
 204     VCMProtectionCallback* protection_callback,
 205     VCMQMSettingsCallback* qmsettings_callback) {
 206   CriticalSectionScoped lock(crit_sect_.get());
 207   // TODO(holmer): Consider putting this threshold only on the video bitrate,
 208   // and not on protection.
 209   if (max_bit_rate_ > 0 &&
 210       target_bitrate > static_cast<uint32_t>(max_bit_rate_)) {
 211     target_bitrate = max_bit_rate_;
 212   }
 213   VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod();
 214   float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
 215   loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
 216   loss_prot_logic_->UpdateRtt(round_trip_time_ms);
 217   loss_prot_logic_->UpdateResidualPacketLoss(static_cast<float>(fraction_lost));
 218
 219   // Get frame rate for encoder: this is the actual/sent frame rate.
 220   float actual_frame_rate = SentFrameRateInternal();
 221
 222   // Sanity check.
 223   if (actual_frame_rate < 1.0) {
 224     actual_frame_rate = 1.0;
 225   }
 226
 227   // Update frame rate for the loss protection logic class: frame rate should
 228   // be the actual/sent rate.
 229   loss_prot_logic_->UpdateFrameRate(actual_frame_rate);
 230
 231   fraction_lost_ = fraction_lost;
 232
 233   // Returns the filtered packet loss, used for the protection setting.
 234   // The filtered loss may be the received loss (no filter), or some
 235   // filtered value (average or max window filter).
 236   // Use max window filter for now.
 237   FilterPacketLossMode filter_mode = kMaxFilter;
 238   uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss(
 239       clock_->TimeInMilliseconds(), filter_mode, fraction_lost);
 240
 241   // For now use the filtered loss for computing the robustness settings.
 242   loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc);
 243
 244   // Rate cost of the protection methods.
 245   uint32_t protection_overhead_bps = 0;
 246
 247   // Update protection settings, when applicable.
 248   float sent_video_rate_kbps = 0.0f;
 249   if (selected_method) {
 250     // Update protection method with content metrics.
 251     selected_method->UpdateContentMetrics(content_->ShortTermAvgData());
 252
 253     // Update method will compute the robustness settings for the given
 254     // protection method and the overhead cost
 255     // the protection method is set by the user via SetVideoProtection.
 256     loss_prot_logic_->UpdateMethod();
 257
 258     // Update protection callback with protection settings.
 259     uint32_t sent_video_rate_bps = 0;
 260     uint32_t sent_nack_rate_bps = 0;
 261     uint32_t sent_fec_rate_bps = 0;
 262     // Get the bit cost of protection method, based on the amount of
 263     // overhead data actually transmitted (including headers) the last
 264     // second.
 265     if (protection_callback) {
 266       UpdateProtectionCallback(selected_method,
 267                                &sent_video_rate_bps,
 268                                &sent_nack_rate_bps,
 269                                &sent_fec_rate_bps,
 270                                protection_callback);
 271     }
 272     uint32_t sent_total_rate_bps =
 273         sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps;
 274     // Estimate the overhead costs of the next second as staying the same
 275     // wrt the source bitrate.
 276     if (sent_total_rate_bps > 0) {
 277       protection_overhead_bps = static_cast<uint32_t>(
 278           target_bitrate *
 279               static_cast<double>(sent_nack_rate_bps + sent_fec_rate_bps) /
 280               sent_total_rate_bps +
 281           0.5);
 282     }
 283     // Cap the overhead estimate to 50%.
 284     if (protection_overhead_bps > target_bitrate / 2)
 285       protection_overhead_bps = target_bitrate / 2;
 286
 287     // Get the effective packet loss for encoder ER when applicable. Should be
 288     // passed to encoder via fraction_lost.
 289     packet_loss_enc = selected_method->RequiredPacketLossER();
 290     sent_video_rate_kbps = static_cast<float>(sent_video_rate_bps) / 1000.0f;
 291   }
 292
 293   // Source coding rate: total rate - protection overhead.
 294   target_bit_rate_ = target_bitrate - protection_overhead_bps;
 295
 296   // Update encoding rates following protection settings.
 297   float target_video_bitrate_kbps =
 298       static_cast<float>(target_bit_rate_) / 1000.0f;
 299   frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_);
 300
 301   if (enable_qm_ && qmsettings_callback) {
 302     // Update QM with rates.
 303     qm_resolution_->UpdateRates(target_video_bitrate_kbps,
 304                                 sent_video_rate_kbps,
 305                                 incoming_frame_rate_,
 306                                 fraction_lost_);
 307     // Check for QM selection.
 308     bool select_qm = CheckStatusForQMchange();
 309     if (select_qm) {
 310       SelectQuality(qmsettings_callback);
 311     }
 312     // Reset the short-term averaged content data.
 313     content_->ResetShortTermAvgData();
 314   }
 315
 316   CheckSuspendConditions();
 317
 318   return target_bit_rate_;
 319 }
 320
 321 void MediaOptimization::EnableProtectionMethod(bool enable,
 322                                                VCMProtectionMethodEnum method) {
 323   CriticalSectionScoped lock(crit_sect_.get());
 324   bool updated = false;
 325   if (enable) {
 326     updated = loss_prot_logic_->SetMethod(method);
 327   } else {
 328     loss_prot_logic_->RemoveMethod(method);
 329   }
 330   if (updated) {
 331     loss_prot_logic_->UpdateMethod();
 332   }
 333 }
 334
 335 uint32_t MediaOptimization::InputFrameRate() {
 336   CriticalSectionScoped lock(crit_sect_.get());
 337   return InputFrameRateInternal();
 338 }
 339
 340 uint32_t MediaOptimization::InputFrameRateInternal() {
 341   ProcessIncomingFrameRate(clock_->TimeInMilliseconds());
 342   return uint32_t(incoming_frame_rate_ + 0.5f);
 343 }
 344
 345 uint32_t MediaOptimization::SentFrameRate() {
 346   CriticalSectionScoped lock(crit_sect_.get());
 347   return SentFrameRateInternal();
 348 }
 349
 350 uint32_t MediaOptimization::SentFrameRateInternal() {
 351   PurgeOldFrameSamples(clock_->TimeInMilliseconds());
 352   UpdateSentFramerate();
 353   return avg_sent_framerate_;
 354 }
 355
 356 uint32_t MediaOptimization::SentBitRate() {
 357   CriticalSectionScoped lock(crit_sect_.get());
 358   const int64_t now_ms = clock_->TimeInMilliseconds();
 359   PurgeOldFrameSamples(now_ms);
 360   UpdateSentBitrate(now_ms);
 361   return avg_sent_bit_rate_bps_;
 362 }
 363
 364 VCMFrameCount MediaOptimization::SentFrameCount() {
 365   CriticalSectionScoped lock(crit_sect_.get());
 366   VCMFrameCount count;
 367   count.numDeltaFrames = delta_frame_cnt_;
 368   count.numKeyFrames = key_frame_cnt_;
 369   return count;
 370 }
 371
 372 int32_t MediaOptimization::UpdateWithEncodedData(int encoded_length,
 373                                                  uint32_t timestamp,
 374                                                  FrameType encoded_frame_type) {
 375   CriticalSectionScoped lock(crit_sect_.get());
 376   const int64_t now_ms = clock_->TimeInMilliseconds();
 377   PurgeOldFrameSamples(now_ms);
 378   if (encoded_frame_samples_.size() > 0 &&
 379       encoded_frame_samples_.back().timestamp == timestamp) {
 380     // Frames having the same timestamp are generated from the same input
 381     // frame. We don't want to double count them, but only increment the
 382     // size_bytes.
 383     encoded_frame_samples_.back().size_bytes += encoded_length;
 384     encoded_frame_samples_.back().time_complete_ms = now_ms;
 385   } else {
 386     encoded_frame_samples_.push_back(
 387         EncodedFrameSample(encoded_length, timestamp, now_ms));
 388   }
 389   UpdateSentBitrate(now_ms);
 390   UpdateSentFramerate();
 391   if (encoded_length > 0) {
 392     const bool delta_frame = (encoded_frame_type != kVideoFrameKey);
 393
 394     frame_dropper_->Fill(encoded_length, delta_frame);
 395     if (max_payload_size_ > 0 && encoded_length > 0) {
 396       const float min_packets_per_frame =
 397           encoded_length / static_cast<float>(max_payload_size_);
 398       if (delta_frame) {
 399         loss_prot_logic_->UpdatePacketsPerFrame(min_packets_per_frame,
 400                                                 clock_->TimeInMilliseconds());
 401       } else {
 402         loss_prot_logic_->UpdatePacketsPerFrameKey(
 403             min_packets_per_frame, clock_->TimeInMilliseconds());
 404       }
 405
 406       if (enable_qm_) {
 407         // Update quality select with encoded length.
 408         qm_resolution_->UpdateEncodedSize(encoded_length, encoded_frame_type);
 409       }
 410     }
 411     if (!delta_frame && encoded_length > 0) {
 412       loss_prot_logic_->UpdateKeyFrameSize(static_cast<float>(encoded_length));
 413     }
 414
 415     // Updating counters.
 416     if (delta_frame) {
 417       delta_frame_cnt_++;
 418     } else {
 419       key_frame_cnt_++;
 420     }
 421   }
 422
 423   return VCM_OK;
 424 }
 425
 426 void MediaOptimization::EnableQM(bool enable) {
 427   CriticalSectionScoped lock(crit_sect_.get());
 428   enable_qm_ = enable;
 429 }
 430
 431 void MediaOptimization::EnableFrameDropper(bool enable) {
 432   CriticalSectionScoped lock(crit_sect_.get());
 433   frame_dropper_->Enable(enable);
 434 }
 435
 436 void MediaOptimization::SuspendBelowMinBitrate(int threshold_bps,
 437                                                int window_bps) {
 438   CriticalSectionScoped lock(crit_sect_.get());
 439   assert(threshold_bps > 0 && window_bps >= 0);
 440   suspension_threshold_bps_ = threshold_bps;
 441   suspension_window_bps_ = window_bps;
 442   suspension_enabled_ = true;
 443   video_suspended_ = false;
 444 }
 445
 446 bool MediaOptimization::IsVideoSuspended() const {
 447   CriticalSectionScoped lock(crit_sect_.get());
 448   return video_suspended_;
 449 }
 450
 451 bool MediaOptimization::DropFrame() {
 452   CriticalSectionScoped lock(crit_sect_.get());
 453   UpdateIncomingFrameRate();
 454   // Leak appropriate number of bytes.
 455   frame_dropper_->Leak((uint32_t)(InputFrameRateInternal() + 0.5f));
 456   if (video_suspended_) {
 457     return true;  // Drop all frames when muted.
 458   }
 459   return frame_dropper_->DropFrame();
 460 }
 461
 462 void MediaOptimization::UpdateContentData(
 463     const VideoContentMetrics* content_metrics) {
 464   CriticalSectionScoped lock(crit_sect_.get());
 465   // Updating content metrics.
 466   if (content_metrics == NULL) {
 467     // Disable QM if metrics are NULL.
 468     enable_qm_ = false;
 469     qm_resolution_->Reset();
 470   } else {
 471     content_->UpdateContentData(content_metrics);
 472   }
 473 }
 474
 475 void MediaOptimization::UpdateIncomingFrameRate() {
 476   int64_t now = clock_->TimeInMilliseconds();
 477   if (incoming_frame_times_[0] == 0) {
 478     // No shifting if this is the first time.
 479   } else {
 480     // Shift all times one step.
 481     for (int32_t i = (kFrameCountHistorySize - 2); i >= 0; i--) {
 482       incoming_frame_times_[i + 1] = incoming_frame_times_[i];
 483     }
 484   }
 485   incoming_frame_times_[0] = now;
 486   ProcessIncomingFrameRate(now);
 487 }
 488
 489 int32_t MediaOptimization::SelectQuality(
 490     VCMQMSettingsCallback* video_qmsettings_callback) {
 491   // Reset quantities for QM select.
 492   qm_resolution_->ResetQM();
 493
 494   // Update QM will long-term averaged content metrics.
 495   qm_resolution_->UpdateContent(content_->LongTermAvgData());
 496
 497   // Select quality mode.
 498   VCMResolutionScale* qm = NULL;
 499   int32_t ret = qm_resolution_->SelectResolution(&qm);
 500   if (ret < 0) {
 501     return ret;
 502   }
 503
 504   // Check for updates to spatial/temporal modes.
 505   QMUpdate(qm, video_qmsettings_callback);
 506
 507   // Reset all the rate and related frame counters quantities.
 508   qm_resolution_->ResetRates();
 509
 510   // Reset counters.
 511   last_qm_update_time_ = clock_->TimeInMilliseconds();
 512
 513   // Reset content metrics.
 514   content_->Reset();
 515
 516   return VCM_OK;
 517 }
 518
 519 void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) {
 520   while (!encoded_frame_samples_.empty()) {
 521     if (now_ms - encoded_frame_samples_.front().time_complete_ms >
 522         kBitrateAverageWinMs) {
 523       encoded_frame_samples_.pop_front();
 524     } else {
 525       break;
 526     }
 527   }
 528 }
 529
 530 void MediaOptimization::UpdateSentBitrate(int64_t now_ms) {
 531   if (encoded_frame_samples_.empty()) {
 532     avg_sent_bit_rate_bps_ = 0;
 533     return;
 534   }
 535   int framesize_sum = 0;
 536   for (FrameSampleList::iterator it = encoded_frame_samples_.begin();
 537        it != encoded_frame_samples_.end();
 538        ++it) {
 539     framesize_sum += it->size_bytes;
 540   }
 541   float denom = static_cast<float>(
 542       now_ms - encoded_frame_samples_.front().time_complete_ms);
 543   if (denom >= 1.0f) {
 544     avg_sent_bit_rate_bps_ =
 545         static_cast<uint32_t>(framesize_sum * 8 * 1000 / denom + 0.5f);
 546   } else {
 547     avg_sent_bit_rate_bps_ = framesize_sum * 8;
 548   }
 549 }
 550
 551 void MediaOptimization::UpdateSentFramerate() {
 552   if (encoded_frame_samples_.size() <= 1) {
 553     avg_sent_framerate_ = encoded_frame_samples_.size();
 554     return;
 555   }
 556   int denom = encoded_frame_samples_.back().timestamp -
 557               encoded_frame_samples_.front().timestamp;
 558   if (denom > 0) {
 559     avg_sent_framerate_ =
 560         (90000 * (encoded_frame_samples_.size() - 1) + denom / 2) / denom;
 561   } else {
 562     avg_sent_framerate_ = encoded_frame_samples_.size();
 563   }
 564 }
 565
 566 bool MediaOptimization::QMUpdate(
 567     VCMResolutionScale* qm,
 568     VCMQMSettingsCallback* video_qmsettings_callback) {
 569   // Check for no change.
 570   if (!qm->change_resolution_spatial && !qm->change_resolution_temporal) {
 571     return false;
 572   }
 573
 574   // Check for change in frame rate.
 575   if (qm->change_resolution_temporal) {
 576     incoming_frame_rate_ = qm->frame_rate;
 577     // Reset frame rate estimate.
 578     memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
 579   }
 580
 581   // Check for change in frame size.
 582   if (qm->change_resolution_spatial) {
 583     codec_width_ = qm->codec_width;
 584     codec_height_ = qm->codec_height;
 585   }
 586
 587   LOG(LS_INFO) << "Media optimizer requests the video resolution to be changed "
 588                   "to " << qm->codec_width << "x" << qm->codec_height << "@"
 589                << qm->frame_rate;
 590
 591   // Update VPM with new target frame rate and frame size.
 592   // Note: use |qm->frame_rate| instead of |_incoming_frame_rate| for updating
 593   // target frame rate in VPM frame dropper. The quantity |_incoming_frame_rate|
 594   // will vary/fluctuate, and since we don't want to change the state of the
 595   // VPM frame dropper, unless a temporal action was selected, we use the
 596   // quantity |qm->frame_rate| for updating.
 597   video_qmsettings_callback->SetVideoQMSettings(
 598       qm->frame_rate, codec_width_, codec_height_);
 599   content_->UpdateFrameRate(qm->frame_rate);
 600   qm_resolution_->UpdateCodecParameters(
 601       qm->frame_rate, codec_width_, codec_height_);
 602   return true;
 603 }
 604
 605 // Check timing constraints and look for significant change in:
 606 // (1) scene content,
 607 // (2) target bit rate.
 608 bool MediaOptimization::CheckStatusForQMchange() {
 609   bool status = true;
 610
 611   // Check that we do not call QMSelect too often, and that we waited some time
 612   // (to sample the metrics) from the event last_change_time
 613   // last_change_time is the time where user changed the size/rate/frame rate
 614   // (via SetEncodingData).
 615   int64_t now = clock_->TimeInMilliseconds();
 616   if ((now - last_qm_update_time_) < kQmMinIntervalMs ||
 617       (now - last_change_time_) < kQmMinIntervalMs) {
 618     status = false;
 619   }
 620
 621   return status;
 622 }
 623
 624 // Allowing VCM to keep track of incoming frame rate.
 625 void MediaOptimization::ProcessIncomingFrameRate(int64_t now) {
 626   int32_t num = 0;
 627   int32_t nr_of_frames = 0;
 628   for (num = 1; num < (kFrameCountHistorySize - 1); ++num) {
 629     if (incoming_frame_times_[num] <= 0 ||
 630         // don't use data older than 2 s
 631         now - incoming_frame_times_[num] > kFrameHistoryWinMs) {
 632       break;
 633     } else {
 634       nr_of_frames++;
 635     }
 636   }
 637   if (num > 1) {
 638     const int64_t diff = now - incoming_frame_times_[num - 1];
 639     incoming_frame_rate_ = 1.0;
 640     if (diff > 0) {
 641       incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast<float>(diff);
 642     }
 643   }
 644 }
 645
 646 void MediaOptimization::CheckSuspendConditions() {
 647   // Check conditions for SuspendBelowMinBitrate. |target_bit_rate_| is in bps.
 648   if (suspension_enabled_) {
 649     if (!video_suspended_) {
 650       // Check if we just went below the threshold.
 651       if (target_bit_rate_ < suspension_threshold_bps_) {
 652         video_suspended_ = true;
 653       }
 654     } else {
 655       // Video is already suspended. Check if we just went over the threshold
 656       // with a margin.
 657       if (target_bit_rate_ >
 658           suspension_threshold_bps_ + suspension_window_bps_) {
 659         video_suspended_ = false;
 660       }
 661     }
 662   }
 663 }
 664
 665 }  // namespace media_optimization
 666 }  // namespace webrtc