src/third_party/webrtc/video_engine/vie_encoder.cc

   1 /*
   2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include "webrtc/video_engine/vie_encoder.h"
  12
  13 #include <assert.h>
  14
  15 #include <algorithm>
  16
  17 #include "webrtc/common_video/interface/video_image.h"
  18 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
  19 #include "webrtc/modules/pacing/include/paced_sender.h"
  20 #include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h"
  21 #include "webrtc/modules/utility/interface/process_thread.h"
  22 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
  23 #include "webrtc/modules/video_coding/main/interface/video_coding.h"
  24 #include "webrtc/modules/video_coding/main/interface/video_coding_defines.h"
  25 #include "webrtc/modules/video_coding/main/source/encoded_frame.h"
  26 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
  27 #include "webrtc/system_wrappers/interface/logging.h"
  28 #include "webrtc/system_wrappers/interface/tick_util.h"
  29 #include "webrtc/system_wrappers/interface/trace_event.h"
  30 #include "webrtc/video_engine/include/vie_codec.h"
  31 #include "webrtc/video_engine/include/vie_image_process.h"
  32 #include "webrtc/frame_callback.h"
  33 #include "webrtc/video_engine/vie_defines.h"
  34
  35 namespace webrtc {
  36
  37 // Pace in kbits/s until we receive first estimate.
  38 static const int kInitialPace = 2000;
  39
  40 // Pacing-rate relative to our target send rate.
  41 // Multiplicative factor that is applied to the target bitrate to calculate the
  42 // number of bytes that can be transmitted per interval.
  43 // Increasing this factor will result in lower delays in cases of bitrate
  44 // overshoots from the encoder.
  45 static const float kPaceMultiplier = 2.5f;
  46
  47 // Margin on when we pause the encoder when the pacing buffer overflows relative
  48 // to the configured buffer delay.
  49 static const float kEncoderPausePacerMargin = 2.0f;
  50
  51 // Don't stop the encoder unless the delay is above this configured value.
  52 static const int kMinPacingDelayMs = 200;
  53
  54 // Allow packets to be transmitted in up to 2 times max video bitrate if the
  55 // bandwidth estimate allows it.
  56 // TODO(holmer): Expose transmission start, min and max bitrates in the
  57 // VideoEngine API and remove the kTransmissionMaxBitrateMultiplier.
  58 static const int kTransmissionMaxBitrateMultiplier = 2;
  59
  60 static const float kStopPaddingThresholdMs = 2000;
  61
  62 std::vector<uint32_t> AllocateStreamBitrates(
  63     uint32_t total_bitrate,
  64     const SimulcastStream* stream_configs,
  65     size_t number_of_streams) {
  66   if (number_of_streams == 0) {
  67     std::vector<uint32_t> stream_bitrates(1, 0);
  68     stream_bitrates[0] = total_bitrate;
  69     return stream_bitrates;
  70   }
  71   std::vector<uint32_t> stream_bitrates(number_of_streams, 0);
  72   uint32_t bitrate_remainder = total_bitrate;
  73   for (size_t i = 0; i < stream_bitrates.size() && bitrate_remainder > 0; ++i) {
  74     if (stream_configs[i].maxBitrate * 1000 > bitrate_remainder) {
  75       stream_bitrates[i] = bitrate_remainder;
  76     } else {
  77       stream_bitrates[i] = stream_configs[i].maxBitrate * 1000;
  78     }
  79     bitrate_remainder -= stream_bitrates[i];
  80   }
  81   return stream_bitrates;
  82 }
  83
  84 class QMVideoSettingsCallback : public VCMQMSettingsCallback {
  85  public:
  86   explicit QMVideoSettingsCallback(VideoProcessingModule* vpm);
  87
  88   ~QMVideoSettingsCallback();
  89
  90   // Update VPM with QM (quality modes: frame size & frame rate) settings.
  91   int32_t SetVideoQMSettings(const uint32_t frame_rate,
  92                              const uint32_t width,
  93                              const uint32_t height);
  94
  95  private:
  96   VideoProcessingModule* vpm_;
  97 };
  98
  99 class ViEBitrateObserver : public BitrateObserver {
 100  public:
 101   explicit ViEBitrateObserver(ViEEncoder* owner)
 102       : owner_(owner) {
 103   }
 104   virtual ~ViEBitrateObserver() {}
 105   // Implements BitrateObserver.
 106   virtual void OnNetworkChanged(const uint32_t bitrate_bps,
 107                                 const uint8_t fraction_lost,
 108                                 const uint32_t rtt) {
 109     owner_->OnNetworkChanged(bitrate_bps, fraction_lost, rtt);
 110   }
 111  private:
 112   ViEEncoder* owner_;
 113 };
 114
 115 class ViEPacedSenderCallback : public PacedSender::Callback {
 116  public:
 117   explicit ViEPacedSenderCallback(ViEEncoder* owner)
 118       : owner_(owner) {
 119   }
 120   virtual ~ViEPacedSenderCallback() {}
 121   virtual bool TimeToSendPacket(uint32_t ssrc, uint16_t sequence_number,
 122                                 int64_t capture_time_ms, bool retransmission) {
 123     return owner_->TimeToSendPacket(ssrc, sequence_number, capture_time_ms,
 124                                     retransmission);
 125   }
 126   virtual int TimeToSendPadding(int bytes) {
 127     return owner_->TimeToSendPadding(bytes);
 128   }
 129  private:
 130   ViEEncoder* owner_;
 131 };
 132
 133 ViEEncoder::ViEEncoder(int32_t engine_id,
 134                        int32_t channel_id,
 135                        uint32_t number_of_cores,
 136                        const Config& config,
 137                        ProcessThread& module_process_thread,
 138                        BitrateController* bitrate_controller)
 139   : engine_id_(engine_id),
 140     channel_id_(channel_id),
 141     number_of_cores_(number_of_cores),
 142     vcm_(*webrtc::VideoCodingModule::Create()),
 143     vpm_(*webrtc::VideoProcessingModule::Create(ViEModuleId(engine_id,
 144                                                             channel_id))),
 145     callback_cs_(CriticalSectionWrapper::CreateCriticalSection()),
 146     data_cs_(CriticalSectionWrapper::CreateCriticalSection()),
 147     bitrate_controller_(bitrate_controller),
 148     time_of_last_incoming_frame_ms_(0),
 149     send_padding_(false),
 150     min_transmit_bitrate_kbps_(0),
 151     target_delay_ms_(0),
 152     network_is_transmitting_(true),
 153     encoder_paused_(false),
 154     encoder_paused_and_dropped_frame_(false),
 155     fec_enabled_(false),
 156     nack_enabled_(false),
 157     codec_observer_(NULL),
 158     effect_filter_(NULL),
 159     module_process_thread_(module_process_thread),
 160     has_received_sli_(false),
 161     picture_id_sli_(0),
 162     has_received_rpsi_(false),
 163     picture_id_rpsi_(0),
 164     qm_callback_(NULL),
 165     video_suspended_(false),
 166     pre_encode_callback_(NULL) {
 167   RtpRtcp::Configuration configuration;
 168   configuration.id = ViEModuleId(engine_id_, channel_id_);
 169   configuration.audio = false;  // Video.
 170
 171   default_rtp_rtcp_.reset(RtpRtcp::CreateRtpRtcp(configuration));
 172   bitrate_observer_.reset(new ViEBitrateObserver(this));
 173   pacing_callback_.reset(new ViEPacedSenderCallback(this));
 174   paced_sender_.reset(
 175       new PacedSender(pacing_callback_.get(), kInitialPace, kPaceMultiplier));
 176 }
 177
 178 bool ViEEncoder::Init() {
 179   if (vcm_.InitializeSender() != 0) {
 180     return false;
 181   }
 182   vpm_.EnableTemporalDecimation(true);
 183
 184   // Enable/disable content analysis: off by default for now.
 185   vpm_.EnableContentAnalysis(false);
 186
 187   if (module_process_thread_.RegisterModule(&vcm_) != 0 ||
 188       module_process_thread_.RegisterModule(default_rtp_rtcp_.get()) != 0 ||
 189       module_process_thread_.RegisterModule(paced_sender_.get()) != 0) {
 190     return false;
 191   }
 192   if (qm_callback_) {
 193     delete qm_callback_;
 194   }
 195   qm_callback_ = new QMVideoSettingsCallback(&vpm_);
 196
 197 #ifdef VIDEOCODEC_VP8
 198   VideoCodec video_codec;
 199   if (vcm_.Codec(webrtc::kVideoCodecVP8, &video_codec) != VCM_OK) {
 200     return false;
 201   }
 202   {
 203     CriticalSectionScoped cs(data_cs_.get());
 204     send_padding_ = video_codec.numberOfSimulcastStreams > 1;
 205   }
 206   if (vcm_.RegisterSendCodec(&video_codec, number_of_cores_,
 207                              default_rtp_rtcp_->MaxDataPayloadLength()) != 0) {
 208     return false;
 209   }
 210   if (default_rtp_rtcp_->RegisterSendPayload(video_codec) != 0) {
 211     return false;
 212   }
 213 #else
 214   VideoCodec video_codec;
 215   if (vcm_.Codec(webrtc::kVideoCodecI420, &video_codec) == VCM_OK) {
 216     {
 217       CriticalSectionScoped cs(data_cs_.get());
 218       send_padding_ = video_codec.numberOfSimulcastStreams > 1;
 219     }
 220     vcm_.RegisterSendCodec(&video_codec, number_of_cores_,
 221                            default_rtp_rtcp_->MaxDataPayloadLength());
 222     default_rtp_rtcp_->RegisterSendPayload(video_codec);
 223   } else {
 224     return false;
 225   }
 226 #endif
 227
 228   if (vcm_.RegisterTransportCallback(this) != 0) {
 229     return false;
 230   }
 231   if (vcm_.RegisterSendStatisticsCallback(this) != 0) {
 232     return false;
 233   }
 234   if (vcm_.RegisterVideoQMCallback(qm_callback_) != 0) {
 235     return false;
 236   }
 237   return true;
 238 }
 239
 240 ViEEncoder::~ViEEncoder() {
 241   if (bitrate_controller_) {
 242     bitrate_controller_->RemoveBitrateObserver(bitrate_observer_.get());
 243   }
 244   module_process_thread_.DeRegisterModule(&vcm_);
 245   module_process_thread_.DeRegisterModule(&vpm_);
 246   module_process_thread_.DeRegisterModule(default_rtp_rtcp_.get());
 247   module_process_thread_.DeRegisterModule(paced_sender_.get());
 248   VideoCodingModule::Destroy(&vcm_);
 249   VideoProcessingModule::Destroy(&vpm_);
 250   delete qm_callback_;
 251 }
 252
 253 int ViEEncoder::Owner() const {
 254   return channel_id_;
 255 }
 256
 257 void ViEEncoder::SetNetworkTransmissionState(bool is_transmitting) {
 258   {
 259     CriticalSectionScoped cs(data_cs_.get());
 260     network_is_transmitting_ = is_transmitting;
 261   }
 262   if (is_transmitting) {
 263     paced_sender_->Resume();
 264   } else {
 265     paced_sender_->Pause();
 266   }
 267 }
 268
 269 void ViEEncoder::Pause() {
 270   CriticalSectionScoped cs(data_cs_.get());
 271   encoder_paused_ = true;
 272 }
 273
 274 void ViEEncoder::Restart() {
 275   CriticalSectionScoped cs(data_cs_.get());
 276   encoder_paused_ = false;
 277 }
 278
 279 uint8_t ViEEncoder::NumberOfCodecs() {
 280   return vcm_.NumberOfCodecs();
 281 }
 282
 283 int32_t ViEEncoder::GetCodec(uint8_t list_index, VideoCodec* video_codec) {
 284   if (vcm_.Codec(list_index, video_codec) != 0) {
 285     return -1;
 286   }
 287   return 0;
 288 }
 289
 290 int32_t ViEEncoder::RegisterExternalEncoder(webrtc::VideoEncoder* encoder,
 291                                             uint8_t pl_type,
 292                                             bool internal_source) {
 293   if (encoder == NULL)
 294     return -1;
 295
 296   if (vcm_.RegisterExternalEncoder(encoder, pl_type, internal_source) !=
 297       VCM_OK) {
 298     return -1;
 299   }
 300   return 0;
 301 }
 302
 303 int32_t ViEEncoder::DeRegisterExternalEncoder(uint8_t pl_type) {
 304   webrtc::VideoCodec current_send_codec;
 305   if (vcm_.SendCodec(&current_send_codec) == VCM_OK) {
 306     uint32_t current_bitrate_bps = 0;
 307     if (vcm_.Bitrate(&current_bitrate_bps) != 0) {
 308       LOG(LS_WARNING) << "Failed to get the current encoder target bitrate.";
 309     }
 310     current_send_codec.startBitrate = (current_bitrate_bps + 500) / 1000;
 311   }
 312
 313   if (vcm_.RegisterExternalEncoder(NULL, pl_type) != VCM_OK) {
 314     return -1;
 315   }
 316
 317   // If the external encoder is the current send codec, use vcm internal
 318   // encoder.
 319   if (current_send_codec.plType == pl_type) {
 320     uint16_t max_data_payload_length =
 321         default_rtp_rtcp_->MaxDataPayloadLength();
 322     {
 323       CriticalSectionScoped cs(data_cs_.get());
 324       send_padding_ = current_send_codec.numberOfSimulcastStreams > 1;
 325     }
 326     // TODO(mflodman): Unfortunately the VideoCodec that VCM has cached a
 327     // raw pointer to an |extra_options| that's long gone.  Clearing it here is
 328     // a hack to prevent the following code from crashing.  This should be fixed
 329     // for realz.  https://code.google.com/p/chromium/issues/detail?id=348222
 330     current_send_codec.extra_options = NULL;
 331     if (vcm_.RegisterSendCodec(&current_send_codec, number_of_cores_,
 332                                max_data_payload_length) != VCM_OK) {
 333       return -1;
 334     }
 335   }
 336   return 0;
 337 }
 338
 339 int32_t ViEEncoder::SetEncoder(const webrtc::VideoCodec& video_codec) {
 340   // Setting target width and height for VPM.
 341   if (vpm_.SetTargetResolution(video_codec.width, video_codec.height,
 342                                video_codec.maxFramerate) != VPM_OK) {
 343     return -1;
 344   }
 345
 346   if (default_rtp_rtcp_->RegisterSendPayload(video_codec) != 0) {
 347     return -1;
 348   }
 349   // Convert from kbps to bps.
 350   std::vector<uint32_t> stream_bitrates = AllocateStreamBitrates(
 351       video_codec.startBitrate * 1000,
 352       video_codec.simulcastStream,
 353       video_codec.numberOfSimulcastStreams);
 354   default_rtp_rtcp_->SetTargetSendBitrate(stream_bitrates);
 355
 356   uint16_t max_data_payload_length =
 357       default_rtp_rtcp_->MaxDataPayloadLength();
 358
 359   {
 360     CriticalSectionScoped cs(data_cs_.get());
 361     send_padding_ = video_codec.numberOfSimulcastStreams > 1;
 362   }
 363   if (vcm_.RegisterSendCodec(&video_codec, number_of_cores_,
 364                              max_data_payload_length) != VCM_OK) {
 365     return -1;
 366   }
 367
 368   // Set this module as sending right away, let the slave module in the channel
 369   // start and stop sending.
 370   if (default_rtp_rtcp_->Sending() == false) {
 371     if (default_rtp_rtcp_->SetSendingStatus(true) != 0) {
 372       return -1;
 373     }
 374   }
 375   bitrate_controller_->SetBitrateObserver(bitrate_observer_.get(),
 376                                           video_codec.startBitrate * 1000,
 377                                           video_codec.minBitrate * 1000,
 378                                           kTransmissionMaxBitrateMultiplier *
 379                                           video_codec.maxBitrate * 1000);
 380
 381   CriticalSectionScoped crit(data_cs_.get());
 382   int pad_up_to_bitrate_kbps = video_codec.startBitrate;
 383   if (pad_up_to_bitrate_kbps < min_transmit_bitrate_kbps_)
 384     pad_up_to_bitrate_kbps = min_transmit_bitrate_kbps_;
 385
 386   paced_sender_->UpdateBitrate(kPaceMultiplier * video_codec.startBitrate,
 387                                pad_up_to_bitrate_kbps);
 388
 389   return 0;
 390 }
 391
 392 int32_t ViEEncoder::GetEncoder(VideoCodec* video_codec) {
 393   if (vcm_.SendCodec(video_codec) != 0) {
 394     return -1;
 395   }
 396   return 0;
 397 }
 398
 399 int32_t ViEEncoder::GetCodecConfigParameters(
 400     unsigned char config_parameters[kConfigParameterSize],
 401     unsigned char& config_parameters_size) {
 402   int32_t num_parameters =
 403       vcm_.CodecConfigParameters(config_parameters, kConfigParameterSize);
 404   if (num_parameters <= 0) {
 405     config_parameters_size = 0;
 406     return -1;
 407   }
 408   config_parameters_size = static_cast<unsigned char>(num_parameters);
 409   return 0;
 410 }
 411
 412 int32_t ViEEncoder::ScaleInputImage(bool enable) {
 413   VideoFrameResampling resampling_mode = kFastRescaling;
 414   // TODO(mflodman) What?
 415   if (enable) {
 416     // kInterpolation is currently not supported.
 417     LOG_F(LS_ERROR) << "Not supported.";
 418     return -1;
 419   }
 420   vpm_.SetInputFrameResampleMode(resampling_mode);
 421
 422   return 0;
 423 }
 424
 425 bool ViEEncoder::TimeToSendPacket(uint32_t ssrc,
 426                                   uint16_t sequence_number,
 427                                   int64_t capture_time_ms,
 428                                   bool retransmission) {
 429   return default_rtp_rtcp_->TimeToSendPacket(ssrc, sequence_number,
 430                                              capture_time_ms, retransmission);
 431 }
 432
 433 int ViEEncoder::TimeToSendPadding(int bytes) {
 434   bool send_padding;
 435   {
 436     CriticalSectionScoped cs(data_cs_.get());
 437     send_padding =
 438         send_padding_ || video_suspended_ || min_transmit_bitrate_kbps_ > 0;
 439   }
 440   if (send_padding) {
 441     return default_rtp_rtcp_->TimeToSendPadding(bytes);
 442   }
 443   return 0;
 444 }
 445
 446 bool ViEEncoder::EncoderPaused() const {
 447   // Pause video if paused by caller or as long as the network is down or the
 448   // pacer queue has grown too large in buffered mode.
 449   if (encoder_paused_) {
 450     return true;
 451   }
 452   if (target_delay_ms_ > 0) {
 453     // Buffered mode.
 454     // TODO(pwestin): Workaround until nack is configured as a time and not
 455     // number of packets.
 456     return paced_sender_->QueueInMs() >=
 457         std::max(static_cast<int>(target_delay_ms_ * kEncoderPausePacerMargin),
 458                  kMinPacingDelayMs);
 459   }
 460   return !network_is_transmitting_;
 461 }
 462
 463 RtpRtcp* ViEEncoder::SendRtpRtcpModule() {
 464   return default_rtp_rtcp_.get();
 465 }
 466
 467 void ViEEncoder::DeliverFrame(int id,
 468                               I420VideoFrame* video_frame,
 469                               int num_csrcs,
 470                               const uint32_t CSRC[kRtpCsrcSize]) {
 471   if (default_rtp_rtcp_->SendingMedia() == false) {
 472     // We've paused or we have no channels attached, don't encode.
 473     return;
 474   }
 475   {
 476     CriticalSectionScoped cs(data_cs_.get());
 477     time_of_last_incoming_frame_ms_ = TickTime::MillisecondTimestamp();
 478     if (EncoderPaused()) {
 479       if (!encoder_paused_and_dropped_frame_) {
 480         TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this);
 481       }
 482       encoder_paused_and_dropped_frame_ = true;
 483       return;
 484     }
 485     if (encoder_paused_and_dropped_frame_) {
 486       TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this);
 487     }
 488     encoder_paused_and_dropped_frame_ = false;
 489   }
 490
 491   // Convert render time, in ms, to RTP timestamp.
 492   const int kMsToRtpTimestamp = 90;
 493   const uint32_t time_stamp =
 494       kMsToRtpTimestamp *
 495       static_cast<uint32_t>(video_frame->render_time_ms());
 496
 497   TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame->render_time_ms(),
 498                           "Encode");
 499   video_frame->set_timestamp(time_stamp);
 500   {
 501     CriticalSectionScoped cs(callback_cs_.get());
 502     if (effect_filter_) {
 503       unsigned int length = CalcBufferSize(kI420,
 504                                            video_frame->width(),
 505                                            video_frame->height());
 506       scoped_ptr<uint8_t[]> video_buffer(new uint8_t[length]);
 507       ExtractBuffer(*video_frame, length, video_buffer.get());
 508       effect_filter_->Transform(length,
 509                                 video_buffer.get(),
 510                                 video_frame->ntp_time_ms(),
 511                                 video_frame->timestamp(),
 512                                 video_frame->width(),
 513                                 video_frame->height());
 514     }
 515   }
 516
 517   // Make sure the CSRC list is correct.
 518   if (num_csrcs > 0) {
 519     uint32_t tempCSRC[kRtpCsrcSize];
 520     for (int i = 0; i < num_csrcs; i++) {
 521       if (CSRC[i] == 1) {
 522         tempCSRC[i] = default_rtp_rtcp_->SSRC();
 523       } else {
 524         tempCSRC[i] = CSRC[i];
 525       }
 526     }
 527     default_rtp_rtcp_->SetCSRCs(tempCSRC, (uint8_t) num_csrcs);
 528   }
 529   // Pass frame via preprocessor.
 530   I420VideoFrame* decimated_frame = NULL;
 531   const int ret = vpm_.PreprocessFrame(*video_frame, &decimated_frame);
 532   if (ret == 1) {
 533     // Drop this frame.
 534     return;
 535   }
 536   if (ret != VPM_OK) {
 537     return;
 538   }
 539   // Frame was not sampled => use original.
 540   if (decimated_frame == NULL)  {
 541     decimated_frame = video_frame;
 542   }
 543
 544   {
 545     CriticalSectionScoped cs(callback_cs_.get());
 546     if (pre_encode_callback_)
 547       pre_encode_callback_->FrameCallback(decimated_frame);
 548   }
 549
 550 #ifdef VIDEOCODEC_VP8
 551   if (vcm_.SendCodec() == webrtc::kVideoCodecVP8) {
 552     webrtc::CodecSpecificInfo codec_specific_info;
 553     codec_specific_info.codecType = webrtc::kVideoCodecVP8;
 554     codec_specific_info.codecSpecific.VP8.hasReceivedRPSI =
 555         has_received_rpsi_;
 556     codec_specific_info.codecSpecific.VP8.hasReceivedSLI =
 557         has_received_sli_;
 558     codec_specific_info.codecSpecific.VP8.pictureIdRPSI =
 559         picture_id_rpsi_;
 560     codec_specific_info.codecSpecific.VP8.pictureIdSLI  =
 561         picture_id_sli_;
 562     has_received_sli_ = false;
 563     has_received_rpsi_ = false;
 564
 565     vcm_.AddVideoFrame(*decimated_frame, vpm_.ContentMetrics(),
 566                        &codec_specific_info);
 567     return;
 568   }
 569 #endif
 570   vcm_.AddVideoFrame(*decimated_frame);
 571 }
 572
 573 void ViEEncoder::DelayChanged(int id, int frame_delay) {
 574   default_rtp_rtcp_->SetCameraDelay(frame_delay);
 575 }
 576
 577 int ViEEncoder::GetPreferedFrameSettings(int* width,
 578                                          int* height,
 579                                          int* frame_rate) {
 580   webrtc::VideoCodec video_codec;
 581   memset(&video_codec, 0, sizeof(video_codec));
 582   if (vcm_.SendCodec(&video_codec) != VCM_OK) {
 583     return -1;
 584   }
 585
 586   *width = video_codec.width;
 587   *height = video_codec.height;
 588   *frame_rate = video_codec.maxFramerate;
 589   return 0;
 590 }
 591
 592 int ViEEncoder::SendKeyFrame() {
 593   return vcm_.IntraFrameRequest(0);
 594 }
 595
 596 int32_t ViEEncoder::SendCodecStatistics(
 597     uint32_t* num_key_frames, uint32_t* num_delta_frames) {
 598   webrtc::VCMFrameCount sent_frames;
 599   if (vcm_.SentFrameCount(sent_frames) != VCM_OK) {
 600     return -1;
 601   }
 602   *num_key_frames = sent_frames.numKeyFrames;
 603   *num_delta_frames = sent_frames.numDeltaFrames;
 604   return 0;
 605 }
 606
 607 int32_t ViEEncoder::PacerQueuingDelayMs() const {
 608   return paced_sender_->QueueInMs();
 609 }
 610
 611 int ViEEncoder::CodecTargetBitrate(uint32_t* bitrate) const {
 612   if (vcm_.Bitrate(bitrate) != 0)
 613     return -1;
 614   return 0;
 615 }
 616
 617 int32_t ViEEncoder::UpdateProtectionMethod(bool enable_nack) {
 618   bool fec_enabled = false;
 619   uint8_t dummy_ptype_red = 0;
 620   uint8_t dummy_ptypeFEC = 0;
 621
 622   // Updated protection method to VCM to get correct packetization sizes.
 623   // FEC has larger overhead than NACK -> set FEC if used.
 624   int32_t error = default_rtp_rtcp_->GenericFECStatus(fec_enabled,
 625                                                       dummy_ptype_red,
 626                                                       dummy_ptypeFEC);
 627   if (error) {
 628     return -1;
 629   }
 630   if (fec_enabled_ == fec_enabled && nack_enabled_ == enable_nack) {
 631     // No change needed, we're already in correct state.
 632     return 0;
 633   }
 634   fec_enabled_ = fec_enabled;
 635   nack_enabled_ = enable_nack;
 636
 637   // Set Video Protection for VCM.
 638   if (fec_enabled && nack_enabled_) {
 639     vcm_.SetVideoProtection(webrtc::kProtectionNackFEC, true);
 640   } else {
 641     vcm_.SetVideoProtection(webrtc::kProtectionFEC, fec_enabled_);
 642     vcm_.SetVideoProtection(webrtc::kProtectionNackSender, nack_enabled_);
 643     vcm_.SetVideoProtection(webrtc::kProtectionNackFEC, false);
 644   }
 645
 646   if (fec_enabled_ || nack_enabled_) {
 647     vcm_.RegisterProtectionCallback(this);
 648     // The send codec must be registered to set correct MTU.
 649     webrtc::VideoCodec codec;
 650     if (vcm_.SendCodec(&codec) == 0) {
 651       uint16_t max_pay_load = default_rtp_rtcp_->MaxDataPayloadLength();
 652       uint32_t current_bitrate_bps = 0;
 653       if (vcm_.Bitrate(&current_bitrate_bps) != 0) {
 654         LOG_F(LS_WARNING) <<
 655             "Failed to get the current encoder target bitrate.";
 656       }
 657       // Convert to start bitrate in kbps.
 658       codec.startBitrate = (current_bitrate_bps + 500) / 1000;
 659       if (vcm_.RegisterSendCodec(&codec, number_of_cores_, max_pay_load) != 0) {
 660         return -1;
 661       }
 662     }
 663     return 0;
 664   } else {
 665     // FEC and NACK are disabled.
 666     vcm_.RegisterProtectionCallback(NULL);
 667   }
 668   return 0;
 669 }
 670
 671 void ViEEncoder::SetSenderBufferingMode(int target_delay_ms) {
 672   {
 673     CriticalSectionScoped cs(data_cs_.get());
 674     target_delay_ms_ = target_delay_ms;
 675   }
 676   if (target_delay_ms > 0) {
 677     // Disable external frame-droppers.
 678     vcm_.EnableFrameDropper(false);
 679     vpm_.EnableTemporalDecimation(false);
 680     // We don't put any limits on the pacer queue when running in buffered mode
 681     // since the encoder will be paused if the queue grow too large.
 682     paced_sender_->set_max_queue_length_ms(-1);
 683   } else {
 684     // Real-time mode - enable frame droppers.
 685     vpm_.EnableTemporalDecimation(true);
 686     vcm_.EnableFrameDropper(true);
 687     paced_sender_->set_max_queue_length_ms(
 688         PacedSender::kDefaultMaxQueueLengthMs);
 689   }
 690 }
 691
 692 int32_t ViEEncoder::SendData(
 693     const FrameType frame_type,
 694     const uint8_t payload_type,
 695     const uint32_t time_stamp,
 696     int64_t capture_time_ms,
 697     const uint8_t* payload_data,
 698     const uint32_t payload_size,
 699     const webrtc::RTPFragmentationHeader& fragmentation_header,
 700     const RTPVideoHeader* rtp_video_hdr) {
 701   // New encoded data, hand over to the rtp module.
 702   return default_rtp_rtcp_->SendOutgoingData(frame_type,
 703                                              payload_type,
 704                                              time_stamp,
 705                                              capture_time_ms,
 706                                              payload_data,
 707                                              payload_size,
 708                                              &fragmentation_header,
 709                                              rtp_video_hdr);
 710 }
 711
 712 int32_t ViEEncoder::ProtectionRequest(
 713     const FecProtectionParams* delta_fec_params,
 714     const FecProtectionParams* key_fec_params,
 715     uint32_t* sent_video_rate_bps,
 716     uint32_t* sent_nack_rate_bps,
 717     uint32_t* sent_fec_rate_bps) {
 718   default_rtp_rtcp_->SetFecParameters(delta_fec_params, key_fec_params);
 719   default_rtp_rtcp_->BitrateSent(NULL, sent_video_rate_bps, sent_fec_rate_bps,
 720                                 sent_nack_rate_bps);
 721   return 0;
 722 }
 723
 724 int32_t ViEEncoder::SendStatistics(const uint32_t bit_rate,
 725                                    const uint32_t frame_rate) {
 726   CriticalSectionScoped cs(callback_cs_.get());
 727   if (codec_observer_) {
 728     codec_observer_->OutgoingRate(channel_id_, frame_rate, bit_rate);
 729   }
 730   return 0;
 731 }
 732
 733 int32_t ViEEncoder::RegisterCodecObserver(ViEEncoderObserver* observer) {
 734   CriticalSectionScoped cs(callback_cs_.get());
 735   if (observer && codec_observer_) {
 736     LOG_F(LS_ERROR) << "Observer already set.";
 737     return -1;
 738   }
 739   codec_observer_ = observer;
 740   return 0;
 741 }
 742
 743 void ViEEncoder::OnReceivedSLI(uint32_t /*ssrc*/,
 744                                uint8_t picture_id) {
 745   picture_id_sli_ = picture_id;
 746   has_received_sli_ = true;
 747 }
 748
 749 void ViEEncoder::OnReceivedRPSI(uint32_t /*ssrc*/,
 750                                 uint64_t picture_id) {
 751   picture_id_rpsi_ = picture_id;
 752   has_received_rpsi_ = true;
 753 }
 754
 755 void ViEEncoder::OnReceivedIntraFrameRequest(uint32_t ssrc) {
 756   // Key frame request from remote side, signal to VCM.
 757   TRACE_EVENT0("webrtc", "OnKeyFrameRequest");
 758
 759   int idx = 0;
 760   {
 761     CriticalSectionScoped cs(data_cs_.get());
 762     std::map<unsigned int, int>::iterator stream_it = ssrc_streams_.find(ssrc);
 763     if (stream_it == ssrc_streams_.end()) {
 764       LOG_F(LS_WARNING) << "ssrc not found: " << ssrc << ", map size "
 765                         << ssrc_streams_.size();
 766       return;
 767     }
 768     std::map<unsigned int, int64_t>::iterator time_it =
 769         time_last_intra_request_ms_.find(ssrc);
 770     if (time_it == time_last_intra_request_ms_.end()) {
 771       time_last_intra_request_ms_[ssrc] = 0;
 772     }
 773
 774     int64_t now = TickTime::MillisecondTimestamp();
 775     if (time_last_intra_request_ms_[ssrc] + kViEMinKeyRequestIntervalMs > now) {
 776       return;
 777     }
 778     time_last_intra_request_ms_[ssrc] = now;
 779     idx = stream_it->second;
 780   }
 781   // Release the critsect before triggering key frame.
 782   vcm_.IntraFrameRequest(idx);
 783 }
 784
 785 void ViEEncoder::OnLocalSsrcChanged(uint32_t old_ssrc, uint32_t new_ssrc) {
 786   CriticalSectionScoped cs(data_cs_.get());
 787   std::map<unsigned int, int>::iterator it = ssrc_streams_.find(old_ssrc);
 788   if (it == ssrc_streams_.end()) {
 789     return;
 790   }
 791
 792   ssrc_streams_[new_ssrc] = it->second;
 793   ssrc_streams_.erase(it);
 794
 795   std::map<unsigned int, int64_t>::iterator time_it =
 796       time_last_intra_request_ms_.find(old_ssrc);
 797   int64_t last_intra_request_ms = 0;
 798   if (time_it != time_last_intra_request_ms_.end()) {
 799     last_intra_request_ms = time_it->second;
 800     time_last_intra_request_ms_.erase(time_it);
 801   }
 802   time_last_intra_request_ms_[new_ssrc] = last_intra_request_ms;
 803 }
 804
 805 bool ViEEncoder::SetSsrcs(const std::list<unsigned int>& ssrcs) {
 806   VideoCodec codec;
 807   if (vcm_.SendCodec(&codec) != 0)
 808     return false;
 809
 810   if (codec.numberOfSimulcastStreams > 0 &&
 811       ssrcs.size() != codec.numberOfSimulcastStreams) {
 812     return false;
 813   }
 814
 815   CriticalSectionScoped cs(data_cs_.get());
 816   ssrc_streams_.clear();
 817   time_last_intra_request_ms_.clear();
 818   int idx = 0;
 819   for (std::list<unsigned int>::const_iterator it = ssrcs.begin();
 820        it != ssrcs.end(); ++it, ++idx) {
 821     unsigned int ssrc = *it;
 822     ssrc_streams_[ssrc] = idx;
 823   }
 824   return true;
 825 }
 826
 827 void ViEEncoder::SetMinTransmitBitrate(int min_transmit_bitrate_kbps) {
 828   assert(min_transmit_bitrate_kbps >= 0);
 829   CriticalSectionScoped crit(data_cs_.get());
 830   min_transmit_bitrate_kbps_ = min_transmit_bitrate_kbps;
 831 }
 832
 833 // Called from ViEBitrateObserver.
 834 void ViEEncoder::OnNetworkChanged(const uint32_t bitrate_bps,
 835                                   const uint8_t fraction_lost,
 836                                   const uint32_t round_trip_time_ms) {
 837   LOG(LS_VERBOSE) << "OnNetworkChanged, bitrate" << bitrate_bps
 838                   << " packet loss " << fraction_lost
 839                   << " rtt " << round_trip_time_ms;
 840   vcm_.SetChannelParameters(bitrate_bps, fraction_lost, round_trip_time_ms);
 841   bool video_is_suspended = vcm_.VideoSuspended();
 842   int bitrate_kbps = bitrate_bps / 1000;
 843   VideoCodec send_codec;
 844   if (vcm_.SendCodec(&send_codec) != 0) {
 845     return;
 846   }
 847   SimulcastStream* stream_configs = send_codec.simulcastStream;
 848   // Allocate the bandwidth between the streams.
 849   std::vector<uint32_t> stream_bitrates = AllocateStreamBitrates(
 850       bitrate_bps,
 851       stream_configs,
 852       send_codec.numberOfSimulcastStreams);
 853   // Find the max amount of padding we can allow ourselves to send at this
 854   // point, based on which streams are currently active and what our current
 855   // available bandwidth is.
 856   int pad_up_to_bitrate_kbps = 0;
 857   if (send_codec.numberOfSimulcastStreams == 0) {
 858     pad_up_to_bitrate_kbps = send_codec.minBitrate;
 859   } else {
 860     pad_up_to_bitrate_kbps =
 861         stream_configs[send_codec.numberOfSimulcastStreams - 1].minBitrate;
 862     for (int i = 0; i < send_codec.numberOfSimulcastStreams - 1; ++i) {
 863       pad_up_to_bitrate_kbps += stream_configs[i].targetBitrate;
 864     }
 865   }
 866
 867   // Disable padding if only sending one stream and video isn't suspended and
 868   // min-transmit bitrate isn't used (applied later).
 869   if (!video_is_suspended && send_codec.numberOfSimulcastStreams <= 1)
 870     pad_up_to_bitrate_kbps = 0;
 871
 872   {
 873     CriticalSectionScoped cs(data_cs_.get());
 874     // The amount of padding should decay to zero if no frames are being
 875     // captured unless a min-transmit bitrate is used.
 876     int64_t now_ms = TickTime::MillisecondTimestamp();
 877     if (now_ms - time_of_last_incoming_frame_ms_ > kStopPaddingThresholdMs)
 878       pad_up_to_bitrate_kbps = 0;
 879
 880     // Pad up to min bitrate.
 881     if (pad_up_to_bitrate_kbps < min_transmit_bitrate_kbps_)
 882       pad_up_to_bitrate_kbps = min_transmit_bitrate_kbps_;
 883
 884     // Padding may never exceed bitrate estimate.
 885     if (pad_up_to_bitrate_kbps > bitrate_kbps)
 886       pad_up_to_bitrate_kbps = bitrate_kbps;
 887
 888     paced_sender_->UpdateBitrate(kPaceMultiplier * bitrate_kbps,
 889                                  pad_up_to_bitrate_kbps);
 890     default_rtp_rtcp_->SetTargetSendBitrate(stream_bitrates);
 891     if (video_suspended_ == video_is_suspended)
 892       return;
 893     video_suspended_ = video_is_suspended;
 894   }
 895
 896   // Video suspend-state changed, inform codec observer.
 897   CriticalSectionScoped crit(callback_cs_.get());
 898   if (codec_observer_) {
 899     LOG(LS_INFO) << "Video suspended " << video_is_suspended
 900                  << " for channel " << channel_id_;
 901     codec_observer_->SuspendChange(channel_id_, video_is_suspended);
 902   }
 903 }
 904
 905 PacedSender* ViEEncoder::GetPacedSender() {
 906   return paced_sender_.get();
 907 }
 908
 909 int32_t ViEEncoder::RegisterEffectFilter(ViEEffectFilter* effect_filter) {
 910   CriticalSectionScoped cs(callback_cs_.get());
 911   if (effect_filter != NULL && effect_filter_ != NULL) {
 912     LOG_F(LS_ERROR) << "Filter already set.";
 913     return -1;
 914   }
 915   effect_filter_ = effect_filter;
 916   return 0;
 917 }
 918
 919 int ViEEncoder::StartDebugRecording(const char* fileNameUTF8) {
 920   return vcm_.StartDebugRecording(fileNameUTF8);
 921 }
 922
 923 int ViEEncoder::StopDebugRecording() {
 924   return vcm_.StopDebugRecording();
 925 }
 926
 927 void ViEEncoder::SuspendBelowMinBitrate() {
 928   vcm_.SuspendBelowMinBitrate();
 929   bitrate_controller_->EnforceMinBitrate(false);
 930 }
 931
 932 void ViEEncoder::RegisterPreEncodeCallback(
 933     I420FrameCallback* pre_encode_callback) {
 934   CriticalSectionScoped cs(callback_cs_.get());
 935   pre_encode_callback_ = pre_encode_callback;
 936 }
 937
 938 void ViEEncoder::DeRegisterPreEncodeCallback() {
 939   CriticalSectionScoped cs(callback_cs_.get());
 940   pre_encode_callback_ = NULL;
 941 }
 942
 943 void ViEEncoder::RegisterPostEncodeImageCallback(
 944       EncodedImageCallback* post_encode_callback) {
 945   vcm_.RegisterPostEncodeImageCallback(post_encode_callback);
 946 }
 947
 948 void ViEEncoder::DeRegisterPostEncodeImageCallback() {
 949   vcm_.RegisterPostEncodeImageCallback(NULL);
 950 }
 951
 952 QMVideoSettingsCallback::QMVideoSettingsCallback(VideoProcessingModule* vpm)
 953     : vpm_(vpm) {
 954 }
 955
 956 QMVideoSettingsCallback::~QMVideoSettingsCallback() {
 957 }
 958
 959 int32_t QMVideoSettingsCallback::SetVideoQMSettings(
 960     const uint32_t frame_rate,
 961     const uint32_t width,
 962     const uint32_t height) {
 963   return vpm_->SetTargetResolution(width, height, frame_rate);
 964 }
 965
 966 }  // namespace webrtc