src/third_party/webrtc/video/video_send_stream.cc

   1 /*
   2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include "webrtc/video/video_send_stream.h"
  12
  13 #include <string>
  14 #include <vector>
  15
  16 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
  17 #include "webrtc/video_engine/include/vie_base.h"
  18 #include "webrtc/video_engine/include/vie_capture.h"
  19 #include "webrtc/video_engine/include/vie_codec.h"
  20 #include "webrtc/video_engine/include/vie_external_codec.h"
  21 #include "webrtc/video_engine/include/vie_image_process.h"
  22 #include "webrtc/video_engine/include/vie_network.h"
  23 #include "webrtc/video_engine/include/vie_rtp_rtcp.h"
  24 #include "webrtc/video_engine/vie_defines.h"
  25 #include "webrtc/video_send_stream.h"
  26
  27 namespace webrtc {
  28 namespace internal {
  29
  30 VideoSendStream::VideoSendStream(newapi::Transport* transport,
  31                                  CpuOveruseObserver* overuse_observer,
  32                                  webrtc::VideoEngine* video_engine,
  33                                  const VideoSendStream::Config& config,
  34                                  int base_channel)
  35     : transport_adapter_(transport),
  36       encoded_frame_proxy_(config.post_encode_callback),
  37       codec_lock_(CriticalSectionWrapper::CreateCriticalSection()),
  38       config_(config),
  39       external_codec_(NULL),
  40       channel_(-1) {
  41   video_engine_base_ = ViEBase::GetInterface(video_engine);
  42   video_engine_base_->CreateChannel(channel_, base_channel);
  43   assert(channel_ != -1);
  44
  45   rtp_rtcp_ = ViERTP_RTCP::GetInterface(video_engine);
  46   assert(rtp_rtcp_ != NULL);
  47
  48   assert(config_.rtp.ssrcs.size() > 0);
  49   if (config_.suspend_below_min_bitrate)
  50     config_.pacing = true;
  51   rtp_rtcp_->SetTransmissionSmoothingStatus(channel_, config_.pacing);
  52
  53   assert(config_.rtp.min_transmit_bitrate_bps >= 0);
  54   rtp_rtcp_->SetMinTransmitBitrate(channel_,
  55                                    config_.rtp.min_transmit_bitrate_bps / 1000);
  56
  57   for (size_t i = 0; i < config_.rtp.extensions.size(); ++i) {
  58     const std::string& extension = config_.rtp.extensions[i].name;
  59     int id = config_.rtp.extensions[i].id;
  60     if (extension == RtpExtension::kTOffset) {
  61       if (rtp_rtcp_->SetSendTimestampOffsetStatus(channel_, true, id) != 0)
  62         abort();
  63     } else if (extension == RtpExtension::kAbsSendTime) {
  64       if (rtp_rtcp_->SetSendAbsoluteSendTimeStatus(channel_, true, id) != 0)
  65         abort();
  66     } else {
  67       abort();  // Unsupported extension.
  68     }
  69   }
  70
  71   rtp_rtcp_->SetRembStatus(channel_, true, false);
  72
  73   // Enable NACK, FEC or both.
  74   if (config_.rtp.fec.red_payload_type != -1) {
  75     assert(config_.rtp.fec.ulpfec_payload_type != -1);
  76     if (config_.rtp.nack.rtp_history_ms > 0) {
  77       rtp_rtcp_->SetHybridNACKFECStatus(
  78           channel_,
  79           true,
  80           static_cast<unsigned char>(config_.rtp.fec.red_payload_type),
  81           static_cast<unsigned char>(config_.rtp.fec.ulpfec_payload_type));
  82     } else {
  83       rtp_rtcp_->SetFECStatus(
  84           channel_,
  85           true,
  86           static_cast<unsigned char>(config_.rtp.fec.red_payload_type),
  87           static_cast<unsigned char>(config_.rtp.fec.ulpfec_payload_type));
  88     }
  89   } else {
  90     rtp_rtcp_->SetNACKStatus(channel_, config_.rtp.nack.rtp_history_ms > 0);
  91   }
  92
  93   char rtcp_cname[ViERTP_RTCP::KMaxRTCPCNameLength];
  94   assert(config_.rtp.c_name.length() < ViERTP_RTCP::KMaxRTCPCNameLength);
  95   strncpy(rtcp_cname, config_.rtp.c_name.c_str(), sizeof(rtcp_cname) - 1);
  96   rtcp_cname[sizeof(rtcp_cname) - 1] = '\0';
  97
  98   rtp_rtcp_->SetRTCPCName(channel_, rtcp_cname);
  99
 100   capture_ = ViECapture::GetInterface(video_engine);
 101   capture_->AllocateExternalCaptureDevice(capture_id_, external_capture_);
 102   capture_->ConnectCaptureDevice(capture_id_, channel_);
 103
 104   network_ = ViENetwork::GetInterface(video_engine);
 105   assert(network_ != NULL);
 106
 107   network_->RegisterSendTransport(channel_, transport_adapter_);
 108   // 28 to match packet overhead in ModuleRtpRtcpImpl.
 109   network_->SetMTU(channel_,
 110                    static_cast<unsigned int>(config_.rtp.max_packet_size + 28));
 111
 112   assert(config.encoder_settings.encoder != NULL);
 113   assert(config.encoder_settings.payload_type >= 0);
 114   assert(config.encoder_settings.payload_type <= 127);
 115   external_codec_ = ViEExternalCodec::GetInterface(video_engine);
 116   if (external_codec_->RegisterExternalSendCodec(
 117           channel_,
 118           config.encoder_settings.payload_type,
 119           config.encoder_settings.encoder,
 120           false) != 0) {
 121     abort();
 122   }
 123
 124   codec_ = ViECodec::GetInterface(video_engine);
 125   if (!ReconfigureVideoEncoder(config_.encoder_settings.streams,
 126                                config_.encoder_settings.encoder_settings)) {
 127     abort();
 128   }
 129
 130   if (overuse_observer)
 131     video_engine_base_->RegisterCpuOveruseObserver(channel_, overuse_observer);
 132
 133   image_process_ = ViEImageProcess::GetInterface(video_engine);
 134   image_process_->RegisterPreEncodeCallback(channel_,
 135                                             config_.pre_encode_callback);
 136   if (config_.post_encode_callback) {
 137     image_process_->RegisterPostEncodeImageCallback(channel_,
 138                                                     &encoded_frame_proxy_);
 139   }
 140
 141   if (config_.suspend_below_min_bitrate) {
 142     codec_->SuspendBelowMinBitrate(channel_);
 143   }
 144
 145   stats_proxy_.reset(new SendStatisticsProxy(config, this));
 146
 147   rtp_rtcp_->RegisterSendChannelRtcpStatisticsCallback(channel_,
 148                                                        stats_proxy_.get());
 149   rtp_rtcp_->RegisterSendChannelRtpStatisticsCallback(channel_,
 150                                                       stats_proxy_.get());
 151   rtp_rtcp_->RegisterSendBitrateObserver(channel_, stats_proxy_.get());
 152   rtp_rtcp_->RegisterSendFrameCountObserver(channel_, stats_proxy_.get());
 153
 154   codec_->RegisterEncoderObserver(channel_, *stats_proxy_);
 155   capture_->RegisterObserver(capture_id_, *stats_proxy_);
 156 }
 157
 158 VideoSendStream::~VideoSendStream() {
 159   capture_->DeregisterObserver(capture_id_);
 160   codec_->DeregisterEncoderObserver(channel_);
 161
 162   rtp_rtcp_->DeregisterSendFrameCountObserver(channel_, stats_proxy_.get());
 163   rtp_rtcp_->DeregisterSendBitrateObserver(channel_, stats_proxy_.get());
 164   rtp_rtcp_->DeregisterSendChannelRtpStatisticsCallback(channel_,
 165                                                         stats_proxy_.get());
 166   rtp_rtcp_->DeregisterSendChannelRtcpStatisticsCallback(channel_,
 167                                                          stats_proxy_.get());
 168
 169   image_process_->DeRegisterPreEncodeCallback(channel_);
 170
 171   network_->DeregisterSendTransport(channel_);
 172
 173   capture_->DisconnectCaptureDevice(channel_);
 174   capture_->ReleaseCaptureDevice(capture_id_);
 175
 176   external_codec_->DeRegisterExternalSendCodec(
 177       channel_, config_.encoder_settings.payload_type);
 178
 179   video_engine_base_->DeleteChannel(channel_);
 180
 181   image_process_->Release();
 182   video_engine_base_->Release();
 183   capture_->Release();
 184   codec_->Release();
 185   if (external_codec_)
 186     external_codec_->Release();
 187   network_->Release();
 188   rtp_rtcp_->Release();
 189 }
 190
 191 void VideoSendStream::PutFrame(const I420VideoFrame& frame) {
 192   input_frame_.CopyFrame(frame);
 193   SwapFrame(&input_frame_);
 194 }
 195
 196 void VideoSendStream::SwapFrame(I420VideoFrame* frame) {
 197   // TODO(pbos): Warn if frame is "too far" into the future, or too old. This
 198   //             would help detect if frame's being used without NTP.
 199   //             TO REVIEWER: Is there any good check for this? Should it be
 200   //             skipped?
 201   if (frame != &input_frame_)
 202     input_frame_.SwapFrame(frame);
 203
 204   // TODO(pbos): Local rendering should not be done on the capture thread.
 205   if (config_.local_renderer != NULL)
 206     config_.local_renderer->RenderFrame(input_frame_, 0);
 207
 208   external_capture_->SwapFrame(&input_frame_);
 209 }
 210
 211 VideoSendStreamInput* VideoSendStream::Input() { return this; }
 212
 213 void VideoSendStream::StartSending() {
 214   transport_adapter_.Enable();
 215   video_engine_base_->StartSend(channel_);
 216   video_engine_base_->StartReceive(channel_);
 217 }
 218
 219 void VideoSendStream::StopSending() {
 220   video_engine_base_->StopSend(channel_);
 221   video_engine_base_->StopReceive(channel_);
 222   transport_adapter_.Disable();
 223 }
 224
 225 bool VideoSendStream::ReconfigureVideoEncoder(
 226     const std::vector<VideoStream>& streams,
 227     void* encoder_settings) {
 228   assert(!streams.empty());
 229   assert(config_.rtp.ssrcs.size() >= streams.size());
 230   // TODO(pbos): Wire encoder_settings.
 231   assert(encoder_settings == NULL);
 232
 233   // VideoStreams in config_.encoder_settings need to be locked.
 234   CriticalSectionScoped crit(codec_lock_.get());
 235
 236   VideoCodec video_codec;
 237   memset(&video_codec, 0, sizeof(video_codec));
 238   video_codec.codecType =
 239       (config_.encoder_settings.payload_name == "VP8" ? kVideoCodecVP8
 240                                                       : kVideoCodecGeneric);
 241
 242   if (video_codec.codecType == kVideoCodecVP8) {
 243     video_codec.codecSpecific.VP8.resilience = kResilientStream;
 244     video_codec.codecSpecific.VP8.numberOfTemporalLayers = 1;
 245     video_codec.codecSpecific.VP8.denoisingOn = true;
 246     video_codec.codecSpecific.VP8.errorConcealmentOn = false;
 247     video_codec.codecSpecific.VP8.automaticResizeOn = false;
 248     video_codec.codecSpecific.VP8.frameDroppingOn = true;
 249     video_codec.codecSpecific.VP8.keyFrameInterval = 3000;
 250   }
 251
 252   strncpy(video_codec.plName,
 253           config_.encoder_settings.payload_name.c_str(),
 254           kPayloadNameSize - 1);
 255   video_codec.plName[kPayloadNameSize - 1] = '\0';
 256   video_codec.plType = config_.encoder_settings.payload_type;
 257   video_codec.numberOfSimulcastStreams =
 258       static_cast<unsigned char>(streams.size());
 259   video_codec.minBitrate = streams[0].min_bitrate_bps / 1000;
 260   assert(streams.size() <= kMaxSimulcastStreams);
 261   for (size_t i = 0; i < streams.size(); ++i) {
 262     SimulcastStream* sim_stream = &video_codec.simulcastStream[i];
 263     assert(streams[i].width > 0);
 264     assert(streams[i].height > 0);
 265     assert(streams[i].max_framerate > 0);
 266     // Different framerates not supported per stream at the moment.
 267     assert(streams[i].max_framerate == streams[0].max_framerate);
 268     assert(streams[i].min_bitrate_bps >= 0);
 269     assert(streams[i].target_bitrate_bps >= streams[i].min_bitrate_bps);
 270     assert(streams[i].max_bitrate_bps >= streams[i].target_bitrate_bps);
 271     assert(streams[i].max_qp >= 0);
 272
 273     sim_stream->width = static_cast<unsigned short>(streams[i].width);
 274     sim_stream->height = static_cast<unsigned short>(streams[i].height);
 275     sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000;
 276     sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000;
 277     sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000;
 278     sim_stream->qpMax = streams[i].max_qp;
 279     // TODO(pbos): Implement mapping for temporal layers.
 280     assert(streams[i].temporal_layers.empty());
 281
 282     video_codec.width = std::max(video_codec.width,
 283                                  static_cast<unsigned short>(streams[i].width));
 284     video_codec.height = std::max(
 285         video_codec.height, static_cast<unsigned short>(streams[i].height));
 286     video_codec.minBitrate =
 287         std::min(video_codec.minBitrate,
 288                  static_cast<unsigned int>(streams[i].min_bitrate_bps / 1000));
 289     video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000;
 290     video_codec.qpMax = std::max(video_codec.qpMax,
 291                                  static_cast<unsigned int>(streams[i].max_qp));
 292   }
 293
 294   if (video_codec.minBitrate < kViEMinCodecBitrate)
 295     video_codec.minBitrate = kViEMinCodecBitrate;
 296   if (video_codec.maxBitrate < kViEMinCodecBitrate)
 297     video_codec.maxBitrate = kViEMinCodecBitrate;
 298
 299   video_codec.startBitrate = 300;
 300
 301   if (video_codec.startBitrate < video_codec.minBitrate)
 302     video_codec.startBitrate = video_codec.minBitrate;
 303   if (video_codec.startBitrate > video_codec.maxBitrate)
 304     video_codec.startBitrate = video_codec.maxBitrate;
 305
 306   assert(config_.encoder_settings.streams[0].max_framerate > 0);
 307   video_codec.maxFramerate = config_.encoder_settings.streams[0].max_framerate;
 308
 309   if (codec_->SetSendCodec(channel_, video_codec) != 0)
 310     return false;
 311
 312   for (size_t i = 0; i < config_.rtp.ssrcs.size(); ++i) {
 313     rtp_rtcp_->SetLocalSSRC(channel_,
 314                             config_.rtp.ssrcs[i],
 315                             kViEStreamTypeNormal,
 316                             static_cast<unsigned char>(i));
 317   }
 318
 319   config_.encoder_settings.streams = streams;
 320   config_.encoder_settings.encoder_settings = encoder_settings;
 321
 322   if (config_.rtp.rtx.ssrcs.empty())
 323     return true;
 324
 325   // Set up RTX.
 326   assert(config_.rtp.rtx.ssrcs.size() == config_.rtp.ssrcs.size());
 327   for (size_t i = 0; i < config_.rtp.ssrcs.size(); ++i) {
 328     rtp_rtcp_->SetLocalSSRC(channel_,
 329                             config_.rtp.rtx.ssrcs[i],
 330                             kViEStreamTypeRtx,
 331                             static_cast<unsigned char>(i));
 332   }
 333
 334   if (config_.rtp.rtx.payload_type != 0)
 335     rtp_rtcp_->SetRtxSendPayloadType(channel_, config_.rtp.rtx.payload_type);
 336
 337   return true;
 338 }
 339
 340 bool VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
 341   return network_->ReceivedRTCPPacket(
 342              channel_, packet, static_cast<int>(length)) == 0;
 343 }
 344
 345 VideoSendStream::Stats VideoSendStream::GetStats() const {
 346   return stats_proxy_->GetStats();
 347 }
 348
 349 bool VideoSendStream::GetSendSideDelay(VideoSendStream::Stats* stats) {
 350   return codec_->GetSendSideDelay(
 351       channel_, &stats->avg_delay_ms, &stats->max_delay_ms);
 352 }
 353
 354 std::string VideoSendStream::GetCName() {
 355   char rtcp_cname[ViERTP_RTCP::KMaxRTCPCNameLength];
 356   rtp_rtcp_->GetRTCPCName(channel_, rtcp_cname);
 357   return rtcp_cname;
 358 }
 359
 360 }  // namespace internal
 361 }  // namespace webrtc