src/media/cast/video_sender/video_sender.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/video_sender/video_sender.h"
   6
   7 #include <list>
   8
   9 #include "base/bind.h"
  10 #include "base/logging.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "media/cast/cast_defines.h"
  13 #include "media/cast/pacing/paced_sender.h"
  14 #include "media/cast/video_sender/video_encoder.h"
  15
  16 namespace media {
  17 namespace cast {
  18
  19 const int64 kMinSchedulingDelayMs = 1;
  20
  21 class LocalRtcpVideoSenderFeedback : public RtcpSenderFeedback {
  22  public:
  23   explicit LocalRtcpVideoSenderFeedback(VideoSender* video_sender)
  24       : video_sender_(video_sender) {
  25   }
  26
  27   virtual void OnReceivedReportBlock(
  28       const RtcpReportBlock& report_block) OVERRIDE {}
  29
  30   virtual void OnReceivedRpsi(uint8 payload_type,
  31                               uint64 picture_id) OVERRIDE {
  32     NOTIMPLEMENTED();
  33   }
  34
  35   virtual void OnReceivedRemb(uint32 bitrate) OVERRIDE {
  36     NOTIMPLEMENTED();
  37   }
  38
  39   virtual void OnReceivedNackRequest(
  40       const std::list<uint16>& nack_sequence_numbers) OVERRIDE {
  41     NOTIMPLEMENTED();
  42   }
  43
  44   virtual void OnReceivedIntraFrameRequest() OVERRIDE {
  45     video_sender_->OnReceivedIntraFrameRequest();
  46   }
  47
  48   virtual void OnReceivedCastFeedback(
  49       const RtcpCastMessage& cast_feedback) OVERRIDE {
  50     video_sender_->OnReceivedCastFeedback(cast_feedback);
  51   }
  52
  53  private:
  54   VideoSender* video_sender_;
  55 };
  56
  57 class LocalRtpVideoSenderStatistics : public RtpSenderStatistics {
  58  public:
  59   explicit LocalRtpVideoSenderStatistics(RtpSender* rtp_sender)
  60      : rtp_sender_(rtp_sender) {
  61   }
  62
  63   virtual void GetStatistics(const base::TimeTicks& now,
  64                              RtcpSenderInfo* sender_info) OVERRIDE {
  65     rtp_sender_->RtpStatistics(now, sender_info);
  66   }
  67
  68  private:
  69   RtpSender* rtp_sender_;
  70 };
  71
  72 VideoSender::VideoSender(
  73     scoped_refptr<CastEnvironment> cast_environment,
  74     const VideoSenderConfig& video_config,
  75     VideoEncoderController* const video_encoder_controller,
  76     PacedPacketSender* const paced_packet_sender)
  77     : incoming_feedback_ssrc_(video_config.incoming_feedback_ssrc),
  78       rtp_max_delay_(
  79           base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)),
  80       max_frame_rate_(video_config.max_frame_rate),
  81       cast_environment_(cast_environment),
  82       rtcp_feedback_(new LocalRtcpVideoSenderFeedback(this)),
  83       rtp_sender_(new RtpSender(cast_environment->Clock(), NULL, &video_config,
  84                                 paced_packet_sender)),
  85       last_acked_frame_id_(-1),
  86       last_sent_frame_id_(-1),
  87       last_sent_key_frame_id_(-1),
  88       duplicate_ack_(0),
  89       last_skip_count_(0),
  90       congestion_control_(cast_environment->Clock(),
  91                           video_config.congestion_control_back_off,
  92                           video_config.max_bitrate,
  93                           video_config.min_bitrate,
  94                           video_config.start_bitrate),
  95       weak_factory_(this) {
  96   max_unacked_frames_ = static_cast<uint8>(video_config.rtp_max_delay_ms *
  97       video_config.max_frame_rate / 1000) + 1;
  98   VLOG(1) << "max_unacked_frames " << static_cast<int>(max_unacked_frames_);
  99   DCHECK_GT(max_unacked_frames_, 0) << "Invalid argument";
 100
 101   rtp_video_sender_statistics_.reset(
 102       new LocalRtpVideoSenderStatistics(rtp_sender_.get()));
 103
 104   if (video_config.use_external_encoder) {
 105     DCHECK(video_encoder_controller) << "Invalid argument";
 106     video_encoder_controller_ = video_encoder_controller;
 107   } else {
 108     video_encoder_ = new VideoEncoder(cast_environment, video_config,
 109         max_unacked_frames_);
 110     video_encoder_controller_ = video_encoder_.get();
 111   }
 112   rtcp_.reset(new Rtcp(
 113       cast_environment_->Clock(),
 114       rtcp_feedback_.get(),
 115       paced_packet_sender,
 116       rtp_video_sender_statistics_.get(),
 117       NULL,
 118       video_config.rtcp_mode,
 119       base::TimeDelta::FromMilliseconds(video_config.rtcp_interval),
 120       true,
 121       video_config.sender_ssrc,
 122       video_config.rtcp_c_name));
 123
 124   rtcp_->SetRemoteSSRC(video_config.incoming_feedback_ssrc);
 125   ScheduleNextRtcpReport();
 126   ScheduleNextResendCheck();
 127   ScheduleNextSkippedFramesCheck();
 128 }
 129
 130 VideoSender::~VideoSender() {}
 131
 132 void VideoSender::InsertRawVideoFrame(
 133     const I420VideoFrame* video_frame,
 134     const base::TimeTicks& capture_time,
 135     const base::Closure callback) {
 136   DCHECK(video_encoder_.get()) << "Invalid state";
 137
 138   if (!video_encoder_->EncodeVideoFrame(video_frame, capture_time,
 139       base::Bind(&VideoSender::SendEncodedVideoFrameMainThread,
 140           weak_factory_.GetWeakPtr()), callback)) {
 141     VLOG(0) << "Failed to InsertRawVideoFrame";
 142     cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback);
 143   }
 144 }
 145
 146 void VideoSender::InsertCodedVideoFrame(const EncodedVideoFrame* encoded_frame,
 147                                         const base::TimeTicks& capture_time,
 148                                         const base::Closure callback) {
 149   DCHECK(!video_encoder_.get()) << "Invalid state";
 150   DCHECK(encoded_frame) << "Invalid argument";
 151
 152   SendEncodedVideoFrame(encoded_frame, capture_time);
 153   cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback);
 154 }
 155
 156 void VideoSender::SendEncodedVideoFrameMainThread(
 157     scoped_ptr<EncodedVideoFrame> video_frame,
 158     const base::TimeTicks& capture_time) {
 159   SendEncodedVideoFrame(video_frame.get(), capture_time);
 160 }
 161
 162 void VideoSender::SendEncodedVideoFrame(const EncodedVideoFrame* encoded_frame,
 163                                         const base::TimeTicks& capture_time) {
 164   last_send_time_ = cast_environment_->Clock()->NowTicks();
 165   rtp_sender_->IncomingEncodedVideoFrame(encoded_frame, capture_time);
 166   if (encoded_frame->key_frame) {
 167     VLOG(1) << "Send encoded key frame; frame_id:"
 168             << static_cast<int>(encoded_frame->frame_id);
 169     last_sent_key_frame_id_ = encoded_frame->frame_id;
 170   }
 171   last_sent_frame_id_ = encoded_frame->frame_id;
 172   UpdateFramesInFlight();
 173 }
 174
 175 void VideoSender::OnReceivedIntraFrameRequest() {
 176   if (last_sent_key_frame_id_ != -1) {
 177     DCHECK_GE(255, last_sent_key_frame_id_);
 178     DCHECK_LE(0, last_sent_key_frame_id_);
 179
 180     uint8 frames_in_flight = static_cast<uint8>(last_sent_frame_id_) -
 181                              static_cast<uint8>(last_sent_key_frame_id_);
 182     if (frames_in_flight < (max_unacked_frames_ - 1)) return;
 183   }
 184   video_encoder_controller_->GenerateKeyFrame();
 185   last_acked_frame_id_ = -1;
 186   last_sent_frame_id_ = -1;
 187 }
 188
 189 void VideoSender::IncomingRtcpPacket(const uint8* packet, size_t length,
 190                                      const base::Closure callback) {
 191   rtcp_->IncomingRtcpPacket(packet, length);
 192   cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback);
 193 }
 194
 195 void VideoSender::ScheduleNextRtcpReport() {
 196   base::TimeDelta time_to_next = rtcp_->TimeToSendNextRtcpReport() -
 197      cast_environment_->Clock()->NowTicks();
 198
 199   time_to_next = std::max(time_to_next,
 200       base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 201
 202   cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
 203       base::Bind(&VideoSender::SendRtcpReport, weak_factory_.GetWeakPtr()),
 204                  time_to_next);
 205 }
 206
 207 void VideoSender::SendRtcpReport() {
 208   rtcp_->SendRtcpReport(incoming_feedback_ssrc_);
 209   ScheduleNextRtcpReport();
 210 }
 211
 212 void VideoSender::ScheduleNextResendCheck() {
 213   base::TimeDelta time_to_next;
 214   if (last_send_time_.is_null()) {
 215     time_to_next = rtp_max_delay_;
 216   } else {
 217     time_to_next = last_send_time_ - cast_environment_->Clock()->NowTicks() +
 218         rtp_max_delay_;
 219   }
 220   time_to_next = std::max(time_to_next,
 221       base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 222
 223   cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
 224       base::Bind(&VideoSender::ResendCheck, weak_factory_.GetWeakPtr()),
 225                  time_to_next);
 226 }
 227
 228 void VideoSender::ResendCheck() {
 229   if (!last_send_time_.is_null() && last_sent_frame_id_ != -1) {
 230     base::TimeDelta time_since_last_send =
 231        cast_environment_->Clock()->NowTicks() - last_send_time_;
 232     if (time_since_last_send > rtp_max_delay_) {
 233       if (last_acked_frame_id_ == -1) {
 234         // We have not received any ack, send a key frame.
 235          video_encoder_controller_->GenerateKeyFrame();
 236         last_acked_frame_id_ = -1;
 237         last_sent_frame_id_ = -1;
 238         UpdateFramesInFlight();
 239       } else {
 240         DCHECK_GE(255, last_acked_frame_id_);
 241         DCHECK_LE(0, last_acked_frame_id_);
 242
 243         uint8 frame_id = static_cast<uint8>(last_acked_frame_id_ + 1);
 244         VLOG(1) << "ACK timeout resend frame:" << static_cast<int>(frame_id);
 245         ResendFrame(frame_id);
 246       }
 247     }
 248   }
 249   ScheduleNextResendCheck();
 250 }
 251
 252 void VideoSender::ScheduleNextSkippedFramesCheck() {
 253   base::TimeDelta time_to_next;
 254   if (last_checked_skip_count_time_.is_null()) {
 255     time_to_next =
 256         base::TimeDelta::FromMilliseconds(kSkippedFramesCheckPeriodkMs);
 257   } else {
 258     time_to_next = last_checked_skip_count_time_ -
 259          cast_environment_->Clock()->NowTicks() +
 260          base::TimeDelta::FromMilliseconds(kSkippedFramesCheckPeriodkMs);
 261   }
 262   time_to_next = std::max(time_to_next,
 263       base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 264
 265   cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
 266       base::Bind(&VideoSender::SkippedFramesCheck, weak_factory_.GetWeakPtr()),
 267                  time_to_next);
 268 }
 269
 270 void VideoSender::SkippedFramesCheck() {
 271   int skip_count = video_encoder_controller_->NumberOfSkippedFrames();
 272   if (skip_count - last_skip_count_ >
 273       kSkippedFramesThreshold * max_frame_rate_) {
 274       // TODO(pwestin): Propagate this up to the application.
 275   }
 276   last_skip_count_ = skip_count;
 277   last_checked_skip_count_time_ = cast_environment_->Clock()->NowTicks();
 278   ScheduleNextSkippedFramesCheck();
 279 }
 280
 281 void VideoSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) {
 282   base::TimeDelta rtt;
 283   base::TimeDelta avg_rtt;
 284   base::TimeDelta min_rtt;
 285   base::TimeDelta max_rtt;
 286
 287   if (rtcp_->Rtt(&rtt, &avg_rtt, &min_rtt, &max_rtt)) {
 288     // Don't use a RTT lower than our average.
 289     rtt = std::max(rtt, avg_rtt);
 290   } else {
 291     // We have no measured value use default.
 292     rtt = base::TimeDelta::FromMilliseconds(kStartRttMs);
 293   }
 294   if (cast_feedback.missing_frames_and_packets_.empty()) {
 295     // No lost packets.
 296     int resend_frame = -1;
 297     if (last_sent_frame_id_ == -1) return;
 298
 299     video_encoder_controller_->LatestFrameIdToReference(
 300         cast_feedback.ack_frame_id_);
 301
 302     if (static_cast<uint8>(last_acked_frame_id_ + 1) ==
 303         cast_feedback.ack_frame_id_) {
 304       uint32 new_bitrate = 0;
 305       if (congestion_control_.OnAck(rtt, &new_bitrate)) {
 306         video_encoder_controller_->SetBitRate(new_bitrate);
 307       }
 308     }
 309     if (last_acked_frame_id_ == cast_feedback.ack_frame_id_ &&
 310         // We only count duplicate ACKs when we have sent newer frames.
 311         IsNewerFrameId(last_sent_frame_id_, last_acked_frame_id_)) {
 312       duplicate_ack_++;
 313     } else {
 314       duplicate_ack_ = 0;
 315     }
 316     if (duplicate_ack_ >= 2 && duplicate_ack_ % 3 == 2) {
 317       // Resend last ACK + 1 frame.
 318       resend_frame = static_cast<uint8>(last_acked_frame_id_ + 1);
 319     }
 320     if (resend_frame != -1) {
 321       DCHECK_GE(255, resend_frame);
 322       DCHECK_LE(0, resend_frame);
 323       VLOG(1) << "Received duplicate ACK for frame:"
 324               << static_cast<int>(resend_frame);
 325       ResendFrame(static_cast<uint8>(resend_frame));
 326     }
 327   } else {
 328     rtp_sender_->ResendPackets(cast_feedback.missing_frames_and_packets_);
 329     last_send_time_ = cast_environment_->Clock()->NowTicks();
 330
 331     uint32 new_bitrate = 0;
 332     if (congestion_control_.OnNack(rtt, &new_bitrate)) {
 333       video_encoder_controller_->SetBitRate(new_bitrate);
 334     }
 335   }
 336   ReceivedAck(cast_feedback.ack_frame_id_);
 337 }
 338
 339 void VideoSender::ReceivedAck(uint8 acked_frame_id) {
 340   VLOG(1) << "ReceivedAck:" << static_cast<int>(acked_frame_id);
 341   last_acked_frame_id_ = acked_frame_id;
 342   UpdateFramesInFlight();
 343 }
 344
 345 void VideoSender::UpdateFramesInFlight() {
 346   if (last_sent_frame_id_ != -1) {
 347     DCHECK_GE(255, last_sent_frame_id_);
 348     DCHECK_LE(0, last_sent_frame_id_);
 349     uint8 frames_in_flight;
 350     if (last_acked_frame_id_ != -1) {
 351       DCHECK_GE(255, last_acked_frame_id_);
 352       DCHECK_LE(0, last_acked_frame_id_);
 353       frames_in_flight = static_cast<uint8>(last_sent_frame_id_) -
 354                          static_cast<uint8>(last_acked_frame_id_);
 355     } else {
 356       frames_in_flight = last_sent_frame_id_ + 1;
 357     }
 358     VLOG(1) << "Frames in flight; last sent: " << last_sent_frame_id_
 359             << " last acked:" << last_acked_frame_id_;
 360     if (frames_in_flight >= max_unacked_frames_) {
 361       video_encoder_controller_->SkipNextFrame(true);
 362       return;
 363     }
 364   }
 365   video_encoder_controller_->SkipNextFrame(false);
 366 }
 367
 368 void VideoSender::ResendFrame(uint8 resend_frame_id) {
 369   MissingFramesAndPacketsMap missing_frames_and_packets;
 370   PacketIdSet missing;
 371   missing_frames_and_packets.insert(std::make_pair(resend_frame_id, missing));
 372   rtp_sender_->ResendPackets(missing_frames_and_packets);
 373   last_send_time_ = cast_environment_->Clock()->NowTicks();
 374 }
 375
 376 }  // namespace cast
 377 }  // namespace media