src/third_party/webrtc/modules/video_coding/main/source/receiver.cc

   1 /*
   2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include "webrtc/modules/video_coding/main/source/receiver.h"
  12
  13 #include <assert.h>
  14
  15 #include <cstdlib>
  16
  17 #include "webrtc/modules/video_coding/main/source/encoded_frame.h"
  18 #include "webrtc/modules/video_coding/main/source/internal_defines.h"
  19 #include "webrtc/modules/video_coding/main/source/media_opt_util.h"
  20 #include "webrtc/system_wrappers/interface/clock.h"
  21 #include "webrtc/system_wrappers/interface/logging.h"
  22 #include "webrtc/system_wrappers/interface/trace_event.h"
  23
  24 namespace webrtc {
  25
  26 enum { kMaxReceiverDelayMs = 10000 };
  27
  28 VCMReceiver::VCMReceiver(VCMTiming* timing,
  29                          Clock* clock,
  30                          EventFactory* event_factory,
  31                          bool master)
  32     : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
  33       clock_(clock),
  34       master_(master),
  35       jitter_buffer_(clock_, event_factory),
  36       timing_(timing),
  37       render_wait_event_(event_factory->CreateEvent()),
  38       state_(kPassive),
  39       max_video_delay_ms_(kMaxVideoDelayMs) {}
  40
  41 VCMReceiver::~VCMReceiver() {
  42   render_wait_event_->Set();
  43   delete crit_sect_;
  44 }
  45
  46 void VCMReceiver::Reset() {
  47   CriticalSectionScoped cs(crit_sect_);
  48   if (!jitter_buffer_.Running()) {
  49     jitter_buffer_.Start();
  50   } else {
  51     jitter_buffer_.Flush();
  52   }
  53   render_wait_event_->Reset();
  54   if (master_) {
  55     state_ = kReceiving;
  56   } else {
  57     state_ = kPassive;
  58   }
  59 }
  60
  61 int32_t VCMReceiver::Initialize() {
  62   Reset();
  63   CriticalSectionScoped cs(crit_sect_);
  64   if (!master_) {
  65     SetNackMode(kNoNack, -1, -1);
  66   }
  67   return VCM_OK;
  68 }
  69
  70 void VCMReceiver::UpdateRtt(uint32_t rtt) {
  71   jitter_buffer_.UpdateRtt(rtt);
  72 }
  73
  74 int32_t VCMReceiver::InsertPacket(const VCMPacket& packet,
  75                                   uint16_t frame_width,
  76                                   uint16_t frame_height) {
  77   // Insert the packet into the jitter buffer. The packet can either be empty or
  78   // contain media at this point.
  79   bool retransmitted = false;
  80   const VCMFrameBufferEnum ret = jitter_buffer_.InsertPacket(packet,
  81                                                              &retransmitted);
  82   if (ret == kOldPacket) {
  83     return VCM_OK;
  84   } else if (ret == kFlushIndicator) {
  85     return VCM_FLUSH_INDICATOR;
  86   } else if (ret < 0) {
  87     return VCM_JITTER_BUFFER_ERROR;
  88   }
  89   if (ret == kCompleteSession && !retransmitted) {
  90     // We don't want to include timestamps which have suffered from
  91     // retransmission here, since we compensate with extra retransmission
  92     // delay within the jitter estimate.
  93     timing_->IncomingTimestamp(packet.timestamp, clock_->TimeInMilliseconds());
  94   }
  95   return VCM_OK;
  96 }
  97
  98 VCMEncodedFrame* VCMReceiver::FrameForDecoding(
  99     uint16_t max_wait_time_ms,
 100     int64_t& next_render_time_ms,
 101     bool render_timing,
 102     VCMReceiver* dual_receiver) {
 103   const int64_t start_time_ms = clock_->TimeInMilliseconds();
 104   uint32_t frame_timestamp = 0;
 105   // Exhaust wait time to get a complete frame for decoding.
 106   bool found_frame = jitter_buffer_.NextCompleteTimestamp(
 107       max_wait_time_ms, &frame_timestamp);
 108
 109   if (!found_frame) {
 110     // Get an incomplete frame when enabled.
 111     const bool dual_receiver_enabled_and_passive = (dual_receiver != NULL &&
 112         dual_receiver->State() == kPassive &&
 113         dual_receiver->NackMode() == kNack);
 114     if (dual_receiver_enabled_and_passive &&
 115         !jitter_buffer_.CompleteSequenceWithNextFrame()) {
 116       // Jitter buffer state might get corrupt with this frame.
 117       dual_receiver->CopyJitterBufferStateFromReceiver(*this);
 118     }
 119     found_frame = jitter_buffer_.NextMaybeIncompleteTimestamp(
 120         &frame_timestamp);
 121   }
 122
 123   if (!found_frame) {
 124     return NULL;
 125   }
 126
 127   // We have a frame - Set timing and render timestamp.
 128   timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs());
 129   const int64_t now_ms = clock_->TimeInMilliseconds();
 130   timing_->UpdateCurrentDelay(frame_timestamp);
 131   next_render_time_ms = timing_->RenderTimeMs(frame_timestamp, now_ms);
 132   // Check render timing.
 133   bool timing_error = false;
 134   // Assume that render timing errors are due to changes in the video stream.
 135   if (next_render_time_ms < 0) {
 136     timing_error = true;
 137   } else if (std::abs(next_render_time_ms - now_ms) > max_video_delay_ms_) {
 138     int frame_delay = static_cast<int>(std::abs(next_render_time_ms - now_ms));
 139     LOG(LS_WARNING) << "A frame about to be decoded is out of the configured "
 140                     << "delay bounds (" << frame_delay << " > "
 141                     << max_video_delay_ms_
 142                     << "). Resetting the video jitter buffer.";
 143     timing_error = true;
 144   } else if (static_cast<int>(timing_->TargetVideoDelay()) >
 145              max_video_delay_ms_) {
 146     LOG(LS_WARNING) << "The video target delay has grown larger than "
 147                     << max_video_delay_ms_ << " ms. Resetting jitter buffer.";
 148     timing_error = true;
 149   }
 150
 151   if (timing_error) {
 152     // Timing error => reset timing and flush the jitter buffer.
 153     jitter_buffer_.Flush();
 154     timing_->Reset();
 155     return NULL;
 156   }
 157
 158   if (!render_timing) {
 159     // Decode frame as close as possible to the render timestamp.
 160     const int32_t available_wait_time = max_wait_time_ms -
 161         static_cast<int32_t>(clock_->TimeInMilliseconds() - start_time_ms);
 162     uint16_t new_max_wait_time = static_cast<uint16_t>(
 163         VCM_MAX(available_wait_time, 0));
 164     uint32_t wait_time_ms = timing_->MaxWaitingTime(
 165         next_render_time_ms, clock_->TimeInMilliseconds());
 166     if (new_max_wait_time < wait_time_ms) {
 167       // We're not allowed to wait until the frame is supposed to be rendered,
 168       // waiting as long as we're allowed to avoid busy looping, and then return
 169       // NULL. Next call to this function might return the frame.
 170       render_wait_event_->Wait(max_wait_time_ms);
 171       return NULL;
 172     }
 173     // Wait until it's time to render.
 174     render_wait_event_->Wait(wait_time_ms);
 175   }
 176
 177   // Extract the frame from the jitter buffer and set the render time.
 178   VCMEncodedFrame* frame = jitter_buffer_.ExtractAndSetDecode(frame_timestamp);
 179   if (frame == NULL) {
 180     return NULL;
 181   }
 182   frame->SetRenderTime(next_render_time_ms);
 183   TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame->TimeStamp(),
 184                           "SetRenderTS", "render_time", next_render_time_ms);
 185   if (dual_receiver != NULL) {
 186     dual_receiver->UpdateState(*frame);
 187   }
 188   if (!frame->Complete()) {
 189     // Update stats for incomplete frames.
 190     bool retransmitted = false;
 191     const int64_t last_packet_time_ms =
 192         jitter_buffer_.LastPacketTime(frame, &retransmitted);
 193     if (last_packet_time_ms >= 0 && !retransmitted) {
 194       // We don't want to include timestamps which have suffered from
 195       // retransmission here, since we compensate with extra retransmission
 196       // delay within the jitter estimate.
 197       timing_->IncomingTimestamp(frame_timestamp, last_packet_time_ms);
 198     }
 199   }
 200   return frame;
 201 }
 202
 203 void VCMReceiver::ReleaseFrame(VCMEncodedFrame* frame) {
 204   jitter_buffer_.ReleaseFrame(frame);
 205 }
 206
 207 void VCMReceiver::ReceiveStatistics(uint32_t* bitrate,
 208                                     uint32_t* framerate) {
 209   assert(bitrate);
 210   assert(framerate);
 211   jitter_buffer_.IncomingRateStatistics(framerate, bitrate);
 212 }
 213
 214 void VCMReceiver::ReceivedFrameCount(VCMFrameCount* frame_count) const {
 215   assert(frame_count);
 216   std::map<FrameType, uint32_t> counts(jitter_buffer_.FrameStatistics());
 217   frame_count->numDeltaFrames = counts[kVideoFrameDelta];
 218   frame_count->numKeyFrames = counts[kVideoFrameKey];
 219 }
 220
 221 uint32_t VCMReceiver::DiscardedPackets() const {
 222   return jitter_buffer_.num_discarded_packets();
 223 }
 224
 225 void VCMReceiver::SetNackMode(VCMNackMode nackMode,
 226                               int low_rtt_nack_threshold_ms,
 227                               int high_rtt_nack_threshold_ms) {
 228   CriticalSectionScoped cs(crit_sect_);
 229   // Default to always having NACK enabled in hybrid mode.
 230   jitter_buffer_.SetNackMode(nackMode, low_rtt_nack_threshold_ms,
 231                              high_rtt_nack_threshold_ms);
 232   if (!master_) {
 233     state_ = kPassive;  // The dual decoder defaults to passive.
 234   }
 235 }
 236
 237 void VCMReceiver::SetNackSettings(size_t max_nack_list_size,
 238                                   int max_packet_age_to_nack,
 239                                   int max_incomplete_time_ms) {
 240   jitter_buffer_.SetNackSettings(max_nack_list_size,
 241                                  max_packet_age_to_nack,
 242                                  max_incomplete_time_ms);
 243 }
 244
 245 VCMNackMode VCMReceiver::NackMode() const {
 246   CriticalSectionScoped cs(crit_sect_);
 247   return jitter_buffer_.nack_mode();
 248 }
 249
 250 VCMNackStatus VCMReceiver::NackList(uint16_t* nack_list,
 251                                     uint16_t size,
 252                                     uint16_t* nack_list_length) {
 253   bool request_key_frame = false;
 254   uint16_t* internal_nack_list = jitter_buffer_.GetNackList(
 255       nack_list_length, &request_key_frame);
 256   assert(*nack_list_length <= size);
 257   if (internal_nack_list != NULL && *nack_list_length > 0) {
 258     memcpy(nack_list, internal_nack_list, *nack_list_length * sizeof(uint16_t));
 259   }
 260   if (request_key_frame) {
 261     return kNackKeyFrameRequest;
 262   }
 263   return kNackOk;
 264 }
 265
 266 // Decide whether we should change decoder state. This should be done if the
 267 // dual decoder has caught up with the decoder decoding with packet losses.
 268 bool VCMReceiver::DualDecoderCaughtUp(VCMEncodedFrame* dual_frame,
 269                                       VCMReceiver& dual_receiver) const {
 270   if (dual_frame == NULL) {
 271     return false;
 272   }
 273   if (jitter_buffer_.LastDecodedTimestamp() == dual_frame->TimeStamp()) {
 274     dual_receiver.UpdateState(kWaitForPrimaryDecode);
 275     return true;
 276   }
 277   return false;
 278 }
 279
 280 void VCMReceiver::CopyJitterBufferStateFromReceiver(
 281     const VCMReceiver& receiver) {
 282   jitter_buffer_.CopyFrom(receiver.jitter_buffer_);
 283 }
 284
 285 VCMReceiverState VCMReceiver::State() const {
 286   CriticalSectionScoped cs(crit_sect_);
 287   return state_;
 288 }
 289
 290 void VCMReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
 291   jitter_buffer_.SetDecodeErrorMode(decode_error_mode);
 292 }
 293
 294 VCMDecodeErrorMode VCMReceiver::DecodeErrorMode() const {
 295   return jitter_buffer_.decode_error_mode();
 296 }
 297
 298 int VCMReceiver::SetMinReceiverDelay(int desired_delay_ms) {
 299   CriticalSectionScoped cs(crit_sect_);
 300   if (desired_delay_ms < 0 || desired_delay_ms > kMaxReceiverDelayMs) {
 301     return -1;
 302   }
 303   max_video_delay_ms_ = desired_delay_ms + kMaxVideoDelayMs;
 304   // Initializing timing to the desired delay.
 305   timing_->set_min_playout_delay(desired_delay_ms);
 306   return 0;
 307 }
 308
 309 int VCMReceiver::RenderBufferSizeMs() {
 310   uint32_t timestamp_start = 0u;
 311   uint32_t timestamp_end = 0u;
 312   // Render timestamps are computed just prior to decoding. Therefore this is
 313   // only an estimate based on frames' timestamps and current timing state.
 314   jitter_buffer_.RenderBufferSize(&timestamp_start, &timestamp_end);
 315   if (timestamp_start == timestamp_end) {
 316     return 0;
 317   }
 318   // Update timing.
 319   const int64_t now_ms = clock_->TimeInMilliseconds();
 320   timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs());
 321   // Get render timestamps.
 322   uint32_t render_start = timing_->RenderTimeMs(timestamp_start, now_ms);
 323   uint32_t render_end = timing_->RenderTimeMs(timestamp_end, now_ms);
 324   return render_end - render_start;
 325 }
 326
 327 void VCMReceiver::UpdateState(VCMReceiverState new_state) {
 328   CriticalSectionScoped cs(crit_sect_);
 329   assert(!(state_ == kPassive && new_state == kWaitForPrimaryDecode));
 330   state_ = new_state;
 331 }
 332
 333 void VCMReceiver::UpdateState(const VCMEncodedFrame& frame) {
 334   if (jitter_buffer_.nack_mode() == kNoNack) {
 335     // Dual decoder mode has not been enabled.
 336     return;
 337   }
 338   // Update the dual receiver state.
 339   if (frame.Complete() && frame.FrameType() == kVideoFrameKey) {
 340     UpdateState(kPassive);
 341   }
 342   if (State() == kWaitForPrimaryDecode &&
 343       frame.Complete() && !frame.MissingFrame()) {
 344     UpdateState(kPassive);
 345   }
 346   if (frame.MissingFrame() || !frame.Complete()) {
 347     // State was corrupted, enable dual receiver.
 348     UpdateState(kReceiving);
 349   }
 350 }
 351 }  // namespace webrtc