src/third_party/webrtc/modules/video_coding/main/source/receiver.cc

   1 /*
   2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include "webrtc/modules/video_coding/main/source/receiver.h"
  12
  13 #include <assert.h>
  14
  15 #include <cstdlib>
  16
  17 #include "webrtc/modules/video_coding/main/source/encoded_frame.h"
  18 #include "webrtc/modules/video_coding/main/source/internal_defines.h"
  19 #include "webrtc/modules/video_coding/main/source/media_opt_util.h"
  20 #include "webrtc/system_wrappers/interface/clock.h"
  21 #include "webrtc/system_wrappers/interface/trace.h"
  22 #include "webrtc/system_wrappers/interface/trace_event.h"
  23
  24 namespace webrtc {
  25
  26 enum { kMaxReceiverDelayMs = 10000 };
  27
  28 VCMReceiver::VCMReceiver(VCMTiming* timing,
  29                          Clock* clock,
  30                          EventFactory* event_factory,
  31                          int32_t vcm_id,
  32                          int32_t receiver_id,
  33                          bool master)
  34     : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
  35       vcm_id_(vcm_id),
  36       clock_(clock),
  37       receiver_id_(receiver_id),
  38       master_(master),
  39       jitter_buffer_(clock_, event_factory, vcm_id, receiver_id, master),
  40       timing_(timing),
  41       render_wait_event_(event_factory->CreateEvent()),
  42       state_(kPassive),
  43       max_video_delay_ms_(kMaxVideoDelayMs) {}
  44
  45 VCMReceiver::~VCMReceiver() {
  46   render_wait_event_->Set();
  47   delete crit_sect_;
  48 }
  49
  50 void VCMReceiver::Reset() {
  51   CriticalSectionScoped cs(crit_sect_);
  52   if (!jitter_buffer_.Running()) {
  53     jitter_buffer_.Start();
  54   } else {
  55     jitter_buffer_.Flush();
  56   }
  57   render_wait_event_->Reset();
  58   if (master_) {
  59     state_ = kReceiving;
  60   } else {
  61     state_ = kPassive;
  62   }
  63 }
  64
  65 int32_t VCMReceiver::Initialize() {
  66   Reset();
  67   CriticalSectionScoped cs(crit_sect_);
  68   if (!master_) {
  69     SetNackMode(kNoNack, -1, -1);
  70   }
  71   return VCM_OK;
  72 }
  73
  74 void VCMReceiver::UpdateRtt(uint32_t rtt) {
  75   jitter_buffer_.UpdateRtt(rtt);
  76 }
  77
  78 int32_t VCMReceiver::InsertPacket(const VCMPacket& packet,
  79                                   uint16_t frame_width,
  80                                   uint16_t frame_height) {
  81   if (packet.frameType == kVideoFrameKey) {
  82     WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideoCoding,
  83                  VCMId(vcm_id_, receiver_id_),
  84                  "Inserting key frame packet seqnum=%u, timestamp=%u",
  85                  packet.seqNum, packet.timestamp);
  86   }
  87
  88   // Insert the packet into the jitter buffer. The packet can either be empty or
  89   // contain media at this point.
  90   bool retransmitted = false;
  91   const VCMFrameBufferEnum ret = jitter_buffer_.InsertPacket(packet,
  92                                                              &retransmitted);
  93   if (ret == kOldPacket) {
  94     return VCM_OK;
  95   } else if (ret == kFlushIndicator) {
  96     return VCM_FLUSH_INDICATOR;
  97   } else if (ret < 0) {
  98     WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoCoding,
  99                  VCMId(vcm_id_, receiver_id_),
 100                  "Error inserting packet seqnum=%u, timestamp=%u",
 101                  packet.seqNum, packet.timestamp);
 102     return VCM_JITTER_BUFFER_ERROR;
 103   }
 104   if (ret == kCompleteSession && !retransmitted) {
 105     // We don't want to include timestamps which have suffered from
 106     // retransmission here, since we compensate with extra retransmission
 107     // delay within the jitter estimate.
 108     timing_->IncomingTimestamp(packet.timestamp, clock_->TimeInMilliseconds());
 109   }
 110   if (master_) {
 111     // Only trace the primary receiver to make it possible to parse and plot
 112     // the trace file.
 113     WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding,
 114                  VCMId(vcm_id_, receiver_id_),
 115                  "Packet seqnum=%u timestamp=%u inserted at %u",
 116                  packet.seqNum, packet.timestamp,
 117                  MaskWord64ToUWord32(clock_->TimeInMilliseconds()));
 118   }
 119   return VCM_OK;
 120 }
 121
 122 VCMEncodedFrame* VCMReceiver::FrameForDecoding(
 123     uint16_t max_wait_time_ms,
 124     int64_t& next_render_time_ms,
 125     bool render_timing,
 126     VCMReceiver* dual_receiver) {
 127   const int64_t start_time_ms = clock_->TimeInMilliseconds();
 128   uint32_t frame_timestamp = 0;
 129   // Exhaust wait time to get a complete frame for decoding.
 130   bool found_frame = jitter_buffer_.NextCompleteTimestamp(
 131       max_wait_time_ms, &frame_timestamp);
 132
 133   if (!found_frame) {
 134     // Get an incomplete frame when enabled.
 135     const bool dual_receiver_enabled_and_passive = (dual_receiver != NULL &&
 136         dual_receiver->State() == kPassive &&
 137         dual_receiver->NackMode() == kNack);
 138     if (dual_receiver_enabled_and_passive &&
 139         !jitter_buffer_.CompleteSequenceWithNextFrame()) {
 140       // Jitter buffer state might get corrupt with this frame.
 141       dual_receiver->CopyJitterBufferStateFromReceiver(*this);
 142     }
 143     found_frame = jitter_buffer_.NextMaybeIncompleteTimestamp(
 144         &frame_timestamp);
 145   }
 146
 147   if (!found_frame) {
 148     return NULL;
 149   }
 150
 151   // We have a frame - Set timing and render timestamp.
 152   timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs());
 153   const int64_t now_ms = clock_->TimeInMilliseconds();
 154   timing_->UpdateCurrentDelay(frame_timestamp);
 155   next_render_time_ms = timing_->RenderTimeMs(frame_timestamp, now_ms);
 156   // Check render timing.
 157   bool timing_error = false;
 158   // Assume that render timing errors are due to changes in the video stream.
 159   if (next_render_time_ms < 0) {
 160     timing_error = true;
 161   } else if (std::abs(next_render_time_ms - now_ms) > max_video_delay_ms_) {
 162     WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding,
 163                  VCMId(vcm_id_, receiver_id_),
 164                  "This frame is out of our delay bounds, resetting jitter "
 165                  "buffer: %d > %d",
 166                  static_cast<int>(std::abs(next_render_time_ms - now_ms)),
 167                  max_video_delay_ms_);
 168     timing_error = true;
 169   } else if (static_cast<int>(timing_->TargetVideoDelay()) >
 170              max_video_delay_ms_) {
 171     WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding,
 172                  VCMId(vcm_id_, receiver_id_),
 173                  "More than %u ms target delay. Flushing jitter buffer and"
 174                  "resetting timing.", max_video_delay_ms_);
 175     timing_error = true;
 176   }
 177
 178   if (timing_error) {
 179     // Timing error => reset timing and flush the jitter buffer.
 180     jitter_buffer_.Flush();
 181     timing_->Reset();
 182     return NULL;
 183   }
 184
 185   if (!render_timing) {
 186     // Decode frame as close as possible to the render timestamp.
 187     const int32_t available_wait_time = max_wait_time_ms -
 188         static_cast<int32_t>(clock_->TimeInMilliseconds() - start_time_ms);
 189     uint16_t new_max_wait_time = static_cast<uint16_t>(
 190         VCM_MAX(available_wait_time, 0));
 191     uint32_t wait_time_ms = timing_->MaxWaitingTime(
 192         next_render_time_ms, clock_->TimeInMilliseconds());
 193     if (new_max_wait_time < wait_time_ms) {
 194       // We're not allowed to wait until the frame is supposed to be rendered,
 195       // waiting as long as we're allowed to avoid busy looping, and then return
 196       // NULL. Next call to this function might return the frame.
 197       render_wait_event_->Wait(max_wait_time_ms);
 198       return NULL;
 199     }
 200     // Wait until it's time to render.
 201     render_wait_event_->Wait(wait_time_ms);
 202   }
 203
 204   // Extract the frame from the jitter buffer and set the render time.
 205   VCMEncodedFrame* frame = jitter_buffer_.ExtractAndSetDecode(frame_timestamp);
 206   if (frame == NULL) {
 207     return NULL;
 208   }
 209   frame->SetRenderTime(next_render_time_ms);
 210   TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame->TimeStamp(),
 211                           "SetRenderTS", "render_time", next_render_time_ms);
 212   if (dual_receiver != NULL) {
 213     dual_receiver->UpdateState(*frame);
 214   }
 215   if (!frame->Complete()) {
 216     // Update stats for incomplete frames.
 217     bool retransmitted = false;
 218     const int64_t last_packet_time_ms =
 219         jitter_buffer_.LastPacketTime(frame, &retransmitted);
 220     if (last_packet_time_ms >= 0 && !retransmitted) {
 221       // We don't want to include timestamps which have suffered from
 222       // retransmission here, since we compensate with extra retransmission
 223       // delay within the jitter estimate.
 224       timing_->IncomingTimestamp(frame_timestamp, last_packet_time_ms);
 225     }
 226   }
 227   return frame;
 228 }
 229
 230 void VCMReceiver::ReleaseFrame(VCMEncodedFrame* frame) {
 231   jitter_buffer_.ReleaseFrame(frame);
 232 }
 233
 234 void VCMReceiver::ReceiveStatistics(uint32_t* bitrate,
 235                                     uint32_t* framerate) {
 236   assert(bitrate);
 237   assert(framerate);
 238   jitter_buffer_.IncomingRateStatistics(framerate, bitrate);
 239 }
 240
 241 void VCMReceiver::ReceivedFrameCount(VCMFrameCount* frame_count) const {
 242   assert(frame_count);
 243   std::map<FrameType, uint32_t> counts(jitter_buffer_.FrameStatistics());
 244   frame_count->numDeltaFrames = counts[kVideoFrameDelta];
 245   frame_count->numKeyFrames = counts[kVideoFrameKey];
 246 }
 247
 248 uint32_t VCMReceiver::DiscardedPackets() const {
 249   return jitter_buffer_.num_discarded_packets();
 250 }
 251
 252 void VCMReceiver::SetNackMode(VCMNackMode nackMode,
 253                               int low_rtt_nack_threshold_ms,
 254                               int high_rtt_nack_threshold_ms) {
 255   CriticalSectionScoped cs(crit_sect_);
 256   // Default to always having NACK enabled in hybrid mode.
 257   jitter_buffer_.SetNackMode(nackMode, low_rtt_nack_threshold_ms,
 258                              high_rtt_nack_threshold_ms);
 259   if (!master_) {
 260     state_ = kPassive;  // The dual decoder defaults to passive.
 261   }
 262 }
 263
 264 void VCMReceiver::SetNackSettings(size_t max_nack_list_size,
 265                                   int max_packet_age_to_nack,
 266                                   int max_incomplete_time_ms) {
 267   jitter_buffer_.SetNackSettings(max_nack_list_size,
 268                                  max_packet_age_to_nack,
 269                                  max_incomplete_time_ms);
 270 }
 271
 272 VCMNackMode VCMReceiver::NackMode() const {
 273   CriticalSectionScoped cs(crit_sect_);
 274   return jitter_buffer_.nack_mode();
 275 }
 276
 277 VCMNackStatus VCMReceiver::NackList(uint16_t* nack_list,
 278                                     uint16_t size,
 279                                     uint16_t* nack_list_length) {
 280   bool request_key_frame = false;
 281   uint16_t* internal_nack_list = jitter_buffer_.GetNackList(
 282       nack_list_length, &request_key_frame);
 283   if (*nack_list_length > size) {
 284     *nack_list_length = 0;
 285     return kNackNeedMoreMemory;
 286   }
 287   if (internal_nack_list != NULL && *nack_list_length > 0) {
 288     memcpy(nack_list, internal_nack_list, *nack_list_length * sizeof(uint16_t));
 289   }
 290   if (request_key_frame) {
 291     return kNackKeyFrameRequest;
 292   }
 293   return kNackOk;
 294 }
 295
 296 // Decide whether we should change decoder state. This should be done if the
 297 // dual decoder has caught up with the decoder decoding with packet losses.
 298 bool VCMReceiver::DualDecoderCaughtUp(VCMEncodedFrame* dual_frame,
 299                                       VCMReceiver& dual_receiver) const {
 300   if (dual_frame == NULL) {
 301     return false;
 302   }
 303   if (jitter_buffer_.LastDecodedTimestamp() == dual_frame->TimeStamp()) {
 304     dual_receiver.UpdateState(kWaitForPrimaryDecode);
 305     return true;
 306   }
 307   return false;
 308 }
 309
 310 void VCMReceiver::CopyJitterBufferStateFromReceiver(
 311     const VCMReceiver& receiver) {
 312   jitter_buffer_.CopyFrom(receiver.jitter_buffer_);
 313 }
 314
 315 VCMReceiverState VCMReceiver::State() const {
 316   CriticalSectionScoped cs(crit_sect_);
 317   return state_;
 318 }
 319
 320 void VCMReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
 321   jitter_buffer_.SetDecodeErrorMode(decode_error_mode);
 322 }
 323
 324 VCMDecodeErrorMode VCMReceiver::DecodeErrorMode() const {
 325   return jitter_buffer_.decode_error_mode();
 326 }
 327
 328 int VCMReceiver::SetMinReceiverDelay(int desired_delay_ms) {
 329   CriticalSectionScoped cs(crit_sect_);
 330   if (desired_delay_ms < 0 || desired_delay_ms > kMaxReceiverDelayMs) {
 331     return -1;
 332   }
 333   max_video_delay_ms_ = desired_delay_ms + kMaxVideoDelayMs;
 334   // Initializing timing to the desired delay.
 335   timing_->set_min_playout_delay(desired_delay_ms);
 336   return 0;
 337 }
 338
 339 int VCMReceiver::RenderBufferSizeMs() {
 340   uint32_t timestamp_start = 0u;
 341   uint32_t timestamp_end = 0u;
 342   // Render timestamps are computed just prior to decoding. Therefore this is
 343   // only an estimate based on frames' timestamps and current timing state.
 344   jitter_buffer_.RenderBufferSize(&timestamp_start, &timestamp_end);
 345   if (timestamp_start == timestamp_end) {
 346     return 0;
 347   }
 348   // Update timing.
 349   const int64_t now_ms = clock_->TimeInMilliseconds();
 350   timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs());
 351   // Get render timestamps.
 352   uint32_t render_start = timing_->RenderTimeMs(timestamp_start, now_ms);
 353   uint32_t render_end = timing_->RenderTimeMs(timestamp_end, now_ms);
 354   return render_end - render_start;
 355 }
 356
 357 void VCMReceiver::UpdateState(VCMReceiverState new_state) {
 358   CriticalSectionScoped cs(crit_sect_);
 359   assert(!(state_ == kPassive && new_state == kWaitForPrimaryDecode));
 360   state_ = new_state;
 361 }
 362
 363 void VCMReceiver::UpdateState(const VCMEncodedFrame& frame) {
 364   if (jitter_buffer_.nack_mode() == kNoNack) {
 365     // Dual decoder mode has not been enabled.
 366     return;
 367   }
 368   // Update the dual receiver state.
 369   if (frame.Complete() && frame.FrameType() == kVideoFrameKey) {
 370     UpdateState(kPassive);
 371   }
 372   if (State() == kWaitForPrimaryDecode &&
 373       frame.Complete() && !frame.MissingFrame()) {
 374     UpdateState(kPassive);
 375   }
 376   if (frame.MissingFrame() || !frame.Complete()) {
 377     // State was corrupted, enable dual receiver.
 378     UpdateState(kReceiving);
 379   }
 380 }
 381 }  // namespace webrtc