2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
11 #include "webrtc/modules/video_coding/main/source/receiver.h"
17 #include "webrtc/modules/video_coding/main/source/encoded_frame.h"
18 #include "webrtc/modules/video_coding/main/source/internal_defines.h"
19 #include "webrtc/modules/video_coding/main/source/media_opt_util.h"
20 #include "webrtc/system_wrappers/interface/clock.h"
21 #include "webrtc/system_wrappers/interface/logging.h"
22 #include "webrtc/system_wrappers/interface/trace_event.h"
26 enum { kMaxReceiverDelayMs = 10000 };
28 VCMReceiver::VCMReceiver(VCMTiming* timing,
30 EventFactory* event_factory,
32 : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
35 jitter_buffer_(clock_, event_factory),
37 render_wait_event_(event_factory->CreateEvent()),
39 max_video_delay_ms_(kMaxVideoDelayMs) {}
41 VCMReceiver::~VCMReceiver() {
42 render_wait_event_->Set();
46 void VCMReceiver::Reset() {
47 CriticalSectionScoped cs(crit_sect_);
48 if (!jitter_buffer_.Running()) {
49 jitter_buffer_.Start();
51 jitter_buffer_.Flush();
53 render_wait_event_->Reset();
61 int32_t VCMReceiver::Initialize() {
63 CriticalSectionScoped cs(crit_sect_);
65 SetNackMode(kNoNack, -1, -1);
70 void VCMReceiver::UpdateRtt(uint32_t rtt) {
71 jitter_buffer_.UpdateRtt(rtt);
74 int32_t VCMReceiver::InsertPacket(const VCMPacket& packet,
76 uint16_t frame_height) {
77 // Insert the packet into the jitter buffer. The packet can either be empty or
78 // contain media at this point.
79 bool retransmitted = false;
80 const VCMFrameBufferEnum ret = jitter_buffer_.InsertPacket(packet,
82 if (ret == kOldPacket) {
84 } else if (ret == kFlushIndicator) {
85 return VCM_FLUSH_INDICATOR;
87 return VCM_JITTER_BUFFER_ERROR;
89 if (ret == kCompleteSession && !retransmitted) {
90 // We don't want to include timestamps which have suffered from
91 // retransmission here, since we compensate with extra retransmission
92 // delay within the jitter estimate.
93 timing_->IncomingTimestamp(packet.timestamp, clock_->TimeInMilliseconds());
98 VCMEncodedFrame* VCMReceiver::FrameForDecoding(
99 uint16_t max_wait_time_ms,
100 int64_t& next_render_time_ms,
102 VCMReceiver* dual_receiver) {
103 const int64_t start_time_ms = clock_->TimeInMilliseconds();
104 uint32_t frame_timestamp = 0;
105 // Exhaust wait time to get a complete frame for decoding.
106 bool found_frame = jitter_buffer_.NextCompleteTimestamp(
107 max_wait_time_ms, &frame_timestamp);
110 // Get an incomplete frame when enabled.
111 const bool dual_receiver_enabled_and_passive = (dual_receiver != NULL &&
112 dual_receiver->State() == kPassive &&
113 dual_receiver->NackMode() == kNack);
114 if (dual_receiver_enabled_and_passive &&
115 !jitter_buffer_.CompleteSequenceWithNextFrame()) {
116 // Jitter buffer state might get corrupt with this frame.
117 dual_receiver->CopyJitterBufferStateFromReceiver(*this);
119 found_frame = jitter_buffer_.NextMaybeIncompleteTimestamp(
127 // We have a frame - Set timing and render timestamp.
128 timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs());
129 const int64_t now_ms = clock_->TimeInMilliseconds();
130 timing_->UpdateCurrentDelay(frame_timestamp);
131 next_render_time_ms = timing_->RenderTimeMs(frame_timestamp, now_ms);
132 // Check render timing.
133 bool timing_error = false;
134 // Assume that render timing errors are due to changes in the video stream.
135 if (next_render_time_ms < 0) {
137 } else if (std::abs(next_render_time_ms - now_ms) > max_video_delay_ms_) {
138 int frame_delay = static_cast<int>(std::abs(next_render_time_ms - now_ms));
139 LOG(LS_WARNING) << "A frame about to be decoded is out of the configured "
140 << "delay bounds (" << frame_delay << " > "
141 << max_video_delay_ms_
142 << "). Resetting the video jitter buffer.";
144 } else if (static_cast<int>(timing_->TargetVideoDelay()) >
145 max_video_delay_ms_) {
146 LOG(LS_WARNING) << "The video target delay has grown larger than "
147 << max_video_delay_ms_ << " ms. Resetting jitter buffer.";
152 // Timing error => reset timing and flush the jitter buffer.
153 jitter_buffer_.Flush();
158 if (!render_timing) {
159 // Decode frame as close as possible to the render timestamp.
160 const int32_t available_wait_time = max_wait_time_ms -
161 static_cast<int32_t>(clock_->TimeInMilliseconds() - start_time_ms);
162 uint16_t new_max_wait_time = static_cast<uint16_t>(
163 VCM_MAX(available_wait_time, 0));
164 uint32_t wait_time_ms = timing_->MaxWaitingTime(
165 next_render_time_ms, clock_->TimeInMilliseconds());
166 if (new_max_wait_time < wait_time_ms) {
167 // We're not allowed to wait until the frame is supposed to be rendered,
168 // waiting as long as we're allowed to avoid busy looping, and then return
169 // NULL. Next call to this function might return the frame.
170 render_wait_event_->Wait(max_wait_time_ms);
173 // Wait until it's time to render.
174 render_wait_event_->Wait(wait_time_ms);
177 // Extract the frame from the jitter buffer and set the render time.
178 VCMEncodedFrame* frame = jitter_buffer_.ExtractAndSetDecode(frame_timestamp);
182 frame->SetRenderTime(next_render_time_ms);
183 TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame->TimeStamp(),
184 "SetRenderTS", "render_time", next_render_time_ms);
185 if (dual_receiver != NULL) {
186 dual_receiver->UpdateState(*frame);
188 if (!frame->Complete()) {
189 // Update stats for incomplete frames.
190 bool retransmitted = false;
191 const int64_t last_packet_time_ms =
192 jitter_buffer_.LastPacketTime(frame, &retransmitted);
193 if (last_packet_time_ms >= 0 && !retransmitted) {
194 // We don't want to include timestamps which have suffered from
195 // retransmission here, since we compensate with extra retransmission
196 // delay within the jitter estimate.
197 timing_->IncomingTimestamp(frame_timestamp, last_packet_time_ms);
203 void VCMReceiver::ReleaseFrame(VCMEncodedFrame* frame) {
204 jitter_buffer_.ReleaseFrame(frame);
207 void VCMReceiver::ReceiveStatistics(uint32_t* bitrate,
208 uint32_t* framerate) {
211 jitter_buffer_.IncomingRateStatistics(framerate, bitrate);
214 void VCMReceiver::ReceivedFrameCount(VCMFrameCount* frame_count) const {
216 std::map<FrameType, uint32_t> counts(jitter_buffer_.FrameStatistics());
217 frame_count->numDeltaFrames = counts[kVideoFrameDelta];
218 frame_count->numKeyFrames = counts[kVideoFrameKey];
221 uint32_t VCMReceiver::DiscardedPackets() const {
222 return jitter_buffer_.num_discarded_packets();
225 void VCMReceiver::SetNackMode(VCMNackMode nackMode,
226 int low_rtt_nack_threshold_ms,
227 int high_rtt_nack_threshold_ms) {
228 CriticalSectionScoped cs(crit_sect_);
229 // Default to always having NACK enabled in hybrid mode.
230 jitter_buffer_.SetNackMode(nackMode, low_rtt_nack_threshold_ms,
231 high_rtt_nack_threshold_ms);
233 state_ = kPassive; // The dual decoder defaults to passive.
237 void VCMReceiver::SetNackSettings(size_t max_nack_list_size,
238 int max_packet_age_to_nack,
239 int max_incomplete_time_ms) {
240 jitter_buffer_.SetNackSettings(max_nack_list_size,
241 max_packet_age_to_nack,
242 max_incomplete_time_ms);
245 VCMNackMode VCMReceiver::NackMode() const {
246 CriticalSectionScoped cs(crit_sect_);
247 return jitter_buffer_.nack_mode();
250 VCMNackStatus VCMReceiver::NackList(uint16_t* nack_list,
252 uint16_t* nack_list_length) {
253 bool request_key_frame = false;
254 uint16_t* internal_nack_list = jitter_buffer_.GetNackList(
255 nack_list_length, &request_key_frame);
256 assert(*nack_list_length <= size);
257 if (internal_nack_list != NULL && *nack_list_length > 0) {
258 memcpy(nack_list, internal_nack_list, *nack_list_length * sizeof(uint16_t));
260 if (request_key_frame) {
261 return kNackKeyFrameRequest;
266 // Decide whether we should change decoder state. This should be done if the
267 // dual decoder has caught up with the decoder decoding with packet losses.
268 bool VCMReceiver::DualDecoderCaughtUp(VCMEncodedFrame* dual_frame,
269 VCMReceiver& dual_receiver) const {
270 if (dual_frame == NULL) {
273 if (jitter_buffer_.LastDecodedTimestamp() == dual_frame->TimeStamp()) {
274 dual_receiver.UpdateState(kWaitForPrimaryDecode);
280 void VCMReceiver::CopyJitterBufferStateFromReceiver(
281 const VCMReceiver& receiver) {
282 jitter_buffer_.CopyFrom(receiver.jitter_buffer_);
285 VCMReceiverState VCMReceiver::State() const {
286 CriticalSectionScoped cs(crit_sect_);
290 void VCMReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
291 jitter_buffer_.SetDecodeErrorMode(decode_error_mode);
294 VCMDecodeErrorMode VCMReceiver::DecodeErrorMode() const {
295 return jitter_buffer_.decode_error_mode();
298 int VCMReceiver::SetMinReceiverDelay(int desired_delay_ms) {
299 CriticalSectionScoped cs(crit_sect_);
300 if (desired_delay_ms < 0 || desired_delay_ms > kMaxReceiverDelayMs) {
303 max_video_delay_ms_ = desired_delay_ms + kMaxVideoDelayMs;
304 // Initializing timing to the desired delay.
305 timing_->set_min_playout_delay(desired_delay_ms);
309 int VCMReceiver::RenderBufferSizeMs() {
310 uint32_t timestamp_start = 0u;
311 uint32_t timestamp_end = 0u;
312 // Render timestamps are computed just prior to decoding. Therefore this is
313 // only an estimate based on frames' timestamps and current timing state.
314 jitter_buffer_.RenderBufferSize(×tamp_start, ×tamp_end);
315 if (timestamp_start == timestamp_end) {
319 const int64_t now_ms = clock_->TimeInMilliseconds();
320 timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs());
321 // Get render timestamps.
322 uint32_t render_start = timing_->RenderTimeMs(timestamp_start, now_ms);
323 uint32_t render_end = timing_->RenderTimeMs(timestamp_end, now_ms);
324 return render_end - render_start;
327 void VCMReceiver::UpdateState(VCMReceiverState new_state) {
328 CriticalSectionScoped cs(crit_sect_);
329 assert(!(state_ == kPassive && new_state == kWaitForPrimaryDecode));
333 void VCMReceiver::UpdateState(const VCMEncodedFrame& frame) {
334 if (jitter_buffer_.nack_mode() == kNoNack) {
335 // Dual decoder mode has not been enabled.
338 // Update the dual receiver state.
339 if (frame.Complete() && frame.FrameType() == kVideoFrameKey) {
340 UpdateState(kPassive);
342 if (State() == kWaitForPrimaryDecode &&
343 frame.Complete() && !frame.MissingFrame()) {
344 UpdateState(kPassive);
346 if (frame.MissingFrame() || !frame.Complete()) {
347 // State was corrupted, enable dual receiver.
348 UpdateState(kReceiving);
351 } // namespace webrtc