#include "media/cast/sender/vp8_encoder.h"
-#include <vector>
-
#include "base/logging.h"
#include "media/base/video_frame.h"
#include "media/cast/cast_defines.h"
namespace media {
namespace cast {
-static const uint32 kMinIntra = 300;
+namespace {
+
+// After a pause in the video stream, what is the maximum duration amount to
+// pass to the encoder for the next frame (in terms of 1/max_fps sized periods)?
+// This essentially controls the encoded size of the first frame that follows a
+// pause in the video stream.
+const int kRestartFramePeriods = 3;
-Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config,
- int max_unacked_frames)
+} // namespace
+
+Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config)
: cast_config_(video_config),
use_multiple_video_buffers_(
cast_config_.max_number_of_video_buffers_used ==
kNumberOfVp8VideoBuffers),
+ raw_image_(nullptr),
key_frame_requested_(true),
- first_frame_received_(false),
last_encoded_frame_id_(kStartFrameId),
last_acked_frame_id_(kStartFrameId),
- frame_id_to_reference_(kStartFrameId - 1),
undroppable_frames_(0) {
+ config_.g_timebase.den = 0; // Not initialized.
+
// VP8 have 3 buffers available for prediction, with
// max_number_of_video_buffers_used set to 1 we maximize the coding efficiency
// however in this mode we can not skip frames in the receiver to catch up
}
Vp8Encoder::~Vp8Encoder() {
- vpx_codec_destroy(encoder_.get());
+ DCHECK(thread_checker_.CalledOnValidThread());
+ if (is_initialized())
+ vpx_codec_destroy(&encoder_);
vpx_img_free(raw_image_);
}
void Vp8Encoder::Initialize() {
DCHECK(thread_checker_.CalledOnValidThread());
- config_.reset(new vpx_codec_enc_cfg_t());
- encoder_.reset(new vpx_codec_ctx_t());
+ DCHECK(!is_initialized());
// Creating a wrapper to the image - setting image data to NULL. Actual
// pointer will be set during encode. Setting align to 1, as it is
buffer_state_[i].frame_id = kStartFrameId;
buffer_state_[i].state = kBufferStartState;
}
- InitEncode(cast_config_.number_of_encode_threads);
-}
-void Vp8Encoder::InitEncode(int number_of_encode_threads) {
- DCHECK(thread_checker_.CalledOnValidThread());
// Populate encoder configuration with default values.
- if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), config_.get(), 0)) {
- DCHECK(false) << "Invalid return value";
+ if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &config_, 0)) {
+ NOTREACHED() << "Invalid return value";
+ config_.g_timebase.den = 0; // Do not call vpx_codec_destroy() in dtor.
+ return;
}
- config_->g_w = cast_config_.width;
- config_->g_h = cast_config_.height;
- config_->rc_target_bitrate = cast_config_.start_bitrate / 1000; // In kbit/s.
-
- // Setting the codec time base.
- config_->g_timebase.num = 1;
- config_->g_timebase.den = kVideoFrequency;
- config_->g_lag_in_frames = 0;
- config_->kf_mode = VPX_KF_DISABLED;
+
+ config_.g_threads = cast_config_.number_of_encode_threads;
+ config_.g_w = cast_config_.width;
+ config_.g_h = cast_config_.height;
+ // Set the timebase to match that of base::TimeDelta.
+ config_.g_timebase.num = 1;
+ config_.g_timebase.den = base::Time::kMicrosecondsPerSecond;
if (use_multiple_video_buffers_) {
// We must enable error resilience when we use multiple buffers, due to
// codec requirements.
- config_->g_error_resilient = 1;
+ config_.g_error_resilient = 1;
}
- config_->g_threads = number_of_encode_threads;
+ config_.g_pass = VPX_RC_ONE_PASS;
+ config_.g_lag_in_frames = 0; // Immediate data output for each frame.
// Rate control settings.
- // Never allow the encoder to drop frame internally.
- config_->rc_dropframe_thresh = 0;
- config_->rc_end_usage = VPX_CBR;
- config_->g_pass = VPX_RC_ONE_PASS;
- config_->rc_resize_allowed = 0;
- config_->rc_min_quantizer = cast_config_.min_qp;
- config_->rc_max_quantizer = cast_config_.max_qp;
- config_->rc_undershoot_pct = 100;
- config_->rc_overshoot_pct = 15;
- config_->rc_buf_initial_sz = 500;
- config_->rc_buf_optimal_sz = 600;
- config_->rc_buf_sz = 1000;
-
- // set the maximum target size of any key-frame.
- uint32 rc_max_intra_target = MaxIntraTarget(config_->rc_buf_optimal_sz);
+ config_.rc_dropframe_thresh = 0; // The encoder may not drop any frames.
+ config_.rc_resize_allowed = 0; // TODO(miu): Why not? Investigate this.
+ config_.rc_end_usage = VPX_CBR;
+ config_.rc_target_bitrate = cast_config_.start_bitrate / 1000; // In kbit/s.
+ config_.rc_min_quantizer = cast_config_.min_qp;
+ config_.rc_max_quantizer = cast_config_.max_qp;
+ // TODO(miu): Revisit these now that the encoder is being successfully
+ // micro-managed.
+ config_.rc_undershoot_pct = 100;
+ config_.rc_overshoot_pct = 15;
+ // TODO(miu): Document why these rc_buf_*_sz values were chosen and/or
+ // research for better values. Should they be computed from the target
+ // playout delay?
+ config_.rc_buf_initial_sz = 500;
+ config_.rc_buf_optimal_sz = 600;
+ config_.rc_buf_sz = 1000;
+
+ config_.kf_mode = VPX_KF_DISABLED;
+
vpx_codec_flags_t flags = 0;
- if (vpx_codec_enc_init(
- encoder_.get(), vpx_codec_vp8_cx(), config_.get(), flags)) {
- DCHECK(false) << "vpx_codec_enc_init() failed.";
- encoder_.reset();
+ if (vpx_codec_enc_init(&encoder_, vpx_codec_vp8_cx(), &config_, flags)) {
+ NOTREACHED() << "vpx_codec_enc_init() failed.";
+ config_.g_timebase.den = 0; // Do not call vpx_codec_destroy() in dtor.
return;
}
- vpx_codec_control(encoder_.get(), VP8E_SET_STATIC_THRESHOLD, 1);
- vpx_codec_control(encoder_.get(), VP8E_SET_NOISE_SENSITIVITY, 0);
- vpx_codec_control(encoder_.get(), VP8E_SET_CPUUSED, -6);
- vpx_codec_control(
- encoder_.get(), VP8E_SET_MAX_INTRA_BITRATE_PCT, rc_max_intra_target);
+
+ // Raise the threshold for considering macroblocks as static. The default is
+ // zero, so this setting makes the encoder less sensitive to motion. This
+ // lowers the probability of needing to utilize more CPU to search for motion
+ // vectors.
+ vpx_codec_control(&encoder_, VP8E_SET_STATIC_THRESHOLD, 1);
+
+ // Improve quality by enabling sets of codec features that utilize more CPU.
+ // The default is zero, with increasingly more CPU to be used as the value is
+ // more negative.
+ // TODO(miu): Document why this value was chosen and expected behaviors.
+ // Should this be dynamic w.r.t. hardware performance?
+ vpx_codec_control(&encoder_, VP8E_SET_CPUUSED, -6);
}
-bool Vp8Encoder::Encode(const scoped_refptr<media::VideoFrame>& video_frame,
- EncodedFrame* encoded_image) {
+void Vp8Encoder::Encode(const scoped_refptr<media::VideoFrame>& video_frame,
+ const base::TimeTicks& reference_time,
+ EncodedFrame* encoded_frame) {
DCHECK(thread_checker_.CalledOnValidThread());
+ DCHECK(encoded_frame);
+
+ CHECK(is_initialized()); // No illegal reference to |config_| or |encoder_|.
+
// Image in vpx_image_t format.
// Input image is const. VP8's raw image is not defined as const.
raw_image_->planes[VPX_PLANE_Y] =
GetCodecUpdateFlags(buffer_to_update, &flags);
}
- // Note: The duration does not reflect the real time between frames. This is
- // done to keep the encoder happy.
- //
- // TODO(miu): This is a semi-hack. We should consider using
- // |video_frame->timestamp()| instead.
- uint32 duration = kVideoFrequency / cast_config_.max_frame_rate;
-
- // Note: Timestamp here is used for bitrate calculation. The absolute value
- // is not important.
- if (!first_frame_received_) {
- first_frame_received_ = true;
- first_frame_timestamp_ = video_frame->timestamp();
- }
-
- vpx_codec_pts_t timestamp =
- (video_frame->timestamp() - first_frame_timestamp_).InMicroseconds() *
- kVideoFrequency / base::Time::kMicrosecondsPerSecond;
-
- if (vpx_codec_encode(encoder_.get(),
- raw_image_,
- timestamp,
- duration,
- flags,
- VPX_DL_REALTIME) != VPX_CODEC_OK) {
- LOG(ERROR) << "Failed to encode for once.";
- return false;
- }
-
- // Get encoded frame.
+ // The frame duration given to the VP8 codec affects a number of important
+ // behaviors, including: per-frame bandwidth, CPU time spent encoding,
+ // temporal quality trade-offs, and key/golden/alt-ref frame generation
+ // intervals. Use the actual amount of time between the current and previous
+ // frames as a prediction for the next frame's duration, but bound the
+ // prediction to account for the fact that the frame rate can be highly
+ // variable, including long pauses in the video stream.
+ const base::TimeDelta minimum_frame_duration =
+ base::TimeDelta::FromSecondsD(1.0 / cast_config_.max_frame_rate);
+ const base::TimeDelta maximum_frame_duration =
+ base::TimeDelta::FromSecondsD(static_cast<double>(kRestartFramePeriods) /
+ cast_config_.max_frame_rate);
+ const base::TimeDelta last_frame_duration =
+ video_frame->timestamp() - last_frame_timestamp_;
+ const base::TimeDelta predicted_frame_duration =
+ std::max(minimum_frame_duration,
+ std::min(maximum_frame_duration, last_frame_duration));
+ last_frame_timestamp_ = video_frame->timestamp();
+
+ // Encode the frame. The presentation time stamp argument here is fixed to
+ // zero to force the encoder to base its single-frame bandwidth calculations
+ // entirely on |predicted_frame_duration| and the target bitrate setting being
+ // micro-managed via calls to UpdateRates().
+ CHECK_EQ(vpx_codec_encode(&encoder_,
+ raw_image_,
+ 0,
+ predicted_frame_duration.InMicroseconds(),
+ flags,
+ VPX_DL_REALTIME),
+ VPX_CODEC_OK)
+ << "BUG: Invalid arguments passed to vpx_codec_encode().";
+
+ // Pull data from the encoder, populating a new EncodedFrame.
+ encoded_frame->frame_id = ++last_encoded_frame_id_;
const vpx_codec_cx_pkt_t* pkt = NULL;
vpx_codec_iter_t iter = NULL;
- bool is_key_frame = false;
- while ((pkt = vpx_codec_get_cx_data(encoder_.get(), &iter)) != NULL) {
+ while ((pkt = vpx_codec_get_cx_data(&encoder_, &iter)) != NULL) {
if (pkt->kind != VPX_CODEC_CX_FRAME_PKT)
continue;
- encoded_image->data.assign(
+ if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) {
+ // TODO(hubbe): Replace "dependency" with a "bool is_key_frame".
+ encoded_frame->dependency = EncodedFrame::KEY;
+ encoded_frame->referenced_frame_id = encoded_frame->frame_id;
+ } else {
+ encoded_frame->dependency = EncodedFrame::DEPENDENT;
+ // Frame dependencies could theoretically be relaxed by looking for the
+ // VPX_FRAME_IS_DROPPABLE flag, but in recent testing (Oct 2014), this
+ // flag never seems to be set.
+ encoded_frame->referenced_frame_id = latest_frame_id_to_reference;
+ }
+ encoded_frame->rtp_timestamp =
+ TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency);
+ encoded_frame->reference_time = reference_time;
+ encoded_frame->data.assign(
static_cast<const uint8*>(pkt->data.frame.buf),
static_cast<const uint8*>(pkt->data.frame.buf) + pkt->data.frame.sz);
- is_key_frame = !!(pkt->data.frame.flags & VPX_FRAME_IS_KEY);
break; // Done, since all data is provided in one CX_FRAME_PKT packet.
}
- // Don't update frame_id for zero size frames.
- if (encoded_image->data.empty())
- return true;
-
- // Populate the encoded frame.
- encoded_image->frame_id = ++last_encoded_frame_id_;
- if (is_key_frame) {
- // TODO(Hubbe): Replace "dependency" with a "bool is_key_frame".
- encoded_image->dependency = EncodedFrame::KEY;
- encoded_image->referenced_frame_id = encoded_image->frame_id;
- } else {
- encoded_image->dependency = EncodedFrame::DEPENDENT;
- encoded_image->referenced_frame_id = latest_frame_id_to_reference;
- }
+ DCHECK(!encoded_frame->data.empty())
+ << "BUG: Encoder must provide data since lagged encoding is disabled.";
- DVLOG(1) << "VP8 encoded frame_id " << encoded_image->frame_id
- << ", sized:" << encoded_image->data.size();
+ DVLOG(2) << "VP8 encoded frame_id " << encoded_frame->frame_id
+ << ", sized:" << encoded_frame->data.size();
- if (is_key_frame) {
+ if (encoded_frame->dependency == EncodedFrame::KEY) {
key_frame_requested_ = false;
for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
buffer_state_[i].state = kBufferSent;
- buffer_state_[i].frame_id = encoded_image->frame_id;
+ buffer_state_[i].frame_id = encoded_frame->frame_id;
}
} else {
if (buffer_to_update != kNoBuffer) {
buffer_state_[buffer_to_update].state = kBufferSent;
- buffer_state_[buffer_to_update].frame_id = encoded_image->frame_id;
+ buffer_state_[buffer_to_update].frame_id = encoded_frame->frame_id;
}
}
- return true;
}
uint32 Vp8Encoder::GetCodecReferenceFlags(vpx_codec_flags_t* flags) {
void Vp8Encoder::UpdateRates(uint32 new_bitrate) {
DCHECK(thread_checker_.CalledOnValidThread());
+
+ if (!is_initialized())
+ return;
+
uint32 new_bitrate_kbit = new_bitrate / 1000;
- if (config_->rc_target_bitrate == new_bitrate_kbit)
+ if (config_.rc_target_bitrate == new_bitrate_kbit)
return;
- config_->rc_target_bitrate = new_bitrate_kbit;
+ config_.rc_target_bitrate = new_bitrate_kbit;
// Update encoder context.
- if (vpx_codec_enc_config_set(encoder_.get(), config_.get())) {
- DCHECK(false) << "Invalid return value";
+ if (vpx_codec_enc_config_set(&encoder_, &config_)) {
+ NOTREACHED() << "Invalid return value";
}
+
+ VLOG(1) << "VP8 new rc_target_bitrate: " << new_bitrate_kbit << " kbps";
}
void Vp8Encoder::LatestFrameIdToReference(uint32 frame_id) {
if (!use_multiple_video_buffers_)
return;
- VLOG(1) << "VP8 ok to reference frame:" << static_cast<int>(frame_id);
+ VLOG(2) << "VP8 ok to reference frame:" << static_cast<int>(frame_id);
for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
if (frame_id == buffer_state_[i].frame_id) {
buffer_state_[i].state = kBufferAcked;
key_frame_requested_ = true;
}
-// Calculate the max size of the key frame relative to a normal delta frame.
-uint32 Vp8Encoder::MaxIntraTarget(uint32 optimal_buffer_size_ms) const {
- // Set max to the optimal buffer level (normalized by target BR),
- // and scaled by a scale_parameter.
- // Max target size = scalePar * optimalBufferSize * targetBR[Kbps].
- // This values is presented in percentage of perFrameBw:
- // perFrameBw = targetBR[Kbps] * 1000 / frameRate.
- // The target in % is as follows:
-
- float scale_parameter = 0.5;
- uint32 target_pct = optimal_buffer_size_ms * scale_parameter *
- cast_config_.max_frame_rate / 10;
-
- // Don't go below 3 times the per frame bandwidth.
- return std::max(target_pct, kMinIntra);
-}
-
} // namespace cast
} // namespace media
projects / platform / framework / web / crosswalk.git / blobdiff