1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/cast/video_sender/external_video_encoder.h"
8 #include "base/logging.h"
9 #include "base/memory/scoped_vector.h"
10 #include "base/memory/shared_memory.h"
11 #include "base/message_loop/message_loop.h"
12 #include "media/base/video_frame.h"
13 #include "media/base/video_util.h"
14 #include "media/cast/cast_defines.h"
15 #include "media/cast/logging/logging_defines.h"
16 #include "media/cast/transport/cast_transport_config.h"
17 #include "media/video/video_encode_accelerator.h"
21 class LocalVideoEncodeAcceleratorClient;
26 static const int kOutputBufferCount = 3;
28 void LogFrameEncodedEvent(
29 const scoped_refptr<media::cast::CastEnvironment>& cast_environment,
30 base::TimeTicks event_time,
31 media::cast::RtpTimestamp rtp_timestamp,
33 cast_environment->Logging()->InsertFrameEvent(
34 event_time, media::cast::kVideoFrameEncoded, rtp_timestamp, frame_id);
37 // Proxy this call to ExternalVideoEncoder on the cast main thread.
38 void ProxyCreateVideoEncodeAccelerator(
39 const scoped_refptr<media::cast::CastEnvironment>& cast_environment,
40 const base::WeakPtr<media::cast::ExternalVideoEncoder>& weak_ptr,
41 const media::cast::CreateVideoEncodeMemoryCallback&
42 create_video_encode_mem_cb,
43 scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,
44 scoped_ptr<media::VideoEncodeAccelerator> vea) {
45 cast_environment->PostTask(
46 media::cast::CastEnvironment::MAIN,
49 &media::cast::ExternalVideoEncoder::OnCreateVideoEncodeAccelerator,
51 create_video_encode_mem_cb,
60 // Container for the associated data of a video frame being processed.
61 struct EncodedFrameReturnData {
62 EncodedFrameReturnData(base::TimeTicks c_time,
63 VideoEncoder::FrameEncodedCallback callback) {
64 capture_time = c_time;
65 frame_encoded_callback = callback;
67 base::TimeTicks capture_time;
68 VideoEncoder::FrameEncodedCallback frame_encoded_callback;
71 // The ExternalVideoEncoder class can be deleted directly by cast, while
72 // LocalVideoEncodeAcceleratorClient stays around long enough to properly shut
73 // down the VideoEncodeAccelerator.
74 class LocalVideoEncodeAcceleratorClient
75 : public VideoEncodeAccelerator::Client,
76 public base::RefCountedThreadSafe<LocalVideoEncodeAcceleratorClient> {
78 LocalVideoEncodeAcceleratorClient(
79 scoped_refptr<CastEnvironment> cast_environment,
80 scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,
81 scoped_ptr<media::VideoEncodeAccelerator> vea,
82 const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
83 const base::WeakPtr<ExternalVideoEncoder>& weak_owner)
84 : cast_environment_(cast_environment),
85 encoder_task_runner_(encoder_task_runner),
86 video_encode_accelerator_(vea.Pass()),
87 create_video_encode_memory_cb_(create_video_encode_mem_cb),
88 weak_owner_(weak_owner),
89 last_encoded_frame_id_(kStartFrameId) {
90 DCHECK(encoder_task_runner_);
93 // Initialize the real HW encoder.
94 void Initialize(const VideoSenderConfig& video_config) {
95 DCHECK(encoder_task_runner_);
96 DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
98 VideoCodecProfile output_profile = media::VIDEO_CODEC_PROFILE_UNKNOWN;
99 switch (video_config.codec) {
100 case transport::kVp8:
101 output_profile = media::VP8PROFILE_MAIN;
103 case transport::kH264:
104 output_profile = media::H264PROFILE_MAIN;
107 codec_ = video_config.codec;
108 max_frame_rate_ = video_config.max_frame_rate;
110 // Asynchronous initialization call; NotifyInitializeDone or NotifyError
111 // will be called once the HW is initialized.
112 video_encode_accelerator_->Initialize(
113 media::VideoFrame::I420,
114 gfx::Size(video_config.width, video_config.height),
116 video_config.start_bitrate,
122 DCHECK(encoder_task_runner_);
123 DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
125 if (video_encode_accelerator_) {
126 video_encode_accelerator_.release()->Destroy();
130 void SetBitRate(uint32 bit_rate) {
131 DCHECK(encoder_task_runner_);
132 DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
134 video_encode_accelerator_->RequestEncodingParametersChange(bit_rate,
138 void EncodeVideoFrame(
139 const scoped_refptr<media::VideoFrame>& video_frame,
140 const base::TimeTicks& capture_time,
141 bool key_frame_requested,
142 const VideoEncoder::FrameEncodedCallback& frame_encoded_callback) {
143 DCHECK(encoder_task_runner_);
144 DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
146 encoded_frame_data_storage_.push_back(
147 EncodedFrameReturnData(capture_time, frame_encoded_callback));
149 // BitstreamBufferReady will be called once the encoder is done.
150 video_encode_accelerator_->Encode(video_frame, key_frame_requested);
154 virtual void NotifyInitializeDone() OVERRIDE {
155 DCHECK(encoder_task_runner_);
156 DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
158 // Wait until shared memory is allocated to indicate that encoder is
162 virtual void NotifyError(VideoEncodeAccelerator::Error error) OVERRIDE {
163 DCHECK(encoder_task_runner_);
164 DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
165 VLOG(1) << "ExternalVideoEncoder NotifyError: " << error;
167 if (video_encode_accelerator_) {
168 video_encode_accelerator_.release()->Destroy();
170 cast_environment_->PostTask(
171 CastEnvironment::MAIN,
173 base::Bind(&ExternalVideoEncoder::EncoderError, weak_owner_));
176 // Called to allocate the input and output buffers.
177 virtual void RequireBitstreamBuffers(unsigned int input_count,
178 const gfx::Size& input_coded_size,
179 size_t output_buffer_size) OVERRIDE {
180 DCHECK(encoder_task_runner_);
181 DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
182 DCHECK(video_encode_accelerator_);
184 for (int j = 0; j < kOutputBufferCount; ++j) {
185 create_video_encode_memory_cb_.Run(
187 base::Bind(&LocalVideoEncodeAcceleratorClient::OnCreateSharedMemory,
192 // Encoder has encoded a frame and it's available in one of out output
194 virtual void BitstreamBufferReady(int32 bitstream_buffer_id,
196 bool key_frame) OVERRIDE {
197 DCHECK(encoder_task_runner_);
198 DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
199 if (bitstream_buffer_id < 0 ||
200 bitstream_buffer_id >= static_cast<int32>(output_buffers_.size())) {
202 VLOG(1) << "BitstreamBufferReady(): invalid bitstream_buffer_id="
203 << bitstream_buffer_id;
204 NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
207 base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
208 if (payload_size > output_buffer->mapped_size()) {
210 VLOG(1) << "BitstreamBufferReady(): invalid payload_size = "
212 NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
215 if (encoded_frame_data_storage_.empty()) {
217 NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
220 scoped_ptr<transport::EncodedVideoFrame> encoded_frame(
221 new transport::EncodedVideoFrame());
223 encoded_frame->codec = codec_;
224 encoded_frame->key_frame = key_frame;
225 encoded_frame->last_referenced_frame_id = last_encoded_frame_id_;
226 last_encoded_frame_id_++;
227 encoded_frame->frame_id = last_encoded_frame_id_;
228 encoded_frame->rtp_timestamp =
229 GetVideoRtpTimestamp(encoded_frame_data_storage_.front().capture_time);
232 encoded_frame->last_referenced_frame_id = encoded_frame->frame_id;
235 encoded_frame->data.insert(
236 0, static_cast<const char*>(output_buffer->memory()), payload_size);
238 cast_environment_->PostTask(
239 CastEnvironment::MAIN,
241 base::Bind(&LogFrameEncodedEvent,
243 cast_environment_->Clock()->NowTicks(),
244 encoded_frame->rtp_timestamp,
245 encoded_frame->frame_id));
247 cast_environment_->PostTask(
248 CastEnvironment::MAIN,
250 base::Bind(encoded_frame_data_storage_.front().frame_encoded_callback,
251 base::Passed(&encoded_frame),
252 encoded_frame_data_storage_.front().capture_time));
254 encoded_frame_data_storage_.pop_front();
256 // We need to re-add the output buffer to the encoder after we are done
258 video_encode_accelerator_->UseOutputBitstreamBuffer(media::BitstreamBuffer(
260 output_buffers_[bitstream_buffer_id]->handle(),
261 output_buffers_[bitstream_buffer_id]->mapped_size()));
265 // Note: This method can be called on any thread.
266 void OnCreateSharedMemory(scoped_ptr<base::SharedMemory> memory) {
267 encoder_task_runner_->PostTask(
269 base::Bind(&LocalVideoEncodeAcceleratorClient::ReceivedSharedMemory,
271 base::Passed(&memory)));
274 void ReceivedSharedMemory(scoped_ptr<base::SharedMemory> memory) {
275 DCHECK(encoder_task_runner_);
276 DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
278 output_buffers_.push_back(memory.release());
280 // Wait until all requested buffers are received.
281 if (output_buffers_.size() < kOutputBufferCount)
284 // Immediately provide all output buffers to the VEA.
285 for (size_t i = 0; i < output_buffers_.size(); ++i) {
286 video_encode_accelerator_->UseOutputBitstreamBuffer(
287 media::BitstreamBuffer(static_cast<int32>(i),
288 output_buffers_[i]->handle(),
289 output_buffers_[i]->mapped_size()));
292 cast_environment_->PostTask(
293 CastEnvironment::MAIN,
295 base::Bind(&ExternalVideoEncoder::EncoderInitialized, weak_owner_));
298 friend class base::RefCountedThreadSafe<LocalVideoEncodeAcceleratorClient>;
300 virtual ~LocalVideoEncodeAcceleratorClient() {}
302 const scoped_refptr<CastEnvironment> cast_environment_;
303 scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner_;
304 scoped_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
305 const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_;
306 const base::WeakPtr<ExternalVideoEncoder> weak_owner_;
308 transport::VideoCodec codec_;
309 uint32 last_encoded_frame_id_;
311 // Shared memory buffers for output with the VideoAccelerator.
312 ScopedVector<base::SharedMemory> output_buffers_;
315 std::list<EncodedFrameReturnData> encoded_frame_data_storage_;
317 DISALLOW_COPY_AND_ASSIGN(LocalVideoEncodeAcceleratorClient);
320 ExternalVideoEncoder::ExternalVideoEncoder(
321 scoped_refptr<CastEnvironment> cast_environment,
322 const VideoSenderConfig& video_config,
323 const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
324 const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb)
325 : video_config_(video_config),
326 cast_environment_(cast_environment),
327 encoder_active_(false),
328 key_frame_requested_(false),
329 skip_next_frame_(false),
331 weak_factory_(this) {
332 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
334 create_vea_cb.Run(base::Bind(&ProxyCreateVideoEncodeAccelerator,
336 weak_factory_.GetWeakPtr(),
337 create_video_encode_mem_cb));
340 ExternalVideoEncoder::~ExternalVideoEncoder() {
341 encoder_task_runner_->PostTask(
343 base::Bind(&LocalVideoEncodeAcceleratorClient::Destroy,
344 video_accelerator_client_));
347 void ExternalVideoEncoder::EncoderInitialized() {
348 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
349 encoder_active_ = true;
352 void ExternalVideoEncoder::EncoderError() {
353 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
354 encoder_active_ = false;
357 void ExternalVideoEncoder::OnCreateVideoEncodeAccelerator(
358 const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
359 scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,
360 scoped_ptr<media::VideoEncodeAccelerator> vea) {
361 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
362 encoder_task_runner_ = encoder_task_runner;
364 video_accelerator_client_ =
365 new LocalVideoEncodeAcceleratorClient(cast_environment_,
368 create_video_encode_mem_cb,
369 weak_factory_.GetWeakPtr());
370 encoder_task_runner_->PostTask(
372 base::Bind(&LocalVideoEncodeAcceleratorClient::Initialize,
373 video_accelerator_client_,
377 bool ExternalVideoEncoder::EncodeVideoFrame(
378 const scoped_refptr<media::VideoFrame>& video_frame,
379 const base::TimeTicks& capture_time,
380 const FrameEncodedCallback& frame_encoded_callback) {
381 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
383 if (!encoder_active_)
386 if (skip_next_frame_) {
387 VLOG(1) << "Skip encoding frame";
389 skip_next_frame_ = false;
392 base::TimeTicks now = cast_environment_->Clock()->NowTicks();
393 cast_environment_->Logging()->InsertFrameEvent(
395 kVideoFrameSentToEncoder,
396 GetVideoRtpTimestamp(capture_time),
399 encoder_task_runner_->PostTask(
401 base::Bind(&LocalVideoEncodeAcceleratorClient::EncodeVideoFrame,
402 video_accelerator_client_,
405 key_frame_requested_,
406 frame_encoded_callback));
408 key_frame_requested_ = false;
412 // Inform the encoder about the new target bit rate.
413 void ExternalVideoEncoder::SetBitRate(int new_bit_rate) {
414 encoder_task_runner_->PostTask(
416 base::Bind(&LocalVideoEncodeAcceleratorClient::SetBitRate,
417 video_accelerator_client_,
421 // Inform the encoder to not encode the next frame.
422 void ExternalVideoEncoder::SkipNextFrame(bool skip_next_frame) {
423 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
424 skip_next_frame_ = skip_next_frame;
427 // Inform the encoder to encode the next frame as a key frame.
428 void ExternalVideoEncoder::GenerateKeyFrame() {
429 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
430 key_frame_requested_ = true;
433 // Inform the encoder to only reference frames older or equal to frame_id;
434 void ExternalVideoEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) {
435 // Do nothing not supported.
438 int ExternalVideoEncoder::NumberOfSkippedFrames() const {
439 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));