src/media/cast/video_sender/external_video_encoder.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/video_sender/external_video_encoder.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/memory/scoped_vector.h"
  10 #include "base/memory/shared_memory.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "media/base/video_frame.h"
  13 #include "media/base/video_util.h"
  14 #include "media/cast/cast_defines.h"
  15 #include "media/cast/logging/logging_defines.h"
  16 #include "media/cast/transport/cast_transport_config.h"
  17 #include "media/video/video_encode_accelerator.h"
  18
  19 namespace media {
  20 namespace cast {
  21 class LocalVideoEncodeAcceleratorClient;
  22 }  // namespace cast
  23 }  // namespace media
  24
  25 namespace {
  26 static const size_t kOutputBufferCount = 3;
  27
  28 void LogFrameEncodedEvent(
  29     const scoped_refptr<media::cast::CastEnvironment>& cast_environment,
  30     base::TimeTicks event_time,
  31     media::cast::RtpTimestamp rtp_timestamp,
  32     uint32 frame_id) {
  33   cast_environment->Logging()->InsertFrameEvent(
  34       event_time, media::cast::kVideoFrameEncoded, rtp_timestamp, frame_id);
  35 }
  36
  37 // Proxy this call to ExternalVideoEncoder on the cast main thread.
  38 void ProxyCreateVideoEncodeAccelerator(
  39     const scoped_refptr<media::cast::CastEnvironment>& cast_environment,
  40     const base::WeakPtr<media::cast::ExternalVideoEncoder>& weak_ptr,
  41     const media::cast::CreateVideoEncodeMemoryCallback&
  42         create_video_encode_mem_cb,
  43     scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,
  44     scoped_ptr<media::VideoEncodeAccelerator> vea) {
  45   cast_environment->PostTask(
  46       media::cast::CastEnvironment::MAIN,
  47       FROM_HERE,
  48       base::Bind(
  49           &media::cast::ExternalVideoEncoder::OnCreateVideoEncodeAccelerator,
  50           weak_ptr,
  51           create_video_encode_mem_cb,
  52           encoder_task_runner,
  53           base::Passed(&vea)));
  54 }
  55 }  // namespace
  56
  57 namespace media {
  58 namespace cast {
  59
  60 // Container for the associated data of a video frame being processed.
  61 struct EncodedFrameReturnData {
  62   EncodedFrameReturnData(base::TimeTicks c_time,
  63                          VideoEncoder::FrameEncodedCallback callback) {
  64     capture_time = c_time;
  65     frame_encoded_callback = callback;
  66   }
  67   base::TimeTicks capture_time;
  68   VideoEncoder::FrameEncodedCallback frame_encoded_callback;
  69 };
  70
  71 // The ExternalVideoEncoder class can be deleted directly by cast, while
  72 // LocalVideoEncodeAcceleratorClient stays around long enough to properly shut
  73 // down the VideoEncodeAccelerator.
  74 class LocalVideoEncodeAcceleratorClient
  75     : public VideoEncodeAccelerator::Client,
  76       public base::RefCountedThreadSafe<LocalVideoEncodeAcceleratorClient> {
  77  public:
  78   LocalVideoEncodeAcceleratorClient(
  79       scoped_refptr<CastEnvironment> cast_environment,
  80       scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,
  81       scoped_ptr<media::VideoEncodeAccelerator> vea,
  82       const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
  83       const base::WeakPtr<ExternalVideoEncoder>& weak_owner)
  84       : cast_environment_(cast_environment),
  85         encoder_task_runner_(encoder_task_runner),
  86         video_encode_accelerator_(vea.Pass()),
  87         create_video_encode_memory_cb_(create_video_encode_mem_cb),
  88         weak_owner_(weak_owner),
  89         last_encoded_frame_id_(kStartFrameId),
  90         key_frame_encountered_(false) {
  91     DCHECK(encoder_task_runner_);
  92   }
  93
  94   // Initialize the real HW encoder.
  95   void Initialize(const VideoSenderConfig& video_config) {
  96     DCHECK(encoder_task_runner_);
  97     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
  98
  99     VideoCodecProfile output_profile = media::VIDEO_CODEC_PROFILE_UNKNOWN;
 100     switch (video_config.codec) {
 101       case transport::kVp8:
 102         output_profile = media::VP8PROFILE_MAIN;
 103         break;
 104       case transport::kH264:
 105         output_profile = media::H264PROFILE_MAIN;
 106         break;
 107       case transport::kFakeSoftwareVideo:
 108         NOTREACHED() << "Fake software video encoder cannot be external";
 109         break;
 110     }
 111     codec_ = video_config.codec;
 112     max_frame_rate_ = video_config.max_frame_rate;
 113
 114     if (!video_encode_accelerator_->Initialize(
 115             media::VideoFrame::I420,
 116             gfx::Size(video_config.width, video_config.height),
 117             output_profile,
 118             video_config.start_bitrate,
 119             this)) {
 120       NotifyError(VideoEncodeAccelerator::kInvalidArgumentError);
 121       return;
 122     }
 123
 124     // Wait until shared memory is allocated to indicate that encoder is
 125     // initialized.
 126   }
 127
 128   // Free the HW.
 129   void Destroy() {
 130     DCHECK(encoder_task_runner_);
 131     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 132
 133     if (video_encode_accelerator_) {
 134       video_encode_accelerator_.release()->Destroy();
 135     }
 136   }
 137
 138   void SetBitRate(uint32 bit_rate) {
 139     DCHECK(encoder_task_runner_);
 140     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 141
 142     video_encode_accelerator_->RequestEncodingParametersChange(bit_rate,
 143                                                                max_frame_rate_);
 144   }
 145
 146   void EncodeVideoFrame(
 147       const scoped_refptr<media::VideoFrame>& video_frame,
 148       const base::TimeTicks& capture_time,
 149       bool key_frame_requested,
 150       const VideoEncoder::FrameEncodedCallback& frame_encoded_callback) {
 151     DCHECK(encoder_task_runner_);
 152     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 153
 154     encoded_frame_data_storage_.push_back(
 155         EncodedFrameReturnData(capture_time, frame_encoded_callback));
 156
 157     // BitstreamBufferReady will be called once the encoder is done.
 158     video_encode_accelerator_->Encode(video_frame, key_frame_requested);
 159   }
 160
 161  protected:
 162   virtual void NotifyError(VideoEncodeAccelerator::Error error) OVERRIDE {
 163     DCHECK(encoder_task_runner_);
 164     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 165     VLOG(1) << "ExternalVideoEncoder NotifyError: " << error;
 166
 167     if (video_encode_accelerator_) {
 168       video_encode_accelerator_.release()->Destroy();
 169     }
 170     cast_environment_->PostTask(
 171         CastEnvironment::MAIN,
 172         FROM_HERE,
 173         base::Bind(&ExternalVideoEncoder::EncoderError, weak_owner_));
 174   }
 175
 176   // Called to allocate the input and output buffers.
 177   virtual void RequireBitstreamBuffers(unsigned int input_count,
 178                                        const gfx::Size& input_coded_size,
 179                                        size_t output_buffer_size) OVERRIDE {
 180     DCHECK(encoder_task_runner_);
 181     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 182     DCHECK(video_encode_accelerator_);
 183
 184     for (size_t j = 0; j < kOutputBufferCount; ++j) {
 185       create_video_encode_memory_cb_.Run(
 186           output_buffer_size,
 187           base::Bind(&LocalVideoEncodeAcceleratorClient::OnCreateSharedMemory,
 188                      this));
 189     }
 190   }
 191
 192   // Encoder has encoded a frame and it's available in one of out output
 193   // buffers.
 194   virtual void BitstreamBufferReady(int32 bitstream_buffer_id,
 195                                     size_t payload_size,
 196                                     bool key_frame) OVERRIDE {
 197     DCHECK(encoder_task_runner_);
 198     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 199     if (bitstream_buffer_id < 0 ||
 200         bitstream_buffer_id >= static_cast<int32>(output_buffers_.size())) {
 201       NOTREACHED();
 202       VLOG(1) << "BitstreamBufferReady(): invalid bitstream_buffer_id="
 203               << bitstream_buffer_id;
 204       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 205       return;
 206     }
 207     base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
 208     if (payload_size > output_buffer->mapped_size()) {
 209       NOTREACHED();
 210       VLOG(1) << "BitstreamBufferReady(): invalid payload_size = "
 211               << payload_size;
 212       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 213       return;
 214     }
 215     if (key_frame)
 216       key_frame_encountered_ = true;
 217     if (!key_frame_encountered_) {
 218       // Do not send video until we have encountered the first key frame.
 219       // Save the bitstream buffer in |stream_header_| to be sent later along
 220       // with the first key frame.
 221       stream_header_.append(static_cast<const char*>(output_buffer->memory()),
 222                             payload_size);
 223     } else if (!encoded_frame_data_storage_.empty()) {
 224       scoped_ptr<transport::EncodedVideoFrame> encoded_frame(
 225           new transport::EncodedVideoFrame());
 226
 227       encoded_frame->codec = codec_;
 228       encoded_frame->key_frame = key_frame;
 229       encoded_frame->last_referenced_frame_id = last_encoded_frame_id_;
 230       last_encoded_frame_id_++;
 231       encoded_frame->frame_id = last_encoded_frame_id_;
 232       encoded_frame->rtp_timestamp = GetVideoRtpTimestamp(
 233           encoded_frame_data_storage_.front().capture_time);
 234       if (key_frame) {
 235         // Self referenced.
 236         encoded_frame->last_referenced_frame_id = encoded_frame->frame_id;
 237       }
 238
 239       if (!stream_header_.empty()) {
 240         encoded_frame->data = stream_header_;
 241         stream_header_.clear();
 242       }
 243       encoded_frame->data.append(
 244           static_cast<const char*>(output_buffer->memory()), payload_size);
 245
 246       cast_environment_->PostTask(
 247           CastEnvironment::MAIN,
 248           FROM_HERE,
 249           base::Bind(&LogFrameEncodedEvent,
 250                      cast_environment_,
 251                      cast_environment_->Clock()->NowTicks(),
 252                      encoded_frame->rtp_timestamp,
 253                      encoded_frame->frame_id));
 254
 255       cast_environment_->PostTask(
 256           CastEnvironment::MAIN,
 257           FROM_HERE,
 258           base::Bind(encoded_frame_data_storage_.front().frame_encoded_callback,
 259                      base::Passed(&encoded_frame),
 260                      encoded_frame_data_storage_.front().capture_time));
 261
 262       encoded_frame_data_storage_.pop_front();
 263     } else {
 264       VLOG(1) << "BitstreamBufferReady(): no encoded frame data available";
 265     }
 266
 267     // We need to re-add the output buffer to the encoder after we are done
 268     // with it.
 269     video_encode_accelerator_->UseOutputBitstreamBuffer(media::BitstreamBuffer(
 270         bitstream_buffer_id,
 271         output_buffers_[bitstream_buffer_id]->handle(),
 272         output_buffers_[bitstream_buffer_id]->mapped_size()));
 273   }
 274
 275  private:
 276   // Note: This method can be called on any thread.
 277   void OnCreateSharedMemory(scoped_ptr<base::SharedMemory> memory) {
 278     encoder_task_runner_->PostTask(
 279         FROM_HERE,
 280         base::Bind(&LocalVideoEncodeAcceleratorClient::ReceivedSharedMemory,
 281                    this,
 282                    base::Passed(&memory)));
 283   }
 284
 285   void ReceivedSharedMemory(scoped_ptr<base::SharedMemory> memory) {
 286     DCHECK(encoder_task_runner_);
 287     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 288
 289     output_buffers_.push_back(memory.release());
 290
 291     // Wait until all requested buffers are received.
 292     if (output_buffers_.size() < kOutputBufferCount)
 293       return;
 294
 295     // Immediately provide all output buffers to the VEA.
 296     for (size_t i = 0; i < output_buffers_.size(); ++i) {
 297       video_encode_accelerator_->UseOutputBitstreamBuffer(
 298           media::BitstreamBuffer(static_cast<int32>(i),
 299                                  output_buffers_[i]->handle(),
 300                                  output_buffers_[i]->mapped_size()));
 301     }
 302
 303     cast_environment_->PostTask(
 304         CastEnvironment::MAIN,
 305         FROM_HERE,
 306         base::Bind(&ExternalVideoEncoder::EncoderInitialized, weak_owner_));
 307   }
 308
 309   friend class base::RefCountedThreadSafe<LocalVideoEncodeAcceleratorClient>;
 310
 311   virtual ~LocalVideoEncodeAcceleratorClient() {}
 312
 313   const scoped_refptr<CastEnvironment> cast_environment_;
 314   scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner_;
 315   scoped_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
 316   const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_;
 317   const base::WeakPtr<ExternalVideoEncoder> weak_owner_;
 318   int max_frame_rate_;
 319   transport::VideoCodec codec_;
 320   uint32 last_encoded_frame_id_;
 321   bool key_frame_encountered_;
 322   std::string stream_header_;
 323
 324   // Shared memory buffers for output with the VideoAccelerator.
 325   ScopedVector<base::SharedMemory> output_buffers_;
 326
 327   // FIFO list.
 328   std::list<EncodedFrameReturnData> encoded_frame_data_storage_;
 329
 330   DISALLOW_COPY_AND_ASSIGN(LocalVideoEncodeAcceleratorClient);
 331 };
 332
 333 ExternalVideoEncoder::ExternalVideoEncoder(
 334     scoped_refptr<CastEnvironment> cast_environment,
 335     const VideoSenderConfig& video_config,
 336     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
 337     const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb)
 338     : video_config_(video_config),
 339       cast_environment_(cast_environment),
 340       encoder_active_(false),
 341       key_frame_requested_(false),
 342       skip_next_frame_(false),
 343       skip_count_(0),
 344       weak_factory_(this) {
 345   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 346
 347   create_vea_cb.Run(base::Bind(&ProxyCreateVideoEncodeAccelerator,
 348                                cast_environment,
 349                                weak_factory_.GetWeakPtr(),
 350                                create_video_encode_mem_cb));
 351 }
 352
 353 ExternalVideoEncoder::~ExternalVideoEncoder() {
 354   encoder_task_runner_->PostTask(
 355       FROM_HERE,
 356       base::Bind(&LocalVideoEncodeAcceleratorClient::Destroy,
 357                  video_accelerator_client_));
 358 }
 359
 360 void ExternalVideoEncoder::EncoderInitialized() {
 361   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 362   encoder_active_ = true;
 363 }
 364
 365 void ExternalVideoEncoder::EncoderError() {
 366   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 367   encoder_active_ = false;
 368 }
 369
 370 void ExternalVideoEncoder::OnCreateVideoEncodeAccelerator(
 371     const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
 372     scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,
 373     scoped_ptr<media::VideoEncodeAccelerator> vea) {
 374   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 375   encoder_task_runner_ = encoder_task_runner;
 376
 377   video_accelerator_client_ =
 378       new LocalVideoEncodeAcceleratorClient(cast_environment_,
 379                                             encoder_task_runner,
 380                                             vea.Pass(),
 381                                             create_video_encode_mem_cb,
 382                                             weak_factory_.GetWeakPtr());
 383   encoder_task_runner_->PostTask(
 384       FROM_HERE,
 385       base::Bind(&LocalVideoEncodeAcceleratorClient::Initialize,
 386                  video_accelerator_client_,
 387                  video_config_));
 388 }
 389
 390 bool ExternalVideoEncoder::EncodeVideoFrame(
 391     const scoped_refptr<media::VideoFrame>& video_frame,
 392     const base::TimeTicks& capture_time,
 393     const FrameEncodedCallback& frame_encoded_callback) {
 394   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 395
 396   if (!encoder_active_)
 397     return false;
 398
 399   if (skip_next_frame_) {
 400     VLOG(1) << "Skip encoding frame";
 401     ++skip_count_;
 402     return false;
 403   }
 404
 405   encoder_task_runner_->PostTask(
 406       FROM_HERE,
 407       base::Bind(&LocalVideoEncodeAcceleratorClient::EncodeVideoFrame,
 408                  video_accelerator_client_,
 409                  video_frame,
 410                  capture_time,
 411                  key_frame_requested_,
 412                  frame_encoded_callback));
 413
 414   key_frame_requested_ = false;
 415   return true;
 416 }
 417
 418 // Inform the encoder about the new target bit rate.
 419 void ExternalVideoEncoder::SetBitRate(int new_bit_rate) {
 420   encoder_task_runner_->PostTask(
 421       FROM_HERE,
 422       base::Bind(&LocalVideoEncodeAcceleratorClient::SetBitRate,
 423                  video_accelerator_client_,
 424                  new_bit_rate));
 425 }
 426
 427 // Inform the encoder to not encode the next frame.
 428 void ExternalVideoEncoder::SkipNextFrame(bool skip_next_frame) {
 429   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 430   skip_next_frame_ = skip_next_frame;
 431 }
 432
 433 // Inform the encoder to encode the next frame as a key frame.
 434 void ExternalVideoEncoder::GenerateKeyFrame() {
 435   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 436   key_frame_requested_ = true;
 437 }
 438
 439 // Inform the encoder to only reference frames older or equal to frame_id;
 440 void ExternalVideoEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) {
 441   // Do nothing not supported.
 442 }
 443
 444 int ExternalVideoEncoder::NumberOfSkippedFrames() const {
 445   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 446   return skip_count_;
 447 }
 448
 449 }  //  namespace cast
 450 }  //  namespace media