src/media/cast/sender/external_video_encoder.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/sender/external_video_encoder.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/memory/scoped_vector.h"
  10 #include "base/memory/shared_memory.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "base/metrics/histogram.h"
  13 #include "media/base/video_frame.h"
  14 #include "media/base/video_util.h"
  15 #include "media/cast/cast_defines.h"
  16 #include "media/cast/logging/logging_defines.h"
  17 #include "media/cast/net/cast_transport_config.h"
  18 #include "media/video/video_encode_accelerator.h"
  19
  20 namespace media {
  21 namespace cast {
  22 class LocalVideoEncodeAcceleratorClient;
  23 }  // namespace cast
  24 }  // namespace media
  25
  26 namespace {
  27 static const size_t kOutputBufferCount = 3;
  28
  29 void LogFrameEncodedEvent(
  30     const scoped_refptr<media::cast::CastEnvironment>& cast_environment,
  31     base::TimeTicks event_time,
  32     media::cast::RtpTimestamp rtp_timestamp,
  33     uint32 frame_id) {
  34   cast_environment->Logging()->InsertFrameEvent(
  35       event_time, media::cast::FRAME_ENCODED, media::cast::VIDEO_EVENT,
  36       rtp_timestamp, frame_id);
  37 }
  38 }  // namespace
  39
  40 namespace media {
  41 namespace cast {
  42
  43 // Container for the associated data of a video frame being processed.
  44 struct EncodedFrameReturnData {
  45   EncodedFrameReturnData(base::TimeTicks c_time,
  46                          VideoEncoder::FrameEncodedCallback callback) {
  47     capture_time = c_time;
  48     frame_encoded_callback = callback;
  49   }
  50   base::TimeTicks capture_time;
  51   VideoEncoder::FrameEncodedCallback frame_encoded_callback;
  52 };
  53
  54 // The ExternalVideoEncoder class can be deleted directly by cast, while
  55 // LocalVideoEncodeAcceleratorClient stays around long enough to properly shut
  56 // down the VideoEncodeAccelerator.
  57 class LocalVideoEncodeAcceleratorClient
  58     : public VideoEncodeAccelerator::Client,
  59       public base::RefCountedThreadSafe<LocalVideoEncodeAcceleratorClient> {
  60  public:
  61   // Create an instance of this class and post a task to create
  62   // video_encode_accelerator_. A ref to |this| will be kept, awaiting reply
  63   // via ProxyCreateVideoEncodeAccelerator, which will provide us with the
  64   // encoder task runner and vea instance. We cannot be destroyed until we
  65   // receive the reply, otherwise the VEA object created may leak.
  66   static scoped_refptr<LocalVideoEncodeAcceleratorClient> Create(
  67       scoped_refptr<CastEnvironment> cast_environment,
  68       const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
  69       const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
  70       const base::WeakPtr<ExternalVideoEncoder>& weak_owner) {
  71     scoped_refptr<LocalVideoEncodeAcceleratorClient> client(
  72         new LocalVideoEncodeAcceleratorClient(
  73             cast_environment, create_video_encode_mem_cb, weak_owner));
  74
  75     // This will keep a ref to |client|, if weak_owner is destroyed before
  76     // ProxyCreateVideoEncodeAccelerator is called, we will stay alive until
  77     // we can properly destroy the VEA.
  78     create_vea_cb.Run(base::Bind(
  79         &LocalVideoEncodeAcceleratorClient::OnCreateVideoEncodeAcceleratorProxy,
  80         client));
  81
  82     return client;
  83   }
  84
  85   // Initialize the real HW encoder.
  86   void Initialize(const VideoSenderConfig& video_config) {
  87     DCHECK(encoder_task_runner_.get());
  88     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
  89
  90     VideoCodecProfile output_profile = media::VIDEO_CODEC_PROFILE_UNKNOWN;
  91     switch (video_config.codec) {
  92       case CODEC_VIDEO_VP8:
  93         output_profile = media::VP8PROFILE_ANY;
  94         break;
  95       case CODEC_VIDEO_H264:
  96         output_profile = media::H264PROFILE_MAIN;
  97         break;
  98       case CODEC_VIDEO_FAKE:
  99         NOTREACHED() << "Fake software video encoder cannot be external";
 100         break;
 101       default:
 102         NOTREACHED() << "Video codec not specified or not supported";
 103         break;
 104     }
 105     max_frame_rate_ = video_config.max_frame_rate;
 106
 107     bool result = video_encode_accelerator_->Initialize(
 108         media::VideoFrame::I420,
 109         gfx::Size(video_config.width, video_config.height),
 110         output_profile,
 111         video_config.start_bitrate,
 112         this);
 113
 114     UMA_HISTOGRAM_BOOLEAN("Cast.Sender.VideoEncodeAcceleratorInitializeSuccess",
 115                           result);
 116     if (!result) {
 117       cast_environment_->PostTask(
 118           CastEnvironment::MAIN,
 119           FROM_HERE,
 120           base::Bind(&ExternalVideoEncoder::EncoderInitialized, weak_owner_,
 121                      false));
 122       return;
 123     }
 124
 125     // Wait until shared memory is allocated to indicate that encoder is
 126     // initialized.
 127   }
 128
 129   // Destroy the VEA on the correct thread.
 130   void Destroy() {
 131     DCHECK(encoder_task_runner_.get());
 132     if (!video_encode_accelerator_)
 133       return;
 134
 135     if (encoder_task_runner_->RunsTasksOnCurrentThread()) {
 136       video_encode_accelerator_.reset();
 137     } else {
 138       // We do this instead of just reposting to encoder_task_runner_, because
 139       // we are called from the destructor.
 140       encoder_task_runner_->PostTask(
 141           FROM_HERE,
 142           base::Bind(&DestroyVideoEncodeAcceleratorOnEncoderThread,
 143                      base::Passed(&video_encode_accelerator_)));
 144     }
 145   }
 146
 147   void SetBitRate(uint32 bit_rate) {
 148     DCHECK(encoder_task_runner_.get());
 149     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 150
 151     video_encode_accelerator_->RequestEncodingParametersChange(bit_rate,
 152                                                                max_frame_rate_);
 153   }
 154
 155   void EncodeVideoFrame(
 156       const scoped_refptr<media::VideoFrame>& video_frame,
 157       const base::TimeTicks& capture_time,
 158       bool key_frame_requested,
 159       const VideoEncoder::FrameEncodedCallback& frame_encoded_callback) {
 160     DCHECK(encoder_task_runner_.get());
 161     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 162
 163     encoded_frame_data_storage_.push_back(
 164         EncodedFrameReturnData(capture_time, frame_encoded_callback));
 165
 166     // BitstreamBufferReady will be called once the encoder is done.
 167     video_encode_accelerator_->Encode(video_frame, key_frame_requested);
 168   }
 169
 170  protected:
 171   virtual void NotifyError(VideoEncodeAccelerator::Error error) OVERRIDE {
 172     DCHECK(encoder_task_runner_.get());
 173     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 174     VLOG(1) << "ExternalVideoEncoder NotifyError: " << error;
 175
 176     cast_environment_->PostTask(
 177         CastEnvironment::MAIN,
 178         FROM_HERE,
 179         base::Bind(&ExternalVideoEncoder::EncoderError, weak_owner_));
 180   }
 181
 182   // Called to allocate the input and output buffers.
 183   virtual void RequireBitstreamBuffers(unsigned int input_count,
 184                                        const gfx::Size& input_coded_size,
 185                                        size_t output_buffer_size) OVERRIDE {
 186     DCHECK(encoder_task_runner_.get());
 187     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 188     DCHECK(video_encode_accelerator_);
 189
 190     for (size_t j = 0; j < kOutputBufferCount; ++j) {
 191       create_video_encode_memory_cb_.Run(
 192           output_buffer_size,
 193           base::Bind(&LocalVideoEncodeAcceleratorClient::OnCreateSharedMemory,
 194                      this));
 195     }
 196   }
 197
 198   // Encoder has encoded a frame and it's available in one of out output
 199   // buffers.
 200   virtual void BitstreamBufferReady(int32 bitstream_buffer_id,
 201                                     size_t payload_size,
 202                                     bool key_frame) OVERRIDE {
 203     DCHECK(encoder_task_runner_.get());
 204     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 205     if (bitstream_buffer_id < 0 ||
 206         bitstream_buffer_id >= static_cast<int32>(output_buffers_.size())) {
 207       NOTREACHED();
 208       VLOG(1) << "BitstreamBufferReady(): invalid bitstream_buffer_id="
 209               << bitstream_buffer_id;
 210       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 211       return;
 212     }
 213     base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
 214     if (payload_size > output_buffer->mapped_size()) {
 215       NOTREACHED();
 216       VLOG(1) << "BitstreamBufferReady(): invalid payload_size = "
 217               << payload_size;
 218       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 219       return;
 220     }
 221     if (key_frame)
 222       key_frame_encountered_ = true;
 223     if (!key_frame_encountered_) {
 224       // Do not send video until we have encountered the first key frame.
 225       // Save the bitstream buffer in |stream_header_| to be sent later along
 226       // with the first key frame.
 227       stream_header_.append(static_cast<const char*>(output_buffer->memory()),
 228                             payload_size);
 229     } else if (!encoded_frame_data_storage_.empty()) {
 230       scoped_ptr<EncodedFrame> encoded_frame(
 231           new EncodedFrame());
 232       encoded_frame->dependency = key_frame ? EncodedFrame::KEY :
 233           EncodedFrame::DEPENDENT;
 234       encoded_frame->frame_id = ++last_encoded_frame_id_;
 235       if (key_frame)
 236         encoded_frame->referenced_frame_id = encoded_frame->frame_id;
 237       else
 238         encoded_frame->referenced_frame_id = encoded_frame->frame_id - 1;
 239       encoded_frame->reference_time =
 240           encoded_frame_data_storage_.front().capture_time;
 241       encoded_frame->rtp_timestamp =
 242           GetVideoRtpTimestamp(encoded_frame->reference_time);
 243       if (!stream_header_.empty()) {
 244         encoded_frame->data = stream_header_;
 245         stream_header_.clear();
 246       }
 247       encoded_frame->data.append(
 248           static_cast<const char*>(output_buffer->memory()), payload_size);
 249
 250       cast_environment_->PostTask(
 251           CastEnvironment::MAIN,
 252           FROM_HERE,
 253           base::Bind(&LogFrameEncodedEvent,
 254                      cast_environment_,
 255                      cast_environment_->Clock()->NowTicks(),
 256                      encoded_frame->rtp_timestamp,
 257                      encoded_frame->frame_id));
 258
 259       cast_environment_->PostTask(
 260           CastEnvironment::MAIN,
 261           FROM_HERE,
 262           base::Bind(encoded_frame_data_storage_.front().frame_encoded_callback,
 263                      base::Passed(&encoded_frame)));
 264
 265       encoded_frame_data_storage_.pop_front();
 266     } else {
 267       VLOG(1) << "BitstreamBufferReady(): no encoded frame data available";
 268     }
 269
 270     // We need to re-add the output buffer to the encoder after we are done
 271     // with it.
 272     video_encode_accelerator_->UseOutputBitstreamBuffer(media::BitstreamBuffer(
 273         bitstream_buffer_id,
 274         output_buffers_[bitstream_buffer_id]->handle(),
 275         output_buffers_[bitstream_buffer_id]->mapped_size()));
 276   }
 277
 278  private:
 279   LocalVideoEncodeAcceleratorClient(
 280       scoped_refptr<CastEnvironment> cast_environment,
 281       const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
 282       const base::WeakPtr<ExternalVideoEncoder>& weak_owner)
 283       : cast_environment_(cast_environment),
 284         create_video_encode_memory_cb_(create_video_encode_mem_cb),
 285         weak_owner_(weak_owner),
 286         last_encoded_frame_id_(kStartFrameId),
 287         key_frame_encountered_(false) {}
 288
 289   // Trampoline VEA creation callback to OnCreateVideoEncodeAccelerator()
 290   // on encoder_task_runner. Normally we would just repost the same method to
 291   // it, and would not need a separate proxy method, but we can't
 292   // ThreadTaskRunnerHandle::Get() in unittests just yet.
 293   void OnCreateVideoEncodeAcceleratorProxy(
 294       scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,
 295       scoped_ptr<media::VideoEncodeAccelerator> vea) {
 296     encoder_task_runner->PostTask(
 297         FROM_HERE,
 298         base::Bind(&media::cast::LocalVideoEncodeAcceleratorClient::
 299                        OnCreateVideoEncodeAccelerator,
 300                    this,
 301                    encoder_task_runner,
 302                    base::Passed(&vea)));
 303   }
 304
 305   void OnCreateVideoEncodeAccelerator(
 306       scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,
 307       scoped_ptr<media::VideoEncodeAccelerator> vea) {
 308     encoder_task_runner_ = encoder_task_runner;
 309     video_encode_accelerator_.reset(vea.release());
 310
 311     cast_environment_->PostTask(
 312         CastEnvironment::MAIN,
 313         FROM_HERE,
 314         base::Bind(&ExternalVideoEncoder::OnCreateVideoEncodeAccelerator,
 315                    weak_owner_,
 316                    encoder_task_runner_));
 317   }
 318
 319   // Note: This method can be called on any thread.
 320   void OnCreateSharedMemory(scoped_ptr<base::SharedMemory> memory) {
 321     encoder_task_runner_->PostTask(
 322         FROM_HERE,
 323         base::Bind(&LocalVideoEncodeAcceleratorClient::ReceivedSharedMemory,
 324                    this,
 325                    base::Passed(&memory)));
 326   }
 327
 328   void ReceivedSharedMemory(scoped_ptr<base::SharedMemory> memory) {
 329     DCHECK(encoder_task_runner_.get());
 330     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 331
 332     output_buffers_.push_back(memory.release());
 333
 334     // Wait until all requested buffers are received.
 335     if (output_buffers_.size() < kOutputBufferCount)
 336       return;
 337
 338     // Immediately provide all output buffers to the VEA.
 339     for (size_t i = 0; i < output_buffers_.size(); ++i) {
 340       video_encode_accelerator_->UseOutputBitstreamBuffer(
 341           media::BitstreamBuffer(static_cast<int32>(i),
 342                                  output_buffers_[i]->handle(),
 343                                  output_buffers_[i]->mapped_size()));
 344     }
 345
 346     cast_environment_->PostTask(
 347         CastEnvironment::MAIN,
 348         FROM_HERE,
 349         base::Bind(&ExternalVideoEncoder::EncoderInitialized, weak_owner_,
 350                    true));
 351   }
 352
 353   static void DestroyVideoEncodeAcceleratorOnEncoderThread(
 354       scoped_ptr<media::VideoEncodeAccelerator> vea) {
 355     // VEA::~VEA specialization takes care of calling Destroy() on the VEA impl.
 356   }
 357
 358   friend class base::RefCountedThreadSafe<LocalVideoEncodeAcceleratorClient>;
 359
 360   virtual ~LocalVideoEncodeAcceleratorClient() {
 361     Destroy();
 362     DCHECK(!video_encode_accelerator_);
 363   }
 364
 365   const scoped_refptr<CastEnvironment> cast_environment_;
 366   scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner_;
 367   scoped_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
 368   const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_;
 369   const base::WeakPtr<ExternalVideoEncoder> weak_owner_;
 370   int max_frame_rate_;
 371   uint32 last_encoded_frame_id_;
 372   bool key_frame_encountered_;
 373   std::string stream_header_;
 374
 375   // Shared memory buffers for output with the VideoAccelerator.
 376   ScopedVector<base::SharedMemory> output_buffers_;
 377
 378   // FIFO list.
 379   std::list<EncodedFrameReturnData> encoded_frame_data_storage_;
 380
 381   DISALLOW_COPY_AND_ASSIGN(LocalVideoEncodeAcceleratorClient);
 382 };
 383
 384 ExternalVideoEncoder::ExternalVideoEncoder(
 385     scoped_refptr<CastEnvironment> cast_environment,
 386     const VideoSenderConfig& video_config,
 387     const CastInitializationCallback& initialization_cb,
 388     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
 389     const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb)
 390     : video_config_(video_config),
 391       cast_environment_(cast_environment),
 392       encoder_active_(false),
 393       key_frame_requested_(false),
 394       initialization_cb_(initialization_cb),
 395       weak_factory_(this) {
 396   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 397
 398   video_accelerator_client_ =
 399       LocalVideoEncodeAcceleratorClient::Create(cast_environment_,
 400                                                 create_vea_cb,
 401                                                 create_video_encode_mem_cb,
 402                                                 weak_factory_.GetWeakPtr());
 403   DCHECK(video_accelerator_client_.get());
 404 }
 405
 406 ExternalVideoEncoder::~ExternalVideoEncoder() {
 407 }
 408
 409 void ExternalVideoEncoder::EncoderInitialized(bool success) {
 410   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 411   encoder_active_ = success;
 412   DCHECK(!initialization_cb_.is_null());
 413   initialization_cb_.Run(
 414       success ?
 415       STATUS_VIDEO_INITIALIZED : STATUS_HW_VIDEO_ENCODER_NOT_SUPPORTED);
 416   initialization_cb_.Reset();
 417 }
 418
 419 void ExternalVideoEncoder::EncoderError() {
 420   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 421   encoder_active_ = false;
 422 }
 423
 424 void ExternalVideoEncoder::OnCreateVideoEncodeAccelerator(
 425     scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner) {
 426   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 427   encoder_task_runner_ = encoder_task_runner;
 428
 429   encoder_task_runner_->PostTask(
 430       FROM_HERE,
 431       base::Bind(&LocalVideoEncodeAcceleratorClient::Initialize,
 432                  video_accelerator_client_,
 433                  video_config_));
 434 }
 435
 436 bool ExternalVideoEncoder::EncodeVideoFrame(
 437     const scoped_refptr<media::VideoFrame>& video_frame,
 438     const base::TimeTicks& capture_time,
 439     const FrameEncodedCallback& frame_encoded_callback) {
 440   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 441
 442   if (!encoder_active_)
 443     return false;
 444
 445   encoder_task_runner_->PostTask(
 446       FROM_HERE,
 447       base::Bind(&LocalVideoEncodeAcceleratorClient::EncodeVideoFrame,
 448                  video_accelerator_client_,
 449                  video_frame,
 450                  capture_time,
 451                  key_frame_requested_,
 452                  frame_encoded_callback));
 453
 454   key_frame_requested_ = false;
 455   return true;
 456 }
 457
 458 // Inform the encoder about the new target bit rate.
 459 void ExternalVideoEncoder::SetBitRate(int new_bit_rate) {
 460   if (!encoder_active_) {
 461     // If we receive SetBitRate() before VEA creation callback is invoked,
 462     // cache the new bit rate in the encoder config and use the new settings
 463     // to initialize VEA.
 464     video_config_.start_bitrate = new_bit_rate;
 465     return;
 466   }
 467
 468   encoder_task_runner_->PostTask(
 469       FROM_HERE,
 470       base::Bind(&LocalVideoEncodeAcceleratorClient::SetBitRate,
 471                  video_accelerator_client_,
 472                  new_bit_rate));
 473 }
 474
 475 // Inform the encoder to encode the next frame as a key frame.
 476 void ExternalVideoEncoder::GenerateKeyFrame() {
 477   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 478   key_frame_requested_ = true;
 479 }
 480
 481 // Inform the encoder to only reference frames older or equal to frame_id;
 482 void ExternalVideoEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) {
 483   // Do nothing not supported.
 484 }
 485 }  //  namespace cast
 486 }  //  namespace media