src/media/cast/sender/external_video_encoder.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/sender/external_video_encoder.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/memory/scoped_vector.h"
  10 #include "base/memory/shared_memory.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "base/metrics/histogram.h"
  13 #include "media/base/video_frame.h"
  14 #include "media/base/video_util.h"
  15 #include "media/cast/cast_defines.h"
  16 #include "media/cast/logging/logging_defines.h"
  17 #include "media/cast/net/cast_transport_config.h"
  18 #include "media/video/video_encode_accelerator.h"
  19
  20 namespace media {
  21 namespace cast {
  22 class LocalVideoEncodeAcceleratorClient;
  23 }  // namespace cast
  24 }  // namespace media
  25
  26 namespace {
  27 static const size_t kOutputBufferCount = 3;
  28
  29 void LogFrameEncodedEvent(
  30     const scoped_refptr<media::cast::CastEnvironment>& cast_environment,
  31     base::TimeTicks event_time,
  32     media::cast::RtpTimestamp rtp_timestamp,
  33     uint32 frame_id) {
  34   cast_environment->Logging()->InsertFrameEvent(
  35       event_time, media::cast::FRAME_ENCODED, media::cast::VIDEO_EVENT,
  36       rtp_timestamp, frame_id);
  37 }
  38 }  // namespace
  39
  40 namespace media {
  41 namespace cast {
  42
  43 // Container for the associated data of a video frame being processed.
  44 struct InProgressFrameEncode {
  45   const RtpTimestamp rtp_timestamp;
  46   const base::TimeTicks reference_time;
  47   const VideoEncoder::FrameEncodedCallback frame_encoded_callback;
  48
  49   InProgressFrameEncode(RtpTimestamp rtp,
  50                         base::TimeTicks r_time,
  51                         VideoEncoder::FrameEncodedCallback callback)
  52       : rtp_timestamp(rtp),
  53         reference_time(r_time),
  54         frame_encoded_callback(callback) {}
  55 };
  56
  57 // The ExternalVideoEncoder class can be deleted directly by cast, while
  58 // LocalVideoEncodeAcceleratorClient stays around long enough to properly shut
  59 // down the VideoEncodeAccelerator.
  60 class LocalVideoEncodeAcceleratorClient
  61     : public VideoEncodeAccelerator::Client,
  62       public base::RefCountedThreadSafe<LocalVideoEncodeAcceleratorClient> {
  63  public:
  64   // Create an instance of this class and post a task to create
  65   // video_encode_accelerator_. A ref to |this| will be kept, awaiting reply
  66   // via ProxyCreateVideoEncodeAccelerator, which will provide us with the
  67   // encoder task runner and vea instance. We cannot be destroyed until we
  68   // receive the reply, otherwise the VEA object created may leak.
  69   static scoped_refptr<LocalVideoEncodeAcceleratorClient> Create(
  70       scoped_refptr<CastEnvironment> cast_environment,
  71       const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
  72       const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
  73       const base::WeakPtr<ExternalVideoEncoder>& weak_owner) {
  74     scoped_refptr<LocalVideoEncodeAcceleratorClient> client(
  75         new LocalVideoEncodeAcceleratorClient(
  76             cast_environment, create_video_encode_mem_cb, weak_owner));
  77
  78     // This will keep a ref to |client|, if weak_owner is destroyed before
  79     // ProxyCreateVideoEncodeAccelerator is called, we will stay alive until
  80     // we can properly destroy the VEA.
  81     create_vea_cb.Run(base::Bind(
  82         &LocalVideoEncodeAcceleratorClient::OnCreateVideoEncodeAcceleratorProxy,
  83         client));
  84
  85     return client;
  86   }
  87
  88   // Initialize the real HW encoder.
  89   void Initialize(const VideoSenderConfig& video_config) {
  90     DCHECK(encoder_task_runner_.get());
  91     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
  92
  93     VideoCodecProfile output_profile = media::VIDEO_CODEC_PROFILE_UNKNOWN;
  94     switch (video_config.codec) {
  95       case CODEC_VIDEO_VP8:
  96         output_profile = media::VP8PROFILE_ANY;
  97         break;
  98       case CODEC_VIDEO_H264:
  99         output_profile = media::H264PROFILE_MAIN;
 100         break;
 101       case CODEC_VIDEO_FAKE:
 102         NOTREACHED() << "Fake software video encoder cannot be external";
 103         break;
 104       default:
 105         NOTREACHED() << "Video codec not specified or not supported";
 106         break;
 107     }
 108     max_frame_rate_ = video_config.max_frame_rate;
 109
 110     bool result = video_encode_accelerator_->Initialize(
 111         media::VideoFrame::I420,
 112         gfx::Size(video_config.width, video_config.height),
 113         output_profile,
 114         video_config.start_bitrate,
 115         this);
 116
 117     UMA_HISTOGRAM_BOOLEAN("Cast.Sender.VideoEncodeAcceleratorInitializeSuccess",
 118                           result);
 119     if (!result) {
 120       cast_environment_->PostTask(
 121           CastEnvironment::MAIN,
 122           FROM_HERE,
 123           base::Bind(&ExternalVideoEncoder::EncoderInitialized, weak_owner_,
 124                      false));
 125       return;
 126     }
 127
 128     // Wait until shared memory is allocated to indicate that encoder is
 129     // initialized.
 130   }
 131
 132   // Destroy the VEA on the correct thread.
 133   void Destroy() {
 134     DCHECK(encoder_task_runner_.get());
 135     if (!video_encode_accelerator_)
 136       return;
 137
 138     if (encoder_task_runner_->RunsTasksOnCurrentThread()) {
 139       video_encode_accelerator_.reset();
 140     } else {
 141       // We do this instead of just reposting to encoder_task_runner_, because
 142       // we are called from the destructor.
 143       encoder_task_runner_->PostTask(
 144           FROM_HERE,
 145           base::Bind(&DestroyVideoEncodeAcceleratorOnEncoderThread,
 146                      base::Passed(&video_encode_accelerator_)));
 147     }
 148   }
 149
 150   void SetBitRate(uint32 bit_rate) {
 151     DCHECK(encoder_task_runner_.get());
 152     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 153
 154     video_encode_accelerator_->RequestEncodingParametersChange(bit_rate,
 155                                                                max_frame_rate_);
 156   }
 157
 158   void EncodeVideoFrame(
 159       const scoped_refptr<media::VideoFrame>& video_frame,
 160       const base::TimeTicks& reference_time,
 161       bool key_frame_requested,
 162       const VideoEncoder::FrameEncodedCallback& frame_encoded_callback) {
 163     DCHECK(encoder_task_runner_.get());
 164     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 165
 166     in_progress_frame_encodes_.push_back(InProgressFrameEncode(
 167         TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency),
 168         reference_time,
 169         frame_encoded_callback));
 170
 171     // BitstreamBufferReady will be called once the encoder is done.
 172     video_encode_accelerator_->Encode(video_frame, key_frame_requested);
 173   }
 174
 175  protected:
 176   void NotifyError(VideoEncodeAccelerator::Error error) override {
 177     DCHECK(encoder_task_runner_.get());
 178     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 179     VLOG(1) << "ExternalVideoEncoder NotifyError: " << error;
 180
 181     cast_environment_->PostTask(
 182         CastEnvironment::MAIN,
 183         FROM_HERE,
 184         base::Bind(&ExternalVideoEncoder::EncoderError, weak_owner_));
 185   }
 186
 187   // Called to allocate the input and output buffers.
 188   void RequireBitstreamBuffers(unsigned int input_count,
 189                                const gfx::Size& input_coded_size,
 190                                size_t output_buffer_size) override {
 191     DCHECK(encoder_task_runner_.get());
 192     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 193     DCHECK(video_encode_accelerator_);
 194
 195     for (size_t j = 0; j < kOutputBufferCount; ++j) {
 196       create_video_encode_memory_cb_.Run(
 197           output_buffer_size,
 198           base::Bind(&LocalVideoEncodeAcceleratorClient::OnCreateSharedMemory,
 199                      this));
 200     }
 201   }
 202
 203   // Encoder has encoded a frame and it's available in one of out output
 204   // buffers.
 205   void BitstreamBufferReady(int32 bitstream_buffer_id,
 206                             size_t payload_size,
 207                             bool key_frame) override {
 208     DCHECK(encoder_task_runner_.get());
 209     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 210     if (bitstream_buffer_id < 0 ||
 211         bitstream_buffer_id >= static_cast<int32>(output_buffers_.size())) {
 212       NOTREACHED();
 213       VLOG(1) << "BitstreamBufferReady(): invalid bitstream_buffer_id="
 214               << bitstream_buffer_id;
 215       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 216       return;
 217     }
 218     base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
 219     if (payload_size > output_buffer->mapped_size()) {
 220       NOTREACHED();
 221       VLOG(1) << "BitstreamBufferReady(): invalid payload_size = "
 222               << payload_size;
 223       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 224       return;
 225     }
 226     if (key_frame)
 227       key_frame_encountered_ = true;
 228     if (!key_frame_encountered_) {
 229       // Do not send video until we have encountered the first key frame.
 230       // Save the bitstream buffer in |stream_header_| to be sent later along
 231       // with the first key frame.
 232       stream_header_.append(static_cast<const char*>(output_buffer->memory()),
 233                             payload_size);
 234     } else if (!in_progress_frame_encodes_.empty()) {
 235       const InProgressFrameEncode& request = in_progress_frame_encodes_.front();
 236
 237       scoped_ptr<EncodedFrame> encoded_frame(new EncodedFrame());
 238       encoded_frame->dependency = key_frame ? EncodedFrame::KEY :
 239           EncodedFrame::DEPENDENT;
 240       encoded_frame->frame_id = ++last_encoded_frame_id_;
 241       if (key_frame)
 242         encoded_frame->referenced_frame_id = encoded_frame->frame_id;
 243       else
 244         encoded_frame->referenced_frame_id = encoded_frame->frame_id - 1;
 245       encoded_frame->rtp_timestamp = request.rtp_timestamp;
 246       encoded_frame->reference_time = request.reference_time;
 247       if (!stream_header_.empty()) {
 248         encoded_frame->data = stream_header_;
 249         stream_header_.clear();
 250       }
 251       encoded_frame->data.append(
 252           static_cast<const char*>(output_buffer->memory()), payload_size);
 253
 254       cast_environment_->PostTask(
 255           CastEnvironment::MAIN,
 256           FROM_HERE,
 257           base::Bind(&LogFrameEncodedEvent,
 258                      cast_environment_,
 259                      cast_environment_->Clock()->NowTicks(),
 260                      encoded_frame->rtp_timestamp,
 261                      encoded_frame->frame_id));
 262
 263       cast_environment_->PostTask(
 264           CastEnvironment::MAIN,
 265           FROM_HERE,
 266           base::Bind(request.frame_encoded_callback,
 267                      base::Passed(&encoded_frame)));
 268
 269       in_progress_frame_encodes_.pop_front();
 270     } else {
 271       VLOG(1) << "BitstreamBufferReady(): no encoded frame data available";
 272     }
 273
 274     // We need to re-add the output buffer to the encoder after we are done
 275     // with it.
 276     video_encode_accelerator_->UseOutputBitstreamBuffer(media::BitstreamBuffer(
 277         bitstream_buffer_id,
 278         output_buffers_[bitstream_buffer_id]->handle(),
 279         output_buffers_[bitstream_buffer_id]->mapped_size()));
 280   }
 281
 282  private:
 283   LocalVideoEncodeAcceleratorClient(
 284       scoped_refptr<CastEnvironment> cast_environment,
 285       const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
 286       const base::WeakPtr<ExternalVideoEncoder>& weak_owner)
 287       : cast_environment_(cast_environment),
 288         create_video_encode_memory_cb_(create_video_encode_mem_cb),
 289         weak_owner_(weak_owner),
 290         last_encoded_frame_id_(kStartFrameId),
 291         key_frame_encountered_(false) {}
 292
 293   // Trampoline VEA creation callback to OnCreateVideoEncodeAccelerator()
 294   // on encoder_task_runner. Normally we would just repost the same method to
 295   // it, and would not need a separate proxy method, but we can't
 296   // ThreadTaskRunnerHandle::Get() in unittests just yet.
 297   void OnCreateVideoEncodeAcceleratorProxy(
 298       scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,
 299       scoped_ptr<media::VideoEncodeAccelerator> vea) {
 300     encoder_task_runner->PostTask(
 301         FROM_HERE,
 302         base::Bind(&media::cast::LocalVideoEncodeAcceleratorClient::
 303                        OnCreateVideoEncodeAccelerator,
 304                    this,
 305                    encoder_task_runner,
 306                    base::Passed(&vea)));
 307   }
 308
 309   void OnCreateVideoEncodeAccelerator(
 310       scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,
 311       scoped_ptr<media::VideoEncodeAccelerator> vea) {
 312     encoder_task_runner_ = encoder_task_runner;
 313     video_encode_accelerator_.reset(vea.release());
 314
 315     cast_environment_->PostTask(
 316         CastEnvironment::MAIN,
 317         FROM_HERE,
 318         base::Bind(&ExternalVideoEncoder::OnCreateVideoEncodeAccelerator,
 319                    weak_owner_,
 320                    encoder_task_runner_));
 321   }
 322
 323   // Note: This method can be called on any thread.
 324   void OnCreateSharedMemory(scoped_ptr<base::SharedMemory> memory) {
 325     encoder_task_runner_->PostTask(
 326         FROM_HERE,
 327         base::Bind(&LocalVideoEncodeAcceleratorClient::ReceivedSharedMemory,
 328                    this,
 329                    base::Passed(&memory)));
 330   }
 331
 332   void ReceivedSharedMemory(scoped_ptr<base::SharedMemory> memory) {
 333     DCHECK(encoder_task_runner_.get());
 334     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 335
 336     output_buffers_.push_back(memory.release());
 337
 338     // Wait until all requested buffers are received.
 339     if (output_buffers_.size() < kOutputBufferCount)
 340       return;
 341
 342     // Immediately provide all output buffers to the VEA.
 343     for (size_t i = 0; i < output_buffers_.size(); ++i) {
 344       video_encode_accelerator_->UseOutputBitstreamBuffer(
 345           media::BitstreamBuffer(static_cast<int32>(i),
 346                                  output_buffers_[i]->handle(),
 347                                  output_buffers_[i]->mapped_size()));
 348     }
 349
 350     cast_environment_->PostTask(
 351         CastEnvironment::MAIN,
 352         FROM_HERE,
 353         base::Bind(&ExternalVideoEncoder::EncoderInitialized, weak_owner_,
 354                    true));
 355   }
 356
 357   static void DestroyVideoEncodeAcceleratorOnEncoderThread(
 358       scoped_ptr<media::VideoEncodeAccelerator> vea) {
 359     // VEA::~VEA specialization takes care of calling Destroy() on the VEA impl.
 360   }
 361
 362   friend class base::RefCountedThreadSafe<LocalVideoEncodeAcceleratorClient>;
 363
 364   ~LocalVideoEncodeAcceleratorClient() override {
 365     Destroy();
 366     DCHECK(!video_encode_accelerator_);
 367   }
 368
 369   const scoped_refptr<CastEnvironment> cast_environment_;
 370   scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner_;
 371   scoped_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
 372   const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_;
 373   const base::WeakPtr<ExternalVideoEncoder> weak_owner_;
 374   int max_frame_rate_;
 375   uint32 last_encoded_frame_id_;
 376   bool key_frame_encountered_;
 377   std::string stream_header_;
 378
 379   // Shared memory buffers for output with the VideoAccelerator.
 380   ScopedVector<base::SharedMemory> output_buffers_;
 381
 382   // FIFO list.
 383   std::list<InProgressFrameEncode> in_progress_frame_encodes_;
 384
 385   DISALLOW_COPY_AND_ASSIGN(LocalVideoEncodeAcceleratorClient);
 386 };
 387
 388 ExternalVideoEncoder::ExternalVideoEncoder(
 389     scoped_refptr<CastEnvironment> cast_environment,
 390     const VideoSenderConfig& video_config,
 391     const CastInitializationCallback& initialization_cb,
 392     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
 393     const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb)
 394     : video_config_(video_config),
 395       cast_environment_(cast_environment),
 396       encoder_active_(false),
 397       key_frame_requested_(false),
 398       initialization_cb_(initialization_cb),
 399       weak_factory_(this) {
 400   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 401
 402   video_accelerator_client_ =
 403       LocalVideoEncodeAcceleratorClient::Create(cast_environment_,
 404                                                 create_vea_cb,
 405                                                 create_video_encode_mem_cb,
 406                                                 weak_factory_.GetWeakPtr());
 407   DCHECK(video_accelerator_client_.get());
 408 }
 409
 410 ExternalVideoEncoder::~ExternalVideoEncoder() {
 411 }
 412
 413 void ExternalVideoEncoder::EncoderInitialized(bool success) {
 414   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 415   encoder_active_ = success;
 416   DCHECK(!initialization_cb_.is_null());
 417   initialization_cb_.Run(
 418       success ?
 419       STATUS_VIDEO_INITIALIZED : STATUS_HW_VIDEO_ENCODER_NOT_SUPPORTED);
 420   initialization_cb_.Reset();
 421 }
 422
 423 void ExternalVideoEncoder::EncoderError() {
 424   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 425   encoder_active_ = false;
 426 }
 427
 428 void ExternalVideoEncoder::OnCreateVideoEncodeAccelerator(
 429     scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner) {
 430   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 431   encoder_task_runner_ = encoder_task_runner;
 432
 433   encoder_task_runner_->PostTask(
 434       FROM_HERE,
 435       base::Bind(&LocalVideoEncodeAcceleratorClient::Initialize,
 436                  video_accelerator_client_,
 437                  video_config_));
 438 }
 439
 440 bool ExternalVideoEncoder::EncodeVideoFrame(
 441     const scoped_refptr<media::VideoFrame>& video_frame,
 442     const base::TimeTicks& reference_time,
 443     const FrameEncodedCallback& frame_encoded_callback) {
 444   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 445
 446   if (!encoder_active_)
 447     return false;
 448
 449   encoder_task_runner_->PostTask(
 450       FROM_HERE,
 451       base::Bind(&LocalVideoEncodeAcceleratorClient::EncodeVideoFrame,
 452                  video_accelerator_client_,
 453                  video_frame,
 454                  reference_time,
 455                  key_frame_requested_,
 456                  frame_encoded_callback));
 457
 458   key_frame_requested_ = false;
 459   return true;
 460 }
 461
 462 // Inform the encoder about the new target bit rate.
 463 void ExternalVideoEncoder::SetBitRate(int new_bit_rate) {
 464   if (!encoder_active_) {
 465     // If we receive SetBitRate() before VEA creation callback is invoked,
 466     // cache the new bit rate in the encoder config and use the new settings
 467     // to initialize VEA.
 468     video_config_.start_bitrate = new_bit_rate;
 469     return;
 470   }
 471
 472   encoder_task_runner_->PostTask(
 473       FROM_HERE,
 474       base::Bind(&LocalVideoEncodeAcceleratorClient::SetBitRate,
 475                  video_accelerator_client_,
 476                  new_bit_rate));
 477 }
 478
 479 // Inform the encoder to encode the next frame as a key frame.
 480 void ExternalVideoEncoder::GenerateKeyFrame() {
 481   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 482   key_frame_requested_ = true;
 483 }
 484
 485 // Inform the encoder to only reference frames older or equal to frame_id;
 486 void ExternalVideoEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) {
 487   // Do nothing not supported.
 488 }
 489 }  //  namespace cast
 490 }  //  namespace media