src/media/cast/video_sender/external_video_encoder.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/video_sender/external_video_encoder.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/memory/scoped_vector.h"
  10 #include "base/memory/shared_memory.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "media/base/video_frame.h"
  13 #include "media/base/video_util.h"
  14 #include "media/cast/cast_defines.h"
  15 #include "media/cast/transport/cast_transport_config.h"
  16 #include "media/video/video_encode_accelerator.h"
  17
  18 namespace {
  19 // We allocate more input buffers than what is asked for by
  20 // RequireBitstreamBuffers() due to potential threading timing.
  21 static const int kInputBufferExtraCount = 1;
  22 static const int kOutputBufferCount = 3;
  23
  24 void LogFrameEncodedEvent(media::cast::CastEnvironment* const cast_environment,
  25                           const base::TimeTicks& capture_time) {
  26   cast_environment->Logging()->InsertFrameEvent(
  27       cast_environment->Clock()->NowTicks(),
  28       media::cast::kVideoFrameEncoded,
  29       media::cast::GetVideoRtpTimestamp(capture_time),
  30       media::cast::kFrameIdUnknown);
  31 }
  32 }  // namespace
  33
  34 namespace media {
  35 namespace cast {
  36
  37 // Container for the associated data of a video frame being processed.
  38 struct EncodedFrameReturnData {
  39   EncodedFrameReturnData(base::TimeTicks c_time,
  40                          VideoEncoder::FrameEncodedCallback callback) {
  41     capture_time = c_time;
  42     frame_encoded_callback = callback;
  43   }
  44   base::TimeTicks capture_time;
  45   VideoEncoder::FrameEncodedCallback frame_encoded_callback;
  46 };
  47
  48 // The ExternalVideoEncoder class can be deleted directly by cast, while
  49 // LocalVideoEncodeAcceleratorClient stays around long enough to properly shut
  50 // down the VideoEncodeAccelerator.
  51 class LocalVideoEncodeAcceleratorClient
  52     : public VideoEncodeAccelerator::Client,
  53       public base::RefCountedThreadSafe<LocalVideoEncodeAcceleratorClient> {
  54  public:
  55   LocalVideoEncodeAcceleratorClient(
  56       scoped_refptr<CastEnvironment> cast_environment,
  57       scoped_refptr<GpuVideoAcceleratorFactories> gpu_factories,
  58       const base::WeakPtr<ExternalVideoEncoder>& weak_owner)
  59       : cast_environment_(cast_environment),
  60         gpu_factories_(gpu_factories),
  61         encoder_task_runner_(gpu_factories->GetTaskRunner()),
  62         weak_owner_(weak_owner),
  63         last_encoded_frame_id_(kStartFrameId) {
  64     DCHECK(encoder_task_runner_);
  65   }
  66
  67   // Initialize the real HW encoder.
  68   void Initialize(const VideoSenderConfig& video_config) {
  69     DCHECK(gpu_factories_);
  70     DCHECK(encoder_task_runner_);
  71
  72     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
  73
  74     video_encode_accelerator_ =
  75         gpu_factories_->CreateVideoEncodeAccelerator(this).Pass();
  76     if (!video_encode_accelerator_)
  77       return;
  78
  79     VideoCodecProfile output_profile = media::VIDEO_CODEC_PROFILE_UNKNOWN;
  80     switch (video_config.codec) {
  81       case transport::kVp8:
  82         output_profile = media::VP8PROFILE_MAIN;
  83         break;
  84       case transport::kH264:
  85         output_profile = media::H264PROFILE_MAIN;
  86         break;
  87     }
  88     codec_ = video_config.codec;
  89     max_frame_rate_ = video_config.max_frame_rate;
  90
  91     // Asynchronous initialization call; NotifyInitializeDone or NotifyError
  92     // will be called once the HW is initialized.
  93     video_encode_accelerator_->Initialize(
  94         media::VideoFrame::I420,
  95         gfx::Size(video_config.width, video_config.height),
  96         output_profile,
  97         video_config.start_bitrate);
  98   }
  99
 100   // Free the HW.
 101   void Destroy() {
 102     DCHECK(encoder_task_runner_);
 103     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 104
 105     if (video_encode_accelerator_) {
 106       video_encode_accelerator_.release()->Destroy();
 107     }
 108   }
 109
 110   void SetBitRate(uint32 bit_rate) {
 111     DCHECK(encoder_task_runner_);
 112     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 113
 114     video_encode_accelerator_->RequestEncodingParametersChange(bit_rate,
 115                                                                max_frame_rate_);
 116   }
 117
 118   void EncodeVideoFrame(
 119       const scoped_refptr<media::VideoFrame>& video_frame,
 120       const base::TimeTicks& capture_time,
 121       bool key_frame_requested,
 122       const VideoEncoder::FrameEncodedCallback& frame_encoded_callback) {
 123     DCHECK(encoder_task_runner_);
 124     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 125
 126     if (input_buffers_free_.empty()) {
 127       NOTREACHED();
 128       VLOG(2) << "EncodeVideoFrame(): drop frame due to no hw buffers";
 129       return;
 130     }
 131     const int index = input_buffers_free_.back();
 132     base::SharedMemory* input_buffer = input_buffers_[index];
 133
 134     // TODO(pwestin): this allocation and copy can be removed once we don't
 135     // pass the video frame through liblingle.
 136
 137     scoped_refptr<media::VideoFrame> frame =
 138         media::VideoFrame::WrapExternalPackedMemory(
 139             video_frame->format(),
 140             video_frame->coded_size(),
 141             video_frame->visible_rect(),
 142             video_frame->natural_size(),
 143             reinterpret_cast<uint8*>(input_buffer->memory()),
 144             input_buffer->mapped_size(),
 145             input_buffer->handle(),
 146             video_frame->GetTimestamp(),
 147             base::Bind(&LocalVideoEncodeAcceleratorClient::FinishedWithInBuffer,
 148                        this,
 149                        index));
 150
 151     if (!frame) {
 152       VLOG(1) << "EncodeVideoFrame(): failed to create frame";
 153       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 154       return;
 155     }
 156     // Do a stride copy of the input frame to match the input requirements.
 157     media::CopyYPlane(video_frame->data(VideoFrame::kYPlane),
 158                       video_frame->stride(VideoFrame::kYPlane),
 159                       video_frame->natural_size().height(),
 160                       frame.get());
 161     media::CopyUPlane(video_frame->data(VideoFrame::kUPlane),
 162                       video_frame->stride(VideoFrame::kUPlane),
 163                       video_frame->natural_size().height(),
 164                       frame.get());
 165     media::CopyVPlane(video_frame->data(VideoFrame::kVPlane),
 166                       video_frame->stride(VideoFrame::kVPlane),
 167                       video_frame->natural_size().height(),
 168                       frame.get());
 169
 170     encoded_frame_data_storage_.push_back(
 171         EncodedFrameReturnData(capture_time, frame_encoded_callback));
 172
 173     // BitstreamBufferReady will be called once the encoder is done.
 174     video_encode_accelerator_->Encode(frame, key_frame_requested);
 175   }
 176
 177  protected:
 178   virtual void NotifyInitializeDone() OVERRIDE {
 179     DCHECK(encoder_task_runner_);
 180     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 181
 182     cast_environment_->PostTask(
 183         CastEnvironment::MAIN,
 184         FROM_HERE,
 185         base::Bind(&ExternalVideoEncoder::EncoderInitialized, weak_owner_));
 186   }
 187
 188   virtual void NotifyError(VideoEncodeAccelerator::Error error) OVERRIDE {
 189     DCHECK(encoder_task_runner_);
 190     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 191     VLOG(1) << "ExternalVideoEncoder NotifyError: " << error;
 192
 193     if (video_encode_accelerator_) {
 194       video_encode_accelerator_.release()->Destroy();
 195     }
 196     cast_environment_->PostTask(
 197         CastEnvironment::MAIN,
 198         FROM_HERE,
 199         base::Bind(&ExternalVideoEncoder::EncoderError, weak_owner_));
 200   }
 201
 202   // Called to allocate the input and output buffers.
 203   virtual void RequireBitstreamBuffers(unsigned int input_count,
 204                                        const gfx::Size& input_coded_size,
 205                                        size_t output_buffer_size) OVERRIDE {
 206     DCHECK(encoder_task_runner_);
 207     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 208     DCHECK(video_encode_accelerator_);
 209
 210     for (unsigned int i = 0; i < input_count + kInputBufferExtraCount; ++i) {
 211       base::SharedMemory* shm =
 212           gpu_factories_->CreateSharedMemory(media::VideoFrame::AllocationSize(
 213               media::VideoFrame::I420, input_coded_size));
 214       if (!shm) {
 215         VLOG(1) << "RequireBitstreamBuffers(): failed to create input buffer ";
 216         return;
 217       }
 218       input_buffers_.push_back(shm);
 219       input_buffers_free_.push_back(i);
 220     }
 221
 222     for (int j = 0; j < kOutputBufferCount; ++j) {
 223       base::SharedMemory* shm =
 224           gpu_factories_->CreateSharedMemory(output_buffer_size);
 225       if (!shm) {
 226         VLOG(1) << "RequireBitstreamBuffers(): failed to create input buffer ";
 227         return;
 228       }
 229       output_buffers_.push_back(shm);
 230     }
 231     // Immediately provide all output buffers to the VEA.
 232     for (size_t i = 0; i < output_buffers_.size(); ++i) {
 233       video_encode_accelerator_->UseOutputBitstreamBuffer(
 234           media::BitstreamBuffer(static_cast<int32>(i),
 235                                  output_buffers_[i]->handle(),
 236                                  output_buffers_[i]->mapped_size()));
 237     }
 238   }
 239
 240   // Encoder has encoded a frame and it's available in one of out output
 241   // buffers.
 242   virtual void BitstreamBufferReady(int32 bitstream_buffer_id,
 243                                     size_t payload_size,
 244                                     bool key_frame) OVERRIDE {
 245     DCHECK(encoder_task_runner_);
 246     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 247     if (bitstream_buffer_id < 0 ||
 248         bitstream_buffer_id >= static_cast<int32>(output_buffers_.size())) {
 249       NOTREACHED();
 250       VLOG(1) << "BitstreamBufferReady(): invalid bitstream_buffer_id="
 251               << bitstream_buffer_id;
 252       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 253       return;
 254     }
 255     base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
 256     if (payload_size > output_buffer->mapped_size()) {
 257       NOTREACHED();
 258       VLOG(1) << "BitstreamBufferReady(): invalid payload_size = "
 259               << payload_size;
 260       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 261       return;
 262     }
 263     if (encoded_frame_data_storage_.empty()) {
 264       NOTREACHED();
 265       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 266       return;
 267     }
 268     scoped_ptr<transport::EncodedVideoFrame> encoded_frame(
 269         new transport::EncodedVideoFrame());
 270
 271     encoded_frame->codec = codec_;
 272     encoded_frame->key_frame = key_frame;
 273     encoded_frame->last_referenced_frame_id = last_encoded_frame_id_;
 274     last_encoded_frame_id_++;
 275     encoded_frame->frame_id = last_encoded_frame_id_;
 276     encoded_frame->rtp_timestamp =
 277         GetVideoRtpTimestamp(encoded_frame_data_storage_.front().capture_time);
 278     if (key_frame) {
 279       // Self referenced.
 280       encoded_frame->last_referenced_frame_id = encoded_frame->frame_id;
 281     }
 282
 283     encoded_frame->data.insert(
 284         0, static_cast<const char*>(output_buffer->memory()), payload_size);
 285
 286     cast_environment_->PostTask(
 287         CastEnvironment::MAIN,
 288         FROM_HERE,
 289         base::Bind(LogFrameEncodedEvent,
 290                    cast_environment_,
 291                    encoded_frame_data_storage_.front().capture_time));
 292
 293     cast_environment_->PostTask(
 294         CastEnvironment::MAIN,
 295         FROM_HERE,
 296         base::Bind(encoded_frame_data_storage_.front().frame_encoded_callback,
 297                    base::Passed(&encoded_frame),
 298                    encoded_frame_data_storage_.front().capture_time));
 299
 300     encoded_frame_data_storage_.pop_front();
 301
 302     // We need to re-add the output buffer to the encoder after we are done
 303     // with it.
 304     video_encode_accelerator_->UseOutputBitstreamBuffer(media::BitstreamBuffer(
 305         bitstream_buffer_id,
 306         output_buffers_[bitstream_buffer_id]->handle(),
 307         output_buffers_[bitstream_buffer_id]->mapped_size()));
 308   }
 309
 310  private:
 311   // Encoder is done with the provided input buffer.
 312   void FinishedWithInBuffer(int input_index) {
 313     DCHECK(encoder_task_runner_);
 314     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 315     DCHECK_GE(input_index, 0);
 316     DCHECK_LT(input_index, static_cast<int>(input_buffers_.size()));
 317     VLOG(2) << "EncodeFrameFinished(): index=" << input_index;
 318     input_buffers_free_.push_back(input_index);
 319   }
 320
 321   friend class base::RefCountedThreadSafe<LocalVideoEncodeAcceleratorClient>;
 322
 323   virtual ~LocalVideoEncodeAcceleratorClient() {}
 324
 325   const scoped_refptr<CastEnvironment> cast_environment_;
 326   scoped_refptr<GpuVideoAcceleratorFactories> gpu_factories_;
 327   scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner_;
 328   const base::WeakPtr<ExternalVideoEncoder> weak_owner_;
 329
 330   scoped_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
 331   int max_frame_rate_;
 332   transport::VideoCodec codec_;
 333   uint32 last_encoded_frame_id_;
 334
 335   // Shared memory buffers for input/output with the VideoAccelerator.
 336   ScopedVector<base::SharedMemory> input_buffers_;
 337   ScopedVector<base::SharedMemory> output_buffers_;
 338
 339   // Input buffers ready to be filled with input from Encode(). As a LIFO since
 340   // we don't care about ordering.
 341   std::vector<int> input_buffers_free_;
 342
 343   // FIFO list.
 344   std::list<EncodedFrameReturnData> encoded_frame_data_storage_;
 345
 346   DISALLOW_COPY_AND_ASSIGN(LocalVideoEncodeAcceleratorClient);
 347 };
 348
 349 ExternalVideoEncoder::ExternalVideoEncoder(
 350     scoped_refptr<CastEnvironment> cast_environment,
 351     const VideoSenderConfig& video_config,
 352     scoped_refptr<GpuVideoAcceleratorFactories> gpu_factories)
 353     : video_config_(video_config),
 354       cast_environment_(cast_environment),
 355       encoder_active_(false),
 356       key_frame_requested_(false),
 357       skip_next_frame_(false),
 358       skip_count_(0),
 359       encoder_task_runner_(gpu_factories->GetTaskRunner()),
 360       weak_factory_(this) {
 361   DCHECK(gpu_factories);
 362   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 363   video_accelerator_client_ = new LocalVideoEncodeAcceleratorClient(
 364       cast_environment, gpu_factories, weak_factory_.GetWeakPtr());
 365
 366   encoder_task_runner_->PostTask(
 367       FROM_HERE,
 368       base::Bind(&LocalVideoEncodeAcceleratorClient::Initialize,
 369                  video_accelerator_client_,
 370                  video_config));
 371 }
 372
 373 ExternalVideoEncoder::~ExternalVideoEncoder() {
 374   encoder_task_runner_->PostTask(
 375       FROM_HERE,
 376       base::Bind(&LocalVideoEncodeAcceleratorClient::Destroy,
 377                  video_accelerator_client_));
 378 }
 379
 380 void ExternalVideoEncoder::EncoderInitialized() {
 381   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 382   encoder_active_ = true;
 383 }
 384
 385 void ExternalVideoEncoder::EncoderError() {
 386   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 387   encoder_active_ = false;
 388 }
 389
 390 bool ExternalVideoEncoder::EncodeVideoFrame(
 391     const scoped_refptr<media::VideoFrame>& video_frame,
 392     const base::TimeTicks& capture_time,
 393     const FrameEncodedCallback& frame_encoded_callback) {
 394   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 395
 396   if (!encoder_active_)
 397     return false;
 398
 399   if (skip_next_frame_) {
 400     VLOG(1) << "Skip encoding frame";
 401     ++skip_count_;
 402     skip_next_frame_ = false;
 403     return false;
 404   }
 405   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 406   cast_environment_->Logging()->InsertFrameEvent(
 407       now,
 408       kVideoFrameSentToEncoder,
 409       GetVideoRtpTimestamp(capture_time),
 410       kFrameIdUnknown);
 411
 412   encoder_task_runner_->PostTask(
 413       FROM_HERE,
 414       base::Bind(&LocalVideoEncodeAcceleratorClient::EncodeVideoFrame,
 415                  video_accelerator_client_,
 416                  video_frame,
 417                  capture_time,
 418                  key_frame_requested_,
 419                  frame_encoded_callback));
 420
 421   key_frame_requested_ = false;
 422   return true;
 423 }
 424
 425 // Inform the encoder about the new target bit rate.
 426 void ExternalVideoEncoder::SetBitRate(int new_bit_rate) {
 427   encoder_task_runner_->PostTask(
 428       FROM_HERE,
 429       base::Bind(&LocalVideoEncodeAcceleratorClient::SetBitRate,
 430                  video_accelerator_client_,
 431                  new_bit_rate));
 432 }
 433
 434 // Inform the encoder to not encode the next frame.
 435 void ExternalVideoEncoder::SkipNextFrame(bool skip_next_frame) {
 436   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 437   skip_next_frame_ = skip_next_frame;
 438 }
 439
 440 // Inform the encoder to encode the next frame as a key frame.
 441 void ExternalVideoEncoder::GenerateKeyFrame() {
 442   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 443   key_frame_requested_ = true;
 444 }
 445
 446 // Inform the encoder to only reference frames older or equal to frame_id;
 447 void ExternalVideoEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) {
 448   // Do nothing not supported.
 449 }
 450
 451 int ExternalVideoEncoder::NumberOfSkippedFrames() const {
 452   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 453   return skip_count_;
 454 }
 455
 456 }  //  namespace cast
 457 }  //  namespace media