src/content/renderer/pepper/pepper_media_stream_video_track_host.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/pepper/pepper_media_stream_video_track_host.h"
   6
   7 #include "base/base64.h"
   8 #include "base/logging.h"
   9 #include "base/rand_util.h"
  10 #include "base/strings/utf_string_conversions.h"
  11 #include "content/renderer/media/media_stream_video_track.h"
  12 #include "media/base/bind_to_current_loop.h"
  13 #include "media/base/yuv_convert.h"
  14 #include "ppapi/c/pp_errors.h"
  15 #include "ppapi/c/ppb_media_stream_video_track.h"
  16 #include "ppapi/c/ppb_video_frame.h"
  17 #include "ppapi/host/dispatch_host_message.h"
  18 #include "ppapi/host/host_message_context.h"
  19 #include "ppapi/proxy/ppapi_messages.h"
  20 #include "ppapi/shared_impl/media_stream_buffer.h"
  21
  22 // IS_ALIGNED is also defined in
  23 // third_party/libjingle/overrides/talk/base/basictypes.h
  24 // TODO(ronghuawu): Avoid undef.
  25 #undef IS_ALIGNED
  26 #include "third_party/libyuv/include/libyuv.h"
  27
  28 using media::VideoFrame;
  29 using ppapi::host::HostMessageContext;
  30 using ppapi::MediaStreamVideoTrackShared;
  31
  32 namespace {
  33
  34 const int32_t kDefaultNumberOfBuffers = 4;
  35 const int32_t kMaxNumberOfBuffers = 8;
  36 // Filter mode for scaling frames.
  37 const libyuv::FilterMode kFilterMode = libyuv::kFilterBox;
  38
  39 const char kPepperVideoSourceName[] = "PepperVideoSourceName";
  40
  41 // Default config for output mode.
  42 const int kDefaultOutputFrameRate = 30;
  43
  44 media::VideoPixelFormat ToPixelFormat(PP_VideoFrame_Format format) {
  45   switch (format) {
  46     case PP_VIDEOFRAME_FORMAT_YV12:
  47       return media::PIXEL_FORMAT_YV12;
  48     case PP_VIDEOFRAME_FORMAT_I420:
  49       return media::PIXEL_FORMAT_I420;
  50     default:
  51       DVLOG(1) << "Unsupported pixel format " << format;
  52       return media::PIXEL_FORMAT_UNKNOWN;
  53   }
  54 }
  55
  56 PP_VideoFrame_Format ToPpapiFormat(VideoFrame::Format format) {
  57   switch (format) {
  58     case VideoFrame::YV12:
  59       return PP_VIDEOFRAME_FORMAT_YV12;
  60     case VideoFrame::I420:
  61       return PP_VIDEOFRAME_FORMAT_I420;
  62     default:
  63       DVLOG(1) << "Unsupported pixel format " << format;
  64       return PP_VIDEOFRAME_FORMAT_UNKNOWN;
  65   }
  66 }
  67
  68 VideoFrame::Format FromPpapiFormat(PP_VideoFrame_Format format) {
  69   switch (format) {
  70     case PP_VIDEOFRAME_FORMAT_YV12:
  71       return VideoFrame::YV12;
  72     case PP_VIDEOFRAME_FORMAT_I420:
  73       return VideoFrame::I420;
  74     default:
  75       DVLOG(1) << "Unsupported pixel format " << format;
  76       return VideoFrame::UNKNOWN;
  77   }
  78 }
  79
  80 // Compute size base on the size of frame received from MediaStreamVideoSink
  81 // and size specified by plugin.
  82 gfx::Size GetTargetSize(const gfx::Size& source, const gfx::Size& plugin) {
  83   return gfx::Size(plugin.width() ? plugin.width() : source.width(),
  84                    plugin.height() ? plugin.height() : source.height());
  85 }
  86
  87 // Compute format base on the format of frame received from MediaStreamVideoSink
  88 // and format specified by plugin.
  89 PP_VideoFrame_Format GetTargetFormat(PP_VideoFrame_Format source,
  90                                      PP_VideoFrame_Format plugin) {
  91   return plugin != PP_VIDEOFRAME_FORMAT_UNKNOWN ? plugin : source;
  92 }
  93
  94 void ConvertFromMediaVideoFrame(const scoped_refptr<media::VideoFrame>& src,
  95                                 PP_VideoFrame_Format dst_format,
  96                                 const gfx::Size& dst_size,
  97                                 uint8_t* dst) {
  98   CHECK(src->format() == VideoFrame::YV12 || src->format() == VideoFrame::I420);
  99   if (dst_format == PP_VIDEOFRAME_FORMAT_BGRA) {
 100     if (src->coded_size() == dst_size) {
 101       libyuv::I420ToARGB(src->data(VideoFrame::kYPlane),
 102                          src->stride(VideoFrame::kYPlane),
 103                          src->data(VideoFrame::kUPlane),
 104                          src->stride(VideoFrame::kUPlane),
 105                          src->data(VideoFrame::kVPlane),
 106                          src->stride(VideoFrame::kVPlane),
 107                          dst,
 108                          dst_size.width() * 4,
 109                          dst_size.width(),
 110                          dst_size.height());
 111     } else {
 112       media::ScaleYUVToRGB32(src->data(VideoFrame::kYPlane),
 113                              src->data(VideoFrame::kUPlane),
 114                              src->data(VideoFrame::kVPlane),
 115                              dst,
 116                              src->coded_size().width(),
 117                              src->coded_size().height(),
 118                              dst_size.width(),
 119                              dst_size.height(),
 120                              src->stride(VideoFrame::kYPlane),
 121                              src->stride(VideoFrame::kUPlane),
 122                              dst_size.width() * 4,
 123                              media::YV12,
 124                              media::ROTATE_0,
 125                              media::FILTER_BILINEAR);
 126     }
 127   } else if (dst_format == PP_VIDEOFRAME_FORMAT_YV12 ||
 128              dst_format == PP_VIDEOFRAME_FORMAT_I420) {
 129     static const size_t kPlanesOrder[][3] = {
 130         {VideoFrame::kYPlane, VideoFrame::kVPlane,
 131          VideoFrame::kUPlane},  // YV12
 132         {VideoFrame::kYPlane, VideoFrame::kUPlane,
 133          VideoFrame::kVPlane},  // I420
 134     };
 135     const int plane_order = (dst_format == PP_VIDEOFRAME_FORMAT_YV12) ? 0 : 1;
 136     int dst_width = dst_size.width();
 137     int dst_height = dst_size.height();
 138     libyuv::ScalePlane(src->data(kPlanesOrder[plane_order][0]),
 139                        src->stride(kPlanesOrder[plane_order][0]),
 140                        src->coded_size().width(),
 141                        src->coded_size().height(),
 142                        dst,
 143                        dst_width,
 144                        dst_width,
 145                        dst_height,
 146                        kFilterMode);
 147     dst += dst_width * dst_height;
 148     const int src_halfwidth = (src->coded_size().width() + 1) >> 1;
 149     const int src_halfheight = (src->coded_size().height() + 1) >> 1;
 150     const int dst_halfwidth = (dst_width + 1) >> 1;
 151     const int dst_halfheight = (dst_height + 1) >> 1;
 152     libyuv::ScalePlane(src->data(kPlanesOrder[plane_order][1]),
 153                        src->stride(kPlanesOrder[plane_order][1]),
 154                        src_halfwidth,
 155                        src_halfheight,
 156                        dst,
 157                        dst_halfwidth,
 158                        dst_halfwidth,
 159                        dst_halfheight,
 160                        kFilterMode);
 161     dst += dst_halfwidth * dst_halfheight;
 162     libyuv::ScalePlane(src->data(kPlanesOrder[plane_order][2]),
 163                        src->stride(kPlanesOrder[plane_order][2]),
 164                        src_halfwidth,
 165                        src_halfheight,
 166                        dst,
 167                        dst_halfwidth,
 168                        dst_halfwidth,
 169                        dst_halfheight,
 170                        kFilterMode);
 171   } else {
 172     NOTREACHED();
 173   }
 174 }
 175
 176 }  // namespace
 177
 178 namespace content {
 179
 180 // Internal class used for delivering video frames on the IO-thread to
 181 // the MediaStreamVideoSource implementation.
 182 class PepperMediaStreamVideoTrackHost::FrameDeliverer
 183     : public base::RefCountedThreadSafe<FrameDeliverer> {
 184  public:
 185   FrameDeliverer(
 186       const scoped_refptr<base::MessageLoopProxy>& io_message_loop_proxy,
 187       const VideoCaptureDeliverFrameCB& new_frame_callback);
 188
 189   void DeliverVideoFrame(const scoped_refptr<media::VideoFrame>& frame,
 190                          const media::VideoCaptureFormat& format);
 191
 192  private:
 193   friend class base::RefCountedThreadSafe<FrameDeliverer>;
 194   virtual ~FrameDeliverer();
 195
 196   void DeliverFrameOnIO(const scoped_refptr<media::VideoFrame>& frame,
 197                         const media::VideoCaptureFormat& format);
 198
 199   scoped_refptr<base::MessageLoopProxy> io_message_loop_;
 200   VideoCaptureDeliverFrameCB new_frame_callback_;
 201
 202   DISALLOW_COPY_AND_ASSIGN(FrameDeliverer);
 203 };
 204
 205 PepperMediaStreamVideoTrackHost::FrameDeliverer::FrameDeliverer(
 206     const scoped_refptr<base::MessageLoopProxy>& io_message_loop_proxy,
 207     const VideoCaptureDeliverFrameCB& new_frame_callback)
 208     : io_message_loop_(io_message_loop_proxy),
 209       new_frame_callback_(new_frame_callback) {
 210 }
 211
 212 PepperMediaStreamVideoTrackHost::FrameDeliverer::~FrameDeliverer() {
 213 }
 214
 215 void PepperMediaStreamVideoTrackHost::FrameDeliverer::DeliverVideoFrame(
 216     const scoped_refptr<media::VideoFrame>& frame,
 217     const media::VideoCaptureFormat& format) {
 218   io_message_loop_->PostTask(
 219       FROM_HERE,
 220       base::Bind(&FrameDeliverer::DeliverFrameOnIO,
 221                  this, frame, format));
 222 }
 223
 224 void PepperMediaStreamVideoTrackHost::FrameDeliverer::DeliverFrameOnIO(
 225      const scoped_refptr<media::VideoFrame>& frame,
 226      const media::VideoCaptureFormat& format) {
 227   DCHECK(io_message_loop_->BelongsToCurrentThread());
 228   new_frame_callback_.Run(frame, format);
 229 }
 230
 231 PepperMediaStreamVideoTrackHost::PepperMediaStreamVideoTrackHost(
 232     RendererPpapiHost* host,
 233     PP_Instance instance,
 234     PP_Resource resource,
 235     const blink::WebMediaStreamTrack& track)
 236     : PepperMediaStreamTrackHostBase(host, instance, resource),
 237       track_(track),
 238       connected_(false),
 239       number_of_buffers_(kDefaultNumberOfBuffers),
 240       source_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 241       plugin_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 242       frame_data_size_(0),
 243       type_(kRead),
 244       output_started_(false),
 245       weak_factory_(this) {
 246   DCHECK(!track_.isNull());
 247 }
 248
 249 PepperMediaStreamVideoTrackHost::PepperMediaStreamVideoTrackHost(
 250     RendererPpapiHost* host,
 251     PP_Instance instance,
 252     PP_Resource resource)
 253     : PepperMediaStreamTrackHostBase(host, instance, resource),
 254       connected_(false),
 255       number_of_buffers_(kDefaultNumberOfBuffers),
 256       source_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 257       plugin_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 258       frame_data_size_(0),
 259       type_(kWrite),
 260       output_started_(false),
 261       weak_factory_(this) {
 262   InitBlinkTrack();
 263   DCHECK(!track_.isNull());
 264 }
 265
 266 bool PepperMediaStreamVideoTrackHost::IsMediaStreamVideoTrackHost() {
 267   return true;
 268 }
 269
 270 PepperMediaStreamVideoTrackHost::~PepperMediaStreamVideoTrackHost() {
 271   OnClose();
 272 }
 273
 274 void PepperMediaStreamVideoTrackHost::InitBuffers() {
 275   gfx::Size size = GetTargetSize(source_frame_size_, plugin_frame_size_);
 276   DCHECK(!size.IsEmpty());
 277
 278   PP_VideoFrame_Format format =
 279       GetTargetFormat(source_frame_format_, plugin_frame_format_);
 280   DCHECK_NE(format, PP_VIDEOFRAME_FORMAT_UNKNOWN);
 281
 282   if (format == PP_VIDEOFRAME_FORMAT_BGRA) {
 283     frame_data_size_ = size.width() * size.height() * 4;
 284   } else {
 285     frame_data_size_ =
 286         VideoFrame::AllocationSize(FromPpapiFormat(format), size);
 287   }
 288
 289   DCHECK_GT(frame_data_size_, 0U);
 290   int32_t buffer_size =
 291       sizeof(ppapi::MediaStreamBuffer::Video) + frame_data_size_;
 292   bool result = PepperMediaStreamTrackHostBase::InitBuffers(number_of_buffers_,
 293                                                             buffer_size,
 294                                                             type_);
 295   CHECK(result);
 296
 297   if (type_ == kWrite) {
 298     for (int32_t i = 0; i < buffer_manager()->number_of_buffers(); ++i) {
 299       ppapi::MediaStreamBuffer::Video* buffer =
 300           &(buffer_manager()->GetBufferPointer(i)->video);
 301       buffer->header.size = buffer_manager()->buffer_size();
 302       buffer->header.type = ppapi::MediaStreamBuffer::TYPE_VIDEO;
 303       buffer->format = format;
 304       buffer->size.width = size.width();
 305       buffer->size.height = size.height();
 306       buffer->data_size = frame_data_size_;
 307     }
 308
 309     // Make all the frames avaiable to the plugin.
 310     std::vector<int32_t> indices = buffer_manager()->DequeueBuffers();
 311     SendEnqueueBuffersMessageToPlugin(indices);
 312   }
 313 }
 314
 315 void PepperMediaStreamVideoTrackHost::OnClose() {
 316   if (connected_) {
 317     MediaStreamVideoSink::RemoveFromVideoTrack(this, track_);
 318     weak_factory_.InvalidateWeakPtrs();
 319     connected_ = false;
 320   }
 321 }
 322
 323 int32_t PepperMediaStreamVideoTrackHost::OnHostMsgEnqueueBuffer(
 324     ppapi::host::HostMessageContext* context, int32_t index) {
 325   if (type_ == kRead) {
 326     return PepperMediaStreamTrackHostBase::OnHostMsgEnqueueBuffer(context,
 327                                                                   index);
 328   } else {
 329     return SendFrameToTrack(index);
 330   }
 331 }
 332
 333 int32_t PepperMediaStreamVideoTrackHost::SendFrameToTrack(int32_t index) {
 334   DCHECK_EQ(type_, kWrite);
 335
 336   if (output_started_) {
 337     // Sends the frame to blink video track.
 338     ppapi::MediaStreamBuffer::Video* pp_frame =
 339         &(buffer_manager()->GetBufferPointer(index)->video);
 340
 341     int32 y_stride = plugin_frame_size_.width();
 342     int32 uv_stride = (plugin_frame_size_.width() + 1) / 2;
 343     uint8* y_data = static_cast<uint8*>(pp_frame->data);
 344     // Default to I420
 345     uint8* u_data = y_data + plugin_frame_size_.GetArea();
 346     uint8* v_data = y_data + (plugin_frame_size_.GetArea() * 5 / 4);
 347     if (plugin_frame_format_ == PP_VIDEOFRAME_FORMAT_YV12) {
 348       // Swap u and v for YV12.
 349       uint8* tmp = u_data;
 350       u_data = v_data;
 351       v_data = tmp;
 352     }
 353
 354     int64 ts_ms = static_cast<int64>(pp_frame->timestamp *
 355                                      base::Time::kMillisecondsPerSecond);
 356     scoped_refptr<VideoFrame> frame = media::VideoFrame::WrapExternalYuvData(
 357         FromPpapiFormat(plugin_frame_format_),
 358         plugin_frame_size_,
 359         gfx::Rect(plugin_frame_size_),
 360         plugin_frame_size_,
 361         y_stride,
 362         uv_stride,
 363         uv_stride,
 364         y_data,
 365         u_data,
 366         v_data,
 367         base::TimeDelta::FromMilliseconds(ts_ms),
 368         base::Closure());
 369
 370     frame_deliverer_->DeliverVideoFrame(
 371         frame,
 372         media::VideoCaptureFormat(plugin_frame_size_,
 373                                   kDefaultOutputFrameRate,
 374                                   ToPixelFormat(plugin_frame_format_)));
 375   }
 376
 377   // Makes the frame available again for plugin.
 378   SendEnqueueBufferMessageToPlugin(index);
 379   return PP_OK;
 380 }
 381
 382 void PepperMediaStreamVideoTrackHost::OnVideoFrame(
 383     const scoped_refptr<VideoFrame>& frame,
 384     const media::VideoCaptureFormat& format) {
 385   DCHECK(frame);
 386   // TODO(penghuang): Check |frame->end_of_stream()| and close the track.
 387   PP_VideoFrame_Format ppformat = ToPpapiFormat(frame->format());
 388   if (ppformat == PP_VIDEOFRAME_FORMAT_UNKNOWN)
 389     return;
 390
 391   if (source_frame_size_.IsEmpty()) {
 392     source_frame_size_ = frame->coded_size();
 393     source_frame_format_ = ppformat;
 394     InitBuffers();
 395   }
 396
 397   int32_t index = buffer_manager()->DequeueBuffer();
 398   // Drop frames if the underlying buffer is full.
 399   if (index < 0) {
 400     DVLOG(1) << "A frame is dropped.";
 401     return;
 402   }
 403
 404   CHECK(frame->coded_size() == source_frame_size_) << "Frame size is changed";
 405   CHECK_EQ(ppformat, source_frame_format_) << "Frame format is changed.";
 406
 407   gfx::Size size = GetTargetSize(source_frame_size_, plugin_frame_size_);
 408   ppformat =
 409       GetTargetFormat(source_frame_format_, plugin_frame_format_);
 410   ppapi::MediaStreamBuffer::Video* buffer =
 411       &(buffer_manager()->GetBufferPointer(index)->video);
 412   buffer->header.size = buffer_manager()->buffer_size();
 413   buffer->header.type = ppapi::MediaStreamBuffer::TYPE_VIDEO;
 414   buffer->timestamp = frame->timestamp().InSecondsF();
 415   buffer->format = ppformat;
 416   buffer->size.width = size.width();
 417   buffer->size.height = size.height();
 418   buffer->data_size = frame_data_size_;
 419   ConvertFromMediaVideoFrame(frame, ppformat, size, buffer->data);
 420
 421   SendEnqueueBufferMessageToPlugin(index);
 422 }
 423
 424 void PepperMediaStreamVideoTrackHost::GetCurrentSupportedFormats(
 425     int max_requested_width, int max_requested_height,
 426     const VideoCaptureDeviceFormatsCB& callback) {
 427   if (type_ != kWrite) {
 428     DVLOG(1) << "GetCurrentSupportedFormats is only supported in output mode.";
 429     callback.Run(media::VideoCaptureFormats());
 430     return;
 431   }
 432
 433   media::VideoCaptureFormats formats;
 434   formats.push_back(
 435       media::VideoCaptureFormat(plugin_frame_size_,
 436                                 kDefaultOutputFrameRate,
 437                                 ToPixelFormat(plugin_frame_format_)));
 438   callback.Run(formats);
 439 }
 440
 441 void PepperMediaStreamVideoTrackHost::StartSourceImpl(
 442     const media::VideoCaptureParams& params,
 443     const VideoCaptureDeliverFrameCB& frame_callback) {
 444   output_started_ = true;
 445   frame_deliverer_ = new FrameDeliverer(io_message_loop(), frame_callback);
 446 }
 447
 448 void PepperMediaStreamVideoTrackHost::StopSourceImpl() {
 449   output_started_ = false;
 450   frame_deliverer_ = NULL;
 451 }
 452
 453 void PepperMediaStreamVideoTrackHost::DidConnectPendingHostToResource() {
 454   if (!connected_) {
 455     MediaStreamVideoSink::AddToVideoTrack(
 456         this,
 457         media::BindToCurrentLoop(
 458             base::Bind(
 459                 &PepperMediaStreamVideoTrackHost::OnVideoFrame,
 460                 weak_factory_.GetWeakPtr())),
 461         track_);
 462     connected_ = true;
 463   }
 464 }
 465
 466 int32_t PepperMediaStreamVideoTrackHost::OnResourceMessageReceived(
 467     const IPC::Message& msg,
 468     HostMessageContext* context) {
 469   IPC_BEGIN_MESSAGE_MAP(PepperMediaStreamVideoTrackHost, msg)
 470   PPAPI_DISPATCH_HOST_RESOURCE_CALL(
 471       PpapiHostMsg_MediaStreamVideoTrack_Configure, OnHostMsgConfigure)
 472   IPC_END_MESSAGE_MAP()
 473   return PepperMediaStreamTrackHostBase::OnResourceMessageReceived(msg,
 474                                                                    context);
 475 }
 476
 477 int32_t PepperMediaStreamVideoTrackHost::OnHostMsgConfigure(
 478     HostMessageContext* context,
 479     const MediaStreamVideoTrackShared::Attributes& attributes) {
 480   CHECK(MediaStreamVideoTrackShared::VerifyAttributes(attributes));
 481
 482   bool changed = false;
 483   gfx::Size new_size(attributes.width, attributes.height);
 484   if (GetTargetSize(source_frame_size_, plugin_frame_size_) !=
 485       GetTargetSize(source_frame_size_, new_size)) {
 486     changed = true;
 487   }
 488   plugin_frame_size_ = new_size;
 489
 490   int32_t buffers = attributes.buffers
 491                         ? std::min(kMaxNumberOfBuffers, attributes.buffers)
 492                         : kDefaultNumberOfBuffers;
 493   if (buffers != number_of_buffers_)
 494     changed = true;
 495   number_of_buffers_ = buffers;
 496
 497   if (GetTargetFormat(source_frame_format_, plugin_frame_format_) !=
 498       GetTargetFormat(source_frame_format_, attributes.format)) {
 499     changed = true;
 500   }
 501   plugin_frame_format_ = attributes.format;
 502
 503   // If the first frame has been received, we will re-initialize buffers with
 504   // new settings. Otherwise, we will initialize buffer when we receive
 505   // the first frame, because plugin can only provide part of attributes
 506   // which are not enough to initialize buffers.
 507   if (changed && (type_ == kWrite || !source_frame_size_.IsEmpty()))
 508     InitBuffers();
 509
 510   // TODO(ronghuawu): Ask the owner of DOMMediaStreamTrackToResource why
 511   // source id instead of track id is used there.
 512   const std::string id = track_.source().id().utf8();
 513   context->reply_msg = PpapiPluginMsg_MediaStreamVideoTrack_ConfigureReply(id);
 514   return PP_OK;
 515 }
 516
 517 void PepperMediaStreamVideoTrackHost::InitBlinkTrack() {
 518   std::string source_id;
 519   base::Base64Encode(base::RandBytesAsString(64), &source_id);
 520   blink::WebMediaStreamSource webkit_source;
 521   webkit_source.initialize(base::UTF8ToUTF16(source_id),
 522                            blink::WebMediaStreamSource::TypeVideo,
 523                            base::UTF8ToUTF16(kPepperVideoSourceName));
 524   webkit_source.setExtraData(this);
 525
 526   const bool enabled = true;
 527   blink::WebMediaConstraints constraints;
 528   constraints.initialize();
 529   track_ = MediaStreamVideoTrack::CreateVideoTrack(
 530        this, constraints,
 531        base::Bind(
 532            &PepperMediaStreamVideoTrackHost::OnTrackStarted,
 533            base::Unretained(this)),
 534        enabled);
 535 }
 536
 537 void PepperMediaStreamVideoTrackHost::OnTrackStarted(
 538     MediaStreamSource* source, bool success) {
 539   DVLOG(3) << "OnTrackStarted result: " << success;
 540 }
 541
 542 }  // namespace content