src/media/base/video_frame.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/base/video_frame.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/callback_helpers.h"
  11 #include "base/logging.h"
  12 #include "base/memory/aligned_memory.h"
  13 #include "base/strings/string_piece.h"
  14 #include "gpu/command_buffer/common/mailbox_holder.h"
  15 #include "media/base/limits.h"
  16 #include "media/base/video_util.h"
  17 #include "third_party/skia/include/core/SkBitmap.h"
  18
  19 namespace media {
  20
  21 static inline size_t RoundUp(size_t value, size_t alignment) {
  22   // Check that |alignment| is a power of 2.
  23   DCHECK((alignment + (alignment - 1)) == (alignment | (alignment - 1)));
  24   return ((value + (alignment - 1)) & ~(alignment - 1));
  25 }
  26
  27 // static
  28 scoped_refptr<VideoFrame> VideoFrame::CreateFrame(
  29     VideoFrame::Format format,
  30     const gfx::Size& coded_size,
  31     const gfx::Rect& visible_rect,
  32     const gfx::Size& natural_size,
  33     base::TimeDelta timestamp) {
  34   // Since we're creating a new YUV frame (and allocating memory for it
  35   // ourselves), we can pad the requested |coded_size| if necessary if the
  36   // request does not line up on sample boundaries.
  37   gfx::Size new_coded_size(coded_size);
  38   switch (format) {
  39     case VideoFrame::YV12:
  40     case VideoFrame::YV12A:
  41     case VideoFrame::I420:
  42     case VideoFrame::YV12J:
  43       new_coded_size.set_height((new_coded_size.height() + 1) / 2 * 2);
  44     // Fallthrough.
  45     case VideoFrame::YV16:
  46       new_coded_size.set_width((new_coded_size.width() + 1) / 2 * 2);
  47       break;
  48     default:
  49       LOG(FATAL) << "Only YUV formats supported: " << format;
  50       return NULL;
  51   }
  52   DCHECK(IsValidConfig(format, new_coded_size, visible_rect, natural_size));
  53   scoped_refptr<VideoFrame> frame(new VideoFrame(
  54       format, new_coded_size, visible_rect, natural_size, timestamp, false));
  55   frame->AllocateYUV();
  56   return frame;
  57 }
  58
  59 // static
  60 std::string VideoFrame::FormatToString(VideoFrame::Format format) {
  61   switch (format) {
  62     case VideoFrame::UNKNOWN:
  63       return "UNKNOWN";
  64     case VideoFrame::YV12:
  65       return "YV12";
  66     case VideoFrame::YV16:
  67       return "YV16";
  68     case VideoFrame::I420:
  69       return "I420";
  70     case VideoFrame::NATIVE_TEXTURE:
  71       return "NATIVE_TEXTURE";
  72 #if defined(VIDEO_HOLE)
  73     case VideoFrame::HOLE:
  74       return "HOLE";
  75 #endif  // defined(VIDEO_HOLE)
  76     case VideoFrame::YV12A:
  77       return "YV12A";
  78     case VideoFrame::YV12J:
  79       return "YV12J";
  80   }
  81   NOTREACHED() << "Invalid videoframe format provided: " << format;
  82   return "";
  83 }
  84
  85 // static
  86 bool VideoFrame::IsValidConfig(VideoFrame::Format format,
  87                                const gfx::Size& coded_size,
  88                                const gfx::Rect& visible_rect,
  89                                const gfx::Size& natural_size) {
  90   // Check maximum limits for all formats.
  91   if (coded_size.GetArea() > limits::kMaxCanvas ||
  92       coded_size.width() > limits::kMaxDimension ||
  93       coded_size.height() > limits::kMaxDimension ||
  94       visible_rect.x() < 0 || visible_rect.y() < 0 ||
  95       visible_rect.right() > coded_size.width() ||
  96       visible_rect.bottom() > coded_size.height() ||
  97       natural_size.GetArea() > limits::kMaxCanvas ||
  98       natural_size.width() > limits::kMaxDimension ||
  99       natural_size.height() > limits::kMaxDimension)
 100     return false;
 101
 102   // Check format-specific width/height requirements.
 103   switch (format) {
 104     case VideoFrame::UNKNOWN:
 105       return (coded_size.IsEmpty() && visible_rect.IsEmpty() &&
 106               natural_size.IsEmpty());
 107     case VideoFrame::YV12:
 108     case VideoFrame::YV12J:
 109     case VideoFrame::I420:
 110     case VideoFrame::YV12A:
 111       // YUV formats have width/height requirements due to chroma subsampling.
 112       if (static_cast<size_t>(coded_size.height()) <
 113           RoundUp(visible_rect.bottom(), 2))
 114         return false;
 115     // Fallthrough.
 116     case VideoFrame::YV16:
 117       if (static_cast<size_t>(coded_size.width()) <
 118           RoundUp(visible_rect.right(), 2))
 119         return false;
 120       break;
 121     case VideoFrame::NATIVE_TEXTURE:
 122 #if defined(VIDEO_HOLE)
 123     case VideoFrame::HOLE:
 124 #endif  // defined(VIDEO_HOLE)
 125       // NATIVE_TEXTURE and HOLE have no software-allocated buffers and are
 126       // allowed to skip the below check and be empty.
 127       return true;
 128   }
 129
 130   // Check that software-allocated buffer formats are not empty.
 131   return (!coded_size.IsEmpty() && !visible_rect.IsEmpty() &&
 132           !natural_size.IsEmpty());
 133 }
 134
 135 // static
 136 scoped_refptr<VideoFrame> VideoFrame::WrapNativeTexture(
 137     scoped_ptr<gpu::MailboxHolder> mailbox_holder,
 138     const ReleaseMailboxCB& mailbox_holder_release_cb,
 139     const gfx::Size& coded_size,
 140     const gfx::Rect& visible_rect,
 141     const gfx::Size& natural_size,
 142     base::TimeDelta timestamp,
 143     const ReadPixelsCB& read_pixels_cb) {
 144   scoped_refptr<VideoFrame> frame(new VideoFrame(NATIVE_TEXTURE,
 145                                                  coded_size,
 146                                                  visible_rect,
 147                                                  natural_size,
 148                                                  timestamp,
 149                                                  false));
 150   frame->mailbox_holder_ = mailbox_holder.Pass();
 151   frame->mailbox_holder_release_cb_ = mailbox_holder_release_cb;
 152   frame->read_pixels_cb_ = read_pixels_cb;
 153
 154   return frame;
 155 }
 156
 157 void VideoFrame::ReadPixelsFromNativeTexture(const SkBitmap& pixels) {
 158   DCHECK_EQ(format_, NATIVE_TEXTURE);
 159   if (!read_pixels_cb_.is_null())
 160     read_pixels_cb_.Run(pixels);
 161 }
 162
 163 // static
 164 scoped_refptr<VideoFrame> VideoFrame::WrapExternalPackedMemory(
 165     Format format,
 166     const gfx::Size& coded_size,
 167     const gfx::Rect& visible_rect,
 168     const gfx::Size& natural_size,
 169     uint8* data,
 170     size_t data_size,
 171     base::SharedMemoryHandle handle,
 172     base::TimeDelta timestamp,
 173     const base::Closure& no_longer_needed_cb) {
 174   if (!IsValidConfig(format, coded_size, visible_rect, natural_size))
 175     return NULL;
 176   if (data_size < AllocationSize(format, coded_size))
 177     return NULL;
 178
 179   switch (format) {
 180     case I420: {
 181       scoped_refptr<VideoFrame> frame(new VideoFrame(
 182           format, coded_size, visible_rect, natural_size, timestamp, false));
 183       frame->shared_memory_handle_ = handle;
 184       frame->strides_[kYPlane] = coded_size.width();
 185       frame->strides_[kUPlane] = coded_size.width() / 2;
 186       frame->strides_[kVPlane] = coded_size.width() / 2;
 187       frame->data_[kYPlane] = data;
 188       frame->data_[kUPlane] = data + coded_size.GetArea();
 189       frame->data_[kVPlane] = data + (coded_size.GetArea() * 5 / 4);
 190       frame->no_longer_needed_cb_ = no_longer_needed_cb;
 191       return frame;
 192     }
 193     default:
 194       NOTIMPLEMENTED();
 195       return NULL;
 196   }
 197 }
 198
 199 // static
 200 scoped_refptr<VideoFrame> VideoFrame::WrapExternalYuvData(
 201     Format format,
 202     const gfx::Size& coded_size,
 203     const gfx::Rect& visible_rect,
 204     const gfx::Size& natural_size,
 205     int32 y_stride,
 206     int32 u_stride,
 207     int32 v_stride,
 208     uint8* y_data,
 209     uint8* u_data,
 210     uint8* v_data,
 211     base::TimeDelta timestamp,
 212     const base::Closure& no_longer_needed_cb) {
 213   if (!IsValidConfig(format, coded_size, visible_rect, natural_size))
 214     return NULL;
 215
 216   scoped_refptr<VideoFrame> frame(new VideoFrame(
 217       format, coded_size, visible_rect, natural_size, timestamp, false));
 218   frame->strides_[kYPlane] = y_stride;
 219   frame->strides_[kUPlane] = u_stride;
 220   frame->strides_[kVPlane] = v_stride;
 221   frame->data_[kYPlane] = y_data;
 222   frame->data_[kUPlane] = u_data;
 223   frame->data_[kVPlane] = v_data;
 224   frame->no_longer_needed_cb_ = no_longer_needed_cb;
 225   return frame;
 226 }
 227
 228 // static
 229 scoped_refptr<VideoFrame> VideoFrame::WrapVideoFrame(
 230       const scoped_refptr<VideoFrame>& frame,
 231       const gfx::Rect& visible_rect,
 232       const gfx::Size& natural_size,
 233       const base::Closure& no_longer_needed_cb) {
 234   DCHECK(frame->visible_rect().Contains(visible_rect));
 235   scoped_refptr<VideoFrame> wrapped_frame(new VideoFrame(
 236       frame->format(), frame->coded_size(), visible_rect, natural_size,
 237       frame->GetTimestamp(), frame->end_of_stream()));
 238
 239   for (size_t i = 0; i < NumPlanes(frame->format()); ++i) {
 240     wrapped_frame->strides_[i] = frame->stride(i);
 241     wrapped_frame->data_[i] = frame->data(i);
 242   }
 243
 244   wrapped_frame->no_longer_needed_cb_ = no_longer_needed_cb;
 245   return wrapped_frame;
 246 }
 247
 248 // static
 249 scoped_refptr<VideoFrame> VideoFrame::CreateEOSFrame() {
 250   return new VideoFrame(VideoFrame::UNKNOWN,
 251                         gfx::Size(),
 252                         gfx::Rect(),
 253                         gfx::Size(),
 254                         kNoTimestamp(),
 255                         true);
 256 }
 257
 258 // static
 259 scoped_refptr<VideoFrame> VideoFrame::CreateColorFrame(
 260     const gfx::Size& size,
 261     uint8 y, uint8 u, uint8 v,
 262     base::TimeDelta timestamp) {
 263   scoped_refptr<VideoFrame> frame = VideoFrame::CreateFrame(
 264       VideoFrame::YV12, size, gfx::Rect(size), size, timestamp);
 265   FillYUV(frame.get(), y, u, v);
 266   return frame;
 267 }
 268
 269 // static
 270 scoped_refptr<VideoFrame> VideoFrame::CreateBlackFrame(const gfx::Size& size) {
 271   const uint8 kBlackY = 0x00;
 272   const uint8 kBlackUV = 0x80;
 273   const base::TimeDelta kZero;
 274   return CreateColorFrame(size, kBlackY, kBlackUV, kBlackUV, kZero);
 275 }
 276
 277 #if defined(VIDEO_HOLE)
 278 // This block and other blocks wrapped around #if defined(VIDEO_HOLE) is not
 279 // maintained by the general compositor team. Please contact the following
 280 // people instead:
 281 //
 282 // wonsik@chromium.org
 283 // ycheo@chromium.org
 284
 285 // static
 286 scoped_refptr<VideoFrame> VideoFrame::CreateHoleFrame(
 287     const gfx::Size& size) {
 288   DCHECK(IsValidConfig(VideoFrame::HOLE, size, gfx::Rect(size), size));
 289   scoped_refptr<VideoFrame> frame(new VideoFrame(
 290       VideoFrame::HOLE, size, gfx::Rect(size), size, base::TimeDelta(), false));
 291   return frame;
 292 }
 293 #endif  // defined(VIDEO_HOLE)
 294
 295 // static
 296 size_t VideoFrame::NumPlanes(Format format) {
 297   switch (format) {
 298     case VideoFrame::NATIVE_TEXTURE:
 299 #if defined(VIDEO_HOLE)
 300     case VideoFrame::HOLE:
 301 #endif  // defined(VIDEO_HOLE)
 302       return 0;
 303     case VideoFrame::YV12:
 304     case VideoFrame::YV16:
 305     case VideoFrame::I420:
 306     case VideoFrame::YV12J:
 307       return 3;
 308     case VideoFrame::YV12A:
 309       return 4;
 310     case VideoFrame::UNKNOWN:
 311       break;
 312   }
 313   NOTREACHED() << "Unsupported video frame format: " << format;
 314   return 0;
 315 }
 316
 317
 318 // static
 319 size_t VideoFrame::AllocationSize(Format format, const gfx::Size& coded_size) {
 320   size_t total = 0;
 321   for (size_t i = 0; i < NumPlanes(format); ++i)
 322     total += PlaneAllocationSize(format, i, coded_size);
 323   return total;
 324 }
 325
 326 // static
 327 gfx::Size VideoFrame::PlaneSize(Format format,
 328                                 size_t plane,
 329                                 const gfx::Size& coded_size) {
 330   const int width = RoundUp(coded_size.width(), 2);
 331   const int height = RoundUp(coded_size.height(), 2);
 332   switch (format) {
 333     case VideoFrame::YV12:
 334     case VideoFrame::YV12J:
 335     case VideoFrame::I420: {
 336       switch (plane) {
 337         case VideoFrame::kYPlane:
 338           return gfx::Size(width, height);
 339         case VideoFrame::kUPlane:
 340         case VideoFrame::kVPlane:
 341           return gfx::Size(width / 2, height / 2);
 342         default:
 343           break;
 344       }
 345     }
 346     case VideoFrame::YV12A: {
 347       switch (plane) {
 348         case VideoFrame::kYPlane:
 349         case VideoFrame::kAPlane:
 350           return gfx::Size(width, height);
 351         case VideoFrame::kUPlane:
 352         case VideoFrame::kVPlane:
 353           return gfx::Size(width / 2, height / 2);
 354         default:
 355           break;
 356       }
 357     }
 358     case VideoFrame::YV16: {
 359       switch (plane) {
 360         case VideoFrame::kYPlane:
 361           return gfx::Size(width, height);
 362         case VideoFrame::kUPlane:
 363         case VideoFrame::kVPlane:
 364           return gfx::Size(width / 2, height);
 365         default:
 366           break;
 367       }
 368     }
 369     case VideoFrame::UNKNOWN:
 370     case VideoFrame::NATIVE_TEXTURE:
 371 #if defined(VIDEO_HOLE)
 372     case VideoFrame::HOLE:
 373 #endif  // defined(VIDEO_HOLE)
 374       break;
 375   }
 376   NOTREACHED() << "Unsupported video frame format/plane: "
 377                << format << "/" << plane;
 378   return gfx::Size();
 379 }
 380
 381 size_t VideoFrame::PlaneAllocationSize(Format format,
 382                                        size_t plane,
 383                                        const gfx::Size& coded_size) {
 384   // VideoFrame formats are (so far) all YUV and 1 byte per sample.
 385   return PlaneSize(format, plane, coded_size).GetArea();
 386 }
 387
 388 // Release data allocated by AllocateYUV().
 389 static void ReleaseData(uint8* data) {
 390   DCHECK(data);
 391   base::AlignedFree(data);
 392 }
 393
 394 void VideoFrame::AllocateYUV() {
 395   DCHECK(format_ == VideoFrame::YV12 || format_ == VideoFrame::YV16 ||
 396          format_ == VideoFrame::YV12A || format_ == VideoFrame::I420 ||
 397          format_ == VideoFrame::YV12J);
 398   // Align Y rows at least at 16 byte boundaries.  The stride for both
 399   // YV12 and YV16 is 1/2 of the stride of Y.  For YV12, every row of bytes for
 400   // U and V applies to two rows of Y (one byte of UV for 4 bytes of Y), so in
 401   // the case of YV12 the strides are identical for the same width surface, but
 402   // the number of bytes allocated for YV12 is 1/2 the amount for U & V as
 403   // YV16. We also round the height of the surface allocated to be an even
 404   // number to avoid any potential of faulting by code that attempts to access
 405   // the Y values of the final row, but assumes that the last row of U & V
 406   // applies to a full two rows of Y. YV12A is the same as YV12, but with an
 407   // additional alpha plane that has the same size and alignment as the Y plane.
 408
 409   size_t y_stride = RoundUp(row_bytes(VideoFrame::kYPlane),
 410                             kFrameSizeAlignment);
 411   size_t uv_stride = RoundUp(row_bytes(VideoFrame::kUPlane),
 412                              kFrameSizeAlignment);
 413   // The *2 here is because some formats (e.g. h264) allow interlaced coding,
 414   // and then the size needs to be a multiple of two macroblocks (vertically).
 415   // See libavcodec/utils.c:avcodec_align_dimensions2().
 416   size_t y_height = RoundUp(coded_size_.height(), kFrameSizeAlignment * 2);
 417   size_t uv_height =
 418       (format_ == VideoFrame::YV12 || format_ == VideoFrame::YV12A ||
 419        format_ == VideoFrame::I420)
 420           ? y_height / 2
 421           : y_height;
 422   size_t y_bytes = y_height * y_stride;
 423   size_t uv_bytes = uv_height * uv_stride;
 424   size_t a_bytes = format_ == VideoFrame::YV12A ? y_bytes : 0;
 425
 426   // The extra line of UV being allocated is because h264 chroma MC
 427   // overreads by one line in some cases, see libavcodec/utils.c:
 428   // avcodec_align_dimensions2() and libavcodec/x86/h264_chromamc.asm:
 429   // put_h264_chroma_mc4_ssse3().
 430   uint8* data = reinterpret_cast<uint8*>(
 431       base::AlignedAlloc(
 432           y_bytes + (uv_bytes * 2 + uv_stride) + a_bytes + kFrameSizePadding,
 433           kFrameAddressAlignment));
 434   no_longer_needed_cb_ = base::Bind(&ReleaseData, data);
 435   COMPILE_ASSERT(0 == VideoFrame::kYPlane, y_plane_data_must_be_index_0);
 436   data_[VideoFrame::kYPlane] = data;
 437   data_[VideoFrame::kUPlane] = data + y_bytes;
 438   data_[VideoFrame::kVPlane] = data + y_bytes + uv_bytes;
 439   strides_[VideoFrame::kYPlane] = y_stride;
 440   strides_[VideoFrame::kUPlane] = uv_stride;
 441   strides_[VideoFrame::kVPlane] = uv_stride;
 442   if (format_ == YV12A) {
 443     data_[VideoFrame::kAPlane] = data + y_bytes + (2 * uv_bytes);
 444     strides_[VideoFrame::kAPlane] = y_stride;
 445   }
 446 }
 447
 448 VideoFrame::VideoFrame(VideoFrame::Format format,
 449                        const gfx::Size& coded_size,
 450                        const gfx::Rect& visible_rect,
 451                        const gfx::Size& natural_size,
 452                        base::TimeDelta timestamp,
 453                        bool end_of_stream)
 454     : format_(format),
 455       coded_size_(coded_size),
 456       visible_rect_(visible_rect),
 457       natural_size_(natural_size),
 458       shared_memory_handle_(base::SharedMemory::NULLHandle()),
 459       timestamp_(timestamp),
 460       end_of_stream_(end_of_stream) {
 461   DCHECK(IsValidConfig(format_, coded_size_, visible_rect_, natural_size_));
 462
 463   memset(&strides_, 0, sizeof(strides_));
 464   memset(&data_, 0, sizeof(data_));
 465 }
 466
 467 VideoFrame::~VideoFrame() {
 468   if (!mailbox_holder_release_cb_.is_null()) {
 469     base::ResetAndReturn(&mailbox_holder_release_cb_)
 470         .Run(mailbox_holder_.Pass());
 471   }
 472   if (!no_longer_needed_cb_.is_null())
 473     base::ResetAndReturn(&no_longer_needed_cb_).Run();
 474 }
 475
 476 bool VideoFrame::IsValidPlane(size_t plane) const {
 477   return (plane < NumPlanes(format_));
 478 }
 479
 480 int VideoFrame::stride(size_t plane) const {
 481   DCHECK(IsValidPlane(plane));
 482   return strides_[plane];
 483 }
 484
 485 int VideoFrame::row_bytes(size_t plane) const {
 486   DCHECK(IsValidPlane(plane));
 487   int width = coded_size_.width();
 488   switch (format_) {
 489     // Planar, 8bpp.
 490     case YV12A:
 491       if (plane == kAPlane)
 492         return width;
 493     // Fallthrough.
 494     case YV12:
 495     case YV16:
 496     case I420:
 497     case YV12J:
 498       if (plane == kYPlane)
 499         return width;
 500       return RoundUp(width, 2) / 2;
 501
 502     default:
 503       break;
 504   }
 505
 506   // Intentionally leave out non-production formats.
 507   NOTREACHED() << "Unsupported video frame format: " << format_;
 508   return 0;
 509 }
 510
 511 int VideoFrame::rows(size_t plane) const {
 512   DCHECK(IsValidPlane(plane));
 513   int height = coded_size_.height();
 514   switch (format_) {
 515     case YV16:
 516       return height;
 517
 518     case YV12A:
 519       if (plane == kAPlane)
 520         return height;
 521     // Fallthrough.
 522     case YV12:
 523     case I420:
 524       if (plane == kYPlane)
 525         return height;
 526       return RoundUp(height, 2) / 2;
 527
 528     default:
 529       break;
 530   }
 531
 532   // Intentionally leave out non-production formats.
 533   NOTREACHED() << "Unsupported video frame format: " << format_;
 534   return 0;
 535 }
 536
 537 uint8* VideoFrame::data(size_t plane) const {
 538   DCHECK(IsValidPlane(plane));
 539   return data_[plane];
 540 }
 541
 542 gpu::MailboxHolder* VideoFrame::mailbox_holder() const {
 543   DCHECK_EQ(format_, NATIVE_TEXTURE);
 544   return mailbox_holder_.get();
 545 }
 546
 547 base::SharedMemoryHandle VideoFrame::shared_memory_handle() const {
 548   return shared_memory_handle_;
 549 }
 550
 551 void VideoFrame::HashFrameForTesting(base::MD5Context* context) {
 552   for (int plane = 0; plane < kMaxPlanes; ++plane) {
 553     if (!IsValidPlane(plane))
 554       break;
 555     for (int row = 0; row < rows(plane); ++row) {
 556       base::MD5Update(context, base::StringPiece(
 557           reinterpret_cast<char*>(data(plane) + stride(plane) * row),
 558           row_bytes(plane)));
 559     }
 560   }
 561 }
 562
 563 }  // namespace media