src/cc/resources/video_resource_updater.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "cc/resources/video_resource_updater.h"
   6
   7 #include "base/bind.h"
   8 #include "cc/output/gl_renderer.h"
   9 #include "cc/resources/resource_provider.h"
  10 #include "gpu/GLES2/gl2extchromium.h"
  11 #include "gpu/command_buffer/client/gles2_interface.h"
  12 #include "media/base/video_frame.h"
  13 #include "media/filters/skcanvas_video_renderer.h"
  14 #include "third_party/khronos/GLES2/gl2.h"
  15 #include "third_party/khronos/GLES2/gl2ext.h"
  16 #include "ui/gfx/size_conversions.h"
  17
  18 namespace cc {
  19
  20 const ResourceFormat kYUVResourceFormat = LUMINANCE_8;
  21 const ResourceFormat kRGBResourceFormat = RGBA_8888;
  22
  23 VideoFrameExternalResources::VideoFrameExternalResources() : type(NONE) {}
  24
  25 VideoFrameExternalResources::~VideoFrameExternalResources() {}
  26
  27 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
  28                                            ResourceProvider* resource_provider)
  29     : context_provider_(context_provider),
  30       resource_provider_(resource_provider) {
  31 }
  32
  33 VideoResourceUpdater::~VideoResourceUpdater() {
  34   while (!all_resources_.empty()) {
  35     resource_provider_->DeleteResource(all_resources_.back());
  36     all_resources_.pop_back();
  37   }
  38 }
  39
  40 void VideoResourceUpdater::DeleteResource(unsigned resource_id) {
  41   resource_provider_->DeleteResource(resource_id);
  42   all_resources_.erase(std::remove(all_resources_.begin(),
  43                                    all_resources_.end(),
  44                                    resource_id));
  45 }
  46
  47 VideoFrameExternalResources VideoResourceUpdater::
  48     CreateExternalResourcesFromVideoFrame(
  49         const scoped_refptr<media::VideoFrame>& video_frame) {
  50   if (!VerifyFrame(video_frame))
  51     return VideoFrameExternalResources();
  52
  53   if (video_frame->format() == media::VideoFrame::NATIVE_TEXTURE)
  54     return CreateForHardwarePlanes(video_frame);
  55   else
  56     return CreateForSoftwarePlanes(video_frame);
  57 }
  58
  59 bool VideoResourceUpdater::VerifyFrame(
  60     const scoped_refptr<media::VideoFrame>& video_frame) {
  61   switch (video_frame->format()) {
  62     // Acceptable inputs.
  63     case media::VideoFrame::YV12:
  64     case media::VideoFrame::I420:
  65     case media::VideoFrame::YV12A:
  66     case media::VideoFrame::YV16:
  67     case media::VideoFrame::YV12J:
  68     case media::VideoFrame::NATIVE_TEXTURE:
  69 #if defined(VIDEO_HOLE)
  70     case media::VideoFrame::HOLE:
  71 #endif  // defined(VIDEO_HOLE)
  72       return true;
  73
  74     // Unacceptable inputs. ¯\(°_o)/¯
  75     case media::VideoFrame::UNKNOWN:
  76     case media::VideoFrame::NV12:
  77       break;
  78   }
  79   return false;
  80 }
  81
  82 // For frames that we receive in software format, determine the dimensions of
  83 // each plane in the frame.
  84 static gfx::Size SoftwarePlaneDimension(
  85     const scoped_refptr<media::VideoFrame>& input_frame,
  86     ResourceFormat output_resource_format,
  87     size_t plane_index) {
  88   if (output_resource_format == kYUVResourceFormat) {
  89     return media::VideoFrame::PlaneSize(
  90         input_frame->format(), plane_index, input_frame->coded_size());
  91   }
  92
  93   DCHECK_EQ(output_resource_format, kRGBResourceFormat);
  94   return input_frame->coded_size();
  95 }
  96
  97 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
  98     const scoped_refptr<media::VideoFrame>& video_frame) {
  99   media::VideoFrame::Format input_frame_format = video_frame->format();
 100
 101 #if defined(VIDEO_HOLE)
 102   if (input_frame_format == media::VideoFrame::HOLE) {
 103     VideoFrameExternalResources external_resources;
 104     external_resources.type = VideoFrameExternalResources::HOLE;
 105     return external_resources;
 106   }
 107 #endif  // defined(VIDEO_HOLE)
 108
 109   // Only YUV software video frames are supported.
 110   DCHECK(input_frame_format == media::VideoFrame::YV12 ||
 111          input_frame_format == media::VideoFrame::I420 ||
 112          input_frame_format == media::VideoFrame::YV12A ||
 113          input_frame_format == media::VideoFrame::YV12J ||
 114          input_frame_format == media::VideoFrame::YV16);
 115   if (input_frame_format != media::VideoFrame::YV12 &&
 116       input_frame_format != media::VideoFrame::I420 &&
 117       input_frame_format != media::VideoFrame::YV12A &&
 118       input_frame_format != media::VideoFrame::YV12J &&
 119       input_frame_format != media::VideoFrame::YV16)
 120     return VideoFrameExternalResources();
 121
 122   bool software_compositor = context_provider_ == NULL;
 123
 124   ResourceFormat output_resource_format = kYUVResourceFormat;
 125   size_t output_plane_count = media::VideoFrame::NumPlanes(input_frame_format);
 126
 127   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
 128   // conversion here. That involves an extra copy of each frame to a bitmap.
 129   // Obviously, this is suboptimal and should be addressed once ubercompositor
 130   // starts shaping up.
 131   if (software_compositor) {
 132     output_resource_format = kRGBResourceFormat;
 133     output_plane_count = 1;
 134   }
 135
 136   int max_resource_size = resource_provider_->max_texture_size();
 137   std::vector<PlaneResource> plane_resources;
 138   bool allocation_success = true;
 139
 140   for (size_t i = 0; i < output_plane_count; ++i) {
 141     gfx::Size output_plane_resource_size =
 142         SoftwarePlaneDimension(video_frame, output_resource_format, i);
 143     if (output_plane_resource_size.IsEmpty() ||
 144         output_plane_resource_size.width() > max_resource_size ||
 145         output_plane_resource_size.height() > max_resource_size) {
 146       allocation_success = false;
 147       break;
 148     }
 149
 150     ResourceProvider::ResourceId resource_id = 0;
 151     gpu::Mailbox mailbox;
 152
 153     // Try recycle a previously-allocated resource.
 154     for (size_t i = 0; i < recycled_resources_.size(); ++i) {
 155       bool resource_matches =
 156           recycled_resources_[i].resource_format == output_resource_format &&
 157           recycled_resources_[i].resource_size == output_plane_resource_size;
 158       bool not_in_use =
 159           !software_compositor || !resource_provider_->InUseByConsumer(
 160                                        recycled_resources_[i].resource_id);
 161       if (resource_matches && not_in_use) {
 162         resource_id = recycled_resources_[i].resource_id;
 163         mailbox = recycled_resources_[i].mailbox;
 164         recycled_resources_.erase(recycled_resources_.begin() + i);
 165         break;
 166       }
 167     }
 168
 169     if (resource_id == 0) {
 170       // TODO(danakj): Abstract out hw/sw resource create/delete from
 171       // ResourceProvider and stop using ResourceProvider in this class.
 172       resource_id =
 173           resource_provider_->CreateResource(output_plane_resource_size,
 174                                              GL_CLAMP_TO_EDGE,
 175                                              ResourceProvider::TextureUsageAny,
 176                                              output_resource_format);
 177
 178       DCHECK(mailbox.IsZero());
 179
 180       if (!software_compositor) {
 181         DCHECK(context_provider_);
 182
 183         gpu::gles2::GLES2Interface* gl = context_provider_->ContextGL();
 184
 185         GLC(gl, gl->GenMailboxCHROMIUM(mailbox.name));
 186         ResourceProvider::ScopedWriteLockGL lock(resource_provider_,
 187                                                  resource_id);
 188         GLC(gl, gl->BindTexture(GL_TEXTURE_2D, lock.texture_id()));
 189         GLC(gl, gl->ProduceTextureCHROMIUM(GL_TEXTURE_2D, mailbox.name));
 190         GLC(gl, gl->BindTexture(GL_TEXTURE_2D, 0));
 191       }
 192
 193       if (resource_id)
 194         all_resources_.push_back(resource_id);
 195     }
 196
 197     if (resource_id == 0) {
 198       allocation_success = false;
 199       break;
 200     }
 201
 202     DCHECK(software_compositor || !mailbox.IsZero());
 203     plane_resources.push_back(PlaneResource(resource_id,
 204                                             output_plane_resource_size,
 205                                             output_resource_format,
 206                                             mailbox));
 207   }
 208
 209   if (!allocation_success) {
 210     for (size_t i = 0; i < plane_resources.size(); ++i)
 211       DeleteResource(plane_resources[i].resource_id);
 212     return VideoFrameExternalResources();
 213   }
 214
 215   VideoFrameExternalResources external_resources;
 216
 217   if (software_compositor) {
 218     DCHECK_EQ(plane_resources.size(), 1u);
 219     DCHECK_EQ(plane_resources[0].resource_format, kRGBResourceFormat);
 220     DCHECK(plane_resources[0].mailbox.IsZero());
 221
 222     if (!video_renderer_)
 223       video_renderer_.reset(new media::SkCanvasVideoRenderer);
 224
 225     {
 226       ResourceProvider::ScopedWriteLockSoftware lock(
 227           resource_provider_, plane_resources[0].resource_id);
 228       video_renderer_->Paint(video_frame.get(),
 229                              lock.sk_canvas(),
 230                              video_frame->visible_rect(),
 231                              0xff);
 232     }
 233
 234     RecycleResourceData recycle_data = {
 235       plane_resources[0].resource_id,
 236       plane_resources[0].resource_size,
 237       plane_resources[0].resource_format,
 238       gpu::Mailbox()
 239     };
 240     external_resources.software_resources.push_back(
 241         plane_resources[0].resource_id);
 242     external_resources.software_release_callback =
 243         base::Bind(&RecycleResource, AsWeakPtr(), recycle_data);
 244     external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
 245
 246     return external_resources;
 247   }
 248
 249   for (size_t i = 0; i < plane_resources.size(); ++i) {
 250     // Update each plane's resource id with its content.
 251     DCHECK_EQ(plane_resources[i].resource_format, kYUVResourceFormat);
 252
 253     const uint8_t* input_plane_pixels = video_frame->data(i);
 254
 255     gfx::Rect image_rect(0,
 256                          0,
 257                          video_frame->stride(i),
 258                          plane_resources[i].resource_size.height());
 259     gfx::Rect source_rect(plane_resources[i].resource_size);
 260     resource_provider_->SetPixels(plane_resources[i].resource_id,
 261                                   input_plane_pixels,
 262                                   image_rect,
 263                                   source_rect,
 264                                   gfx::Vector2d());
 265
 266     RecycleResourceData recycle_data = {
 267       plane_resources[i].resource_id,
 268       plane_resources[i].resource_size,
 269       plane_resources[i].resource_format,
 270       plane_resources[i].mailbox
 271     };
 272
 273     external_resources.mailboxes.push_back(
 274         TextureMailbox(plane_resources[i].mailbox, GL_TEXTURE_2D, 0));
 275     external_resources.release_callbacks.push_back(
 276         base::Bind(&RecycleResource, AsWeakPtr(), recycle_data));
 277   }
 278
 279   external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 280   return external_resources;
 281 }
 282
 283 static void ReturnTexture(const scoped_refptr<media::VideoFrame>& frame,
 284                           uint32 sync_point,
 285                           bool lost_resource) {
 286   frame->AppendReleaseSyncPoint(sync_point);
 287 }
 288
 289 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
 290     const scoped_refptr<media::VideoFrame>& video_frame) {
 291   media::VideoFrame::Format frame_format = video_frame->format();
 292
 293   DCHECK_EQ(frame_format, media::VideoFrame::NATIVE_TEXTURE);
 294   if (frame_format != media::VideoFrame::NATIVE_TEXTURE)
 295       return VideoFrameExternalResources();
 296
 297   if (!context_provider_)
 298     return VideoFrameExternalResources();
 299
 300   const gpu::MailboxHolder* mailbox_holder = video_frame->mailbox_holder();
 301   VideoFrameExternalResources external_resources;
 302   switch (mailbox_holder->texture_target) {
 303     case GL_TEXTURE_2D:
 304       external_resources.type = VideoFrameExternalResources::RGB_RESOURCE;
 305       break;
 306     case GL_TEXTURE_EXTERNAL_OES:
 307       external_resources.type =
 308           VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
 309       break;
 310     case GL_TEXTURE_RECTANGLE_ARB:
 311       external_resources.type = VideoFrameExternalResources::IO_SURFACE;
 312       break;
 313     default:
 314       NOTREACHED();
 315       return VideoFrameExternalResources();
 316   }
 317
 318   external_resources.mailboxes.push_back(
 319       TextureMailbox(mailbox_holder->mailbox,
 320                      mailbox_holder->texture_target,
 321                      mailbox_holder->sync_point));
 322   external_resources.release_callbacks.push_back(
 323       base::Bind(&ReturnTexture, video_frame));
 324   return external_resources;
 325 }
 326
 327 // static
 328 void VideoResourceUpdater::RecycleResource(
 329     base::WeakPtr<VideoResourceUpdater> updater,
 330     RecycleResourceData data,
 331     uint32 sync_point,
 332     bool lost_resource) {
 333   if (!updater.get()) {
 334     // Resource was already deleted.
 335     return;
 336   }
 337
 338   ContextProvider* context_provider = updater->context_provider_;
 339   if (context_provider && sync_point) {
 340     GLC(context_provider->ContextGL(),
 341         context_provider->ContextGL()->WaitSyncPointCHROMIUM(sync_point));
 342   }
 343
 344   if (lost_resource) {
 345     updater->DeleteResource(data.resource_id);
 346     return;
 347   }
 348
 349   // Drop recycled resources that are the wrong format.
 350   while (!updater->recycled_resources_.empty() &&
 351          updater->recycled_resources_.back().resource_format !=
 352          data.resource_format) {
 353     updater->DeleteResource(updater->recycled_resources_.back().resource_id);
 354     updater->recycled_resources_.pop_back();
 355   }
 356
 357   PlaneResource recycled_resource(data.resource_id,
 358                                   data.resource_size,
 359                                   data.resource_format,
 360                                   data.mailbox);
 361   updater->recycled_resources_.push_back(recycled_resource);
 362 }
 363
 364 }  // namespace cc