1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/common/gpu/media/rendering_helper.h"
11 #include "base/bind.h"
12 #include "base/callback_helpers.h"
13 #include "base/command_line.h"
14 #include "base/mac/scoped_nsautorelease_pool.h"
15 #include "base/message_loop/message_loop.h"
16 #include "base/strings/stringize_macros.h"
17 #include "base/synchronization/waitable_event.h"
18 #include "base/time/time.h"
19 #include "ui/gl/gl_context.h"
20 #include "ui/gl/gl_implementation.h"
21 #include "ui/gl/gl_surface.h"
22 #include "ui/gl/gl_surface_egl.h"
23 #include "ui/gl/gl_surface_glx.h"
30 #include "ui/gfx/x/x11_types.h"
33 #if !defined(OS_WIN) && defined(ARCH_CPU_X86_FAMILY)
34 #define GL_VARIANT_GLX 1
36 #define GL_VARIANT_EGL 1
39 // Helper for Shader creation.
40 static void CreateShader(GLuint program,
44 GLuint shader = glCreateShader(type);
45 glShaderSource(shader, 1, &source, &size);
46 glCompileShader(shader);
47 int result = GL_FALSE;
48 glGetShaderiv(shader, GL_COMPILE_STATUS, &result);
51 glGetShaderInfoLog(shader, arraysize(log), NULL, log);
54 glAttachShader(program, shader);
55 glDeleteShader(shader);
56 CHECK_EQ(static_cast<int>(glGetError()), GL_NO_ERROR);
61 RenderingHelperParams::RenderingHelperParams() {}
63 RenderingHelperParams::~RenderingHelperParams() {}
65 VideoFrameTexture::VideoFrameTexture(uint32 texture_target,
67 const base::Closure& no_longer_needed_cb)
68 : texture_target_(texture_target),
69 texture_id_(texture_id),
70 no_longer_needed_cb_(no_longer_needed_cb) {
71 DCHECK(!no_longer_needed_cb_.is_null());
74 VideoFrameTexture::~VideoFrameTexture() {
75 base::ResetAndReturn(&no_longer_needed_cb_).Run();
78 RenderingHelper::RenderedVideo::RenderedVideo()
79 : last_frame_rendered(false), is_flushing(false), frames_to_drop(0) {
82 RenderingHelper::RenderedVideo::~RenderedVideo() {
86 bool RenderingHelper::InitializeOneOff() {
87 base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess();
89 cmd_line->AppendSwitchASCII(switches::kUseGL,
90 gfx::kGLImplementationDesktopName);
92 cmd_line->AppendSwitchASCII(switches::kUseGL, gfx::kGLImplementationEGLName);
94 return gfx::GLSurface::InitializeOneOff();
97 RenderingHelper::RenderingHelper() {
98 window_ = gfx::kNullAcceleratedWidget;
102 RenderingHelper::~RenderingHelper() {
103 CHECK_EQ(videos_.size(), 0U) << "Must call UnInitialize before dtor.";
107 void RenderingHelper::Initialize(const RenderingHelperParams& params,
108 base::WaitableEvent* done) {
109 // Use videos_.size() != 0 as a proxy for the class having already been
110 // Initialize()'d, and UnInitialize() before continuing.
111 if (videos_.size()) {
112 base::WaitableEvent done(false, false);
118 base::Bind(&RenderingHelper::RenderContent, base::Unretained(this)));
120 frame_duration_ = params.rendering_fps > 0
121 ? base::TimeDelta::FromSeconds(1) / params.rendering_fps
124 render_as_thumbnails_ = params.render_as_thumbnails;
125 message_loop_ = base::MessageLoop::current();
129 gfx::Size(GetSystemMetrics(SM_CXSCREEN), GetSystemMetrics(SM_CYSCREEN));
130 window_ = CreateWindowEx(0,
132 L"VideoDecodeAcceleratorTest",
133 WS_OVERLAPPEDWINDOW | WS_VISIBLE,
136 screen_size_.width(),
137 screen_size_.height(),
142 #elif defined(USE_X11)
143 Display* display = gfx::GetXDisplay();
144 Screen* screen = DefaultScreenOfDisplay(display);
145 screen_size_ = gfx::Size(XWidthOfScreen(screen), XHeightOfScreen(screen));
149 XSetWindowAttributes window_attributes;
150 memset(&window_attributes, 0, sizeof(window_attributes));
151 window_attributes.background_pixel =
152 BlackPixel(display, DefaultScreen(display));
153 window_attributes.override_redirect = true;
154 int depth = DefaultDepth(display, DefaultScreen(display));
156 window_ = XCreateWindow(display,
157 DefaultRootWindow(display),
160 screen_size_.width(),
161 screen_size_.height(),
162 0 /* border width */,
164 CopyFromParent /* class */,
165 CopyFromParent /* visual */,
166 (CWBackPixel | CWOverrideRedirect),
168 XStoreName(display, window_, "VideoDecodeAcceleratorTest");
169 XSelectInput(display, window_, ExposureMask);
170 XMapWindow(display, window_);
172 #error unknown platform
174 CHECK(window_ != gfx::kNullAcceleratedWidget);
176 gl_surface_ = gfx::GLSurface::CreateViewGLSurface(window_);
177 gl_context_ = gfx::GLContext::CreateGLContext(
178 NULL, gl_surface_.get(), gfx::PreferIntegratedGpu);
179 gl_context_->MakeCurrent(gl_surface_.get());
181 CHECK_GT(params.window_sizes.size(), 0U);
182 videos_.resize(params.window_sizes.size());
183 LayoutRenderingAreas(params.window_sizes);
185 if (render_as_thumbnails_) {
186 CHECK_EQ(videos_.size(), 1U);
188 GLint max_texture_size;
189 glGetIntegerv(GL_MAX_TEXTURE_SIZE, &max_texture_size);
190 CHECK_GE(max_texture_size, params.thumbnails_page_size.width());
191 CHECK_GE(max_texture_size, params.thumbnails_page_size.height());
193 thumbnails_fbo_size_ = params.thumbnails_page_size;
194 thumbnail_size_ = params.thumbnail_size;
196 glGenFramebuffersEXT(1, &thumbnails_fbo_id_);
197 glGenTextures(1, &thumbnails_texture_id_);
198 glBindTexture(GL_TEXTURE_2D, thumbnails_texture_id_);
199 glTexImage2D(GL_TEXTURE_2D,
202 thumbnails_fbo_size_.width(),
203 thumbnails_fbo_size_.height(),
206 GL_UNSIGNED_SHORT_5_6_5,
208 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
209 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
210 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
211 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
212 glBindTexture(GL_TEXTURE_2D, 0);
214 glBindFramebufferEXT(GL_FRAMEBUFFER, thumbnails_fbo_id_);
215 glFramebufferTexture2DEXT(GL_FRAMEBUFFER,
216 GL_COLOR_ATTACHMENT0,
218 thumbnails_texture_id_,
221 GLenum fb_status = glCheckFramebufferStatusEXT(GL_FRAMEBUFFER);
222 CHECK(fb_status == GL_FRAMEBUFFER_COMPLETE) << fb_status;
223 glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
224 glClear(GL_COLOR_BUFFER_BIT);
225 glBindFramebufferEXT(GL_FRAMEBUFFER, 0);
228 // These vertices and texture coords. map (0,0) in the texture to the
229 // bottom left of the viewport. Since we get the video frames with the
230 // the top left at (0,0) we need to flip the texture y coordinate
231 // in the vertex shader for this to be rendered the right way up.
232 // In the case of thumbnail rendering we use the same vertex shader
233 // to render the FBO the screen, where we do not want this flipping.
234 static const float kVertices[] =
235 { -1.f, 1.f, -1.f, -1.f, 1.f, 1.f, 1.f, -1.f, };
236 static const float kTextureCoords[] = { 0, 1, 0, 0, 1, 1, 1, 0, };
237 static const char kVertexShader[] = STRINGIZE(
238 varying vec2 interp_tc;
239 attribute vec4 in_pos;
240 attribute vec2 in_tc;
241 uniform bool tex_flip;
244 interp_tc = vec2(in_tc.x, 1.0 - in_tc.y);
247 gl_Position = in_pos;
251 static const char kFragmentShader[] =
252 "#extension GL_OES_EGL_image_external : enable\n"
253 "precision mediump float;\n"
254 "varying vec2 interp_tc;\n"
255 "uniform sampler2D tex;\n"
256 "#ifdef GL_OES_EGL_image_external\n"
257 "uniform samplerExternalOES tex_external;\n"
260 " vec4 color = texture2D(tex, interp_tc);\n"
261 "#ifdef GL_OES_EGL_image_external\n"
262 " color += texture2D(tex_external, interp_tc);\n"
264 " gl_FragColor = color;\n"
267 static const char kFragmentShader[] = STRINGIZE(
268 varying vec2 interp_tc;
269 uniform sampler2D tex;
271 gl_FragColor = texture2D(tex, interp_tc);
274 program_ = glCreateProgram();
276 program_, GL_VERTEX_SHADER, kVertexShader, arraysize(kVertexShader));
277 CreateShader(program_,
280 arraysize(kFragmentShader));
281 glLinkProgram(program_);
282 int result = GL_FALSE;
283 glGetProgramiv(program_, GL_LINK_STATUS, &result);
286 glGetShaderInfoLog(program_, arraysize(log), NULL, log);
289 glUseProgram(program_);
290 glDeleteProgram(program_);
292 glUniform1i(glGetUniformLocation(program_, "tex_flip"), 0);
293 glUniform1i(glGetUniformLocation(program_, "tex"), 0);
294 GLint tex_external = glGetUniformLocation(program_, "tex_external");
295 if (tex_external != -1) {
296 glUniform1i(tex_external, 1);
298 int pos_location = glGetAttribLocation(program_, "in_pos");
299 glEnableVertexAttribArray(pos_location);
300 glVertexAttribPointer(pos_location, 2, GL_FLOAT, GL_FALSE, 0, kVertices);
301 int tc_location = glGetAttribLocation(program_, "in_tc");
302 glEnableVertexAttribArray(tc_location);
303 glVertexAttribPointer(tc_location, 2, GL_FLOAT, GL_FALSE, 0, kTextureCoords);
308 void RenderingHelper::UnInitialize(base::WaitableEvent* done) {
309 CHECK_EQ(base::MessageLoop::current(), message_loop_);
311 render_task_.Cancel();
313 if (render_as_thumbnails_) {
314 glDeleteTextures(1, &thumbnails_texture_id_);
315 glDeleteFramebuffersEXT(1, &thumbnails_fbo_id_);
318 gl_context_->ReleaseCurrent(gl_surface_.get());
326 void RenderingHelper::CreateTexture(uint32 texture_target,
328 const gfx::Size& size,
329 base::WaitableEvent* done) {
330 if (base::MessageLoop::current() != message_loop_) {
331 message_loop_->PostTask(FROM_HERE,
332 base::Bind(&RenderingHelper::CreateTexture,
333 base::Unretained(this),
340 glGenTextures(1, texture_id);
341 glBindTexture(texture_target, *texture_id);
342 if (texture_target == GL_TEXTURE_2D) {
343 glTexImage2D(GL_TEXTURE_2D,
353 glTexParameteri(texture_target, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
354 glTexParameteri(texture_target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
355 // OpenGLES2.0.25 section 3.8.2 requires CLAMP_TO_EDGE for NPOT textures.
356 glTexParameteri(texture_target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
357 glTexParameteri(texture_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
358 CHECK_EQ(static_cast<int>(glGetError()), GL_NO_ERROR);
362 // Helper function to set GL viewport.
363 static inline void GLSetViewPort(const gfx::Rect& area) {
364 glViewport(area.x(), area.y(), area.width(), area.height());
365 glScissor(area.x(), area.y(), area.width(), area.height());
368 void RenderingHelper::RenderThumbnail(uint32 texture_target,
370 CHECK_EQ(base::MessageLoop::current(), message_loop_);
371 const int width = thumbnail_size_.width();
372 const int height = thumbnail_size_.height();
373 const int thumbnails_in_row = thumbnails_fbo_size_.width() / width;
374 const int thumbnails_in_column = thumbnails_fbo_size_.height() / height;
375 const int row = (frame_count_ / thumbnails_in_row) % thumbnails_in_column;
376 const int col = frame_count_ % thumbnails_in_row;
378 gfx::Rect area(col * width, row * height, width, height);
380 glUniform1i(glGetUniformLocation(program_, "tex_flip"), 0);
381 glBindFramebufferEXT(GL_FRAMEBUFFER, thumbnails_fbo_id_);
383 RenderTexture(texture_target, texture_id);
384 glBindFramebufferEXT(GL_FRAMEBUFFER, 0);
386 // Need to flush the GL commands before we return the tnumbnail texture to
392 void RenderingHelper::QueueVideoFrame(
394 scoped_refptr<VideoFrameTexture> video_frame) {
395 CHECK_EQ(base::MessageLoop::current(), message_loop_);
396 RenderedVideo* video = &videos_[window_id];
397 DCHECK(!video->is_flushing);
399 // Start the rendering task when getting the first frame.
400 if (scheduled_render_time_.is_null() &&
401 (frame_duration_ != base::TimeDelta())) {
402 scheduled_render_time_ = base::TimeTicks::Now();
403 message_loop_->PostTask(FROM_HERE, render_task_.callback());
406 if (video->frames_to_drop > 0) {
407 --video->frames_to_drop;
411 // Pop the last frame if it has been rendered.
412 if (video->last_frame_rendered) {
413 // When last_frame_rendered is true, we should have only one pending frame.
414 // Since we are going to have a new frame, we can release the pending one.
415 DCHECK(video->pending_frames.size() == 1);
416 video->pending_frames.pop();
417 video->last_frame_rendered = false;
420 video->pending_frames.push(video_frame);
423 void RenderingHelper::RenderTexture(uint32 texture_target, uint32 texture_id) {
424 // The ExternalOES sampler is bound to GL_TEXTURE1 and the Texture2D sampler
425 // is bound to GL_TEXTURE0.
426 if (texture_target == GL_TEXTURE_2D) {
427 glActiveTexture(GL_TEXTURE0 + 0);
428 } else if (texture_target == GL_TEXTURE_EXTERNAL_OES) {
429 glActiveTexture(GL_TEXTURE0 + 1);
431 glBindTexture(texture_target, texture_id);
432 glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
433 glBindTexture(texture_target, 0);
434 CHECK_EQ(static_cast<int>(glGetError()), GL_NO_ERROR);
437 void RenderingHelper::DeleteTexture(uint32 texture_id) {
438 CHECK_EQ(base::MessageLoop::current(), message_loop_);
439 glDeleteTextures(1, &texture_id);
440 CHECK_EQ(static_cast<int>(glGetError()), GL_NO_ERROR);
443 void* RenderingHelper::GetGLContext() {
444 return gl_context_->GetHandle();
447 void* RenderingHelper::GetGLDisplay() {
448 return gl_surface_->GetDisplay();
451 void RenderingHelper::Clear() {
453 message_loop_ = NULL;
457 render_as_thumbnails_ = false;
459 thumbnails_fbo_id_ = 0;
460 thumbnails_texture_id_ = 0;
464 DestroyWindow(window_);
466 // Destroy resources acquired in Initialize, in reverse-acquisition order.
468 CHECK(XUnmapWindow(gfx::GetXDisplay(), window_));
469 CHECK(XDestroyWindow(gfx::GetXDisplay(), window_));
472 window_ = gfx::kNullAcceleratedWidget;
475 void RenderingHelper::GetThumbnailsAsRGB(std::vector<unsigned char>* rgb,
477 base::WaitableEvent* done) {
478 CHECK(render_as_thumbnails_);
480 const size_t num_pixels = thumbnails_fbo_size_.GetArea();
481 std::vector<unsigned char> rgba;
482 rgba.resize(num_pixels * 4);
483 glBindFramebufferEXT(GL_FRAMEBUFFER, thumbnails_fbo_id_);
484 glPixelStorei(GL_PACK_ALIGNMENT, 1);
485 // We can only count on GL_RGBA/GL_UNSIGNED_BYTE support.
488 thumbnails_fbo_size_.width(),
489 thumbnails_fbo_size_.height(),
493 glBindFramebufferEXT(GL_FRAMEBUFFER, 0);
494 rgb->resize(num_pixels * 3);
495 // Drop the alpha channel, but check as we go that it is all 0xff.
497 unsigned char* rgb_ptr = &((*rgb)[0]);
498 unsigned char* rgba_ptr = &rgba[0];
499 for (size_t i = 0; i < num_pixels; ++i) {
500 *rgb_ptr++ = *rgba_ptr++;
501 *rgb_ptr++ = *rgba_ptr++;
502 *rgb_ptr++ = *rgba_ptr++;
503 solid = solid && (*rgba_ptr == 0xff);
506 *alpha_solid = solid;
511 void RenderingHelper::Flush(size_t window_id) {
512 videos_[window_id].is_flushing = true;
515 void RenderingHelper::RenderContent() {
516 CHECK_EQ(base::MessageLoop::current(), message_loop_);
518 scheduled_render_time_ += frame_duration_;
519 base::TimeDelta delay = scheduled_render_time_ - base::TimeTicks::Now();
520 message_loop_->PostDelayedTask(
521 FROM_HERE, render_task_.callback(), std::max(delay, base::TimeDelta()));
523 glUniform1i(glGetUniformLocation(program_, "tex_flip"), 1);
525 // Frames that will be returned to the client (via the no_longer_needed_cb)
526 // after this vector falls out of scope at the end of this method. We need
527 // to keep references to them until after SwapBuffers() call below.
528 std::vector<scoped_refptr<VideoFrameTexture> > frames_to_be_returned;
530 if (render_as_thumbnails_) {
531 // In render_as_thumbnails_ mode, we render the FBO content on the
532 // screen instead of the decoded textures.
533 GLSetViewPort(videos_[0].render_area);
534 RenderTexture(GL_TEXTURE_2D, thumbnails_texture_id_);
536 for (size_t i = 0; i < videos_.size(); ++i) {
537 RenderedVideo* video = &videos_[i];
538 if (video->pending_frames.empty())
540 scoped_refptr<VideoFrameTexture> frame = video->pending_frames.front();
541 GLSetViewPort(video->render_area);
542 RenderTexture(frame->texture_target(), frame->texture_id());
544 if (video->last_frame_rendered)
545 ++video->frames_to_drop;
547 if (video->pending_frames.size() > 1 || video->is_flushing) {
548 frames_to_be_returned.push_back(video->pending_frames.front());
549 video->pending_frames.pop();
550 video->last_frame_rendered = false;
552 video->last_frame_rendered = true;
557 gl_surface_->SwapBuffers();
560 // Helper function for the LayoutRenderingAreas(). The |lengths| are the
561 // heights(widths) of the rows(columns). It scales the elements in
562 // |lengths| proportionally so that the sum of them equal to |total_length|.
563 // It also outputs the coordinates of the rows(columns) to |offsets|.
564 static void ScaleAndCalculateOffsets(std::vector<int>* lengths,
565 std::vector<int>* offsets,
567 int sum = std::accumulate(lengths->begin(), lengths->end(), 0);
568 for (size_t i = 0; i < lengths->size(); ++i) {
569 lengths->at(i) = lengths->at(i) * total_length / sum;
570 offsets->at(i) = (i == 0) ? 0 : offsets->at(i - 1) + lengths->at(i - 1);
574 void RenderingHelper::LayoutRenderingAreas(
575 const std::vector<gfx::Size>& window_sizes) {
576 // Find the number of colums and rows.
577 // The smallest n * n or n * (n + 1) > number of windows.
578 size_t cols = sqrt(videos_.size() - 1) + 1;
579 size_t rows = (videos_.size() + cols - 1) / cols;
581 // Find the widths and heights of the grid.
582 std::vector<int> widths(cols);
583 std::vector<int> heights(rows);
584 std::vector<int> offset_x(cols);
585 std::vector<int> offset_y(rows);
587 for (size_t i = 0; i < window_sizes.size(); ++i) {
588 const gfx::Size& size = window_sizes[i];
589 widths[i % cols] = std::max(widths[i % cols], size.width());
590 heights[i / cols] = std::max(heights[i / cols], size.height());
593 ScaleAndCalculateOffsets(&widths, &offset_x, screen_size_.width());
594 ScaleAndCalculateOffsets(&heights, &offset_y, screen_size_.height());
596 // Put each render_area_ in the center of each cell.
597 for (size_t i = 0; i < window_sizes.size(); ++i) {
598 const gfx::Size& size = window_sizes[i];
600 std::min(static_cast<float>(widths[i % cols]) / size.width(),
601 static_cast<float>(heights[i / cols]) / size.height());
603 // Don't scale up the texture.
604 scale = std::min(1.0f, scale);
606 size_t w = scale * size.width();
607 size_t h = scale * size.height();
608 size_t x = offset_x[i % cols] + (widths[i % cols] - w) / 2;
609 size_t y = offset_y[i / cols] + (heights[i / cols] - h) / 2;
610 videos_[i].render_area = gfx::Rect(x, y, w, h);
613 } // namespace content