1 // Copyright (c) 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
12 #include "ppapi/c/pp_errors.h"
13 #include "ppapi/c/ppb_console.h"
14 #include "ppapi/c/ppb_opengles2.h"
15 #include "ppapi/cpp/graphics_3d.h"
16 #include "ppapi/cpp/graphics_3d_client.h"
17 #include "ppapi/cpp/input_event.h"
18 #include "ppapi/cpp/instance.h"
19 #include "ppapi/cpp/module.h"
20 #include "ppapi/cpp/rect.h"
21 #include "ppapi/cpp/var.h"
22 #include "ppapi/cpp/video_decoder.h"
24 // VP8 is more likely to work on different versions of Chrome. Undefine this
26 #define USE_VP8_TESTDATA_INSTEAD_OF_H264
27 #include "ppapi/examples/video_decode/testdata.h"
29 #include "ppapi/lib/gl/include/GLES2/gl2.h"
30 #include "ppapi/lib/gl/include/GLES2/gl2ext.h"
31 #include "ppapi/utility/completion_callback_factory.h"
33 // Use assert as a poor-man's CHECK, even in non-debug mode.
34 // Since <assert.h> redefines assert on every inclusion (it doesn't use
35 // include-guards), make sure this is the last file #include'd in this file.
39 // Assert |context_| isn't holding any GL Errors. Done as a macro instead of a
40 // function to preserve line number information in the failure message.
41 #define assertNoGLError() assert(!gles2_if_->GetError(context_->pp_resource()));
46 Shader() : program(0), texcoord_scale_location(0) {}
50 GLint texcoord_scale_location;
56 struct PendingPicture {
57 PendingPicture(Decoder* decoder, const PP_VideoPicture& picture)
58 : decoder(decoder), picture(picture) {}
62 PP_VideoPicture picture;
65 class MyInstance : public pp::Instance, public pp::Graphics3DClient {
67 MyInstance(PP_Instance instance, pp::Module* module);
68 virtual ~MyInstance();
70 // pp::Instance implementation.
71 virtual void DidChangeView(const pp::Rect& position,
72 const pp::Rect& clip_ignored);
73 virtual bool HandleInputEvent(const pp::InputEvent& event);
75 // pp::Graphics3DClient implementation.
76 virtual void Graphics3DContextLost() {
77 // TODO(vrk/fischman): Properly reset after a lost graphics context. In
78 // particular need to delete context_ and re-create textures.
79 // Probably have to recreate the decoder from scratch, because old textures
80 // can still be outstanding in the decoder!
81 assert(false && "Unexpectedly lost graphics context");
84 void PaintPicture(Decoder* decoder, const PP_VideoPicture& picture);
87 // Log an error to the developer console and stderr by creating a temporary
88 // object of this type and streaming to it. Example usage:
89 // LogError(this).s() << "Hello world: " << 42;
92 LogError(MyInstance* instance) : instance_(instance) {}
94 const std::string& msg = stream_.str();
95 instance_->console_if_->Log(
96 instance_->pp_instance(), PP_LOGLEVEL_ERROR, pp::Var(msg).pp_var());
97 std::cerr << msg << std::endl;
99 // Impl note: it would have been nicer to have LogError derive from
100 // std::ostringstream so that it can be streamed to directly, but lookup
101 // rules turn streamed string literals to hex pointers on output.
102 std::ostringstream& s() { return stream_; }
105 MyInstance* instance_;
106 std::ostringstream stream_;
109 void InitializeDecoders();
111 // GL-related functions.
113 void CreateGLObjects();
114 void Create2DProgramOnce();
115 void CreateRectangleARBProgramOnce();
116 Shader CreateProgram(const char* vertex_shader, const char* fragment_shader);
117 void CreateShader(GLuint program, GLenum type, const char* source, int size);
118 void PaintNextPicture();
119 void PaintFinished(int32_t result);
121 pp::Size plugin_size_;
123 // When decode outpaces render, we queue up decoded pictures for later
125 typedef std::queue<PendingPicture> PendingPictureQueue;
126 PendingPictureQueue pending_pictures_;
128 int num_frames_rendered_;
129 PP_TimeTicks first_frame_delivered_ticks_;
130 PP_TimeTicks last_swap_request_ticks_;
131 PP_TimeTicks swap_ticks_;
132 pp::CompletionCallbackFactory<MyInstance> callback_factory_;
135 const PPB_Console* console_if_;
136 const PPB_Core* core_if_;
137 const PPB_OpenGLES2* gles2_if_;
140 pp::Graphics3D* context_;
141 typedef std::vector<Decoder*> DecoderList;
142 DecoderList video_decoders_;
144 // Shader program to draw GL_TEXTURE_2D target.
146 // Shader program to draw GL_TEXTURE_RECTANGLE_ARB target.
147 Shader shader_rectangle_arb_;
152 Decoder(MyInstance* instance, int id, const pp::Graphics3D& graphics_3d);
155 int id() const { return id_; }
156 bool flushing() const { return flushing_; }
157 bool resetting() const { return resetting_; }
160 void RecyclePicture(const PP_VideoPicture& picture);
162 PP_TimeTicks GetAverageLatency() {
163 return num_pictures_ ? total_latency_ / num_pictures_ : 0;
167 void InitializeDone(int32_t result);
169 void DecodeNextFrame();
170 void DecodeDone(int32_t result);
171 void PictureReady(int32_t result, PP_VideoPicture picture);
172 void FlushDone(int32_t result);
173 void ResetDone(int32_t result);
175 MyInstance* instance_;
178 pp::VideoDecoder* decoder_;
179 pp::CompletionCallbackFactory<Decoder> callback_factory_;
181 size_t encoded_data_next_pos_to_decode_;
182 int next_picture_id_;
186 const PPB_Core* core_if_;
187 static const int kMaxDecodeDelay = 128;
188 PP_TimeTicks decode_time_[kMaxDecodeDelay];
189 PP_TimeTicks total_latency_;
193 #if defined USE_VP8_TESTDATA_INSTEAD_OF_H264
195 // VP8 is stored in an IVF container.
196 // Helpful description: http://wiki.multimedia.cx/index.php?title=IVF
198 static void GetNextFrame(size_t* start_pos, size_t* end_pos) {
199 size_t current_pos = *start_pos;
200 if (current_pos == 0)
201 current_pos = 32; // Skip stream header.
202 uint32_t frame_size = kData[current_pos] + (kData[current_pos + 1] << 8) +
203 (kData[current_pos + 2] << 16) +
204 (kData[current_pos + 3] << 24);
205 current_pos += 12; // Skip frame header.
206 *start_pos = current_pos;
207 *end_pos = current_pos + frame_size;
210 #else // !USE_VP8_TESTDATA_INSTEAD_OF_H264
212 // Returns true if the current position is at the start of a NAL unit.
213 static bool LookingAtNAL(const unsigned char* encoded, size_t pos) {
214 // H264 frames start with 0, 0, 0, 1 in our test data.
215 return pos + 3 < kDataLen && encoded[pos] == 0 && encoded[pos + 1] == 0 &&
216 encoded[pos + 2] == 0 && encoded[pos + 3] == 1;
219 static void GetNextFrame(size_t* start_pos, size_t* end_pos) {
220 assert(LookingAtNAL(kData, *start_pos));
221 *end_pos = *start_pos;
223 while (*end_pos < kDataLen && !LookingAtNAL(kData, *end_pos)) {
228 #endif // USE_VP8_TESTDATA_INSTEAD_OF_H264
230 Decoder::Decoder(MyInstance* instance,
232 const pp::Graphics3D& graphics_3d)
233 : instance_(instance),
235 decoder_(new pp::VideoDecoder(instance)),
236 callback_factory_(this),
237 encoded_data_next_pos_to_decode_(0),
243 core_if_ = static_cast<const PPB_Core*>(
244 pp::Module::Get()->GetBrowserInterface(PPB_CORE_INTERFACE));
246 #if defined USE_VP8_TESTDATA_INSTEAD_OF_H264
247 const PP_VideoProfile kBitstreamProfile = PP_VIDEOPROFILE_VP8MAIN;
249 const PP_VideoProfile kBitstreamProfile = PP_VIDEOPROFILE_H264MAIN;
252 assert(!decoder_->is_null());
253 decoder_->Initialize(graphics_3d,
255 PP_TRUE /* allow_software_fallback */,
256 callback_factory_.NewCallback(&Decoder::InitializeDone));
259 Decoder::~Decoder() {
263 void Decoder::InitializeDone(int32_t result) {
265 assert(result == PP_OK);
269 void Decoder::Start() {
272 encoded_data_next_pos_to_decode_ = 0;
274 // Register callback to get the first picture. We call GetPicture again in
275 // PictureReady to continuously receive pictures as they're decoded.
276 decoder_->GetPicture(
277 callback_factory_.NewCallbackWithOutput(&Decoder::PictureReady));
279 // Start the decode loop.
283 void Decoder::Reset() {
287 decoder_->Reset(callback_factory_.NewCallback(&Decoder::ResetDone));
290 void Decoder::RecyclePicture(const PP_VideoPicture& picture) {
292 decoder_->RecyclePicture(picture);
295 void Decoder::DecodeNextFrame() {
297 if (encoded_data_next_pos_to_decode_ <= kDataLen) {
298 // If we've just reached the end of the bitstream, flush and wait.
299 if (!flushing_ && encoded_data_next_pos_to_decode_ == kDataLen) {
301 decoder_->Flush(callback_factory_.NewCallback(&Decoder::FlushDone));
305 // Find the start of the next frame.
306 size_t start_pos = encoded_data_next_pos_to_decode_;
308 GetNextFrame(&start_pos, &end_pos);
309 encoded_data_next_pos_to_decode_ = end_pos;
310 // Decode the frame. On completion, DecodeDone will call DecodeNextFrame
311 // to implement a decode loop.
312 uint32_t size = static_cast<uint32_t>(end_pos - start_pos);
313 decode_time_[next_picture_id_ % kMaxDecodeDelay] = core_if_->GetTimeTicks();
314 decoder_->Decode(next_picture_id_++,
317 callback_factory_.NewCallback(&Decoder::DecodeDone));
321 void Decoder::DecodeDone(int32_t result) {
323 // Break out of the decode loop on abort.
324 if (result == PP_ERROR_ABORTED)
326 assert(result == PP_OK);
327 if (!flushing_ && !resetting_)
331 void Decoder::PictureReady(int32_t result, PP_VideoPicture picture) {
333 // Break out of the get picture loop on abort.
334 if (result == PP_ERROR_ABORTED)
336 assert(result == PP_OK);
339 PP_TimeTicks latency = core_if_->GetTimeTicks() -
340 decode_time_[picture.decode_id % kMaxDecodeDelay];
341 total_latency_ += latency;
343 decoder_->GetPicture(
344 callback_factory_.NewCallbackWithOutput(&Decoder::PictureReady));
345 instance_->PaintPicture(this, picture);
348 void Decoder::FlushDone(int32_t result) {
350 assert(result == PP_OK || result == PP_ERROR_ABORTED);
355 void Decoder::ResetDone(int32_t result) {
357 assert(result == PP_OK);
364 MyInstance::MyInstance(PP_Instance instance, pp::Module* module)
365 : pp::Instance(instance),
366 pp::Graphics3DClient(this),
368 num_frames_rendered_(0),
369 first_frame_delivered_ticks_(-1),
370 last_swap_request_ticks_(-1),
372 callback_factory_(this),
374 console_if_ = static_cast<const PPB_Console*>(
375 pp::Module::Get()->GetBrowserInterface(PPB_CONSOLE_INTERFACE));
376 core_if_ = static_cast<const PPB_Core*>(
377 pp::Module::Get()->GetBrowserInterface(PPB_CORE_INTERFACE));
378 gles2_if_ = static_cast<const PPB_OpenGLES2*>(
379 pp::Module::Get()->GetBrowserInterface(PPB_OPENGLES2_INTERFACE));
381 RequestInputEvents(PP_INPUTEVENT_CLASS_MOUSE);
384 MyInstance::~MyInstance() {
388 PP_Resource graphics_3d = context_->pp_resource();
389 if (shader_2d_.program)
390 gles2_if_->DeleteProgram(graphics_3d, shader_2d_.program);
391 if (shader_rectangle_arb_.program)
392 gles2_if_->DeleteProgram(graphics_3d, shader_rectangle_arb_.program);
394 for (DecoderList::iterator it = video_decoders_.begin();
395 it != video_decoders_.end();
402 void MyInstance::DidChangeView(const pp::Rect& position,
403 const pp::Rect& clip_ignored) {
404 if (position.width() == 0 || position.height() == 0)
406 if (plugin_size_.width()) {
407 assert(position.size() == plugin_size_);
410 plugin_size_ = position.size();
412 // Initialize graphics.
414 InitializeDecoders();
417 bool MyInstance::HandleInputEvent(const pp::InputEvent& event) {
418 switch (event.GetType()) {
419 case PP_INPUTEVENT_TYPE_MOUSEDOWN: {
420 pp::MouseInputEvent mouse_event(event);
421 // Reset all decoders on mouse down.
422 if (mouse_event.GetButton() == PP_INPUTEVENT_MOUSEBUTTON_LEFT) {
424 for (size_t i = 0; i < video_decoders_.size(); i++) {
425 if (!video_decoders_[i]->resetting())
426 video_decoders_[i]->Reset();
429 // Clear pending pictures.
430 while (!pending_pictures_.empty())
431 pending_pictures_.pop();
441 void MyInstance::InitializeDecoders() {
442 assert(video_decoders_.empty());
443 // Create two decoders with ids 0 and 1.
444 video_decoders_.push_back(new Decoder(this, 0, *context_));
445 video_decoders_.push_back(new Decoder(this, 1, *context_));
448 void MyInstance::PaintPicture(Decoder* decoder,
449 const PP_VideoPicture& picture) {
450 if (first_frame_delivered_ticks_ == -1)
451 assert((first_frame_delivered_ticks_ = core_if_->GetTimeTicks()) != -1);
453 pending_pictures_.push(PendingPicture(decoder, picture));
458 void MyInstance::PaintNextPicture() {
459 assert(!is_painting_);
462 const PendingPicture& next = pending_pictures_.front();
463 Decoder* decoder = next.decoder;
464 const PP_VideoPicture& picture = next.picture;
468 int half_width = plugin_size_.width() / 2;
469 int half_height = plugin_size_.height() / 2;
470 if (decoder->id() != 0) {
475 PP_Resource graphics_3d = context_->pp_resource();
476 if (picture.texture_target == GL_TEXTURE_2D) {
477 Create2DProgramOnce();
478 gles2_if_->UseProgram(graphics_3d, shader_2d_.program);
479 gles2_if_->Uniform2f(
480 graphics_3d, shader_2d_.texcoord_scale_location, 1.0, 1.0);
482 assert(picture.texture_target == GL_TEXTURE_RECTANGLE_ARB);
483 CreateRectangleARBProgramOnce();
484 gles2_if_->UseProgram(graphics_3d, shader_rectangle_arb_.program);
485 gles2_if_->Uniform2f(graphics_3d,
486 shader_rectangle_arb_.texcoord_scale_location,
487 picture.texture_size.width,
488 picture.texture_size.height);
491 gles2_if_->Viewport(graphics_3d, x, y, half_width, half_height);
492 gles2_if_->ActiveTexture(graphics_3d, GL_TEXTURE0);
493 gles2_if_->BindTexture(
494 graphics_3d, picture.texture_target, picture.texture_id);
495 gles2_if_->DrawArrays(graphics_3d, GL_TRIANGLE_STRIP, 0, 4);
497 gles2_if_->UseProgram(graphics_3d, 0);
499 last_swap_request_ticks_ = core_if_->GetTimeTicks();
500 context_->SwapBuffers(
501 callback_factory_.NewCallback(&MyInstance::PaintFinished));
504 void MyInstance::PaintFinished(int32_t result) {
505 assert(result == PP_OK);
506 swap_ticks_ += core_if_->GetTimeTicks() - last_swap_request_ticks_;
507 is_painting_ = false;
508 ++num_frames_rendered_;
509 if (num_frames_rendered_ % 50 == 0) {
510 double elapsed = core_if_->GetTimeTicks() - first_frame_delivered_ticks_;
511 double fps = (elapsed > 0) ? num_frames_rendered_ / elapsed : 1000;
512 double ms_per_swap = (swap_ticks_ * 1e3) / num_frames_rendered_;
513 double secs_average_latency = 0;
514 for (DecoderList::iterator it = video_decoders_.begin();
515 it != video_decoders_.end();
517 secs_average_latency += (*it)->GetAverageLatency();
518 secs_average_latency /= video_decoders_.size();
519 double ms_average_latency = 1000 * secs_average_latency;
520 LogError(this).s() << "Rendered frames: " << num_frames_rendered_
522 << ", with average ms/swap of: " << ms_per_swap
523 << ", with average latency (ms) of: "
524 << ms_average_latency;
527 // If the decoders were reset, this will be empty.
528 if (pending_pictures_.empty())
531 const PendingPicture& next = pending_pictures_.front();
532 Decoder* decoder = next.decoder;
533 const PP_VideoPicture& picture = next.picture;
534 decoder->RecyclePicture(picture);
535 pending_pictures_.pop();
537 // Keep painting as long as we have pictures.
538 if (!pending_pictures_.empty())
542 void MyInstance::InitGL() {
543 assert(plugin_size_.width() && plugin_size_.height());
544 is_painting_ = false;
547 int32_t context_attributes[] = {
548 PP_GRAPHICS3DATTRIB_ALPHA_SIZE, 8,
549 PP_GRAPHICS3DATTRIB_BLUE_SIZE, 8,
550 PP_GRAPHICS3DATTRIB_GREEN_SIZE, 8,
551 PP_GRAPHICS3DATTRIB_RED_SIZE, 8,
552 PP_GRAPHICS3DATTRIB_DEPTH_SIZE, 0,
553 PP_GRAPHICS3DATTRIB_STENCIL_SIZE, 0,
554 PP_GRAPHICS3DATTRIB_SAMPLES, 0,
555 PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0,
556 PP_GRAPHICS3DATTRIB_WIDTH, plugin_size_.width(),
557 PP_GRAPHICS3DATTRIB_HEIGHT, plugin_size_.height(),
558 PP_GRAPHICS3DATTRIB_NONE,
560 context_ = new pp::Graphics3D(this, context_attributes);
561 assert(!context_->is_null());
562 assert(BindGraphics(*context_));
565 gles2_if_->ClearColor(context_->pp_resource(), 1, 0, 0, 1);
566 gles2_if_->Clear(context_->pp_resource(), GL_COLOR_BUFFER_BIT);
573 void MyInstance::CreateGLObjects() {
574 // Assign vertex positions and texture coordinates to buffers for use in
576 static const float kVertices[] = {
577 -1, -1, -1, 1, 1, -1, 1, 1, // Position coordinates.
578 0, 1, 0, 0, 1, 1, 1, 0, // Texture coordinates.
582 gles2_if_->GenBuffers(context_->pp_resource(), 1, &buffer);
583 gles2_if_->BindBuffer(context_->pp_resource(), GL_ARRAY_BUFFER, buffer);
585 gles2_if_->BufferData(context_->pp_resource(),
593 static const char kVertexShader[] =
594 "varying vec2 v_texCoord; \n"
595 "attribute vec4 a_position; \n"
596 "attribute vec2 a_texCoord; \n"
597 "uniform vec2 v_scale; \n"
600 " v_texCoord = v_scale * a_texCoord; \n"
601 " gl_Position = a_position; \n"
604 void MyInstance::Create2DProgramOnce() {
605 if (shader_2d_.program)
607 static const char kFragmentShader2D[] =
608 "precision mediump float; \n"
609 "varying vec2 v_texCoord; \n"
610 "uniform sampler2D s_texture; \n"
613 " gl_FragColor = texture2D(s_texture, v_texCoord); \n"
615 shader_2d_ = CreateProgram(kVertexShader, kFragmentShader2D);
619 void MyInstance::CreateRectangleARBProgramOnce() {
620 if (shader_rectangle_arb_.program)
622 static const char kFragmentShaderRectangle[] =
623 "#extension GL_ARB_texture_rectangle : require\n"
624 "precision mediump float; \n"
625 "varying vec2 v_texCoord; \n"
626 "uniform sampler2DRect s_texture; \n"
629 " gl_FragColor = texture2DRect(s_texture, v_texCoord).rgba; \n"
631 shader_rectangle_arb_ =
632 CreateProgram(kVertexShader, kFragmentShaderRectangle);
635 Shader MyInstance::CreateProgram(const char* vertex_shader,
636 const char* fragment_shader) {
639 // Create shader program.
640 shader.program = gles2_if_->CreateProgram(context_->pp_resource());
642 shader.program, GL_VERTEX_SHADER, vertex_shader, strlen(vertex_shader));
643 CreateShader(shader.program,
646 strlen(fragment_shader));
647 gles2_if_->LinkProgram(context_->pp_resource(), shader.program);
648 gles2_if_->UseProgram(context_->pp_resource(), shader.program);
649 gles2_if_->Uniform1i(
650 context_->pp_resource(),
651 gles2_if_->GetUniformLocation(
652 context_->pp_resource(), shader.program, "s_texture"),
656 shader.texcoord_scale_location = gles2_if_->GetUniformLocation(
657 context_->pp_resource(), shader.program, "v_scale");
659 GLint pos_location = gles2_if_->GetAttribLocation(
660 context_->pp_resource(), shader.program, "a_position");
661 GLint tc_location = gles2_if_->GetAttribLocation(
662 context_->pp_resource(), shader.program, "a_texCoord");
665 gles2_if_->EnableVertexAttribArray(context_->pp_resource(), pos_location);
666 gles2_if_->VertexAttribPointer(
667 context_->pp_resource(), pos_location, 2, GL_FLOAT, GL_FALSE, 0, 0);
668 gles2_if_->EnableVertexAttribArray(context_->pp_resource(), tc_location);
669 gles2_if_->VertexAttribPointer(
670 context_->pp_resource(),
676 static_cast<float*>(0) + 8); // Skip position coordinates.
678 gles2_if_->UseProgram(context_->pp_resource(), 0);
683 void MyInstance::CreateShader(GLuint program,
687 GLuint shader = gles2_if_->CreateShader(context_->pp_resource(), type);
688 gles2_if_->ShaderSource(context_->pp_resource(), shader, 1, &source, &size);
689 gles2_if_->CompileShader(context_->pp_resource(), shader);
690 gles2_if_->AttachShader(context_->pp_resource(), program, shader);
691 gles2_if_->DeleteShader(context_->pp_resource(), shader);
694 // This object is the global object representing this plugin library as long
696 class MyModule : public pp::Module {
698 MyModule() : pp::Module() {}
699 virtual ~MyModule() {}
701 virtual pp::Instance* CreateInstance(PP_Instance instance) {
702 return new MyInstance(instance, this);
706 } // anonymous namespace
709 // Factory function for your specialization of the Module object.
710 Module* CreateModule() {
711 return new MyModule();