1 // Copyright 2014 Samsung Electronics Inc. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/common/gpu/media/tizen/tizen_video_decode_accelerator.h"
7 #include <gst/app/gstappsink.h>
8 #include <gst/app/gstappsrc.h>
10 #include <gst/video/gstvideosink.h>
11 #include <gst/video/video.h>
13 #include "base/bind.h"
14 #include "base/memory/shared_memory.h"
15 #include "base/message_loop/message_loop_proxy.h"
16 #include "base/synchronization/waitable_event.h"
17 #include "base/time/time.h"
18 #if defined(TIZEN_MULTIMEDIA_PIXMAP_SUPPORT)
19 #include "ui/gl/efl_pixmap.h"
22 #if GST_VERSION_MAJOR == 1
23 #include <gst/video/videooverlay.h>
25 #include <gst/interfaces/xoverlay.h>
28 using media::VideoFrame;
32 struct GstElementDeleter {
33 void operator()(GstElement* ptr) const {
35 gst_object_unref(ptr);
39 // Gstreamer elements and names.
40 const char* kDecoderName = "decoder";
41 #if GST_VERSION_MAJOR == 1
42 const char* kDecoderGstElement = "omxh264dec";
44 const char* kDecoderGstElement = "omx_h264dec";
47 // Generating Unique Key from given width and height.
48 int32 ConvertWidthAndHeightToKey(int width, int height) {
49 return ((width << 16) | height);
56 MAX_BITRATE = 2000000, // bps.
57 INPUT_BUFFER_SIZE = MAX_BITRATE / 8, // bytes. 1 sec for H.264 HD video.
58 #if defined(TIZEN_MULTIMEDIA_PIXMAP_SUPPORT)
59 ID_LAST = 0x3FFFFFFF, // wrap round ID after this
63 media::VideoDecodeAccelerator* CreateTizenVideoDecodeAccelerator() {
64 return new TizenVideoDecodeAccelerator();
67 struct TizenVideoDecodeAccelerator::BitstreamBufferRef {
69 base::WeakPtr<media::VideoDecodeAccelerator::Client> client,
70 const scoped_refptr<base::MessageLoopProxy>& client_message_loop_proxy,
71 base::SharedMemory* shm,
75 client_message_loop_proxy_(client_message_loop_proxy),
82 ~BitstreamBufferRef() {
84 client_message_loop_proxy_->PostTask(
87 &media::VideoDecodeAccelerator::Client::NotifyEndOfBitstreamBuffer,
93 static void Destruct(gpointer data) {
95 BitstreamBufferRef* pRef = static_cast<BitstreamBufferRef*>(data);
99 base::WeakPtr<media::VideoDecodeAccelerator::Client> client_;
100 scoped_refptr<base::MessageLoopProxy> client_message_loop_proxy_;
101 scoped_ptr<base::SharedMemory> shm_;
105 GstBuffer* gst_buffer_;
108 struct TizenVideoDecodeAccelerator::Impl {
111 is_destroying_(false),
115 io_message_loop_proxy_(base::MessageLoopProxy::current()),
116 gst_thread_("TizenDecoderThreadGst")
117 #if defined(TIZEN_MULTIMEDIA_PIXMAP_SUPPORT)
118 ,bitstream_buffer_id_(0),
123 damage_handler_(NULL),
124 is_x_window_handle_set_(false)
127 #if defined(TIZEN_MULTIMEDIA_PIXMAP_SUPPORT)
128 xpixmap_buffer_map_.clear();
132 static GstBusSyncReply OnBusMessage(
133 GstBus* bus, GstMessage* msg, gpointer data) {
134 switch (GST_MESSAGE_TYPE(msg)) {
135 case GST_MESSAGE_ERROR: {
137 GError* error = NULL;
138 gst_message_parse_error(msg, &error, &debug);
139 LOG(ERROR) << __FUNCTION__
140 << " GSTError happens from bus at "
141 << GST_OBJECT_NAME(msg->src)
142 << ":" << error->message;
143 LOG(ERROR) << __FUNCTION__
144 << " Debugging Info: "
145 << (debug != NULL ? debug : "none");
150 #if defined(TIZEN_MULTIMEDIA_PIXMAP_SUPPORT)
151 case GST_MESSAGE_ELEMENT: {
152 TizenVideoDecodeAccelerator::Impl* obj_impl =
153 static_cast<TizenVideoDecodeAccelerator::Impl*>(data);
155 if (obj_impl->IsXWindowHandleSet()) {
156 #if GST_VERSION_MAJOR == 1
157 if (gst_is_video_overlay_prepare_window_handle_message(msg)) {
159 if (gst_structure_has_name(msg->structure, "prepare-xid")) {
161 obj_impl->OnXWindowIdPrepared(msg);
162 gst_message_unref(msg);
167 LOG(ERROR) << __FUNCTION__ << "Accelerator is NULL";
178 static void StartFeed(GstAppSrc *source, guint size, gpointer app) {
180 content::TizenVideoDecodeAccelerator::Impl* impl =
181 static_cast<content::TizenVideoDecodeAccelerator::Impl*>(app);
182 impl->can_feed_ = true;
185 static void StopFeed(GstAppSrc *source, gpointer app) {
187 content::TizenVideoDecodeAccelerator::Impl* impl =
188 static_cast<content::TizenVideoDecodeAccelerator::Impl*>(app);
189 impl->can_feed_ = false;
192 #if defined(TIZEN_MULTIMEDIA_PIXMAP_SUPPORT)
193 bool IsXWindowHandleSet() const {return is_x_window_handle_set_;}
194 void OnXWindowIdPrepared(GstMessage* message);
195 void SetXWindowHandle(bool handle_set);
196 void SetPixmap(const int32& gst_width, const int32& gst_height);
197 void DeregisterDamageHandler();
198 static Eina_Bool OnSurfaceChanged(void* ptr_acc, int type, void* event);
199 static void OnSinkCapChanged(
200 GstPad* sink_pad, GParamSpec* gparamspec, void* user_data);
203 volatile bool can_feed_;
204 volatile bool is_destroying_;
205 GstElement* pipeline_;
208 scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy_;
209 scoped_ptr<base::WeakPtrFactory<Client> > io_client_weak_factory_;
210 base::Thread gst_thread_;
211 #if defined(TIZEN_MULTIMEDIA_PIXMAP_SUPPORT)
212 int bitstream_buffer_id_;
213 scoped_refptr<gfx::EflPixmap> pixmap_surface_;
217 Ecore_X_Damage damage_;
218 Ecore_Event_Handler* damage_handler_;
219 bool is_x_window_handle_set_;
220 typedef std::map<int16, scoped_refptr<gfx::EflPixmap> > PixmapSurfaceTizenMap;
221 PixmapSurfaceTizenMap xpixmap_buffer_map_;
225 TizenVideoDecodeAccelerator::TizenVideoDecodeAccelerator()
229 TizenVideoDecodeAccelerator::~TizenVideoDecodeAccelerator() {
232 bool TizenVideoDecodeAccelerator::Initialize(
233 media::VideoCodecProfile profile,
235 GError* error = NULL;
236 GstCaps* video_caps = NULL;
237 GstElement* gst_decoder = NULL;
238 GstElement* gst_parser = NULL;
239 GstBus* gst_bus = NULL;
240 GstPad* video_sink_pad = NULL;
241 scoped_ptr<GstElement, GstElementDeleter> gst_pipeline;
242 static GstAppSrcCallbacks appsrc_callbacks =
243 {&Impl::StartFeed, &Impl::StopFeed, NULL};
244 CHECK(impl_ == NULL);
246 impl_->io_client_weak_factory_.reset(
247 new base::WeakPtrFactory<Client>(client));
250 case media::H264PROFILE_BASELINE:
251 DVLOG(1) << "Initialize(): profile -> H264PROFILE_BASELINE";
253 case media::H264PROFILE_MAIN:
254 DVLOG(1) << "Initialize(): profile -> H264PROFILE_MAIN";
257 LOG(ERROR) << "Initialize(): unsupported profile=" << profile;
261 if (!gst_is_initialized() && !gst_init_check(NULL, NULL, &error)) {
262 LOG(ERROR) << __FUNCTION__ << "cannot initialize gstreamer.";
267 // pipeline initialization.
268 gst_pipeline.reset(gst_pipeline_new("h264_decode"));
270 LOG(ERROR) << __FUNCTION__ << "cannot initialize gst pipeline.";
273 if (!(gst_bus = gst_pipeline_get_bus(GST_PIPELINE(gst_pipeline.get())))) {
276 #if GST_VERSION_MAJOR == 1
277 gst_bus_set_sync_handler(gst_bus, Impl::OnBusMessage, impl_, NULL);
279 gst_bus_set_sync_handler(gst_bus, Impl::OnBusMessage, impl_);
281 gst_object_unref(gst_bus);
283 // appsrc initialization.
284 if (!(impl_->appsrc_ = gst_element_factory_make("appsrc", "src"))) {
285 LOG(ERROR) << __FUNCTION__ << "cannot initialize gst appsrc.";
288 if (!gst_bin_add(GST_BIN(gst_pipeline.get()), impl_->appsrc_)) {
289 gst_object_unref(impl_->appsrc_);
290 impl_->appsrc_ = NULL;
293 gst_app_src_set_max_bytes(GST_APP_SRC(impl_->appsrc_), INPUT_BUFFER_SIZE);
294 gst_app_src_set_callbacks(GST_APP_SRC(impl_->appsrc_), &appsrc_callbacks,
295 static_cast<gpointer>(impl_), NULL);
296 g_object_set(G_OBJECT(impl_->appsrc_),
299 "min-percent", 80, // if buffer below 80%, need-data emits.
300 "stream-type", GST_APP_STREAM_TYPE_STREAM,
302 if (!(video_caps = gst_caps_from_string("video/x-h264,framerate=30/1"))) {
305 gst_app_src_set_caps(GST_APP_SRC(impl_->appsrc_), video_caps);
306 gst_caps_unref(video_caps);
308 #if defined(OS_TIZEN)
309 DVLOG(1) << "######################################";
310 DVLOG(1) << " USING omx_h264dec DECODER " << (unsigned int)this;
311 DVLOG(1) << "######################################";
313 // parser initialization
314 if (!(gst_parser = gst_element_factory_make("h264parse", "h264parse"))) {
315 LOG(ERROR) << " cannot create h264parse.";
318 if(!gst_bin_add(GST_BIN(gst_pipeline.get()), gst_parser)) {
319 LOG(ERROR) << " cannot add h264parse into decoder pipeline.";
320 gst_object_unref(gst_parser);
324 // decoder initialization.
325 if (!(gst_decoder = gst_element_factory_make(kDecoderGstElement, kDecoderName))) {
326 LOG(ERROR) << " cannot create " << kDecoderGstElement << ".";
329 if (!gst_bin_add(GST_BIN(gst_pipeline.get()), gst_decoder)) {
330 LOG(ERROR) << " cannot add " << kDecoderGstElement << " to pipeline.";
331 gst_object_unref(gst_decoder);
335 // sink initialization.
336 if (!(impl_->sink_ = gst_element_factory_make("xvimagesink", "xvimagesink"))) {
337 LOG(ERROR) << __FUNCTION__ << " cannot create xvimagesink.";
340 if (!gst_bin_add(GST_BIN(gst_pipeline.get()), impl_->sink_)) {
341 gst_object_unref(impl_->sink_);
345 g_object_set(impl_->sink_, "rotate", 0, NULL);
346 #if defined(TIZEN_MULTIMEDIA_PIXMAP_SUPPORT)
347 if (!(video_sink_pad = gst_element_get_static_pad(impl_->sink_, "sink"))) {
351 video_sink_pad, "notify::caps", G_CALLBACK(impl_->OnSinkCapChanged), impl_);
352 impl_->SetXWindowHandle(false);
353 gst_object_unref(video_sink_pad);
356 // linking the elements.
357 if (!gst_element_link(impl_->appsrc_, gst_parser)) {
358 LOG(ERROR) << " Source and gst_parser could not be linked";
362 if (!gst_element_link(gst_parser, gst_decoder)) {
363 LOG(ERROR) << " gst_parser and Decoder could not be linked";
366 if (!gst_element_link(gst_decoder, impl_->sink_)) {
367 LOG(ERROR) << __FUNCTION__ << " Decoder and Sink could not be linked";
372 DVLOG(1) << "######################################";
373 DVLOG(1) << " USING ffdec_h264 DECODER";
374 DVLOG(1) << "######################################";
375 GstElement* gst_colorspace = NULL;
377 // decoder initialization
378 if (!(gst_decoder = gst_element_factory_make("ffdec_h264", "H264-decoder"))) {
379 LOG(ERROR) << __FUNCTION__ << " cannot create ffdec_h264.";
382 if (!gst_bin_add(GST_BIN(gst_pipeline.get()), gst_decoder)) {
383 gst_object_unref(gst_decoder);
387 // colorspace initialization
388 if (!(gst_colorspace = gst_element_factory_make("ffmpegcolorspace", "cs"))) {
389 LOG(ERROR) << __FUNCTION__ << " cannot create ffmpegcolorspace.";
392 if (!gst_bin_add(GST_BIN(gst_pipeline.get()), gst_colorspace)) {
393 gst_object_unref(gst_colorspace);
397 if (!(impl_->sink_ = gst_element_factory_make("autovideosink", "sink"))) {
398 LOG(ERROR) << __FUNCTION__ << " cannot create autovideosink.";
401 if (!gst_bin_add(GST_BIN(gst_pipeline.get()), impl_->sink_)) {
402 gst_object_unref(impl_->sink_);
407 if(!gst_element_link_many(impl_->appsrc_, gst_decoder, gst_colorspace,
408 impl_->sink_, NULL)) {
409 LOG(ERROR) << __FUNCTION__ << " Some element could not be linked";
413 if (!impl_->gst_thread_.Start()) {
414 LOG(ERROR) << __FUNCTION__ << " gst_thread_ failed to start";
418 impl_->gst_thread_.message_loop()->PostTask(
420 base::Bind(&TizenVideoDecodeAccelerator::StartDecoder,
421 base::Unretained(this)));
423 GST_DEBUG_BIN_TO_DOT_FILE(
424 GST_BIN(gst_pipeline.get()), GST_DEBUG_GRAPH_SHOW_ALL, "decoder_graph.dot");
426 impl_->pipeline_ = gst_pipeline.release();
430 void TizenVideoDecodeAccelerator::Decode(
431 const media::BitstreamBuffer& bitstream_buffer) {
432 scoped_ptr<BitstreamBufferRef> buffer_ref;
433 scoped_ptr<base::SharedMemory> shm(
434 new base::SharedMemory(bitstream_buffer.handle(), true));
436 if (!shm->Map(bitstream_buffer.size())) {
437 LOG(ERROR) << __FUNCTION__ << " could not map bitstream_buffer";
438 NotifyError(media::VideoDecodeAccelerator::UNREADABLE_INPUT);
442 buffer_ref.reset(new BitstreamBufferRef(
443 impl_->io_client_weak_factory_->GetWeakPtr(),
444 base::MessageLoopProxy::current(),
446 bitstream_buffer.size(),
447 bitstream_buffer.id()));
453 if (impl_->can_feed_ && !impl_->is_destroying_) {
454 impl_->gst_thread_.message_loop()->PostTask(
456 base::Bind(&TizenVideoDecodeAccelerator::OnDecode,
457 base::Unretained(this),
458 base::Passed(&buffer_ref)));
460 DVLOG(2) << __FUNCTION__
461 << " Frame drop on decoder:"
462 << " INPUT Q is FULL";
466 void TizenVideoDecodeAccelerator::AssignPictureBuffers(
467 const std::vector<media::PictureBuffer>& buffers) {
471 void TizenVideoDecodeAccelerator::ReusePictureBuffer(
472 int32 picture_buffer_id) {
476 void TizenVideoDecodeAccelerator::Flush() {
480 void TizenVideoDecodeAccelerator::Reset() {
484 void TizenVideoDecodeAccelerator::Destroy() {
486 #if defined(TIZEN_MULTIMEDIA_PIXMAP_SUPPORT)
487 impl_->SetXWindowHandle(false);
488 impl_->DeregisterDamageHandler();
489 impl_->xpixmap_buffer_map_.clear();
491 if (impl_->gst_thread_.IsRunning()) {
492 impl_->gst_thread_.Stop();
494 gst_app_src_end_of_stream(GST_APP_SRC(impl_->appsrc_));
495 impl_->is_destroying_ = true;
496 if (impl_->pipeline_) {
497 gst_element_set_state(impl_->pipeline_, GST_STATE_NULL);
498 gst_object_unref(GST_OBJECT(impl_->pipeline_));
506 bool TizenVideoDecodeAccelerator::CanDecodeOnIOThread(){
510 void TizenVideoDecodeAccelerator::StartDecoder() {
511 gst_element_set_state(impl_->pipeline_, GST_STATE_PLAYING);
514 #if defined(TIZEN_MULTIMEDIA_PIXMAP_SUPPORT)
515 void TizenVideoDecodeAccelerator::Impl::OnXWindowIdPrepared(
516 GstMessage* message) {
517 #if GST_VERSION_MAJOR == 1
518 const GstStructure* structure = gst_message_get_structure(message);
519 gst_structure_get_int(structure, "video-width", &gst_width_);
520 gst_structure_get_int(structure, "video-height", &gst_height_);
522 gst_structure_get_int(message->structure, "video-width", &gst_width_);
523 gst_structure_get_int(message->structure, "video-height", &gst_height_);
525 SetPixmap(gst_width_, gst_height_);
526 SetXWindowHandle(true);
529 void TizenVideoDecodeAccelerator::Impl::SetXWindowHandle(
531 is_x_window_handle_set_ = handle_set;
534 void TizenVideoDecodeAccelerator::Impl::SetPixmap(
535 const int32& gst_width, const int32& gst_height) {
536 int32 key_wh = ConvertWidthAndHeightToKey(gst_width, gst_height);
537 PixmapSurfaceTizenMap::iterator it = xpixmap_buffer_map_.find(key_wh);
538 if (it != xpixmap_buffer_map_.end()) {
539 pixmap_surface_ = it->second;
540 pixmap_id_ = pixmap_surface_->GetId();
543 gfx::EflPixmap::Create(gfx::EflPixmapBase::UsageType::SURFACE,
544 gfx::Size(gst_width, gst_height));
545 if (pixmap_surface_.get() == NULL) {
546 LOG(ERROR) << __FUNCTION__ << "Failed to create pixmap Surface";
549 pixmap_id_ = pixmap_surface_->GetId();
550 xpixmap_buffer_map_[key_wh] = pixmap_surface_;
552 gst_width_ = gst_width;
553 gst_height_ = gst_height;
554 DeregisterDamageHandler();
556 // Register to get notification from ecore for damage updates.
557 damage_ = ecore_x_damage_new(pixmap_id_,
558 ECORE_X_DAMAGE_REPORT_RAW_RECTANGLES);
559 damage_handler_ = ecore_event_handler_add(ECORE_X_EVENT_DAMAGE_NOTIFY,
562 #if GST_VERSION_MAJOR == 1
563 gst_video_overlay_set_window_handle(GST_VIDEO_OVERLAY(sink_), pixmap_id_);
565 gst_x_overlay_set_window_handle(GST_X_OVERLAY(sink_), pixmap_id_);
569 void TizenVideoDecodeAccelerator::Impl::DeregisterDamageHandler() {
571 ecore_x_damage_free(damage_);
574 if (damage_handler_) {
575 ecore_event_handler_del(damage_handler_);
576 damage_handler_ = NULL;
580 // Callback received when pixmap surface is changed/damaged
581 Eina_Bool TizenVideoDecodeAccelerator::Impl::OnSurfaceChanged(void* ptr_acc,
584 TizenVideoDecodeAccelerator::Impl* self =
585 static_cast<TizenVideoDecodeAccelerator::Impl*>(ptr_acc);
588 media::Picture picture(self->pixmap_id_,
589 self->bitstream_buffer_id_,
590 gfx::Rect(self->gst_width_, self->gst_height_));
592 self->io_message_loop_proxy_->PostTask(
594 base::Bind(&media::VideoDecodeAccelerator::Client::PictureReady,
595 self->io_client_weak_factory_->GetWeakPtr(),
597 self->bitstream_buffer_id_ = (self->bitstream_buffer_id_ + 1) & ID_LAST;
599 LOG(ERROR) << __FUNCTION__ << "Accelerator is NULL";
600 return ECORE_CALLBACK_CANCEL;
602 return ECORE_CALLBACK_PASS_ON;
605 void TizenVideoDecodeAccelerator::Impl::OnSinkCapChanged(
606 GstPad* sink_pad, GParamSpec* gparamspec,void* user_data) {
607 TizenVideoDecodeAccelerator::Impl* self =
608 static_cast<TizenVideoDecodeAccelerator::Impl*>(user_data);
610 LOG(ERROR) << __FUNCTION__ << "Accelerator is NULL";
614 int width = 0, height = 0;
615 #if GST_VERSION_MAJOR == 1
616 GstCaps* caps = gst_pad_get_current_caps(GST_PAD(sink_pad));
619 gst_video_info_init(&info);
621 if (gst_video_info_from_caps(&info, caps)) {
623 height = info.height;
624 if ((self->gst_width_ != width) || (self->gst_height_ != height)) {
625 self->SetPixmap(width, height);
630 if (gst_video_get_size(sink_pad, &width, &height)) {
631 if ((self->gst_width_ != width) || (self->gst_height_ != height)) {
632 self->SetPixmap(width, height);
639 void TizenVideoDecodeAccelerator::OnDecode(
640 scoped_ptr<BitstreamBufferRef> buffer_ref) {
644 #if GST_VERSION_MAJOR == 1
645 buffer_ref->gst_buffer_ =
646 gst_buffer_new_wrapped_full(GST_MEMORY_FLAG_READONLY,
647 static_cast<guint8*>(buffer_ref->shm_->memory()),
651 reinterpret_cast<guint8*>(buffer_ref.get()),
652 BitstreamBufferRef::Destruct);
653 if (!buffer_ref->gst_buffer_ || !GST_IS_BUFFER(buffer_ref->gst_buffer_)) {
654 LOG(ERROR) << " gst_buffer_new_wrapped_full failed to allocate memory.!";
658 if (!(buffer_ref->gst_buffer_ = gst_buffer_new())) {
662 GST_BUFFER_MALLOCDATA(buffer_ref->gst_buffer_) =
663 reinterpret_cast<guint8*>(buffer_ref.get());
664 GST_BUFFER_FREE_FUNC(buffer_ref->gst_buffer_) = BitstreamBufferRef::Destruct;
665 GST_BUFFER_SIZE(buffer_ref->gst_buffer_) = buffer_ref->size_;
666 GST_BUFFER_DATA(buffer_ref->gst_buffer_) =
667 static_cast<guint8*>(buffer_ref->shm_->memory());
670 gst_app_src_push_buffer(GST_APP_SRC(impl_->appsrc_),
671 buffer_ref->gst_buffer_)) {
672 LOG(ERROR) << __FUNCTION__ << " fail to push buffer into decoder pipeline";
676 // lifecycle of buffer_ref will be handled by gstreamer.
677 buffer_ref.release();
680 void TizenVideoDecodeAccelerator::NotifyError(
681 media::VideoDecodeAccelerator::Error error) {
682 if (impl_->io_client_weak_factory_->GetWeakPtr()) {
683 impl_->io_client_weak_factory_->GetWeakPtr()->NotifyError(error);
687 } // namespace content