gst/vaapi/gstvaapidownload.c

   1 /*
   2  *  gstvaapidownload.c - VA-API video downloader
   3  *
   4  *  Copyright (C) 2010-2011 Splitted-Desktop Systems
   5  *  Copyright (C) 2011-2012 Intel Corporation
   6  *
   7  *  This library is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU Lesser General Public License
   9  *  as published by the Free Software Foundation; either version 2.1
  10  *  of the License, or (at your option) any later version.
  11  *
  12  *  This library is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  *  Lesser General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU Lesser General Public
  18  *  License along with this library; if not, write to the Free
  19  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  *  Boston, MA 02110-1301 USA
  21  */
  22
  23 /**
  24  * SECTION:gstvaapidownload
  25  * @short_description: A VA to video flow filter
  26  *
  27  * vaapidownload converts from VA surfaces to raw YUV pixels.
  28  */
  29
  30 #include "config.h"
  31
  32 #include <gst/gst.h>
  33 #include <gst/video/video.h>
  34 #include <gst/video/videocontext.h>
  35 #include <gst/vaapi/gstvaapivideosink.h>
  36 #include <gst/vaapi/gstvaapivideobuffer.h>
  37
  38 #if USE_GLX
  39 #include <gst/vaapi/gstvaapivideobuffer_glx.h>
  40 #define gst_vaapi_video_buffer_new_from_pool(pool) \
  41     gst_vaapi_video_buffer_glx_new_from_pool(pool)
  42 #define gst_vaapi_video_buffer_new_from_buffer(buffer) \
  43     gst_vaapi_video_buffer_glx_new_from_buffer(buffer)
  44 #endif
  45
  46 #include "gstvaapipluginutil.h"
  47 #include "gstvaapidownload.h"
  48
  49 #define GST_PLUGIN_NAME "vaapidownload"
  50 #define GST_PLUGIN_DESC "A VA to video flow filter"
  51
  52 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapidownload);
  53 #define GST_CAT_DEFAULT gst_debug_vaapidownload
  54
  55 /* ElementFactory information */
  56 static const GstElementDetails gst_vaapidownload_details =
  57     GST_ELEMENT_DETAILS(
  58         "VA-API colorspace converter",
  59         "Filter/Converter/Video",
  60         GST_PLUGIN_DESC,
  61         "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
  62
  63 /* Default templates */
  64 static const char gst_vaapidownload_yuv_caps_str[] =
  65     "video/x-raw-yuv, "
  66     "width  = (int) [ 1, MAX ], "
  67     "height = (int) [ 1, MAX ]; ";
  68
  69 static const char gst_vaapidownload_vaapi_caps_str[] =
  70     GST_VAAPI_SURFACE_CAPS;
  71
  72 static GstStaticPadTemplate gst_vaapidownload_sink_factory =
  73     GST_STATIC_PAD_TEMPLATE(
  74         "sink",
  75         GST_PAD_SINK,
  76         GST_PAD_ALWAYS,
  77         GST_STATIC_CAPS(gst_vaapidownload_vaapi_caps_str));
  78
  79 static GstStaticPadTemplate gst_vaapidownload_src_factory =
  80     GST_STATIC_PAD_TEMPLATE(
  81         "src",
  82         GST_PAD_SRC,
  83         GST_PAD_ALWAYS,
  84         GST_STATIC_CAPS(gst_vaapidownload_yuv_caps_str));
  85
  86 typedef struct _TransformSizeCache TransformSizeCache;
  87 struct _TransformSizeCache {
  88     GstCaps            *caps;
  89     guint               size;
  90 };
  91
  92 struct _GstVaapiDownload {
  93     /*< private >*/
  94     GstBaseTransform    parent_instance;
  95
  96     GstVaapiDisplay    *display;
  97     GstCaps            *allowed_caps;
  98     TransformSizeCache  transform_size_cache[2];
  99     GstVaapiVideoPool  *images;
 100     GstVaapiImageFormat image_format;
 101     guint               image_width;
 102     guint               image_height;
 103     unsigned int        images_reset    : 1;
 104 };
 105
 106 struct _GstVaapiDownloadClass {
 107     /*< private >*/
 108     GstBaseTransformClass parent_class;
 109 };
 110
 111 static void
 112 gst_vaapidownload_implements_iface_init(GstImplementsInterfaceClass *iface);
 113
 114 static void
 115 gst_video_context_interface_init(GstVideoContextInterface *iface);
 116
 117 #define GstVideoContextClass GstVideoContextInterface
 118 G_DEFINE_TYPE_WITH_CODE(
 119     GstVaapiDownload,
 120     gst_vaapidownload,
 121     GST_TYPE_BASE_TRANSFORM,
 122     G_IMPLEMENT_INTERFACE(GST_TYPE_IMPLEMENTS_INTERFACE,
 123                           gst_vaapidownload_implements_iface_init);
 124     G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
 125                           gst_video_context_interface_init));
 126
 127 static gboolean
 128 gst_vaapidownload_start(GstBaseTransform *trans);
 129
 130 static gboolean
 131 gst_vaapidownload_stop(GstBaseTransform *trans);
 132
 133 static void
 134 gst_vaapidownload_before_transform(GstBaseTransform *trans, GstBuffer *buffer);
 135
 136 static GstFlowReturn
 137 gst_vaapidownload_transform(
 138     GstBaseTransform *trans,
 139     GstBuffer        *inbuf,
 140     GstBuffer        *outbuf
 141 );
 142
 143 static GstCaps *
 144 gst_vaapidownload_transform_caps(
 145     GstBaseTransform *trans,
 146     GstPadDirection   direction,
 147     GstCaps          *caps
 148 );
 149
 150 static gboolean
 151 gst_vaapidownload_transform_size(
 152     GstBaseTransform *trans,
 153     GstPadDirection   direction,
 154     GstCaps          *caps,
 155     guint             size,
 156     GstCaps          *othercaps,
 157     guint            *othersize
 158 );
 159
 160 static gboolean
 161 gst_vaapidownload_set_caps(
 162     GstBaseTransform *trans,
 163     GstCaps          *incaps,
 164     GstCaps          *outcaps
 165 );
 166
 167 static gboolean
 168 gst_vaapidownload_query(
 169     GstPad   *pad,
 170     GstQuery *query
 171 );
 172
 173 /* GstImplementsInterface interface */
 174
 175 static gboolean
 176 gst_vaapidownload_implements_interface_supported(
 177     GstImplementsInterface *iface,
 178     GType                   type
 179 )
 180 {
 181     return (type == GST_TYPE_VIDEO_CONTEXT);
 182 }
 183
 184 static void
 185 gst_vaapidownload_implements_iface_init(GstImplementsInterfaceClass *iface)
 186 {
 187     iface->supported = gst_vaapidownload_implements_interface_supported;
 188 }
 189
 190 /* GstVideoContext interface */
 191
 192 static void
 193 gst_vaapidownload_set_video_context(GstVideoContext *context, const gchar *type,
 194     const GValue *value)
 195 {
 196   GstVaapiDownload *download = GST_VAAPIDOWNLOAD (context);
 197   gst_vaapi_set_display (type, value, &download->display);
 198 }
 199
 200 static void
 201 gst_video_context_interface_init(GstVideoContextInterface *iface)
 202 {
 203     iface->set_context = gst_vaapidownload_set_video_context;
 204 }
 205
 206 static void
 207 gst_vaapidownload_destroy(GstVaapiDownload *download)
 208 {
 209     guint i;
 210
 211     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
 212         TransformSizeCache * const tsc = &download->transform_size_cache[i];
 213         if (tsc->caps) {
 214             gst_caps_unref(tsc->caps);
 215             tsc->caps = NULL;
 216             tsc->size = 0;
 217         }
 218     }
 219
 220     if (download->allowed_caps) {
 221         gst_caps_unref(download->allowed_caps);
 222         download->allowed_caps = NULL;
 223     }
 224
 225     g_clear_object(&download->images);
 226     g_clear_object(&download->display);
 227 }
 228
 229 static void
 230 gst_vaapidownload_finalize(GObject *object)
 231 {
 232     gst_vaapidownload_destroy(GST_VAAPIDOWNLOAD(object));
 233
 234     G_OBJECT_CLASS(gst_vaapidownload_parent_class)->finalize(object);
 235 }
 236
 237 static void
 238 gst_vaapidownload_class_init(GstVaapiDownloadClass *klass)
 239 {
 240     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
 241     GstBaseTransformClass * const trans_class = GST_BASE_TRANSFORM_CLASS(klass);
 242     GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
 243     GstPadTemplate *pad_template;
 244
 245     GST_DEBUG_CATEGORY_INIT(gst_debug_vaapidownload,
 246                             GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
 247
 248     object_class->finalize        = gst_vaapidownload_finalize;
 249     trans_class->start            = gst_vaapidownload_start;
 250     trans_class->stop             = gst_vaapidownload_stop;
 251     trans_class->before_transform = gst_vaapidownload_before_transform;
 252     trans_class->transform        = gst_vaapidownload_transform;
 253     trans_class->transform_caps   = gst_vaapidownload_transform_caps;
 254     trans_class->transform_size   = gst_vaapidownload_transform_size;
 255     trans_class->set_caps         = gst_vaapidownload_set_caps;
 256
 257     gst_element_class_set_details_simple(
 258         element_class,
 259         gst_vaapidownload_details.longname,
 260         gst_vaapidownload_details.klass,
 261         gst_vaapidownload_details.description,
 262         gst_vaapidownload_details.author
 263     );
 264
 265     /* sink pad */
 266     pad_template = gst_static_pad_template_get(&gst_vaapidownload_sink_factory);
 267     gst_element_class_add_pad_template(element_class, pad_template);
 268     gst_object_unref(pad_template);
 269
 270     /* src pad */
 271     pad_template = gst_static_pad_template_get(&gst_vaapidownload_src_factory);
 272     gst_element_class_add_pad_template(element_class, pad_template);
 273     gst_object_unref(pad_template);
 274 }
 275
 276 static void
 277 gst_vaapidownload_init(GstVaapiDownload *download)
 278 {
 279     GstPad *sinkpad, *srcpad;
 280
 281     download->display           = NULL;
 282     download->allowed_caps      = NULL;
 283     download->images            = NULL;
 284     download->images_reset      = FALSE;
 285     download->image_format      = (GstVaapiImageFormat)0;
 286     download->image_width       = 0;
 287     download->image_height      = 0;
 288
 289     /* Override buffer allocator on sink pad */
 290     sinkpad = gst_element_get_static_pad(GST_ELEMENT(download), "sink");
 291     gst_pad_set_query_function(sinkpad, gst_vaapidownload_query);
 292     gst_object_unref(sinkpad);
 293
 294     /* Override query on src pad */
 295     srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
 296     gst_pad_set_query_function(srcpad, gst_vaapidownload_query);
 297     gst_object_unref(srcpad);
 298 }
 299
 300 static gboolean
 301 gst_vaapidownload_start(GstBaseTransform *trans)
 302 {
 303     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
 304
 305     if (!gst_vaapi_ensure_display(download, &download->display))
 306         return FALSE;
 307     return TRUE;
 308 }
 309
 310 static gboolean
 311 gst_vaapidownload_stop(GstBaseTransform *trans)
 312 {
 313     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
 314
 315     g_clear_object(&download->display);
 316
 317     return TRUE;
 318 }
 319
 320 static GstVaapiImageFormat
 321 get_surface_format(GstVaapiSurface *surface)
 322 {
 323     GstVaapiImage *image;
 324     GstVaapiImageFormat format = GST_VAAPI_IMAGE_NV12;
 325
 326     /* XXX: NV12 is assumed by default */
 327     image = gst_vaapi_surface_derive_image(surface);
 328     if (image) {
 329         format = gst_vaapi_image_get_format(image);
 330         g_object_unref(image);
 331     }
 332     return format;
 333 }
 334
 335 static gboolean
 336 gst_vaapidownload_update_src_caps(GstVaapiDownload *download, GstBuffer *buffer)
 337 {
 338     GstVaapiVideoBuffer *vbuffer;
 339     GstVaapiSurface *surface;
 340     GstVaapiImageFormat format;
 341     GstPad *srcpad;
 342     GstCaps *in_caps, *out_caps;
 343
 344     vbuffer = GST_VAAPI_VIDEO_BUFFER(buffer);
 345     surface = gst_vaapi_video_buffer_get_surface(vbuffer);
 346     if (!surface) {
 347         GST_WARNING("failed to retrieve VA surface from buffer");
 348         return FALSE;
 349     }
 350
 351     format = get_surface_format(surface);
 352     if (format == download->image_format)
 353         return TRUE;
 354
 355     in_caps = GST_BUFFER_CAPS(buffer);
 356     if (!in_caps) {
 357         GST_WARNING("failed to retrieve caps from buffer");
 358         return FALSE;
 359     }
 360
 361     out_caps = gst_vaapi_image_format_get_caps(format);
 362     if (!out_caps) {
 363         GST_WARNING("failed to create caps from format %" GST_FOURCC_FORMAT,
 364                     GST_FOURCC_ARGS(format));
 365         return FALSE;
 366     }
 367
 368     if (!gst_vaapi_append_surface_caps(out_caps, in_caps)) {
 369         gst_caps_unref(out_caps);
 370         return FALSE;
 371     }
 372
 373     /* Try to renegotiate downstream caps */
 374     srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
 375     gst_pad_set_caps(srcpad, out_caps);
 376     gst_object_unref(srcpad);
 377
 378     gst_vaapidownload_set_caps(GST_BASE_TRANSFORM(download), in_caps, out_caps);
 379     gst_caps_replace(&download->allowed_caps, out_caps);
 380     gst_caps_unref(out_caps);
 381     return TRUE;
 382 }
 383
 384 static void
 385 gst_vaapidownload_before_transform(GstBaseTransform *trans, GstBuffer *buffer)
 386 {
 387     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
 388
 389     gst_vaapidownload_update_src_caps(download, buffer);
 390 }
 391
 392 static GstFlowReturn
 393 gst_vaapidownload_transform(
 394     GstBaseTransform *trans,
 395     GstBuffer        *inbuf,
 396     GstBuffer        *outbuf
 397 )
 398 {
 399     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
 400     GstVaapiVideoBuffer *vbuffer;
 401     GstVaapiSurface *surface;
 402     GstVaapiImage *image = NULL;
 403     gboolean success;
 404
 405     vbuffer = GST_VAAPI_VIDEO_BUFFER(inbuf);
 406     surface = gst_vaapi_video_buffer_get_surface(vbuffer);
 407     if (!surface)
 408         return GST_FLOW_UNEXPECTED;
 409
 410     image = gst_vaapi_video_pool_get_object(download->images);
 411     if (!image)
 412         return GST_FLOW_UNEXPECTED;
 413     if (!gst_vaapi_surface_get_image(surface, image))
 414         goto error_get_image;
 415
 416     success = gst_vaapi_image_get_buffer(image, outbuf, NULL);
 417     gst_vaapi_video_pool_put_object(download->images, image);
 418     if (!success)
 419         goto error_get_buffer;
 420     return GST_FLOW_OK;
 421
 422 error_get_image:
 423     {
 424         GST_WARNING("failed to download %" GST_FOURCC_FORMAT " image "
 425                     "from surface 0x%08x",
 426                     GST_FOURCC_ARGS(gst_vaapi_image_get_format(image)),
 427                     gst_vaapi_surface_get_id(surface));
 428         gst_vaapi_video_pool_put_object(download->images, image);
 429         return GST_FLOW_UNEXPECTED;
 430     }
 431
 432 error_get_buffer:
 433     {
 434         GST_WARNING("failed to transfer image to output video buffer");
 435         return GST_FLOW_UNEXPECTED;
 436     }
 437 }
 438
 439 static GstCaps *
 440 gst_vaapidownload_transform_caps(
 441     GstBaseTransform *trans,
 442     GstPadDirection   direction,
 443     GstCaps          *caps
 444 )
 445 {
 446     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
 447     GstPad *srcpad;
 448     GstCaps *allowed_caps, *inter_caps, *out_caps = NULL;
 449     GstStructure *structure;
 450
 451     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
 452
 453     structure = gst_caps_get_structure(caps, 0);
 454
 455     if (direction == GST_PAD_SINK) {
 456         if (!gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
 457             return NULL;
 458         if (!gst_vaapi_ensure_display(download, &download->display))
 459             return NULL;
 460         out_caps = gst_caps_from_string(gst_vaapidownload_yuv_caps_str);
 461
 462         /* Build up allowed caps */
 463         /* XXX: we don't know the decoded surface format yet so we
 464            expose whatever VA images we support */
 465         if (download->allowed_caps)
 466             allowed_caps = gst_caps_ref(download->allowed_caps);
 467         else {
 468             allowed_caps = gst_vaapi_display_get_image_caps(download->display);
 469             if (!allowed_caps)
 470                 return NULL;
 471         }
 472         inter_caps = gst_caps_intersect(out_caps, allowed_caps);
 473         gst_caps_unref(allowed_caps);
 474         gst_caps_unref(out_caps);
 475         out_caps = inter_caps;
 476
 477         /* Intersect with allowed caps from the peer, if any */
 478         srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
 479         allowed_caps = gst_pad_peer_get_caps(srcpad);
 480         if (allowed_caps) {
 481             inter_caps = gst_caps_intersect(out_caps, allowed_caps);
 482             gst_caps_unref(allowed_caps);
 483             gst_caps_unref(out_caps);
 484             out_caps = inter_caps;
 485         }
 486     }
 487     else {
 488         if (!gst_structure_has_name(structure, "video/x-raw-yuv"))
 489             return NULL;
 490         out_caps = gst_caps_from_string(gst_vaapidownload_vaapi_caps_str);
 491
 492         structure = gst_caps_get_structure(out_caps, 0);
 493         gst_structure_set(
 494             structure,
 495             "type", G_TYPE_STRING, "vaapi",
 496             "opengl", G_TYPE_BOOLEAN, USE_GLX,
 497             NULL
 498         );
 499     }
 500
 501     if (!gst_vaapi_append_surface_caps(out_caps, caps)) {
 502         gst_caps_unref(out_caps);
 503         return NULL;
 504     }
 505     return out_caps;
 506 }
 507
 508 static gboolean
 509 gst_vaapidownload_ensure_image_pool(GstVaapiDownload *download, GstCaps *caps)
 510 {
 511     GstStructure * const structure = gst_caps_get_structure(caps, 0);
 512     GstVaapiImageFormat format;
 513     gint width, height;
 514
 515     format = gst_vaapi_image_format_from_caps(caps);
 516     gst_structure_get_int(structure, "width",  &width);
 517     gst_structure_get_int(structure, "height", &height);
 518
 519     if (format != download->image_format ||
 520         width  != download->image_width  ||
 521         height != download->image_height) {
 522         download->image_format = format;
 523         download->image_width  = width;
 524         download->image_height = height;
 525         g_clear_object(&download->images);
 526         download->images = gst_vaapi_image_pool_new(download->display, caps);
 527         if (!download->images)
 528             return FALSE;
 529         download->images_reset = TRUE;
 530     }
 531     return TRUE;
 532 }
 533
 534 static inline gboolean
 535 gst_vaapidownload_negotiate_buffers(
 536     GstVaapiDownload  *download,
 537     GstCaps          *incaps,
 538     GstCaps          *outcaps
 539 )
 540 {
 541     if (!gst_vaapidownload_ensure_image_pool(download, outcaps))
 542         return FALSE;
 543     return TRUE;
 544 }
 545
 546 static gboolean
 547 gst_vaapidownload_set_caps(
 548     GstBaseTransform *trans,
 549     GstCaps          *incaps,
 550     GstCaps          *outcaps
 551 )
 552 {
 553     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
 554
 555     if (!gst_vaapidownload_negotiate_buffers(download, incaps, outcaps))
 556         return FALSE;
 557     return TRUE;
 558 }
 559
 560 static gboolean
 561 gst_vaapidownload_transform_size(
 562     GstBaseTransform *trans,
 563     GstPadDirection   direction,
 564     GstCaps          *caps,
 565     guint             size,
 566     GstCaps          *othercaps,
 567     guint            *othersize
 568 )
 569 {
 570     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
 571     GstStructure * const structure = gst_caps_get_structure(othercaps, 0);
 572     GstVideoFormat format;
 573     gint width, height;
 574     guint i;
 575
 576     /* Lookup in cache */
 577     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
 578         TransformSizeCache * const tsc = &download->transform_size_cache[i];
 579         if (tsc->caps && tsc->caps == othercaps) {
 580             *othersize = tsc->size;
 581             return TRUE;
 582         }
 583     }
 584
 585     /* Compute requested buffer size */
 586     if (gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
 587         *othersize = 0;
 588     else {
 589         if (!gst_video_format_parse_caps(othercaps, &format, &width, &height))
 590             return FALSE;
 591         *othersize = gst_video_format_get_size(format, width, height);
 592     }
 593
 594     /* Update cache */
 595     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
 596         TransformSizeCache * const tsc = &download->transform_size_cache[i];
 597         if (!tsc->caps) {
 598             gst_caps_replace(&tsc->caps, othercaps);
 599             tsc->size = *othersize;
 600         }
 601     }
 602     return TRUE;
 603 }
 604
 605 static gboolean
 606 gst_vaapidownload_query(GstPad *pad, GstQuery *query)
 607 {
 608     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(gst_pad_get_parent_element(pad));
 609     gboolean res;
 610
 611     GST_DEBUG("sharing display %p", download->display);
 612
 613     if (gst_vaapi_reply_to_query(query, download->display))
 614         res = TRUE;
 615     else
 616         res = gst_pad_query_default(pad, query);
 617
 618     g_object_unref(download);
 619     return res;
 620
 621 }