gst/vaapi/gstvaapiupload.c

   1 /*
   2  *  gstvaapiupload.c - VA-API video uploader
   3  *
   4  *  Copyright (C) 2010-2011 Splitted-Desktop Systems
   5  *  Copyright (C) 2011-2012 Intel Corporation
   6  *
   7  *  This library is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU Lesser General Public License
   9  *  as published by the Free Software Foundation; either version 2.1
  10  *  of the License, or (at your option) any later version.
  11  *
  12  *  This library is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  *  Lesser General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU Lesser General Public
  18  *  License along with this library; if not, write to the Free
  19  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  *  Boston, MA 02110-1301 USA
  21  */
  22
  23 /**
  24  * SECTION:gstvaapiupload
  25  * @short_description: A video to VA flow filter
  26  *
  27  * vaapiupload uploads from raw YUV pixels to VA surfaces suitable
  28  * for the vaapisink element, for example.
  29  */
  30
  31 #include "gst/vaapi/sysdeps.h"
  32 #include <gst/gst.h>
  33 #include <gst/video/video.h>
  34 #include <gst/video/videocontext.h>
  35 #include <gst/vaapi/gstvaapivideobuffer.h>
  36 #include <gst/vaapi/gstvaapidebug.h>
  37
  38 #include "gstvaapiupload.h"
  39 #include "gstvaapipluginutil.h"
  40 #include "gstvaapipluginbuffer.h"
  41
  42 #define GST_PLUGIN_NAME "vaapiupload"
  43 #define GST_PLUGIN_DESC "A video to VA flow filter"
  44
  45 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapiupload);
  46 #define GST_CAT_DEFAULT gst_debug_vaapiupload
  47
  48 /* ElementFactory information */
  49 static const GstElementDetails gst_vaapiupload_details =
  50     GST_ELEMENT_DETAILS(
  51         "VA-API colorspace uploader",
  52         "Filter/Converter/Video",
  53         GST_PLUGIN_DESC,
  54         "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
  55
  56 /* Default templates */
  57 static const char gst_vaapiupload_yuv_caps_str[] =
  58     "video/x-raw-yuv, "
  59     "width  = (int) [ 1, MAX ], "
  60     "height = (int) [ 1, MAX ]; ";
  61
  62 static const char gst_vaapiupload_vaapi_caps_str[] =
  63     GST_VAAPI_SURFACE_CAPS;
  64
  65 static GstStaticPadTemplate gst_vaapiupload_sink_factory =
  66     GST_STATIC_PAD_TEMPLATE(
  67         "sink",
  68         GST_PAD_SINK,
  69         GST_PAD_ALWAYS,
  70         GST_STATIC_CAPS(gst_vaapiupload_yuv_caps_str));
  71
  72 static GstStaticPadTemplate gst_vaapiupload_src_factory =
  73     GST_STATIC_PAD_TEMPLATE(
  74         "src",
  75         GST_PAD_SRC,
  76         GST_PAD_ALWAYS,
  77         GST_STATIC_CAPS(gst_vaapiupload_vaapi_caps_str));
  78
  79 static void
  80 gst_vaapiupload_implements_iface_init(GstImplementsInterfaceClass *iface);
  81
  82 static void
  83 gst_video_context_interface_init(GstVideoContextInterface *iface);
  84
  85 #define GstVideoContextClass GstVideoContextInterface
  86 G_DEFINE_TYPE_WITH_CODE(
  87     GstVaapiUpload,
  88     gst_vaapiupload,
  89     GST_TYPE_BASE_TRANSFORM,
  90     G_IMPLEMENT_INTERFACE(GST_TYPE_IMPLEMENTS_INTERFACE,
  91                           gst_vaapiupload_implements_iface_init);
  92     G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
  93                           gst_video_context_interface_init))
  94
  95 /*
  96  * Direct rendering levels (direct-rendering)
  97  * 0: upstream allocated YUV pixels
  98  * 1: vaapiupload allocated YUV pixels (mapped from VA image)
  99  * 2: vaapiupload allocated YUV pixels (mapped from VA surface)
 100  */
 101 #define DIRECT_RENDERING_DEFAULT 2
 102
 103 enum {
 104     PROP_0,
 105
 106     PROP_DIRECT_RENDERING,
 107 };
 108
 109 static gboolean
 110 gst_vaapiupload_start(GstBaseTransform *trans);
 111
 112 static gboolean
 113 gst_vaapiupload_stop(GstBaseTransform *trans);
 114
 115 static GstFlowReturn
 116 gst_vaapiupload_transform(
 117     GstBaseTransform *trans,
 118     GstBuffer        *inbuf,
 119     GstBuffer        *outbuf
 120 );
 121
 122 static GstCaps *
 123 gst_vaapiupload_transform_caps(
 124     GstBaseTransform *trans,
 125     GstPadDirection   direction,
 126     GstCaps          *caps
 127 );
 128
 129 static gboolean
 130 gst_vaapiupload_set_caps(
 131     GstBaseTransform *trans,
 132     GstCaps          *incaps,
 133     GstCaps          *outcaps
 134 );
 135
 136 static gboolean
 137 gst_vaapiupload_get_unit_size(
 138     GstBaseTransform *trans,
 139     GstCaps          *caps,
 140     guint            *size
 141 );
 142
 143 static GstFlowReturn
 144 gst_vaapiupload_sinkpad_buffer_alloc(
 145     GstPad           *pad,
 146     guint64           offset,
 147     guint             size,
 148     GstCaps          *caps,
 149     GstBuffer       **pbuf
 150 );
 151
 152 static GstFlowReturn
 153 gst_vaapiupload_prepare_output_buffer(
 154     GstBaseTransform *trans,
 155     GstBuffer        *inbuf,
 156     gint              size,
 157     GstCaps          *caps,
 158     GstBuffer       **poutbuf
 159 );
 160
 161 static gboolean
 162 gst_vaapiupload_query(
 163     GstPad   *pad,
 164     GstQuery *query
 165 );
 166
 167 /* GstImplementsInterface interface */
 168
 169 static gboolean
 170 gst_vaapiupload_implements_interface_supported(
 171     GstImplementsInterface *iface,
 172     GType                   type
 173 )
 174 {
 175     return (type == GST_TYPE_VIDEO_CONTEXT);
 176 }
 177
 178 static void
 179 gst_vaapiupload_implements_iface_init(GstImplementsInterfaceClass *iface)
 180 {
 181     iface->supported = gst_vaapiupload_implements_interface_supported;
 182 }
 183
 184 /* GstVideoContext interface */
 185
 186 static void
 187 gst_vaapiupload_set_video_context(GstVideoContext *context, const gchar *type,
 188     const GValue *value)
 189 {
 190   GstVaapiUpload *upload = GST_VAAPIUPLOAD (context);
 191   gst_vaapi_set_display (type, value, &upload->display);
 192 }
 193
 194 static void
 195 gst_video_context_interface_init(GstVideoContextInterface *iface)
 196 {
 197     iface->set_context = gst_vaapiupload_set_video_context;
 198 }
 199
 200 static void
 201 gst_vaapiupload_destroy(GstVaapiUpload *upload)
 202 {
 203     g_clear_object(&upload->images);
 204     g_clear_object(&upload->surfaces);
 205     g_clear_object(&upload->display);
 206 }
 207
 208 static void
 209 gst_vaapiupload_finalize(GObject *object)
 210 {
 211     gst_vaapiupload_destroy(GST_VAAPIUPLOAD(object));
 212
 213     G_OBJECT_CLASS(gst_vaapiupload_parent_class)->finalize(object);
 214 }
 215
 216
 217 static void
 218 gst_vaapiupload_set_property(
 219     GObject      *object,
 220     guint         prop_id,
 221     const GValue *value,
 222     GParamSpec   *pspec
 223 )
 224 {
 225     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(object);
 226
 227     switch (prop_id) {
 228     case PROP_DIRECT_RENDERING:
 229         GST_OBJECT_LOCK(upload);
 230         upload->direct_rendering = g_value_get_uint(value);
 231         GST_OBJECT_UNLOCK(upload);
 232         break;
 233     default:
 234         G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
 235         break;
 236     }
 237 }
 238
 239 static void
 240 gst_vaapiupload_get_property(
 241     GObject    *object,
 242     guint       prop_id,
 243     GValue     *value,
 244     GParamSpec *pspec
 245 )
 246 {
 247     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(object);
 248
 249     switch (prop_id) {
 250     case PROP_DIRECT_RENDERING:
 251         g_value_set_uint(value, upload->direct_rendering);
 252         break;
 253     default:
 254         G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
 255         break;
 256     }
 257 }
 258
 259 static void
 260 gst_vaapiupload_class_init(GstVaapiUploadClass *klass)
 261 {
 262     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
 263     GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
 264     GstBaseTransformClass * const trans_class = GST_BASE_TRANSFORM_CLASS(klass);
 265     GstPadTemplate *pad_template;
 266
 267     GST_DEBUG_CATEGORY_INIT(gst_debug_vaapiupload,
 268                             GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
 269
 270     object_class->finalize      = gst_vaapiupload_finalize;
 271     object_class->set_property  = gst_vaapiupload_set_property;
 272     object_class->get_property  = gst_vaapiupload_get_property;
 273
 274     trans_class->start          = gst_vaapiupload_start;
 275     trans_class->stop           = gst_vaapiupload_stop;
 276     trans_class->transform      = gst_vaapiupload_transform;
 277     trans_class->transform_caps = gst_vaapiupload_transform_caps;
 278     trans_class->set_caps       = gst_vaapiupload_set_caps;
 279     trans_class->get_unit_size  = gst_vaapiupload_get_unit_size;
 280     trans_class->prepare_output_buffer = gst_vaapiupload_prepare_output_buffer;
 281
 282     gst_element_class_set_details_simple(
 283         element_class,
 284         gst_vaapiupload_details.longname,
 285         gst_vaapiupload_details.klass,
 286         gst_vaapiupload_details.description,
 287         gst_vaapiupload_details.author
 288     );
 289
 290     /* sink pad */
 291     pad_template = gst_static_pad_template_get(&gst_vaapiupload_sink_factory);
 292     gst_element_class_add_pad_template(element_class, pad_template);
 293     gst_object_unref(pad_template);
 294
 295     /* src pad */
 296     pad_template = gst_static_pad_template_get(&gst_vaapiupload_src_factory);
 297     gst_element_class_add_pad_template(element_class, pad_template);
 298     gst_object_unref(pad_template);
 299
 300     /**
 301      * GstVaapiUpload:direct-rendering:
 302      *
 303      * Selects the direct rendering level.
 304      * <orderedlist>
 305      * <listitem override="0">
 306      *   Disables direct rendering.
 307      * </listitem>
 308      * <listitem>
 309      *   Enables direct rendering to the output buffer. i.e. this
 310      *   tries to use a single buffer for both sink and src pads.
 311      * </listitem>
 312      * <listitem>
 313      *   Enables direct rendering to the underlying surface. i.e. with
 314      *   drivers supporting vaDeriveImage(), the output surface pixels
 315      *   will be modified directly.
 316      * </listitem>
 317      * </orderedlist>
 318      */
 319     g_object_class_install_property
 320         (object_class,
 321          PROP_DIRECT_RENDERING,
 322          g_param_spec_uint("direct-rendering",
 323                            "Direct rendering",
 324                            "Direct rendering level",
 325                            0, 2,
 326                            DIRECT_RENDERING_DEFAULT,
 327                            G_PARAM_READWRITE));
 328 }
 329
 330 static void
 331 gst_vaapiupload_init(GstVaapiUpload *upload)
 332 {
 333     GstPad *sinkpad, *srcpad;
 334
 335     upload->display                    = NULL;
 336     upload->images                     = NULL;
 337     upload->images_reset               = FALSE;
 338     upload->image_width                = 0;
 339     upload->image_height               = 0;
 340     upload->surfaces                   = NULL;
 341     upload->surfaces_reset             = FALSE;
 342     upload->surface_width              = 0;
 343     upload->surface_height             = 0;
 344     upload->direct_rendering_caps      = 0;
 345     upload->direct_rendering           = G_MAXUINT32;
 346     upload->need_manual_upload        = FALSE;
 347
 348     /* Override buffer allocator on sink pad */
 349     sinkpad = gst_element_get_static_pad(GST_ELEMENT(upload), "sink");
 350     gst_pad_set_bufferalloc_function(
 351         sinkpad,
 352         gst_vaapiupload_sinkpad_buffer_alloc
 353     );
 354     gst_pad_set_query_function(sinkpad, gst_vaapiupload_query);
 355     g_object_unref(sinkpad);
 356
 357     /* Override query on src pad */
 358     srcpad = gst_element_get_static_pad(GST_ELEMENT(upload), "src");
 359     gst_pad_set_query_function(srcpad, gst_vaapiupload_query);
 360     g_object_unref(srcpad);
 361 }
 362
 363 static inline gboolean
 364 gst_vaapiupload_ensure_display(GstVaapiUpload *upload)
 365 {
 366     return gst_vaapi_ensure_display(upload, GST_VAAPI_DISPLAY_TYPE_ANY,
 367         &upload->display);
 368 }
 369
 370 static gboolean
 371 gst_vaapiupload_start(GstBaseTransform *trans)
 372 {
 373     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 374
 375     if (!gst_vaapiupload_ensure_display(upload))
 376         return FALSE;
 377     return TRUE;
 378 }
 379
 380 static gboolean
 381 gst_vaapiupload_stop(GstBaseTransform *trans)
 382 {
 383     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 384
 385     g_clear_object(&upload->display);
 386
 387     return TRUE;
 388 }
 389
 390 static GstFlowReturn
 391 gst_vaapiupload_transform(
 392     GstBaseTransform *trans,
 393     GstBuffer        *inbuf,
 394     GstBuffer        *outbuf
 395 )
 396 {
 397     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 398     GstVaapiVideoBuffer *vbuffer;
 399     GstVaapiSurface *surface;
 400     GstVaapiImage *image;
 401     gboolean success;
 402
 403     vbuffer = GST_VAAPI_VIDEO_BUFFER(outbuf);
 404     surface = gst_vaapi_video_buffer_get_surface(vbuffer);
 405     if (!surface)
 406         return GST_FLOW_UNEXPECTED;
 407
 408     if (upload->direct_rendering) {
 409         if (!GST_VAAPI_IS_VIDEO_BUFFER(inbuf)) {
 410             GST_DEBUG("GstVaapiVideoBuffer was expected");
 411             return GST_FLOW_UNEXPECTED;
 412         }
 413
 414         vbuffer = GST_VAAPI_VIDEO_BUFFER(inbuf);
 415         image   = gst_vaapi_video_buffer_get_image(vbuffer);
 416         if (!image)
 417             return GST_FLOW_UNEXPECTED;
 418         if (!gst_vaapi_image_unmap(image))
 419             return GST_FLOW_UNEXPECTED;
 420
 421         if (upload->direct_rendering < 2) {
 422             if (!gst_vaapi_surface_put_image(surface, image))
 423                 goto error_put_image;
 424         }
 425         goto flow_ok;
 426     }
 427
 428     image = gst_vaapi_video_pool_get_object(upload->images);
 429     if (!image)
 430         goto error_put_image;
 431
 432     if (!upload->need_manual_upload) {
 433         gst_vaapi_image_update_from_buffer(image, inbuf, NULL);
 434     } else { /* manually copy data to image*/
 435         success = gst_vaapi_upload_buffer_to_image(image, inbuf);
 436         if (!success)
 437             goto error_put_image;
 438     }
 439
 440     success = gst_vaapi_surface_put_image(surface, image);
 441     gst_vaapi_video_pool_put_object(upload->images, image);
 442     if (!success)
 443         goto error_put_image;
 444
 445 flow_ok:
 446     FPS_CALCULATION(vaapiupload);
 447     return GST_FLOW_OK;
 448
 449 error_put_image:
 450     {
 451         GST_WARNING("failed to upload %" GST_FOURCC_FORMAT " image "
 452                     "to surface 0x%08x",
 453                     GST_FOURCC_ARGS(gst_vaapi_image_get_format(image)),
 454                     gst_vaapi_surface_get_id(surface));
 455         return GST_FLOW_OK;
 456     }
 457 }
 458
 459 static GstCaps *
 460 gst_vaapi_get_other_support_caps(GstVaapiConvert *upload)
 461 {
 462   GstCaps *caps;
 463   caps = gst_caps_from_string(GST_VIDEO_CAPS_YUV("YUY2"));
 464   return caps;
 465 }
 466
 467 static GstCaps *
 468 gst_vaapiupload_transform_caps(
 469     GstBaseTransform *trans,
 470     GstPadDirection   direction,
 471     GstCaps          *caps
 472 )
 473 {
 474     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 475     GstCaps *out_caps = NULL;
 476     GstStructure *structure;
 477
 478     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
 479
 480     structure = gst_caps_get_structure(caps, 0);
 481
 482     if (direction == GST_PAD_SINK) {
 483         if (!gst_structure_has_name(structure, "video/x-raw-yuv"))
 484             return NULL;
 485         out_caps = gst_caps_from_string(gst_vaapiupload_vaapi_caps_str);
 486
 487         structure = gst_caps_get_structure(out_caps, 0);
 488         gst_structure_set(
 489             structure,
 490             "type", G_TYPE_STRING, "vaapi",
 491             "opengl", G_TYPE_BOOLEAN, USE_GLX,
 492             NULL
 493         );
 494     }
 495     else {
 496         if (!gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
 497             return NULL;
 498         out_caps = gst_caps_from_string(gst_vaapiupload_yuv_caps_str);
 499         if (upload->display) {
 500             GstCaps *allowed_caps, *inter_caps, *other_caps;
 501             allowed_caps = gst_vaapi_display_get_image_caps(upload->display);
 502             if (!allowed_caps)
 503                 return NULL;
 504             /* can direct copy other YUV to va surface */
 505             other_caps = gst_vaapi_get_other_support_caps(upload);
 506             gst_caps_merge(allowed_caps, other_caps);
 507
 508             inter_caps = gst_caps_intersect(out_caps, allowed_caps);
 509             gst_caps_unref(allowed_caps);
 510             gst_caps_unref(out_caps);
 511             out_caps = inter_caps;
 512         }
 513     }
 514
 515     if (!gst_vaapi_append_surface_caps(out_caps, caps)) {
 516         gst_caps_unref(out_caps);
 517         return NULL;
 518     }
 519     return out_caps;
 520 }
 521
 522 static gboolean
 523 gst_vaapiupload_ensure_image_pool(GstVaapiUpload *upload, GstCaps *caps)
 524 {
 525     GstStructure * const structure = gst_caps_get_structure(caps, 0);
 526     gint width, height;
 527
 528     gst_structure_get_int(structure, "width",  &width);
 529     gst_structure_get_int(structure, "height", &height);
 530
 531     if (width != upload->image_width || height != upload->image_height) {
 532         upload->image_width  = width;
 533         upload->image_height = height;
 534         g_clear_object(&upload->images);
 535         upload->images = gst_vaapi_image_pool_new(upload->display, caps);
 536         if (!upload->images)
 537             return FALSE;
 538         upload->images_reset = TRUE;
 539     }
 540     return TRUE;
 541 }
 542
 543 static gboolean
 544 gst_vaapiupload_ensure_surface_pool(GstVaapiUpload *upload, GstCaps *caps)
 545 {
 546     GstStructure * const structure = gst_caps_get_structure(caps, 0);
 547     gint width, height;
 548
 549     gst_structure_get_int(structure, "width",  &width);
 550     gst_structure_get_int(structure, "height", &height);
 551
 552     if (width != upload->surface_width || height != upload->surface_height) {
 553         upload->surface_width  = width;
 554         upload->surface_height = height;
 555         g_clear_object(&upload->surfaces);
 556         upload->surfaces = gst_vaapi_surface_pool_new(upload->display, caps);
 557         if (!upload->surfaces)
 558             return FALSE;
 559         upload->surfaces_reset = TRUE;
 560     }
 561     return TRUE;
 562 }
 563
 564 static void
 565 gst_vaapiupload_ensure_direct_rendering_caps(
 566     GstVaapiUpload *upload,
 567     GstCaps         *caps
 568 )
 569 {
 570     GstVaapiSurface *surface;
 571     GstVaapiImage *image;
 572     GstVaapiImageFormat vaformat;
 573     GstVideoFormat vformat;
 574     GstStructure *structure;
 575     gint width, height;
 576
 577     if (!upload->images_reset && !upload->surfaces_reset)
 578         return;
 579
 580     upload->images_reset          = FALSE;
 581     upload->surfaces_reset        = FALSE;
 582     upload->direct_rendering_caps = 0;
 583
 584     structure = gst_caps_get_structure(caps, 0);
 585     if (!structure)
 586         return;
 587     gst_structure_get_int(structure, "width",  &width);
 588     gst_structure_get_int(structure, "height", &height);
 589
 590     /* Translate from Gst video format to VA image format */
 591     if (!gst_video_format_parse_caps(caps, &vformat, NULL, NULL))
 592         return;
 593     if (!gst_video_format_is_yuv(vformat))
 594         return;
 595     vaformat = gst_vaapi_image_format_from_video(vformat);
 596     if (!vaformat)
 597         return;
 598
 599     /* Check if we can alias sink & output buffers (same data_size) */
 600     image = gst_vaapi_video_pool_get_object(upload->images);
 601     if (image) {
 602         if (upload->direct_rendering_caps == 0 &&
 603             (gst_vaapi_image_get_format(image) == vaformat &&
 604              gst_vaapi_image_is_linear(image) &&
 605              (gst_vaapi_image_get_data_size(image) ==
 606               gst_video_format_get_size(vformat, width, height))))
 607             upload->direct_rendering_caps = 1;
 608         gst_vaapi_video_pool_put_object(upload->images, image);
 609     }
 610
 611     /* Check if we can access to the surface pixels directly */
 612     surface = gst_vaapi_video_pool_get_object(upload->surfaces);
 613     if (surface) {
 614         image = gst_vaapi_surface_derive_image(surface);
 615         if (image) {
 616             if (gst_vaapi_image_map(image)) {
 617                 if (upload->direct_rendering_caps == 1 &&
 618                     (gst_vaapi_image_get_format(image) == vaformat &&
 619                      gst_vaapi_image_is_linear(image) &&
 620                      (gst_vaapi_image_get_data_size(image) ==
 621                       gst_video_format_get_size(vformat, width, height))))
 622                     upload->direct_rendering_caps = 2;
 623                 gst_vaapi_image_unmap(image);
 624             }
 625             g_object_unref(image);
 626         }
 627         gst_vaapi_video_pool_put_object(upload->surfaces, surface);
 628     }
 629 }
 630
 631 typedef enum YUV_TYPE {
 632   YUV_UNKOWN = 0,
 633   YUV_411    = 1,
 634   YUV_422    = 2,
 635   YUV_444    = 4
 636 } YUV_TYPE;
 637
 638 static YUV_TYPE
 639 _image_format_to_yuv_type(guint32 fourcc)
 640 {
 641     switch (fourcc) {
 642         case GST_MAKE_FOURCC('N','V','1','2'):
 643         case GST_MAKE_FOURCC('Y','V','1','2'):
 644         case GST_MAKE_FOURCC('I','4','2','0'):
 645         case GST_MAKE_FOURCC('N','V','2','1'):
 646         return YUV_411;
 647
 648         case GST_MAKE_FOURCC('Y','U','Y','2'):
 649         case GST_MAKE_FOURCC('Y','V','Y','U'):
 650         return YUV_422;
 651
 652         case GST_MAKE_FOURCC('A','Y','U','V'):
 653         return YUV_UNKOWN;
 654
 655         default:
 656         return YUV_UNKOWN;
 657     }
 658 }
 659
 660 static GstCaps *
 661 _get_nearest_caps(GstCaps *caps_list, GstCaps *src_caps)
 662 {
 663     GstCaps *ret = NULL;
 664     GstStructure *cur_struct, *tmp_struct;
 665     guint32 cur_format, dest_format, tmp_format;
 666     YUV_TYPE cur_type, tmp_type;
 667     const GValue*tmp_val;
 668     guint  n_caps;
 669     guint  i;
 670     guint min_diff, tmp_diff;
 671
 672     cur_struct = gst_caps_get_structure(src_caps, 0);
 673     tmp_val = gst_structure_get_value (cur_struct, "format");
 674     if (!tmp_val)
 675         return NULL;
 676
 677     cur_format = gst_value_get_fourcc(tmp_val);
 678     if((cur_type = _image_format_to_yuv_type(cur_format)) == YUV_UNKOWN)
 679         return NULL;
 680
 681     n_caps = gst_caps_get_size(caps_list);
 682     min_diff = 100;
 683     dest_format = 0;
 684     for (i = 0; i < n_caps; ++i) {
 685       tmp_struct = gst_caps_get_structure(caps_list, i);
 686       tmp_val = gst_structure_get_value (tmp_struct, "format");
 687       if (!tmp_val)
 688           continue;
 689       tmp_format = gst_value_get_fourcc(tmp_val);
 690       if ((tmp_type = _image_format_to_yuv_type(tmp_format)) == YUV_UNKOWN)
 691           continue;
 692       tmp_diff = abs(tmp_type - cur_type);
 693       if (tmp_diff < min_diff) {
 694           min_diff = tmp_diff;
 695           dest_format = tmp_format;
 696       }
 697     }
 698
 699     if (dest_format == 0)
 700       return NULL;
 701
 702     ret = gst_caps_copy(src_caps);
 703     tmp_struct = gst_caps_get_structure(ret, 0);
 704     gst_structure_set(tmp_struct, "format", GST_TYPE_FOURCC, dest_format, NULL);
 705     return ret;
 706 }
 707
 708 static gboolean
 709 gst_vaapiupload_negotiate_buffers(
 710     GstVaapiUpload  *upload,
 711     GstCaps          *incaps,
 712     GstCaps          *outcaps
 713 )
 714 {
 715     guint dr;
 716     gboolean ret = TRUE;
 717     GstCaps *image_allowed_caps = NULL;
 718     GstCaps *image_caps = NULL;
 719
 720     image_allowed_caps = gst_vaapi_display_get_image_caps(upload->display);
 721     if (gst_caps_can_intersect(incaps, image_allowed_caps)) {
 722         image_caps = gst_caps_ref(incaps);
 723         upload->need_manual_upload = FALSE;
 724     } else {
 725         image_caps = _get_nearest_caps(image_allowed_caps, incaps);
 726         upload->need_manual_upload = TRUE;
 727     }
 728
 729     if (!gst_vaapiupload_ensure_image_pool(upload, image_caps))
 730         goto failed;
 731
 732     if (!gst_vaapiupload_ensure_surface_pool(upload, outcaps))
 733         goto failed;
 734
 735     if (upload->direct_rendering && !upload->need_manual_upload)
 736       gst_vaapiupload_ensure_direct_rendering_caps(upload, incaps);
 737     dr = MIN(upload->direct_rendering, upload->direct_rendering_caps);
 738     if (upload->direct_rendering != dr) {
 739         upload->direct_rendering = dr;
 740         GST_DEBUG("direct-rendering level: %d", dr);
 741     }
 742     ret = TRUE;
 743     goto end;
 744
 745   failed:
 746     ret = FALSE;
 747
 748   end:
 749     gst_caps_unref(image_caps);
 750     gst_caps_unref(image_allowed_caps);
 751     return ret;
 752 }
 753
 754 static gboolean
 755 gst_vaapiupload_set_caps(
 756     GstBaseTransform *trans,
 757     GstCaps          *incaps,
 758     GstCaps          *outcaps
 759 )
 760 {
 761     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 762
 763     if (!gst_vaapiupload_negotiate_buffers(upload, incaps, outcaps))
 764         return FALSE;
 765
 766     GST_INFO("set caps\nIN caps:\n%" GST_PTR_FORMAT "\nOUT caps:\n%" GST_PTR_FORMAT,
 767              incaps, outcaps);
 768     return TRUE;
 769 }
 770
 771 static gboolean
 772 gst_vaapiupload_get_unit_size(
 773     GstBaseTransform *trans,
 774     GstCaps          *caps,
 775     guint            *size
 776 )
 777 {
 778     GstStructure * const structure = gst_caps_get_structure(caps, 0);
 779     GstVideoFormat format;
 780     gint width, height;
 781
 782     if (gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
 783         *size = 0;
 784     else {
 785         if (!gst_video_format_parse_caps(caps, &format, &width, &height))
 786             return FALSE;
 787         *size = gst_video_format_get_size(format, width, height);
 788     }
 789     return TRUE;
 790 }
 791
 792 static GstFlowReturn
 793 gst_vaapiupload_buffer_alloc(
 794     GstBaseTransform *trans,
 795     guint             size,
 796     GstCaps          *caps,
 797     GstBuffer       **pbuf
 798 )
 799 {
 800     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 801     GstBuffer *buffer = NULL;
 802     GstVaapiImage *image = NULL;
 803     GstVaapiSurface *surface = NULL;
 804     GstVaapiVideoBuffer *vbuffer;
 805
 806     /* already checked */
 807     if (!upload->direct_rendering)
 808         return GST_FLOW_OK;
 809
 810     /* Check if we can use direct-rendering */
 811     if (!gst_vaapiupload_negotiate_buffers(upload, caps, caps))
 812         goto error;
 813     if (!upload->direct_rendering)
 814         return GST_FLOW_OK;
 815
 816     switch (upload->direct_rendering) {
 817     case 2:
 818         buffer  = gst_vaapi_video_buffer_new_from_pool(upload->surfaces);
 819         if (!buffer)
 820             goto error;
 821         vbuffer = GST_VAAPI_VIDEO_BUFFER(buffer);
 822
 823         surface = gst_vaapi_video_buffer_get_surface(vbuffer);
 824         image   = gst_vaapi_surface_derive_image(surface);
 825         if (image && gst_vaapi_image_get_data_size(image) == size) {
 826             gst_vaapi_video_buffer_set_image(vbuffer, image);
 827             g_object_unref(image); /* video buffer owns an extra reference */
 828             break;
 829         }
 830
 831         /* We can't use the derive-image optimization. Disable it. */
 832         upload->direct_rendering = 1;
 833         gst_buffer_unref(buffer);
 834         buffer = NULL;
 835
 836     case 1:
 837         buffer  = gst_vaapi_video_buffer_new_from_pool(upload->images);
 838         if (!buffer)
 839             goto error;
 840         vbuffer = GST_VAAPI_VIDEO_BUFFER(buffer);
 841
 842         image   = gst_vaapi_video_buffer_get_image(vbuffer);
 843         break;
 844     }
 845     g_assert(image);
 846
 847     if (!gst_vaapi_image_map(image))
 848         goto error;
 849
 850     GST_BUFFER_DATA(buffer) = gst_vaapi_image_get_plane(image, 0);
 851     GST_BUFFER_SIZE(buffer) = gst_vaapi_image_get_data_size(image);
 852
 853     gst_buffer_set_caps(buffer, caps);
 854     *pbuf = buffer;
 855     return GST_FLOW_OK;
 856
 857 error:
 858     /* We can't use the inout-buffers optimization. Disable it. */
 859     GST_DEBUG("disable in/out buffer optimization");
 860     if (buffer)
 861         gst_buffer_unref(buffer);
 862     upload->direct_rendering = 0;
 863     return GST_FLOW_OK;
 864 }
 865
 866 static GstFlowReturn
 867 gst_vaapiupload_sinkpad_buffer_alloc(
 868     GstPad           *pad,
 869     guint64           offset,
 870     guint             size,
 871     GstCaps          *caps,
 872     GstBuffer       **pbuf
 873 )
 874 {
 875     GstBaseTransform *trans;
 876     GstFlowReturn ret;
 877
 878     trans = GST_BASE_TRANSFORM(gst_pad_get_parent_element(pad));
 879     if (!trans)
 880         return GST_FLOW_UNEXPECTED;
 881
 882     ret = gst_vaapiupload_buffer_alloc(trans, size, caps, pbuf);
 883     g_object_unref(trans);
 884     return ret;
 885 }
 886
 887 static GstFlowReturn
 888 gst_vaapiupload_prepare_output_buffer(
 889     GstBaseTransform *trans,
 890     GstBuffer        *inbuf,
 891     gint              size,
 892     GstCaps          *caps,
 893     GstBuffer       **poutbuf
 894 )
 895 {
 896     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 897     GstBuffer *buffer = NULL;
 898
 899     if (upload->direct_rendering == 2) {
 900         if (GST_VAAPI_IS_VIDEO_BUFFER(inbuf)) {
 901             buffer = gst_vaapi_video_buffer_new_from_buffer(inbuf);
 902             GST_BUFFER_SIZE(buffer) = size;
 903         }
 904         else {
 905             GST_DEBUG("upstream element destroyed our in/out buffer");
 906             upload->direct_rendering = 1;
 907         }
 908     }
 909
 910     if (!buffer) {
 911         buffer = gst_vaapi_video_buffer_new_from_pool(upload->surfaces);
 912         if (!buffer)
 913             return GST_FLOW_UNEXPECTED;
 914         gst_buffer_set_caps(buffer, caps);
 915     }
 916
 917     *poutbuf = buffer;
 918     return GST_FLOW_OK;
 919 }
 920
 921 static gboolean
 922 gst_vaapiupload_query(GstPad *pad, GstQuery *query)
 923 {
 924   GstVaapiUpload *upload = GST_VAAPIUPLOAD (gst_pad_get_parent_element (pad));
 925   gboolean res;
 926
 927   GST_DEBUG ("sharing display %p", upload->display);
 928
 929   if (gst_vaapi_reply_to_query (query, upload->display))
 930     res = TRUE;
 931   else
 932     res = gst_pad_query_default (pad, query);
 933
 934   g_object_unref (upload);
 935   return res;
 936 }