gst/vaapi/gstvaapiupload.c

   1 /*
   2  *  gstvaapiupload.c - VA-API video uploader
   3  *
   4  *  Copyright (C) 2010-2011 Splitted-Desktop Systems
   5  *  Copyright (C) 2011-2012 Intel Corporation
   6  *
   7  *  This library is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU Lesser General Public License
   9  *  as published by the Free Software Foundation; either version 2.1
  10  *  of the License, or (at your option) any later version.
  11  *
  12  *  This library is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  *  Lesser General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU Lesser General Public
  18  *  License along with this library; if not, write to the Free
  19  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  *  Boston, MA 02110-1301 USA
  21  */
  22
  23 /**
  24  * SECTION:gstvaapiupload
  25  * @short_description: A video to VA flow filter
  26  *
  27  * vaapiupload uploads from raw YUV pixels to VA surfaces suitable
  28  * for the vaapisink element, for example.
  29  */
  30
  31 #include "config.h"
  32
  33 #include <gst/gst.h>
  34 #include <gst/video/video.h>
  35 #include <gst/video/videocontext.h>
  36 #include <gst/vaapi/gstvaapivideobuffer.h>
  37 #include <gst/vaapi/gstvaapidebug.h>
  38
  39 #include "gstvaapiupload.h"
  40 #include "gstvaapipluginutil.h"
  41 #include "gstvaapipluginbuffer.h"
  42
  43 #define GST_PLUGIN_NAME "vaapiupload"
  44 #define GST_PLUGIN_DESC "A video to VA flow filter"
  45
  46 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapiupload);
  47 #define GST_CAT_DEFAULT gst_debug_vaapiupload
  48
  49 /* ElementFactory information */
  50 static const GstElementDetails gst_vaapiupload_details =
  51     GST_ELEMENT_DETAILS(
  52         "VA-API colorspace uploader",
  53         "Filter/Converter/Video",
  54         GST_PLUGIN_DESC,
  55         "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
  56
  57 /* Default templates */
  58 static const char gst_vaapiupload_yuv_caps_str[] =
  59     "video/x-raw-yuv, "
  60     "width  = (int) [ 1, MAX ], "
  61     "height = (int) [ 1, MAX ]; ";
  62
  63 static const char gst_vaapiupload_vaapi_caps_str[] =
  64     GST_VAAPI_SURFACE_CAPS;
  65
  66 static GstStaticPadTemplate gst_vaapiupload_sink_factory =
  67     GST_STATIC_PAD_TEMPLATE(
  68         "sink",
  69         GST_PAD_SINK,
  70         GST_PAD_ALWAYS,
  71         GST_STATIC_CAPS(gst_vaapiupload_yuv_caps_str));
  72
  73 static GstStaticPadTemplate gst_vaapiupload_src_factory =
  74     GST_STATIC_PAD_TEMPLATE(
  75         "src",
  76         GST_PAD_SRC,
  77         GST_PAD_ALWAYS,
  78         GST_STATIC_CAPS(gst_vaapiupload_vaapi_caps_str));
  79
  80 static void
  81 gst_vaapiupload_implements_iface_init(GstImplementsInterfaceClass *iface);
  82
  83 static void
  84 gst_video_context_interface_init(GstVideoContextInterface *iface);
  85
  86 #define GstVideoContextClass GstVideoContextInterface
  87 G_DEFINE_TYPE_WITH_CODE(
  88     GstVaapiUpload,
  89     gst_vaapiupload,
  90     GST_TYPE_BASE_TRANSFORM,
  91     G_IMPLEMENT_INTERFACE(GST_TYPE_IMPLEMENTS_INTERFACE,
  92                           gst_vaapiupload_implements_iface_init);
  93     G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
  94                           gst_video_context_interface_init));
  95
  96 /*
  97  * Direct rendering levels (direct-rendering)
  98  * 0: upstream allocated YUV pixels
  99  * 1: vaapiupload allocated YUV pixels (mapped from VA image)
 100  * 2: vaapiupload allocated YUV pixels (mapped from VA surface)
 101  */
 102 #define DIRECT_RENDERING_DEFAULT 2
 103
 104 enum {
 105     PROP_0,
 106
 107     PROP_DIRECT_RENDERING,
 108 };
 109
 110 static gboolean
 111 gst_vaapiupload_start(GstBaseTransform *trans);
 112
 113 static gboolean
 114 gst_vaapiupload_stop(GstBaseTransform *trans);
 115
 116 static GstFlowReturn
 117 gst_vaapiupload_transform(
 118     GstBaseTransform *trans,
 119     GstBuffer        *inbuf,
 120     GstBuffer        *outbuf
 121 );
 122
 123 static GstCaps *
 124 gst_vaapiupload_transform_caps(
 125     GstBaseTransform *trans,
 126     GstPadDirection   direction,
 127     GstCaps          *caps
 128 );
 129
 130 static gboolean
 131 gst_vaapiupload_set_caps(
 132     GstBaseTransform *trans,
 133     GstCaps          *incaps,
 134     GstCaps          *outcaps
 135 );
 136
 137 static gboolean
 138 gst_vaapiupload_get_unit_size(
 139     GstBaseTransform *trans,
 140     GstCaps          *caps,
 141     guint            *size
 142 );
 143
 144 static GstFlowReturn
 145 gst_vaapiupload_sinkpad_buffer_alloc(
 146     GstPad           *pad,
 147     guint64           offset,
 148     guint             size,
 149     GstCaps          *caps,
 150     GstBuffer       **pbuf
 151 );
 152
 153 static GstFlowReturn
 154 gst_vaapiupload_prepare_output_buffer(
 155     GstBaseTransform *trans,
 156     GstBuffer        *inbuf,
 157     gint              size,
 158     GstCaps          *caps,
 159     GstBuffer       **poutbuf
 160 );
 161
 162 static gboolean
 163 gst_vaapiupload_query(
 164     GstPad   *pad,
 165     GstQuery *query
 166 );
 167
 168 /* GstImplementsInterface interface */
 169
 170 static gboolean
 171 gst_vaapiupload_implements_interface_supported(
 172     GstImplementsInterface *iface,
 173     GType                   type
 174 )
 175 {
 176     return (type == GST_TYPE_VIDEO_CONTEXT);
 177 }
 178
 179 static void
 180 gst_vaapiupload_implements_iface_init(GstImplementsInterfaceClass *iface)
 181 {
 182     iface->supported = gst_vaapiupload_implements_interface_supported;
 183 }
 184
 185 /* GstVideoContext interface */
 186
 187 static void
 188 gst_vaapiupload_set_video_context(GstVideoContext *context, const gchar *type,
 189     const GValue *value)
 190 {
 191   GstVaapiUpload *upload = GST_VAAPIUPLOAD (context);
 192   gst_vaapi_set_display (type, value, &upload->display);
 193 }
 194
 195 static void
 196 gst_video_context_interface_init(GstVideoContextInterface *iface)
 197 {
 198     iface->set_context = gst_vaapiupload_set_video_context;
 199 }
 200
 201 static void
 202 gst_vaapiupload_destroy(GstVaapiUpload *upload)
 203 {
 204     g_clear_object(&upload->images);
 205     g_clear_object(&upload->surfaces);
 206     g_clear_object(&upload->display);
 207 }
 208
 209 static void
 210 gst_vaapiupload_finalize(GObject *object)
 211 {
 212     gst_vaapiupload_destroy(GST_VAAPIUPLOAD(object));
 213
 214     G_OBJECT_CLASS(gst_vaapiupload_parent_class)->finalize(object);
 215 }
 216
 217
 218 static void
 219 gst_vaapiupload_set_property(
 220     GObject      *object,
 221     guint         prop_id,
 222     const GValue *value,
 223     GParamSpec   *pspec
 224 )
 225 {
 226     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(object);
 227
 228     switch (prop_id) {
 229     case PROP_DIRECT_RENDERING:
 230         GST_OBJECT_LOCK(upload);
 231         upload->direct_rendering = g_value_get_uint(value);
 232         GST_OBJECT_UNLOCK(upload);
 233         break;
 234     default:
 235         G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
 236         break;
 237     }
 238 }
 239
 240 static void
 241 gst_vaapiupload_get_property(
 242     GObject    *object,
 243     guint       prop_id,
 244     GValue     *value,
 245     GParamSpec *pspec
 246 )
 247 {
 248     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(object);
 249
 250     switch (prop_id) {
 251     case PROP_DIRECT_RENDERING:
 252         g_value_set_uint(value, upload->direct_rendering);
 253         break;
 254     default:
 255         G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
 256         break;
 257     }
 258 }
 259
 260 static void
 261 gst_vaapiupload_class_init(GstVaapiUploadClass *klass)
 262 {
 263     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
 264     GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
 265     GstBaseTransformClass * const trans_class = GST_BASE_TRANSFORM_CLASS(klass);
 266     GstPadTemplate *pad_template;
 267
 268     GST_DEBUG_CATEGORY_INIT(gst_debug_vaapiupload,
 269                             GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
 270
 271     object_class->finalize      = gst_vaapiupload_finalize;
 272     object_class->set_property  = gst_vaapiupload_set_property;
 273     object_class->get_property  = gst_vaapiupload_get_property;
 274
 275     trans_class->start          = gst_vaapiupload_start;
 276     trans_class->stop           = gst_vaapiupload_stop;
 277     trans_class->transform      = gst_vaapiupload_transform;
 278     trans_class->transform_caps = gst_vaapiupload_transform_caps;
 279     trans_class->set_caps       = gst_vaapiupload_set_caps;
 280     trans_class->get_unit_size  = gst_vaapiupload_get_unit_size;
 281     trans_class->prepare_output_buffer = gst_vaapiupload_prepare_output_buffer;
 282
 283     gst_element_class_set_details_simple(
 284         element_class,
 285         gst_vaapiupload_details.longname,
 286         gst_vaapiupload_details.klass,
 287         gst_vaapiupload_details.description,
 288         gst_vaapiupload_details.author
 289     );
 290
 291     /* sink pad */
 292     pad_template = gst_static_pad_template_get(&gst_vaapiupload_sink_factory);
 293     gst_element_class_add_pad_template(element_class, pad_template);
 294     gst_object_unref(pad_template);
 295
 296     /* src pad */
 297     pad_template = gst_static_pad_template_get(&gst_vaapiupload_src_factory);
 298     gst_element_class_add_pad_template(element_class, pad_template);
 299     gst_object_unref(pad_template);
 300
 301     /**
 302      * GstVaapiUpload:direct-rendering:
 303      *
 304      * Selects the direct rendering level.
 305      * <orderedlist>
 306      * <listitem override="0">
 307      *   Disables direct rendering.
 308      * </listitem>
 309      * <listitem>
 310      *   Enables direct rendering to the output buffer. i.e. this
 311      *   tries to use a single buffer for both sink and src pads.
 312      * </listitem>
 313      * <listitem>
 314      *   Enables direct rendering to the underlying surface. i.e. with
 315      *   drivers supporting vaDeriveImage(), the output surface pixels
 316      *   will be modified directly.
 317      * </listitem>
 318      * </orderedlist>
 319      */
 320     g_object_class_install_property
 321         (object_class,
 322          PROP_DIRECT_RENDERING,
 323          g_param_spec_uint("direct-rendering",
 324                            "Direct rendering",
 325                            "Direct rendering level",
 326                            0, 2,
 327                            DIRECT_RENDERING_DEFAULT,
 328                            G_PARAM_READWRITE));
 329 }
 330
 331 static void
 332 gst_vaapiupload_init(GstVaapiUpload *upload)
 333 {
 334     GstPad *sinkpad, *srcpad;
 335
 336     upload->display                    = NULL;
 337     upload->images                     = NULL;
 338     upload->images_reset               = FALSE;
 339     upload->image_width                = 0;
 340     upload->image_height               = 0;
 341     upload->surfaces                   = NULL;
 342     upload->surfaces_reset             = FALSE;
 343     upload->surface_width              = 0;
 344     upload->surface_height             = 0;
 345     upload->direct_rendering_caps      = 0;
 346     upload->direct_rendering           = G_MAXUINT32;
 347     upload->need_manual_upload        = FALSE;
 348
 349     /* Override buffer allocator on sink pad */
 350     sinkpad = gst_element_get_static_pad(GST_ELEMENT(upload), "sink");
 351     gst_pad_set_bufferalloc_function(
 352         sinkpad,
 353         gst_vaapiupload_sinkpad_buffer_alloc
 354     );
 355     gst_pad_set_query_function(sinkpad, gst_vaapiupload_query);
 356     g_object_unref(sinkpad);
 357
 358     /* Override query on src pad */
 359     srcpad = gst_element_get_static_pad(GST_ELEMENT(upload), "src");
 360     gst_pad_set_query_function(srcpad, gst_vaapiupload_query);
 361     g_object_unref(srcpad);
 362 }
 363
 364 static inline gboolean
 365 gst_vaapiupload_ensure_display(GstVaapiUpload *upload)
 366 {
 367     return gst_vaapi_ensure_display(upload, GST_VAAPI_DISPLAY_TYPE_ANY,
 368         &upload->display);
 369 }
 370
 371 static gboolean
 372 gst_vaapiupload_start(GstBaseTransform *trans)
 373 {
 374     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 375
 376     if (!gst_vaapiupload_ensure_display(upload))
 377         return FALSE;
 378     return TRUE;
 379 }
 380
 381 static gboolean
 382 gst_vaapiupload_stop(GstBaseTransform *trans)
 383 {
 384     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 385
 386     g_clear_object(&upload->display);
 387
 388     return TRUE;
 389 }
 390
 391 static GstFlowReturn
 392 gst_vaapiupload_transform(
 393     GstBaseTransform *trans,
 394     GstBuffer        *inbuf,
 395     GstBuffer        *outbuf
 396 )
 397 {
 398     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 399     GstVaapiVideoBuffer *vbuffer;
 400     GstVaapiSurface *surface;
 401     GstVaapiImage *image;
 402     gboolean success;
 403
 404     vbuffer = GST_VAAPI_VIDEO_BUFFER(outbuf);
 405     surface = gst_vaapi_video_buffer_get_surface(vbuffer);
 406     if (!surface)
 407         return GST_FLOW_UNEXPECTED;
 408
 409     if (upload->direct_rendering) {
 410         if (!GST_VAAPI_IS_VIDEO_BUFFER(inbuf)) {
 411             GST_DEBUG("GstVaapiVideoBuffer was expected");
 412             return GST_FLOW_UNEXPECTED;
 413         }
 414
 415         vbuffer = GST_VAAPI_VIDEO_BUFFER(inbuf);
 416         image   = gst_vaapi_video_buffer_get_image(vbuffer);
 417         if (!image)
 418             return GST_FLOW_UNEXPECTED;
 419         if (!gst_vaapi_image_unmap(image))
 420             return GST_FLOW_UNEXPECTED;
 421
 422         if (upload->direct_rendering < 2) {
 423             if (!gst_vaapi_surface_put_image(surface, image))
 424                 goto error_put_image;
 425         }
 426         goto flow_ok;
 427     }
 428
 429     /* PVR need directly copy image to surface */
 430     image = gst_vaapi_surface_derive_image(surface);
 431     //image = gst_vaapi_video_pool_get_object(upload->images);
 432     if (!image)
 433         goto error_put_image;
 434
 435     if (!upload->need_manual_upload) {
 436         success = gst_vaapi_image_update_from_buffer(image, inbuf, NULL);
 437     } else { /* manually copy data to image*/
 438         success = gst_vaapi_convert_buffer_to_image(image, inbuf);
 439     }
 440     if (!success)
 441             goto error_put_image;
 442
 443     //success = gst_vaapi_surface_put_image(surface, image);
 444     //gst_vaapi_video_pool_put_object(upload->images, image);
 445     //if (!success)
 446     //    goto error_put_image;
 447     g_object_unref(image);
 448
 449 flow_ok:
 450     FPS_CALCULATION(vaapiupload);
 451     return GST_FLOW_OK;
 452
 453 error_put_image:
 454     {
 455         GST_WARNING("failed to upload %" GST_FOURCC_FORMAT " image "
 456                     "to surface 0x%08x",
 457                     GST_FOURCC_ARGS(gst_vaapi_image_get_format(image)),
 458                     gst_vaapi_surface_get_id(surface));
 459         if (!upload->direct_rendering && image)
 460             g_object_unref(image);
 461         return GST_FLOW_OK;
 462     }
 463 }
 464
 465 static GstCaps *
 466 gst_vaapi_get_other_support_caps(GstVaapiUpload *upload)
 467 {
 468   GstCaps *caps;
 469   caps = gst_caps_from_string(GST_VIDEO_CAPS_YUV("YUY2"));
 470   return caps;
 471 }
 472
 473 static GstCaps *
 474 gst_vaapiupload_transform_caps(
 475     GstBaseTransform *trans,
 476     GstPadDirection   direction,
 477     GstCaps          *caps
 478 )
 479 {
 480     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 481     GstCaps *out_caps = NULL;
 482     GstStructure *structure;
 483
 484     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
 485
 486     structure = gst_caps_get_structure(caps, 0);
 487
 488     if (direction == GST_PAD_SINK) {
 489         if (!gst_structure_has_name(structure, "video/x-raw-yuv"))
 490             return NULL;
 491         out_caps = gst_caps_from_string(gst_vaapiupload_vaapi_caps_str);
 492
 493         structure = gst_caps_get_structure(out_caps, 0);
 494         gst_structure_set(
 495             structure,
 496             "type", G_TYPE_STRING, "vaapi",
 497             "opengl", G_TYPE_BOOLEAN, USE_GLX,
 498             NULL
 499         );
 500     }
 501     else {
 502         if (!gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
 503             return NULL;
 504         out_caps = gst_caps_from_string(gst_vaapiupload_yuv_caps_str);
 505         if (upload->display) {
 506             GstCaps *allowed_caps, *inter_caps, *other_caps;
 507             allowed_caps = gst_vaapi_display_get_image_caps(upload->display);
 508             if (!allowed_caps)
 509                 return NULL;
 510             /* can direct copy other YUV to va surface */
 511             other_caps = gst_vaapi_get_other_support_caps(upload);
 512             gst_caps_merge(allowed_caps, other_caps);
 513
 514             inter_caps = gst_caps_intersect(out_caps, allowed_caps);
 515             gst_caps_unref(allowed_caps);
 516             gst_caps_unref(out_caps);
 517             out_caps = inter_caps;
 518         }
 519     }
 520
 521     if (!gst_vaapi_append_surface_caps(out_caps, caps)) {
 522         gst_caps_unref(out_caps);
 523         return NULL;
 524     }
 525     return out_caps;
 526 }
 527
 528 static gboolean
 529 gst_vaapiupload_ensure_image_pool(GstVaapiUpload *upload, GstCaps *caps)
 530 {
 531     GstStructure * const structure = gst_caps_get_structure(caps, 0);
 532     gint width, height;
 533
 534     gst_structure_get_int(structure, "width",  &width);
 535     gst_structure_get_int(structure, "height", &height);
 536
 537     if (width != upload->image_width || height != upload->image_height) {
 538         upload->image_width  = width;
 539         upload->image_height = height;
 540         g_clear_object(&upload->images);
 541         upload->images = gst_vaapi_image_pool_new(upload->display, caps);
 542         if (!upload->images)
 543             return FALSE;
 544         upload->images_reset = TRUE;
 545     }
 546     return TRUE;
 547 }
 548
 549 static gboolean
 550 gst_vaapiupload_ensure_surface_pool(GstVaapiUpload *upload, GstCaps *caps)
 551 {
 552     GstStructure * const structure = gst_caps_get_structure(caps, 0);
 553     gint width, height;
 554
 555     gst_structure_get_int(structure, "width",  &width);
 556     gst_structure_get_int(structure, "height", &height);
 557
 558     if (width != upload->surface_width || height != upload->surface_height) {
 559         upload->surface_width  = width;
 560         upload->surface_height = height;
 561         g_clear_object(&upload->surfaces);
 562         upload->surfaces = gst_vaapi_surface_pool_new(upload->display, caps);
 563         if (!upload->surfaces)
 564             return FALSE;
 565         upload->surfaces_reset = TRUE;
 566     }
 567     return TRUE;
 568 }
 569
 570 static void
 571 gst_vaapiupload_ensure_direct_rendering_caps(
 572     GstVaapiUpload *upload,
 573     GstCaps         *caps
 574 )
 575 {
 576     GstVaapiSurface *surface;
 577     GstVaapiImage *image;
 578     GstVaapiImageFormat vaformat;
 579     GstVideoFormat vformat;
 580     GstStructure *structure;
 581     gint width, height;
 582
 583     if (!upload->images_reset && !upload->surfaces_reset)
 584         return;
 585
 586     upload->images_reset          = FALSE;
 587     upload->surfaces_reset        = FALSE;
 588     upload->direct_rendering_caps = 0;
 589
 590     structure = gst_caps_get_structure(caps, 0);
 591     if (!structure)
 592         return;
 593     gst_structure_get_int(structure, "width",  &width);
 594     gst_structure_get_int(structure, "height", &height);
 595
 596     /* Translate from Gst video format to VA image format */
 597     if (!gst_video_format_parse_caps(caps, &vformat, NULL, NULL))
 598         return;
 599     if (!gst_video_format_is_yuv(vformat))
 600         return;
 601     vaformat = gst_vaapi_image_format_from_video(vformat);
 602     if (!vaformat)
 603         return;
 604
 605     /* Check if we can alias sink & output buffers (same data_size) */
 606     image = gst_vaapi_video_pool_get_object(upload->images);
 607     if (image) {
 608         if (upload->direct_rendering_caps == 0 &&
 609             (gst_vaapi_image_get_format(image) == vaformat &&
 610              gst_vaapi_image_is_linear(image) &&
 611              (gst_vaapi_image_get_data_size(image) ==
 612               gst_video_format_get_size(vformat, width, height))))
 613             upload->direct_rendering_caps = 1;
 614         gst_vaapi_video_pool_put_object(upload->images, image);
 615     }
 616
 617     /* Check if we can access to the surface pixels directly */
 618     surface = gst_vaapi_video_pool_get_object(upload->surfaces);
 619     if (surface) {
 620         image = gst_vaapi_surface_derive_image(surface);
 621         if (image) {
 622             if (gst_vaapi_image_map(image)) {
 623                 if (upload->direct_rendering_caps == 1 &&
 624                     (gst_vaapi_image_get_format(image) == vaformat &&
 625                      gst_vaapi_image_is_linear(image) &&
 626                      (gst_vaapi_image_get_data_size(image) ==
 627                       gst_video_format_get_size(vformat, width, height))))
 628                     upload->direct_rendering_caps = 2;
 629                 gst_vaapi_image_unmap(image);
 630             }
 631             g_object_unref(image);
 632         }
 633         gst_vaapi_video_pool_put_object(upload->surfaces, surface);
 634     }
 635 }
 636
 637 typedef enum YUV_TYPE {
 638   YUV_UNKOWN = 0,
 639   YUV_411    = 1,
 640   YUV_422    = 2,
 641   YUV_444    = 4
 642 } YUV_TYPE;
 643
 644 static YUV_TYPE
 645 _image_format_to_yuv_type(guint32 fourcc)
 646 {
 647     switch (fourcc) {
 648         case GST_MAKE_FOURCC('N','V','1','2'):
 649         case GST_MAKE_FOURCC('Y','V','1','2'):
 650         case GST_MAKE_FOURCC('I','4','2','0'):
 651         case GST_MAKE_FOURCC('N','V','2','1'):
 652         return YUV_411;
 653
 654         case GST_MAKE_FOURCC('Y','U','Y','2'):
 655         case GST_MAKE_FOURCC('Y','V','Y','U'):
 656         return YUV_422;
 657
 658         case GST_MAKE_FOURCC('A','Y','U','V'):
 659         return YUV_UNKOWN;
 660
 661         default:
 662         return YUV_UNKOWN;
 663     }
 664 }
 665
 666 static GstCaps *
 667 _get_nearest_caps(GstCaps *caps_list, GstCaps *src_caps)
 668 {
 669     GstCaps *ret = NULL;
 670     GstStructure *cur_struct, *tmp_struct;
 671     guint32 cur_format, dest_format, tmp_format;
 672     YUV_TYPE cur_type, tmp_type;
 673     const GValue*tmp_val;
 674     guint  n_caps;
 675     guint  i;
 676     guint min_diff, tmp_diff;
 677
 678     cur_struct = gst_caps_get_structure(src_caps, 0);
 679     tmp_val = gst_structure_get_value (cur_struct, "format");
 680     if (!tmp_val)
 681         return NULL;
 682
 683     cur_format = gst_value_get_fourcc(tmp_val);
 684     if((cur_type = _image_format_to_yuv_type(cur_format)) == YUV_UNKOWN)
 685         return NULL;
 686
 687     n_caps = gst_caps_get_size(caps_list);
 688     min_diff = 100;
 689     dest_format = 0;
 690     for (i = 0; i < n_caps; ++i) {
 691       tmp_struct = gst_caps_get_structure(caps_list, i);
 692       tmp_val = gst_structure_get_value (tmp_struct, "format");
 693       if (!tmp_val)
 694           continue;
 695       tmp_format = gst_value_get_fourcc(tmp_val);
 696       if ((tmp_type = _image_format_to_yuv_type(tmp_format)) == YUV_UNKOWN)
 697           continue;
 698       tmp_diff = abs(tmp_type - cur_type);
 699       if (tmp_diff < min_diff) {
 700           min_diff = tmp_diff;
 701           dest_format = tmp_format;
 702       }
 703     }
 704
 705     if (dest_format == 0)
 706       return NULL;
 707
 708     ret = gst_caps_copy(src_caps);
 709     tmp_struct = gst_caps_get_structure(ret, 0);
 710     gst_structure_set(tmp_struct, "format", GST_TYPE_FOURCC, dest_format, NULL);
 711     return ret;
 712 }
 713
 714 static gboolean
 715 gst_vaapiupload_negotiate_buffers(
 716     GstVaapiUpload  *upload,
 717     GstCaps          *incaps,
 718     GstCaps          *outcaps
 719 )
 720 {
 721     guint dr;
 722     gboolean ret = TRUE;
 723     GstCaps *image_allowed_caps = NULL;
 724     GstCaps *image_caps = NULL;
 725
 726     image_allowed_caps = gst_vaapi_display_get_image_caps(upload->display);
 727     if (gst_caps_can_intersect(incaps, image_allowed_caps)) {
 728         image_caps = gst_caps_ref(incaps);
 729         upload->need_manual_upload = FALSE;
 730     } else {
 731         image_caps = _get_nearest_caps(image_allowed_caps, incaps);
 732         upload->need_manual_upload = TRUE;
 733     }
 734
 735     if (!gst_vaapiupload_ensure_image_pool(upload, image_caps))
 736         goto failed;
 737
 738     if (!gst_vaapiupload_ensure_surface_pool(upload, outcaps))
 739         goto failed;
 740
 741     gst_vaapi_video_pool_set_capacity(upload->surfaces, 6);
 742     gst_vaapi_video_pool_reserve(upload->surfaces, 4);
 743
 744     if (upload->direct_rendering && !upload->need_manual_upload)
 745       gst_vaapiupload_ensure_direct_rendering_caps(upload, incaps);
 746     dr = MIN(upload->direct_rendering, upload->direct_rendering_caps);
 747     if (upload->direct_rendering != dr) {
 748         upload->direct_rendering = dr;
 749         GST_DEBUG("direct-rendering level: %d", dr);
 750     }
 751     ret = TRUE;
 752     goto end;
 753
 754   failed:
 755     ret = FALSE;
 756
 757   end:
 758     gst_caps_unref(image_caps);
 759     gst_caps_unref(image_allowed_caps);
 760     return ret;
 761 }
 762
 763 static gboolean
 764 gst_vaapiupload_set_caps(
 765     GstBaseTransform *trans,
 766     GstCaps          *incaps,
 767     GstCaps          *outcaps
 768 )
 769 {
 770     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 771
 772     if (!gst_vaapiupload_negotiate_buffers(upload, incaps, outcaps))
 773         return FALSE;
 774
 775     GST_INFO("set caps\nIN caps:\n%" GST_PTR_FORMAT "\nOUT caps:\n%" GST_PTR_FORMAT,
 776              incaps, outcaps);
 777     return TRUE;
 778 }
 779
 780 static gboolean
 781 gst_vaapiupload_get_unit_size(
 782     GstBaseTransform *trans,
 783     GstCaps          *caps,
 784     guint            *size
 785 )
 786 {
 787     GstStructure * const structure = gst_caps_get_structure(caps, 0);
 788     GstVideoFormat format;
 789     gint width, height;
 790
 791     if (gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
 792         *size = 0;
 793     else {
 794         if (!gst_video_format_parse_caps(caps, &format, &width, &height))
 795             return FALSE;
 796         *size = gst_video_format_get_size(format, width, height);
 797     }
 798     return TRUE;
 799 }
 800
 801 static GstFlowReturn
 802 gst_vaapiupload_buffer_alloc(
 803     GstBaseTransform *trans,
 804     guint             size,
 805     GstCaps          *caps,
 806     GstBuffer       **pbuf
 807 )
 808 {
 809     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 810     GstBuffer *buffer = NULL;
 811     GstVaapiImage *image = NULL;
 812     GstVaapiSurface *surface = NULL;
 813     GstVaapiVideoBuffer *vbuffer;
 814
 815     /* already checked */
 816     if (!upload->direct_rendering)
 817         return GST_FLOW_OK;
 818
 819     /* Check if we can use direct-rendering */
 820     if (!gst_vaapiupload_negotiate_buffers(upload, caps, caps))
 821         goto error;
 822     if (!upload->direct_rendering)
 823         return GST_FLOW_OK;
 824
 825     switch (upload->direct_rendering) {
 826     case 2:
 827         buffer  = gst_vaapi_video_buffer_new_from_pool(upload->surfaces);
 828         if (!buffer)
 829             goto error;
 830         vbuffer = GST_VAAPI_VIDEO_BUFFER(buffer);
 831
 832         surface = gst_vaapi_video_buffer_get_surface(vbuffer);
 833         image   = gst_vaapi_surface_derive_image(surface);
 834         if (image && gst_vaapi_image_get_data_size(image) == size) {
 835             gst_vaapi_video_buffer_set_image(vbuffer, image);
 836             g_object_unref(image); /* video buffer owns an extra reference */
 837             break;
 838         }
 839
 840         /* We can't use the derive-image optimization. Disable it. */
 841         upload->direct_rendering = 1;
 842         gst_buffer_unref(buffer);
 843         buffer = NULL;
 844
 845     case 1:
 846         buffer  = gst_vaapi_video_buffer_new_from_pool(upload->images);
 847         if (!buffer)
 848             goto error;
 849         vbuffer = GST_VAAPI_VIDEO_BUFFER(buffer);
 850
 851         image   = gst_vaapi_video_buffer_get_image(vbuffer);
 852         break;
 853     }
 854     g_assert(image);
 855
 856     if (!gst_vaapi_image_map(image))
 857         goto error;
 858
 859     GST_BUFFER_DATA(buffer) = gst_vaapi_image_get_plane(image, 0);
 860     GST_BUFFER_SIZE(buffer) = gst_vaapi_image_get_data_size(image);
 861
 862     gst_buffer_set_caps(buffer, caps);
 863     *pbuf = buffer;
 864     return GST_FLOW_OK;
 865
 866 error:
 867     /* We can't use the inout-buffers optimization. Disable it. */
 868     GST_DEBUG("disable in/out buffer optimization");
 869     if (buffer)
 870         gst_buffer_unref(buffer);
 871     upload->direct_rendering = 0;
 872     return GST_FLOW_OK;
 873 }
 874
 875 static GstFlowReturn
 876 gst_vaapiupload_sinkpad_buffer_alloc(
 877     GstPad           *pad,
 878     guint64           offset,
 879     guint             size,
 880     GstCaps          *caps,
 881     GstBuffer       **pbuf
 882 )
 883 {
 884     GstBaseTransform *trans;
 885     GstFlowReturn ret;
 886
 887     trans = GST_BASE_TRANSFORM(gst_pad_get_parent_element(pad));
 888     if (!trans)
 889         return GST_FLOW_UNEXPECTED;
 890
 891     ret = gst_vaapiupload_buffer_alloc(trans, size, caps, pbuf);
 892     g_object_unref(trans);
 893     return ret;
 894 }
 895
 896 static GstFlowReturn
 897 gst_vaapiupload_prepare_output_buffer(
 898     GstBaseTransform *trans,
 899     GstBuffer        *inbuf,
 900     gint              size,
 901     GstCaps          *caps,
 902     GstBuffer       **poutbuf
 903 )
 904 {
 905     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 906     GstBuffer *buffer = NULL;
 907
 908     if (upload->direct_rendering == 2) {
 909         if (GST_VAAPI_IS_VIDEO_BUFFER(inbuf)) {
 910             buffer = gst_vaapi_video_buffer_new_from_buffer(inbuf);
 911             GST_BUFFER_SIZE(buffer) = size;
 912         }
 913         else {
 914             GST_DEBUG("upstream element destroyed our in/out buffer");
 915             upload->direct_rendering = 1;
 916         }
 917     }
 918
 919     if (!buffer) {
 920         buffer = gst_vaapi_video_buffer_new_from_pool(upload->surfaces);
 921         if (!buffer)
 922             return GST_FLOW_UNEXPECTED;
 923         gst_buffer_set_caps(buffer, caps);
 924     }
 925
 926     *poutbuf = buffer;
 927     return GST_FLOW_OK;
 928 }
 929
 930 static gboolean
 931 gst_vaapiupload_query(GstPad *pad, GstQuery *query)
 932 {
 933   GstVaapiUpload *upload = GST_VAAPIUPLOAD (gst_pad_get_parent_element (pad));
 934   gboolean res;
 935
 936   GST_DEBUG ("sharing display %p", upload->display);
 937
 938   if (gst_vaapi_reply_to_query (query, upload->display))
 939     res = TRUE;
 940   else
 941     res = gst_pad_query_default (pad, query);
 942
 943   g_object_unref (upload);
 944   return res;
 945 }