drivers/media/common/videobuf2/videobuf2-core.c

   1 /*
   2  * videobuf2-core.c - video buffer 2 core framework
   3  *
   4  * Copyright (C) 2010 Samsung Electronics
   5  *
   6  * Author: Pawel Osciak <pawel@osciak.com>
   7  *         Marek Szyprowski <m.szyprowski@samsung.com>
   8  *
   9  * The vb2_thread implementation was based on code from videobuf-dvb.c:
  10  *      (c) 2004 Gerd Knorr <kraxel@bytesex.org> [SUSE Labs]
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License as published by
  14  * the Free Software Foundation.
  15  */
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/err.h>
  20 #include <linux/kernel.h>
  21 #include <linux/module.h>
  22 #include <linux/mm.h>
  23 #include <linux/poll.h>
  24 #include <linux/slab.h>
  25 #include <linux/sched.h>
  26 #include <linux/freezer.h>
  27 #include <linux/kthread.h>
  28
  29 #include <media/videobuf2-core.h>
  30 #include <media/v4l2-mc.h>
  31
  32 #include <trace/events/vb2.h>
  33
  34 static int debug;
  35 module_param(debug, int, 0644);
  36
  37 #define dprintk(q, level, fmt, arg...)                                  \
  38         do {                                                            \
  39                 if (debug >= level)                                     \
  40                         pr_info("[%s] %s: " fmt, (q)->name, __func__,   \
  41                                 ## arg);                                \
  42         } while (0)
  43
  44 #ifdef CONFIG_VIDEO_ADV_DEBUG
  45
  46 /*
  47  * If advanced debugging is on, then count how often each op is called
  48  * successfully, which can either be per-buffer or per-queue.
  49  *
  50  * This makes it easy to check that the 'init' and 'cleanup'
  51  * (and variations thereof) stay balanced.
  52  */
  53
  54 #define log_memop(vb, op)                                               \
  55         dprintk((vb)->vb2_queue, 2, "call_memop(%d, %s)%s\n",           \
  56                 (vb)->index, #op,                                       \
  57                 (vb)->vb2_queue->mem_ops->op ? "" : " (nop)")
  58
  59 #define call_memop(vb, op, args...)                                     \
  60 ({                                                                      \
  61         struct vb2_queue *_q = (vb)->vb2_queue;                         \
  62         int err;                                                        \
  63                                                                         \
  64         log_memop(vb, op);                                              \
  65         err = _q->mem_ops->op ? _q->mem_ops->op(args) : 0;              \
  66         if (!err)                                                       \
  67                 (vb)->cnt_mem_ ## op++;                                 \
  68         err;                                                            \
  69 })
  70
  71 #define call_ptr_memop(op, vb, args...)                                 \
  72 ({                                                                      \
  73         struct vb2_queue *_q = (vb)->vb2_queue;                         \
  74         void *ptr;                                                      \
  75                                                                         \
  76         log_memop(vb, op);                                              \
  77         ptr = _q->mem_ops->op ? _q->mem_ops->op(vb, args) : NULL;       \
  78         if (!IS_ERR_OR_NULL(ptr))                                       \
  79                 (vb)->cnt_mem_ ## op++;                                 \
  80         ptr;                                                            \
  81 })
  82
  83 #define call_void_memop(vb, op, args...)                                \
  84 ({                                                                      \
  85         struct vb2_queue *_q = (vb)->vb2_queue;                         \
  86                                                                         \
  87         log_memop(vb, op);                                              \
  88         if (_q->mem_ops->op)                                            \
  89                 _q->mem_ops->op(args);                                  \
  90         (vb)->cnt_mem_ ## op++;                                         \
  91 })
  92
  93 #define log_qop(q, op)                                                  \
  94         dprintk(q, 2, "call_qop(%s)%s\n", #op,                          \
  95                 (q)->ops->op ? "" : " (nop)")
  96
  97 #define call_qop(q, op, args...)                                        \
  98 ({                                                                      \
  99         int err;                                                        \
 100                                                                         \
 101         log_qop(q, op);                                                 \
 102         err = (q)->ops->op ? (q)->ops->op(args) : 0;                    \
 103         if (!err)                                                       \
 104                 (q)->cnt_ ## op++;                                      \
 105         err;                                                            \
 106 })
 107
 108 #define call_void_qop(q, op, args...)                                   \
 109 ({                                                                      \
 110         log_qop(q, op);                                                 \
 111         if ((q)->ops->op)                                               \
 112                 (q)->ops->op(args);                                     \
 113         (q)->cnt_ ## op++;                                              \
 114 })
 115
 116 #define log_vb_qop(vb, op, args...)                                     \
 117         dprintk((vb)->vb2_queue, 2, "call_vb_qop(%d, %s)%s\n",          \
 118                 (vb)->index, #op,                                       \
 119                 (vb)->vb2_queue->ops->op ? "" : " (nop)")
 120
 121 #define call_vb_qop(vb, op, args...)                                    \
 122 ({                                                                      \
 123         int err;                                                        \
 124                                                                         \
 125         log_vb_qop(vb, op);                                             \
 126         err = (vb)->vb2_queue->ops->op ?                                \
 127                 (vb)->vb2_queue->ops->op(args) : 0;                     \
 128         if (!err)                                                       \
 129                 (vb)->cnt_ ## op++;                                     \
 130         err;                                                            \
 131 })
 132
 133 #define call_void_vb_qop(vb, op, args...)                               \
 134 ({                                                                      \
 135         log_vb_qop(vb, op);                                             \
 136         if ((vb)->vb2_queue->ops->op)                                   \
 137                 (vb)->vb2_queue->ops->op(args);                         \
 138         (vb)->cnt_ ## op++;                                             \
 139 })
 140
 141 #else
 142
 143 #define call_memop(vb, op, args...)                                     \
 144         ((vb)->vb2_queue->mem_ops->op ?                                 \
 145                 (vb)->vb2_queue->mem_ops->op(args) : 0)
 146
 147 #define call_ptr_memop(op, vb, args...)                                 \
 148         ((vb)->vb2_queue->mem_ops->op ?                                 \
 149                 (vb)->vb2_queue->mem_ops->op(vb, args) : NULL)
 150
 151 #define call_void_memop(vb, op, args...)                                \
 152         do {                                                            \
 153                 if ((vb)->vb2_queue->mem_ops->op)                       \
 154                         (vb)->vb2_queue->mem_ops->op(args);             \
 155         } while (0)
 156
 157 #define call_qop(q, op, args...)                                        \
 158         ((q)->ops->op ? (q)->ops->op(args) : 0)
 159
 160 #define call_void_qop(q, op, args...)                                   \
 161         do {                                                            \
 162                 if ((q)->ops->op)                                       \
 163                         (q)->ops->op(args);                             \
 164         } while (0)
 165
 166 #define call_vb_qop(vb, op, args...)                                    \
 167         ((vb)->vb2_queue->ops->op ? (vb)->vb2_queue->ops->op(args) : 0)
 168
 169 #define call_void_vb_qop(vb, op, args...)                               \
 170         do {                                                            \
 171                 if ((vb)->vb2_queue->ops->op)                           \
 172                         (vb)->vb2_queue->ops->op(args);                 \
 173         } while (0)
 174
 175 #endif
 176
 177 #define call_bufop(q, op, args...)                                      \
 178 ({                                                                      \
 179         int ret = 0;                                                    \
 180         if (q && q->buf_ops && q->buf_ops->op)                          \
 181                 ret = q->buf_ops->op(args);                             \
 182         ret;                                                            \
 183 })
 184
 185 #define call_void_bufop(q, op, args...)                                 \
 186 ({                                                                      \
 187         if (q && q->buf_ops && q->buf_ops->op)                          \
 188                 q->buf_ops->op(args);                                   \
 189 })
 190
 191 static void __vb2_queue_cancel(struct vb2_queue *q);
 192 static void __enqueue_in_driver(struct vb2_buffer *vb);
 193
 194 static const char *vb2_state_name(enum vb2_buffer_state s)
 195 {
 196         static const char * const state_names[] = {
 197                 [VB2_BUF_STATE_DEQUEUED] = "dequeued",
 198                 [VB2_BUF_STATE_IN_REQUEST] = "in request",
 199                 [VB2_BUF_STATE_PREPARING] = "preparing",
 200                 [VB2_BUF_STATE_QUEUED] = "queued",
 201                 [VB2_BUF_STATE_ACTIVE] = "active",
 202                 [VB2_BUF_STATE_DONE] = "done",
 203                 [VB2_BUF_STATE_ERROR] = "error",
 204         };
 205
 206         if ((unsigned int)(s) < ARRAY_SIZE(state_names))
 207                 return state_names[s];
 208         return "unknown";
 209 }
 210
 211 /*
 212  * __vb2_buf_mem_alloc() - allocate video memory for the given buffer
 213  */
 214 static int __vb2_buf_mem_alloc(struct vb2_buffer *vb)
 215 {
 216         struct vb2_queue *q = vb->vb2_queue;
 217         void *mem_priv;
 218         int plane;
 219         int ret = -ENOMEM;
 220
 221         /*
 222          * Allocate memory for all planes in this buffer
 223          * NOTE: mmapped areas should be page aligned
 224          */
 225         for (plane = 0; plane < vb->num_planes; ++plane) {
 226                 /* Memops alloc requires size to be page aligned. */
 227                 unsigned long size = PAGE_ALIGN(vb->planes[plane].length);
 228
 229                 /* Did it wrap around? */
 230                 if (size < vb->planes[plane].length)
 231                         goto free;
 232
 233                 mem_priv = call_ptr_memop(alloc,
 234                                           vb,
 235                                           q->alloc_devs[plane] ? : q->dev,
 236                                           size);
 237                 if (IS_ERR_OR_NULL(mem_priv)) {
 238                         if (mem_priv)
 239                                 ret = PTR_ERR(mem_priv);
 240                         goto free;
 241                 }
 242
 243                 /* Associate allocator private data with this plane */
 244                 vb->planes[plane].mem_priv = mem_priv;
 245         }
 246
 247         return 0;
 248 free:
 249         /* Free already allocated memory if one of the allocations failed */
 250         for (; plane > 0; --plane) {
 251                 call_void_memop(vb, put, vb->planes[plane - 1].mem_priv);
 252                 vb->planes[plane - 1].mem_priv = NULL;
 253         }
 254
 255         return ret;
 256 }
 257
 258 /*
 259  * __vb2_buf_mem_free() - free memory of the given buffer
 260  */
 261 static void __vb2_buf_mem_free(struct vb2_buffer *vb)
 262 {
 263         unsigned int plane;
 264
 265         for (plane = 0; plane < vb->num_planes; ++plane) {
 266                 call_void_memop(vb, put, vb->planes[plane].mem_priv);
 267                 vb->planes[plane].mem_priv = NULL;
 268                 dprintk(vb->vb2_queue, 3, "freed plane %d of buffer %d\n",
 269                         plane, vb->index);
 270         }
 271 }
 272
 273 /*
 274  * __vb2_buf_userptr_put() - release userspace memory associated with
 275  * a USERPTR buffer
 276  */
 277 static void __vb2_buf_userptr_put(struct vb2_buffer *vb)
 278 {
 279         unsigned int plane;
 280
 281         for (plane = 0; plane < vb->num_planes; ++plane) {
 282                 if (vb->planes[plane].mem_priv)
 283                         call_void_memop(vb, put_userptr, vb->planes[plane].mem_priv);
 284                 vb->planes[plane].mem_priv = NULL;
 285         }
 286 }
 287
 288 /*
 289  * __vb2_plane_dmabuf_put() - release memory associated with
 290  * a DMABUF shared plane
 291  */
 292 static void __vb2_plane_dmabuf_put(struct vb2_buffer *vb, struct vb2_plane *p)
 293 {
 294         if (!p->mem_priv)
 295                 return;
 296
 297         if (p->dbuf_mapped)
 298                 call_void_memop(vb, unmap_dmabuf, p->mem_priv);
 299
 300         call_void_memop(vb, detach_dmabuf, p->mem_priv);
 301         dma_buf_put(p->dbuf);
 302         p->mem_priv = NULL;
 303         p->dbuf = NULL;
 304         p->dbuf_mapped = 0;
 305 }
 306
 307 /*
 308  * __vb2_buf_dmabuf_put() - release memory associated with
 309  * a DMABUF shared buffer
 310  */
 311 static void __vb2_buf_dmabuf_put(struct vb2_buffer *vb)
 312 {
 313         unsigned int plane;
 314
 315         for (plane = 0; plane < vb->num_planes; ++plane)
 316                 __vb2_plane_dmabuf_put(vb, &vb->planes[plane]);
 317 }
 318
 319 /*
 320  * __vb2_buf_mem_prepare() - call ->prepare() on buffer's private memory
 321  * to sync caches
 322  */
 323 static void __vb2_buf_mem_prepare(struct vb2_buffer *vb)
 324 {
 325         unsigned int plane;
 326
 327         if (vb->synced)
 328                 return;
 329
 330         if (vb->need_cache_sync_on_prepare) {
 331                 for (plane = 0; plane < vb->num_planes; ++plane)
 332                         call_void_memop(vb, prepare,
 333                                         vb->planes[plane].mem_priv);
 334         }
 335         vb->synced = 1;
 336 }
 337
 338 /*
 339  * __vb2_buf_mem_finish() - call ->finish on buffer's private memory
 340  * to sync caches
 341  */
 342 static void __vb2_buf_mem_finish(struct vb2_buffer *vb)
 343 {
 344         unsigned int plane;
 345
 346         if (!vb->synced)
 347                 return;
 348
 349         if (vb->need_cache_sync_on_finish) {
 350                 for (plane = 0; plane < vb->num_planes; ++plane)
 351                         call_void_memop(vb, finish,
 352                                         vb->planes[plane].mem_priv);
 353         }
 354         vb->synced = 0;
 355 }
 356
 357 /*
 358  * __setup_offsets() - setup unique offsets ("cookies") for every plane in
 359  * the buffer.
 360  */
 361 static void __setup_offsets(struct vb2_buffer *vb)
 362 {
 363         struct vb2_queue *q = vb->vb2_queue;
 364         unsigned int plane;
 365         unsigned long off = 0;
 366
 367         if (vb->index) {
 368                 struct vb2_buffer *prev = q->bufs[vb->index - 1];
 369                 struct vb2_plane *p = &prev->planes[prev->num_planes - 1];
 370
 371                 off = PAGE_ALIGN(p->m.offset + p->length);
 372         }
 373
 374         for (plane = 0; plane < vb->num_planes; ++plane) {
 375                 vb->planes[plane].m.offset = off;
 376
 377                 dprintk(q, 3, "buffer %d, plane %d offset 0x%08lx\n",
 378                                 vb->index, plane, off);
 379
 380                 off += vb->planes[plane].length;
 381                 off = PAGE_ALIGN(off);
 382         }
 383 }
 384
 385 /*
 386  * __vb2_queue_alloc() - allocate videobuf buffer structures and (for MMAP type)
 387  * video buffer memory for all buffers/planes on the queue and initializes the
 388  * queue
 389  *
 390  * Returns the number of buffers successfully allocated.
 391  */
 392 static int __vb2_queue_alloc(struct vb2_queue *q, enum vb2_memory memory,
 393                              unsigned int num_buffers, unsigned int num_planes,
 394                              const unsigned plane_sizes[VB2_MAX_PLANES])
 395 {
 396         unsigned int buffer, plane;
 397         struct vb2_buffer *vb;
 398         int ret;
 399
 400         /* Ensure that q->num_buffers+num_buffers is below VB2_MAX_FRAME */
 401         num_buffers = min_t(unsigned int, num_buffers,
 402                             VB2_MAX_FRAME - q->num_buffers);
 403
 404         for (buffer = 0; buffer < num_buffers; ++buffer) {
 405                 /* Allocate videobuf buffer structures */
 406                 vb = kzalloc(q->buf_struct_size, GFP_KERNEL);
 407                 if (!vb) {
 408                         dprintk(q, 1, "memory alloc for buffer struct failed\n");
 409                         break;
 410                 }
 411
 412                 vb->state = VB2_BUF_STATE_DEQUEUED;
 413                 vb->vb2_queue = q;
 414                 vb->num_planes = num_planes;
 415                 vb->index = q->num_buffers + buffer;
 416                 vb->type = q->type;
 417                 vb->memory = memory;
 418                 /*
 419                  * We need to set these flags here so that the videobuf2 core
 420                  * will call ->prepare()/->finish() cache sync/flush on vb2
 421                  * buffers when appropriate. However, we can avoid explicit
 422                  * ->prepare() and ->finish() cache sync for DMABUF buffers,
 423                  * because DMA exporter takes care of it.
 424                  */
 425                 if (q->memory != VB2_MEMORY_DMABUF) {
 426                         vb->need_cache_sync_on_prepare = 1;
 427                         vb->need_cache_sync_on_finish = 1;
 428                 }
 429                 for (plane = 0; plane < num_planes; ++plane) {
 430                         vb->planes[plane].length = plane_sizes[plane];
 431                         vb->planes[plane].min_length = plane_sizes[plane];
 432                 }
 433                 call_void_bufop(q, init_buffer, vb);
 434
 435                 q->bufs[vb->index] = vb;
 436
 437                 /* Allocate video buffer memory for the MMAP type */
 438                 if (memory == VB2_MEMORY_MMAP) {
 439                         ret = __vb2_buf_mem_alloc(vb);
 440                         if (ret) {
 441                                 dprintk(q, 1, "failed allocating memory for buffer %d\n",
 442                                         buffer);
 443                                 q->bufs[vb->index] = NULL;
 444                                 kfree(vb);
 445                                 break;
 446                         }
 447                         __setup_offsets(vb);
 448                         /*
 449                          * Call the driver-provided buffer initialization
 450                          * callback, if given. An error in initialization
 451                          * results in queue setup failure.
 452                          */
 453                         ret = call_vb_qop(vb, buf_init, vb);
 454                         if (ret) {
 455                                 dprintk(q, 1, "buffer %d %p initialization failed\n",
 456                                         buffer, vb);
 457                                 __vb2_buf_mem_free(vb);
 458                                 q->bufs[vb->index] = NULL;
 459                                 kfree(vb);
 460                                 break;
 461                         }
 462                 }
 463         }
 464
 465         dprintk(q, 3, "allocated %d buffers, %d plane(s) each\n",
 466                 buffer, num_planes);
 467
 468         return buffer;
 469 }
 470
 471 /*
 472  * __vb2_free_mem() - release all video buffer memory for a given queue
 473  */
 474 static void __vb2_free_mem(struct vb2_queue *q, unsigned int buffers)
 475 {
 476         unsigned int buffer;
 477         struct vb2_buffer *vb;
 478
 479         for (buffer = q->num_buffers - buffers; buffer < q->num_buffers;
 480              ++buffer) {
 481                 vb = q->bufs[buffer];
 482                 if (!vb)
 483                         continue;
 484
 485                 /* Free MMAP buffers or release USERPTR buffers */
 486                 if (q->memory == VB2_MEMORY_MMAP)
 487                         __vb2_buf_mem_free(vb);
 488                 else if (q->memory == VB2_MEMORY_DMABUF)
 489                         __vb2_buf_dmabuf_put(vb);
 490                 else
 491                         __vb2_buf_userptr_put(vb);
 492         }
 493 }
 494
 495 /*
 496  * __vb2_queue_free() - free buffers at the end of the queue - video memory and
 497  * related information, if no buffers are left return the queue to an
 498  * uninitialized state. Might be called even if the queue has already been freed.
 499  */
 500 static int __vb2_queue_free(struct vb2_queue *q, unsigned int buffers)
 501 {
 502         unsigned int buffer;
 503
 504         /*
 505          * Sanity check: when preparing a buffer the queue lock is released for
 506          * a short while (see __buf_prepare for the details), which would allow
 507          * a race with a reqbufs which can call this function. Removing the
 508          * buffers from underneath __buf_prepare is obviously a bad idea, so we
 509          * check if any of the buffers is in the state PREPARING, and if so we
 510          * just return -EAGAIN.
 511          */
 512         for (buffer = q->num_buffers - buffers; buffer < q->num_buffers;
 513              ++buffer) {
 514                 if (q->bufs[buffer] == NULL)
 515                         continue;
 516                 if (q->bufs[buffer]->state == VB2_BUF_STATE_PREPARING) {
 517                         dprintk(q, 1, "preparing buffers, cannot free\n");
 518                         return -EAGAIN;
 519                 }
 520         }
 521
 522         /* Call driver-provided cleanup function for each buffer, if provided */
 523         for (buffer = q->num_buffers - buffers; buffer < q->num_buffers;
 524              ++buffer) {
 525                 struct vb2_buffer *vb = q->bufs[buffer];
 526
 527                 if (vb && vb->planes[0].mem_priv)
 528                         call_void_vb_qop(vb, buf_cleanup, vb);
 529         }
 530
 531         /* Release video buffer memory */
 532         __vb2_free_mem(q, buffers);
 533
 534 #ifdef CONFIG_VIDEO_ADV_DEBUG
 535         /*
 536          * Check that all the calls were balances during the life-time of this
 537          * queue. If not (or if the debug level is 1 or up), then dump the
 538          * counters to the kernel log.
 539          */
 540         if (q->num_buffers) {
 541                 bool unbalanced = q->cnt_start_streaming != q->cnt_stop_streaming ||
 542                                   q->cnt_wait_prepare != q->cnt_wait_finish;
 543
 544                 if (unbalanced || debug) {
 545                         pr_info("counters for queue %p:%s\n", q,
 546                                 unbalanced ? " UNBALANCED!" : "");
 547                         pr_info("     setup: %u start_streaming: %u stop_streaming: %u\n",
 548                                 q->cnt_queue_setup, q->cnt_start_streaming,
 549                                 q->cnt_stop_streaming);
 550                         pr_info("     wait_prepare: %u wait_finish: %u\n",
 551                                 q->cnt_wait_prepare, q->cnt_wait_finish);
 552                 }
 553                 q->cnt_queue_setup = 0;
 554                 q->cnt_wait_prepare = 0;
 555                 q->cnt_wait_finish = 0;
 556                 q->cnt_start_streaming = 0;
 557                 q->cnt_stop_streaming = 0;
 558         }
 559         for (buffer = 0; buffer < q->num_buffers; ++buffer) {
 560                 struct vb2_buffer *vb = q->bufs[buffer];
 561                 bool unbalanced = vb->cnt_mem_alloc != vb->cnt_mem_put ||
 562                                   vb->cnt_mem_prepare != vb->cnt_mem_finish ||
 563                                   vb->cnt_mem_get_userptr != vb->cnt_mem_put_userptr ||
 564                                   vb->cnt_mem_attach_dmabuf != vb->cnt_mem_detach_dmabuf ||
 565                                   vb->cnt_mem_map_dmabuf != vb->cnt_mem_unmap_dmabuf ||
 566                                   vb->cnt_buf_queue != vb->cnt_buf_done ||
 567                                   vb->cnt_buf_prepare != vb->cnt_buf_finish ||
 568                                   vb->cnt_buf_init != vb->cnt_buf_cleanup;
 569
 570                 if (unbalanced || debug) {
 571                         pr_info("   counters for queue %p, buffer %d:%s\n",
 572                                 q, buffer, unbalanced ? " UNBALANCED!" : "");
 573                         pr_info("     buf_init: %u buf_cleanup: %u buf_prepare: %u buf_finish: %u\n",
 574                                 vb->cnt_buf_init, vb->cnt_buf_cleanup,
 575                                 vb->cnt_buf_prepare, vb->cnt_buf_finish);
 576                         pr_info("     buf_out_validate: %u buf_queue: %u buf_done: %u buf_request_complete: %u\n",
 577                                 vb->cnt_buf_out_validate, vb->cnt_buf_queue,
 578                                 vb->cnt_buf_done, vb->cnt_buf_request_complete);
 579                         pr_info("     alloc: %u put: %u prepare: %u finish: %u mmap: %u\n",
 580                                 vb->cnt_mem_alloc, vb->cnt_mem_put,
 581                                 vb->cnt_mem_prepare, vb->cnt_mem_finish,
 582                                 vb->cnt_mem_mmap);
 583                         pr_info("     get_userptr: %u put_userptr: %u\n",
 584                                 vb->cnt_mem_get_userptr, vb->cnt_mem_put_userptr);
 585                         pr_info("     attach_dmabuf: %u detach_dmabuf: %u map_dmabuf: %u unmap_dmabuf: %u\n",
 586                                 vb->cnt_mem_attach_dmabuf, vb->cnt_mem_detach_dmabuf,
 587                                 vb->cnt_mem_map_dmabuf, vb->cnt_mem_unmap_dmabuf);
 588                         pr_info("     get_dmabuf: %u num_users: %u vaddr: %u cookie: %u\n",
 589                                 vb->cnt_mem_get_dmabuf,
 590                                 vb->cnt_mem_num_users,
 591                                 vb->cnt_mem_vaddr,
 592                                 vb->cnt_mem_cookie);
 593                 }
 594         }
 595 #endif
 596
 597         /* Free videobuf buffers */
 598         for (buffer = q->num_buffers - buffers; buffer < q->num_buffers;
 599              ++buffer) {
 600                 kfree(q->bufs[buffer]);
 601                 q->bufs[buffer] = NULL;
 602         }
 603
 604         q->num_buffers -= buffers;
 605         if (!q->num_buffers) {
 606                 q->memory = VB2_MEMORY_UNKNOWN;
 607                 INIT_LIST_HEAD(&q->queued_list);
 608         }
 609         return 0;
 610 }
 611
 612 bool vb2_buffer_in_use(struct vb2_queue *q, struct vb2_buffer *vb)
 613 {
 614         unsigned int plane;
 615         for (plane = 0; plane < vb->num_planes; ++plane) {
 616                 void *mem_priv = vb->planes[plane].mem_priv;
 617                 /*
 618                  * If num_users() has not been provided, call_memop
 619                  * will return 0, apparently nobody cares about this
 620                  * case anyway. If num_users() returns more than 1,
 621                  * we are not the only user of the plane's memory.
 622                  */
 623                 if (mem_priv && call_memop(vb, num_users, mem_priv) > 1)
 624                         return true;
 625         }
 626         return false;
 627 }
 628 EXPORT_SYMBOL(vb2_buffer_in_use);
 629
 630 /*
 631  * __buffers_in_use() - return true if any buffers on the queue are in use and
 632  * the queue cannot be freed (by the means of REQBUFS(0)) call
 633  */
 634 static bool __buffers_in_use(struct vb2_queue *q)
 635 {
 636         unsigned int buffer;
 637         for (buffer = 0; buffer < q->num_buffers; ++buffer) {
 638                 if (vb2_buffer_in_use(q, q->bufs[buffer]))
 639                         return true;
 640         }
 641         return false;
 642 }
 643
 644 void vb2_core_querybuf(struct vb2_queue *q, unsigned int index, void *pb)
 645 {
 646         call_void_bufop(q, fill_user_buffer, q->bufs[index], pb);
 647 }
 648 EXPORT_SYMBOL_GPL(vb2_core_querybuf);
 649
 650 /*
 651  * __verify_userptr_ops() - verify that all memory operations required for
 652  * USERPTR queue type have been provided
 653  */
 654 static int __verify_userptr_ops(struct vb2_queue *q)
 655 {
 656         if (!(q->io_modes & VB2_USERPTR) || !q->mem_ops->get_userptr ||
 657             !q->mem_ops->put_userptr)
 658                 return -EINVAL;
 659
 660         return 0;
 661 }
 662
 663 /*
 664  * __verify_mmap_ops() - verify that all memory operations required for
 665  * MMAP queue type have been provided
 666  */
 667 static int __verify_mmap_ops(struct vb2_queue *q)
 668 {
 669         if (!(q->io_modes & VB2_MMAP) || !q->mem_ops->alloc ||
 670             !q->mem_ops->put || !q->mem_ops->mmap)
 671                 return -EINVAL;
 672
 673         return 0;
 674 }
 675
 676 /*
 677  * __verify_dmabuf_ops() - verify that all memory operations required for
 678  * DMABUF queue type have been provided
 679  */
 680 static int __verify_dmabuf_ops(struct vb2_queue *q)
 681 {
 682         if (!(q->io_modes & VB2_DMABUF) || !q->mem_ops->attach_dmabuf ||
 683             !q->mem_ops->detach_dmabuf  || !q->mem_ops->map_dmabuf ||
 684             !q->mem_ops->unmap_dmabuf)
 685                 return -EINVAL;
 686
 687         return 0;
 688 }
 689
 690 int vb2_verify_memory_type(struct vb2_queue *q,
 691                 enum vb2_memory memory, unsigned int type)
 692 {
 693         if (memory != VB2_MEMORY_MMAP && memory != VB2_MEMORY_USERPTR &&
 694             memory != VB2_MEMORY_DMABUF) {
 695                 dprintk(q, 1, "unsupported memory type\n");
 696                 return -EINVAL;
 697         }
 698
 699         if (type != q->type) {
 700                 dprintk(q, 1, "requested type is incorrect\n");
 701                 return -EINVAL;
 702         }
 703
 704         /*
 705          * Make sure all the required memory ops for given memory type
 706          * are available.
 707          */
 708         if (memory == VB2_MEMORY_MMAP && __verify_mmap_ops(q)) {
 709                 dprintk(q, 1, "MMAP for current setup unsupported\n");
 710                 return -EINVAL;
 711         }
 712
 713         if (memory == VB2_MEMORY_USERPTR && __verify_userptr_ops(q)) {
 714                 dprintk(q, 1, "USERPTR for current setup unsupported\n");
 715                 return -EINVAL;
 716         }
 717
 718         if (memory == VB2_MEMORY_DMABUF && __verify_dmabuf_ops(q)) {
 719                 dprintk(q, 1, "DMABUF for current setup unsupported\n");
 720                 return -EINVAL;
 721         }
 722
 723         /*
 724          * Place the busy tests at the end: -EBUSY can be ignored when
 725          * create_bufs is called with count == 0, but count == 0 should still
 726          * do the memory and type validation.
 727          */
 728         if (vb2_fileio_is_active(q)) {
 729                 dprintk(q, 1, "file io in progress\n");
 730                 return -EBUSY;
 731         }
 732         return 0;
 733 }
 734 EXPORT_SYMBOL(vb2_verify_memory_type);
 735
 736 int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory,
 737                      unsigned int *count)
 738 {
 739         unsigned int num_buffers, allocated_buffers, num_planes = 0;
 740         unsigned plane_sizes[VB2_MAX_PLANES] = { };
 741         unsigned int i;
 742         int ret;
 743
 744         if (q->streaming) {
 745                 dprintk(q, 1, "streaming active\n");
 746                 return -EBUSY;
 747         }
 748
 749         if (q->waiting_in_dqbuf && *count) {
 750                 dprintk(q, 1, "another dup()ped fd is waiting for a buffer\n");
 751                 return -EBUSY;
 752         }
 753
 754         if (*count == 0 || q->num_buffers != 0 ||
 755             (q->memory != VB2_MEMORY_UNKNOWN && q->memory != memory)) {
 756                 /*
 757                  * We already have buffers allocated, so first check if they
 758                  * are not in use and can be freed.
 759                  */
 760                 mutex_lock(&q->mmap_lock);
 761                 if (debug && q->memory == VB2_MEMORY_MMAP &&
 762                     __buffers_in_use(q))
 763                         dprintk(q, 1, "memory in use, orphaning buffers\n");
 764
 765                 /*
 766                  * Call queue_cancel to clean up any buffers in the
 767                  * QUEUED state which is possible if buffers were prepared or
 768                  * queued without ever calling STREAMON.
 769                  */
 770                 __vb2_queue_cancel(q);
 771                 ret = __vb2_queue_free(q, q->num_buffers);
 772                 mutex_unlock(&q->mmap_lock);
 773                 if (ret)
 774                         return ret;
 775
 776                 /*
 777                  * In case of REQBUFS(0) return immediately without calling
 778                  * driver's queue_setup() callback and allocating resources.
 779                  */
 780                 if (*count == 0)
 781                         return 0;
 782         }
 783
 784         /*
 785          * Make sure the requested values and current defaults are sane.
 786          */
 787         WARN_ON(q->min_buffers_needed > VB2_MAX_FRAME);
 788         num_buffers = max_t(unsigned int, *count, q->min_buffers_needed);
 789         num_buffers = min_t(unsigned int, num_buffers, VB2_MAX_FRAME);
 790         memset(q->alloc_devs, 0, sizeof(q->alloc_devs));
 791         /*
 792          * Set this now to ensure that drivers see the correct q->memory value
 793          * in the queue_setup op.
 794          */
 795         mutex_lock(&q->mmap_lock);
 796         q->memory = memory;
 797         mutex_unlock(&q->mmap_lock);
 798
 799         /*
 800          * Ask the driver how many buffers and planes per buffer it requires.
 801          * Driver also sets the size and allocator context for each plane.
 802          */
 803         ret = call_qop(q, queue_setup, q, &num_buffers, &num_planes,
 804                        plane_sizes, q->alloc_devs);
 805         if (ret)
 806                 goto error;
 807
 808         /* Check that driver has set sane values */
 809         if (WARN_ON(!num_planes)) {
 810                 ret = -EINVAL;
 811                 goto error;
 812         }
 813
 814         for (i = 0; i < num_planes; i++)
 815                 if (WARN_ON(!plane_sizes[i])) {
 816                         ret = -EINVAL;
 817                         goto error;
 818                 }
 819
 820         /* Finally, allocate buffers and video memory */
 821         allocated_buffers =
 822                 __vb2_queue_alloc(q, memory, num_buffers, num_planes, plane_sizes);
 823         if (allocated_buffers == 0) {
 824                 dprintk(q, 1, "memory allocation failed\n");
 825                 ret = -ENOMEM;
 826                 goto error;
 827         }
 828
 829         /*
 830          * There is no point in continuing if we can't allocate the minimum
 831          * number of buffers needed by this vb2_queue.
 832          */
 833         if (allocated_buffers < q->min_buffers_needed)
 834                 ret = -ENOMEM;
 835
 836         /*
 837          * Check if driver can handle the allocated number of buffers.
 838          */
 839         if (!ret && allocated_buffers < num_buffers) {
 840                 num_buffers = allocated_buffers;
 841                 /*
 842                  * num_planes is set by the previous queue_setup(), but since it
 843                  * signals to queue_setup() whether it is called from create_bufs()
 844                  * vs reqbufs() we zero it here to signal that queue_setup() is
 845                  * called for the reqbufs() case.
 846                  */
 847                 num_planes = 0;
 848
 849                 ret = call_qop(q, queue_setup, q, &num_buffers,
 850                                &num_planes, plane_sizes, q->alloc_devs);
 851
 852                 if (!ret && allocated_buffers < num_buffers)
 853                         ret = -ENOMEM;
 854
 855                 /*
 856                  * Either the driver has accepted a smaller number of buffers,
 857                  * or .queue_setup() returned an error
 858                  */
 859         }
 860
 861         mutex_lock(&q->mmap_lock);
 862         q->num_buffers = allocated_buffers;
 863
 864         if (ret < 0) {
 865                 /*
 866                  * Note: __vb2_queue_free() will subtract 'allocated_buffers'
 867                  * from q->num_buffers and it will reset q->memory to
 868                  * VB2_MEMORY_UNKNOWN.
 869                  */
 870                 __vb2_queue_free(q, allocated_buffers);
 871                 mutex_unlock(&q->mmap_lock);
 872                 return ret;
 873         }
 874         mutex_unlock(&q->mmap_lock);
 875
 876         /*
 877          * Return the number of successfully allocated buffers
 878          * to the userspace.
 879          */
 880         *count = allocated_buffers;
 881         q->waiting_for_buffers = !q->is_output;
 882
 883         return 0;
 884
 885 error:
 886         mutex_lock(&q->mmap_lock);
 887         q->memory = VB2_MEMORY_UNKNOWN;
 888         mutex_unlock(&q->mmap_lock);
 889         return ret;
 890 }
 891 EXPORT_SYMBOL_GPL(vb2_core_reqbufs);
 892
 893 int vb2_core_create_bufs(struct vb2_queue *q, enum vb2_memory memory,
 894                          unsigned int *count,
 895                          unsigned int requested_planes,
 896                          const unsigned int requested_sizes[])
 897 {
 898         unsigned int num_planes = 0, num_buffers, allocated_buffers;
 899         unsigned plane_sizes[VB2_MAX_PLANES] = { };
 900         bool no_previous_buffers = !q->num_buffers;
 901         int ret;
 902
 903         if (q->num_buffers == VB2_MAX_FRAME) {
 904                 dprintk(q, 1, "maximum number of buffers already allocated\n");
 905                 return -ENOBUFS;
 906         }
 907
 908         if (no_previous_buffers) {
 909                 if (q->waiting_in_dqbuf && *count) {
 910                         dprintk(q, 1, "another dup()ped fd is waiting for a buffer\n");
 911                         return -EBUSY;
 912                 }
 913                 memset(q->alloc_devs, 0, sizeof(q->alloc_devs));
 914                 /*
 915                  * Set this now to ensure that drivers see the correct q->memory
 916                  * value in the queue_setup op.
 917                  */
 918                 mutex_lock(&q->mmap_lock);
 919                 q->memory = memory;
 920                 mutex_unlock(&q->mmap_lock);
 921                 q->waiting_for_buffers = !q->is_output;
 922         } else {
 923                 if (q->memory != memory) {
 924                         dprintk(q, 1, "memory model mismatch\n");
 925                         return -EINVAL;
 926                 }
 927         }
 928
 929         num_buffers = min(*count, VB2_MAX_FRAME - q->num_buffers);
 930
 931         if (requested_planes && requested_sizes) {
 932                 num_planes = requested_planes;
 933                 memcpy(plane_sizes, requested_sizes, sizeof(plane_sizes));
 934         }
 935
 936         /*
 937          * Ask the driver, whether the requested number of buffers, planes per
 938          * buffer and their sizes are acceptable
 939          */
 940         ret = call_qop(q, queue_setup, q, &num_buffers,
 941                        &num_planes, plane_sizes, q->alloc_devs);
 942         if (ret)
 943                 goto error;
 944
 945         /* Finally, allocate buffers and video memory */
 946         allocated_buffers = __vb2_queue_alloc(q, memory, num_buffers,
 947                                 num_planes, plane_sizes);
 948         if (allocated_buffers == 0) {
 949                 dprintk(q, 1, "memory allocation failed\n");
 950                 ret = -ENOMEM;
 951                 goto error;
 952         }
 953
 954         /*
 955          * Check if driver can handle the so far allocated number of buffers.
 956          */
 957         if (allocated_buffers < num_buffers) {
 958                 num_buffers = allocated_buffers;
 959
 960                 /*
 961                  * q->num_buffers contains the total number of buffers, that the
 962                  * queue driver has set up
 963                  */
 964                 ret = call_qop(q, queue_setup, q, &num_buffers,
 965                                &num_planes, plane_sizes, q->alloc_devs);
 966
 967                 if (!ret && allocated_buffers < num_buffers)
 968                         ret = -ENOMEM;
 969
 970                 /*
 971                  * Either the driver has accepted a smaller number of buffers,
 972                  * or .queue_setup() returned an error
 973                  */
 974         }
 975
 976         mutex_lock(&q->mmap_lock);
 977         q->num_buffers += allocated_buffers;
 978
 979         if (ret < 0) {
 980                 /*
 981                  * Note: __vb2_queue_free() will subtract 'allocated_buffers'
 982                  * from q->num_buffers and it will reset q->memory to
 983                  * VB2_MEMORY_UNKNOWN.
 984                  */
 985                 __vb2_queue_free(q, allocated_buffers);
 986                 mutex_unlock(&q->mmap_lock);
 987                 return -ENOMEM;
 988         }
 989         mutex_unlock(&q->mmap_lock);
 990
 991         /*
 992          * Return the number of successfully allocated buffers
 993          * to the userspace.
 994          */
 995         *count = allocated_buffers;
 996
 997         return 0;
 998
 999 error:
1000         if (no_previous_buffers) {
1001                 mutex_lock(&q->mmap_lock);
1002                 q->memory = VB2_MEMORY_UNKNOWN;
1003                 mutex_unlock(&q->mmap_lock);
1004         }
1005         return ret;
1006 }
1007 EXPORT_SYMBOL_GPL(vb2_core_create_bufs);
1008
1009 void *vb2_plane_vaddr(struct vb2_buffer *vb, unsigned int plane_no)
1010 {
1011         if (plane_no >= vb->num_planes || !vb->planes[plane_no].mem_priv)
1012                 return NULL;
1013
1014         return call_ptr_memop(vaddr, vb, vb->planes[plane_no].mem_priv);
1015
1016 }
1017 EXPORT_SYMBOL_GPL(vb2_plane_vaddr);
1018
1019 void *vb2_plane_cookie(struct vb2_buffer *vb, unsigned int plane_no)
1020 {
1021         if (plane_no >= vb->num_planes || !vb->planes[plane_no].mem_priv)
1022                 return NULL;
1023
1024         return call_ptr_memop(cookie, vb, vb->planes[plane_no].mem_priv);
1025 }
1026 EXPORT_SYMBOL_GPL(vb2_plane_cookie);
1027
1028 void vb2_buffer_done(struct vb2_buffer *vb, enum vb2_buffer_state state)
1029 {
1030         struct vb2_queue *q = vb->vb2_queue;
1031         unsigned long flags;
1032
1033         if (WARN_ON(vb->state != VB2_BUF_STATE_ACTIVE))
1034                 return;
1035
1036         if (WARN_ON(state != VB2_BUF_STATE_DONE &&
1037                     state != VB2_BUF_STATE_ERROR &&
1038                     state != VB2_BUF_STATE_QUEUED))
1039                 state = VB2_BUF_STATE_ERROR;
1040
1041 #ifdef CONFIG_VIDEO_ADV_DEBUG
1042         /*
1043          * Although this is not a callback, it still does have to balance
1044          * with the buf_queue op. So update this counter manually.
1045          */
1046         vb->cnt_buf_done++;
1047 #endif
1048         dprintk(q, 4, "done processing on buffer %d, state: %s\n",
1049                 vb->index, vb2_state_name(state));
1050
1051         if (state != VB2_BUF_STATE_QUEUED)
1052                 __vb2_buf_mem_finish(vb);
1053
1054         spin_lock_irqsave(&q->done_lock, flags);
1055         if (state == VB2_BUF_STATE_QUEUED) {
1056                 vb->state = VB2_BUF_STATE_QUEUED;
1057         } else {
1058                 /* Add the buffer to the done buffers list */
1059                 list_add_tail(&vb->done_entry, &q->done_list);
1060                 vb->state = state;
1061         }
1062         atomic_dec(&q->owned_by_drv_count);
1063
1064         if (state != VB2_BUF_STATE_QUEUED && vb->req_obj.req) {
1065                 media_request_object_unbind(&vb->req_obj);
1066                 media_request_object_put(&vb->req_obj);
1067         }
1068
1069         spin_unlock_irqrestore(&q->done_lock, flags);
1070
1071         trace_vb2_buf_done(q, vb);
1072
1073         switch (state) {
1074         case VB2_BUF_STATE_QUEUED:
1075                 return;
1076         default:
1077                 /* Inform any processes that may be waiting for buffers */
1078                 wake_up(&q->done_wq);
1079                 break;
1080         }
1081 }
1082 EXPORT_SYMBOL_GPL(vb2_buffer_done);
1083
1084 void vb2_discard_done(struct vb2_queue *q)
1085 {
1086         struct vb2_buffer *vb;
1087         unsigned long flags;
1088
1089         spin_lock_irqsave(&q->done_lock, flags);
1090         list_for_each_entry(vb, &q->done_list, done_entry)
1091                 vb->state = VB2_BUF_STATE_ERROR;
1092         spin_unlock_irqrestore(&q->done_lock, flags);
1093 }
1094 EXPORT_SYMBOL_GPL(vb2_discard_done);
1095
1096 /*
1097  * __prepare_mmap() - prepare an MMAP buffer
1098  */
1099 static int __prepare_mmap(struct vb2_buffer *vb)
1100 {
1101         int ret = 0;
1102
1103         ret = call_bufop(vb->vb2_queue, fill_vb2_buffer,
1104                          vb, vb->planes);
1105         return ret ? ret : call_vb_qop(vb, buf_prepare, vb);
1106 }
1107
1108 /*
1109  * __prepare_userptr() - prepare a USERPTR buffer
1110  */
1111 static int __prepare_userptr(struct vb2_buffer *vb)
1112 {
1113         struct vb2_plane planes[VB2_MAX_PLANES];
1114         struct vb2_queue *q = vb->vb2_queue;
1115         void *mem_priv;
1116         unsigned int plane;
1117         int ret = 0;
1118         bool reacquired = vb->planes[0].mem_priv == NULL;
1119
1120         memset(planes, 0, sizeof(planes[0]) * vb->num_planes);
1121         /* Copy relevant information provided by the userspace */
1122         ret = call_bufop(vb->vb2_queue, fill_vb2_buffer,
1123                          vb, planes);
1124         if (ret)
1125                 return ret;
1126
1127         for (plane = 0; plane < vb->num_planes; ++plane) {
1128                 /* Skip the plane if already verified */
1129                 if (vb->planes[plane].m.userptr &&
1130                         vb->planes[plane].m.userptr == planes[plane].m.userptr
1131                         && vb->planes[plane].length == planes[plane].length)
1132                         continue;
1133
1134                 dprintk(q, 3, "userspace address for plane %d changed, reacquiring memory\n",
1135                         plane);
1136
1137                 /* Check if the provided plane buffer is large enough */
1138                 if (planes[plane].length < vb->planes[plane].min_length) {
1139                         dprintk(q, 1, "provided buffer size %u is less than setup size %u for plane %d\n",
1140                                                 planes[plane].length,
1141                                                 vb->planes[plane].min_length,
1142                                                 plane);
1143                         ret = -EINVAL;
1144                         goto err;
1145                 }
1146
1147                 /* Release previously acquired memory if present */
1148                 if (vb->planes[plane].mem_priv) {
1149                         if (!reacquired) {
1150                                 reacquired = true;
1151                                 vb->copied_timestamp = 0;
1152                                 call_void_vb_qop(vb, buf_cleanup, vb);
1153                         }
1154                         call_void_memop(vb, put_userptr, vb->planes[plane].mem_priv);
1155                 }
1156
1157                 vb->planes[plane].mem_priv = NULL;
1158                 vb->planes[plane].bytesused = 0;
1159                 vb->planes[plane].length = 0;
1160                 vb->planes[plane].m.userptr = 0;
1161                 vb->planes[plane].data_offset = 0;
1162
1163                 /* Acquire each plane's memory */
1164                 mem_priv = call_ptr_memop(get_userptr,
1165                                           vb,
1166                                           q->alloc_devs[plane] ? : q->dev,
1167                                           planes[plane].m.userptr,
1168                                           planes[plane].length);
1169                 if (IS_ERR(mem_priv)) {
1170                         dprintk(q, 1, "failed acquiring userspace memory for plane %d\n",
1171                                 plane);
1172                         ret = PTR_ERR(mem_priv);
1173                         goto err;
1174                 }
1175                 vb->planes[plane].mem_priv = mem_priv;
1176         }
1177
1178         /*
1179          * Now that everything is in order, copy relevant information
1180          * provided by userspace.
1181          */
1182         for (plane = 0; plane < vb->num_planes; ++plane) {
1183                 vb->planes[plane].bytesused = planes[plane].bytesused;
1184                 vb->planes[plane].length = planes[plane].length;
1185                 vb->planes[plane].m.userptr = planes[plane].m.userptr;
1186                 vb->planes[plane].data_offset = planes[plane].data_offset;
1187         }
1188
1189         if (reacquired) {
1190                 /*
1191                  * One or more planes changed, so we must call buf_init to do
1192                  * the driver-specific initialization on the newly acquired
1193                  * buffer, if provided.
1194                  */
1195                 ret = call_vb_qop(vb, buf_init, vb);
1196                 if (ret) {
1197                         dprintk(q, 1, "buffer initialization failed\n");
1198                         goto err;
1199                 }
1200         }
1201
1202         ret = call_vb_qop(vb, buf_prepare, vb);
1203         if (ret) {
1204                 dprintk(q, 1, "buffer preparation failed\n");
1205                 call_void_vb_qop(vb, buf_cleanup, vb);
1206                 goto err;
1207         }
1208
1209         return 0;
1210 err:
1211         /* In case of errors, release planes that were already acquired */
1212         for (plane = 0; plane < vb->num_planes; ++plane) {
1213                 if (vb->planes[plane].mem_priv)
1214                         call_void_memop(vb, put_userptr,
1215                                 vb->planes[plane].mem_priv);
1216                 vb->planes[plane].mem_priv = NULL;
1217                 vb->planes[plane].m.userptr = 0;
1218                 vb->planes[plane].length = 0;
1219         }
1220
1221         return ret;
1222 }
1223
1224 /*
1225  * __prepare_dmabuf() - prepare a DMABUF buffer
1226  */
1227 static int __prepare_dmabuf(struct vb2_buffer *vb)
1228 {
1229         struct vb2_plane planes[VB2_MAX_PLANES];
1230         struct vb2_queue *q = vb->vb2_queue;
1231         void *mem_priv;
1232         unsigned int plane;
1233         int ret = 0;
1234         bool reacquired = vb->planes[0].mem_priv == NULL;
1235
1236         memset(planes, 0, sizeof(planes[0]) * vb->num_planes);
1237         /* Copy relevant information provided by the userspace */
1238         ret = call_bufop(vb->vb2_queue, fill_vb2_buffer,
1239                          vb, planes);
1240         if (ret)
1241                 return ret;
1242
1243         for (plane = 0; plane < vb->num_planes; ++plane) {
1244                 struct dma_buf *dbuf = dma_buf_get(planes[plane].m.fd);
1245
1246                 if (IS_ERR_OR_NULL(dbuf)) {
1247                         dprintk(q, 1, "invalid dmabuf fd for plane %d\n",
1248                                 plane);
1249                         ret = -EINVAL;
1250                         goto err;
1251                 }
1252
1253                 /* use DMABUF size if length is not provided */
1254                 if (planes[plane].length == 0)
1255                         planes[plane].length = dbuf->size;
1256
1257                 if (planes[plane].length < vb->planes[plane].min_length) {
1258                         dprintk(q, 1, "invalid dmabuf length %u for plane %d, minimum length %u\n",
1259                                 planes[plane].length, plane,
1260                                 vb->planes[plane].min_length);
1261                         dma_buf_put(dbuf);
1262                         ret = -EINVAL;
1263                         goto err;
1264                 }
1265
1266                 /* Skip the plane if already verified */
1267                 if (dbuf == vb->planes[plane].dbuf &&
1268                         vb->planes[plane].length == planes[plane].length) {
1269                         dma_buf_put(dbuf);
1270                         continue;
1271                 }
1272
1273                 dprintk(q, 3, "buffer for plane %d changed\n", plane);
1274
1275                 if (!reacquired) {
1276                         reacquired = true;
1277                         vb->copied_timestamp = 0;
1278                         call_void_vb_qop(vb, buf_cleanup, vb);
1279                 }
1280
1281                 /* Release previously acquired memory if present */
1282                 __vb2_plane_dmabuf_put(vb, &vb->planes[plane]);
1283                 vb->planes[plane].bytesused = 0;
1284                 vb->planes[plane].length = 0;
1285                 vb->planes[plane].m.fd = 0;
1286                 vb->planes[plane].data_offset = 0;
1287
1288                 /* Acquire each plane's memory */
1289                 mem_priv = call_ptr_memop(attach_dmabuf,
1290                                           vb,
1291                                           q->alloc_devs[plane] ? : q->dev,
1292                                           dbuf,
1293                                           planes[plane].length);
1294                 if (IS_ERR(mem_priv)) {
1295                         dprintk(q, 1, "failed to attach dmabuf\n");
1296                         ret = PTR_ERR(mem_priv);
1297                         dma_buf_put(dbuf);
1298                         goto err;
1299                 }
1300
1301                 vb->planes[plane].dbuf = dbuf;
1302                 vb->planes[plane].mem_priv = mem_priv;
1303         }
1304
1305         /*
1306          * This pins the buffer(s) with dma_buf_map_attachment()). It's done
1307          * here instead just before the DMA, while queueing the buffer(s) so
1308          * userspace knows sooner rather than later if the dma-buf map fails.
1309          */
1310         for (plane = 0; plane < vb->num_planes; ++plane) {
1311                 if (vb->planes[plane].dbuf_mapped)
1312                         continue;
1313
1314                 ret = call_memop(vb, map_dmabuf, vb->planes[plane].mem_priv);
1315                 if (ret) {
1316                         dprintk(q, 1, "failed to map dmabuf for plane %d\n",
1317                                 plane);
1318                         goto err;
1319                 }
1320                 vb->planes[plane].dbuf_mapped = 1;
1321         }
1322
1323         /*
1324          * Now that everything is in order, copy relevant information
1325          * provided by userspace.
1326          */
1327         for (plane = 0; plane < vb->num_planes; ++plane) {
1328                 vb->planes[plane].bytesused = planes[plane].bytesused;
1329                 vb->planes[plane].length = planes[plane].length;
1330                 vb->planes[plane].m.fd = planes[plane].m.fd;
1331                 vb->planes[plane].data_offset = planes[plane].data_offset;
1332         }
1333
1334         if (reacquired) {
1335                 /*
1336                  * Call driver-specific initialization on the newly acquired buffer,
1337                  * if provided.
1338                  */
1339                 ret = call_vb_qop(vb, buf_init, vb);
1340                 if (ret) {
1341                         dprintk(q, 1, "buffer initialization failed\n");
1342                         goto err;
1343                 }
1344         }
1345
1346         ret = call_vb_qop(vb, buf_prepare, vb);
1347         if (ret) {
1348                 dprintk(q, 1, "buffer preparation failed\n");
1349                 call_void_vb_qop(vb, buf_cleanup, vb);
1350                 goto err;
1351         }
1352
1353         return 0;
1354 err:
1355         /* In case of errors, release planes that were already acquired */
1356         __vb2_buf_dmabuf_put(vb);
1357
1358         return ret;
1359 }
1360
1361 /*
1362  * __enqueue_in_driver() - enqueue a vb2_buffer in driver for processing
1363  */
1364 static void __enqueue_in_driver(struct vb2_buffer *vb)
1365 {
1366         struct vb2_queue *q = vb->vb2_queue;
1367
1368         vb->state = VB2_BUF_STATE_ACTIVE;
1369         atomic_inc(&q->owned_by_drv_count);
1370
1371         trace_vb2_buf_queue(q, vb);
1372
1373         call_void_vb_qop(vb, buf_queue, vb);
1374 }
1375
1376 static int __buf_prepare(struct vb2_buffer *vb)
1377 {
1378         struct vb2_queue *q = vb->vb2_queue;
1379         enum vb2_buffer_state orig_state = vb->state;
1380         int ret;
1381
1382         if (q->error) {
1383                 dprintk(q, 1, "fatal error occurred on queue\n");
1384                 return -EIO;
1385         }
1386
1387         if (vb->prepared)
1388                 return 0;
1389         WARN_ON(vb->synced);
1390
1391         if (q->is_output) {
1392                 ret = call_vb_qop(vb, buf_out_validate, vb);
1393                 if (ret) {
1394                         dprintk(q, 1, "buffer validation failed\n");
1395                         return ret;
1396                 }
1397         }
1398
1399         vb->state = VB2_BUF_STATE_PREPARING;
1400
1401         switch (q->memory) {
1402         case VB2_MEMORY_MMAP:
1403                 ret = __prepare_mmap(vb);
1404                 break;
1405         case VB2_MEMORY_USERPTR:
1406                 ret = __prepare_userptr(vb);
1407                 break;
1408         case VB2_MEMORY_DMABUF:
1409                 ret = __prepare_dmabuf(vb);
1410                 break;
1411         default:
1412                 WARN(1, "Invalid queue type\n");
1413                 ret = -EINVAL;
1414                 break;
1415         }
1416
1417         if (ret) {
1418                 dprintk(q, 1, "buffer preparation failed: %d\n", ret);
1419                 vb->state = orig_state;
1420                 return ret;
1421         }
1422
1423         __vb2_buf_mem_prepare(vb);
1424         vb->prepared = 1;
1425         vb->state = orig_state;
1426
1427         return 0;
1428 }
1429
1430 static int vb2_req_prepare(struct media_request_object *obj)
1431 {
1432         struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
1433         int ret;
1434
1435         if (WARN_ON(vb->state != VB2_BUF_STATE_IN_REQUEST))
1436                 return -EINVAL;
1437
1438         mutex_lock(vb->vb2_queue->lock);
1439         ret = __buf_prepare(vb);
1440         mutex_unlock(vb->vb2_queue->lock);
1441         return ret;
1442 }
1443
1444 static void __vb2_dqbuf(struct vb2_buffer *vb);
1445
1446 static void vb2_req_unprepare(struct media_request_object *obj)
1447 {
1448         struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
1449
1450         mutex_lock(vb->vb2_queue->lock);
1451         __vb2_dqbuf(vb);
1452         vb->state = VB2_BUF_STATE_IN_REQUEST;
1453         mutex_unlock(vb->vb2_queue->lock);
1454         WARN_ON(!vb->req_obj.req);
1455 }
1456
1457 int vb2_core_qbuf(struct vb2_queue *q, unsigned int index, void *pb,
1458                   struct media_request *req);
1459
1460 static void vb2_req_queue(struct media_request_object *obj)
1461 {
1462         struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
1463
1464         mutex_lock(vb->vb2_queue->lock);
1465         vb2_core_qbuf(vb->vb2_queue, vb->index, NULL, NULL);
1466         mutex_unlock(vb->vb2_queue->lock);
1467 }
1468
1469 static void vb2_req_unbind(struct media_request_object *obj)
1470 {
1471         struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
1472
1473         if (vb->state == VB2_BUF_STATE_IN_REQUEST)
1474                 call_void_bufop(vb->vb2_queue, init_buffer, vb);
1475 }
1476
1477 static void vb2_req_release(struct media_request_object *obj)
1478 {
1479         struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
1480
1481         if (vb->state == VB2_BUF_STATE_IN_REQUEST) {
1482                 vb->state = VB2_BUF_STATE_DEQUEUED;
1483                 if (vb->request)
1484                         media_request_put(vb->request);
1485                 vb->request = NULL;
1486         }
1487 }
1488
1489 static const struct media_request_object_ops vb2_core_req_ops = {
1490         .prepare = vb2_req_prepare,
1491         .unprepare = vb2_req_unprepare,
1492         .queue = vb2_req_queue,
1493         .unbind = vb2_req_unbind,
1494         .release = vb2_req_release,
1495 };
1496
1497 bool vb2_request_object_is_buffer(struct media_request_object *obj)
1498 {
1499         return obj->ops == &vb2_core_req_ops;
1500 }
1501 EXPORT_SYMBOL_GPL(vb2_request_object_is_buffer);
1502
1503 unsigned int vb2_request_buffer_cnt(struct media_request *req)
1504 {
1505         struct media_request_object *obj;
1506         unsigned long flags;
1507         unsigned int buffer_cnt = 0;
1508
1509         spin_lock_irqsave(&req->lock, flags);
1510         list_for_each_entry(obj, &req->objects, list)
1511                 if (vb2_request_object_is_buffer(obj))
1512                         buffer_cnt++;
1513         spin_unlock_irqrestore(&req->lock, flags);
1514
1515         return buffer_cnt;
1516 }
1517 EXPORT_SYMBOL_GPL(vb2_request_buffer_cnt);
1518
1519 int vb2_core_prepare_buf(struct vb2_queue *q, unsigned int index, void *pb)
1520 {
1521         struct vb2_buffer *vb;
1522         int ret;
1523
1524         vb = q->bufs[index];
1525         if (vb->state != VB2_BUF_STATE_DEQUEUED) {
1526                 dprintk(q, 1, "invalid buffer state %s\n",
1527                         vb2_state_name(vb->state));
1528                 return -EINVAL;
1529         }
1530         if (vb->prepared) {
1531                 dprintk(q, 1, "buffer already prepared\n");
1532                 return -EINVAL;
1533         }
1534
1535         ret = __buf_prepare(vb);
1536         if (ret)
1537                 return ret;
1538
1539         /* Fill buffer information for the userspace */
1540         call_void_bufop(q, fill_user_buffer, vb, pb);
1541
1542         dprintk(q, 2, "prepare of buffer %d succeeded\n", vb->index);
1543
1544         return 0;
1545 }
1546 EXPORT_SYMBOL_GPL(vb2_core_prepare_buf);
1547
1548 /*
1549  * vb2_start_streaming() - Attempt to start streaming.
1550  * @q:          videobuf2 queue
1551  *
1552  * Attempt to start streaming. When this function is called there must be
1553  * at least q->min_buffers_needed buffers queued up (i.e. the minimum
1554  * number of buffers required for the DMA engine to function). If the
1555  * @start_streaming op fails it is supposed to return all the driver-owned
1556  * buffers back to vb2 in state QUEUED. Check if that happened and if
1557  * not warn and reclaim them forcefully.
1558  */
1559 static int vb2_start_streaming(struct vb2_queue *q)
1560 {
1561         struct vb2_buffer *vb;
1562         int ret;
1563
1564         /*
1565          * If any buffers were queued before streamon,
1566          * we can now pass them to driver for processing.
1567          */
1568         list_for_each_entry(vb, &q->queued_list, queued_entry)
1569                 __enqueue_in_driver(vb);
1570
1571         /* Tell the driver to start streaming */
1572         q->start_streaming_called = 1;
1573         ret = call_qop(q, start_streaming, q,
1574                        atomic_read(&q->owned_by_drv_count));
1575         if (!ret)
1576                 return 0;
1577
1578         q->start_streaming_called = 0;
1579
1580         dprintk(q, 1, "driver refused to start streaming\n");
1581         /*
1582          * If you see this warning, then the driver isn't cleaning up properly
1583          * after a failed start_streaming(). See the start_streaming()
1584          * documentation in videobuf2-core.h for more information how buffers
1585          * should be returned to vb2 in start_streaming().
1586          */
1587         if (WARN_ON(atomic_read(&q->owned_by_drv_count))) {
1588                 unsigned i;
1589
1590                 /*
1591                  * Forcefully reclaim buffers if the driver did not
1592                  * correctly return them to vb2.
1593                  */
1594                 for (i = 0; i < q->num_buffers; ++i) {
1595                         vb = q->bufs[i];
1596                         if (vb->state == VB2_BUF_STATE_ACTIVE)
1597                                 vb2_buffer_done(vb, VB2_BUF_STATE_QUEUED);
1598                 }
1599                 /* Must be zero now */
1600                 WARN_ON(atomic_read(&q->owned_by_drv_count));
1601         }
1602         /*
1603          * If done_list is not empty, then start_streaming() didn't call
1604          * vb2_buffer_done(vb, VB2_BUF_STATE_QUEUED) but STATE_ERROR or
1605          * STATE_DONE.
1606          */
1607         WARN_ON(!list_empty(&q->done_list));
1608         return ret;
1609 }
1610
1611 int vb2_core_qbuf(struct vb2_queue *q, unsigned int index, void *pb,
1612                   struct media_request *req)
1613 {
1614         struct vb2_buffer *vb;
1615         enum vb2_buffer_state orig_state;
1616         int ret;
1617
1618         if (q->error) {
1619                 dprintk(q, 1, "fatal error occurred on queue\n");
1620                 return -EIO;
1621         }
1622
1623         vb = q->bufs[index];
1624
1625         if (!req && vb->state != VB2_BUF_STATE_IN_REQUEST &&
1626             q->requires_requests) {
1627                 dprintk(q, 1, "qbuf requires a request\n");
1628                 return -EBADR;
1629         }
1630
1631         if ((req && q->uses_qbuf) ||
1632             (!req && vb->state != VB2_BUF_STATE_IN_REQUEST &&
1633              q->uses_requests)) {
1634                 dprintk(q, 1, "queue in wrong mode (qbuf vs requests)\n");
1635                 return -EBUSY;
1636         }
1637
1638         if (req) {
1639                 int ret;
1640
1641                 q->uses_requests = 1;
1642                 if (vb->state != VB2_BUF_STATE_DEQUEUED) {
1643                         dprintk(q, 1, "buffer %d not in dequeued state\n",
1644                                 vb->index);
1645                         return -EINVAL;
1646                 }
1647
1648                 if (q->is_output && !vb->prepared) {
1649                         ret = call_vb_qop(vb, buf_out_validate, vb);
1650                         if (ret) {
1651                                 dprintk(q, 1, "buffer validation failed\n");
1652                                 return ret;
1653                         }
1654                 }
1655
1656                 media_request_object_init(&vb->req_obj);
1657
1658                 /* Make sure the request is in a safe state for updating. */
1659                 ret = media_request_lock_for_update(req);
1660                 if (ret)
1661                         return ret;
1662                 ret = media_request_object_bind(req, &vb2_core_req_ops,
1663                                                 q, true, &vb->req_obj);
1664                 media_request_unlock_for_update(req);
1665                 if (ret)
1666                         return ret;
1667
1668                 vb->state = VB2_BUF_STATE_IN_REQUEST;
1669
1670                 /*
1671                  * Increment the refcount and store the request.
1672                  * The request refcount is decremented again when the
1673                  * buffer is dequeued. This is to prevent vb2_buffer_done()
1674                  * from freeing the request from interrupt context, which can
1675                  * happen if the application closed the request fd after
1676                  * queueing the request.
1677                  */
1678                 media_request_get(req);
1679                 vb->request = req;
1680
1681                 /* Fill buffer information for the userspace */
1682                 if (pb) {
1683                         call_void_bufop(q, copy_timestamp, vb, pb);
1684                         call_void_bufop(q, fill_user_buffer, vb, pb);
1685                 }
1686
1687                 dprintk(q, 2, "qbuf of buffer %d succeeded\n", vb->index);
1688                 return 0;
1689         }
1690
1691         if (vb->state != VB2_BUF_STATE_IN_REQUEST)
1692                 q->uses_qbuf = 1;
1693
1694         switch (vb->state) {
1695         case VB2_BUF_STATE_DEQUEUED:
1696         case VB2_BUF_STATE_IN_REQUEST:
1697                 if (!vb->prepared) {
1698                         ret = __buf_prepare(vb);
1699                         if (ret)
1700                                 return ret;
1701                 }
1702                 break;
1703         case VB2_BUF_STATE_PREPARING:
1704                 dprintk(q, 1, "buffer still being prepared\n");
1705                 return -EINVAL;
1706         default:
1707                 dprintk(q, 1, "invalid buffer state %s\n",
1708                         vb2_state_name(vb->state));
1709                 return -EINVAL;
1710         }
1711
1712         /*
1713          * Add to the queued buffers list, a buffer will stay on it until
1714          * dequeued in dqbuf.
1715          */
1716         orig_state = vb->state;
1717         list_add_tail(&vb->queued_entry, &q->queued_list);
1718         q->queued_count++;
1719         q->waiting_for_buffers = false;
1720         vb->state = VB2_BUF_STATE_QUEUED;
1721
1722         if (pb)
1723                 call_void_bufop(q, copy_timestamp, vb, pb);
1724
1725         trace_vb2_qbuf(q, vb);
1726
1727         /*
1728          * If already streaming, give the buffer to driver for processing.
1729          * If not, the buffer will be given to driver on next streamon.
1730          */
1731         if (q->start_streaming_called)
1732                 __enqueue_in_driver(vb);
1733
1734         /* Fill buffer information for the userspace */
1735         if (pb)
1736                 call_void_bufop(q, fill_user_buffer, vb, pb);
1737
1738         /*
1739          * If streamon has been called, and we haven't yet called
1740          * start_streaming() since not enough buffers were queued, and
1741          * we now have reached the minimum number of queued buffers,
1742          * then we can finally call start_streaming().
1743          */
1744         if (q->streaming && !q->start_streaming_called &&
1745             q->queued_count >= q->min_buffers_needed) {
1746                 ret = vb2_start_streaming(q);
1747                 if (ret) {
1748                         /*
1749                          * Since vb2_core_qbuf will return with an error,
1750                          * we should return it to state DEQUEUED since
1751                          * the error indicates that the buffer wasn't queued.
1752                          */
1753                         list_del(&vb->queued_entry);
1754                         q->queued_count--;
1755                         vb->state = orig_state;
1756                         return ret;
1757                 }
1758         }
1759
1760         dprintk(q, 2, "qbuf of buffer %d succeeded\n", vb->index);
1761         return 0;
1762 }
1763 EXPORT_SYMBOL_GPL(vb2_core_qbuf);
1764
1765 /*
1766  * __vb2_wait_for_done_vb() - wait for a buffer to become available
1767  * for dequeuing
1768  *
1769  * Will sleep if required for nonblocking == false.
1770  */
1771 static int __vb2_wait_for_done_vb(struct vb2_queue *q, int nonblocking)
1772 {
1773         /*
1774          * All operations on vb_done_list are performed under done_lock
1775          * spinlock protection. However, buffers may be removed from
1776          * it and returned to userspace only while holding both driver's
1777          * lock and the done_lock spinlock. Thus we can be sure that as
1778          * long as we hold the driver's lock, the list will remain not
1779          * empty if list_empty() check succeeds.
1780          */
1781
1782         for (;;) {
1783                 int ret;
1784
1785                 if (q->waiting_in_dqbuf) {
1786                         dprintk(q, 1, "another dup()ped fd is waiting for a buffer\n");
1787                         return -EBUSY;
1788                 }
1789
1790                 if (!q->streaming) {
1791                         dprintk(q, 1, "streaming off, will not wait for buffers\n");
1792                         return -EINVAL;
1793                 }
1794
1795                 if (q->error) {
1796                         dprintk(q, 1, "Queue in error state, will not wait for buffers\n");
1797                         return -EIO;
1798                 }
1799
1800                 if (q->last_buffer_dequeued) {
1801                         dprintk(q, 3, "last buffer dequeued already, will not wait for buffers\n");
1802                         return -EPIPE;
1803                 }
1804
1805                 if (!list_empty(&q->done_list)) {
1806                         /*
1807                          * Found a buffer that we were waiting for.
1808                          */
1809                         break;
1810                 }
1811
1812                 if (nonblocking) {
1813                         dprintk(q, 3, "nonblocking and no buffers to dequeue, will not wait\n");
1814                         return -EAGAIN;
1815                 }
1816
1817                 q->waiting_in_dqbuf = 1;
1818                 /*
1819                  * We are streaming and blocking, wait for another buffer to
1820                  * become ready or for streamoff. Driver's lock is released to
1821                  * allow streamoff or qbuf to be called while waiting.
1822                  */
1823                 call_void_qop(q, wait_prepare, q);
1824
1825                 /*
1826                  * All locks have been released, it is safe to sleep now.
1827                  */
1828                 dprintk(q, 3, "will sleep waiting for buffers\n");
1829                 ret = wait_event_interruptible(q->done_wq,
1830                                 !list_empty(&q->done_list) || !q->streaming ||
1831                                 q->error);
1832
1833                 /*
1834                  * We need to reevaluate both conditions again after reacquiring
1835                  * the locks or return an error if one occurred.
1836                  */
1837                 call_void_qop(q, wait_finish, q);
1838                 q->waiting_in_dqbuf = 0;
1839                 if (ret) {
1840                         dprintk(q, 1, "sleep was interrupted\n");
1841                         return ret;
1842                 }
1843         }
1844         return 0;
1845 }
1846
1847 /*
1848  * __vb2_get_done_vb() - get a buffer ready for dequeuing
1849  *
1850  * Will sleep if required for nonblocking == false.
1851  */
1852 static int __vb2_get_done_vb(struct vb2_queue *q, struct vb2_buffer **vb,
1853                              void *pb, int nonblocking)
1854 {
1855         unsigned long flags;
1856         int ret = 0;
1857
1858         /*
1859          * Wait for at least one buffer to become available on the done_list.
1860          */
1861         ret = __vb2_wait_for_done_vb(q, nonblocking);
1862         if (ret)
1863                 return ret;
1864
1865         /*
1866          * Driver's lock has been held since we last verified that done_list
1867          * is not empty, so no need for another list_empty(done_list) check.
1868          */
1869         spin_lock_irqsave(&q->done_lock, flags);
1870         *vb = list_first_entry(&q->done_list, struct vb2_buffer, done_entry);
1871         /*
1872          * Only remove the buffer from done_list if all planes can be
1873          * handled. Some cases such as V4L2 file I/O and DVB have pb
1874          * == NULL; skip the check then as there's nothing to verify.
1875          */
1876         if (pb)
1877                 ret = call_bufop(q, verify_planes_array, *vb, pb);
1878         if (!ret)
1879                 list_del(&(*vb)->done_entry);
1880         spin_unlock_irqrestore(&q->done_lock, flags);
1881
1882         return ret;
1883 }
1884
1885 int vb2_wait_for_all_buffers(struct vb2_queue *q)
1886 {
1887         if (!q->streaming) {
1888                 dprintk(q, 1, "streaming off, will not wait for buffers\n");
1889                 return -EINVAL;
1890         }
1891
1892         if (q->start_streaming_called)
1893                 wait_event(q->done_wq, !atomic_read(&q->owned_by_drv_count));
1894         return 0;
1895 }
1896 EXPORT_SYMBOL_GPL(vb2_wait_for_all_buffers);
1897
1898 /*
1899  * __vb2_dqbuf() - bring back the buffer to the DEQUEUED state
1900  */
1901 static void __vb2_dqbuf(struct vb2_buffer *vb)
1902 {
1903         struct vb2_queue *q = vb->vb2_queue;
1904
1905         /* nothing to do if the buffer is already dequeued */
1906         if (vb->state == VB2_BUF_STATE_DEQUEUED)
1907                 return;
1908
1909         vb->state = VB2_BUF_STATE_DEQUEUED;
1910
1911         call_void_bufop(q, init_buffer, vb);
1912 }
1913
1914 int vb2_core_dqbuf(struct vb2_queue *q, unsigned int *pindex, void *pb,
1915                    bool nonblocking)
1916 {
1917         struct vb2_buffer *vb = NULL;
1918         int ret;
1919
1920         ret = __vb2_get_done_vb(q, &vb, pb, nonblocking);
1921         if (ret < 0)
1922                 return ret;
1923
1924         switch (vb->state) {
1925         case VB2_BUF_STATE_DONE:
1926                 dprintk(q, 3, "returning done buffer\n");
1927                 break;
1928         case VB2_BUF_STATE_ERROR:
1929                 dprintk(q, 3, "returning done buffer with errors\n");
1930                 break;
1931         default:
1932                 dprintk(q, 1, "invalid buffer state %s\n",
1933                         vb2_state_name(vb->state));
1934                 return -EINVAL;
1935         }
1936
1937         call_void_vb_qop(vb, buf_finish, vb);
1938         vb->prepared = 0;
1939
1940         if (pindex)
1941                 *pindex = vb->index;
1942
1943         /* Fill buffer information for the userspace */
1944         if (pb)
1945                 call_void_bufop(q, fill_user_buffer, vb, pb);
1946
1947         /* Remove from videobuf queue */
1948         list_del(&vb->queued_entry);
1949         q->queued_count--;
1950
1951         trace_vb2_dqbuf(q, vb);
1952
1953         /* go back to dequeued state */
1954         __vb2_dqbuf(vb);
1955
1956         if (WARN_ON(vb->req_obj.req)) {
1957                 media_request_object_unbind(&vb->req_obj);
1958                 media_request_object_put(&vb->req_obj);
1959         }
1960         if (vb->request)
1961                 media_request_put(vb->request);
1962         vb->request = NULL;
1963
1964         dprintk(q, 2, "dqbuf of buffer %d, state: %s\n",
1965                 vb->index, vb2_state_name(vb->state));
1966
1967         return 0;
1968
1969 }
1970 EXPORT_SYMBOL_GPL(vb2_core_dqbuf);
1971
1972 /*
1973  * __vb2_queue_cancel() - cancel and stop (pause) streaming
1974  *
1975  * Removes all queued buffers from driver's queue and all buffers queued by
1976  * userspace from videobuf's queue. Returns to state after reqbufs.
1977  */
1978 static void __vb2_queue_cancel(struct vb2_queue *q)
1979 {
1980         unsigned int i;
1981
1982         /*
1983          * Tell driver to stop all transactions and release all queued
1984          * buffers.
1985          */
1986         if (q->start_streaming_called)
1987                 call_void_qop(q, stop_streaming, q);
1988
1989         /*
1990          * If you see this warning, then the driver isn't cleaning up properly
1991          * in stop_streaming(). See the stop_streaming() documentation in
1992          * videobuf2-core.h for more information how buffers should be returned
1993          * to vb2 in stop_streaming().
1994          */
1995         if (WARN_ON(atomic_read(&q->owned_by_drv_count))) {
1996                 for (i = 0; i < q->num_buffers; ++i)
1997                         if (q->bufs[i]->state == VB2_BUF_STATE_ACTIVE) {
1998                                 pr_warn("driver bug: stop_streaming operation is leaving buf %p in active state\n",
1999                                         q->bufs[i]);
2000                                 vb2_buffer_done(q->bufs[i], VB2_BUF_STATE_ERROR);
2001                         }
2002                 /* Must be zero now */
2003                 WARN_ON(atomic_read(&q->owned_by_drv_count));
2004         }
2005
2006         q->streaming = 0;
2007         q->start_streaming_called = 0;
2008         q->queued_count = 0;
2009         q->error = 0;
2010         q->uses_requests = 0;
2011         q->uses_qbuf = 0;
2012
2013         /*
2014          * Remove all buffers from videobuf's list...
2015          */
2016         INIT_LIST_HEAD(&q->queued_list);
2017         /*
2018          * ...and done list; userspace will not receive any buffers it
2019          * has not already dequeued before initiating cancel.
2020          */
2021         INIT_LIST_HEAD(&q->done_list);
2022         atomic_set(&q->owned_by_drv_count, 0);
2023         wake_up_all(&q->done_wq);
2024
2025         /*
2026          * Reinitialize all buffers for next use.
2027          * Make sure to call buf_finish for any queued buffers. Normally
2028          * that's done in dqbuf, but that's not going to happen when we
2029          * cancel the whole queue. Note: this code belongs here, not in
2030          * __vb2_dqbuf() since in vb2_core_dqbuf() there is a critical
2031          * call to __fill_user_buffer() after buf_finish(). That order can't
2032          * be changed, so we can't move the buf_finish() to __vb2_dqbuf().
2033          */
2034         for (i = 0; i < q->num_buffers; ++i) {
2035                 struct vb2_buffer *vb = q->bufs[i];
2036                 struct media_request *req = vb->req_obj.req;
2037
2038                 /*
2039                  * If a request is associated with this buffer, then
2040                  * call buf_request_cancel() to give the driver to complete()
2041                  * related request objects. Otherwise those objects would
2042                  * never complete.
2043                  */
2044                 if (req) {
2045                         enum media_request_state state;
2046                         unsigned long flags;
2047
2048                         spin_lock_irqsave(&req->lock, flags);
2049                         state = req->state;
2050                         spin_unlock_irqrestore(&req->lock, flags);
2051
2052                         if (state == MEDIA_REQUEST_STATE_QUEUED)
2053                                 call_void_vb_qop(vb, buf_request_complete, vb);
2054                 }
2055
2056                 __vb2_buf_mem_finish(vb);
2057
2058                 if (vb->prepared) {
2059                         call_void_vb_qop(vb, buf_finish, vb);
2060                         vb->prepared = 0;
2061                 }
2062                 __vb2_dqbuf(vb);
2063
2064                 if (vb->req_obj.req) {
2065                         media_request_object_unbind(&vb->req_obj);
2066                         media_request_object_put(&vb->req_obj);
2067                 }
2068                 if (vb->request)
2069                         media_request_put(vb->request);
2070                 vb->request = NULL;
2071                 vb->copied_timestamp = 0;
2072         }
2073 }
2074
2075 int vb2_core_streamon(struct vb2_queue *q, unsigned int type)
2076 {
2077         int ret;
2078
2079         if (type != q->type) {
2080                 dprintk(q, 1, "invalid stream type\n");
2081                 return -EINVAL;
2082         }
2083
2084         if (q->streaming) {
2085                 dprintk(q, 3, "already streaming\n");
2086                 return 0;
2087         }
2088
2089         if (!q->num_buffers) {
2090                 dprintk(q, 1, "no buffers have been allocated\n");
2091                 return -EINVAL;
2092         }
2093
2094         if (q->num_buffers < q->min_buffers_needed) {
2095                 dprintk(q, 1, "need at least %u allocated buffers\n",
2096                                 q->min_buffers_needed);
2097                 return -EINVAL;
2098         }
2099
2100         /*
2101          * Tell driver to start streaming provided sufficient buffers
2102          * are available.
2103          */
2104         if (q->queued_count >= q->min_buffers_needed) {
2105                 ret = v4l_vb2q_enable_media_source(q);
2106                 if (ret)
2107                         return ret;
2108                 ret = vb2_start_streaming(q);
2109                 if (ret)
2110                         return ret;
2111         }
2112
2113         q->streaming = 1;
2114
2115         dprintk(q, 3, "successful\n");
2116         return 0;
2117 }
2118 EXPORT_SYMBOL_GPL(vb2_core_streamon);
2119
2120 void vb2_queue_error(struct vb2_queue *q)
2121 {
2122         q->error = 1;
2123
2124         wake_up_all(&q->done_wq);
2125 }
2126 EXPORT_SYMBOL_GPL(vb2_queue_error);
2127
2128 int vb2_core_streamoff(struct vb2_queue *q, unsigned int type)
2129 {
2130         if (type != q->type) {
2131                 dprintk(q, 1, "invalid stream type\n");
2132                 return -EINVAL;
2133         }
2134
2135         /*
2136          * Cancel will pause streaming and remove all buffers from the driver
2137          * and videobuf, effectively returning control over them to userspace.
2138          *
2139          * Note that we do this even if q->streaming == 0: if you prepare or
2140          * queue buffers, and then call streamoff without ever having called
2141          * streamon, you would still expect those buffers to be returned to
2142          * their normal dequeued state.
2143          */
2144         __vb2_queue_cancel(q);
2145         q->waiting_for_buffers = !q->is_output;
2146         q->last_buffer_dequeued = false;
2147
2148         dprintk(q, 3, "successful\n");
2149         return 0;
2150 }
2151 EXPORT_SYMBOL_GPL(vb2_core_streamoff);
2152
2153 /*
2154  * __find_plane_by_offset() - find plane associated with the given offset off
2155  */
2156 static int __find_plane_by_offset(struct vb2_queue *q, unsigned long off,
2157                         unsigned int *_buffer, unsigned int *_plane)
2158 {
2159         struct vb2_buffer *vb;
2160         unsigned int buffer, plane;
2161
2162         /*
2163          * Sanity checks to ensure the lock is held, MEMORY_MMAP is
2164          * used and fileio isn't active.
2165          */
2166         lockdep_assert_held(&q->mmap_lock);
2167
2168         if (q->memory != VB2_MEMORY_MMAP) {
2169                 dprintk(q, 1, "queue is not currently set up for mmap\n");
2170                 return -EINVAL;
2171         }
2172
2173         if (vb2_fileio_is_active(q)) {
2174                 dprintk(q, 1, "file io in progress\n");
2175                 return -EBUSY;
2176         }
2177
2178         /*
2179          * Go over all buffers and their planes, comparing the given offset
2180          * with an offset assigned to each plane. If a match is found,
2181          * return its buffer and plane numbers.
2182          */
2183         for (buffer = 0; buffer < q->num_buffers; ++buffer) {
2184                 vb = q->bufs[buffer];
2185
2186                 for (plane = 0; plane < vb->num_planes; ++plane) {
2187                         if (vb->planes[plane].m.offset == off) {
2188                                 *_buffer = buffer;
2189                                 *_plane = plane;
2190                                 return 0;
2191                         }
2192                 }
2193         }
2194
2195         return -EINVAL;
2196 }
2197
2198 int vb2_core_expbuf_dmabuf(struct vb2_queue *q, unsigned int type,
2199                            unsigned int index, unsigned int plane,
2200                            unsigned int flags, struct dma_buf **dmabuf)
2201 {
2202         struct vb2_buffer *vb = NULL;
2203         struct vb2_plane *vb_plane;
2204         struct dma_buf *dbuf;
2205
2206         if (q->memory != VB2_MEMORY_MMAP) {
2207                 dprintk(q, 1, "queue is not currently set up for mmap\n");
2208                 return -EINVAL;
2209         }
2210
2211         if (!q->mem_ops->get_dmabuf) {
2212                 dprintk(q, 1, "queue does not support DMA buffer exporting\n");
2213                 return -EINVAL;
2214         }
2215
2216         if (flags & ~(O_CLOEXEC | O_ACCMODE)) {
2217                 dprintk(q, 1, "queue does support only O_CLOEXEC and access mode flags\n");
2218                 return -EINVAL;
2219         }
2220
2221         if (type != q->type) {
2222                 dprintk(q, 1, "invalid buffer type\n");
2223                 return -EINVAL;
2224         }
2225
2226         if (index >= q->num_buffers) {
2227                 dprintk(q, 1, "buffer index out of range\n");
2228                 return -EINVAL;
2229         }
2230
2231         vb = q->bufs[index];
2232
2233         if (plane >= vb->num_planes) {
2234                 dprintk(q, 1, "buffer plane out of range\n");
2235                 return -EINVAL;
2236         }
2237
2238         if (vb2_fileio_is_active(q)) {
2239                 dprintk(q, 1, "expbuf: file io in progress\n");
2240                 return -EBUSY;
2241         }
2242
2243         vb_plane = &vb->planes[plane];
2244
2245         dbuf = call_ptr_memop(get_dmabuf,
2246                               vb,
2247                               vb_plane->mem_priv,
2248                               flags & O_ACCMODE);
2249         if (IS_ERR_OR_NULL(dbuf)) {
2250                 dprintk(q, 1, "failed to export buffer %d, plane %d\n",
2251                         index, plane);
2252                 return -EINVAL;
2253         }
2254
2255         *dmabuf = dbuf;
2256         return 0;
2257 }
2258 EXPORT_SYMBOL_GPL(vb2_core_expbuf_dmabuf);
2259
2260 int vb2_core_expbuf(struct vb2_queue *q, int *fd, unsigned int type,
2261                     unsigned int index, unsigned int plane, unsigned int flags)
2262 {
2263         struct dma_buf *dbuf;
2264         int ret;
2265
2266         ret = vb2_core_expbuf_dmabuf(q, type, index, plane, flags, &dbuf);
2267         if (ret)
2268                 return ret;
2269
2270         ret = dma_buf_fd(dbuf, flags & ~O_ACCMODE);
2271         if (ret < 0) {
2272                 dprintk(q, 3, "buffer %d, plane %d failed to export (%d)\n",
2273                         index, plane, ret);
2274                 dma_buf_put(dbuf);
2275                 return ret;
2276         }
2277
2278         dprintk(q, 3, "buffer %d, plane %d exported as %d descriptor\n",
2279                 index, plane, ret);
2280         *fd = ret;
2281
2282         return 0;
2283 }
2284 EXPORT_SYMBOL_GPL(vb2_core_expbuf);
2285
2286 int vb2_mmap(struct vb2_queue *q, struct vm_area_struct *vma)
2287 {
2288         unsigned long off = vma->vm_pgoff << PAGE_SHIFT;
2289         struct vb2_buffer *vb;
2290         unsigned int buffer = 0, plane = 0;
2291         int ret;
2292         unsigned long length;
2293
2294         /*
2295          * Check memory area access mode.
2296          */
2297         if (!(vma->vm_flags & VM_SHARED)) {
2298                 dprintk(q, 1, "invalid vma flags, VM_SHARED needed\n");
2299                 return -EINVAL;
2300         }
2301         if (q->is_output) {
2302                 if (!(vma->vm_flags & VM_WRITE)) {
2303                         dprintk(q, 1, "invalid vma flags, VM_WRITE needed\n");
2304                         return -EINVAL;
2305                 }
2306         } else {
2307                 if (!(vma->vm_flags & VM_READ)) {
2308                         dprintk(q, 1, "invalid vma flags, VM_READ needed\n");
2309                         return -EINVAL;
2310                 }
2311         }
2312
2313         mutex_lock(&q->mmap_lock);
2314
2315         /*
2316          * Find the plane corresponding to the offset passed by userspace. This
2317          * will return an error if not MEMORY_MMAP or file I/O is in progress.
2318          */
2319         ret = __find_plane_by_offset(q, off, &buffer, &plane);
2320         if (ret)
2321                 goto unlock;
2322
2323         vb = q->bufs[buffer];
2324
2325         /*
2326          * MMAP requires page_aligned buffers.
2327          * The buffer length was page_aligned at __vb2_buf_mem_alloc(),
2328          * so, we need to do the same here.
2329          */
2330         length = PAGE_ALIGN(vb->planes[plane].length);
2331         if (length < (vma->vm_end - vma->vm_start)) {
2332                 dprintk(q, 1,
2333                         "MMAP invalid, as it would overflow buffer length\n");
2334                 ret = -EINVAL;
2335                 goto unlock;
2336         }
2337
2338         /*
2339          * vm_pgoff is treated in V4L2 API as a 'cookie' to select a buffer,
2340          * not as a in-buffer offset. We always want to mmap a whole buffer
2341          * from its beginning.
2342          */
2343         vma->vm_pgoff = 0;
2344
2345         ret = call_memop(vb, mmap, vb->planes[plane].mem_priv, vma);
2346
2347 unlock:
2348         mutex_unlock(&q->mmap_lock);
2349         if (ret)
2350                 return ret;
2351
2352         dprintk(q, 3, "buffer %d, plane %d successfully mapped\n", buffer, plane);
2353         return 0;
2354 }
2355 EXPORT_SYMBOL_GPL(vb2_mmap);
2356
2357 #ifndef CONFIG_MMU
2358 unsigned long vb2_get_unmapped_area(struct vb2_queue *q,
2359                                     unsigned long addr,
2360                                     unsigned long len,
2361                                     unsigned long pgoff,
2362                                     unsigned long flags)
2363 {
2364         unsigned long off = pgoff << PAGE_SHIFT;
2365         struct vb2_buffer *vb;
2366         unsigned int buffer, plane;
2367         void *vaddr;
2368         int ret;
2369
2370         mutex_lock(&q->mmap_lock);
2371
2372         /*
2373          * Find the plane corresponding to the offset passed by userspace. This
2374          * will return an error if not MEMORY_MMAP or file I/O is in progress.
2375          */
2376         ret = __find_plane_by_offset(q, off, &buffer, &plane);
2377         if (ret)
2378                 goto unlock;
2379
2380         vb = q->bufs[buffer];
2381
2382         vaddr = vb2_plane_vaddr(vb, plane);
2383         mutex_unlock(&q->mmap_lock);
2384         return vaddr ? (unsigned long)vaddr : -EINVAL;
2385
2386 unlock:
2387         mutex_unlock(&q->mmap_lock);
2388         return ret;
2389 }
2390 EXPORT_SYMBOL_GPL(vb2_get_unmapped_area);
2391 #endif
2392
2393 int vb2_core_queue_init(struct vb2_queue *q)
2394 {
2395         /*
2396          * Sanity check
2397          */
2398         if (WARN_ON(!q)                   ||
2399             WARN_ON(!q->ops)              ||
2400             WARN_ON(!q->mem_ops)          ||
2401             WARN_ON(!q->type)             ||
2402             WARN_ON(!q->io_modes)         ||
2403             WARN_ON(!q->ops->queue_setup) ||
2404             WARN_ON(!q->ops->buf_queue))
2405                 return -EINVAL;
2406
2407         if (WARN_ON(q->requires_requests && !q->supports_requests))
2408                 return -EINVAL;
2409
2410         INIT_LIST_HEAD(&q->queued_list);
2411         INIT_LIST_HEAD(&q->done_list);
2412         spin_lock_init(&q->done_lock);
2413         mutex_init(&q->mmap_lock);
2414         init_waitqueue_head(&q->done_wq);
2415
2416         q->memory = VB2_MEMORY_UNKNOWN;
2417
2418         if (q->buf_struct_size == 0)
2419                 q->buf_struct_size = sizeof(struct vb2_buffer);
2420
2421         if (q->bidirectional)
2422                 q->dma_dir = DMA_BIDIRECTIONAL;
2423         else
2424                 q->dma_dir = q->is_output ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
2425
2426         if (q->name[0] == '\0')
2427                 snprintf(q->name, sizeof(q->name), "%s-%p",
2428                          q->is_output ? "out" : "cap", q);
2429
2430         return 0;
2431 }
2432 EXPORT_SYMBOL_GPL(vb2_core_queue_init);
2433
2434 static int __vb2_init_fileio(struct vb2_queue *q, int read);
2435 static int __vb2_cleanup_fileio(struct vb2_queue *q);
2436 void vb2_core_queue_release(struct vb2_queue *q)
2437 {
2438         __vb2_cleanup_fileio(q);
2439         __vb2_queue_cancel(q);
2440         mutex_lock(&q->mmap_lock);
2441         __vb2_queue_free(q, q->num_buffers);
2442         mutex_unlock(&q->mmap_lock);
2443 }
2444 EXPORT_SYMBOL_GPL(vb2_core_queue_release);
2445
2446 __poll_t vb2_core_poll(struct vb2_queue *q, struct file *file,
2447                 poll_table *wait)
2448 {
2449         __poll_t req_events = poll_requested_events(wait);
2450         struct vb2_buffer *vb = NULL;
2451         unsigned long flags;
2452
2453         /*
2454          * poll_wait() MUST be called on the first invocation on all the
2455          * potential queues of interest, even if we are not interested in their
2456          * events during this first call. Failure to do so will result in
2457          * queue's events to be ignored because the poll_table won't be capable
2458          * of adding new wait queues thereafter.
2459          */
2460         poll_wait(file, &q->done_wq, wait);
2461
2462         if (!q->is_output && !(req_events & (EPOLLIN | EPOLLRDNORM)))
2463                 return 0;
2464         if (q->is_output && !(req_events & (EPOLLOUT | EPOLLWRNORM)))
2465                 return 0;
2466
2467         /*
2468          * Start file I/O emulator only if streaming API has not been used yet.
2469          */
2470         if (q->num_buffers == 0 && !vb2_fileio_is_active(q)) {
2471                 if (!q->is_output && (q->io_modes & VB2_READ) &&
2472                                 (req_events & (EPOLLIN | EPOLLRDNORM))) {
2473                         if (__vb2_init_fileio(q, 1))
2474                                 return EPOLLERR;
2475                 }
2476                 if (q->is_output && (q->io_modes & VB2_WRITE) &&
2477                                 (req_events & (EPOLLOUT | EPOLLWRNORM))) {
2478                         if (__vb2_init_fileio(q, 0))
2479                                 return EPOLLERR;
2480                         /*
2481                          * Write to OUTPUT queue can be done immediately.
2482                          */
2483                         return EPOLLOUT | EPOLLWRNORM;
2484                 }
2485         }
2486
2487         /*
2488          * There is nothing to wait for if the queue isn't streaming, or if the
2489          * error flag is set.
2490          */
2491         if (!vb2_is_streaming(q) || q->error)
2492                 return EPOLLERR;
2493
2494         /*
2495          * If this quirk is set and QBUF hasn't been called yet then
2496          * return EPOLLERR as well. This only affects capture queues, output
2497          * queues will always initialize waiting_for_buffers to false.
2498          * This quirk is set by V4L2 for backwards compatibility reasons.
2499          */
2500         if (q->quirk_poll_must_check_waiting_for_buffers &&
2501             q->waiting_for_buffers && (req_events & (EPOLLIN | EPOLLRDNORM)))
2502                 return EPOLLERR;
2503
2504         /*
2505          * For output streams you can call write() as long as there are fewer
2506          * buffers queued than there are buffers available.
2507          */
2508         if (q->is_output && q->fileio && q->queued_count < q->num_buffers)
2509                 return EPOLLOUT | EPOLLWRNORM;
2510
2511         if (list_empty(&q->done_list)) {
2512                 /*
2513                  * If the last buffer was dequeued from a capture queue,
2514                  * return immediately. DQBUF will return -EPIPE.
2515                  */
2516                 if (q->last_buffer_dequeued)
2517                         return EPOLLIN | EPOLLRDNORM;
2518         }
2519
2520         /*
2521          * Take first buffer available for dequeuing.
2522          */
2523         spin_lock_irqsave(&q->done_lock, flags);
2524         if (!list_empty(&q->done_list))
2525                 vb = list_first_entry(&q->done_list, struct vb2_buffer,
2526                                         done_entry);
2527         spin_unlock_irqrestore(&q->done_lock, flags);
2528
2529         if (vb && (vb->state == VB2_BUF_STATE_DONE
2530                         || vb->state == VB2_BUF_STATE_ERROR)) {
2531                 return (q->is_output) ?
2532                                 EPOLLOUT | EPOLLWRNORM :
2533                                 EPOLLIN | EPOLLRDNORM;
2534         }
2535         return 0;
2536 }
2537 EXPORT_SYMBOL_GPL(vb2_core_poll);
2538
2539 /*
2540  * struct vb2_fileio_buf - buffer context used by file io emulator
2541  *
2542  * vb2 provides a compatibility layer and emulator of file io (read and
2543  * write) calls on top of streaming API. This structure is used for
2544  * tracking context related to the buffers.
2545  */
2546 struct vb2_fileio_buf {
2547         void *vaddr;
2548         unsigned int size;
2549         unsigned int pos;
2550         unsigned int queued:1;
2551 };
2552
2553 /*
2554  * struct vb2_fileio_data - queue context used by file io emulator
2555  *
2556  * @cur_index:  the index of the buffer currently being read from or
2557  *              written to. If equal to q->num_buffers then a new buffer
2558  *              must be dequeued.
2559  * @initial_index: in the read() case all buffers are queued up immediately
2560  *              in __vb2_init_fileio() and __vb2_perform_fileio() just cycles
2561  *              buffers. However, in the write() case no buffers are initially
2562  *              queued, instead whenever a buffer is full it is queued up by
2563  *              __vb2_perform_fileio(). Only once all available buffers have
2564  *              been queued up will __vb2_perform_fileio() start to dequeue
2565  *              buffers. This means that initially __vb2_perform_fileio()
2566  *              needs to know what buffer index to use when it is queuing up
2567  *              the buffers for the first time. That initial index is stored
2568  *              in this field. Once it is equal to q->num_buffers all
2569  *              available buffers have been queued and __vb2_perform_fileio()
2570  *              should start the normal dequeue/queue cycle.
2571  *
2572  * vb2 provides a compatibility layer and emulator of file io (read and
2573  * write) calls on top of streaming API. For proper operation it required
2574  * this structure to save the driver state between each call of the read
2575  * or write function.
2576  */
2577 struct vb2_fileio_data {
2578         unsigned int count;
2579         unsigned int type;
2580         unsigned int memory;
2581         struct vb2_fileio_buf bufs[VB2_MAX_FRAME];
2582         unsigned int cur_index;
2583         unsigned int initial_index;
2584         unsigned int q_count;
2585         unsigned int dq_count;
2586         unsigned read_once:1;
2587         unsigned write_immediately:1;
2588 };
2589
2590 /*
2591  * __vb2_init_fileio() - initialize file io emulator
2592  * @q:          videobuf2 queue
2593  * @read:       mode selector (1 means read, 0 means write)
2594  */
2595 static int __vb2_init_fileio(struct vb2_queue *q, int read)
2596 {
2597         struct vb2_fileio_data *fileio;
2598         int i, ret;
2599         unsigned int count = 0;
2600
2601         /*
2602          * Sanity check
2603          */
2604         if (WARN_ON((read && !(q->io_modes & VB2_READ)) ||
2605                     (!read && !(q->io_modes & VB2_WRITE))))
2606                 return -EINVAL;
2607
2608         /*
2609          * Check if device supports mapping buffers to kernel virtual space.
2610          */
2611         if (!q->mem_ops->vaddr)
2612                 return -EBUSY;
2613
2614         /*
2615          * Check if streaming api has not been already activated.
2616          */
2617         if (q->streaming || q->num_buffers > 0)
2618                 return -EBUSY;
2619
2620         /*
2621          * Start with count 1, driver can increase it in queue_setup()
2622          */
2623         count = 1;
2624
2625         dprintk(q, 3, "setting up file io: mode %s, count %d, read_once %d, write_immediately %d\n",
2626                 (read) ? "read" : "write", count, q->fileio_read_once,
2627                 q->fileio_write_immediately);
2628
2629         fileio = kzalloc(sizeof(*fileio), GFP_KERNEL);
2630         if (fileio == NULL)
2631                 return -ENOMEM;
2632
2633         fileio->read_once = q->fileio_read_once;
2634         fileio->write_immediately = q->fileio_write_immediately;
2635
2636         /*
2637          * Request buffers and use MMAP type to force driver
2638          * to allocate buffers by itself.
2639          */
2640         fileio->count = count;
2641         fileio->memory = VB2_MEMORY_MMAP;
2642         fileio->type = q->type;
2643         q->fileio = fileio;
2644         ret = vb2_core_reqbufs(q, fileio->memory, &fileio->count);
2645         if (ret)
2646                 goto err_kfree;
2647
2648         /*
2649          * Check if plane_count is correct
2650          * (multiplane buffers are not supported).
2651          */
2652         if (q->bufs[0]->num_planes != 1) {
2653                 ret = -EBUSY;
2654                 goto err_reqbufs;
2655         }
2656
2657         /*
2658          * Get kernel address of each buffer.
2659          */
2660         for (i = 0; i < q->num_buffers; i++) {
2661                 fileio->bufs[i].vaddr = vb2_plane_vaddr(q->bufs[i], 0);
2662                 if (fileio->bufs[i].vaddr == NULL) {
2663                         ret = -EINVAL;
2664                         goto err_reqbufs;
2665                 }
2666                 fileio->bufs[i].size = vb2_plane_size(q->bufs[i], 0);
2667         }
2668
2669         /*
2670          * Read mode requires pre queuing of all buffers.
2671          */
2672         if (read) {
2673                 /*
2674                  * Queue all buffers.
2675                  */
2676                 for (i = 0; i < q->num_buffers; i++) {
2677                         ret = vb2_core_qbuf(q, i, NULL, NULL);
2678                         if (ret)
2679                                 goto err_reqbufs;
2680                         fileio->bufs[i].queued = 1;
2681                 }
2682                 /*
2683                  * All buffers have been queued, so mark that by setting
2684                  * initial_index to q->num_buffers
2685                  */
2686                 fileio->initial_index = q->num_buffers;
2687                 fileio->cur_index = q->num_buffers;
2688         }
2689
2690         /*
2691          * Start streaming.
2692          */
2693         ret = vb2_core_streamon(q, q->type);
2694         if (ret)
2695                 goto err_reqbufs;
2696
2697         return ret;
2698
2699 err_reqbufs:
2700         fileio->count = 0;
2701         vb2_core_reqbufs(q, fileio->memory, &fileio->count);
2702
2703 err_kfree:
2704         q->fileio = NULL;
2705         kfree(fileio);
2706         return ret;
2707 }
2708
2709 /*
2710  * __vb2_cleanup_fileio() - free resourced used by file io emulator
2711  * @q:          videobuf2 queue
2712  */
2713 static int __vb2_cleanup_fileio(struct vb2_queue *q)
2714 {
2715         struct vb2_fileio_data *fileio = q->fileio;
2716
2717         if (fileio) {
2718                 vb2_core_streamoff(q, q->type);
2719                 q->fileio = NULL;
2720                 fileio->count = 0;
2721                 vb2_core_reqbufs(q, fileio->memory, &fileio->count);
2722                 kfree(fileio);
2723                 dprintk(q, 3, "file io emulator closed\n");
2724         }
2725         return 0;
2726 }
2727
2728 /*
2729  * __vb2_perform_fileio() - perform a single file io (read or write) operation
2730  * @q:          videobuf2 queue
2731  * @data:       pointed to target userspace buffer
2732  * @count:      number of bytes to read or write
2733  * @ppos:       file handle position tracking pointer
2734  * @nonblock:   mode selector (1 means blocking calls, 0 means nonblocking)
2735  * @read:       access mode selector (1 means read, 0 means write)
2736  */
2737 static size_t __vb2_perform_fileio(struct vb2_queue *q, char __user *data, size_t count,
2738                 loff_t *ppos, int nonblock, int read)
2739 {
2740         struct vb2_fileio_data *fileio;
2741         struct vb2_fileio_buf *buf;
2742         bool is_multiplanar = q->is_multiplanar;
2743         /*
2744          * When using write() to write data to an output video node the vb2 core
2745          * should copy timestamps if V4L2_BUF_FLAG_TIMESTAMP_COPY is set. Nobody
2746          * else is able to provide this information with the write() operation.
2747          */
2748         bool copy_timestamp = !read && q->copy_timestamp;
2749         unsigned index;
2750         int ret;
2751
2752         dprintk(q, 3, "mode %s, offset %ld, count %zd, %sblocking\n",
2753                 read ? "read" : "write", (long)*ppos, count,
2754                 nonblock ? "non" : "");
2755
2756         if (!data)
2757                 return -EINVAL;
2758
2759         if (q->waiting_in_dqbuf) {
2760                 dprintk(q, 3, "another dup()ped fd is %s\n",
2761                         read ? "reading" : "writing");
2762                 return -EBUSY;
2763         }
2764
2765         /*
2766          * Initialize emulator on first call.
2767          */
2768         if (!vb2_fileio_is_active(q)) {
2769                 ret = __vb2_init_fileio(q, read);
2770                 dprintk(q, 3, "vb2_init_fileio result: %d\n", ret);
2771                 if (ret)
2772                         return ret;
2773         }
2774         fileio = q->fileio;
2775
2776         /*
2777          * Check if we need to dequeue the buffer.
2778          */
2779         index = fileio->cur_index;
2780         if (index >= q->num_buffers) {
2781                 struct vb2_buffer *b;
2782
2783                 /*
2784                  * Call vb2_dqbuf to get buffer back.
2785                  */
2786                 ret = vb2_core_dqbuf(q, &index, NULL, nonblock);
2787                 dprintk(q, 5, "vb2_dqbuf result: %d\n", ret);
2788                 if (ret)
2789                         return ret;
2790                 fileio->dq_count += 1;
2791
2792                 fileio->cur_index = index;
2793                 buf = &fileio->bufs[index];
2794                 b = q->bufs[index];
2795
2796                 /*
2797                  * Get number of bytes filled by the driver
2798                  */
2799                 buf->pos = 0;
2800                 buf->queued = 0;
2801                 buf->size = read ? vb2_get_plane_payload(q->bufs[index], 0)
2802                                  : vb2_plane_size(q->bufs[index], 0);
2803                 /* Compensate for data_offset on read in the multiplanar case. */
2804                 if (is_multiplanar && read &&
2805                                 b->planes[0].data_offset < buf->size) {
2806                         buf->pos = b->planes[0].data_offset;
2807                         buf->size -= buf->pos;
2808                 }
2809         } else {
2810                 buf = &fileio->bufs[index];
2811         }
2812
2813         /*
2814          * Limit count on last few bytes of the buffer.
2815          */
2816         if (buf->pos + count > buf->size) {
2817                 count = buf->size - buf->pos;
2818                 dprintk(q, 5, "reducing read count: %zd\n", count);
2819         }
2820
2821         /*
2822          * Transfer data to userspace.
2823          */
2824         dprintk(q, 3, "copying %zd bytes - buffer %d, offset %u\n",
2825                 count, index, buf->pos);
2826         if (read)
2827                 ret = copy_to_user(data, buf->vaddr + buf->pos, count);
2828         else
2829                 ret = copy_from_user(buf->vaddr + buf->pos, data, count);
2830         if (ret) {
2831                 dprintk(q, 3, "error copying data\n");
2832                 return -EFAULT;
2833         }
2834
2835         /*
2836          * Update counters.
2837          */
2838         buf->pos += count;
2839         *ppos += count;
2840
2841         /*
2842          * Queue next buffer if required.
2843          */
2844         if (buf->pos == buf->size || (!read && fileio->write_immediately)) {
2845                 struct vb2_buffer *b = q->bufs[index];
2846
2847                 /*
2848                  * Check if this is the last buffer to read.
2849                  */
2850                 if (read && fileio->read_once && fileio->dq_count == 1) {
2851                         dprintk(q, 3, "read limit reached\n");
2852                         return __vb2_cleanup_fileio(q);
2853                 }
2854
2855                 /*
2856                  * Call vb2_qbuf and give buffer to the driver.
2857                  */
2858                 b->planes[0].bytesused = buf->pos;
2859
2860                 if (copy_timestamp)
2861                         b->timestamp = ktime_get_ns();
2862                 ret = vb2_core_qbuf(q, index, NULL, NULL);
2863                 dprintk(q, 5, "vb2_dbuf result: %d\n", ret);
2864                 if (ret)
2865                         return ret;
2866
2867                 /*
2868                  * Buffer has been queued, update the status
2869                  */
2870                 buf->pos = 0;
2871                 buf->queued = 1;
2872                 buf->size = vb2_plane_size(q->bufs[index], 0);
2873                 fileio->q_count += 1;
2874                 /*
2875                  * If we are queuing up buffers for the first time, then
2876                  * increase initial_index by one.
2877                  */
2878                 if (fileio->initial_index < q->num_buffers)
2879                         fileio->initial_index++;
2880                 /*
2881                  * The next buffer to use is either a buffer that's going to be
2882                  * queued for the first time (initial_index < q->num_buffers)
2883                  * or it is equal to q->num_buffers, meaning that the next
2884                  * time we need to dequeue a buffer since we've now queued up
2885                  * all the 'first time' buffers.
2886                  */
2887                 fileio->cur_index = fileio->initial_index;
2888         }
2889
2890         /*
2891          * Return proper number of bytes processed.
2892          */
2893         if (ret == 0)
2894                 ret = count;
2895         return ret;
2896 }
2897
2898 size_t vb2_read(struct vb2_queue *q, char __user *data, size_t count,
2899                 loff_t *ppos, int nonblocking)
2900 {
2901         return __vb2_perform_fileio(q, data, count, ppos, nonblocking, 1);
2902 }
2903 EXPORT_SYMBOL_GPL(vb2_read);
2904
2905 size_t vb2_write(struct vb2_queue *q, const char __user *data, size_t count,
2906                 loff_t *ppos, int nonblocking)
2907 {
2908         return __vb2_perform_fileio(q, (char __user *) data, count,
2909                                                         ppos, nonblocking, 0);
2910 }
2911 EXPORT_SYMBOL_GPL(vb2_write);
2912
2913 struct vb2_threadio_data {
2914         struct task_struct *thread;
2915         vb2_thread_fnc fnc;
2916         void *priv;
2917         bool stop;
2918 };
2919
2920 static int vb2_thread(void *data)
2921 {
2922         struct vb2_queue *q = data;
2923         struct vb2_threadio_data *threadio = q->threadio;
2924         bool copy_timestamp = false;
2925         unsigned prequeue = 0;
2926         unsigned index = 0;
2927         int ret = 0;
2928
2929         if (q->is_output) {
2930                 prequeue = q->num_buffers;
2931                 copy_timestamp = q->copy_timestamp;
2932         }
2933
2934         set_freezable();
2935
2936         for (;;) {
2937                 struct vb2_buffer *vb;
2938
2939                 /*
2940                  * Call vb2_dqbuf to get buffer back.
2941                  */
2942                 if (prequeue) {
2943                         vb = q->bufs[index++];
2944                         prequeue--;
2945                 } else {
2946                         call_void_qop(q, wait_finish, q);
2947                         if (!threadio->stop)
2948                                 ret = vb2_core_dqbuf(q, &index, NULL, 0);
2949                         call_void_qop(q, wait_prepare, q);
2950                         dprintk(q, 5, "file io: vb2_dqbuf result: %d\n", ret);
2951                         if (!ret)
2952                                 vb = q->bufs[index];
2953                 }
2954                 if (ret || threadio->stop)
2955                         break;
2956                 try_to_freeze();
2957
2958                 if (vb->state != VB2_BUF_STATE_ERROR)
2959                         if (threadio->fnc(vb, threadio->priv))
2960                                 break;
2961                 call_void_qop(q, wait_finish, q);
2962                 if (copy_timestamp)
2963                         vb->timestamp = ktime_get_ns();
2964                 if (!threadio->stop)
2965                         ret = vb2_core_qbuf(q, vb->index, NULL, NULL);
2966                 call_void_qop(q, wait_prepare, q);
2967                 if (ret || threadio->stop)
2968                         break;
2969         }
2970
2971         /* Hmm, linux becomes *very* unhappy without this ... */
2972         while (!kthread_should_stop()) {
2973                 set_current_state(TASK_INTERRUPTIBLE);
2974                 schedule();
2975         }
2976         return 0;
2977 }
2978
2979 /*
2980  * This function should not be used for anything else but the videobuf2-dvb
2981  * support. If you think you have another good use-case for this, then please
2982  * contact the linux-media mailinglist first.
2983  */
2984 int vb2_thread_start(struct vb2_queue *q, vb2_thread_fnc fnc, void *priv,
2985                      const char *thread_name)
2986 {
2987         struct vb2_threadio_data *threadio;
2988         int ret = 0;
2989
2990         if (q->threadio)
2991                 return -EBUSY;
2992         if (vb2_is_busy(q))
2993                 return -EBUSY;
2994         if (WARN_ON(q->fileio))
2995                 return -EBUSY;
2996
2997         threadio = kzalloc(sizeof(*threadio), GFP_KERNEL);
2998         if (threadio == NULL)
2999                 return -ENOMEM;
3000         threadio->fnc = fnc;
3001         threadio->priv = priv;
3002
3003         ret = __vb2_init_fileio(q, !q->is_output);
3004         dprintk(q, 3, "file io: vb2_init_fileio result: %d\n", ret);
3005         if (ret)
3006                 goto nomem;
3007         q->threadio = threadio;
3008         threadio->thread = kthread_run(vb2_thread, q, "vb2-%s", thread_name);
3009         if (IS_ERR(threadio->thread)) {
3010                 ret = PTR_ERR(threadio->thread);
3011                 threadio->thread = NULL;
3012                 goto nothread;
3013         }
3014         return 0;
3015
3016 nothread:
3017         __vb2_cleanup_fileio(q);
3018 nomem:
3019         kfree(threadio);
3020         return ret;
3021 }
3022 EXPORT_SYMBOL_GPL(vb2_thread_start);
3023
3024 int vb2_thread_stop(struct vb2_queue *q)
3025 {
3026         struct vb2_threadio_data *threadio = q->threadio;
3027         int err;
3028
3029         if (threadio == NULL)
3030                 return 0;
3031         threadio->stop = true;
3032         /* Wake up all pending sleeps in the thread */
3033         vb2_queue_error(q);
3034         err = kthread_stop(threadio->thread);
3035         __vb2_cleanup_fileio(q);
3036         threadio->thread = NULL;
3037         kfree(threadio);
3038         q->threadio = NULL;
3039         return err;
3040 }
3041 EXPORT_SYMBOL_GPL(vb2_thread_stop);
3042
3043 MODULE_DESCRIPTION("Media buffer core framework");
3044 MODULE_AUTHOR("Pawel Osciak <pawel@osciak.com>, Marek Szyprowski");
3045 MODULE_LICENSE("GPL");