drivers/gpu/drm/virtio/virtgpu_submit.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright (C) 2015 Red Hat, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Authors:
   7  *    Dave Airlie
   8  *    Alon Levy
   9  */
  10
  11 #include <linux/dma-fence-unwrap.h>
  12 #include <linux/file.h>
  13 #include <linux/sync_file.h>
  14 #include <linux/uaccess.h>
  15
  16 #include <drm/drm_file.h>
  17 #include <drm/drm_syncobj.h>
  18 #include <drm/virtgpu_drm.h>
  19
  20 #include "virtgpu_drv.h"
  21
  22 struct virtio_gpu_submit_post_dep {
  23         struct drm_syncobj *syncobj;
  24         struct dma_fence_chain *chain;
  25         u64 point;
  26 };
  27
  28 struct virtio_gpu_submit {
  29         struct virtio_gpu_submit_post_dep *post_deps;
  30         unsigned int num_out_syncobjs;
  31
  32         struct drm_syncobj **in_syncobjs;
  33         unsigned int num_in_syncobjs;
  34
  35         struct virtio_gpu_object_array *buflist;
  36         struct drm_virtgpu_execbuffer *exbuf;
  37         struct virtio_gpu_fence *out_fence;
  38         struct virtio_gpu_fpriv *vfpriv;
  39         struct virtio_gpu_device *vgdev;
  40         struct sync_file *sync_file;
  41         struct drm_file *file;
  42         int out_fence_fd;
  43         u64 fence_ctx;
  44         u32 ring_idx;
  45         void *buf;
  46 };
  47
  48 static int virtio_gpu_do_fence_wait(struct virtio_gpu_submit *submit,
  49                                     struct dma_fence *in_fence)
  50 {
  51         u32 context = submit->fence_ctx + submit->ring_idx;
  52
  53         if (dma_fence_match_context(in_fence, context))
  54                 return 0;
  55
  56         return dma_fence_wait(in_fence, true);
  57 }
  58
  59 static int virtio_gpu_dma_fence_wait(struct virtio_gpu_submit *submit,
  60                                      struct dma_fence *fence)
  61 {
  62         struct dma_fence_unwrap itr;
  63         struct dma_fence *f;
  64         int err;
  65
  66         dma_fence_unwrap_for_each(f, &itr, fence) {
  67                 err = virtio_gpu_do_fence_wait(submit, f);
  68                 if (err)
  69                         return err;
  70         }
  71
  72         return 0;
  73 }
  74
  75 static void virtio_gpu_free_syncobjs(struct drm_syncobj **syncobjs,
  76                                      u32 nr_syncobjs)
  77 {
  78         u32 i = nr_syncobjs;
  79
  80         while (i--) {
  81                 if (syncobjs[i])
  82                         drm_syncobj_put(syncobjs[i]);
  83         }
  84
  85         kvfree(syncobjs);
  86 }
  87
  88 static int
  89 virtio_gpu_parse_deps(struct virtio_gpu_submit *submit)
  90 {
  91         struct drm_virtgpu_execbuffer *exbuf = submit->exbuf;
  92         struct drm_virtgpu_execbuffer_syncobj syncobj_desc;
  93         size_t syncobj_stride = exbuf->syncobj_stride;
  94         u32 num_in_syncobjs = exbuf->num_in_syncobjs;
  95         struct drm_syncobj **syncobjs;
  96         int ret = 0, i;
  97
  98         if (!num_in_syncobjs)
  99                 return 0;
 100
 101         /*
 102          * kvalloc at first tries to allocate memory using kmalloc and
 103          * falls back to vmalloc only on failure. It also uses __GFP_NOWARN
 104          * internally for allocations larger than a page size, preventing
 105          * storm of KMSG warnings.
 106          */
 107         syncobjs = kvcalloc(num_in_syncobjs, sizeof(*syncobjs), GFP_KERNEL);
 108         if (!syncobjs)
 109                 return -ENOMEM;
 110
 111         for (i = 0; i < num_in_syncobjs; i++) {
 112                 u64 address = exbuf->in_syncobjs + i * syncobj_stride;
 113                 struct dma_fence *fence;
 114
 115                 memset(&syncobj_desc, 0, sizeof(syncobj_desc));
 116
 117                 if (copy_from_user(&syncobj_desc,
 118                                    u64_to_user_ptr(address),
 119                                    min(syncobj_stride, sizeof(syncobj_desc)))) {
 120                         ret = -EFAULT;
 121                         break;
 122                 }
 123
 124                 if (syncobj_desc.flags & ~VIRTGPU_EXECBUF_SYNCOBJ_FLAGS) {
 125                         ret = -EINVAL;
 126                         break;
 127                 }
 128
 129                 ret = drm_syncobj_find_fence(submit->file, syncobj_desc.handle,
 130                                              syncobj_desc.point, 0, &fence);
 131                 if (ret)
 132                         break;
 133
 134                 ret = virtio_gpu_dma_fence_wait(submit, fence);
 135
 136                 dma_fence_put(fence);
 137                 if (ret)
 138                         break;
 139
 140                 if (syncobj_desc.flags & VIRTGPU_EXECBUF_SYNCOBJ_RESET) {
 141                         syncobjs[i] = drm_syncobj_find(submit->file,
 142                                                        syncobj_desc.handle);
 143                         if (!syncobjs[i]) {
 144                                 ret = -EINVAL;
 145                                 break;
 146                         }
 147                 }
 148         }
 149
 150         if (ret) {
 151                 virtio_gpu_free_syncobjs(syncobjs, i);
 152                 return ret;
 153         }
 154
 155         submit->num_in_syncobjs = num_in_syncobjs;
 156         submit->in_syncobjs = syncobjs;
 157
 158         return ret;
 159 }
 160
 161 static void virtio_gpu_reset_syncobjs(struct drm_syncobj **syncobjs,
 162                                       u32 nr_syncobjs)
 163 {
 164         u32 i;
 165
 166         for (i = 0; i < nr_syncobjs; i++) {
 167                 if (syncobjs[i])
 168                         drm_syncobj_replace_fence(syncobjs[i], NULL);
 169         }
 170 }
 171
 172 static void
 173 virtio_gpu_free_post_deps(struct virtio_gpu_submit_post_dep *post_deps,
 174                           u32 nr_syncobjs)
 175 {
 176         u32 i = nr_syncobjs;
 177
 178         while (i--) {
 179                 kfree(post_deps[i].chain);
 180                 drm_syncobj_put(post_deps[i].syncobj);
 181         }
 182
 183         kvfree(post_deps);
 184 }
 185
 186 static int virtio_gpu_parse_post_deps(struct virtio_gpu_submit *submit)
 187 {
 188         struct drm_virtgpu_execbuffer *exbuf = submit->exbuf;
 189         struct drm_virtgpu_execbuffer_syncobj syncobj_desc;
 190         struct virtio_gpu_submit_post_dep *post_deps;
 191         u32 num_out_syncobjs = exbuf->num_out_syncobjs;
 192         size_t syncobj_stride = exbuf->syncobj_stride;
 193         int ret = 0, i;
 194
 195         if (!num_out_syncobjs)
 196                 return 0;
 197
 198         post_deps = kvcalloc(num_out_syncobjs, sizeof(*post_deps), GFP_KERNEL);
 199         if (!post_deps)
 200                 return -ENOMEM;
 201
 202         for (i = 0; i < num_out_syncobjs; i++) {
 203                 u64 address = exbuf->out_syncobjs + i * syncobj_stride;
 204
 205                 memset(&syncobj_desc, 0, sizeof(syncobj_desc));
 206
 207                 if (copy_from_user(&syncobj_desc,
 208                                    u64_to_user_ptr(address),
 209                                    min(syncobj_stride, sizeof(syncobj_desc)))) {
 210                         ret = -EFAULT;
 211                         break;
 212                 }
 213
 214                 post_deps[i].point = syncobj_desc.point;
 215
 216                 if (syncobj_desc.flags) {
 217                         ret = -EINVAL;
 218                         break;
 219                 }
 220
 221                 if (syncobj_desc.point) {
 222                         post_deps[i].chain = dma_fence_chain_alloc();
 223                         if (!post_deps[i].chain) {
 224                                 ret = -ENOMEM;
 225                                 break;
 226                         }
 227                 }
 228
 229                 post_deps[i].syncobj = drm_syncobj_find(submit->file,
 230                                                         syncobj_desc.handle);
 231                 if (!post_deps[i].syncobj) {
 232                         kfree(post_deps[i].chain);
 233                         ret = -EINVAL;
 234                         break;
 235                 }
 236         }
 237
 238         if (ret) {
 239                 virtio_gpu_free_post_deps(post_deps, i);
 240                 return ret;
 241         }
 242
 243         submit->num_out_syncobjs = num_out_syncobjs;
 244         submit->post_deps = post_deps;
 245
 246         return 0;
 247 }
 248
 249 static void
 250 virtio_gpu_process_post_deps(struct virtio_gpu_submit *submit)
 251 {
 252         struct virtio_gpu_submit_post_dep *post_deps = submit->post_deps;
 253
 254         if (post_deps) {
 255                 struct dma_fence *fence = &submit->out_fence->f;
 256                 u32 i;
 257
 258                 for (i = 0; i < submit->num_out_syncobjs; i++) {
 259                         if (post_deps[i].chain) {
 260                                 drm_syncobj_add_point(post_deps[i].syncobj,
 261                                                       post_deps[i].chain,
 262                                                       fence, post_deps[i].point);
 263                                 post_deps[i].chain = NULL;
 264                         } else {
 265                                 drm_syncobj_replace_fence(post_deps[i].syncobj,
 266                                                           fence);
 267                         }
 268                 }
 269         }
 270 }
 271
 272 static int virtio_gpu_fence_event_create(struct drm_device *dev,
 273                                          struct drm_file *file,
 274                                          struct virtio_gpu_fence *fence,
 275                                          u32 ring_idx)
 276 {
 277         struct virtio_gpu_fence_event *e = NULL;
 278         int ret;
 279
 280         e = kzalloc(sizeof(*e), GFP_KERNEL);
 281         if (!e)
 282                 return -ENOMEM;
 283
 284         e->event.type = VIRTGPU_EVENT_FENCE_SIGNALED;
 285         e->event.length = sizeof(e->event);
 286
 287         ret = drm_event_reserve_init(dev, file, &e->base, &e->event);
 288         if (ret) {
 289                 kfree(e);
 290                 return ret;
 291         }
 292
 293         fence->e = e;
 294
 295         return 0;
 296 }
 297
 298 static int virtio_gpu_init_submit_buflist(struct virtio_gpu_submit *submit)
 299 {
 300         struct drm_virtgpu_execbuffer *exbuf = submit->exbuf;
 301         u32 *bo_handles;
 302
 303         if (!exbuf->num_bo_handles)
 304                 return 0;
 305
 306         bo_handles = kvmalloc_array(exbuf->num_bo_handles, sizeof(*bo_handles),
 307                                     GFP_KERNEL);
 308         if (!bo_handles)
 309                 return -ENOMEM;
 310
 311         if (copy_from_user(bo_handles, u64_to_user_ptr(exbuf->bo_handles),
 312                            exbuf->num_bo_handles * sizeof(*bo_handles))) {
 313                 kvfree(bo_handles);
 314                 return -EFAULT;
 315         }
 316
 317         submit->buflist = virtio_gpu_array_from_handles(submit->file, bo_handles,
 318                                                         exbuf->num_bo_handles);
 319         if (!submit->buflist) {
 320                 kvfree(bo_handles);
 321                 return -ENOENT;
 322         }
 323
 324         kvfree(bo_handles);
 325
 326         return 0;
 327 }
 328
 329 static void virtio_gpu_cleanup_submit(struct virtio_gpu_submit *submit)
 330 {
 331         virtio_gpu_reset_syncobjs(submit->in_syncobjs, submit->num_in_syncobjs);
 332         virtio_gpu_free_syncobjs(submit->in_syncobjs, submit->num_in_syncobjs);
 333         virtio_gpu_free_post_deps(submit->post_deps, submit->num_out_syncobjs);
 334
 335         if (!IS_ERR(submit->buf))
 336                 kvfree(submit->buf);
 337
 338         if (submit->buflist)
 339                 virtio_gpu_array_put_free(submit->buflist);
 340
 341         if (submit->out_fence_fd >= 0)
 342                 put_unused_fd(submit->out_fence_fd);
 343
 344         if (submit->out_fence)
 345                 dma_fence_put(&submit->out_fence->f);
 346
 347         if (submit->sync_file)
 348                 fput(submit->sync_file->file);
 349 }
 350
 351 static void virtio_gpu_submit(struct virtio_gpu_submit *submit)
 352 {
 353         virtio_gpu_cmd_submit(submit->vgdev, submit->buf, submit->exbuf->size,
 354                               submit->vfpriv->ctx_id, submit->buflist,
 355                               submit->out_fence);
 356         virtio_gpu_notify(submit->vgdev);
 357 }
 358
 359 static void virtio_gpu_complete_submit(struct virtio_gpu_submit *submit)
 360 {
 361         submit->buf = NULL;
 362         submit->buflist = NULL;
 363         submit->sync_file = NULL;
 364         submit->out_fence_fd = -1;
 365 }
 366
 367 static int virtio_gpu_init_submit(struct virtio_gpu_submit *submit,
 368                                   struct drm_virtgpu_execbuffer *exbuf,
 369                                   struct drm_device *dev,
 370                                   struct drm_file *file,
 371                                   u64 fence_ctx, u32 ring_idx)
 372 {
 373         struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
 374         struct virtio_gpu_device *vgdev = dev->dev_private;
 375         struct virtio_gpu_fence *out_fence;
 376         bool drm_fence_event;
 377         int err;
 378
 379         memset(submit, 0, sizeof(*submit));
 380
 381         if ((exbuf->flags & VIRTGPU_EXECBUF_RING_IDX) &&
 382             (vfpriv->ring_idx_mask & BIT_ULL(ring_idx)))
 383                 drm_fence_event = true;
 384         else
 385                 drm_fence_event = false;
 386
 387         if ((exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) ||
 388             exbuf->num_out_syncobjs ||
 389             exbuf->num_bo_handles ||
 390             drm_fence_event)
 391                 out_fence = virtio_gpu_fence_alloc(vgdev, fence_ctx, ring_idx);
 392         else
 393                 out_fence = NULL;
 394
 395         if (drm_fence_event) {
 396                 err = virtio_gpu_fence_event_create(dev, file, out_fence, ring_idx);
 397                 if (err) {
 398                         dma_fence_put(&out_fence->f);
 399                         return err;
 400                 }
 401         }
 402
 403         submit->out_fence = out_fence;
 404         submit->fence_ctx = fence_ctx;
 405         submit->ring_idx = ring_idx;
 406         submit->out_fence_fd = -1;
 407         submit->vfpriv = vfpriv;
 408         submit->vgdev = vgdev;
 409         submit->exbuf = exbuf;
 410         submit->file = file;
 411
 412         err = virtio_gpu_init_submit_buflist(submit);
 413         if (err)
 414                 return err;
 415
 416         submit->buf = vmemdup_user(u64_to_user_ptr(exbuf->command), exbuf->size);
 417         if (IS_ERR(submit->buf))
 418                 return PTR_ERR(submit->buf);
 419
 420         if (exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) {
 421                 err = get_unused_fd_flags(O_CLOEXEC);
 422                 if (err < 0)
 423                         return err;
 424
 425                 submit->out_fence_fd = err;
 426
 427                 submit->sync_file = sync_file_create(&out_fence->f);
 428                 if (!submit->sync_file)
 429                         return -ENOMEM;
 430         }
 431
 432         return 0;
 433 }
 434
 435 static int virtio_gpu_wait_in_fence(struct virtio_gpu_submit *submit)
 436 {
 437         int ret = 0;
 438
 439         if (submit->exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_IN) {
 440                 struct dma_fence *in_fence =
 441                                 sync_file_get_fence(submit->exbuf->fence_fd);
 442                 if (!in_fence)
 443                         return -EINVAL;
 444
 445                 /*
 446                  * Wait if the fence is from a foreign context, or if the
 447                  * fence array contains any fence from a foreign context.
 448                  */
 449                 ret = virtio_gpu_dma_fence_wait(submit, in_fence);
 450
 451                 dma_fence_put(in_fence);
 452         }
 453
 454         return ret;
 455 }
 456
 457 static void virtio_gpu_install_out_fence_fd(struct virtio_gpu_submit *submit)
 458 {
 459         if (submit->sync_file) {
 460                 submit->exbuf->fence_fd = submit->out_fence_fd;
 461                 fd_install(submit->out_fence_fd, submit->sync_file->file);
 462         }
 463 }
 464
 465 static int virtio_gpu_lock_buflist(struct virtio_gpu_submit *submit)
 466 {
 467         if (submit->buflist)
 468                 return virtio_gpu_array_lock_resv(submit->buflist);
 469
 470         return 0;
 471 }
 472
 473 int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data,
 474                                 struct drm_file *file)
 475 {
 476         struct virtio_gpu_device *vgdev = dev->dev_private;
 477         struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
 478         u64 fence_ctx = vgdev->fence_drv.context;
 479         struct drm_virtgpu_execbuffer *exbuf = data;
 480         struct virtio_gpu_submit submit;
 481         u32 ring_idx = 0;
 482         int ret = -EINVAL;
 483
 484         if (!vgdev->has_virgl_3d)
 485                 return -ENOSYS;
 486
 487         if (exbuf->flags & ~VIRTGPU_EXECBUF_FLAGS)
 488                 return ret;
 489
 490         if (exbuf->flags & VIRTGPU_EXECBUF_RING_IDX) {
 491                 if (exbuf->ring_idx >= vfpriv->num_rings)
 492                         return ret;
 493
 494                 if (!vfpriv->base_fence_ctx)
 495                         return ret;
 496
 497                 fence_ctx = vfpriv->base_fence_ctx;
 498                 ring_idx = exbuf->ring_idx;
 499         }
 500
 501         virtio_gpu_create_context(dev, file);
 502
 503         ret = virtio_gpu_init_submit(&submit, exbuf, dev, file,
 504                                      fence_ctx, ring_idx);
 505         if (ret)
 506                 goto cleanup;
 507
 508         ret = virtio_gpu_parse_post_deps(&submit);
 509         if (ret)
 510                 goto cleanup;
 511
 512         ret = virtio_gpu_parse_deps(&submit);
 513         if (ret)
 514                 goto cleanup;
 515
 516         /*
 517          * Await in-fences in the end of the job submission path to
 518          * optimize the path by proceeding directly to the submission
 519          * to virtio after the waits.
 520          */
 521         ret = virtio_gpu_wait_in_fence(&submit);
 522         if (ret)
 523                 goto cleanup;
 524
 525         ret = virtio_gpu_lock_buflist(&submit);
 526         if (ret)
 527                 goto cleanup;
 528
 529         virtio_gpu_submit(&submit);
 530
 531         /*
 532          * Set up usr-out data after submitting the job to optimize
 533          * the job submission path.
 534          */
 535         virtio_gpu_install_out_fence_fd(&submit);
 536         virtio_gpu_process_post_deps(&submit);
 537         virtio_gpu_complete_submit(&submit);
 538 cleanup:
 539         virtio_gpu_cleanup_submit(&submit);
 540
 541         return ret;
 542 }