amdgpu/amdgpu_cs.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22 */
  23 #include <stdlib.h>
  24 #include <stdio.h>
  25 #include <string.h>
  26 #include <errno.h>
  27 #include <pthread.h>
  28 #include <sched.h>
  29 #include <sys/ioctl.h>
  30
  31 #include "xf86drm.h"
  32 #include "amdgpu_drm.h"
  33 #include "amdgpu_internal.h"
  34
  35 /**
  36  * Create an IB buffer.
  37  *
  38  * \param   dev - \c [in] Device handle
  39  * \param   context - \c [in] GPU Context
  40  * \param   ib_size - \c [in] Size of allocation
  41  * \param   ib - \c [out] return the pointer to the created IB buffer
  42  *
  43  * \return  0 on success otherwise POSIX Error code
  44 */
  45 static int amdgpu_cs_create_ib(amdgpu_device_handle dev,
  46                                amdgpu_context_handle context,
  47                                enum amdgpu_cs_ib_size ib_size,
  48                                amdgpu_ib_handle *ib)
  49 {
  50         struct amdgpu_bo_alloc_request alloc_buffer;
  51         struct amdgpu_bo_alloc_result info;
  52         int r;
  53         void *cpu;
  54         struct amdgpu_ib *new_ib;
  55
  56         memset(&alloc_buffer, 0, sizeof(alloc_buffer));
  57
  58         switch (ib_size) {
  59         case amdgpu_cs_ib_size_4K:
  60                 alloc_buffer.alloc_size = 4 * 1024;
  61                 break;
  62         case amdgpu_cs_ib_size_16K:
  63                 alloc_buffer.alloc_size = 16 * 1024;
  64                 break;
  65         case amdgpu_cs_ib_size_32K:
  66                 alloc_buffer.alloc_size = 32 * 1024;
  67                 break;
  68         case amdgpu_cs_ib_size_64K:
  69                 alloc_buffer.alloc_size = 64 * 1024;
  70                 break;
  71         case amdgpu_cs_ib_size_128K:
  72                 alloc_buffer.alloc_size = 128 * 1024;
  73                 break;
  74         default:
  75                 return -EINVAL;
  76         }
  77
  78         alloc_buffer.phys_alignment = 4 * 1024;
  79
  80         alloc_buffer.preferred_heap = AMDGPU_GEM_DOMAIN_GTT;
  81
  82         r = amdgpu_bo_alloc(dev,
  83                             &alloc_buffer,
  84                             &info);
  85         if (r)
  86                 return r;
  87
  88         r = amdgpu_bo_cpu_map(info.buf_handle, &cpu);
  89         if (r) {
  90                 amdgpu_bo_free(info.buf_handle);
  91                 return r;
  92         }
  93
  94         new_ib = malloc(sizeof(struct amdgpu_ib));
  95         if (NULL == new_ib) {
  96                 amdgpu_bo_cpu_unmap(info.buf_handle);
  97                 amdgpu_bo_free(info.buf_handle);
  98                 return -ENOMEM;
  99         }
 100
 101         new_ib->buf_handle = info.buf_handle;
 102         new_ib->cpu = cpu;
 103         new_ib->virtual_mc_base_address = info.virtual_mc_base_address;
 104         new_ib->ib_size = ib_size;
 105         *ib = new_ib;
 106         return 0;
 107 }
 108
 109 /**
 110  * Destroy an IB buffer.
 111  *
 112  * \param   dev - \c [in]  Device handle
 113  * \param   ib - \c [in] the IB buffer
 114  *
 115  * \return  0 on success otherwise POSIX Error code
 116 */
 117 static int amdgpu_cs_destroy_ib(amdgpu_device_handle dev,
 118                                 amdgpu_ib_handle ib)
 119 {
 120         int r;
 121         r = amdgpu_bo_cpu_unmap(ib->buf_handle);
 122         if (r)
 123                 return r;
 124
 125         r = amdgpu_bo_free(ib->buf_handle);
 126         if (r)
 127                 return r;
 128
 129         free(ib);
 130         return 0;
 131 }
 132
 133 /**
 134  * Initialize IB pools to empty.
 135  *
 136  * \param   context - \c [in]  GPU Context
 137  *
 138  * \return  0 on success otherwise POSIX Error code
 139 */
 140 static int amdgpu_cs_init_ib_pool(amdgpu_context_handle context)
 141 {
 142         int i;
 143         int r;
 144
 145         r = pthread_mutex_init(&context->pool_mutex, NULL);
 146         if (r)
 147                 return r;
 148
 149         for (i = 0; i < AMDGPU_CS_IB_SIZE_NUM; i++)
 150                 LIST_INITHEAD(&context->ib_pools[i]);
 151
 152         return 0;
 153 }
 154
 155 /**
 156  * Allocate an IB buffer from IB pools.
 157  *
 158  * \param   dev - \c [in]  Device handle
 159  * \param   context - \c [in] GPU Context
 160  * \param   ib_size - \c [in]  Size of allocation
 161  * \param   ib - \c [out] return the pointer to the allocated IB buffer
 162  *
 163  * \return  0 on success otherwise POSIX Error code
 164 */
 165 static int amdgpu_cs_alloc_from_ib_pool(amdgpu_device_handle dev,
 166                                         amdgpu_context_handle context,
 167                                         enum amdgpu_cs_ib_size ib_size,
 168                                         amdgpu_ib_handle *ib)
 169 {
 170         int r;
 171         struct list_head *head;
 172         head = &context->ib_pools[ib_size];
 173
 174         r = -ENOMEM;
 175         pthread_mutex_lock(&context->pool_mutex);
 176         if (!LIST_IS_EMPTY(head)) {
 177                 *ib = LIST_ENTRY(struct amdgpu_ib, head->next, list_node);
 178                 LIST_DEL(&(*ib)->list_node);
 179                 r = 0;
 180         }
 181         pthread_mutex_unlock(&context->pool_mutex);
 182
 183         return r;
 184 }
 185
 186 /**
 187  * Free an IB buffer to IB pools.
 188  *
 189  * \param   context - \c [in]  GPU Context
 190  * \param   ib - \c [in] the IB buffer
 191  *
 192  * \return  N/A
 193 */
 194 static void amdgpu_cs_free_to_ib_pool(amdgpu_context_handle context,
 195                                       amdgpu_ib_handle ib)
 196 {
 197         struct list_head *head;
 198         head = &context->ib_pools[ib->ib_size];
 199         pthread_mutex_lock(&context->pool_mutex);
 200         LIST_ADD(&ib->list_node, head);
 201         pthread_mutex_unlock(&context->pool_mutex);
 202         return;
 203 }
 204
 205 /**
 206  * Destroy all IB buffers in pools
 207  *
 208  * \param   dev - \c [in]  Device handle
 209  * \param   context - \c [in]  GPU Context
 210  *
 211  * \return  0 on success otherwise POSIX Error code
 212 */
 213 static int amdgpu_cs_destroy_ib_pool(amdgpu_device_handle dev,
 214                                      amdgpu_context_handle context)
 215 {
 216         int i;
 217         int r;
 218         struct list_head *head;
 219         struct amdgpu_ib *next;
 220         struct amdgpu_ib *storage;
 221
 222         r = 0;
 223         pthread_mutex_lock(&context->pool_mutex);
 224         for (i = 0; i < AMDGPU_CS_IB_SIZE_NUM; i++) {
 225                 head = &context->ib_pools[i];
 226                 LIST_FOR_EACH_ENTRY_SAFE(next, storage, head, list_node) {
 227                         r = amdgpu_cs_destroy_ib(dev, next);
 228                         if (r)
 229                                 break;
 230                 }
 231         }
 232         pthread_mutex_unlock(&context->pool_mutex);
 233         pthread_mutex_destroy(&context->pool_mutex);
 234         return r;
 235 }
 236
 237 /**
 238  * Initialize pending IB lists
 239  *
 240  * \param   context - \c [in]  GPU Context
 241  *
 242  * \return  0 on success otherwise POSIX Error code
 243 */
 244 static int amdgpu_cs_init_pendings(amdgpu_context_handle context)
 245 {
 246         unsigned ip, inst;
 247         uint32_t ring;
 248         int r;
 249
 250         r = pthread_mutex_init(&context->pendings_mutex, NULL);
 251         if (r)
 252                 return r;
 253
 254         for (ip = 0; ip < AMDGPU_HW_IP_NUM; ip++)
 255                 for (inst = 0; inst < AMDGPU_HW_IP_INSTANCE_MAX_COUNT; inst++)
 256                         for (ring = 0; ring < AMDGPU_CS_MAX_RINGS; ring++)
 257                                 LIST_INITHEAD(&context->pendings[ip][inst][ring]);
 258
 259         LIST_INITHEAD(&context->freed);
 260         return 0;
 261 }
 262
 263 /**
 264  * Free pending IBs
 265  *
 266  * \param   dev - \c [in]  Device handle
 267  * \param   context - \c [in]  GPU Context
 268  *
 269  * \return  0 on success otherwise POSIX Error code
 270 */
 271 static int amdgpu_cs_destroy_pendings(amdgpu_device_handle dev,
 272                                       amdgpu_context_handle context)
 273 {
 274         int ip, inst;
 275         uint32_t ring;
 276         int r;
 277         struct amdgpu_ib *next;
 278         struct amdgpu_ib *s;
 279         struct list_head *head;
 280
 281         r = 0;
 282         pthread_mutex_lock(&context->pendings_mutex);
 283         for (ip = 0; ip < AMDGPU_HW_IP_NUM; ip++)
 284                 for (inst = 0; inst < AMDGPU_HW_IP_INSTANCE_MAX_COUNT; inst++)
 285                         for (ring = 0; ring < AMDGPU_CS_MAX_RINGS; ring++) {
 286                                 head = &context->pendings[ip][inst][ring];
 287                                 LIST_FOR_EACH_ENTRY_SAFE(next, s, head, list_node) {
 288                                         r = amdgpu_cs_destroy_ib(dev, next);
 289                                         if (r)
 290                                                 break;
 291                                 }
 292                         }
 293
 294         head = &context->freed;
 295         LIST_FOR_EACH_ENTRY_SAFE(next, s, head, list_node) {
 296                 r = amdgpu_cs_destroy_ib(dev, next);
 297                 if (r)
 298                         break;
 299         }
 300
 301         pthread_mutex_unlock(&context->pendings_mutex);
 302         pthread_mutex_destroy(&context->pendings_mutex);
 303         return r;
 304 }
 305
 306 /**
 307  * Add IB to pending IB lists without holding sequence_mutex.
 308  *
 309  * \param   context - \c [in]  GPU Context
 310  * \param   ib - \c [in]  ib to added to pending lists
 311  * \param   ip - \c [in]  hw ip block
 312  * \param   ip_instance - \c [in]  instance of the hw ip block
 313  * \param   ring - \c [in]  Ring of hw ip
 314  *
 315  * \return  N/A
 316 */
 317 static void amdgpu_cs_add_pending(amdgpu_context_handle context,
 318                                   amdgpu_ib_handle ib,
 319                                   unsigned ip, unsigned ip_instance,
 320                                   uint32_t ring)
 321 {
 322         struct list_head *head;
 323         pthread_mutex_lock(&context->pendings_mutex);
 324         head = &context->pendings[ip][ip_instance][ring];
 325         LIST_ADDTAIL(&ib->list_node, head);
 326         pthread_mutex_unlock(&context->pendings_mutex);
 327         return;
 328 }
 329
 330 /**
 331  * Garbage collector on a pending IB list without holding pendings_mutex.
 332  * This function by itself is not multithread safe.
 333  *
 334  * \param   context - \c [in]  GPU Context
 335  * \param   ip - \c [in]  hw ip block
 336  * \param   ip_instance - \c [in]  instance of the hw ip block
 337  * \param   ring - \c [in]  Ring of hw ip
 338  * \param   expired_fence - \c [in]  fence expired
 339  *
 340  * \return  N/A
 341  * \note Hold pendings_mutex before calling this function.
 342 */
 343 static void amdgpu_cs_pending_gc_not_safe(amdgpu_context_handle context,
 344                                           unsigned ip, unsigned ip_instance,
 345                                           uint32_t ring,
 346                                           uint64_t expired_fence)
 347 {
 348         struct list_head *head;
 349         struct amdgpu_ib *next;
 350         struct amdgpu_ib *s;
 351         int r;
 352
 353         head = &context->pendings[ip][ip_instance][ring];
 354         LIST_FOR_EACH_ENTRY_SAFE(next, s, head, list_node)
 355                 if (next->cs_handle <= expired_fence) {
 356                         LIST_DEL(&next->list_node);
 357                         amdgpu_cs_free_to_ib_pool(context, next);
 358                 } else {
 359                         /* The pending list is a sorted list.
 360                            There is no need to continue. */
 361                         break;
 362                 }
 363
 364         /* walk the freed list as well */
 365         head = &context->freed;
 366         LIST_FOR_EACH_ENTRY_SAFE(next, s, head, list_node) {
 367                 bool busy;
 368
 369                 r = amdgpu_bo_wait_for_idle(next->buf_handle, 0, &busy);
 370                 if (r || busy)
 371                         break;
 372
 373                 LIST_DEL(&next->list_node);
 374                 amdgpu_cs_free_to_ib_pool(context, next);
 375         }
 376
 377         return;
 378 }
 379
 380 /**
 381  * Garbage collector on a pending IB list
 382  *
 383  * \param   context - \c [in]  GPU Context
 384  * \param   ip - \c [in]  hw ip block
 385  * \param   ip_instance - \c [in]  instance of the hw ip block
 386  * \param   ring - \c [in]  Ring of hw ip
 387  * \param   expired_fence - \c [in]  fence expired
 388  *
 389  * \return  N/A
 390 */
 391 static void amdgpu_cs_pending_gc(amdgpu_context_handle context,
 392                                  unsigned ip, unsigned ip_instance,
 393                                  uint32_t ring,
 394                                  uint64_t expired_fence)
 395 {
 396         pthread_mutex_lock(&context->pendings_mutex);
 397         amdgpu_cs_pending_gc_not_safe(context, ip, ip_instance, ring,
 398                                       expired_fence);
 399         pthread_mutex_unlock(&context->pendings_mutex);
 400         return;
 401 }
 402
 403 /**
 404  * Garbage collector on all pending IB lists
 405  *
 406  * \param   context - \c [in]  GPU Context
 407  *
 408  * \return  N/A
 409 */
 410 static void amdgpu_cs_all_pending_gc(amdgpu_context_handle context)
 411 {
 412         unsigned ip, inst;
 413         uint32_t ring;
 414         uint64_t expired_fences[AMDGPU_HW_IP_NUM][AMDGPU_HW_IP_INSTANCE_MAX_COUNT][AMDGPU_CS_MAX_RINGS];
 415
 416         pthread_mutex_lock(&context->sequence_mutex);
 417         for (ip = 0; ip < AMDGPU_HW_IP_NUM; ip++)
 418                 for (inst = 0; inst < AMDGPU_HW_IP_INSTANCE_MAX_COUNT; inst++)
 419                         for (ring = 0; ring < AMDGPU_CS_MAX_RINGS; ring++)
 420                                 expired_fences[ip][inst][ring] =
 421                                         context->expired_fences[ip][inst][ring];
 422         pthread_mutex_unlock(&context->sequence_mutex);
 423
 424         pthread_mutex_lock(&context->pendings_mutex);
 425         for (ip = 0; ip < AMDGPU_HW_IP_NUM; ip++)
 426                 for (inst = 0; inst < AMDGPU_HW_IP_INSTANCE_MAX_COUNT; inst++)
 427                         for (ring = 0; ring < AMDGPU_CS_MAX_RINGS; ring++)
 428                                 amdgpu_cs_pending_gc_not_safe(context, ip, inst, ring,
 429                                         expired_fences[ip][inst][ring]);
 430         pthread_mutex_unlock(&context->pendings_mutex);
 431 }
 432
 433 /**
 434  * Allocate an IB buffer
 435  * If there is no free IB buffer in pools, create one.
 436  *
 437  * \param   dev - \c [in] Device handle
 438  * \param   context - \c [in] GPU Context
 439  * \param   ib_size - \c [in] Size of allocation
 440  * \param   ib - \c [out] return the pointer to the allocated IB buffer
 441  *
 442  * \return  0 on success otherwise POSIX Error code
 443 */
 444 static int amdgpu_cs_alloc_ib_local(amdgpu_device_handle dev,
 445                                     amdgpu_context_handle context,
 446                                     enum amdgpu_cs_ib_size ib_size,
 447                                     amdgpu_ib_handle *ib)
 448 {
 449         int r;
 450
 451         r = amdgpu_cs_alloc_from_ib_pool(dev, context, ib_size, ib);
 452         if (!r)
 453                 return r;
 454
 455         amdgpu_cs_all_pending_gc(context);
 456
 457         /* Retry to allocate from free IB pools after garbage collector. */
 458         r = amdgpu_cs_alloc_from_ib_pool(dev, context, ib_size, ib);
 459         if (!r)
 460                 return r;
 461
 462         /* There is no suitable IB in free pools. Create one. */
 463         r = amdgpu_cs_create_ib(dev, context, ib_size, ib);
 464         return r;
 465 }
 466
 467 int amdgpu_cs_alloc_ib(amdgpu_device_handle dev,
 468                        amdgpu_context_handle context,
 469                        enum amdgpu_cs_ib_size ib_size,
 470                        struct amdgpu_cs_ib_alloc_result *output)
 471 {
 472         int r;
 473         amdgpu_ib_handle ib;
 474
 475         if (NULL == dev)
 476                 return -EINVAL;
 477         if (NULL == context)
 478                 return -EINVAL;
 479         if (NULL == output)
 480                 return -EINVAL;
 481         if (ib_size >= AMDGPU_CS_IB_SIZE_NUM)
 482                 return -EINVAL;
 483
 484         r = amdgpu_cs_alloc_ib_local(dev, context, ib_size, &ib);
 485         if (!r) {
 486                 output->handle = ib;
 487                 output->cpu = ib->cpu;
 488                 output->mc_address = ib->virtual_mc_base_address;
 489         }
 490
 491         return r;
 492 }
 493
 494 int amdgpu_cs_free_ib(amdgpu_device_handle dev,
 495                       amdgpu_context_handle context,
 496                       amdgpu_ib_handle handle)
 497 {
 498         if (NULL == dev)
 499                 return -EINVAL;
 500         if (NULL == context)
 501                 return -EINVAL;
 502         if (NULL == handle)
 503                 return -EINVAL;
 504
 505         pthread_mutex_lock(&context->pendings_mutex);
 506         LIST_ADD(&handle->list_node, &context->freed);
 507         pthread_mutex_unlock(&context->pendings_mutex);
 508         return 0;
 509 }
 510
 511 /**
 512  * Create command submission context
 513  *
 514  * \param   dev - \c [in] amdgpu device handle
 515  * \param   context - \c [out] amdgpu context handle
 516  *
 517  * \return  0 on success otherwise POSIX Error code
 518 */
 519 int amdgpu_cs_ctx_create(amdgpu_device_handle dev,
 520                          amdgpu_context_handle *context)
 521 {
 522         struct amdgpu_context *gpu_context;
 523         union drm_amdgpu_ctx args;
 524         int r;
 525
 526         if (NULL == dev)
 527                 return -EINVAL;
 528         if (NULL == context)
 529                 return -EINVAL;
 530
 531         gpu_context = calloc(1, sizeof(struct amdgpu_context));
 532         if (NULL == gpu_context)
 533                 return -ENOMEM;
 534
 535         r = pthread_mutex_init(&gpu_context->sequence_mutex, NULL);
 536         if (r)
 537                 goto error_mutex;
 538
 539         r = amdgpu_cs_init_ib_pool(gpu_context);
 540         if (r)
 541                 goto error_pool;
 542
 543         r = amdgpu_cs_init_pendings(gpu_context);
 544         if (r)
 545                 goto error_pendings;
 546
 547         r = amdgpu_cs_alloc_ib_local(dev, gpu_context, amdgpu_cs_ib_size_4K,
 548                                      &gpu_context->fence_ib);
 549         if (r)
 550                 goto error_fence_ib;
 551
 552
 553         memset(&args, 0, sizeof(args));
 554         args.in.op = AMDGPU_CTX_OP_ALLOC_CTX;
 555         r = drmCommandWriteRead(dev->fd, DRM_AMDGPU_CTX, &args, sizeof(args));
 556         if (r)
 557                 goto error_kernel;
 558
 559         gpu_context->id = args.out.alloc.ctx_id;
 560         *context = (amdgpu_context_handle)gpu_context;
 561
 562         return 0;
 563
 564 error_kernel:
 565         amdgpu_cs_free_ib(dev, gpu_context, gpu_context->fence_ib);
 566
 567 error_fence_ib:
 568         amdgpu_cs_destroy_pendings(dev, gpu_context);
 569
 570 error_pendings:
 571         amdgpu_cs_destroy_ib_pool(dev, gpu_context);
 572
 573 error_pool:
 574         pthread_mutex_destroy(&gpu_context->sequence_mutex);
 575
 576 error_mutex:
 577         free(gpu_context);
 578         return r;
 579 }
 580
 581 /**
 582  * Release command submission context
 583  *
 584  * \param   dev - \c [in] amdgpu device handle
 585  * \param   context - \c [in] amdgpu context handle
 586  *
 587  * \return  0 on success otherwise POSIX Error code
 588 */
 589 int amdgpu_cs_ctx_free(amdgpu_device_handle dev,
 590                        amdgpu_context_handle context)
 591 {
 592         int r;
 593         union drm_amdgpu_ctx args;
 594
 595         if (NULL == dev)
 596                 return -EINVAL;
 597         if (NULL == context)
 598                 return -EINVAL;
 599
 600         r = amdgpu_cs_free_ib(dev, context, context->fence_ib);
 601         if (r)
 602                 return r;
 603
 604         r = amdgpu_cs_destroy_pendings(dev, context);
 605         if (r)
 606                 return r;
 607
 608         r = amdgpu_cs_destroy_ib_pool(dev, context);
 609         if (r)
 610                 return r;
 611
 612         pthread_mutex_destroy(&context->sequence_mutex);
 613
 614         /* now deal with kernel side */
 615         memset(&args, 0, sizeof(args));
 616         args.in.op = AMDGPU_CTX_OP_FREE_CTX;
 617         args.in.ctx_id = context->id;
 618         r = drmCommandWriteRead(dev->fd, DRM_AMDGPU_CTX, &args, sizeof(args));
 619
 620         free(context);
 621
 622         return r;
 623 }
 624
 625 static int amdgpu_cs_create_bo_list(amdgpu_device_handle dev,
 626                                     amdgpu_context_handle context,
 627                                     struct amdgpu_cs_request *request,
 628                                     amdgpu_ib_handle fence_ib,
 629                                     uint32_t *handle)
 630 {
 631         struct drm_amdgpu_bo_list_entry *list;
 632         union drm_amdgpu_bo_list args;
 633         unsigned num_resources;
 634         unsigned i;
 635         int r;
 636
 637         num_resources = request->number_of_resources;
 638         if (fence_ib)
 639                 ++num_resources;
 640
 641         list = alloca(sizeof(struct drm_amdgpu_bo_list_entry) * num_resources);
 642
 643         memset(&args, 0, sizeof(args));
 644         args.in.operation = AMDGPU_BO_LIST_OP_CREATE;
 645         args.in.bo_number = num_resources;
 646         args.in.bo_info_size = sizeof(struct drm_amdgpu_bo_list_entry);
 647         args.in.bo_info_ptr = (uint64_t)(uintptr_t)list;
 648
 649         for (i = 0; i < request->number_of_resources; i++) {
 650                 list[i].bo_handle = request->resources[i]->handle;
 651                 if (request->resource_flags)
 652                         list[i].bo_priority = request->resource_flags[i];
 653                 else
 654                         list[i].bo_priority = 0;
 655         }
 656
 657         if (fence_ib)
 658                 list[i].bo_handle = fence_ib->buf_handle->handle;
 659
 660         r = drmCommandWriteRead(dev->fd, DRM_AMDGPU_BO_LIST,
 661                                 &args, sizeof(args));
 662         if (r)
 663                 return r;
 664
 665         *handle = args.out.list_handle;
 666         return 0;
 667 }
 668
 669 static int amdgpu_cs_free_bo_list(amdgpu_device_handle dev, uint32_t handle)
 670 {
 671         union drm_amdgpu_bo_list args;
 672         int r;
 673
 674         memset(&args, 0, sizeof(args));
 675         args.in.operation = AMDGPU_BO_LIST_OP_DESTROY;
 676         args.in.list_handle = handle;
 677
 678         r = drmCommandWriteRead(dev->fd, DRM_AMDGPU_BO_LIST,
 679                                 &args, sizeof(args));
 680
 681         return r;
 682 }
 683
 684 static uint32_t amdgpu_cs_fence_index(unsigned ip, unsigned ring)
 685 {
 686         return ip * AMDGPU_CS_MAX_RINGS + ring;
 687 }
 688
 689 /**
 690  * Submit command to kernel DRM
 691  * \param   dev - \c [in]  Device handle
 692  * \param   context - \c [in]  GPU Context
 693  * \param   ibs_request - \c [in]  Pointer to submission requests
 694  * \param   fence - \c [out] return fence for this submission
 695  *
 696  * \return  0 on success otherwise POSIX Error code
 697  * \sa amdgpu_cs_submit()
 698 */
 699 static int amdgpu_cs_submit_one(amdgpu_device_handle dev,
 700                                 amdgpu_context_handle context,
 701                                 struct amdgpu_cs_request *ibs_request,
 702                                 uint64_t *fence)
 703 {
 704         int r;
 705         uint32_t i, size;
 706         union drm_amdgpu_cs cs;
 707         uint64_t *chunk_array;
 708         struct drm_amdgpu_cs_chunk *chunks;
 709         struct drm_amdgpu_cs_chunk_data *chunk_data;
 710         uint32_t bo_list_handle;
 711
 712         if (ibs_request->ip_type >= AMDGPU_HW_IP_NUM)
 713                 return -EINVAL;
 714         if (ibs_request->ring >= AMDGPU_CS_MAX_RINGS)
 715                 return -EINVAL;
 716         if (ibs_request->number_of_ibs > AMDGPU_CS_MAX_IBS_PER_SUBMIT)
 717                 return -EINVAL;
 718
 719         size = (ibs_request->number_of_ibs + 1) * ((sizeof(uint64_t) +
 720                 sizeof(struct drm_amdgpu_cs_chunk) +
 721                 sizeof(struct drm_amdgpu_cs_chunk_data)) +
 722                ibs_request->number_of_resources + 1) *
 723                sizeof(struct drm_amdgpu_bo_list_entry);
 724         chunk_array = malloc(size);
 725         if (NULL == chunk_array)
 726                 return -ENOMEM;
 727         memset(chunk_array, 0, size);
 728
 729         chunks = (struct drm_amdgpu_cs_chunk *)(chunk_array + ibs_request->number_of_ibs + 1);
 730         chunk_data = (struct drm_amdgpu_cs_chunk_data *)(chunks + ibs_request->number_of_ibs + 1);
 731
 732         memset(&cs, 0, sizeof(cs));
 733         cs.in.chunks = (uint64_t)(uintptr_t)chunk_array;
 734         cs.in.ctx_id = context->id;
 735         cs.in.num_chunks = ibs_request->number_of_ibs;
 736         /* IB chunks */
 737         for (i = 0; i < ibs_request->number_of_ibs; i++) {
 738                 struct amdgpu_cs_ib_info *ib;
 739                 chunk_array[i] = (uint64_t)(uintptr_t)&chunks[i];
 740                 chunks[i].chunk_id = AMDGPU_CHUNK_ID_IB;
 741                 chunks[i].length_dw = sizeof(struct drm_amdgpu_cs_chunk_ib) / 4;
 742                 chunks[i].chunk_data = (uint64_t)(uintptr_t)&chunk_data[i];
 743
 744                 ib = &ibs_request->ibs[i];
 745
 746                 chunk_data[i].ib_data.handle = ib->ib_handle->buf_handle->handle;
 747                 chunk_data[i].ib_data.va_start = ib->ib_handle->virtual_mc_base_address;
 748                 chunk_data[i].ib_data.ib_bytes = ib->size * 4;
 749                 chunk_data[i].ib_data.ip_type = ibs_request->ip_type;
 750                 chunk_data[i].ib_data.ip_instance = ibs_request->ip_instance;
 751                 chunk_data[i].ib_data.ring = ibs_request->ring;
 752
 753                 if (ib->flags & AMDGPU_CS_GFX_IB_CE)
 754                         chunk_data[i].ib_data.flags = AMDGPU_IB_FLAG_CE;
 755         }
 756
 757         r = amdgpu_cs_create_bo_list(dev, context, ibs_request, NULL,
 758                                      &bo_list_handle);
 759         if (r)
 760                 goto error_unlock;
 761
 762         cs.in.bo_list_handle = bo_list_handle;
 763         pthread_mutex_lock(&context->sequence_mutex);
 764
 765         if (ibs_request->ip_type != AMDGPU_HW_IP_UVD &&
 766             ibs_request->ip_type != AMDGPU_HW_IP_VCE) {
 767                 i = cs.in.num_chunks++;
 768
 769                 /* fence chunk */
 770                 chunk_array[i] = (uint64_t)(uintptr_t)&chunks[i];
 771                 chunks[i].chunk_id = AMDGPU_CHUNK_ID_FENCE;
 772                 chunks[i].length_dw = sizeof(struct drm_amdgpu_cs_chunk_fence) / 4;
 773                 chunks[i].chunk_data = (uint64_t)(uintptr_t)&chunk_data[i];
 774
 775                 /* fence bo handle */
 776                 chunk_data[i].fence_data.handle = context->fence_ib->buf_handle->handle;
 777                 /* offset */
 778                 chunk_data[i].fence_data.offset = amdgpu_cs_fence_index(
 779                         ibs_request->ip_type, ibs_request->ring);
 780                 chunk_data[i].fence_data.offset *= sizeof(uint64_t);
 781         }
 782
 783         r = drmCommandWriteRead(dev->fd, DRM_AMDGPU_CS,
 784                                 &cs, sizeof(cs));
 785         if (r)
 786                 goto error_unlock;
 787
 788
 789         /* Hold sequence_mutex while adding record to the pending list.
 790            So the pending list is a sorted list according to fence value. */
 791
 792         for (i = 0; i < ibs_request->number_of_ibs; i++) {
 793                 struct amdgpu_cs_ib_info *ib;
 794
 795                 ib = &ibs_request->ibs[i];
 796                 if (ib->flags & AMDGPU_CS_REUSE_IB)
 797                         continue;
 798
 799                 ib->ib_handle->cs_handle = cs.out.handle;
 800
 801                 amdgpu_cs_add_pending(context, ib->ib_handle, ibs_request->ip_type,
 802                                       ibs_request->ip_instance,
 803                                       ibs_request->ring);
 804         }
 805
 806         *fence = cs.out.handle;
 807
 808         pthread_mutex_unlock(&context->sequence_mutex);
 809
 810         r = amdgpu_cs_free_bo_list(dev, bo_list_handle);
 811         if (r)
 812                 goto error_free;
 813
 814         free(chunk_array);
 815         return 0;
 816
 817 error_unlock:
 818         pthread_mutex_unlock(&context->sequence_mutex);
 819
 820 error_free:
 821         free(chunk_array);
 822         return r;
 823 }
 824
 825 int amdgpu_cs_submit(amdgpu_device_handle  dev,
 826                      amdgpu_context_handle context,
 827                      uint64_t flags,
 828                      struct amdgpu_cs_request *ibs_request,
 829                      uint32_t number_of_requests,
 830                      uint64_t *fences)
 831 {
 832         int r;
 833         uint32_t i;
 834
 835         if (NULL == dev)
 836                 return -EINVAL;
 837         if (NULL == context)
 838                 return -EINVAL;
 839         if (NULL == ibs_request)
 840                 return -EINVAL;
 841         if (NULL == fences)
 842                 return -EINVAL;
 843
 844         r = 0;
 845         for (i = 0; i < number_of_requests; i++) {
 846                 r = amdgpu_cs_submit_one(dev, context, ibs_request, fences);
 847                 if (r)
 848                         break;
 849                 fences++;
 850                 ibs_request++;
 851         }
 852
 853         return r;
 854 }
 855
 856 /**
 857  * Calculate absolute timeout.
 858  *
 859  * \param   timeout - \c [in] timeout in nanoseconds.
 860  *
 861  * \return  absolute timeout in nanoseconds
 862 */
 863 uint64_t amdgpu_cs_calculate_timeout(uint64_t timeout)
 864 {
 865         int r;
 866
 867         if (timeout != AMDGPU_TIMEOUT_INFINITE) {
 868                 struct timespec current;
 869                 r = clock_gettime(CLOCK_MONOTONIC, &current);
 870                 if (r)
 871                         return r;
 872
 873                 timeout += ((uint64_t)current.tv_sec) * 1000000000ull;
 874                 timeout += current.tv_nsec;
 875         }
 876         return timeout;
 877 }
 878
 879 static int amdgpu_ioctl_wait_cs(amdgpu_device_handle dev,
 880                                 unsigned ip,
 881                                 unsigned ip_instance,
 882                                 uint32_t ring,
 883                                 uint64_t handle,
 884                                 uint64_t timeout_ns,
 885                                 bool *busy)
 886 {
 887         union drm_amdgpu_wait_cs args;
 888         int r;
 889
 890         memset(&args, 0, sizeof(args));
 891         args.in.handle = handle;
 892         args.in.ip_type = ip;
 893         args.in.ip_instance = ip_instance;
 894         args.in.ring = ring;
 895         args.in.timeout = amdgpu_cs_calculate_timeout(timeout_ns);
 896
 897         /* Handle errors manually here because of timeout */
 898         r = ioctl(dev->fd, DRM_IOCTL_AMDGPU_WAIT_CS, &args);
 899         if (r == -1 && (errno == EINTR || errno == EAGAIN)) {
 900                 *busy = true;
 901                 return 0;
 902         } else if (r)
 903                 return -errno;
 904
 905         *busy = args.out.status;
 906         return 0;
 907 }
 908
 909 int amdgpu_cs_query_fence_status(amdgpu_device_handle dev,
 910                                  struct amdgpu_cs_query_fence *fence,
 911                                  uint32_t *expired)
 912 {
 913         amdgpu_context_handle context;
 914         uint64_t *signaled_fence;
 915         uint64_t *expired_fence;
 916         unsigned ip_type, ip_instance;
 917         uint32_t ring;
 918         bool busy = true;
 919         int r;
 920
 921         if (NULL == dev)
 922                 return -EINVAL;
 923         if (NULL == fence)
 924                 return -EINVAL;
 925         if (NULL == expired)
 926                 return -EINVAL;
 927         if (NULL == fence->context)
 928                 return -EINVAL;
 929         if (fence->ip_type >= AMDGPU_HW_IP_NUM)
 930                 return -EINVAL;
 931         if (fence->ring >= AMDGPU_CS_MAX_RINGS)
 932                 return -EINVAL;
 933
 934         context = fence->context;
 935         ip_type = fence->ip_type;
 936         ip_instance = fence->ip_instance;
 937         ring = fence->ring;
 938         signaled_fence = context->fence_ib->cpu;
 939         signaled_fence += amdgpu_cs_fence_index(ip_type, ring);
 940         expired_fence = &context->expired_fences[ip_type][ip_instance][ring];
 941         *expired = false;
 942
 943         pthread_mutex_lock(&context->sequence_mutex);
 944         if (fence->fence <= *expired_fence) {
 945                 /* This fence value is expired already. */
 946                 pthread_mutex_unlock(&context->sequence_mutex);
 947                 *expired = true;
 948                 return 0;
 949         }
 950
 951         if (fence->fence <= *signaled_fence) {
 952                 /* This fence value is signaled already. */
 953                 *expired_fence = *signaled_fence;
 954                 pthread_mutex_unlock(&context->sequence_mutex);
 955                 amdgpu_cs_pending_gc(context, ip_type, ip_instance, ring,
 956                                      fence->fence);
 957                 *expired = true;
 958                 return 0;
 959         }
 960
 961         pthread_mutex_unlock(&context->sequence_mutex);
 962
 963         r = amdgpu_ioctl_wait_cs(dev, ip_type, ip_instance, ring,
 964                                  fence->fence, fence->timeout_ns, &busy);
 965         if (!r && !busy) {
 966                 *expired = true;
 967                 pthread_mutex_lock(&context->sequence_mutex);
 968                 /* The thread doesn't hold sequence_mutex. Other thread could
 969                    update *expired_fence already. Check whether there is a
 970                    newerly expired fence. */
 971                 if (fence->fence > *expired_fence) {
 972                         *expired_fence = fence->fence;
 973                         pthread_mutex_unlock(&context->sequence_mutex);
 974                         amdgpu_cs_pending_gc(context, ip_type, ip_instance,
 975                                              ring, fence->fence);
 976                 } else {
 977                         pthread_mutex_unlock(&context->sequence_mutex);
 978                 }
 979         }
 980
 981         return r;
 982 }
 983