drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/dma-fence-array.h>
  29 #include <linux/interval_tree_generic.h>
  30 #include <drm/drmP.h>
  31 #include <drm/amdgpu_drm.h>
  32 #include "amdgpu.h"
  33 #include "amdgpu_trace.h"
  34
  35 /*
  36  * GPUVM
  37  * GPUVM is similar to the legacy gart on older asics, however
  38  * rather than there being a single global gart table
  39  * for the entire GPU, there are multiple VM page tables active
  40  * at any given time.  The VM page tables can contain a mix
  41  * vram pages and system memory pages and system memory pages
  42  * can be mapped as snooped (cached system pages) or unsnooped
  43  * (uncached system pages).
  44  * Each VM has an ID associated with it and there is a page table
  45  * associated with each VMID.  When execting a command buffer,
  46  * the kernel tells the the ring what VMID to use for that command
  47  * buffer.  VMIDs are allocated dynamically as commands are submitted.
  48  * The userspace drivers maintain their own address space and the kernel
  49  * sets up their pages tables accordingly when they submit their
  50  * command buffers and a VMID is assigned.
  51  * Cayman/Trinity support up to 8 active VMs at any given time;
  52  * SI supports 16.
  53  */
  54
  55 #define START(node) ((node)->start)
  56 #define LAST(node) ((node)->last)
  57
  58 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
  59                      START, LAST, static, amdgpu_vm_it)
  60
  61 #undef START
  62 #undef LAST
  63
  64 /* Local structure. Encapsulate some VM table update parameters to reduce
  65  * the number of function parameters
  66  */
  67 struct amdgpu_pte_update_params {
  68         /* amdgpu device we do this update for */
  69         struct amdgpu_device *adev;
  70         /* optional amdgpu_vm we do this update for */
  71         struct amdgpu_vm *vm;
  72         /* address where to copy page table entries from */
  73         uint64_t src;
  74         /* indirect buffer to fill with commands */
  75         struct amdgpu_ib *ib;
  76         /* Function which actually does the update */
  77         void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe,
  78                      uint64_t addr, unsigned count, uint32_t incr,
  79                      uint64_t flags);
  80         /* indicate update pt or its shadow */
  81         bool shadow;
  82 };
  83
  84 /* Helper to disable partial resident texture feature from a fence callback */
  85 struct amdgpu_prt_cb {
  86         struct amdgpu_device *adev;
  87         struct dma_fence_cb cb;
  88 };
  89
  90 /**
  91  * amdgpu_vm_num_entries - return the number of entries in a PD/PT
  92  *
  93  * @adev: amdgpu_device pointer
  94  *
  95  * Calculate the number of entries in a page directory or page table.
  96  */
  97 static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
  98                                       unsigned level)
  99 {
 100         if (level == 0)
 101                 /* For the root directory */
 102                 return adev->vm_manager.max_pfn >>
 103                         (adev->vm_manager.block_size *
 104                          adev->vm_manager.num_level);
 105         else if (level == adev->vm_manager.num_level)
 106                 /* For the page tables on the leaves */
 107                 return AMDGPU_VM_PTE_COUNT(adev);
 108         else
 109                 /* Everything in between */
 110                 return 1 << adev->vm_manager.block_size;
 111 }
 112
 113 /**
 114  * amdgpu_vm_bo_size - returns the size of the BOs in bytes
 115  *
 116  * @adev: amdgpu_device pointer
 117  *
 118  * Calculate the size of the BO for a page directory or page table in bytes.
 119  */
 120 static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level)
 121 {
 122         return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);
 123 }
 124
 125 /**
 126  * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
 127  *
 128  * @vm: vm providing the BOs
 129  * @validated: head of validation list
 130  * @entry: entry to add
 131  *
 132  * Add the page directory to the list of BOs to
 133  * validate for command submission.
 134  */
 135 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 136                          struct list_head *validated,
 137                          struct amdgpu_bo_list_entry *entry)
 138 {
 139         entry->robj = vm->root.bo;
 140         entry->priority = 0;
 141         entry->tv.bo = &entry->robj->tbo;
 142         entry->tv.shared = true;
 143         entry->user_pages = NULL;
 144         list_add(&entry->tv.head, validated);
 145 }
 146
 147 /**
 148  * amdgpu_vm_validate_layer - validate a single page table level
 149  *
 150  * @parent: parent page table level
 151  * @validate: callback to do the validation
 152  * @param: parameter for the validation callback
 153  *
 154  * Validate the page table BOs on command submission if neccessary.
 155  */
 156 static int amdgpu_vm_validate_level(struct amdgpu_vm_pt *parent,
 157                                     int (*validate)(void *, struct amdgpu_bo *),
 158                                     void *param)
 159 {
 160         unsigned i;
 161         int r;
 162
 163         if (!parent->entries)
 164                 return 0;
 165
 166         for (i = 0; i <= parent->last_entry_used; ++i) {
 167                 struct amdgpu_vm_pt *entry = &parent->entries[i];
 168
 169                 if (!entry->bo)
 170                         continue;
 171
 172                 r = validate(param, entry->bo);
 173                 if (r)
 174                         return r;
 175
 176                 /*
 177                  * Recurse into the sub directory. This is harmless because we
 178                  * have only a maximum of 5 layers.
 179                  */
 180                 r = amdgpu_vm_validate_level(entry, validate, param);
 181                 if (r)
 182                         return r;
 183         }
 184
 185         return r;
 186 }
 187
 188 /**
 189  * amdgpu_vm_validate_pt_bos - validate the page table BOs
 190  *
 191  * @adev: amdgpu device pointer
 192  * @vm: vm providing the BOs
 193  * @validate: callback to do the validation
 194  * @param: parameter for the validation callback
 195  *
 196  * Validate the page table BOs on command submission if neccessary.
 197  */
 198 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 199                               int (*validate)(void *p, struct amdgpu_bo *bo),
 200                               void *param)
 201 {
 202         uint64_t num_evictions;
 203
 204         /* We only need to validate the page tables
 205          * if they aren't already valid.
 206          */
 207         num_evictions = atomic64_read(&adev->num_evictions);
 208         if (num_evictions == vm->last_eviction_counter)
 209                 return 0;
 210
 211         return amdgpu_vm_validate_level(&vm->root, validate, param);
 212 }
 213
 214 /**
 215  * amdgpu_vm_move_level_in_lru - move one level of PT BOs to the LRU tail
 216  *
 217  * @adev: amdgpu device instance
 218  * @vm: vm providing the BOs
 219  *
 220  * Move the PT BOs to the tail of the LRU.
 221  */
 222 static void amdgpu_vm_move_level_in_lru(struct amdgpu_vm_pt *parent)
 223 {
 224         unsigned i;
 225
 226         if (!parent->entries)
 227                 return;
 228
 229         for (i = 0; i <= parent->last_entry_used; ++i) {
 230                 struct amdgpu_vm_pt *entry = &parent->entries[i];
 231
 232                 if (!entry->bo)
 233                         continue;
 234
 235                 ttm_bo_move_to_lru_tail(&entry->bo->tbo);
 236                 amdgpu_vm_move_level_in_lru(entry);
 237         }
 238 }
 239
 240 /**
 241  * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail
 242  *
 243  * @adev: amdgpu device instance
 244  * @vm: vm providing the BOs
 245  *
 246  * Move the PT BOs to the tail of the LRU.
 247  */
 248 void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
 249                                   struct amdgpu_vm *vm)
 250 {
 251         struct ttm_bo_global *glob = adev->mman.bdev.glob;
 252
 253         spin_lock(&glob->lru_lock);
 254         amdgpu_vm_move_level_in_lru(&vm->root);
 255         spin_unlock(&glob->lru_lock);
 256 }
 257
 258  /**
 259  * amdgpu_vm_alloc_levels - allocate the PD/PT levels
 260  *
 261  * @adev: amdgpu_device pointer
 262  * @vm: requested vm
 263  * @saddr: start of the address range
 264  * @eaddr: end of the address range
 265  *
 266  * Make sure the page directories and page tables are allocated
 267  */
 268 static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 269                                   struct amdgpu_vm *vm,
 270                                   struct amdgpu_vm_pt *parent,
 271                                   uint64_t saddr, uint64_t eaddr,
 272                                   unsigned level)
 273 {
 274         unsigned shift = (adev->vm_manager.num_level - level) *
 275                 adev->vm_manager.block_size;
 276         unsigned pt_idx, from, to;
 277         int r;
 278
 279         if (!parent->entries) {
 280                 unsigned num_entries = amdgpu_vm_num_entries(adev, level);
 281
 282                 parent->entries = drm_calloc_large(num_entries,
 283                                                    sizeof(struct amdgpu_vm_pt));
 284                 if (!parent->entries)
 285                         return -ENOMEM;
 286                 memset(parent->entries, 0 , sizeof(struct amdgpu_vm_pt));
 287         }
 288
 289         from = saddr >> shift;
 290         to = eaddr >> shift;
 291         if (from >= amdgpu_vm_num_entries(adev, level) ||
 292             to >= amdgpu_vm_num_entries(adev, level))
 293                 return -EINVAL;
 294
 295         if (to > parent->last_entry_used)
 296                 parent->last_entry_used = to;
 297
 298         ++level;
 299         saddr = saddr & ((1 << shift) - 1);
 300         eaddr = eaddr & ((1 << shift) - 1);
 301
 302         /* walk over the address space and allocate the page tables */
 303         for (pt_idx = from; pt_idx <= to; ++pt_idx) {
 304                 struct reservation_object *resv = vm->root.bo->tbo.resv;
 305                 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
 306                 struct amdgpu_bo *pt;
 307
 308                 if (!entry->bo) {
 309                         r = amdgpu_bo_create(adev,
 310                                              amdgpu_vm_bo_size(adev, level),
 311                                              AMDGPU_GPU_PAGE_SIZE, true,
 312                                              AMDGPU_GEM_DOMAIN_VRAM,
 313                                              AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
 314                                              AMDGPU_GEM_CREATE_SHADOW |
 315                                              AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
 316                                              AMDGPU_GEM_CREATE_VRAM_CLEARED,
 317                                              NULL, resv, &pt);
 318                         if (r)
 319                                 return r;
 320
 321                         /* Keep a reference to the root directory to avoid
 322                         * freeing them up in the wrong order.
 323                         */
 324                         pt->parent = amdgpu_bo_ref(vm->root.bo);
 325
 326                         entry->bo = pt;
 327                         entry->addr = 0;
 328                 }
 329
 330                 if (level < adev->vm_manager.num_level) {
 331                         uint64_t sub_saddr = (pt_idx == from) ? saddr : 0;
 332                         uint64_t sub_eaddr = (pt_idx == to) ? eaddr :
 333                                 ((1 << shift) - 1);
 334                         r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr,
 335                                                    sub_eaddr, level);
 336                         if (r)
 337                                 return r;
 338                 }
 339         }
 340
 341         return 0;
 342 }
 343
 344 /**
 345  * amdgpu_vm_alloc_pts - Allocate page tables.
 346  *
 347  * @adev: amdgpu_device pointer
 348  * @vm: VM to allocate page tables for
 349  * @saddr: Start address which needs to be allocated
 350  * @size: Size from start address we need.
 351  *
 352  * Make sure the page tables are allocated.
 353  */
 354 int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
 355                         struct amdgpu_vm *vm,
 356                         uint64_t saddr, uint64_t size)
 357 {
 358         uint64_t last_pfn;
 359         uint64_t eaddr;
 360
 361         /* validate the parameters */
 362         if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK)
 363                 return -EINVAL;
 364
 365         eaddr = saddr + size - 1;
 366         last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
 367         if (last_pfn >= adev->vm_manager.max_pfn) {
 368                 dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n",
 369                         last_pfn, adev->vm_manager.max_pfn);
 370                 return -EINVAL;
 371         }
 372
 373         saddr /= AMDGPU_GPU_PAGE_SIZE;
 374         eaddr /= AMDGPU_GPU_PAGE_SIZE;
 375
 376         return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr, 0);
 377 }
 378
 379 /**
 380  * amdgpu_vm_had_gpu_reset - check if reset occured since last use
 381  *
 382  * @adev: amdgpu_device pointer
 383  * @id: VMID structure
 384  *
 385  * Check if GPU reset occured since last use of the VMID.
 386  */
 387 static bool amdgpu_vm_had_gpu_reset(struct amdgpu_device *adev,
 388                                     struct amdgpu_vm_id *id)
 389 {
 390         return id->current_gpu_reset_count !=
 391                 atomic_read(&adev->gpu_reset_counter);
 392 }
 393
 394 static bool amdgpu_vm_reserved_vmid_ready(struct amdgpu_vm *vm, unsigned vmhub)
 395 {
 396         return !!vm->reserved_vmid[vmhub];
 397 }
 398
 399 /* idr_mgr->lock must be held */
 400 static int amdgpu_vm_grab_reserved_vmid_locked(struct amdgpu_vm *vm,
 401                                                struct amdgpu_ring *ring,
 402                                                struct amdgpu_sync *sync,
 403                                                struct dma_fence *fence,
 404                                                struct amdgpu_job *job)
 405 {
 406         struct amdgpu_device *adev = ring->adev;
 407         unsigned vmhub = ring->funcs->vmhub;
 408         uint64_t fence_context = adev->fence_context + ring->idx;
 409         struct amdgpu_vm_id *id = vm->reserved_vmid[vmhub];
 410         struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 411         struct dma_fence *updates = sync->last_vm_update;
 412         int r = 0;
 413         struct dma_fence *flushed, *tmp;
 414         bool needs_flush = false;
 415
 416         flushed  = id->flushed_updates;
 417         if ((amdgpu_vm_had_gpu_reset(adev, id)) ||
 418             (atomic64_read(&id->owner) != vm->client_id) ||
 419             (job->vm_pd_addr != id->pd_gpu_addr) ||
 420             (updates && (!flushed || updates->context != flushed->context ||
 421                         dma_fence_is_later(updates, flushed))) ||
 422             (!id->last_flush || (id->last_flush->context != fence_context &&
 423                                  !dma_fence_is_signaled(id->last_flush)))) {
 424                 needs_flush = true;
 425                 /* to prevent one context starved by another context */
 426                 id->pd_gpu_addr = 0;
 427                 tmp = amdgpu_sync_peek_fence(&id->active, ring);
 428                 if (tmp) {
 429                         r = amdgpu_sync_fence(adev, sync, tmp);
 430                         return r;
 431                 }
 432         }
 433
 434         /* Good we can use this VMID. Remember this submission as
 435         * user of the VMID.
 436         */
 437         r = amdgpu_sync_fence(ring->adev, &id->active, fence);
 438         if (r)
 439                 goto out;
 440
 441         if (updates && (!flushed || updates->context != flushed->context ||
 442                         dma_fence_is_later(updates, flushed))) {
 443                 dma_fence_put(id->flushed_updates);
 444                 id->flushed_updates = dma_fence_get(updates);
 445         }
 446         id->pd_gpu_addr = job->vm_pd_addr;
 447         atomic64_set(&id->owner, vm->client_id);
 448         job->vm_needs_flush = needs_flush;
 449         if (needs_flush) {
 450                 dma_fence_put(id->last_flush);
 451                 id->last_flush = NULL;
 452         }
 453         job->vm_id = id - id_mgr->ids;
 454         trace_amdgpu_vm_grab_id(vm, ring, job);
 455 out:
 456         return r;
 457 }
 458
 459 /**
 460  * amdgpu_vm_grab_id - allocate the next free VMID
 461  *
 462  * @vm: vm to allocate id for
 463  * @ring: ring we want to submit job to
 464  * @sync: sync object where we add dependencies
 465  * @fence: fence protecting ID from reuse
 466  *
 467  * Allocate an id for the vm, adding fences to the sync obj as necessary.
 468  */
 469 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 470                       struct amdgpu_sync *sync, struct dma_fence *fence,
 471                       struct amdgpu_job *job)
 472 {
 473         struct amdgpu_device *adev = ring->adev;
 474         unsigned vmhub = ring->funcs->vmhub;
 475         struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 476         uint64_t fence_context = adev->fence_context + ring->idx;
 477         struct dma_fence *updates = sync->last_vm_update;
 478         struct amdgpu_vm_id *id, *idle;
 479         struct dma_fence **fences;
 480         unsigned i;
 481         int r = 0;
 482
 483         mutex_lock(&id_mgr->lock);
 484         if (amdgpu_vm_reserved_vmid_ready(vm, vmhub)) {
 485                 r = amdgpu_vm_grab_reserved_vmid_locked(vm, ring, sync, fence, job);
 486                 mutex_unlock(&id_mgr->lock);
 487                 return r;
 488         }
 489         fences = kmalloc_array(sizeof(void *), id_mgr->num_ids, GFP_KERNEL);
 490         if (!fences) {
 491                 mutex_unlock(&id_mgr->lock);
 492                 return -ENOMEM;
 493         }
 494         /* Check if we have an idle VMID */
 495         i = 0;
 496         list_for_each_entry(idle, &id_mgr->ids_lru, list) {
 497                 fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
 498                 if (!fences[i])
 499                         break;
 500                 ++i;
 501         }
 502
 503         /* If we can't find a idle VMID to use, wait till one becomes available */
 504         if (&idle->list == &id_mgr->ids_lru) {
 505                 u64 fence_context = adev->vm_manager.fence_context + ring->idx;
 506                 unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
 507                 struct dma_fence_array *array;
 508                 unsigned j;
 509
 510                 for (j = 0; j < i; ++j)
 511                         dma_fence_get(fences[j]);
 512
 513                 array = dma_fence_array_create(i, fences, fence_context,
 514                                            seqno, true);
 515                 if (!array) {
 516                         for (j = 0; j < i; ++j)
 517                                 dma_fence_put(fences[j]);
 518                         kfree(fences);
 519                         r = -ENOMEM;
 520                         goto error;
 521                 }
 522
 523
 524                 r = amdgpu_sync_fence(ring->adev, sync, &array->base);
 525                 dma_fence_put(&array->base);
 526                 if (r)
 527                         goto error;
 528
 529                 mutex_unlock(&id_mgr->lock);
 530                 return 0;
 531
 532         }
 533         kfree(fences);
 534
 535         job->vm_needs_flush = false;
 536         /* Check if we can use a VMID already assigned to this VM */
 537         list_for_each_entry_reverse(id, &id_mgr->ids_lru, list) {
 538                 struct dma_fence *flushed;
 539                 bool needs_flush = false;
 540
 541                 /* Check all the prerequisites to using this VMID */
 542                 if (amdgpu_vm_had_gpu_reset(adev, id))
 543                         continue;
 544
 545                 if (atomic64_read(&id->owner) != vm->client_id)
 546                         continue;
 547
 548                 if (job->vm_pd_addr != id->pd_gpu_addr)
 549                         continue;
 550
 551                 if (!id->last_flush ||
 552                     (id->last_flush->context != fence_context &&
 553                      !dma_fence_is_signaled(id->last_flush)))
 554                         needs_flush = true;
 555
 556                 flushed  = id->flushed_updates;
 557                 if (updates && (!flushed || dma_fence_is_later(updates, flushed)))
 558                         needs_flush = true;
 559
 560                 /* Concurrent flushes are only possible starting with Vega10 */
 561                 if (adev->asic_type < CHIP_VEGA10 && needs_flush)
 562                         continue;
 563
 564                 /* Good we can use this VMID. Remember this submission as
 565                  * user of the VMID.
 566                  */
 567                 r = amdgpu_sync_fence(ring->adev, &id->active, fence);
 568                 if (r)
 569                         goto error;
 570
 571                 if (updates && (!flushed || dma_fence_is_later(updates, flushed))) {
 572                         dma_fence_put(id->flushed_updates);
 573                         id->flushed_updates = dma_fence_get(updates);
 574                 }
 575
 576                 if (needs_flush)
 577                         goto needs_flush;
 578                 else
 579                         goto no_flush_needed;
 580
 581         };
 582
 583         /* Still no ID to use? Then use the idle one found earlier */
 584         id = idle;
 585
 586         /* Remember this submission as user of the VMID */
 587         r = amdgpu_sync_fence(ring->adev, &id->active, fence);
 588         if (r)
 589                 goto error;
 590
 591         id->pd_gpu_addr = job->vm_pd_addr;
 592         dma_fence_put(id->flushed_updates);
 593         id->flushed_updates = dma_fence_get(updates);
 594         atomic64_set(&id->owner, vm->client_id);
 595
 596 needs_flush:
 597         job->vm_needs_flush = true;
 598         dma_fence_put(id->last_flush);
 599         id->last_flush = NULL;
 600
 601 no_flush_needed:
 602         list_move_tail(&id->list, &id_mgr->ids_lru);
 603
 604         job->vm_id = id - id_mgr->ids;
 605         trace_amdgpu_vm_grab_id(vm, ring, job);
 606
 607 error:
 608         mutex_unlock(&id_mgr->lock);
 609         return r;
 610 }
 611
 612 static void amdgpu_vm_free_reserved_vmid(struct amdgpu_device *adev,
 613                                           struct amdgpu_vm *vm,
 614                                           unsigned vmhub)
 615 {
 616         struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 617
 618         mutex_lock(&id_mgr->lock);
 619         if (vm->reserved_vmid[vmhub]) {
 620                 list_add(&vm->reserved_vmid[vmhub]->list,
 621                         &id_mgr->ids_lru);
 622                 vm->reserved_vmid[vmhub] = NULL;
 623                 atomic_dec(&id_mgr->reserved_vmid_num);
 624         }
 625         mutex_unlock(&id_mgr->lock);
 626 }
 627
 628 static int amdgpu_vm_alloc_reserved_vmid(struct amdgpu_device *adev,
 629                                          struct amdgpu_vm *vm,
 630                                          unsigned vmhub)
 631 {
 632         struct amdgpu_vm_id_manager *id_mgr;
 633         struct amdgpu_vm_id *idle;
 634         int r = 0;
 635
 636         id_mgr = &adev->vm_manager.id_mgr[vmhub];
 637         mutex_lock(&id_mgr->lock);
 638         if (vm->reserved_vmid[vmhub])
 639                 goto unlock;
 640         if (atomic_inc_return(&id_mgr->reserved_vmid_num) >
 641             AMDGPU_VM_MAX_RESERVED_VMID) {
 642                 DRM_ERROR("Over limitation of reserved vmid\n");
 643                 atomic_dec(&id_mgr->reserved_vmid_num);
 644                 r = -EINVAL;
 645                 goto unlock;
 646         }
 647         /* Select the first entry VMID */
 648         idle = list_first_entry(&id_mgr->ids_lru, struct amdgpu_vm_id, list);
 649         list_del_init(&idle->list);
 650         vm->reserved_vmid[vmhub] = idle;
 651         mutex_unlock(&id_mgr->lock);
 652
 653         return 0;
 654 unlock:
 655         mutex_unlock(&id_mgr->lock);
 656         return r;
 657 }
 658
 659 static bool amdgpu_vm_ring_has_compute_vm_bug(struct amdgpu_ring *ring)
 660 {
 661         struct amdgpu_device *adev = ring->adev;
 662         const struct amdgpu_ip_block *ip_block;
 663
 664         if (ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)
 665                 /* only compute rings */
 666                 return false;
 667
 668         ip_block = amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
 669         if (!ip_block)
 670                 return false;
 671
 672         if (ip_block->version->major <= 7) {
 673                 /* gfx7 has no workaround */
 674                 return true;
 675         } else if (ip_block->version->major == 8) {
 676                 if (adev->gfx.mec_fw_version >= 673)
 677                         /* gfx8 is fixed in MEC firmware 673 */
 678                         return false;
 679                 else
 680                         return true;
 681         }
 682         return false;
 683 }
 684
 685 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
 686                                   struct amdgpu_job *job)
 687 {
 688         struct amdgpu_device *adev = ring->adev;
 689         unsigned vmhub = ring->funcs->vmhub;
 690         struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 691         struct amdgpu_vm_id *id;
 692         bool gds_switch_needed;
 693         bool vm_flush_needed = job->vm_needs_flush ||
 694                 amdgpu_vm_ring_has_compute_vm_bug(ring);
 695
 696         if (job->vm_id == 0)
 697                 return false;
 698         id = &id_mgr->ids[job->vm_id];
 699         gds_switch_needed = ring->funcs->emit_gds_switch && (
 700                 id->gds_base != job->gds_base ||
 701                 id->gds_size != job->gds_size ||
 702                 id->gws_base != job->gws_base ||
 703                 id->gws_size != job->gws_size ||
 704                 id->oa_base != job->oa_base ||
 705                 id->oa_size != job->oa_size);
 706
 707         if (amdgpu_vm_had_gpu_reset(adev, id))
 708                 return true;
 709         if (!vm_flush_needed && !gds_switch_needed)
 710                 return false;
 711         return true;
 712 }
 713
 714 /**
 715  * amdgpu_vm_flush - hardware flush the vm
 716  *
 717  * @ring: ring to use for flush
 718  * @vm_id: vmid number to use
 719  * @pd_addr: address of the page directory
 720  *
 721  * Emit a VM flush when it is necessary.
 722  */
 723 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
 724 {
 725         struct amdgpu_device *adev = ring->adev;
 726         unsigned vmhub = ring->funcs->vmhub;
 727         struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 728         struct amdgpu_vm_id *id = &id_mgr->ids[job->vm_id];
 729         bool gds_switch_needed = ring->funcs->emit_gds_switch && (
 730                 id->gds_base != job->gds_base ||
 731                 id->gds_size != job->gds_size ||
 732                 id->gws_base != job->gws_base ||
 733                 id->gws_size != job->gws_size ||
 734                 id->oa_base != job->oa_base ||
 735                 id->oa_size != job->oa_size);
 736         bool vm_flush_needed = job->vm_needs_flush;
 737         unsigned patch_offset = 0;
 738         int r;
 739
 740         if (amdgpu_vm_had_gpu_reset(adev, id)) {
 741                 gds_switch_needed = true;
 742                 vm_flush_needed = true;
 743         }
 744
 745         if (!vm_flush_needed && !gds_switch_needed)
 746                 return 0;
 747
 748         if (ring->funcs->init_cond_exec)
 749                 patch_offset = amdgpu_ring_init_cond_exec(ring);
 750
 751         if (ring->funcs->emit_vm_flush && vm_flush_needed) {
 752                 struct dma_fence *fence;
 753
 754                 trace_amdgpu_vm_flush(ring, job->vm_id, job->vm_pd_addr);
 755                 amdgpu_ring_emit_vm_flush(ring, job->vm_id, job->vm_pd_addr);
 756
 757                 r = amdgpu_fence_emit(ring, &fence);
 758                 if (r)
 759                         return r;
 760
 761                 mutex_lock(&id_mgr->lock);
 762                 dma_fence_put(id->last_flush);
 763                 id->last_flush = fence;
 764                 id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
 765                 mutex_unlock(&id_mgr->lock);
 766         }
 767
 768         if (ring->funcs->emit_gds_switch && gds_switch_needed) {
 769                 id->gds_base = job->gds_base;
 770                 id->gds_size = job->gds_size;
 771                 id->gws_base = job->gws_base;
 772                 id->gws_size = job->gws_size;
 773                 id->oa_base = job->oa_base;
 774                 id->oa_size = job->oa_size;
 775                 amdgpu_ring_emit_gds_switch(ring, job->vm_id, job->gds_base,
 776                                             job->gds_size, job->gws_base,
 777                                             job->gws_size, job->oa_base,
 778                                             job->oa_size);
 779         }
 780
 781         if (ring->funcs->patch_cond_exec)
 782                 amdgpu_ring_patch_cond_exec(ring, patch_offset);
 783
 784         /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
 785         if (ring->funcs->emit_switch_buffer) {
 786                 amdgpu_ring_emit_switch_buffer(ring);
 787                 amdgpu_ring_emit_switch_buffer(ring);
 788         }
 789         return 0;
 790 }
 791
 792 /**
 793  * amdgpu_vm_reset_id - reset VMID to zero
 794  *
 795  * @adev: amdgpu device structure
 796  * @vm_id: vmid number to use
 797  *
 798  * Reset saved GDW, GWS and OA to force switch on next flush.
 799  */
 800 void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vmhub,
 801                         unsigned vmid)
 802 {
 803         struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 804         struct amdgpu_vm_id *id = &id_mgr->ids[vmid];
 805
 806         atomic64_set(&id->owner, 0);
 807         id->gds_base = 0;
 808         id->gds_size = 0;
 809         id->gws_base = 0;
 810         id->gws_size = 0;
 811         id->oa_base = 0;
 812         id->oa_size = 0;
 813 }
 814
 815 /**
 816  * amdgpu_vm_reset_all_id - reset VMID to zero
 817  *
 818  * @adev: amdgpu device structure
 819  *
 820  * Reset VMID to force flush on next use
 821  */
 822 void amdgpu_vm_reset_all_ids(struct amdgpu_device *adev)
 823 {
 824         unsigned i, j;
 825
 826         for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
 827                 struct amdgpu_vm_id_manager *id_mgr =
 828                         &adev->vm_manager.id_mgr[i];
 829
 830                 for (j = 1; j < id_mgr->num_ids; ++j)
 831                         amdgpu_vm_reset_id(adev, i, j);
 832         }
 833 }
 834
 835 /**
 836  * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
 837  *
 838  * @vm: requested vm
 839  * @bo: requested buffer object
 840  *
 841  * Find @bo inside the requested vm.
 842  * Search inside the @bos vm list for the requested vm
 843  * Returns the found bo_va or NULL if none is found
 844  *
 845  * Object has to be reserved!
 846  */
 847 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
 848                                        struct amdgpu_bo *bo)
 849 {
 850         struct amdgpu_bo_va *bo_va;
 851
 852         list_for_each_entry(bo_va, &bo->va, bo_list) {
 853                 if (bo_va->vm == vm) {
 854                         return bo_va;
 855                 }
 856         }
 857         return NULL;
 858 }
 859
 860 /**
 861  * amdgpu_vm_do_set_ptes - helper to call the right asic function
 862  *
 863  * @params: see amdgpu_pte_update_params definition
 864  * @pe: addr of the page entry
 865  * @addr: dst addr to write into pe
 866  * @count: number of page entries to update
 867  * @incr: increase next addr by incr bytes
 868  * @flags: hw access flags
 869  *
 870  * Traces the parameters and calls the right asic functions
 871  * to setup the page table using the DMA.
 872  */
 873 static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
 874                                   uint64_t pe, uint64_t addr,
 875                                   unsigned count, uint32_t incr,
 876                                   uint64_t flags)
 877 {
 878         trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
 879
 880         if (count < 3) {
 881                 amdgpu_vm_write_pte(params->adev, params->ib, pe,
 882                                     addr | flags, count, incr);
 883
 884         } else {
 885                 amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
 886                                       count, incr, flags);
 887         }
 888 }
 889
 890 /**
 891  * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
 892  *
 893  * @params: see amdgpu_pte_update_params definition
 894  * @pe: addr of the page entry
 895  * @addr: dst addr to write into pe
 896  * @count: number of page entries to update
 897  * @incr: increase next addr by incr bytes
 898  * @flags: hw access flags
 899  *
 900  * Traces the parameters and calls the DMA function to copy the PTEs.
 901  */
 902 static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
 903                                    uint64_t pe, uint64_t addr,
 904                                    unsigned count, uint32_t incr,
 905                                    uint64_t flags)
 906 {
 907         uint64_t src = (params->src + (addr >> 12) * 8);
 908
 909
 910         trace_amdgpu_vm_copy_ptes(pe, src, count);
 911
 912         amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
 913 }
 914
 915 /**
 916  * amdgpu_vm_map_gart - Resolve gart mapping of addr
 917  *
 918  * @pages_addr: optional DMA address to use for lookup
 919  * @addr: the unmapped addr
 920  *
 921  * Look up the physical address of the page that the pte resolves
 922  * to and return the pointer for the page table entry.
 923  */
 924 static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
 925 {
 926         uint64_t result;
 927
 928         /* page table offset */
 929         result = pages_addr[addr >> PAGE_SHIFT];
 930
 931         /* in case cpu page size != gpu page size*/
 932         result |= addr & (~PAGE_MASK);
 933
 934         result &= 0xFFFFFFFFFFFFF000ULL;
 935
 936         return result;
 937 }
 938
 939 /*
 940  * amdgpu_vm_update_level - update a single level in the hierarchy
 941  *
 942  * @adev: amdgpu_device pointer
 943  * @vm: requested vm
 944  * @parent: parent directory
 945  *
 946  * Makes sure all entries in @parent are up to date.
 947  * Returns 0 for success, error for failure.
 948  */
 949 static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 950                                   struct amdgpu_vm *vm,
 951                                   struct amdgpu_vm_pt *parent,
 952                                   unsigned level)
 953 {
 954         struct amdgpu_bo *shadow;
 955         struct amdgpu_ring *ring;
 956         uint64_t pd_addr, shadow_addr;
 957         uint32_t incr = amdgpu_vm_bo_size(adev, level + 1);
 958         uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
 959         unsigned count = 0, pt_idx, ndw;
 960         struct amdgpu_job *job;
 961         struct amdgpu_pte_update_params params;
 962         struct dma_fence *fence = NULL;
 963
 964         int r;
 965
 966         if (!parent->entries)
 967                 return 0;
 968         ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
 969
 970         /* padding, etc. */
 971         ndw = 64;
 972
 973         /* assume the worst case */
 974         ndw += parent->last_entry_used * 6;
 975
 976         pd_addr = amdgpu_bo_gpu_offset(parent->bo);
 977
 978         shadow = parent->bo->shadow;
 979         if (shadow) {
 980                 r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
 981                 if (r)
 982                         return r;
 983                 shadow_addr = amdgpu_bo_gpu_offset(shadow);
 984                 ndw *= 2;
 985         } else {
 986                 shadow_addr = 0;
 987         }
 988
 989         r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
 990         if (r)
 991                 return r;
 992
 993         memset(&params, 0, sizeof(params));
 994         params.adev = adev;
 995         params.ib = &job->ibs[0];
 996
 997         /* walk over the address space and update the directory */
 998         for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
 999                 struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
1000                 uint64_t pde, pt;
1001
1002                 if (bo == NULL)
1003                         continue;
1004
1005                 if (bo->shadow) {
1006                         struct amdgpu_bo *pt_shadow = bo->shadow;
1007
1008                         r = amdgpu_ttm_bind(&pt_shadow->tbo,
1009                                             &pt_shadow->tbo.mem);
1010                         if (r)
1011                                 return r;
1012                 }
1013
1014                 pt = amdgpu_bo_gpu_offset(bo);
1015                 if (parent->entries[pt_idx].addr == pt)
1016                         continue;
1017
1018                 parent->entries[pt_idx].addr = pt;
1019
1020                 pde = pd_addr + pt_idx * 8;
1021                 if (((last_pde + 8 * count) != pde) ||
1022                     ((last_pt + incr * count) != pt) ||
1023                     (count == AMDGPU_VM_MAX_UPDATE_SIZE)) {
1024
1025                         if (count) {
1026                                 uint64_t entry;
1027
1028                                 entry = amdgpu_gart_get_vm_pde(adev, last_pt);
1029                                 if (shadow)
1030                                         amdgpu_vm_do_set_ptes(&params,
1031                                                               last_shadow,
1032                                                               entry, count,
1033                                                               incr,
1034                                                               AMDGPU_PTE_VALID);
1035
1036                                 amdgpu_vm_do_set_ptes(&params, last_pde,
1037                                                       entry, count, incr,
1038                                                       AMDGPU_PTE_VALID);
1039                         }
1040
1041                         count = 1;
1042                         last_pde = pde;
1043                         last_shadow = shadow_addr + pt_idx * 8;
1044                         last_pt = pt;
1045                 } else {
1046                         ++count;
1047                 }
1048         }
1049
1050         if (count) {
1051                 uint64_t entry;
1052
1053                 entry = amdgpu_gart_get_vm_pde(adev, last_pt);
1054
1055                 if (vm->root.bo->shadow)
1056                         amdgpu_vm_do_set_ptes(&params, last_shadow, entry,
1057                                               count, incr, AMDGPU_PTE_VALID);
1058
1059                 amdgpu_vm_do_set_ptes(&params, last_pde, entry,
1060                                       count, incr, AMDGPU_PTE_VALID);
1061         }
1062
1063         if (params.ib->length_dw == 0) {
1064                 amdgpu_job_free(job);
1065         } else {
1066                 amdgpu_ring_pad_ib(ring, params.ib);
1067                 amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
1068                                  AMDGPU_FENCE_OWNER_VM);
1069                 if (shadow)
1070                         amdgpu_sync_resv(adev, &job->sync, shadow->tbo.resv,
1071                                          AMDGPU_FENCE_OWNER_VM);
1072
1073                 WARN_ON(params.ib->length_dw > ndw);
1074                 r = amdgpu_job_submit(job, ring, &vm->entity,
1075                                 AMDGPU_FENCE_OWNER_VM, &fence);
1076                 if (r)
1077                         goto error_free;
1078
1079                 amdgpu_bo_fence(parent->bo, fence, true);
1080                 dma_fence_put(vm->last_dir_update);
1081                 vm->last_dir_update = dma_fence_get(fence);
1082                 dma_fence_put(fence);
1083         }
1084         /*
1085          * Recurse into the subdirectories. This recursion is harmless because
1086          * we only have a maximum of 5 layers.
1087          */
1088         for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
1089                 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
1090
1091                 if (!entry->bo)
1092                         continue;
1093
1094                 r = amdgpu_vm_update_level(adev, vm, entry, level + 1);
1095                 if (r)
1096                         return r;
1097         }
1098
1099         return 0;
1100
1101 error_free:
1102         amdgpu_job_free(job);
1103         return r;
1104 }
1105
1106 /*
1107  * amdgpu_vm_invalidate_level - mark all PD levels as invalid
1108  *
1109  * @parent: parent PD
1110  *
1111  * Mark all PD level as invalid after an error.
1112  */
1113 static void amdgpu_vm_invalidate_level(struct amdgpu_vm_pt *parent)
1114 {
1115         unsigned pt_idx;
1116
1117         /*
1118          * Recurse into the subdirectories. This recursion is harmless because
1119          * we only have a maximum of 5 layers.
1120          */
1121         for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
1122                 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
1123
1124                 if (!entry->bo)
1125                         continue;
1126
1127                 entry->addr = ~0ULL;
1128                 amdgpu_vm_invalidate_level(entry);
1129         }
1130 }
1131
1132 /*
1133  * amdgpu_vm_update_directories - make sure that all directories are valid
1134  *
1135  * @adev: amdgpu_device pointer
1136  * @vm: requested vm
1137  *
1138  * Makes sure all directories are up to date.
1139  * Returns 0 for success, error for failure.
1140  */
1141 int amdgpu_vm_update_directories(struct amdgpu_device *adev,
1142                                  struct amdgpu_vm *vm)
1143 {
1144         int r;
1145
1146         r = amdgpu_vm_update_level(adev, vm, &vm->root, 0);
1147         if (r)
1148                 amdgpu_vm_invalidate_level(&vm->root);
1149
1150         return r;
1151 }
1152
1153 /**
1154  * amdgpu_vm_find_pt - find the page table for an address
1155  *
1156  * @p: see amdgpu_pte_update_params definition
1157  * @addr: virtual address in question
1158  *
1159  * Find the page table BO for a virtual address, return NULL when none found.
1160  */
1161 static struct amdgpu_bo *amdgpu_vm_get_pt(struct amdgpu_pte_update_params *p,
1162                                           uint64_t addr)
1163 {
1164         struct amdgpu_vm_pt *entry = &p->vm->root;
1165         unsigned idx, level = p->adev->vm_manager.num_level;
1166
1167         while (entry->entries) {
1168                 idx = addr >> (p->adev->vm_manager.block_size * level--);
1169                 idx %= amdgpu_bo_size(entry->bo) / 8;
1170                 entry = &entry->entries[idx];
1171         }
1172
1173         if (level)
1174                 return NULL;
1175
1176         return entry->bo;
1177 }
1178
1179 /**
1180  * amdgpu_vm_update_ptes - make sure that page tables are valid
1181  *
1182  * @params: see amdgpu_pte_update_params definition
1183  * @vm: requested vm
1184  * @start: start of GPU address range
1185  * @end: end of GPU address range
1186  * @dst: destination address to map to, the next dst inside the function
1187  * @flags: mapping flags
1188  *
1189  * Update the page tables in the range @start - @end.
1190  * Returns 0 for success, -EINVAL for failure.
1191  */
1192 static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
1193                                   uint64_t start, uint64_t end,
1194                                   uint64_t dst, uint64_t flags)
1195 {
1196         struct amdgpu_device *adev = params->adev;
1197         const uint64_t mask = AMDGPU_VM_PTE_COUNT(adev) - 1;
1198
1199         uint64_t cur_pe_start, cur_nptes, cur_dst;
1200         uint64_t addr; /* next GPU address to be updated */
1201         struct amdgpu_bo *pt;
1202         unsigned nptes; /* next number of ptes to be updated */
1203         uint64_t next_pe_start;
1204
1205         /* initialize the variables */
1206         addr = start;
1207         pt = amdgpu_vm_get_pt(params, addr);
1208         if (!pt) {
1209                 pr_err("PT not found, aborting update_ptes\n");
1210                 return -EINVAL;
1211         }
1212
1213         if (params->shadow) {
1214                 if (!pt->shadow)
1215                         return 0;
1216                 pt = pt->shadow;
1217         }
1218         if ((addr & ~mask) == (end & ~mask))
1219                 nptes = end - addr;
1220         else
1221                 nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask);
1222
1223         cur_pe_start = amdgpu_bo_gpu_offset(pt);
1224         cur_pe_start += (addr & mask) * 8;
1225         cur_nptes = nptes;
1226         cur_dst = dst;
1227
1228         /* for next ptb*/
1229         addr += nptes;
1230         dst += nptes * AMDGPU_GPU_PAGE_SIZE;
1231
1232         /* walk over the address space and update the page tables */
1233         while (addr < end) {
1234                 pt = amdgpu_vm_get_pt(params, addr);
1235                 if (!pt) {
1236                         pr_err("PT not found, aborting update_ptes\n");
1237                         return -EINVAL;
1238                 }
1239
1240                 if (params->shadow) {
1241                         if (!pt->shadow)
1242                                 return 0;
1243                         pt = pt->shadow;
1244                 }
1245
1246                 if ((addr & ~mask) == (end & ~mask))
1247                         nptes = end - addr;
1248                 else
1249                         nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask);
1250
1251                 next_pe_start = amdgpu_bo_gpu_offset(pt);
1252                 next_pe_start += (addr & mask) * 8;
1253
1254                 if ((cur_pe_start + 8 * cur_nptes) == next_pe_start &&
1255                     ((cur_nptes + nptes) <= AMDGPU_VM_MAX_UPDATE_SIZE)) {
1256                         /* The next ptb is consecutive to current ptb.
1257                          * Don't call the update function now.
1258                          * Will update two ptbs together in future.
1259                         */
1260                         cur_nptes += nptes;
1261                 } else {
1262                         params->func(params, cur_pe_start, cur_dst, cur_nptes,
1263                                      AMDGPU_GPU_PAGE_SIZE, flags);
1264
1265                         cur_pe_start = next_pe_start;
1266                         cur_nptes = nptes;
1267                         cur_dst = dst;
1268                 }
1269
1270                 /* for next ptb*/
1271                 addr += nptes;
1272                 dst += nptes * AMDGPU_GPU_PAGE_SIZE;
1273         }
1274
1275         params->func(params, cur_pe_start, cur_dst, cur_nptes,
1276                      AMDGPU_GPU_PAGE_SIZE, flags);
1277
1278         return 0;
1279 }
1280
1281 /*
1282  * amdgpu_vm_frag_ptes - add fragment information to PTEs
1283  *
1284  * @params: see amdgpu_pte_update_params definition
1285  * @vm: requested vm
1286  * @start: first PTE to handle
1287  * @end: last PTE to handle
1288  * @dst: addr those PTEs should point to
1289  * @flags: hw mapping flags
1290  * Returns 0 for success, -EINVAL for failure.
1291  */
1292 static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params  *params,
1293                                 uint64_t start, uint64_t end,
1294                                 uint64_t dst, uint64_t flags)
1295 {
1296         int r;
1297
1298         /**
1299          * The MC L1 TLB supports variable sized pages, based on a fragment
1300          * field in the PTE. When this field is set to a non-zero value, page
1301          * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
1302          * flags are considered valid for all PTEs within the fragment range
1303          * and corresponding mappings are assumed to be physically contiguous.
1304          *
1305          * The L1 TLB can store a single PTE for the whole fragment,
1306          * significantly increasing the space available for translation
1307          * caching. This leads to large improvements in throughput when the
1308          * TLB is under pressure.
1309          *
1310          * The L2 TLB distributes small and large fragments into two
1311          * asymmetric partitions. The large fragment cache is significantly
1312          * larger. Thus, we try to use large fragments wherever possible.
1313          * Userspace can support this by aligning virtual base address and
1314          * allocation size to the fragment size.
1315          */
1316
1317         /* SI and newer are optimized for 64KB */
1318         uint64_t frag_flags = AMDGPU_PTE_FRAG(AMDGPU_LOG2_PAGES_PER_FRAG);
1319         uint64_t frag_align = 1 << AMDGPU_LOG2_PAGES_PER_FRAG;
1320
1321         uint64_t frag_start = ALIGN(start, frag_align);
1322         uint64_t frag_end = end & ~(frag_align - 1);
1323
1324         /* system pages are non continuously */
1325         if (params->src || !(flags & AMDGPU_PTE_VALID) ||
1326             (frag_start >= frag_end))
1327                 return amdgpu_vm_update_ptes(params, start, end, dst, flags);
1328
1329         /* handle the 4K area at the beginning */
1330         if (start != frag_start) {
1331                 r = amdgpu_vm_update_ptes(params, start, frag_start,
1332                                           dst, flags);
1333                 if (r)
1334                         return r;
1335                 dst += (frag_start - start) * AMDGPU_GPU_PAGE_SIZE;
1336         }
1337
1338         /* handle the area in the middle */
1339         r = amdgpu_vm_update_ptes(params, frag_start, frag_end, dst,
1340                                   flags | frag_flags);
1341         if (r)
1342                 return r;
1343
1344         /* handle the 4K area at the end */
1345         if (frag_end != end) {
1346                 dst += (frag_end - frag_start) * AMDGPU_GPU_PAGE_SIZE;
1347                 r = amdgpu_vm_update_ptes(params, frag_end, end, dst, flags);
1348         }
1349         return r;
1350 }
1351
1352 /**
1353  * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
1354  *
1355  * @adev: amdgpu_device pointer
1356  * @exclusive: fence we need to sync to
1357  * @src: address where to copy page table entries from
1358  * @pages_addr: DMA addresses to use for mapping
1359  * @vm: requested vm
1360  * @start: start of mapped range
1361  * @last: last mapped entry
1362  * @flags: flags for the entries
1363  * @addr: addr to set the area to
1364  * @fence: optional resulting fence
1365  *
1366  * Fill in the page table entries between @start and @last.
1367  * Returns 0 for success, -EINVAL for failure.
1368  */
1369 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
1370                                        struct dma_fence *exclusive,
1371                                        uint64_t src,
1372                                        dma_addr_t *pages_addr,
1373                                        struct amdgpu_vm *vm,
1374                                        uint64_t start, uint64_t last,
1375                                        uint64_t flags, uint64_t addr,
1376                                        struct dma_fence **fence)
1377 {
1378         struct amdgpu_ring *ring;
1379         void *owner = AMDGPU_FENCE_OWNER_VM;
1380         unsigned nptes, ncmds, ndw;
1381         struct amdgpu_job *job;
1382         struct amdgpu_pte_update_params params;
1383         struct dma_fence *f = NULL;
1384         int r;
1385
1386         memset(&params, 0, sizeof(params));
1387         params.adev = adev;
1388         params.vm = vm;
1389         params.src = src;
1390
1391         ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
1392
1393         /* sync to everything on unmapping */
1394         if (!(flags & AMDGPU_PTE_VALID))
1395                 owner = AMDGPU_FENCE_OWNER_UNDEFINED;
1396
1397         nptes = last - start + 1;
1398
1399         /*
1400          * reserve space for one command every (1 << BLOCK_SIZE)
1401          *  entries or 2k dwords (whatever is smaller)
1402          */
1403         ncmds = (nptes >> min(adev->vm_manager.block_size, 11u)) + 1;
1404
1405         /* padding, etc. */
1406         ndw = 64;
1407
1408         if (src) {
1409                 /* only copy commands needed */
1410                 ndw += ncmds * 7;
1411
1412                 params.func = amdgpu_vm_do_copy_ptes;
1413
1414         } else if (pages_addr) {
1415                 /* copy commands needed */
1416                 ndw += ncmds * 7;
1417
1418                 /* and also PTEs */
1419                 ndw += nptes * 2;
1420
1421                 params.func = amdgpu_vm_do_copy_ptes;
1422
1423         } else {
1424                 /* set page commands needed */
1425                 ndw += ncmds * 10;
1426
1427                 /* two extra commands for begin/end of fragment */
1428                 ndw += 2 * 10;
1429
1430                 params.func = amdgpu_vm_do_set_ptes;
1431         }
1432
1433         r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
1434         if (r)
1435                 return r;
1436
1437         params.ib = &job->ibs[0];
1438
1439         if (!src && pages_addr) {
1440                 uint64_t *pte;
1441                 unsigned i;
1442
1443                 /* Put the PTEs at the end of the IB. */
1444                 i = ndw - nptes * 2;
1445                 pte= (uint64_t *)&(job->ibs->ptr[i]);
1446                 params.src = job->ibs->gpu_addr + i * 4;
1447
1448                 for (i = 0; i < nptes; ++i) {
1449                         pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
1450                                                     AMDGPU_GPU_PAGE_SIZE);
1451                         pte[i] |= flags;
1452                 }
1453                 addr = 0;
1454         }
1455
1456         r = amdgpu_sync_fence(adev, &job->sync, exclusive);
1457         if (r)
1458                 goto error_free;
1459
1460         r = amdgpu_sync_resv(adev, &job->sync, vm->root.bo->tbo.resv,
1461                              owner);
1462         if (r)
1463                 goto error_free;
1464
1465         r = reservation_object_reserve_shared(vm->root.bo->tbo.resv);
1466         if (r)
1467                 goto error_free;
1468
1469         params.shadow = true;
1470         r = amdgpu_vm_frag_ptes(&params, start, last + 1, addr, flags);
1471         if (r)
1472                 goto error_free;
1473         params.shadow = false;
1474         r = amdgpu_vm_frag_ptes(&params, start, last + 1, addr, flags);
1475         if (r)
1476                 goto error_free;
1477
1478         amdgpu_ring_pad_ib(ring, params.ib);
1479         WARN_ON(params.ib->length_dw > ndw);
1480         r = amdgpu_job_submit(job, ring, &vm->entity,
1481                               AMDGPU_FENCE_OWNER_VM, &f);
1482         if (r)
1483                 goto error_free;
1484
1485         amdgpu_bo_fence(vm->root.bo, f, true);
1486         dma_fence_put(*fence);
1487         *fence = f;
1488         return 0;
1489
1490 error_free:
1491         amdgpu_job_free(job);
1492         return r;
1493 }
1494
1495 /**
1496  * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
1497  *
1498  * @adev: amdgpu_device pointer
1499  * @exclusive: fence we need to sync to
1500  * @gtt_flags: flags as they are used for GTT
1501  * @pages_addr: DMA addresses to use for mapping
1502  * @vm: requested vm
1503  * @mapping: mapped range and flags to use for the update
1504  * @flags: HW flags for the mapping
1505  * @nodes: array of drm_mm_nodes with the MC addresses
1506  * @fence: optional resulting fence
1507  *
1508  * Split the mapping into smaller chunks so that each update fits
1509  * into a SDMA IB.
1510  * Returns 0 for success, -EINVAL for failure.
1511  */
1512 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1513                                       struct dma_fence *exclusive,
1514                                       uint64_t gtt_flags,
1515                                       dma_addr_t *pages_addr,
1516                                       struct amdgpu_vm *vm,
1517                                       struct amdgpu_bo_va_mapping *mapping,
1518                                       uint64_t flags,
1519                                       struct drm_mm_node *nodes,
1520                                       struct dma_fence **fence)
1521 {
1522         uint64_t pfn, src = 0, start = mapping->start;
1523         int r;
1524
1525         /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1526          * but in case of something, we filter the flags in first place
1527          */
1528         if (!(mapping->flags & AMDGPU_PTE_READABLE))
1529                 flags &= ~AMDGPU_PTE_READABLE;
1530         if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1531                 flags &= ~AMDGPU_PTE_WRITEABLE;
1532
1533         flags &= ~AMDGPU_PTE_EXECUTABLE;
1534         flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1535
1536         flags &= ~AMDGPU_PTE_MTYPE_MASK;
1537         flags |= (mapping->flags & AMDGPU_PTE_MTYPE_MASK);
1538
1539         if ((mapping->flags & AMDGPU_PTE_PRT) &&
1540             (adev->asic_type >= CHIP_VEGA10)) {
1541                 flags |= AMDGPU_PTE_PRT;
1542                 flags &= ~AMDGPU_PTE_VALID;
1543         }
1544
1545         trace_amdgpu_vm_bo_update(mapping);
1546
1547         pfn = mapping->offset >> PAGE_SHIFT;
1548         if (nodes) {
1549                 while (pfn >= nodes->size) {
1550                         pfn -= nodes->size;
1551                         ++nodes;
1552                 }
1553         }
1554
1555         do {
1556                 uint64_t max_entries;
1557                 uint64_t addr, last;
1558
1559                 if (nodes) {
1560                         addr = nodes->start << PAGE_SHIFT;
1561                         max_entries = (nodes->size - pfn) *
1562                                 (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
1563                 } else {
1564                         addr = 0;
1565                         max_entries = S64_MAX;
1566                 }
1567
1568                 if (pages_addr) {
1569                         if (flags == gtt_flags)
1570                                 src = adev->gart.table_addr +
1571                                         (addr >> AMDGPU_GPU_PAGE_SHIFT) * 8;
1572                         else
1573                                 max_entries = min(max_entries, 16ull * 1024ull);
1574                         addr = 0;
1575                 } else if (flags & AMDGPU_PTE_VALID) {
1576                         addr += adev->vm_manager.vram_base_offset;
1577                 }
1578                 addr += pfn << PAGE_SHIFT;
1579
1580                 last = min((uint64_t)mapping->last, start + max_entries - 1);
1581                 r = amdgpu_vm_bo_update_mapping(adev, exclusive,
1582                                                 src, pages_addr, vm,
1583                                                 start, last, flags, addr,
1584                                                 fence);
1585                 if (r)
1586                         return r;
1587
1588                 pfn += last - start + 1;
1589                 if (nodes && nodes->size == pfn) {
1590                         pfn = 0;
1591                         ++nodes;
1592                 }
1593                 start = last + 1;
1594
1595         } while (unlikely(start != mapping->last + 1));
1596
1597         return 0;
1598 }
1599
1600 /**
1601  * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1602  *
1603  * @adev: amdgpu_device pointer
1604  * @bo_va: requested BO and VM object
1605  * @clear: if true clear the entries
1606  *
1607  * Fill in the page table entries for @bo_va.
1608  * Returns 0 for success, -EINVAL for failure.
1609  */
1610 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
1611                         struct amdgpu_bo_va *bo_va,
1612                         bool clear)
1613 {
1614         struct amdgpu_vm *vm = bo_va->vm;
1615         struct amdgpu_bo_va_mapping *mapping;
1616         dma_addr_t *pages_addr = NULL;
1617         uint64_t gtt_flags, flags;
1618         struct ttm_mem_reg *mem;
1619         struct drm_mm_node *nodes;
1620         struct dma_fence *exclusive;
1621         int r;
1622
1623         if (clear || !bo_va->bo) {
1624                 mem = NULL;
1625                 nodes = NULL;
1626                 exclusive = NULL;
1627         } else {
1628                 struct ttm_dma_tt *ttm;
1629
1630                 mem = &bo_va->bo->tbo.mem;
1631                 nodes = mem->mm_node;
1632                 if (mem->mem_type == TTM_PL_TT) {
1633                         ttm = container_of(bo_va->bo->tbo.ttm, struct
1634                                            ttm_dma_tt, ttm);
1635                         pages_addr = ttm->dma_address;
1636                 }
1637                 exclusive = reservation_object_get_excl(bo_va->bo->tbo.resv);
1638         }
1639
1640         if (bo_va->bo) {
1641                 flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
1642                 gtt_flags = (amdgpu_ttm_is_bound(bo_va->bo->tbo.ttm) &&
1643                         adev == amdgpu_ttm_adev(bo_va->bo->tbo.bdev)) ?
1644                         flags : 0;
1645         } else {
1646                 flags = 0x0;
1647                 gtt_flags = ~0x0;
1648         }
1649
1650         spin_lock(&vm->status_lock);
1651         if (!list_empty(&bo_va->vm_status))
1652                 list_splice_init(&bo_va->valids, &bo_va->invalids);
1653         spin_unlock(&vm->status_lock);
1654
1655         list_for_each_entry(mapping, &bo_va->invalids, list) {
1656                 r = amdgpu_vm_bo_split_mapping(adev, exclusive,
1657                                                gtt_flags, pages_addr, vm,
1658                                                mapping, flags, nodes,
1659                                                &bo_va->last_pt_update);
1660                 if (r)
1661                         return r;
1662         }
1663
1664         if (trace_amdgpu_vm_bo_mapping_enabled()) {
1665                 list_for_each_entry(mapping, &bo_va->valids, list)
1666                         trace_amdgpu_vm_bo_mapping(mapping);
1667
1668                 list_for_each_entry(mapping, &bo_va->invalids, list)
1669                         trace_amdgpu_vm_bo_mapping(mapping);
1670         }
1671
1672         spin_lock(&vm->status_lock);
1673         list_splice_init(&bo_va->invalids, &bo_va->valids);
1674         list_del_init(&bo_va->vm_status);
1675         if (clear)
1676                 list_add(&bo_va->vm_status, &vm->cleared);
1677         spin_unlock(&vm->status_lock);
1678
1679         return 0;
1680 }
1681
1682 /**
1683  * amdgpu_vm_update_prt_state - update the global PRT state
1684  */
1685 static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1686 {
1687         unsigned long flags;
1688         bool enable;
1689
1690         spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1691         enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1692         adev->gart.gart_funcs->set_prt(adev, enable);
1693         spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1694 }
1695
1696 /**
1697  * amdgpu_vm_prt_get - add a PRT user
1698  */
1699 static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1700 {
1701         if (!adev->gart.gart_funcs->set_prt)
1702                 return;
1703
1704         if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1705                 amdgpu_vm_update_prt_state(adev);
1706 }
1707
1708 /**
1709  * amdgpu_vm_prt_put - drop a PRT user
1710  */
1711 static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1712 {
1713         if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1714                 amdgpu_vm_update_prt_state(adev);
1715 }
1716
1717 /**
1718  * amdgpu_vm_prt_cb - callback for updating the PRT status
1719  */
1720 static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1721 {
1722         struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1723
1724         amdgpu_vm_prt_put(cb->adev);
1725         kfree(cb);
1726 }
1727
1728 /**
1729  * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
1730  */
1731 static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1732                                  struct dma_fence *fence)
1733 {
1734         struct amdgpu_prt_cb *cb;
1735
1736         if (!adev->gart.gart_funcs->set_prt)
1737                 return;
1738
1739         cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1740         if (!cb) {
1741                 /* Last resort when we are OOM */
1742                 if (fence)
1743                         dma_fence_wait(fence, false);
1744
1745                 amdgpu_vm_prt_put(adev);
1746         } else {
1747                 cb->adev = adev;
1748                 if (!fence || dma_fence_add_callback(fence, &cb->cb,
1749                                                      amdgpu_vm_prt_cb))
1750                         amdgpu_vm_prt_cb(fence, &cb->cb);
1751         }
1752 }
1753
1754 /**
1755  * amdgpu_vm_free_mapping - free a mapping
1756  *
1757  * @adev: amdgpu_device pointer
1758  * @vm: requested vm
1759  * @mapping: mapping to be freed
1760  * @fence: fence of the unmap operation
1761  *
1762  * Free a mapping and make sure we decrease the PRT usage count if applicable.
1763  */
1764 static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1765                                    struct amdgpu_vm *vm,
1766                                    struct amdgpu_bo_va_mapping *mapping,
1767                                    struct dma_fence *fence)
1768 {
1769         if (mapping->flags & AMDGPU_PTE_PRT)
1770                 amdgpu_vm_add_prt_cb(adev, fence);
1771         kfree(mapping);
1772 }
1773
1774 /**
1775  * amdgpu_vm_prt_fini - finish all prt mappings
1776  *
1777  * @adev: amdgpu_device pointer
1778  * @vm: requested vm
1779  *
1780  * Register a cleanup callback to disable PRT support after VM dies.
1781  */
1782 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1783 {
1784         struct reservation_object *resv = vm->root.bo->tbo.resv;
1785         struct dma_fence *excl, **shared;
1786         unsigned i, shared_count;
1787         int r;
1788
1789         r = reservation_object_get_fences_rcu(resv, &excl,
1790                                               &shared_count, &shared);
1791         if (r) {
1792                 /* Not enough memory to grab the fence list, as last resort
1793                  * block for all the fences to complete.
1794                  */
1795                 reservation_object_wait_timeout_rcu(resv, true, false,
1796                                                     MAX_SCHEDULE_TIMEOUT);
1797                 return;
1798         }
1799
1800         /* Add a callback for each fence in the reservation object */
1801         amdgpu_vm_prt_get(adev);
1802         amdgpu_vm_add_prt_cb(adev, excl);
1803
1804         for (i = 0; i < shared_count; ++i) {
1805                 amdgpu_vm_prt_get(adev);
1806                 amdgpu_vm_add_prt_cb(adev, shared[i]);
1807         }
1808
1809         kfree(shared);
1810 }
1811
1812 /**
1813  * amdgpu_vm_clear_freed - clear freed BOs in the PT
1814  *
1815  * @adev: amdgpu_device pointer
1816  * @vm: requested vm
1817  * @fence: optional resulting fence (unchanged if no work needed to be done
1818  * or if an error occurred)
1819  *
1820  * Make sure all freed BOs are cleared in the PT.
1821  * Returns 0 for success.
1822  *
1823  * PTs have to be reserved and mutex must be locked!
1824  */
1825 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1826                           struct amdgpu_vm *vm,
1827                           struct dma_fence **fence)
1828 {
1829         struct amdgpu_bo_va_mapping *mapping;
1830         struct dma_fence *f = NULL;
1831         int r;
1832
1833         while (!list_empty(&vm->freed)) {
1834                 mapping = list_first_entry(&vm->freed,
1835                         struct amdgpu_bo_va_mapping, list);
1836                 list_del(&mapping->list);
1837
1838                 r = amdgpu_vm_bo_update_mapping(adev, NULL, 0, NULL, vm,
1839                                                 mapping->start, mapping->last,
1840                                                 0, 0, &f);
1841                 amdgpu_vm_free_mapping(adev, vm, mapping, f);
1842                 if (r) {
1843                         dma_fence_put(f);
1844                         return r;
1845                 }
1846         }
1847
1848         if (fence && f) {
1849                 dma_fence_put(*fence);
1850                 *fence = f;
1851         } else {
1852                 dma_fence_put(f);
1853         }
1854
1855         return 0;
1856
1857 }
1858
1859 /**
1860  * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT
1861  *
1862  * @adev: amdgpu_device pointer
1863  * @vm: requested vm
1864  *
1865  * Make sure all invalidated BOs are cleared in the PT.
1866  * Returns 0 for success.
1867  *
1868  * PTs have to be reserved and mutex must be locked!
1869  */
1870 int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
1871                              struct amdgpu_vm *vm, struct amdgpu_sync *sync)
1872 {
1873         struct amdgpu_bo_va *bo_va = NULL;
1874         int r = 0;
1875
1876         spin_lock(&vm->status_lock);
1877         while (!list_empty(&vm->invalidated)) {
1878                 bo_va = list_first_entry(&vm->invalidated,
1879                         struct amdgpu_bo_va, vm_status);
1880                 spin_unlock(&vm->status_lock);
1881
1882                 r = amdgpu_vm_bo_update(adev, bo_va, true);
1883                 if (r)
1884                         return r;
1885
1886                 spin_lock(&vm->status_lock);
1887         }
1888         spin_unlock(&vm->status_lock);
1889
1890         if (bo_va)
1891                 r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
1892
1893         return r;
1894 }
1895
1896 /**
1897  * amdgpu_vm_bo_add - add a bo to a specific vm
1898  *
1899  * @adev: amdgpu_device pointer
1900  * @vm: requested vm
1901  * @bo: amdgpu buffer object
1902  *
1903  * Add @bo into the requested vm.
1904  * Add @bo to the list of bos associated with the vm
1905  * Returns newly added bo_va or NULL for failure
1906  *
1907  * Object has to be reserved!
1908  */
1909 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1910                                       struct amdgpu_vm *vm,
1911                                       struct amdgpu_bo *bo)
1912 {
1913         struct amdgpu_bo_va *bo_va;
1914
1915         bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
1916         if (bo_va == NULL) {
1917                 return NULL;
1918         }
1919         bo_va->vm = vm;
1920         bo_va->bo = bo;
1921         bo_va->ref_count = 1;
1922         INIT_LIST_HEAD(&bo_va->bo_list);
1923         INIT_LIST_HEAD(&bo_va->valids);
1924         INIT_LIST_HEAD(&bo_va->invalids);
1925         INIT_LIST_HEAD(&bo_va->vm_status);
1926
1927         if (bo)
1928                 list_add_tail(&bo_va->bo_list, &bo->va);
1929
1930         return bo_va;
1931 }
1932
1933 /**
1934  * amdgpu_vm_bo_map - map bo inside a vm
1935  *
1936  * @adev: amdgpu_device pointer
1937  * @bo_va: bo_va to store the address
1938  * @saddr: where to map the BO
1939  * @offset: requested offset in the BO
1940  * @flags: attributes of pages (read/write/valid/etc.)
1941  *
1942  * Add a mapping of the BO at the specefied addr into the VM.
1943  * Returns 0 for success, error for failure.
1944  *
1945  * Object has to be reserved and unreserved outside!
1946  */
1947 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1948                      struct amdgpu_bo_va *bo_va,
1949                      uint64_t saddr, uint64_t offset,
1950                      uint64_t size, uint64_t flags)
1951 {
1952         struct amdgpu_bo_va_mapping *mapping, *tmp;
1953         struct amdgpu_vm *vm = bo_va->vm;
1954         uint64_t eaddr;
1955
1956         /* validate the parameters */
1957         if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
1958             size == 0 || size & AMDGPU_GPU_PAGE_MASK)
1959                 return -EINVAL;
1960
1961         /* make sure object fit at this offset */
1962         eaddr = saddr + size - 1;
1963         if (saddr >= eaddr ||
1964             (bo_va->bo && offset + size > amdgpu_bo_size(bo_va->bo)))
1965                 return -EINVAL;
1966
1967         saddr /= AMDGPU_GPU_PAGE_SIZE;
1968         eaddr /= AMDGPU_GPU_PAGE_SIZE;
1969
1970         tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1971         if (tmp) {
1972                 /* bo and tmp overlap, invalid addr */
1973                 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1974                         "0x%010Lx-0x%010Lx\n", bo_va->bo, saddr, eaddr,
1975                         tmp->start, tmp->last + 1);
1976                 return -EINVAL;
1977         }
1978
1979         mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1980         if (!mapping)
1981                 return -ENOMEM;
1982
1983         INIT_LIST_HEAD(&mapping->list);
1984         mapping->start = saddr;
1985         mapping->last = eaddr;
1986         mapping->offset = offset;
1987         mapping->flags = flags;
1988
1989         list_add(&mapping->list, &bo_va->invalids);
1990         amdgpu_vm_it_insert(mapping, &vm->va);
1991
1992         if (flags & AMDGPU_PTE_PRT)
1993                 amdgpu_vm_prt_get(adev);
1994
1995         return 0;
1996 }
1997
1998 /**
1999  * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
2000  *
2001  * @adev: amdgpu_device pointer
2002  * @bo_va: bo_va to store the address
2003  * @saddr: where to map the BO
2004  * @offset: requested offset in the BO
2005  * @flags: attributes of pages (read/write/valid/etc.)
2006  *
2007  * Add a mapping of the BO at the specefied addr into the VM. Replace existing
2008  * mappings as we do so.
2009  * Returns 0 for success, error for failure.
2010  *
2011  * Object has to be reserved and unreserved outside!
2012  */
2013 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
2014                              struct amdgpu_bo_va *bo_va,
2015                              uint64_t saddr, uint64_t offset,
2016                              uint64_t size, uint64_t flags)
2017 {
2018         struct amdgpu_bo_va_mapping *mapping;
2019         struct amdgpu_vm *vm = bo_va->vm;
2020         uint64_t eaddr;
2021         int r;
2022
2023         /* validate the parameters */
2024         if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
2025             size == 0 || size & AMDGPU_GPU_PAGE_MASK)
2026                 return -EINVAL;
2027
2028         /* make sure object fit at this offset */
2029         eaddr = saddr + size - 1;
2030         if (saddr >= eaddr ||
2031             (bo_va->bo && offset + size > amdgpu_bo_size(bo_va->bo)))
2032                 return -EINVAL;
2033
2034         /* Allocate all the needed memory */
2035         mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
2036         if (!mapping)
2037                 return -ENOMEM;
2038
2039         r = amdgpu_vm_bo_clear_mappings(adev, bo_va->vm, saddr, size);
2040         if (r) {
2041                 kfree(mapping);
2042                 return r;
2043         }
2044
2045         saddr /= AMDGPU_GPU_PAGE_SIZE;
2046         eaddr /= AMDGPU_GPU_PAGE_SIZE;
2047
2048         mapping->start = saddr;
2049         mapping->last = eaddr;
2050         mapping->offset = offset;
2051         mapping->flags = flags;
2052
2053         list_add(&mapping->list, &bo_va->invalids);
2054         amdgpu_vm_it_insert(mapping, &vm->va);
2055
2056         if (flags & AMDGPU_PTE_PRT)
2057                 amdgpu_vm_prt_get(adev);
2058
2059         return 0;
2060 }
2061
2062 /**
2063  * amdgpu_vm_bo_unmap - remove bo mapping from vm
2064  *
2065  * @adev: amdgpu_device pointer
2066  * @bo_va: bo_va to remove the address from
2067  * @saddr: where to the BO is mapped
2068  *
2069  * Remove a mapping of the BO at the specefied addr from the VM.
2070  * Returns 0 for success, error for failure.
2071  *
2072  * Object has to be reserved and unreserved outside!
2073  */
2074 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
2075                        struct amdgpu_bo_va *bo_va,
2076                        uint64_t saddr)
2077 {
2078         struct amdgpu_bo_va_mapping *mapping;
2079         struct amdgpu_vm *vm = bo_va->vm;
2080         bool valid = true;
2081
2082         saddr /= AMDGPU_GPU_PAGE_SIZE;
2083
2084         list_for_each_entry(mapping, &bo_va->valids, list) {
2085                 if (mapping->start == saddr)
2086                         break;
2087         }
2088
2089         if (&mapping->list == &bo_va->valids) {
2090                 valid = false;
2091
2092                 list_for_each_entry(mapping, &bo_va->invalids, list) {
2093                         if (mapping->start == saddr)
2094                                 break;
2095                 }
2096
2097                 if (&mapping->list == &bo_va->invalids)
2098                         return -ENOENT;
2099         }
2100
2101         list_del(&mapping->list);
2102         amdgpu_vm_it_remove(mapping, &vm->va);
2103         trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2104
2105         if (valid)
2106                 list_add(&mapping->list, &vm->freed);
2107         else
2108                 amdgpu_vm_free_mapping(adev, vm, mapping,
2109                                        bo_va->last_pt_update);
2110
2111         return 0;
2112 }
2113
2114 /**
2115  * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
2116  *
2117  * @adev: amdgpu_device pointer
2118  * @vm: VM structure to use
2119  * @saddr: start of the range
2120  * @size: size of the range
2121  *
2122  * Remove all mappings in a range, split them as appropriate.
2123  * Returns 0 for success, error for failure.
2124  */
2125 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
2126                                 struct amdgpu_vm *vm,
2127                                 uint64_t saddr, uint64_t size)
2128 {
2129         struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
2130         LIST_HEAD(removed);
2131         uint64_t eaddr;
2132
2133         eaddr = saddr + size - 1;
2134         saddr /= AMDGPU_GPU_PAGE_SIZE;
2135         eaddr /= AMDGPU_GPU_PAGE_SIZE;
2136
2137         /* Allocate all the needed memory */
2138         before = kzalloc(sizeof(*before), GFP_KERNEL);
2139         if (!before)
2140                 return -ENOMEM;
2141         INIT_LIST_HEAD(&before->list);
2142
2143         after = kzalloc(sizeof(*after), GFP_KERNEL);
2144         if (!after) {
2145                 kfree(before);
2146                 return -ENOMEM;
2147         }
2148         INIT_LIST_HEAD(&after->list);
2149
2150         /* Now gather all removed mappings */
2151         tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2152         while (tmp) {
2153                 /* Remember mapping split at the start */
2154                 if (tmp->start < saddr) {
2155                         before->start = tmp->start;
2156                         before->last = saddr - 1;
2157                         before->offset = tmp->offset;
2158                         before->flags = tmp->flags;
2159                         list_add(&before->list, &tmp->list);
2160                 }
2161
2162                 /* Remember mapping split at the end */
2163                 if (tmp->last > eaddr) {
2164                         after->start = eaddr + 1;
2165                         after->last = tmp->last;
2166                         after->offset = tmp->offset;
2167                         after->offset += after->start - tmp->start;
2168                         after->flags = tmp->flags;
2169                         list_add(&after->list, &tmp->list);
2170                 }
2171
2172                 list_del(&tmp->list);
2173                 list_add(&tmp->list, &removed);
2174
2175                 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
2176         }
2177
2178         /* And free them up */
2179         list_for_each_entry_safe(tmp, next, &removed, list) {
2180                 amdgpu_vm_it_remove(tmp, &vm->va);
2181                 list_del(&tmp->list);
2182
2183                 if (tmp->start < saddr)
2184                     tmp->start = saddr;
2185                 if (tmp->last > eaddr)
2186                     tmp->last = eaddr;
2187
2188                 list_add(&tmp->list, &vm->freed);
2189                 trace_amdgpu_vm_bo_unmap(NULL, tmp);
2190         }
2191
2192         /* Insert partial mapping before the range */
2193         if (!list_empty(&before->list)) {
2194                 amdgpu_vm_it_insert(before, &vm->va);
2195                 if (before->flags & AMDGPU_PTE_PRT)
2196                         amdgpu_vm_prt_get(adev);
2197         } else {
2198                 kfree(before);
2199         }
2200
2201         /* Insert partial mapping after the range */
2202         if (!list_empty(&after->list)) {
2203                 amdgpu_vm_it_insert(after, &vm->va);
2204                 if (after->flags & AMDGPU_PTE_PRT)
2205                         amdgpu_vm_prt_get(adev);
2206         } else {
2207                 kfree(after);
2208         }
2209
2210         return 0;
2211 }
2212
2213 /**
2214  * amdgpu_vm_bo_rmv - remove a bo to a specific vm
2215  *
2216  * @adev: amdgpu_device pointer
2217  * @bo_va: requested bo_va
2218  *
2219  * Remove @bo_va->bo from the requested vm.
2220  *
2221  * Object have to be reserved!
2222  */
2223 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
2224                       struct amdgpu_bo_va *bo_va)
2225 {
2226         struct amdgpu_bo_va_mapping *mapping, *next;
2227         struct amdgpu_vm *vm = bo_va->vm;
2228
2229         list_del(&bo_va->bo_list);
2230
2231         spin_lock(&vm->status_lock);
2232         list_del(&bo_va->vm_status);
2233         spin_unlock(&vm->status_lock);
2234
2235         list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
2236                 list_del(&mapping->list);
2237                 amdgpu_vm_it_remove(mapping, &vm->va);
2238                 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2239                 list_add(&mapping->list, &vm->freed);
2240         }
2241         list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
2242                 list_del(&mapping->list);
2243                 amdgpu_vm_it_remove(mapping, &vm->va);
2244                 amdgpu_vm_free_mapping(adev, vm, mapping,
2245                                        bo_va->last_pt_update);
2246         }
2247
2248         dma_fence_put(bo_va->last_pt_update);
2249         kfree(bo_va);
2250 }
2251
2252 /**
2253  * amdgpu_vm_bo_invalidate - mark the bo as invalid
2254  *
2255  * @adev: amdgpu_device pointer
2256  * @vm: requested vm
2257  * @bo: amdgpu buffer object
2258  *
2259  * Mark @bo as invalid.
2260  */
2261 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
2262                              struct amdgpu_bo *bo)
2263 {
2264         struct amdgpu_bo_va *bo_va;
2265
2266         list_for_each_entry(bo_va, &bo->va, bo_list) {
2267                 spin_lock(&bo_va->vm->status_lock);
2268                 if (list_empty(&bo_va->vm_status))
2269                         list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
2270                 spin_unlock(&bo_va->vm->status_lock);
2271         }
2272 }
2273
2274 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2275 {
2276         /* Total bits covered by PD + PTs */
2277         unsigned bits = ilog2(vm_size) + 18;
2278
2279         /* Make sure the PD is 4K in size up to 8GB address space.
2280            Above that split equal between PD and PTs */
2281         if (vm_size <= 8)
2282                 return (bits - 9);
2283         else
2284                 return ((bits + 3) / 2);
2285 }
2286
2287 /**
2288  * amdgpu_vm_adjust_size - adjust vm size and block size
2289  *
2290  * @adev: amdgpu_device pointer
2291  * @vm_size: the default vm size if it's set auto
2292  */
2293 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size)
2294 {
2295         /* adjust vm size firstly */
2296         if (amdgpu_vm_size == -1)
2297                 adev->vm_manager.vm_size = vm_size;
2298         else
2299                 adev->vm_manager.vm_size = amdgpu_vm_size;
2300
2301         /* block size depends on vm size */
2302         if (amdgpu_vm_block_size == -1)
2303                 adev->vm_manager.block_size =
2304                         amdgpu_vm_get_block_size(adev->vm_manager.vm_size);
2305         else
2306                 adev->vm_manager.block_size = amdgpu_vm_block_size;
2307
2308         DRM_INFO("vm size is %llu GB, block size is %u-bit\n",
2309                 adev->vm_manager.vm_size, adev->vm_manager.block_size);
2310 }
2311
2312 /**
2313  * amdgpu_vm_init - initialize a vm instance
2314  *
2315  * @adev: amdgpu_device pointer
2316  * @vm: requested vm
2317  *
2318  * Init @vm fields.
2319  */
2320 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2321 {
2322         const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
2323                 AMDGPU_VM_PTE_COUNT(adev) * 8);
2324         unsigned ring_instance;
2325         struct amdgpu_ring *ring;
2326         struct amd_sched_rq *rq;
2327         int r, i;
2328
2329         vm->va = RB_ROOT;
2330         vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
2331         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2332                 vm->reserved_vmid[i] = NULL;
2333         spin_lock_init(&vm->status_lock);
2334         INIT_LIST_HEAD(&vm->invalidated);
2335         INIT_LIST_HEAD(&vm->cleared);
2336         INIT_LIST_HEAD(&vm->freed);
2337
2338         /* create scheduler entity for page table updates */
2339
2340         ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
2341         ring_instance %= adev->vm_manager.vm_pte_num_rings;
2342         ring = adev->vm_manager.vm_pte_rings[ring_instance];
2343         rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
2344         r = amd_sched_entity_init(&ring->sched, &vm->entity,
2345                                   rq, amdgpu_sched_jobs);
2346         if (r)
2347                 return r;
2348
2349         vm->last_dir_update = NULL;
2350
2351         r = amdgpu_bo_create(adev, amdgpu_vm_bo_size(adev, 0), align, true,
2352                              AMDGPU_GEM_DOMAIN_VRAM,
2353                              AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
2354                              AMDGPU_GEM_CREATE_SHADOW |
2355                              AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
2356                              AMDGPU_GEM_CREATE_VRAM_CLEARED,
2357                              NULL, NULL, &vm->root.bo);
2358         if (r)
2359                 goto error_free_sched_entity;
2360
2361         r = amdgpu_bo_reserve(vm->root.bo, false);
2362         if (r)
2363                 goto error_free_root;
2364
2365         vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
2366         amdgpu_bo_unreserve(vm->root.bo);
2367
2368         return 0;
2369
2370 error_free_root:
2371         amdgpu_bo_unref(&vm->root.bo->shadow);
2372         amdgpu_bo_unref(&vm->root.bo);
2373         vm->root.bo = NULL;
2374
2375 error_free_sched_entity:
2376         amd_sched_entity_fini(&ring->sched, &vm->entity);
2377
2378         return r;
2379 }
2380
2381 /**
2382  * amdgpu_vm_free_levels - free PD/PT levels
2383  *
2384  * @level: PD/PT starting level to free
2385  *
2386  * Free the page directory or page table level and all sub levels.
2387  */
2388 static void amdgpu_vm_free_levels(struct amdgpu_vm_pt *level)
2389 {
2390         unsigned i;
2391
2392         if (level->bo) {
2393                 amdgpu_bo_unref(&level->bo->shadow);
2394                 amdgpu_bo_unref(&level->bo);
2395         }
2396
2397         if (level->entries)
2398                 for (i = 0; i <= level->last_entry_used; i++)
2399                         amdgpu_vm_free_levels(&level->entries[i]);
2400
2401         drm_free_large(level->entries);
2402 }
2403
2404 /**
2405  * amdgpu_vm_fini - tear down a vm instance
2406  *
2407  * @adev: amdgpu_device pointer
2408  * @vm: requested vm
2409  *
2410  * Tear down @vm.
2411  * Unbind the VM and remove all bos from the vm bo list
2412  */
2413 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2414 {
2415         struct amdgpu_bo_va_mapping *mapping, *tmp;
2416         bool prt_fini_needed = !!adev->gart.gart_funcs->set_prt;
2417         int i;
2418
2419         amd_sched_entity_fini(vm->entity.sched, &vm->entity);
2420
2421         if (!RB_EMPTY_ROOT(&vm->va)) {
2422                 dev_err(adev->dev, "still active bo inside vm\n");
2423         }
2424         rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, rb) {
2425                 list_del(&mapping->list);
2426                 amdgpu_vm_it_remove(mapping, &vm->va);
2427                 kfree(mapping);
2428         }
2429         list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
2430                 if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
2431                         amdgpu_vm_prt_fini(adev, vm);
2432                         prt_fini_needed = false;
2433                 }
2434
2435                 list_del(&mapping->list);
2436                 amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
2437         }
2438
2439         amdgpu_vm_free_levels(&vm->root);
2440         dma_fence_put(vm->last_dir_update);
2441         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2442                 amdgpu_vm_free_reserved_vmid(adev, vm, i);
2443 }
2444
2445 /**
2446  * amdgpu_vm_manager_init - init the VM manager
2447  *
2448  * @adev: amdgpu_device pointer
2449  *
2450  * Initialize the VM manager structures
2451  */
2452 void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2453 {
2454         unsigned i, j;
2455
2456         for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
2457                 struct amdgpu_vm_id_manager *id_mgr =
2458                         &adev->vm_manager.id_mgr[i];
2459
2460                 mutex_init(&id_mgr->lock);
2461                 INIT_LIST_HEAD(&id_mgr->ids_lru);
2462                 atomic_set(&id_mgr->reserved_vmid_num, 0);
2463
2464                 /* skip over VMID 0, since it is the system VM */
2465                 for (j = 1; j < id_mgr->num_ids; ++j) {
2466                         amdgpu_vm_reset_id(adev, i, j);
2467                         amdgpu_sync_create(&id_mgr->ids[i].active);
2468                         list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru);
2469                 }
2470         }
2471
2472         adev->vm_manager.fence_context =
2473                 dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2474         for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
2475                 adev->vm_manager.seqno[i] = 0;
2476
2477         atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
2478         atomic64_set(&adev->vm_manager.client_counter, 0);
2479         spin_lock_init(&adev->vm_manager.prt_lock);
2480         atomic_set(&adev->vm_manager.num_prt_users, 0);
2481 }
2482
2483 /**
2484  * amdgpu_vm_manager_fini - cleanup VM manager
2485  *
2486  * @adev: amdgpu_device pointer
2487  *
2488  * Cleanup the VM manager and free resources.
2489  */
2490 void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
2491 {
2492         unsigned i, j;
2493
2494         for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
2495                 struct amdgpu_vm_id_manager *id_mgr =
2496                         &adev->vm_manager.id_mgr[i];
2497
2498                 mutex_destroy(&id_mgr->lock);
2499                 for (j = 0; j < AMDGPU_NUM_VM; ++j) {
2500                         struct amdgpu_vm_id *id = &id_mgr->ids[j];
2501
2502                         amdgpu_sync_free(&id->active);
2503                         dma_fence_put(id->flushed_updates);
2504                         dma_fence_put(id->last_flush);
2505                 }
2506         }
2507 }
2508
2509 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
2510 {
2511         union drm_amdgpu_vm *args = data;
2512         struct amdgpu_device *adev = dev->dev_private;
2513         struct amdgpu_fpriv *fpriv = filp->driver_priv;
2514         int r;
2515
2516         switch (args->in.op) {
2517         case AMDGPU_VM_OP_RESERVE_VMID:
2518                 /* current, we only have requirement to reserve vmid from gfxhub */
2519                 r = amdgpu_vm_alloc_reserved_vmid(adev, &fpriv->vm,
2520                                                   AMDGPU_GFXHUB);
2521                 if (r)
2522                         return r;
2523                 break;
2524         case AMDGPU_VM_OP_UNRESERVE_VMID:
2525                 amdgpu_vm_free_reserved_vmid(adev, &fpriv->vm, AMDGPU_GFXHUB);
2526                 break;
2527         default:
2528                 return -EINVAL;
2529         }
2530
2531         return 0;
2532 }