drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c

   1 /*
   2  * Copyright 2014-2018 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22 #include <linux/dma-buf.h>
  23 #include <linux/list.h>
  24 #include <linux/pagemap.h>
  25 #include <linux/sched/mm.h>
  26 #include <linux/sched/task.h>
  27
  28 #include "amdgpu_object.h"
  29 #include "amdgpu_gem.h"
  30 #include "amdgpu_vm.h"
  31 #include "amdgpu_amdkfd.h"
  32 #include "amdgpu_dma_buf.h"
  33 #include <uapi/linux/kfd_ioctl.h>
  34 #include "amdgpu_xgmi.h"
  35 #include "kfd_smi_events.h"
  36
  37 /* Userptr restore delay, just long enough to allow consecutive VM
  38  * changes to accumulate
  39  */
  40 #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1
  41
  42 /*
  43  * Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB
  44  * BO chunk
  45  */
  46 #define VRAM_AVAILABLITY_ALIGN (1 << 21)
  47
  48 /* Impose limit on how much memory KFD can use */
  49 static struct {
  50         uint64_t max_system_mem_limit;
  51         uint64_t max_ttm_mem_limit;
  52         int64_t system_mem_used;
  53         int64_t ttm_mem_used;
  54         spinlock_t mem_limit_lock;
  55 } kfd_mem_limit;
  56
  57 static const char * const domain_bit_to_string[] = {
  58                 "CPU",
  59                 "GTT",
  60                 "VRAM",
  61                 "GDS",
  62                 "GWS",
  63                 "OA"
  64 };
  65
  66 #define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
  67
  68 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);
  69
  70 static bool kfd_mem_is_attached(struct amdgpu_vm *avm,
  71                 struct kgd_mem *mem)
  72 {
  73         struct kfd_mem_attachment *entry;
  74
  75         list_for_each_entry(entry, &mem->attachments, list)
  76                 if (entry->bo_va->base.vm == avm)
  77                         return true;
  78
  79         return false;
  80 }
  81
  82 /* Set memory usage limits. Current, limits are
  83  *  System (TTM + userptr) memory - 15/16th System RAM
  84  *  TTM memory - 3/8th System RAM
  85  */
  86 void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
  87 {
  88         struct sysinfo si;
  89         uint64_t mem;
  90
  91         si_meminfo(&si);
  92         mem = si.freeram - si.freehigh;
  93         mem *= si.mem_unit;
  94
  95         spin_lock_init(&kfd_mem_limit.mem_limit_lock);
  96         kfd_mem_limit.max_system_mem_limit = mem - (mem >> 4);
  97         kfd_mem_limit.max_ttm_mem_limit = (mem >> 1) - (mem >> 3);
  98         pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n",
  99                 (kfd_mem_limit.max_system_mem_limit >> 20),
 100                 (kfd_mem_limit.max_ttm_mem_limit >> 20));
 101 }
 102
 103 void amdgpu_amdkfd_reserve_system_mem(uint64_t size)
 104 {
 105         kfd_mem_limit.system_mem_used += size;
 106 }
 107
 108 /* Estimate page table size needed to represent a given memory size
 109  *
 110  * With 4KB pages, we need one 8 byte PTE for each 4KB of memory
 111  * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB
 112  * of memory (factor 256K, >> 18). ROCm user mode tries to optimize
 113  * for 2MB pages for TLB efficiency. However, small allocations and
 114  * fragmented system memory still need some 4KB pages. We choose a
 115  * compromise that should work in most cases without reserving too
 116  * much memory for page tables unnecessarily (factor 16K, >> 14).
 117  */
 118
 119 #define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM)
 120
 121 /**
 122  * amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size
 123  * of buffer.
 124  *
 125  * @adev: Device to which allocated BO belongs to
 126  * @size: Size of buffer, in bytes, encapsulated by B0. This should be
 127  * equivalent to amdgpu_bo_size(BO)
 128  * @alloc_flag: Flag used in allocating a BO as noted above
 129  *
 130  * Return: returns -ENOMEM in case of error, ZERO otherwise
 131  */
 132 int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 133                 uint64_t size, u32 alloc_flag)
 134 {
 135         uint64_t reserved_for_pt =
 136                 ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
 137         size_t system_mem_needed, ttm_mem_needed, vram_needed;
 138         int ret = 0;
 139
 140         system_mem_needed = 0;
 141         ttm_mem_needed = 0;
 142         vram_needed = 0;
 143         if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
 144                 system_mem_needed = size;
 145                 ttm_mem_needed = size;
 146         } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
 147                 /*
 148                  * Conservatively round up the allocation requirement to 2 MB
 149                  * to avoid fragmentation caused by 4K allocations in the tail
 150                  * 2M BO chunk.
 151                  */
 152                 vram_needed = size;
 153         } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
 154                 system_mem_needed = size;
 155         } else if (!(alloc_flag &
 156                                 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
 157                                  KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
 158                 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
 159                 return -ENOMEM;
 160         }
 161
 162         spin_lock(&kfd_mem_limit.mem_limit_lock);
 163
 164         if (kfd_mem_limit.system_mem_used + system_mem_needed >
 165             kfd_mem_limit.max_system_mem_limit)
 166                 pr_debug("Set no_system_mem_limit=1 if using shared memory\n");
 167
 168         if ((kfd_mem_limit.system_mem_used + system_mem_needed >
 169              kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) ||
 170             (kfd_mem_limit.ttm_mem_used + ttm_mem_needed >
 171              kfd_mem_limit.max_ttm_mem_limit) ||
 172             (adev && adev->kfd.vram_used + vram_needed >
 173              adev->gmc.real_vram_size -
 174              atomic64_read(&adev->vram_pin_size) -
 175              reserved_for_pt)) {
 176                 ret = -ENOMEM;
 177                 goto release;
 178         }
 179
 180         /* Update memory accounting by decreasing available system
 181          * memory, TTM memory and GPU memory as computed above
 182          */
 183         WARN_ONCE(vram_needed && !adev,
 184                   "adev reference can't be null when vram is used");
 185         if (adev) {
 186                 adev->kfd.vram_used += vram_needed;
 187                 adev->kfd.vram_used_aligned += ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN);
 188         }
 189         kfd_mem_limit.system_mem_used += system_mem_needed;
 190         kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
 191
 192 release:
 193         spin_unlock(&kfd_mem_limit.mem_limit_lock);
 194         return ret;
 195 }
 196
 197 void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
 198                 uint64_t size, u32 alloc_flag)
 199 {
 200         spin_lock(&kfd_mem_limit.mem_limit_lock);
 201
 202         if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
 203                 kfd_mem_limit.system_mem_used -= size;
 204                 kfd_mem_limit.ttm_mem_used -= size;
 205         } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
 206                 WARN_ONCE(!adev,
 207                           "adev reference can't be null when alloc mem flags vram is set");
 208                 if (adev) {
 209                         adev->kfd.vram_used -= size;
 210                         adev->kfd.vram_used_aligned -= ALIGN(size, VRAM_AVAILABLITY_ALIGN);
 211                 }
 212         } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
 213                 kfd_mem_limit.system_mem_used -= size;
 214         } else if (!(alloc_flag &
 215                                 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
 216                                  KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
 217                 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
 218                 goto release;
 219         }
 220         WARN_ONCE(adev && adev->kfd.vram_used < 0,
 221                   "KFD VRAM memory accounting unbalanced");
 222         WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,
 223                   "KFD TTM memory accounting unbalanced");
 224         WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
 225                   "KFD system memory accounting unbalanced");
 226
 227 release:
 228         spin_unlock(&kfd_mem_limit.mem_limit_lock);
 229 }
 230
 231 void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
 232 {
 233         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 234         u32 alloc_flags = bo->kfd_bo->alloc_flags;
 235         u64 size = amdgpu_bo_size(bo);
 236
 237         amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags);
 238
 239         kfree(bo->kfd_bo);
 240 }
 241
 242 /**
 243  * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
 244  * about USERPTR or DOOREBELL or MMIO BO.
 245  * @adev: Device for which dmamap BO is being created
 246  * @mem: BO of peer device that is being DMA mapped. Provides parameters
 247  *       in building the dmamap BO
 248  * @bo_out: Output parameter updated with handle of dmamap BO
 249  */
 250 static int
 251 create_dmamap_sg_bo(struct amdgpu_device *adev,
 252                  struct kgd_mem *mem, struct amdgpu_bo **bo_out)
 253 {
 254         struct drm_gem_object *gem_obj;
 255         int ret, align;
 256
 257         ret = amdgpu_bo_reserve(mem->bo, false);
 258         if (ret)
 259                 return ret;
 260
 261         align = 1;
 262         ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
 263                         AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
 264                         ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
 265
 266         amdgpu_bo_unreserve(mem->bo);
 267
 268         if (ret) {
 269                 pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
 270                 return -EINVAL;
 271         }
 272
 273         *bo_out = gem_to_amdgpu_bo(gem_obj);
 274         (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
 275         return ret;
 276 }
 277
 278 /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
 279  *  reservation object.
 280  *
 281  * @bo: [IN] Remove eviction fence(s) from this BO
 282  * @ef: [IN] This eviction fence is removed if it
 283  *  is present in the shared list.
 284  *
 285  * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held.
 286  */
 287 static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,
 288                                         struct amdgpu_amdkfd_fence *ef)
 289 {
 290         struct dma_fence *replacement;
 291
 292         if (!ef)
 293                 return -EINVAL;
 294
 295         /* TODO: Instead of block before we should use the fence of the page
 296          * table update and TLB flush here directly.
 297          */
 298         replacement = dma_fence_get_stub();
 299         dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context,
 300                                 replacement, DMA_RESV_USAGE_READ);
 301         dma_fence_put(replacement);
 302         return 0;
 303 }
 304
 305 int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo)
 306 {
 307         struct amdgpu_bo *root = bo;
 308         struct amdgpu_vm_bo_base *vm_bo;
 309         struct amdgpu_vm *vm;
 310         struct amdkfd_process_info *info;
 311         struct amdgpu_amdkfd_fence *ef;
 312         int ret;
 313
 314         /* we can always get vm_bo from root PD bo.*/
 315         while (root->parent)
 316                 root = root->parent;
 317
 318         vm_bo = root->vm_bo;
 319         if (!vm_bo)
 320                 return 0;
 321
 322         vm = vm_bo->vm;
 323         if (!vm)
 324                 return 0;
 325
 326         info = vm->process_info;
 327         if (!info || !info->eviction_fence)
 328                 return 0;
 329
 330         ef = container_of(dma_fence_get(&info->eviction_fence->base),
 331                         struct amdgpu_amdkfd_fence, base);
 332
 333         BUG_ON(!dma_resv_trylock(bo->tbo.base.resv));
 334         ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef);
 335         dma_resv_unlock(bo->tbo.base.resv);
 336
 337         dma_fence_put(&ef->base);
 338         return ret;
 339 }
 340
 341 static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,
 342                                      bool wait)
 343 {
 344         struct ttm_operation_ctx ctx = { false, false };
 345         int ret;
 346
 347         if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),
 348                  "Called with userptr BO"))
 349                 return -EINVAL;
 350
 351         amdgpu_bo_placement_from_domain(bo, domain);
 352
 353         ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 354         if (ret)
 355                 goto validate_fail;
 356         if (wait)
 357                 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
 358
 359 validate_fail:
 360         return ret;
 361 }
 362
 363 static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo)
 364 {
 365         return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false);
 366 }
 367
 368 /* vm_validate_pt_pd_bos - Validate page table and directory BOs
 369  *
 370  * Page directories are not updated here because huge page handling
 371  * during page table updates can invalidate page directory entries
 372  * again. Page directories are only updated after updating page
 373  * tables.
 374  */
 375 static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm)
 376 {
 377         struct amdgpu_bo *pd = vm->root.bo;
 378         struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
 379         int ret;
 380
 381         ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate_vm_bo, NULL);
 382         if (ret) {
 383                 pr_err("failed to validate PT BOs\n");
 384                 return ret;
 385         }
 386
 387         vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo);
 388
 389         return 0;
 390 }
 391
 392 static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)
 393 {
 394         struct amdgpu_bo *pd = vm->root.bo;
 395         struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
 396         int ret;
 397
 398         ret = amdgpu_vm_update_pdes(adev, vm, false);
 399         if (ret)
 400                 return ret;
 401
 402         return amdgpu_sync_fence(sync, vm->last_update);
 403 }
 404
 405 static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
 406 {
 407         struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);
 408         bool coherent = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT;
 409         bool uncached = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED;
 410         uint32_t mapping_flags;
 411         uint64_t pte_flags;
 412         bool snoop = false;
 413
 414         mapping_flags = AMDGPU_VM_PAGE_READABLE;
 415         if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE)
 416                 mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE;
 417         if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE)
 418                 mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
 419
 420         switch (adev->asic_type) {
 421         case CHIP_ARCTURUS:
 422         case CHIP_ALDEBARAN:
 423                 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
 424                         if (bo_adev == adev) {
 425                                 if (uncached)
 426                                         mapping_flags |= AMDGPU_VM_MTYPE_UC;
 427                                 else if (coherent)
 428                                         mapping_flags |= AMDGPU_VM_MTYPE_CC;
 429                                 else
 430                                         mapping_flags |= AMDGPU_VM_MTYPE_RW;
 431                                 if (adev->asic_type == CHIP_ALDEBARAN &&
 432                                     adev->gmc.xgmi.connected_to_cpu)
 433                                         snoop = true;
 434                         } else {
 435                                 if (uncached || coherent)
 436                                         mapping_flags |= AMDGPU_VM_MTYPE_UC;
 437                                 else
 438                                         mapping_flags |= AMDGPU_VM_MTYPE_NC;
 439                                 if (amdgpu_xgmi_same_hive(adev, bo_adev))
 440                                         snoop = true;
 441                         }
 442                 } else {
 443                         if (uncached || coherent)
 444                                 mapping_flags |= AMDGPU_VM_MTYPE_UC;
 445                         else
 446                                 mapping_flags |= AMDGPU_VM_MTYPE_NC;
 447                         snoop = true;
 448                 }
 449                 break;
 450         default:
 451                 if (uncached || coherent)
 452                         mapping_flags |= AMDGPU_VM_MTYPE_UC;
 453                 else
 454                         mapping_flags |= AMDGPU_VM_MTYPE_NC;
 455
 456                 if (!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM))
 457                         snoop = true;
 458         }
 459
 460         pte_flags = amdgpu_gem_va_map_flags(adev, mapping_flags);
 461         pte_flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
 462
 463         return pte_flags;
 464 }
 465
 466 /**
 467  * create_sg_table() - Create an sg_table for a contiguous DMA addr range
 468  * @addr: The starting address to point to
 469  * @size: Size of memory area in bytes being pointed to
 470  *
 471  * Allocates an instance of sg_table and initializes it to point to memory
 472  * area specified by input parameters. The address used to build is assumed
 473  * to be DMA mapped, if needed.
 474  *
 475  * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
 476  * because they are physically contiguous.
 477  *
 478  * Return: Initialized instance of SG Table or NULL
 479  */
 480 static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
 481 {
 482         struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
 483
 484         if (!sg)
 485                 return NULL;
 486         if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
 487                 kfree(sg);
 488                 return NULL;
 489         }
 490         sg_dma_address(sg->sgl) = addr;
 491         sg->sgl->length = size;
 492 #ifdef CONFIG_NEED_SG_DMA_LENGTH
 493         sg->sgl->dma_length = size;
 494 #endif
 495         return sg;
 496 }
 497
 498 static int
 499 kfd_mem_dmamap_userptr(struct kgd_mem *mem,
 500                        struct kfd_mem_attachment *attachment)
 501 {
 502         enum dma_data_direction direction =
 503                 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
 504                 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 505         struct ttm_operation_ctx ctx = {.interruptible = true};
 506         struct amdgpu_bo *bo = attachment->bo_va->base.bo;
 507         struct amdgpu_device *adev = attachment->adev;
 508         struct ttm_tt *src_ttm = mem->bo->tbo.ttm;
 509         struct ttm_tt *ttm = bo->tbo.ttm;
 510         int ret;
 511
 512         ttm->sg = kmalloc(sizeof(*ttm->sg), GFP_KERNEL);
 513         if (unlikely(!ttm->sg))
 514                 return -ENOMEM;
 515
 516         if (WARN_ON(ttm->num_pages != src_ttm->num_pages))
 517                 return -EINVAL;
 518
 519         /* Same sequence as in amdgpu_ttm_tt_pin_userptr */
 520         ret = sg_alloc_table_from_pages(ttm->sg, src_ttm->pages,
 521                                         ttm->num_pages, 0,
 522                                         (u64)ttm->num_pages << PAGE_SHIFT,
 523                                         GFP_KERNEL);
 524         if (unlikely(ret))
 525                 goto free_sg;
 526
 527         ret = dma_map_sgtable(adev->dev, ttm->sg, direction, 0);
 528         if (unlikely(ret))
 529                 goto release_sg;
 530
 531         drm_prime_sg_to_dma_addr_array(ttm->sg, ttm->dma_address,
 532                                        ttm->num_pages);
 533
 534         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
 535         ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 536         if (ret)
 537                 goto unmap_sg;
 538
 539         return 0;
 540
 541 unmap_sg:
 542         dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0);
 543 release_sg:
 544         pr_err("DMA map userptr failed: %d\n", ret);
 545         sg_free_table(ttm->sg);
 546 free_sg:
 547         kfree(ttm->sg);
 548         ttm->sg = NULL;
 549         return ret;
 550 }
 551
 552 static int
 553 kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
 554 {
 555         struct ttm_operation_ctx ctx = {.interruptible = true};
 556         struct amdgpu_bo *bo = attachment->bo_va->base.bo;
 557
 558         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
 559         return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 560 }
 561
 562 /**
 563  * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
 564  * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
 565  * @attachment: Virtual address attachment of the BO on accessing device
 566  *
 567  * An access request from the device that owns DOORBELL does not require DMA mapping.
 568  * This is because the request doesn't go through PCIe root complex i.e. it instead
 569  * loops back. The need to DMA map arises only when accessing peer device's DOORBELL
 570  *
 571  * In contrast, all access requests for MMIO need to be DMA mapped without regard to
 572  * device ownership. This is because access requests for MMIO go through PCIe root
 573  * complex.
 574  *
 575  * This is accomplished in two steps:
 576  *   - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
 577  *         in updating requesting device's page table
 578  *   - Signal TTM to mark memory pointed to by requesting device's BO as GPU
 579  *         accessible. This allows an update of requesting device's page table
 580  *         with entries associated with DOOREBELL or MMIO memory
 581  *
 582  * This method is invoked in the following contexts:
 583  *   - Mapping of DOORBELL or MMIO BO of same or peer device
 584  *   - Validating an evicted DOOREBELL or MMIO BO on device seeking access
 585  *
 586  * Return: ZERO if successful, NON-ZERO otherwise
 587  */
 588 static int
 589 kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
 590                      struct kfd_mem_attachment *attachment)
 591 {
 592         struct ttm_operation_ctx ctx = {.interruptible = true};
 593         struct amdgpu_bo *bo = attachment->bo_va->base.bo;
 594         struct amdgpu_device *adev = attachment->adev;
 595         struct ttm_tt *ttm = bo->tbo.ttm;
 596         enum dma_data_direction dir;
 597         dma_addr_t dma_addr;
 598         bool mmio;
 599         int ret;
 600
 601         /* Expect SG Table of dmapmap BO to be NULL */
 602         mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
 603         if (unlikely(ttm->sg)) {
 604                 pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
 605                 return -EINVAL;
 606         }
 607
 608         dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
 609                         DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 610         dma_addr = mem->bo->tbo.sg->sgl->dma_address;
 611         pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
 612         pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
 613         dma_addr = dma_map_resource(adev->dev, dma_addr,
 614                         mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
 615         ret = dma_mapping_error(adev->dev, dma_addr);
 616         if (unlikely(ret))
 617                 return ret;
 618         pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
 619
 620         ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
 621         if (unlikely(!ttm->sg)) {
 622                 ret = -ENOMEM;
 623                 goto unmap_sg;
 624         }
 625
 626         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
 627         ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 628         if (unlikely(ret))
 629                 goto free_sg;
 630
 631         return ret;
 632
 633 free_sg:
 634         sg_free_table(ttm->sg);
 635         kfree(ttm->sg);
 636         ttm->sg = NULL;
 637 unmap_sg:
 638         dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
 639                            dir, DMA_ATTR_SKIP_CPU_SYNC);
 640         return ret;
 641 }
 642
 643 static int
 644 kfd_mem_dmamap_attachment(struct kgd_mem *mem,
 645                           struct kfd_mem_attachment *attachment)
 646 {
 647         switch (attachment->type) {
 648         case KFD_MEM_ATT_SHARED:
 649                 return 0;
 650         case KFD_MEM_ATT_USERPTR:
 651                 return kfd_mem_dmamap_userptr(mem, attachment);
 652         case KFD_MEM_ATT_DMABUF:
 653                 return kfd_mem_dmamap_dmabuf(attachment);
 654         case KFD_MEM_ATT_SG:
 655                 return kfd_mem_dmamap_sg_bo(mem, attachment);
 656         default:
 657                 WARN_ON_ONCE(1);
 658         }
 659         return -EINVAL;
 660 }
 661
 662 static void
 663 kfd_mem_dmaunmap_userptr(struct kgd_mem *mem,
 664                          struct kfd_mem_attachment *attachment)
 665 {
 666         enum dma_data_direction direction =
 667                 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
 668                 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 669         struct ttm_operation_ctx ctx = {.interruptible = false};
 670         struct amdgpu_bo *bo = attachment->bo_va->base.bo;
 671         struct amdgpu_device *adev = attachment->adev;
 672         struct ttm_tt *ttm = bo->tbo.ttm;
 673
 674         if (unlikely(!ttm->sg))
 675                 return;
 676
 677         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
 678         ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 679
 680         dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0);
 681         sg_free_table(ttm->sg);
 682         kfree(ttm->sg);
 683         ttm->sg = NULL;
 684 }
 685
 686 static void
 687 kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
 688 {
 689         struct ttm_operation_ctx ctx = {.interruptible = true};
 690         struct amdgpu_bo *bo = attachment->bo_va->base.bo;
 691
 692         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
 693         ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 694 }
 695
 696 /**
 697  * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
 698  * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
 699  * @attachment: Virtual address attachment of the BO on accessing device
 700  *
 701  * The method performs following steps:
 702  *   - Signal TTM to mark memory pointed to by BO as GPU inaccessible
 703  *   - Free SG Table that is used to encapsulate DMA mapped memory of
 704  *          peer device's DOORBELL or MMIO memory
 705  *
 706  * This method is invoked in the following contexts:
 707  *     UNMapping of DOORBELL or MMIO BO on a device having access to its memory
 708  *     Eviction of DOOREBELL or MMIO BO on device having access to its memory
 709  *
 710  * Return: void
 711  */
 712 static void
 713 kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
 714                        struct kfd_mem_attachment *attachment)
 715 {
 716         struct ttm_operation_ctx ctx = {.interruptible = true};
 717         struct amdgpu_bo *bo = attachment->bo_va->base.bo;
 718         struct amdgpu_device *adev = attachment->adev;
 719         struct ttm_tt *ttm = bo->tbo.ttm;
 720         enum dma_data_direction dir;
 721
 722         if (unlikely(!ttm->sg)) {
 723                 pr_err("SG Table of BO is UNEXPECTEDLY NULL");
 724                 return;
 725         }
 726
 727         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
 728         ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 729
 730         dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
 731                                 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 732         dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,
 733                         ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
 734         sg_free_table(ttm->sg);
 735         kfree(ttm->sg);
 736         ttm->sg = NULL;
 737         bo->tbo.sg = NULL;
 738 }
 739
 740 static void
 741 kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
 742                             struct kfd_mem_attachment *attachment)
 743 {
 744         switch (attachment->type) {
 745         case KFD_MEM_ATT_SHARED:
 746                 break;
 747         case KFD_MEM_ATT_USERPTR:
 748                 kfd_mem_dmaunmap_userptr(mem, attachment);
 749                 break;
 750         case KFD_MEM_ATT_DMABUF:
 751                 kfd_mem_dmaunmap_dmabuf(attachment);
 752                 break;
 753         case KFD_MEM_ATT_SG:
 754                 kfd_mem_dmaunmap_sg_bo(mem, attachment);
 755                 break;
 756         default:
 757                 WARN_ON_ONCE(1);
 758         }
 759 }
 760
 761 static int
 762 kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
 763                       struct amdgpu_bo **bo)
 764 {
 765         struct drm_gem_object *gobj;
 766         int ret;
 767
 768         if (!mem->dmabuf) {
 769                 mem->dmabuf = amdgpu_gem_prime_export(&mem->bo->tbo.base,
 770                         mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
 771                                 DRM_RDWR : 0);
 772                 if (IS_ERR(mem->dmabuf)) {
 773                         ret = PTR_ERR(mem->dmabuf);
 774                         mem->dmabuf = NULL;
 775                         return ret;
 776                 }
 777         }
 778
 779         gobj = amdgpu_gem_prime_import(adev_to_drm(adev), mem->dmabuf);
 780         if (IS_ERR(gobj))
 781                 return PTR_ERR(gobj);
 782
 783         *bo = gem_to_amdgpu_bo(gobj);
 784         (*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE;
 785
 786         return 0;
 787 }
 788
 789 /* kfd_mem_attach - Add a BO to a VM
 790  *
 791  * Everything that needs to bo done only once when a BO is first added
 792  * to a VM. It can later be mapped and unmapped many times without
 793  * repeating these steps.
 794  *
 795  * 0. Create BO for DMA mapping, if needed
 796  * 1. Allocate and initialize BO VA entry data structure
 797  * 2. Add BO to the VM
 798  * 3. Determine ASIC-specific PTE flags
 799  * 4. Alloc page tables and directories if needed
 800  * 4a.  Validate new page tables and directories
 801  */
 802 static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
 803                 struct amdgpu_vm *vm, bool is_aql)
 804 {
 805         struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);
 806         unsigned long bo_size = mem->bo->tbo.base.size;
 807         uint64_t va = mem->va;
 808         struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
 809         struct amdgpu_bo *bo[2] = {NULL, NULL};
 810         bool same_hive = false;
 811         int i, ret;
 812
 813         if (!va) {
 814                 pr_err("Invalid VA when adding BO to VM\n");
 815                 return -EINVAL;
 816         }
 817
 818         /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices
 819          *
 820          * The access path of MMIO and DOORBELL BOs of is always over PCIe.
 821          * In contrast the access path of VRAM BOs depens upon the type of
 822          * link that connects the peer device. Access over PCIe is allowed
 823          * if peer device has large BAR. In contrast, access over xGMI is
 824          * allowed for both small and large BAR configurations of peer device
 825          */
 826         if ((adev != bo_adev) &&
 827             ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
 828              (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
 829              (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
 830                 if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)
 831                         same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
 832                 if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev))
 833                         return -EINVAL;
 834         }
 835
 836         for (i = 0; i <= is_aql; i++) {
 837                 attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
 838                 if (unlikely(!attachment[i])) {
 839                         ret = -ENOMEM;
 840                         goto unwind;
 841                 }
 842
 843                 pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
 844                          va + bo_size, vm);
 845
 846                 if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
 847                     (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
 848                     same_hive) {
 849                         /* Mappings on the local GPU, or VRAM mappings in the
 850                          * local hive, or userptr mapping IOMMU direct map mode
 851                          * share the original BO
 852                          */
 853                         attachment[i]->type = KFD_MEM_ATT_SHARED;
 854                         bo[i] = mem->bo;
 855                         drm_gem_object_get(&bo[i]->tbo.base);
 856                 } else if (i > 0) {
 857                         /* Multiple mappings on the same GPU share the BO */
 858                         attachment[i]->type = KFD_MEM_ATT_SHARED;
 859                         bo[i] = bo[0];
 860                         drm_gem_object_get(&bo[i]->tbo.base);
 861                 } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
 862                         /* Create an SG BO to DMA-map userptrs on other GPUs */
 863                         attachment[i]->type = KFD_MEM_ATT_USERPTR;
 864                         ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
 865                         if (ret)
 866                                 goto unwind;
 867                 /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
 868                 } else if (mem->bo->tbo.type == ttm_bo_type_sg) {
 869                         WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL ||
 870                                     mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP),
 871                                   "Handing invalid SG BO in ATTACH request");
 872                         attachment[i]->type = KFD_MEM_ATT_SG;
 873                         ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
 874                         if (ret)
 875                                 goto unwind;
 876                 /* Enable acces to GTT and VRAM BOs of peer devices */
 877                 } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT ||
 878                            mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
 879                         attachment[i]->type = KFD_MEM_ATT_DMABUF;
 880                         ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
 881                         if (ret)
 882                                 goto unwind;
 883                         pr_debug("Employ DMABUF mechanism to enable peer GPU access\n");
 884                 } else {
 885                         WARN_ONCE(true, "Handling invalid ATTACH request");
 886                         ret = -EINVAL;
 887                         goto unwind;
 888                 }
 889
 890                 /* Add BO to VM internal data structures */
 891                 ret = amdgpu_bo_reserve(bo[i], false);
 892                 if (ret) {
 893                         pr_debug("Unable to reserve BO during memory attach");
 894                         goto unwind;
 895                 }
 896                 attachment[i]->bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]);
 897                 amdgpu_bo_unreserve(bo[i]);
 898                 if (unlikely(!attachment[i]->bo_va)) {
 899                         ret = -ENOMEM;
 900                         pr_err("Failed to add BO object to VM. ret == %d\n",
 901                                ret);
 902                         goto unwind;
 903                 }
 904                 attachment[i]->va = va;
 905                 attachment[i]->pte_flags = get_pte_flags(adev, mem);
 906                 attachment[i]->adev = adev;
 907                 list_add(&attachment[i]->list, &mem->attachments);
 908
 909                 va += bo_size;
 910         }
 911
 912         return 0;
 913
 914 unwind:
 915         for (; i >= 0; i--) {
 916                 if (!attachment[i])
 917                         continue;
 918                 if (attachment[i]->bo_va) {
 919                         amdgpu_bo_reserve(bo[i], true);
 920                         amdgpu_vm_bo_del(adev, attachment[i]->bo_va);
 921                         amdgpu_bo_unreserve(bo[i]);
 922                         list_del(&attachment[i]->list);
 923                 }
 924                 if (bo[i])
 925                         drm_gem_object_put(&bo[i]->tbo.base);
 926                 kfree(attachment[i]);
 927         }
 928         return ret;
 929 }
 930
 931 static void kfd_mem_detach(struct kfd_mem_attachment *attachment)
 932 {
 933         struct amdgpu_bo *bo = attachment->bo_va->base.bo;
 934
 935         pr_debug("\t remove VA 0x%llx in entry %p\n",
 936                         attachment->va, attachment);
 937         amdgpu_vm_bo_del(attachment->adev, attachment->bo_va);
 938         drm_gem_object_put(&bo->tbo.base);
 939         list_del(&attachment->list);
 940         kfree(attachment);
 941 }
 942
 943 static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
 944                                 struct amdkfd_process_info *process_info,
 945                                 bool userptr)
 946 {
 947         struct ttm_validate_buffer *entry = &mem->validate_list;
 948         struct amdgpu_bo *bo = mem->bo;
 949
 950         INIT_LIST_HEAD(&entry->head);
 951         entry->num_shared = 1;
 952         entry->bo = &bo->tbo;
 953         mutex_lock(&process_info->lock);
 954         if (userptr)
 955                 list_add_tail(&entry->head, &process_info->userptr_valid_list);
 956         else
 957                 list_add_tail(&entry->head, &process_info->kfd_bo_list);
 958         mutex_unlock(&process_info->lock);
 959 }
 960
 961 static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem,
 962                 struct amdkfd_process_info *process_info)
 963 {
 964         struct ttm_validate_buffer *bo_list_entry;
 965
 966         bo_list_entry = &mem->validate_list;
 967         mutex_lock(&process_info->lock);
 968         list_del(&bo_list_entry->head);
 969         mutex_unlock(&process_info->lock);
 970 }
 971
 972 /* Initializes user pages. It registers the MMU notifier and validates
 973  * the userptr BO in the GTT domain.
 974  *
 975  * The BO must already be on the userptr_valid_list. Otherwise an
 976  * eviction and restore may happen that leaves the new BO unmapped
 977  * with the user mode queues running.
 978  *
 979  * Takes the process_info->lock to protect against concurrent restore
 980  * workers.
 981  *
 982  * Returns 0 for success, negative errno for errors.
 983  */
 984 static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr,
 985                            bool criu_resume)
 986 {
 987         struct amdkfd_process_info *process_info = mem->process_info;
 988         struct amdgpu_bo *bo = mem->bo;
 989         struct ttm_operation_ctx ctx = { true, false };
 990         int ret = 0;
 991
 992         mutex_lock(&process_info->lock);
 993
 994         ret = amdgpu_ttm_tt_set_userptr(&bo->tbo, user_addr, 0);
 995         if (ret) {
 996                 pr_err("%s: Failed to set userptr: %d\n", __func__, ret);
 997                 goto out;
 998         }
 999
1000         ret = amdgpu_mn_register(bo, user_addr);
1001         if (ret) {
1002                 pr_err("%s: Failed to register MMU notifier: %d\n",
1003                        __func__, ret);
1004                 goto out;
1005         }
1006
1007         if (criu_resume) {
1008                 /*
1009                  * During a CRIU restore operation, the userptr buffer objects
1010                  * will be validated in the restore_userptr_work worker at a
1011                  * later stage when it is scheduled by another ioctl called by
1012                  * CRIU master process for the target pid for restore.
1013                  */
1014                 atomic_inc(&mem->invalid);
1015                 mutex_unlock(&process_info->lock);
1016                 return 0;
1017         }
1018
1019         ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages);
1020         if (ret) {
1021                 pr_err("%s: Failed to get user pages: %d\n", __func__, ret);
1022                 goto unregister_out;
1023         }
1024
1025         ret = amdgpu_bo_reserve(bo, true);
1026         if (ret) {
1027                 pr_err("%s: Failed to reserve BO\n", __func__);
1028                 goto release_out;
1029         }
1030         amdgpu_bo_placement_from_domain(bo, mem->domain);
1031         ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1032         if (ret)
1033                 pr_err("%s: failed to validate BO\n", __func__);
1034         amdgpu_bo_unreserve(bo);
1035
1036 release_out:
1037         amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
1038 unregister_out:
1039         if (ret)
1040                 amdgpu_mn_unregister(bo);
1041 out:
1042         mutex_unlock(&process_info->lock);
1043         return ret;
1044 }
1045
1046 /* Reserving a BO and its page table BOs must happen atomically to
1047  * avoid deadlocks. Some operations update multiple VMs at once. Track
1048  * all the reservation info in a context structure. Optionally a sync
1049  * object can track VM updates.
1050  */
1051 struct bo_vm_reservation_context {
1052         struct amdgpu_bo_list_entry kfd_bo; /* BO list entry for the KFD BO */
1053         unsigned int n_vms;                 /* Number of VMs reserved       */
1054         struct amdgpu_bo_list_entry *vm_pd; /* Array of VM BO list entries  */
1055         struct ww_acquire_ctx ticket;       /* Reservation ticket           */
1056         struct list_head list, duplicates;  /* BO lists                     */
1057         struct amdgpu_sync *sync;           /* Pointer to sync object       */
1058         bool reserved;                      /* Whether BOs are reserved     */
1059 };
1060
1061 enum bo_vm_match {
1062         BO_VM_NOT_MAPPED = 0,   /* Match VMs where a BO is not mapped */
1063         BO_VM_MAPPED,           /* Match VMs where a BO is mapped     */
1064         BO_VM_ALL,              /* Match all VMs a BO was added to    */
1065 };
1066
1067 /**
1068  * reserve_bo_and_vm - reserve a BO and a VM unconditionally.
1069  * @mem: KFD BO structure.
1070  * @vm: the VM to reserve.
1071  * @ctx: the struct that will be used in unreserve_bo_and_vms().
1072  */
1073 static int reserve_bo_and_vm(struct kgd_mem *mem,
1074                               struct amdgpu_vm *vm,
1075                               struct bo_vm_reservation_context *ctx)
1076 {
1077         struct amdgpu_bo *bo = mem->bo;
1078         int ret;
1079
1080         WARN_ON(!vm);
1081
1082         ctx->reserved = false;
1083         ctx->n_vms = 1;
1084         ctx->sync = &mem->sync;
1085
1086         INIT_LIST_HEAD(&ctx->list);
1087         INIT_LIST_HEAD(&ctx->duplicates);
1088
1089         ctx->vm_pd = kcalloc(ctx->n_vms, sizeof(*ctx->vm_pd), GFP_KERNEL);
1090         if (!ctx->vm_pd)
1091                 return -ENOMEM;
1092
1093         ctx->kfd_bo.priority = 0;
1094         ctx->kfd_bo.tv.bo = &bo->tbo;
1095         ctx->kfd_bo.tv.num_shared = 1;
1096         list_add(&ctx->kfd_bo.tv.head, &ctx->list);
1097
1098         amdgpu_vm_get_pd_bo(vm, &ctx->list, &ctx->vm_pd[0]);
1099
1100         ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list,
1101                                      false, &ctx->duplicates);
1102         if (ret) {
1103                 pr_err("Failed to reserve buffers in ttm.\n");
1104                 kfree(ctx->vm_pd);
1105                 ctx->vm_pd = NULL;
1106                 return ret;
1107         }
1108
1109         ctx->reserved = true;
1110         return 0;
1111 }
1112
1113 /**
1114  * reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally
1115  * @mem: KFD BO structure.
1116  * @vm: the VM to reserve. If NULL, then all VMs associated with the BO
1117  * is used. Otherwise, a single VM associated with the BO.
1118  * @map_type: the mapping status that will be used to filter the VMs.
1119  * @ctx: the struct that will be used in unreserve_bo_and_vms().
1120  *
1121  * Returns 0 for success, negative for failure.
1122  */
1123 static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
1124                                 struct amdgpu_vm *vm, enum bo_vm_match map_type,
1125                                 struct bo_vm_reservation_context *ctx)
1126 {
1127         struct amdgpu_bo *bo = mem->bo;
1128         struct kfd_mem_attachment *entry;
1129         unsigned int i;
1130         int ret;
1131
1132         ctx->reserved = false;
1133         ctx->n_vms = 0;
1134         ctx->vm_pd = NULL;
1135         ctx->sync = &mem->sync;
1136
1137         INIT_LIST_HEAD(&ctx->list);
1138         INIT_LIST_HEAD(&ctx->duplicates);
1139
1140         list_for_each_entry(entry, &mem->attachments, list) {
1141                 if ((vm && vm != entry->bo_va->base.vm) ||
1142                         (entry->is_mapped != map_type
1143                         && map_type != BO_VM_ALL))
1144                         continue;
1145
1146                 ctx->n_vms++;
1147         }
1148
1149         if (ctx->n_vms != 0) {
1150                 ctx->vm_pd = kcalloc(ctx->n_vms, sizeof(*ctx->vm_pd),
1151                                      GFP_KERNEL);
1152                 if (!ctx->vm_pd)
1153                         return -ENOMEM;
1154         }
1155
1156         ctx->kfd_bo.priority = 0;
1157         ctx->kfd_bo.tv.bo = &bo->tbo;
1158         ctx->kfd_bo.tv.num_shared = 1;
1159         list_add(&ctx->kfd_bo.tv.head, &ctx->list);
1160
1161         i = 0;
1162         list_for_each_entry(entry, &mem->attachments, list) {
1163                 if ((vm && vm != entry->bo_va->base.vm) ||
1164                         (entry->is_mapped != map_type
1165                         && map_type != BO_VM_ALL))
1166                         continue;
1167
1168                 amdgpu_vm_get_pd_bo(entry->bo_va->base.vm, &ctx->list,
1169                                 &ctx->vm_pd[i]);
1170                 i++;
1171         }
1172
1173         ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list,
1174                                      false, &ctx->duplicates);
1175         if (ret) {
1176                 pr_err("Failed to reserve buffers in ttm.\n");
1177                 kfree(ctx->vm_pd);
1178                 ctx->vm_pd = NULL;
1179                 return ret;
1180         }
1181
1182         ctx->reserved = true;
1183         return 0;
1184 }
1185
1186 /**
1187  * unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context
1188  * @ctx: Reservation context to unreserve
1189  * @wait: Optionally wait for a sync object representing pending VM updates
1190  * @intr: Whether the wait is interruptible
1191  *
1192  * Also frees any resources allocated in
1193  * reserve_bo_and_(cond_)vm(s). Returns the status from
1194  * amdgpu_sync_wait.
1195  */
1196 static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx,
1197                                  bool wait, bool intr)
1198 {
1199         int ret = 0;
1200
1201         if (wait)
1202                 ret = amdgpu_sync_wait(ctx->sync, intr);
1203
1204         if (ctx->reserved)
1205                 ttm_eu_backoff_reservation(&ctx->ticket, &ctx->list);
1206         kfree(ctx->vm_pd);
1207
1208         ctx->sync = NULL;
1209
1210         ctx->reserved = false;
1211         ctx->vm_pd = NULL;
1212
1213         return ret;
1214 }
1215
1216 static void unmap_bo_from_gpuvm(struct kgd_mem *mem,
1217                                 struct kfd_mem_attachment *entry,
1218                                 struct amdgpu_sync *sync)
1219 {
1220         struct amdgpu_bo_va *bo_va = entry->bo_va;
1221         struct amdgpu_device *adev = entry->adev;
1222         struct amdgpu_vm *vm = bo_va->base.vm;
1223
1224         amdgpu_vm_bo_unmap(adev, bo_va, entry->va);
1225
1226         amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update);
1227
1228         amdgpu_sync_fence(sync, bo_va->last_pt_update);
1229
1230         kfd_mem_dmaunmap_attachment(mem, entry);
1231 }
1232
1233 static int update_gpuvm_pte(struct kgd_mem *mem,
1234                             struct kfd_mem_attachment *entry,
1235                             struct amdgpu_sync *sync)
1236 {
1237         struct amdgpu_bo_va *bo_va = entry->bo_va;
1238         struct amdgpu_device *adev = entry->adev;
1239         int ret;
1240
1241         ret = kfd_mem_dmamap_attachment(mem, entry);
1242         if (ret)
1243                 return ret;
1244
1245         /* Update the page tables  */
1246         ret = amdgpu_vm_bo_update(adev, bo_va, false);
1247         if (ret) {
1248                 pr_err("amdgpu_vm_bo_update failed\n");
1249                 return ret;
1250         }
1251
1252         return amdgpu_sync_fence(sync, bo_va->last_pt_update);
1253 }
1254
1255 static int map_bo_to_gpuvm(struct kgd_mem *mem,
1256                            struct kfd_mem_attachment *entry,
1257                            struct amdgpu_sync *sync,
1258                            bool no_update_pte)
1259 {
1260         int ret;
1261
1262         /* Set virtual address for the allocation */
1263         ret = amdgpu_vm_bo_map(entry->adev, entry->bo_va, entry->va, 0,
1264                                amdgpu_bo_size(entry->bo_va->base.bo),
1265                                entry->pte_flags);
1266         if (ret) {
1267                 pr_err("Failed to map VA 0x%llx in vm. ret %d\n",
1268                                 entry->va, ret);
1269                 return ret;
1270         }
1271
1272         if (no_update_pte)
1273                 return 0;
1274
1275         ret = update_gpuvm_pte(mem, entry, sync);
1276         if (ret) {
1277                 pr_err("update_gpuvm_pte() failed\n");
1278                 goto update_gpuvm_pte_failed;
1279         }
1280
1281         return 0;
1282
1283 update_gpuvm_pte_failed:
1284         unmap_bo_from_gpuvm(mem, entry, sync);
1285         return ret;
1286 }
1287
1288 static int process_validate_vms(struct amdkfd_process_info *process_info)
1289 {
1290         struct amdgpu_vm *peer_vm;
1291         int ret;
1292
1293         list_for_each_entry(peer_vm, &process_info->vm_list_head,
1294                             vm_list_node) {
1295                 ret = vm_validate_pt_pd_bos(peer_vm);
1296                 if (ret)
1297                         return ret;
1298         }
1299
1300         return 0;
1301 }
1302
1303 static int process_sync_pds_resv(struct amdkfd_process_info *process_info,
1304                                  struct amdgpu_sync *sync)
1305 {
1306         struct amdgpu_vm *peer_vm;
1307         int ret;
1308
1309         list_for_each_entry(peer_vm, &process_info->vm_list_head,
1310                             vm_list_node) {
1311                 struct amdgpu_bo *pd = peer_vm->root.bo;
1312
1313                 ret = amdgpu_sync_resv(NULL, sync, pd->tbo.base.resv,
1314                                        AMDGPU_SYNC_NE_OWNER,
1315                                        AMDGPU_FENCE_OWNER_KFD);
1316                 if (ret)
1317                         return ret;
1318         }
1319
1320         return 0;
1321 }
1322
1323 static int process_update_pds(struct amdkfd_process_info *process_info,
1324                               struct amdgpu_sync *sync)
1325 {
1326         struct amdgpu_vm *peer_vm;
1327         int ret;
1328
1329         list_for_each_entry(peer_vm, &process_info->vm_list_head,
1330                             vm_list_node) {
1331                 ret = vm_update_pds(peer_vm, sync);
1332                 if (ret)
1333                         return ret;
1334         }
1335
1336         return 0;
1337 }
1338
1339 static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
1340                        struct dma_fence **ef)
1341 {
1342         struct amdkfd_process_info *info = NULL;
1343         int ret;
1344
1345         if (!*process_info) {
1346                 info = kzalloc(sizeof(*info), GFP_KERNEL);
1347                 if (!info)
1348                         return -ENOMEM;
1349
1350                 mutex_init(&info->lock);
1351                 INIT_LIST_HEAD(&info->vm_list_head);
1352                 INIT_LIST_HEAD(&info->kfd_bo_list);
1353                 INIT_LIST_HEAD(&info->userptr_valid_list);
1354                 INIT_LIST_HEAD(&info->userptr_inval_list);
1355
1356                 info->eviction_fence =
1357                         amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
1358                                                    current->mm,
1359                                                    NULL);
1360                 if (!info->eviction_fence) {
1361                         pr_err("Failed to create eviction fence\n");
1362                         ret = -ENOMEM;
1363                         goto create_evict_fence_fail;
1364                 }
1365
1366                 info->pid = get_task_pid(current->group_leader, PIDTYPE_PID);
1367                 atomic_set(&info->evicted_bos, 0);
1368                 INIT_DELAYED_WORK(&info->restore_userptr_work,
1369                                   amdgpu_amdkfd_restore_userptr_worker);
1370
1371                 *process_info = info;
1372                 *ef = dma_fence_get(&info->eviction_fence->base);
1373         }
1374
1375         vm->process_info = *process_info;
1376
1377         /* Validate page directory and attach eviction fence */
1378         ret = amdgpu_bo_reserve(vm->root.bo, true);
1379         if (ret)
1380                 goto reserve_pd_fail;
1381         ret = vm_validate_pt_pd_bos(vm);
1382         if (ret) {
1383                 pr_err("validate_pt_pd_bos() failed\n");
1384                 goto validate_pd_fail;
1385         }
1386         ret = amdgpu_bo_sync_wait(vm->root.bo,
1387                                   AMDGPU_FENCE_OWNER_KFD, false);
1388         if (ret)
1389                 goto wait_pd_fail;
1390         ret = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1);
1391         if (ret)
1392                 goto reserve_shared_fail;
1393         amdgpu_bo_fence(vm->root.bo,
1394                         &vm->process_info->eviction_fence->base, true);
1395         amdgpu_bo_unreserve(vm->root.bo);
1396
1397         /* Update process info */
1398         mutex_lock(&vm->process_info->lock);
1399         list_add_tail(&vm->vm_list_node,
1400                         &(vm->process_info->vm_list_head));
1401         vm->process_info->n_vms++;
1402         mutex_unlock(&vm->process_info->lock);
1403
1404         return 0;
1405
1406 reserve_shared_fail:
1407 wait_pd_fail:
1408 validate_pd_fail:
1409         amdgpu_bo_unreserve(vm->root.bo);
1410 reserve_pd_fail:
1411         vm->process_info = NULL;
1412         if (info) {
1413                 /* Two fence references: one in info and one in *ef */
1414                 dma_fence_put(&info->eviction_fence->base);
1415                 dma_fence_put(*ef);
1416                 *ef = NULL;
1417                 *process_info = NULL;
1418                 put_pid(info->pid);
1419 create_evict_fence_fail:
1420                 mutex_destroy(&info->lock);
1421                 kfree(info);
1422         }
1423         return ret;
1424 }
1425
1426 /**
1427  * amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria
1428  * @bo: Handle of buffer object being pinned
1429  * @domain: Domain into which BO should be pinned
1430  *
1431  *   - USERPTR BOs are UNPINNABLE and will return error
1432  *   - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
1433  *     PIN count incremented. It is valid to PIN a BO multiple times
1434  *
1435  * Return: ZERO if successful in pinning, Non-Zero in case of error.
1436  */
1437 static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)
1438 {
1439         int ret = 0;
1440
1441         ret = amdgpu_bo_reserve(bo, false);
1442         if (unlikely(ret))
1443                 return ret;
1444
1445         ret = amdgpu_bo_pin_restricted(bo, domain, 0, 0);
1446         if (ret)
1447                 pr_err("Error in Pinning BO to domain: %d\n", domain);
1448
1449         amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
1450         amdgpu_bo_unreserve(bo);
1451
1452         return ret;
1453 }
1454
1455 /**
1456  * amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria
1457  * @bo: Handle of buffer object being unpinned
1458  *
1459  *   - Is a illegal request for USERPTR BOs and is ignored
1460  *   - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
1461  *     PIN count decremented. Calls to UNPIN must balance calls to PIN
1462  */
1463 static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
1464 {
1465         int ret = 0;
1466
1467         ret = amdgpu_bo_reserve(bo, false);
1468         if (unlikely(ret))
1469                 return;
1470
1471         amdgpu_bo_unpin(bo);
1472         amdgpu_bo_unreserve(bo);
1473 }
1474
1475 int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
1476                                            struct file *filp, u32 pasid,
1477                                            void **process_info,
1478                                            struct dma_fence **ef)
1479 {
1480         struct amdgpu_fpriv *drv_priv;
1481         struct amdgpu_vm *avm;
1482         int ret;
1483
1484         ret = amdgpu_file_to_fpriv(filp, &drv_priv);
1485         if (ret)
1486                 return ret;
1487         avm = &drv_priv->vm;
1488
1489         /* Already a compute VM? */
1490         if (avm->process_info)
1491                 return -EINVAL;
1492
1493         /* Free the original amdgpu allocated pasid,
1494          * will be replaced with kfd allocated pasid.
1495          */
1496         if (avm->pasid) {
1497                 amdgpu_pasid_free(avm->pasid);
1498                 amdgpu_vm_set_pasid(adev, avm, 0);
1499         }
1500
1501         /* Convert VM into a compute VM */
1502         ret = amdgpu_vm_make_compute(adev, avm);
1503         if (ret)
1504                 return ret;
1505
1506         ret = amdgpu_vm_set_pasid(adev, avm, pasid);
1507         if (ret)
1508                 return ret;
1509         /* Initialize KFD part of the VM and process info */
1510         ret = init_kfd_vm(avm, process_info, ef);
1511         if (ret)
1512                 return ret;
1513
1514         amdgpu_vm_set_task_info(avm);
1515
1516         return 0;
1517 }
1518
1519 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
1520                                     struct amdgpu_vm *vm)
1521 {
1522         struct amdkfd_process_info *process_info = vm->process_info;
1523
1524         if (!process_info)
1525                 return;
1526
1527         /* Update process info */
1528         mutex_lock(&process_info->lock);
1529         process_info->n_vms--;
1530         list_del(&vm->vm_list_node);
1531         mutex_unlock(&process_info->lock);
1532
1533         vm->process_info = NULL;
1534
1535         /* Release per-process resources when last compute VM is destroyed */
1536         if (!process_info->n_vms) {
1537                 WARN_ON(!list_empty(&process_info->kfd_bo_list));
1538                 WARN_ON(!list_empty(&process_info->userptr_valid_list));
1539                 WARN_ON(!list_empty(&process_info->userptr_inval_list));
1540
1541                 dma_fence_put(&process_info->eviction_fence->base);
1542                 cancel_delayed_work_sync(&process_info->restore_userptr_work);
1543                 put_pid(process_info->pid);
1544                 mutex_destroy(&process_info->lock);
1545                 kfree(process_info);
1546         }
1547 }
1548
1549 void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
1550                                             void *drm_priv)
1551 {
1552         struct amdgpu_vm *avm;
1553
1554         if (WARN_ON(!adev || !drm_priv))
1555                 return;
1556
1557         avm = drm_priv_to_vm(drm_priv);
1558
1559         pr_debug("Releasing process vm %p\n", avm);
1560
1561         /* The original pasid of amdgpu vm has already been
1562          * released during making a amdgpu vm to a compute vm
1563          * The current pasid is managed by kfd and will be
1564          * released on kfd process destroy. Set amdgpu pasid
1565          * to 0 to avoid duplicate release.
1566          */
1567         amdgpu_vm_release_compute(adev, avm);
1568 }
1569
1570 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv)
1571 {
1572         struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1573         struct amdgpu_bo *pd = avm->root.bo;
1574         struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
1575
1576         if (adev->asic_type < CHIP_VEGA10)
1577                 return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT;
1578         return avm->pd_phys_addr;
1579 }
1580
1581 void amdgpu_amdkfd_block_mmu_notifications(void *p)
1582 {
1583         struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p;
1584
1585         mutex_lock(&pinfo->lock);
1586         WRITE_ONCE(pinfo->block_mmu_notifications, true);
1587         mutex_unlock(&pinfo->lock);
1588 }
1589
1590 int amdgpu_amdkfd_criu_resume(void *p)
1591 {
1592         int ret = 0;
1593         struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p;
1594
1595         mutex_lock(&pinfo->lock);
1596         pr_debug("scheduling work\n");
1597         atomic_inc(&pinfo->evicted_bos);
1598         if (!READ_ONCE(pinfo->block_mmu_notifications)) {
1599                 ret = -EINVAL;
1600                 goto out_unlock;
1601         }
1602         WRITE_ONCE(pinfo->block_mmu_notifications, false);
1603         schedule_delayed_work(&pinfo->restore_userptr_work, 0);
1604
1605 out_unlock:
1606         mutex_unlock(&pinfo->lock);
1607         return ret;
1608 }
1609
1610 size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
1611 {
1612         uint64_t reserved_for_pt =
1613                 ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
1614         size_t available;
1615         spin_lock(&kfd_mem_limit.mem_limit_lock);
1616         available = adev->gmc.real_vram_size
1617                 - adev->kfd.vram_used_aligned
1618                 - atomic64_read(&adev->vram_pin_size)
1619                 - reserved_for_pt;
1620         spin_unlock(&kfd_mem_limit.mem_limit_lock);
1621
1622         return ALIGN_DOWN(available, VRAM_AVAILABLITY_ALIGN);
1623 }
1624
1625 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1626                 struct amdgpu_device *adev, uint64_t va, uint64_t size,
1627                 void *drm_priv, struct kgd_mem **mem,
1628                 uint64_t *offset, uint32_t flags, bool criu_resume)
1629 {
1630         struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1631         enum ttm_bo_type bo_type = ttm_bo_type_device;
1632         struct sg_table *sg = NULL;
1633         uint64_t user_addr = 0;
1634         struct amdgpu_bo *bo;
1635         struct drm_gem_object *gobj = NULL;
1636         u32 domain, alloc_domain;
1637         u64 alloc_flags;
1638         int ret;
1639
1640         /*
1641          * Check on which domain to allocate BO
1642          */
1643         if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
1644                 domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM;
1645                 alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
1646                 alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ?
1647                         AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0;
1648         } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
1649                 domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
1650                 alloc_flags = 0;
1651         } else {
1652                 domain = AMDGPU_GEM_DOMAIN_GTT;
1653                 alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
1654                 alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE;
1655
1656                 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
1657                         if (!offset || !*offset)
1658                                 return -EINVAL;
1659                         user_addr = untagged_addr(*offset);
1660                 } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1661                                     KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1662                         bo_type = ttm_bo_type_sg;
1663                         if (size > UINT_MAX)
1664                                 return -EINVAL;
1665                         sg = create_sg_table(*offset, size);
1666                         if (!sg)
1667                                 return -ENOMEM;
1668                 } else {
1669                         return -EINVAL;
1670                 }
1671         }
1672
1673         *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
1674         if (!*mem) {
1675                 ret = -ENOMEM;
1676                 goto err;
1677         }
1678         INIT_LIST_HEAD(&(*mem)->attachments);
1679         mutex_init(&(*mem)->lock);
1680         (*mem)->aql_queue = !!(flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM);
1681
1682         /* Workaround for AQL queue wraparound bug. Map the same
1683          * memory twice. That means we only actually allocate half
1684          * the memory.
1685          */
1686         if ((*mem)->aql_queue)
1687                 size = size >> 1;
1688
1689         (*mem)->alloc_flags = flags;
1690
1691         amdgpu_sync_create(&(*mem)->sync);
1692
1693         ret = amdgpu_amdkfd_reserve_mem_limit(adev, size, flags);
1694         if (ret) {
1695                 pr_debug("Insufficient memory\n");
1696                 goto err_reserve_limit;
1697         }
1698
1699         pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s\n",
1700                         va, size, domain_string(alloc_domain));
1701
1702         ret = amdgpu_gem_object_create(adev, size, 1, alloc_domain, alloc_flags,
1703                                        bo_type, NULL, &gobj);
1704         if (ret) {
1705                 pr_debug("Failed to create BO on domain %s. ret %d\n",
1706                          domain_string(alloc_domain), ret);
1707                 goto err_bo_create;
1708         }
1709         ret = drm_vma_node_allow(&gobj->vma_node, drm_priv);
1710         if (ret) {
1711                 pr_debug("Failed to allow vma node access. ret %d\n", ret);
1712                 goto err_node_allow;
1713         }
1714         bo = gem_to_amdgpu_bo(gobj);
1715         if (bo_type == ttm_bo_type_sg) {
1716                 bo->tbo.sg = sg;
1717                 bo->tbo.ttm->sg = sg;
1718         }
1719         bo->kfd_bo = *mem;
1720         (*mem)->bo = bo;
1721         if (user_addr)
1722                 bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO;
1723
1724         (*mem)->va = va;
1725         (*mem)->domain = domain;
1726         (*mem)->mapped_to_gpu_memory = 0;
1727         (*mem)->process_info = avm->process_info;
1728         add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr);
1729
1730         if (user_addr) {
1731                 pr_debug("creating userptr BO for user_addr = %llu\n", user_addr);
1732                 ret = init_user_pages(*mem, user_addr, criu_resume);
1733                 if (ret)
1734                         goto allocate_init_user_pages_failed;
1735         } else  if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1736                                 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1737                 ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT);
1738                 if (ret) {
1739                         pr_err("Pinning MMIO/DOORBELL BO during ALLOC FAILED\n");
1740                         goto err_pin_bo;
1741                 }
1742                 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
1743                 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
1744         }
1745
1746         if (offset)
1747                 *offset = amdgpu_bo_mmap_offset(bo);
1748
1749         return 0;
1750
1751 allocate_init_user_pages_failed:
1752 err_pin_bo:
1753         remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
1754         drm_vma_node_revoke(&gobj->vma_node, drm_priv);
1755 err_node_allow:
1756         /* Don't unreserve system mem limit twice */
1757         goto err_reserve_limit;
1758 err_bo_create:
1759         amdgpu_amdkfd_unreserve_mem_limit(adev, size, flags);
1760 err_reserve_limit:
1761         mutex_destroy(&(*mem)->lock);
1762         if (gobj)
1763                 drm_gem_object_put(gobj);
1764         else
1765                 kfree(*mem);
1766 err:
1767         if (sg) {
1768                 sg_free_table(sg);
1769                 kfree(sg);
1770         }
1771         return ret;
1772 }
1773
1774 int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
1775                 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,
1776                 uint64_t *size)
1777 {
1778         struct amdkfd_process_info *process_info = mem->process_info;
1779         unsigned long bo_size = mem->bo->tbo.base.size;
1780         bool use_release_notifier = (mem->bo->kfd_bo == mem);
1781         struct kfd_mem_attachment *entry, *tmp;
1782         struct bo_vm_reservation_context ctx;
1783         struct ttm_validate_buffer *bo_list_entry;
1784         unsigned int mapped_to_gpu_memory;
1785         int ret;
1786         bool is_imported = false;
1787
1788         mutex_lock(&mem->lock);
1789
1790         /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */
1791         if (mem->alloc_flags &
1792             (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1793              KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1794                 amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);
1795         }
1796
1797         mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
1798         is_imported = mem->is_imported;
1799         mutex_unlock(&mem->lock);
1800         /* lock is not needed after this, since mem is unused and will
1801          * be freed anyway
1802          */
1803
1804         if (mapped_to_gpu_memory > 0) {
1805                 pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n",
1806                                 mem->va, bo_size);
1807                 return -EBUSY;
1808         }
1809
1810         /* Make sure restore workers don't access the BO any more */
1811         bo_list_entry = &mem->validate_list;
1812         mutex_lock(&process_info->lock);
1813         list_del(&bo_list_entry->head);
1814         mutex_unlock(&process_info->lock);
1815
1816         /* No more MMU notifiers */
1817         amdgpu_mn_unregister(mem->bo);
1818
1819         ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx);
1820         if (unlikely(ret))
1821                 return ret;
1822
1823         /* The eviction fence should be removed by the last unmap.
1824          * TODO: Log an error condition if the bo still has the eviction fence
1825          * attached
1826          */
1827         amdgpu_amdkfd_remove_eviction_fence(mem->bo,
1828                                         process_info->eviction_fence);
1829         pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va,
1830                 mem->va + bo_size * (1 + mem->aql_queue));
1831
1832         /* Remove from VM internal data structures */
1833         list_for_each_entry_safe(entry, tmp, &mem->attachments, list)
1834                 kfd_mem_detach(entry);
1835
1836         ret = unreserve_bo_and_vms(&ctx, false, false);
1837
1838         /* Free the sync object */
1839         amdgpu_sync_free(&mem->sync);
1840
1841         /* If the SG is not NULL, it's one we created for a doorbell or mmio
1842          * remap BO. We need to free it.
1843          */
1844         if (mem->bo->tbo.sg) {
1845                 sg_free_table(mem->bo->tbo.sg);
1846                 kfree(mem->bo->tbo.sg);
1847         }
1848
1849         /* Update the size of the BO being freed if it was allocated from
1850          * VRAM and is not imported.
1851          */
1852         if (size) {
1853                 if ((mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM) &&
1854                     (!is_imported))
1855                         *size = bo_size;
1856                 else
1857                         *size = 0;
1858         }
1859
1860         /* Free the BO*/
1861         drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv);
1862         if (mem->dmabuf)
1863                 dma_buf_put(mem->dmabuf);
1864         mutex_destroy(&mem->lock);
1865
1866         /* If this releases the last reference, it will end up calling
1867          * amdgpu_amdkfd_release_notify and kfree the mem struct. That's why
1868          * this needs to be the last call here.
1869          */
1870         drm_gem_object_put(&mem->bo->tbo.base);
1871
1872         /*
1873          * For kgd_mem allocated in amdgpu_amdkfd_gpuvm_import_dmabuf(),
1874          * explicitly free it here.
1875          */
1876         if (!use_release_notifier)
1877                 kfree(mem);
1878
1879         return ret;
1880 }
1881
1882 int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1883                 struct amdgpu_device *adev, struct kgd_mem *mem,
1884                 void *drm_priv)
1885 {
1886         struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1887         int ret;
1888         struct amdgpu_bo *bo;
1889         uint32_t domain;
1890         struct kfd_mem_attachment *entry;
1891         struct bo_vm_reservation_context ctx;
1892         unsigned long bo_size;
1893         bool is_invalid_userptr = false;
1894
1895         bo = mem->bo;
1896         if (!bo) {
1897                 pr_err("Invalid BO when mapping memory to GPU\n");
1898                 return -EINVAL;
1899         }
1900
1901         /* Make sure restore is not running concurrently. Since we
1902          * don't map invalid userptr BOs, we rely on the next restore
1903          * worker to do the mapping
1904          */
1905         mutex_lock(&mem->process_info->lock);
1906
1907         /* Lock mmap-sem. If we find an invalid userptr BO, we can be
1908          * sure that the MMU notifier is no longer running
1909          * concurrently and the queues are actually stopped
1910          */
1911         if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
1912                 mmap_write_lock(current->mm);
1913                 is_invalid_userptr = atomic_read(&mem->invalid);
1914                 mmap_write_unlock(current->mm);
1915         }
1916
1917         mutex_lock(&mem->lock);
1918
1919         domain = mem->domain;
1920         bo_size = bo->tbo.base.size;
1921
1922         pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n",
1923                         mem->va,
1924                         mem->va + bo_size * (1 + mem->aql_queue),
1925                         avm, domain_string(domain));
1926
1927         if (!kfd_mem_is_attached(avm, mem)) {
1928                 ret = kfd_mem_attach(adev, mem, avm, mem->aql_queue);
1929                 if (ret)
1930                         goto out;
1931         }
1932
1933         ret = reserve_bo_and_vm(mem, avm, &ctx);
1934         if (unlikely(ret))
1935                 goto out;
1936
1937         /* Userptr can be marked as "not invalid", but not actually be
1938          * validated yet (still in the system domain). In that case
1939          * the queues are still stopped and we can leave mapping for
1940          * the next restore worker
1941          */
1942         if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) &&
1943             bo->tbo.resource->mem_type == TTM_PL_SYSTEM)
1944                 is_invalid_userptr = true;
1945
1946         ret = vm_validate_pt_pd_bos(avm);
1947         if (unlikely(ret))
1948                 goto out_unreserve;
1949
1950         if (mem->mapped_to_gpu_memory == 0 &&
1951             !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
1952                 /* Validate BO only once. The eviction fence gets added to BO
1953                  * the first time it is mapped. Validate will wait for all
1954                  * background evictions to complete.
1955                  */
1956                 ret = amdgpu_amdkfd_bo_validate(bo, domain, true);
1957                 if (ret) {
1958                         pr_debug("Validate failed\n");
1959                         goto out_unreserve;
1960                 }
1961         }
1962
1963         list_for_each_entry(entry, &mem->attachments, list) {
1964                 if (entry->bo_va->base.vm != avm || entry->is_mapped)
1965                         continue;
1966
1967                 pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",
1968                          entry->va, entry->va + bo_size, entry);
1969
1970                 ret = map_bo_to_gpuvm(mem, entry, ctx.sync,
1971                                       is_invalid_userptr);
1972                 if (ret) {
1973                         pr_err("Failed to map bo to gpuvm\n");
1974                         goto out_unreserve;
1975                 }
1976
1977                 ret = vm_update_pds(avm, ctx.sync);
1978                 if (ret) {
1979                         pr_err("Failed to update page directories\n");
1980                         goto out_unreserve;
1981                 }
1982
1983                 entry->is_mapped = true;
1984                 mem->mapped_to_gpu_memory++;
1985                 pr_debug("\t INC mapping count %d\n",
1986                          mem->mapped_to_gpu_memory);
1987         }
1988
1989         if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->tbo.pin_count)
1990                 amdgpu_bo_fence(bo,
1991                                 &avm->process_info->eviction_fence->base,
1992                                 true);
1993         ret = unreserve_bo_and_vms(&ctx, false, false);
1994
1995         goto out;
1996
1997 out_unreserve:
1998         unreserve_bo_and_vms(&ctx, false, false);
1999 out:
2000         mutex_unlock(&mem->process_info->lock);
2001         mutex_unlock(&mem->lock);
2002         return ret;
2003 }
2004
2005 int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
2006                 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv)
2007 {
2008         struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
2009         struct amdkfd_process_info *process_info = avm->process_info;
2010         unsigned long bo_size = mem->bo->tbo.base.size;
2011         struct kfd_mem_attachment *entry;
2012         struct bo_vm_reservation_context ctx;
2013         int ret;
2014
2015         mutex_lock(&mem->lock);
2016
2017         ret = reserve_bo_and_cond_vms(mem, avm, BO_VM_MAPPED, &ctx);
2018         if (unlikely(ret))
2019                 goto out;
2020         /* If no VMs were reserved, it means the BO wasn't actually mapped */
2021         if (ctx.n_vms == 0) {
2022                 ret = -EINVAL;
2023                 goto unreserve_out;
2024         }
2025
2026         ret = vm_validate_pt_pd_bos(avm);
2027         if (unlikely(ret))
2028                 goto unreserve_out;
2029
2030         pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n",
2031                 mem->va,
2032                 mem->va + bo_size * (1 + mem->aql_queue),
2033                 avm);
2034
2035         list_for_each_entry(entry, &mem->attachments, list) {
2036                 if (entry->bo_va->base.vm != avm || !entry->is_mapped)
2037                         continue;
2038
2039                 pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",
2040                          entry->va, entry->va + bo_size, entry);
2041
2042                 unmap_bo_from_gpuvm(mem, entry, ctx.sync);
2043                 entry->is_mapped = false;
2044
2045                 mem->mapped_to_gpu_memory--;
2046                 pr_debug("\t DEC mapping count %d\n",
2047                          mem->mapped_to_gpu_memory);
2048         }
2049
2050         /* If BO is unmapped from all VMs, unfence it. It can be evicted if
2051          * required.
2052          */
2053         if (mem->mapped_to_gpu_memory == 0 &&
2054             !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) &&
2055             !mem->bo->tbo.pin_count)
2056                 amdgpu_amdkfd_remove_eviction_fence(mem->bo,
2057                                                 process_info->eviction_fence);
2058
2059 unreserve_out:
2060         unreserve_bo_and_vms(&ctx, false, false);
2061 out:
2062         mutex_unlock(&mem->lock);
2063         return ret;
2064 }
2065
2066 int amdgpu_amdkfd_gpuvm_sync_memory(
2067                 struct amdgpu_device *adev, struct kgd_mem *mem, bool intr)
2068 {
2069         struct amdgpu_sync sync;
2070         int ret;
2071
2072         amdgpu_sync_create(&sync);
2073
2074         mutex_lock(&mem->lock);
2075         amdgpu_sync_clone(&mem->sync, &sync);
2076         mutex_unlock(&mem->lock);
2077
2078         ret = amdgpu_sync_wait(&sync, intr);
2079         amdgpu_sync_free(&sync);
2080         return ret;
2081 }
2082
2083 /**
2084  * amdgpu_amdkfd_map_gtt_bo_to_gart - Map BO to GART and increment reference count
2085  * @adev: Device to which allocated BO belongs
2086  * @bo: Buffer object to be mapped
2087  *
2088  * Before return, bo reference count is incremented. To release the reference and unpin/
2089  * unmap the BO, call amdgpu_amdkfd_free_gtt_mem.
2090  */
2091 int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_device *adev, struct amdgpu_bo *bo)
2092 {
2093         int ret;
2094
2095         ret = amdgpu_bo_reserve(bo, true);
2096         if (ret) {
2097                 pr_err("Failed to reserve bo. ret %d\n", ret);
2098                 goto err_reserve_bo_failed;
2099         }
2100
2101         ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
2102         if (ret) {
2103                 pr_err("Failed to pin bo. ret %d\n", ret);
2104                 goto err_pin_bo_failed;
2105         }
2106
2107         ret = amdgpu_ttm_alloc_gart(&bo->tbo);
2108         if (ret) {
2109                 pr_err("Failed to bind bo to GART. ret %d\n", ret);
2110                 goto err_map_bo_gart_failed;
2111         }
2112
2113         amdgpu_amdkfd_remove_eviction_fence(
2114                 bo, bo->kfd_bo->process_info->eviction_fence);
2115
2116         amdgpu_bo_unreserve(bo);
2117
2118         bo = amdgpu_bo_ref(bo);
2119
2120         return 0;
2121
2122 err_map_bo_gart_failed:
2123         amdgpu_bo_unpin(bo);
2124 err_pin_bo_failed:
2125         amdgpu_bo_unreserve(bo);
2126 err_reserve_bo_failed:
2127
2128         return ret;
2129 }
2130
2131 /** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Map a GTT BO for kernel CPU access
2132  *
2133  * @mem: Buffer object to be mapped for CPU access
2134  * @kptr[out]: pointer in kernel CPU address space
2135  * @size[out]: size of the buffer
2136  *
2137  * Pins the BO and maps it for kernel CPU access. The eviction fence is removed
2138  * from the BO, since pinned BOs cannot be evicted. The bo must remain on the
2139  * validate_list, so the GPU mapping can be restored after a page table was
2140  * evicted.
2141  *
2142  * Return: 0 on success, error code on failure
2143  */
2144 int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
2145                                              void **kptr, uint64_t *size)
2146 {
2147         int ret;
2148         struct amdgpu_bo *bo = mem->bo;
2149
2150         if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
2151                 pr_err("userptr can't be mapped to kernel\n");
2152                 return -EINVAL;
2153         }
2154
2155         mutex_lock(&mem->process_info->lock);
2156
2157         ret = amdgpu_bo_reserve(bo, true);
2158         if (ret) {
2159                 pr_err("Failed to reserve bo. ret %d\n", ret);
2160                 goto bo_reserve_failed;
2161         }
2162
2163         ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
2164         if (ret) {
2165                 pr_err("Failed to pin bo. ret %d\n", ret);
2166                 goto pin_failed;
2167         }
2168
2169         ret = amdgpu_bo_kmap(bo, kptr);
2170         if (ret) {
2171                 pr_err("Failed to map bo to kernel. ret %d\n", ret);
2172                 goto kmap_failed;
2173         }
2174
2175         amdgpu_amdkfd_remove_eviction_fence(
2176                 bo, mem->process_info->eviction_fence);
2177
2178         if (size)
2179                 *size = amdgpu_bo_size(bo);
2180
2181         amdgpu_bo_unreserve(bo);
2182
2183         mutex_unlock(&mem->process_info->lock);
2184         return 0;
2185
2186 kmap_failed:
2187         amdgpu_bo_unpin(bo);
2188 pin_failed:
2189         amdgpu_bo_unreserve(bo);
2190 bo_reserve_failed:
2191         mutex_unlock(&mem->process_info->lock);
2192
2193         return ret;
2194 }
2195
2196 /** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Unmap a GTT BO for kernel CPU access
2197  *
2198  * @mem: Buffer object to be unmapped for CPU access
2199  *
2200  * Removes the kernel CPU mapping and unpins the BO. It does not restore the
2201  * eviction fence, so this function should only be used for cleanup before the
2202  * BO is destroyed.
2203  */
2204 void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem)
2205 {
2206         struct amdgpu_bo *bo = mem->bo;
2207
2208         amdgpu_bo_reserve(bo, true);
2209         amdgpu_bo_kunmap(bo);
2210         amdgpu_bo_unpin(bo);
2211         amdgpu_bo_unreserve(bo);
2212 }
2213
2214 int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev,
2215                                           struct kfd_vm_fault_info *mem)
2216 {
2217         if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) {
2218                 *mem = *adev->gmc.vm_fault_info;
2219                 mb();
2220                 atomic_set(&adev->gmc.vm_fault_info_updated, 0);
2221         }
2222         return 0;
2223 }
2224
2225 int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device *adev,
2226                                       struct dma_buf *dma_buf,
2227                                       uint64_t va, void *drm_priv,
2228                                       struct kgd_mem **mem, uint64_t *size,
2229                                       uint64_t *mmap_offset)
2230 {
2231         struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
2232         struct drm_gem_object *obj;
2233         struct amdgpu_bo *bo;
2234         int ret;
2235
2236         if (dma_buf->ops != &amdgpu_dmabuf_ops)
2237                 /* Can't handle non-graphics buffers */
2238                 return -EINVAL;
2239
2240         obj = dma_buf->priv;
2241         if (drm_to_adev(obj->dev) != adev)
2242                 /* Can't handle buffers from other devices */
2243                 return -EINVAL;
2244
2245         bo = gem_to_amdgpu_bo(obj);
2246         if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |
2247                                     AMDGPU_GEM_DOMAIN_GTT)))
2248                 /* Only VRAM and GTT BOs are supported */
2249                 return -EINVAL;
2250
2251         *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
2252         if (!*mem)
2253                 return -ENOMEM;
2254
2255         ret = drm_vma_node_allow(&obj->vma_node, drm_priv);
2256         if (ret) {
2257                 kfree(mem);
2258                 return ret;
2259         }
2260
2261         if (size)
2262                 *size = amdgpu_bo_size(bo);
2263
2264         if (mmap_offset)
2265                 *mmap_offset = amdgpu_bo_mmap_offset(bo);
2266
2267         INIT_LIST_HEAD(&(*mem)->attachments);
2268         mutex_init(&(*mem)->lock);
2269
2270         (*mem)->alloc_flags =
2271                 ((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?
2272                 KFD_IOC_ALLOC_MEM_FLAGS_VRAM : KFD_IOC_ALLOC_MEM_FLAGS_GTT)
2273                 | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE
2274                 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
2275
2276         drm_gem_object_get(&bo->tbo.base);
2277         (*mem)->bo = bo;
2278         (*mem)->va = va;
2279         (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?
2280                 AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT;
2281         (*mem)->mapped_to_gpu_memory = 0;
2282         (*mem)->process_info = avm->process_info;
2283         add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false);
2284         amdgpu_sync_create(&(*mem)->sync);
2285         (*mem)->is_imported = true;
2286
2287         return 0;
2288 }
2289
2290 /* Evict a userptr BO by stopping the queues if necessary
2291  *
2292  * Runs in MMU notifier, may be in RECLAIM_FS context. This means it
2293  * cannot do any memory allocations, and cannot take any locks that
2294  * are held elsewhere while allocating memory. Therefore this is as
2295  * simple as possible, using atomic counters.
2296  *
2297  * It doesn't do anything to the BO itself. The real work happens in
2298  * restore, where we get updated page addresses. This function only
2299  * ensures that GPU access to the BO is stopped.
2300  */
2301 int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem,
2302                                 struct mm_struct *mm)
2303 {
2304         struct amdkfd_process_info *process_info = mem->process_info;
2305         int evicted_bos;
2306         int r = 0;
2307
2308         /* Do not process MMU notifications until stage-4 IOCTL is received */
2309         if (READ_ONCE(process_info->block_mmu_notifications))
2310                 return 0;
2311
2312         atomic_inc(&mem->invalid);
2313         evicted_bos = atomic_inc_return(&process_info->evicted_bos);
2314         if (evicted_bos == 1) {
2315                 /* First eviction, stop the queues */
2316                 r = kgd2kfd_quiesce_mm(mm, KFD_QUEUE_EVICTION_TRIGGER_USERPTR);
2317                 if (r)
2318                         pr_err("Failed to quiesce KFD\n");
2319                 schedule_delayed_work(&process_info->restore_userptr_work,
2320                         msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
2321         }
2322
2323         return r;
2324 }
2325
2326 /* Update invalid userptr BOs
2327  *
2328  * Moves invalidated (evicted) userptr BOs from userptr_valid_list to
2329  * userptr_inval_list and updates user pages for all BOs that have
2330  * been invalidated since their last update.
2331  */
2332 static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
2333                                      struct mm_struct *mm)
2334 {
2335         struct kgd_mem *mem, *tmp_mem;
2336         struct amdgpu_bo *bo;
2337         struct ttm_operation_ctx ctx = { false, false };
2338         int invalid, ret;
2339
2340         /* Move all invalidated BOs to the userptr_inval_list and
2341          * release their user pages by migration to the CPU domain
2342          */
2343         list_for_each_entry_safe(mem, tmp_mem,
2344                                  &process_info->userptr_valid_list,
2345                                  validate_list.head) {
2346                 if (!atomic_read(&mem->invalid))
2347                         continue; /* BO is still valid */
2348
2349                 bo = mem->bo;
2350
2351                 if (amdgpu_bo_reserve(bo, true))
2352                         return -EAGAIN;
2353                 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
2354                 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
2355                 amdgpu_bo_unreserve(bo);
2356                 if (ret) {
2357                         pr_err("%s: Failed to invalidate userptr BO\n",
2358                                __func__);
2359                         return -EAGAIN;
2360                 }
2361
2362                 list_move_tail(&mem->validate_list.head,
2363                                &process_info->userptr_inval_list);
2364         }
2365
2366         if (list_empty(&process_info->userptr_inval_list))
2367                 return 0; /* All evicted userptr BOs were freed */
2368
2369         /* Go through userptr_inval_list and update any invalid user_pages */
2370         list_for_each_entry(mem, &process_info->userptr_inval_list,
2371                             validate_list.head) {
2372                 invalid = atomic_read(&mem->invalid);
2373                 if (!invalid)
2374                         /* BO hasn't been invalidated since the last
2375                          * revalidation attempt. Keep its BO list.
2376                          */
2377                         continue;
2378
2379                 bo = mem->bo;
2380
2381                 /* Get updated user pages */
2382                 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages);
2383                 if (ret) {
2384                         pr_debug("Failed %d to get user pages\n", ret);
2385
2386                         /* Return -EFAULT bad address error as success. It will
2387                          * fail later with a VM fault if the GPU tries to access
2388                          * it. Better than hanging indefinitely with stalled
2389                          * user mode queues.
2390                          *
2391                          * Return other error -EBUSY or -ENOMEM to retry restore
2392                          */
2393                         if (ret != -EFAULT)
2394                                 return ret;
2395                 } else {
2396
2397                         /*
2398                          * FIXME: Cannot ignore the return code, must hold
2399                          * notifier_lock
2400                          */
2401                         amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
2402                 }
2403
2404                 /* Mark the BO as valid unless it was invalidated
2405                  * again concurrently.
2406                  */
2407                 if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid)
2408                         return -EAGAIN;
2409         }
2410
2411         return 0;
2412 }
2413
2414 /* Validate invalid userptr BOs
2415  *
2416  * Validates BOs on the userptr_inval_list, and moves them back to the
2417  * userptr_valid_list. Also updates GPUVM page tables with new page
2418  * addresses and waits for the page table updates to complete.
2419  */
2420 static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
2421 {
2422         struct amdgpu_bo_list_entry *pd_bo_list_entries;
2423         struct list_head resv_list, duplicates;
2424         struct ww_acquire_ctx ticket;
2425         struct amdgpu_sync sync;
2426
2427         struct amdgpu_vm *peer_vm;
2428         struct kgd_mem *mem, *tmp_mem;
2429         struct amdgpu_bo *bo;
2430         struct ttm_operation_ctx ctx = { false, false };
2431         int i, ret;
2432
2433         pd_bo_list_entries = kcalloc(process_info->n_vms,
2434                                      sizeof(struct amdgpu_bo_list_entry),
2435                                      GFP_KERNEL);
2436         if (!pd_bo_list_entries) {
2437                 pr_err("%s: Failed to allocate PD BO list entries\n", __func__);
2438                 ret = -ENOMEM;
2439                 goto out_no_mem;
2440         }
2441
2442         INIT_LIST_HEAD(&resv_list);
2443         INIT_LIST_HEAD(&duplicates);
2444
2445         /* Get all the page directory BOs that need to be reserved */
2446         i = 0;
2447         list_for_each_entry(peer_vm, &process_info->vm_list_head,
2448                             vm_list_node)
2449                 amdgpu_vm_get_pd_bo(peer_vm, &resv_list,
2450                                     &pd_bo_list_entries[i++]);
2451         /* Add the userptr_inval_list entries to resv_list */
2452         list_for_each_entry(mem, &process_info->userptr_inval_list,
2453                             validate_list.head) {
2454                 list_add_tail(&mem->resv_list.head, &resv_list);
2455                 mem->resv_list.bo = mem->validate_list.bo;
2456                 mem->resv_list.num_shared = mem->validate_list.num_shared;
2457         }
2458
2459         /* Reserve all BOs and page tables for validation */
2460         ret = ttm_eu_reserve_buffers(&ticket, &resv_list, false, &duplicates);
2461         WARN(!list_empty(&duplicates), "Duplicates should be empty");
2462         if (ret)
2463                 goto out_free;
2464
2465         amdgpu_sync_create(&sync);
2466
2467         ret = process_validate_vms(process_info);
2468         if (ret)
2469                 goto unreserve_out;
2470
2471         /* Validate BOs and update GPUVM page tables */
2472         list_for_each_entry_safe(mem, tmp_mem,
2473                                  &process_info->userptr_inval_list,
2474                                  validate_list.head) {
2475                 struct kfd_mem_attachment *attachment;
2476
2477                 bo = mem->bo;
2478
2479                 /* Validate the BO if we got user pages */
2480                 if (bo->tbo.ttm->pages[0]) {
2481                         amdgpu_bo_placement_from_domain(bo, mem->domain);
2482                         ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
2483                         if (ret) {
2484                                 pr_err("%s: failed to validate BO\n", __func__);
2485                                 goto unreserve_out;
2486                         }
2487                 }
2488
2489                 list_move_tail(&mem->validate_list.head,
2490                                &process_info->userptr_valid_list);
2491
2492                 /* Update mapping. If the BO was not validated
2493                  * (because we couldn't get user pages), this will
2494                  * clear the page table entries, which will result in
2495                  * VM faults if the GPU tries to access the invalid
2496                  * memory.
2497                  */
2498                 list_for_each_entry(attachment, &mem->attachments, list) {
2499                         if (!attachment->is_mapped)
2500                                 continue;
2501
2502                         kfd_mem_dmaunmap_attachment(mem, attachment);
2503                         ret = update_gpuvm_pte(mem, attachment, &sync);
2504                         if (ret) {
2505                                 pr_err("%s: update PTE failed\n", __func__);
2506                                 /* make sure this gets validated again */
2507                                 atomic_inc(&mem->invalid);
2508                                 goto unreserve_out;
2509                         }
2510                 }
2511         }
2512
2513         /* Update page directories */
2514         ret = process_update_pds(process_info, &sync);
2515
2516 unreserve_out:
2517         ttm_eu_backoff_reservation(&ticket, &resv_list);
2518         amdgpu_sync_wait(&sync, false);
2519         amdgpu_sync_free(&sync);
2520 out_free:
2521         kfree(pd_bo_list_entries);
2522 out_no_mem:
2523
2524         return ret;
2525 }
2526
2527 /* Worker callback to restore evicted userptr BOs
2528  *
2529  * Tries to update and validate all userptr BOs. If successful and no
2530  * concurrent evictions happened, the queues are restarted. Otherwise,
2531  * reschedule for another attempt later.
2532  */
2533 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work)
2534 {
2535         struct delayed_work *dwork = to_delayed_work(work);
2536         struct amdkfd_process_info *process_info =
2537                 container_of(dwork, struct amdkfd_process_info,
2538                              restore_userptr_work);
2539         struct task_struct *usertask;
2540         struct mm_struct *mm;
2541         int evicted_bos;
2542
2543         evicted_bos = atomic_read(&process_info->evicted_bos);
2544         if (!evicted_bos)
2545                 return;
2546
2547         /* Reference task and mm in case of concurrent process termination */
2548         usertask = get_pid_task(process_info->pid, PIDTYPE_PID);
2549         if (!usertask)
2550                 return;
2551         mm = get_task_mm(usertask);
2552         if (!mm) {
2553                 put_task_struct(usertask);
2554                 return;
2555         }
2556
2557         mutex_lock(&process_info->lock);
2558
2559         if (update_invalid_user_pages(process_info, mm))
2560                 goto unlock_out;
2561         /* userptr_inval_list can be empty if all evicted userptr BOs
2562          * have been freed. In that case there is nothing to validate
2563          * and we can just restart the queues.
2564          */
2565         if (!list_empty(&process_info->userptr_inval_list)) {
2566                 if (atomic_read(&process_info->evicted_bos) != evicted_bos)
2567                         goto unlock_out; /* Concurrent eviction, try again */
2568
2569                 if (validate_invalid_user_pages(process_info))
2570                         goto unlock_out;
2571         }
2572         /* Final check for concurrent evicton and atomic update. If
2573          * another eviction happens after successful update, it will
2574          * be a first eviction that calls quiesce_mm. The eviction
2575          * reference counting inside KFD will handle this case.
2576          */
2577         if (atomic_cmpxchg(&process_info->evicted_bos, evicted_bos, 0) !=
2578             evicted_bos)
2579                 goto unlock_out;
2580         evicted_bos = 0;
2581         if (kgd2kfd_resume_mm(mm)) {
2582                 pr_err("%s: Failed to resume KFD\n", __func__);
2583                 /* No recovery from this failure. Probably the CP is
2584                  * hanging. No point trying again.
2585                  */
2586         }
2587
2588 unlock_out:
2589         mutex_unlock(&process_info->lock);
2590
2591         /* If validation failed, reschedule another attempt */
2592         if (evicted_bos) {
2593                 schedule_delayed_work(&process_info->restore_userptr_work,
2594                         msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
2595
2596                 kfd_smi_event_queue_restore_rescheduled(mm);
2597         }
2598         mmput(mm);
2599         put_task_struct(usertask);
2600 }
2601
2602 /** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given
2603  *   KFD process identified by process_info
2604  *
2605  * @process_info: amdkfd_process_info of the KFD process
2606  *
2607  * After memory eviction, restore thread calls this function. The function
2608  * should be called when the Process is still valid. BO restore involves -
2609  *
2610  * 1.  Release old eviction fence and create new one
2611  * 2.  Get two copies of PD BO list from all the VMs. Keep one copy as pd_list.
2612  * 3   Use the second PD list and kfd_bo_list to create a list (ctx.list) of
2613  *     BOs that need to be reserved.
2614  * 4.  Reserve all the BOs
2615  * 5.  Validate of PD and PT BOs.
2616  * 6.  Validate all KFD BOs using kfd_bo_list and Map them and add new fence
2617  * 7.  Add fence to all PD and PT BOs.
2618  * 8.  Unreserve all BOs
2619  */
2620 int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef)
2621 {
2622         struct amdgpu_bo_list_entry *pd_bo_list;
2623         struct amdkfd_process_info *process_info = info;
2624         struct amdgpu_vm *peer_vm;
2625         struct kgd_mem *mem;
2626         struct bo_vm_reservation_context ctx;
2627         struct amdgpu_amdkfd_fence *new_fence;
2628         int ret = 0, i;
2629         struct list_head duplicate_save;
2630         struct amdgpu_sync sync_obj;
2631         unsigned long failed_size = 0;
2632         unsigned long total_size = 0;
2633
2634         INIT_LIST_HEAD(&duplicate_save);
2635         INIT_LIST_HEAD(&ctx.list);
2636         INIT_LIST_HEAD(&ctx.duplicates);
2637
2638         pd_bo_list = kcalloc(process_info->n_vms,
2639                              sizeof(struct amdgpu_bo_list_entry),
2640                              GFP_KERNEL);
2641         if (!pd_bo_list)
2642                 return -ENOMEM;
2643
2644         i = 0;
2645         mutex_lock(&process_info->lock);
2646         list_for_each_entry(peer_vm, &process_info->vm_list_head,
2647                         vm_list_node)
2648                 amdgpu_vm_get_pd_bo(peer_vm, &ctx.list, &pd_bo_list[i++]);
2649
2650         /* Reserve all BOs and page tables/directory. Add all BOs from
2651          * kfd_bo_list to ctx.list
2652          */
2653         list_for_each_entry(mem, &process_info->kfd_bo_list,
2654                             validate_list.head) {
2655
2656                 list_add_tail(&mem->resv_list.head, &ctx.list);
2657                 mem->resv_list.bo = mem->validate_list.bo;
2658                 mem->resv_list.num_shared = mem->validate_list.num_shared;
2659         }
2660
2661         ret = ttm_eu_reserve_buffers(&ctx.ticket, &ctx.list,
2662                                      false, &duplicate_save);
2663         if (ret) {
2664                 pr_debug("Memory eviction: TTM Reserve Failed. Try again\n");
2665                 goto ttm_reserve_fail;
2666         }
2667
2668         amdgpu_sync_create(&sync_obj);
2669
2670         /* Validate PDs and PTs */
2671         ret = process_validate_vms(process_info);
2672         if (ret)
2673                 goto validate_map_fail;
2674
2675         ret = process_sync_pds_resv(process_info, &sync_obj);
2676         if (ret) {
2677                 pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n");
2678                 goto validate_map_fail;
2679         }
2680
2681         /* Validate BOs and map them to GPUVM (update VM page tables). */
2682         list_for_each_entry(mem, &process_info->kfd_bo_list,
2683                             validate_list.head) {
2684
2685                 struct amdgpu_bo *bo = mem->bo;
2686                 uint32_t domain = mem->domain;
2687                 struct kfd_mem_attachment *attachment;
2688                 struct dma_resv_iter cursor;
2689                 struct dma_fence *fence;
2690
2691                 total_size += amdgpu_bo_size(bo);
2692
2693                 ret = amdgpu_amdkfd_bo_validate(bo, domain, false);
2694                 if (ret) {
2695                         pr_debug("Memory eviction: Validate BOs failed\n");
2696                         failed_size += amdgpu_bo_size(bo);
2697                         ret = amdgpu_amdkfd_bo_validate(bo,
2698                                                 AMDGPU_GEM_DOMAIN_GTT, false);
2699                         if (ret) {
2700                                 pr_debug("Memory eviction: Try again\n");
2701                                 goto validate_map_fail;
2702                         }
2703                 }
2704                 dma_resv_for_each_fence(&cursor, bo->tbo.base.resv,
2705                                         DMA_RESV_USAGE_KERNEL, fence) {
2706                         ret = amdgpu_sync_fence(&sync_obj, fence);
2707                         if (ret) {
2708                                 pr_debug("Memory eviction: Sync BO fence failed. Try again\n");
2709                                 goto validate_map_fail;
2710                         }
2711                 }
2712                 list_for_each_entry(attachment, &mem->attachments, list) {
2713                         if (!attachment->is_mapped)
2714                                 continue;
2715
2716                         kfd_mem_dmaunmap_attachment(mem, attachment);
2717                         ret = update_gpuvm_pte(mem, attachment, &sync_obj);
2718                         if (ret) {
2719                                 pr_debug("Memory eviction: update PTE failed. Try again\n");
2720                                 goto validate_map_fail;
2721                         }
2722                 }
2723         }
2724
2725         if (failed_size)
2726                 pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size);
2727
2728         /* Update page directories */
2729         ret = process_update_pds(process_info, &sync_obj);
2730         if (ret) {
2731                 pr_debug("Memory eviction: update PDs failed. Try again\n");
2732                 goto validate_map_fail;
2733         }
2734
2735         /* Wait for validate and PT updates to finish */
2736         amdgpu_sync_wait(&sync_obj, false);
2737
2738         /* Release old eviction fence and create new one, because fence only
2739          * goes from unsignaled to signaled, fence cannot be reused.
2740          * Use context and mm from the old fence.
2741          */
2742         new_fence = amdgpu_amdkfd_fence_create(
2743                                 process_info->eviction_fence->base.context,
2744                                 process_info->eviction_fence->mm,
2745                                 NULL);
2746         if (!new_fence) {
2747                 pr_err("Failed to create eviction fence\n");
2748                 ret = -ENOMEM;
2749                 goto validate_map_fail;
2750         }
2751         dma_fence_put(&process_info->eviction_fence->base);
2752         process_info->eviction_fence = new_fence;
2753         *ef = dma_fence_get(&new_fence->base);
2754
2755         /* Attach new eviction fence to all BOs except pinned ones */
2756         list_for_each_entry(mem, &process_info->kfd_bo_list,
2757                 validate_list.head) {
2758                 if (mem->bo->tbo.pin_count)
2759                         continue;
2760
2761                 amdgpu_bo_fence(mem->bo,
2762                         &process_info->eviction_fence->base, true);
2763         }
2764         /* Attach eviction fence to PD / PT BOs */
2765         list_for_each_entry(peer_vm, &process_info->vm_list_head,
2766                             vm_list_node) {
2767                 struct amdgpu_bo *bo = peer_vm->root.bo;
2768
2769                 amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true);
2770         }
2771
2772 validate_map_fail:
2773         ttm_eu_backoff_reservation(&ctx.ticket, &ctx.list);
2774         amdgpu_sync_free(&sync_obj);
2775 ttm_reserve_fail:
2776         mutex_unlock(&process_info->lock);
2777         kfree(pd_bo_list);
2778         return ret;
2779 }
2780
2781 int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem)
2782 {
2783         struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
2784         struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws;
2785         int ret;
2786
2787         if (!info || !gws)
2788                 return -EINVAL;
2789
2790         *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
2791         if (!*mem)
2792                 return -ENOMEM;
2793
2794         mutex_init(&(*mem)->lock);
2795         INIT_LIST_HEAD(&(*mem)->attachments);
2796         (*mem)->bo = amdgpu_bo_ref(gws_bo);
2797         (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS;
2798         (*mem)->process_info = process_info;
2799         add_kgd_mem_to_kfd_bo_list(*mem, process_info, false);
2800         amdgpu_sync_create(&(*mem)->sync);
2801
2802
2803         /* Validate gws bo the first time it is added to process */
2804         mutex_lock(&(*mem)->process_info->lock);
2805         ret = amdgpu_bo_reserve(gws_bo, false);
2806         if (unlikely(ret)) {
2807                 pr_err("Reserve gws bo failed %d\n", ret);
2808                 goto bo_reservation_failure;
2809         }
2810
2811         ret = amdgpu_amdkfd_bo_validate(gws_bo, AMDGPU_GEM_DOMAIN_GWS, true);
2812         if (ret) {
2813                 pr_err("GWS BO validate failed %d\n", ret);
2814                 goto bo_validation_failure;
2815         }
2816         /* GWS resource is shared b/t amdgpu and amdkfd
2817          * Add process eviction fence to bo so they can
2818          * evict each other.
2819          */
2820         ret = dma_resv_reserve_fences(gws_bo->tbo.base.resv, 1);
2821         if (ret)
2822                 goto reserve_shared_fail;
2823         amdgpu_bo_fence(gws_bo, &process_info->eviction_fence->base, true);
2824         amdgpu_bo_unreserve(gws_bo);
2825         mutex_unlock(&(*mem)->process_info->lock);
2826
2827         return ret;
2828
2829 reserve_shared_fail:
2830 bo_validation_failure:
2831         amdgpu_bo_unreserve(gws_bo);
2832 bo_reservation_failure:
2833         mutex_unlock(&(*mem)->process_info->lock);
2834         amdgpu_sync_free(&(*mem)->sync);
2835         remove_kgd_mem_from_kfd_bo_list(*mem, process_info);
2836         amdgpu_bo_unref(&gws_bo);
2837         mutex_destroy(&(*mem)->lock);
2838         kfree(*mem);
2839         *mem = NULL;
2840         return ret;
2841 }
2842
2843 int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem)
2844 {
2845         int ret;
2846         struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
2847         struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
2848         struct amdgpu_bo *gws_bo = kgd_mem->bo;
2849
2850         /* Remove BO from process's validate list so restore worker won't touch
2851          * it anymore
2852          */
2853         remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info);
2854
2855         ret = amdgpu_bo_reserve(gws_bo, false);
2856         if (unlikely(ret)) {
2857                 pr_err("Reserve gws bo failed %d\n", ret);
2858                 //TODO add BO back to validate_list?
2859                 return ret;
2860         }
2861         amdgpu_amdkfd_remove_eviction_fence(gws_bo,
2862                         process_info->eviction_fence);
2863         amdgpu_bo_unreserve(gws_bo);
2864         amdgpu_sync_free(&kgd_mem->sync);
2865         amdgpu_bo_unref(&gws_bo);
2866         mutex_destroy(&kgd_mem->lock);
2867         kfree(mem);
2868         return 0;
2869 }
2870
2871 /* Returns GPU-specific tiling mode information */
2872 int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev,
2873                                 struct tile_config *config)
2874 {
2875         config->gb_addr_config = adev->gfx.config.gb_addr_config;
2876         config->tile_config_ptr = adev->gfx.config.tile_mode_array;
2877         config->num_tile_configs =
2878                         ARRAY_SIZE(adev->gfx.config.tile_mode_array);
2879         config->macro_tile_config_ptr =
2880                         adev->gfx.config.macrotile_mode_array;
2881         config->num_macro_tile_configs =
2882                         ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
2883
2884         /* Those values are not set from GFX9 onwards */
2885         config->num_banks = adev->gfx.config.num_banks;
2886         config->num_ranks = adev->gfx.config.num_ranks;
2887
2888         return 0;
2889 }
2890
2891 bool amdgpu_amdkfd_bo_mapped_to_dev(struct amdgpu_device *adev, struct kgd_mem *mem)
2892 {
2893         struct kfd_mem_attachment *entry;
2894
2895         list_for_each_entry(entry, &mem->attachments, list) {
2896                 if (entry->is_mapped && entry->adev == adev)
2897                         return true;
2898         }
2899         return false;
2900 }
2901
2902 #if defined(CONFIG_DEBUG_FS)
2903
2904 int kfd_debugfs_kfd_mem_limits(struct seq_file *m, void *data)
2905 {
2906
2907         spin_lock(&kfd_mem_limit.mem_limit_lock);
2908         seq_printf(m, "System mem used %lldM out of %lluM\n",
2909                   (kfd_mem_limit.system_mem_used >> 20),
2910                   (kfd_mem_limit.max_system_mem_limit >> 20));
2911         seq_printf(m, "TTM mem used %lldM out of %lluM\n",
2912                   (kfd_mem_limit.ttm_mem_used >> 20),
2913                   (kfd_mem_limit.max_ttm_mem_limit >> 20));
2914         spin_unlock(&kfd_mem_limit.mem_limit_lock);
2915
2916         return 0;
2917 }
2918
2919 #endif