drivers/gpu/drm/i915/gt/intel_ggtt.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020 Intel Corporation
   4  */
   5
   6 #include <asm/set_memory.h>
   7 #include <asm/smp.h>
   8 #include <linux/types.h>
   9 #include <linux/stop_machine.h>
  10
  11 #include <drm/drm_managed.h>
  12 #include <drm/i915_drm.h>
  13 #include <drm/intel-gtt.h>
  14
  15 #include "display/intel_display.h"
  16 #include "gem/i915_gem_lmem.h"
  17
  18 #include "intel_ggtt_gmch.h"
  19 #include "intel_gt.h"
  20 #include "intel_gt_regs.h"
  21 #include "intel_pci_config.h"
  22 #include "i915_drv.h"
  23 #include "i915_pci.h"
  24 #include "i915_scatterlist.h"
  25 #include "i915_utils.h"
  26 #include "i915_vgpu.h"
  27
  28 #include "intel_gtt.h"
  29 #include "gen8_ppgtt.h"
  30
  31 static void i915_ggtt_color_adjust(const struct drm_mm_node *node,
  32                                    unsigned long color,
  33                                    u64 *start,
  34                                    u64 *end)
  35 {
  36         if (i915_node_color_differs(node, color))
  37                 *start += I915_GTT_PAGE_SIZE;
  38
  39         /*
  40          * Also leave a space between the unallocated reserved node after the
  41          * GTT and any objects within the GTT, i.e. we use the color adjustment
  42          * to insert a guard page to prevent prefetches crossing over the
  43          * GTT boundary.
  44          */
  45         node = list_next_entry(node, node_list);
  46         if (node->color != color)
  47                 *end -= I915_GTT_PAGE_SIZE;
  48 }
  49
  50 static int ggtt_init_hw(struct i915_ggtt *ggtt)
  51 {
  52         struct drm_i915_private *i915 = ggtt->vm.i915;
  53
  54         i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT);
  55
  56         ggtt->vm.is_ggtt = true;
  57
  58         /* Only VLV supports read-only GGTT mappings */
  59         ggtt->vm.has_read_only = IS_VALLEYVIEW(i915);
  60
  61         if (!HAS_LLC(i915) && !HAS_PPGTT(i915))
  62                 ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust;
  63
  64         if (ggtt->mappable_end) {
  65                 if (!io_mapping_init_wc(&ggtt->iomap,
  66                                         ggtt->gmadr.start,
  67                                         ggtt->mappable_end)) {
  68                         ggtt->vm.cleanup(&ggtt->vm);
  69                         return -EIO;
  70                 }
  71
  72                 ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start,
  73                                               ggtt->mappable_end);
  74         }
  75
  76         intel_ggtt_init_fences(ggtt);
  77
  78         return 0;
  79 }
  80
  81 /**
  82  * i915_ggtt_init_hw - Initialize GGTT hardware
  83  * @i915: i915 device
  84  */
  85 int i915_ggtt_init_hw(struct drm_i915_private *i915)
  86 {
  87         int ret;
  88
  89         /*
  90          * Note that we use page colouring to enforce a guard page at the
  91          * end of the address space. This is required as the CS may prefetch
  92          * beyond the end of the batch buffer, across the page boundary,
  93          * and beyond the end of the GTT if we do not provide a guard.
  94          */
  95         ret = ggtt_init_hw(to_gt(i915)->ggtt);
  96         if (ret)
  97                 return ret;
  98
  99         return 0;
 100 }
 101
 102 /**
 103  * i915_ggtt_suspend_vm - Suspend the memory mappings for a GGTT or DPT VM
 104  * @vm: The VM to suspend the mappings for
 105  *
 106  * Suspend the memory mappings for all objects mapped to HW via the GGTT or a
 107  * DPT page table.
 108  */
 109 void i915_ggtt_suspend_vm(struct i915_address_space *vm)
 110 {
 111         struct i915_vma *vma, *vn;
 112         int save_skip_rewrite;
 113
 114         drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt);
 115
 116 retry:
 117         i915_gem_drain_freed_objects(vm->i915);
 118
 119         mutex_lock(&vm->mutex);
 120
 121         /*
 122          * Skip rewriting PTE on VMA unbind.
 123          * FIXME: Use an argument to i915_vma_unbind() instead?
 124          */
 125         save_skip_rewrite = vm->skip_pte_rewrite;
 126         vm->skip_pte_rewrite = true;
 127
 128         list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) {
 129                 struct drm_i915_gem_object *obj = vma->obj;
 130
 131                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 132
 133                 if (i915_vma_is_pinned(vma) || !i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
 134                         continue;
 135
 136                 /* unlikely to race when GPU is idle, so no worry about slowpath.. */
 137                 if (WARN_ON(!i915_gem_object_trylock(obj, NULL))) {
 138                         /*
 139                          * No dead objects should appear here, GPU should be
 140                          * completely idle, and userspace suspended
 141                          */
 142                         i915_gem_object_get(obj);
 143
 144                         mutex_unlock(&vm->mutex);
 145
 146                         i915_gem_object_lock(obj, NULL);
 147                         GEM_WARN_ON(i915_vma_unbind(vma));
 148                         i915_gem_object_unlock(obj);
 149                         i915_gem_object_put(obj);
 150
 151                         vm->skip_pte_rewrite = save_skip_rewrite;
 152                         goto retry;
 153                 }
 154
 155                 if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) {
 156                         i915_vma_wait_for_bind(vma);
 157
 158                         __i915_vma_evict(vma, false);
 159                         drm_mm_remove_node(&vma->node);
 160                 }
 161
 162                 i915_gem_object_unlock(obj);
 163         }
 164
 165         vm->clear_range(vm, 0, vm->total);
 166
 167         vm->skip_pte_rewrite = save_skip_rewrite;
 168
 169         mutex_unlock(&vm->mutex);
 170 }
 171
 172 void i915_ggtt_suspend(struct i915_ggtt *ggtt)
 173 {
 174         struct intel_gt *gt;
 175
 176         i915_ggtt_suspend_vm(&ggtt->vm);
 177         ggtt->invalidate(ggtt);
 178
 179         list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
 180                 intel_gt_check_and_clear_faults(gt);
 181 }
 182
 183 void gen6_ggtt_invalidate(struct i915_ggtt *ggtt)
 184 {
 185         struct intel_uncore *uncore = ggtt->vm.gt->uncore;
 186
 187         spin_lock_irq(&uncore->lock);
 188         intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 189         intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6);
 190         spin_unlock_irq(&uncore->lock);
 191 }
 192
 193 static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt)
 194 {
 195         struct intel_uncore *uncore = ggtt->vm.gt->uncore;
 196
 197         /*
 198          * Note that as an uncached mmio write, this will flush the
 199          * WCB of the writes into the GGTT before it triggers the invalidate.
 200          */
 201         intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 202 }
 203
 204 static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
 205 {
 206         struct drm_i915_private *i915 = ggtt->vm.i915;
 207
 208         gen8_ggtt_invalidate(ggtt);
 209
 210         if (GRAPHICS_VER(i915) >= 12) {
 211                 struct intel_gt *gt;
 212
 213                 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
 214                         intel_uncore_write_fw(gt->uncore,
 215                                               GEN12_GUC_TLB_INV_CR,
 216                                               GEN12_GUC_TLB_INV_CR_INVALIDATE);
 217         } else {
 218                 intel_uncore_write_fw(ggtt->vm.gt->uncore,
 219                                       GEN8_GTCR, GEN8_GTCR_INVALIDATE);
 220         }
 221 }
 222
 223 static u64 mtl_ggtt_pte_encode(dma_addr_t addr,
 224                                unsigned int pat_index,
 225                                u32 flags)
 226 {
 227         gen8_pte_t pte = addr | GEN8_PAGE_PRESENT;
 228
 229         WARN_ON_ONCE(addr & ~GEN12_GGTT_PTE_ADDR_MASK);
 230
 231         if (flags & PTE_LM)
 232                 pte |= GEN12_GGTT_PTE_LM;
 233
 234         if (pat_index & BIT(0))
 235                 pte |= MTL_GGTT_PTE_PAT0;
 236
 237         if (pat_index & BIT(1))
 238                 pte |= MTL_GGTT_PTE_PAT1;
 239
 240         return pte;
 241 }
 242
 243 u64 gen8_ggtt_pte_encode(dma_addr_t addr,
 244                          unsigned int pat_index,
 245                          u32 flags)
 246 {
 247         gen8_pte_t pte = addr | GEN8_PAGE_PRESENT;
 248
 249         if (flags & PTE_LM)
 250                 pte |= GEN12_GGTT_PTE_LM;
 251
 252         return pte;
 253 }
 254
 255 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
 256 {
 257         writeq(pte, addr);
 258 }
 259
 260 static void gen8_ggtt_insert_page(struct i915_address_space *vm,
 261                                   dma_addr_t addr,
 262                                   u64 offset,
 263                                   unsigned int pat_index,
 264                                   u32 flags)
 265 {
 266         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 267         gen8_pte_t __iomem *pte =
 268                 (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
 269
 270         gen8_set_pte(pte, ggtt->vm.pte_encode(addr, pat_index, flags));
 271
 272         ggtt->invalidate(ggtt);
 273 }
 274
 275 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
 276                                      struct i915_vma_resource *vma_res,
 277                                      unsigned int pat_index,
 278                                      u32 flags)
 279 {
 280         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 281         const gen8_pte_t pte_encode = ggtt->vm.pte_encode(0, pat_index, flags);
 282         gen8_pte_t __iomem *gte;
 283         gen8_pte_t __iomem *end;
 284         struct sgt_iter iter;
 285         dma_addr_t addr;
 286
 287         /*
 288          * Note that we ignore PTE_READ_ONLY here. The caller must be careful
 289          * not to allow the user to override access to a read only page.
 290          */
 291
 292         gte = (gen8_pte_t __iomem *)ggtt->gsm;
 293         gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE;
 294         end = gte + vma_res->guard / I915_GTT_PAGE_SIZE;
 295         while (gte < end)
 296                 gen8_set_pte(gte++, vm->scratch[0]->encode);
 297         end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE;
 298
 299         for_each_sgt_daddr(addr, iter, vma_res->bi.pages)
 300                 gen8_set_pte(gte++, pte_encode | addr);
 301         GEM_BUG_ON(gte > end);
 302
 303         /* Fill the allocated but "unused" space beyond the end of the buffer */
 304         while (gte < end)
 305                 gen8_set_pte(gte++, vm->scratch[0]->encode);
 306
 307         /*
 308          * We want to flush the TLBs only after we're certain all the PTE
 309          * updates have finished.
 310          */
 311         ggtt->invalidate(ggtt);
 312 }
 313
 314 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
 315                                   u64 start, u64 length)
 316 {
 317         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 318         unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
 319         unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
 320         const gen8_pte_t scratch_pte = vm->scratch[0]->encode;
 321         gen8_pte_t __iomem *gtt_base =
 322                 (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
 323         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
 324         int i;
 325
 326         if (WARN(num_entries > max_entries,
 327                  "First entry = %d; Num entries = %d (max=%d)\n",
 328                  first_entry, num_entries, max_entries))
 329                 num_entries = max_entries;
 330
 331         for (i = 0; i < num_entries; i++)
 332                 gen8_set_pte(&gtt_base[i], scratch_pte);
 333 }
 334
 335 static void gen6_ggtt_insert_page(struct i915_address_space *vm,
 336                                   dma_addr_t addr,
 337                                   u64 offset,
 338                                   unsigned int pat_index,
 339                                   u32 flags)
 340 {
 341         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 342         gen6_pte_t __iomem *pte =
 343                 (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
 344
 345         iowrite32(vm->pte_encode(addr, pat_index, flags), pte);
 346
 347         ggtt->invalidate(ggtt);
 348 }
 349
 350 /*
 351  * Binds an object into the global gtt with the specified cache level.
 352  * The object will be accessible to the GPU via commands whose operands
 353  * reference offsets within the global GTT as well as accessible by the GPU
 354  * through the GMADR mapped BAR (i915->mm.gtt->gtt).
 355  */
 356 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
 357                                      struct i915_vma_resource *vma_res,
 358                                      unsigned int pat_index,
 359                                      u32 flags)
 360 {
 361         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 362         gen6_pte_t __iomem *gte;
 363         gen6_pte_t __iomem *end;
 364         struct sgt_iter iter;
 365         dma_addr_t addr;
 366
 367         gte = (gen6_pte_t __iomem *)ggtt->gsm;
 368         gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE;
 369
 370         end = gte + vma_res->guard / I915_GTT_PAGE_SIZE;
 371         while (gte < end)
 372                 iowrite32(vm->scratch[0]->encode, gte++);
 373         end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE;
 374         for_each_sgt_daddr(addr, iter, vma_res->bi.pages)
 375                 iowrite32(vm->pte_encode(addr, pat_index, flags), gte++);
 376         GEM_BUG_ON(gte > end);
 377
 378         /* Fill the allocated but "unused" space beyond the end of the buffer */
 379         while (gte < end)
 380                 iowrite32(vm->scratch[0]->encode, gte++);
 381
 382         /*
 383          * We want to flush the TLBs only after we're certain all the PTE
 384          * updates have finished.
 385          */
 386         ggtt->invalidate(ggtt);
 387 }
 388
 389 static void nop_clear_range(struct i915_address_space *vm,
 390                             u64 start, u64 length)
 391 {
 392 }
 393
 394 static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
 395 {
 396         /*
 397          * Make sure the internal GAM fifo has been cleared of all GTT
 398          * writes before exiting stop_machine(). This guarantees that
 399          * any aperture accesses waiting to start in another process
 400          * cannot back up behind the GTT writes causing a hang.
 401          * The register can be any arbitrary GAM register.
 402          */
 403         intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6);
 404 }
 405
 406 struct insert_page {
 407         struct i915_address_space *vm;
 408         dma_addr_t addr;
 409         u64 offset;
 410         unsigned int pat_index;
 411 };
 412
 413 static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
 414 {
 415         struct insert_page *arg = _arg;
 416
 417         gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset,
 418                               arg->pat_index, 0);
 419         bxt_vtd_ggtt_wa(arg->vm);
 420
 421         return 0;
 422 }
 423
 424 static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,
 425                                           dma_addr_t addr,
 426                                           u64 offset,
 427                                           unsigned int pat_index,
 428                                           u32 unused)
 429 {
 430         struct insert_page arg = { vm, addr, offset, pat_index };
 431
 432         stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL);
 433 }
 434
 435 struct insert_entries {
 436         struct i915_address_space *vm;
 437         struct i915_vma_resource *vma_res;
 438         unsigned int pat_index;
 439         u32 flags;
 440 };
 441
 442 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
 443 {
 444         struct insert_entries *arg = _arg;
 445
 446         gen8_ggtt_insert_entries(arg->vm, arg->vma_res,
 447                                  arg->pat_index, arg->flags);
 448         bxt_vtd_ggtt_wa(arg->vm);
 449
 450         return 0;
 451 }
 452
 453 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
 454                                              struct i915_vma_resource *vma_res,
 455                                              unsigned int pat_index,
 456                                              u32 flags)
 457 {
 458         struct insert_entries arg = { vm, vma_res, pat_index, flags };
 459
 460         stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);
 461 }
 462
 463 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
 464                                   u64 start, u64 length)
 465 {
 466         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 467         unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
 468         unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
 469         gen6_pte_t scratch_pte, __iomem *gtt_base =
 470                 (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
 471         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
 472         int i;
 473
 474         if (WARN(num_entries > max_entries,
 475                  "First entry = %d; Num entries = %d (max=%d)\n",
 476                  first_entry, num_entries, max_entries))
 477                 num_entries = max_entries;
 478
 479         scratch_pte = vm->scratch[0]->encode;
 480         for (i = 0; i < num_entries; i++)
 481                 iowrite32(scratch_pte, &gtt_base[i]);
 482 }
 483
 484 void intel_ggtt_bind_vma(struct i915_address_space *vm,
 485                          struct i915_vm_pt_stash *stash,
 486                          struct i915_vma_resource *vma_res,
 487                          unsigned int pat_index,
 488                          u32 flags)
 489 {
 490         u32 pte_flags;
 491
 492         if (vma_res->bound_flags & (~flags & I915_VMA_BIND_MASK))
 493                 return;
 494
 495         vma_res->bound_flags |= flags;
 496
 497         /* Applicable to VLV (gen8+ do not support RO in the GGTT) */
 498         pte_flags = 0;
 499         if (vma_res->bi.readonly)
 500                 pte_flags |= PTE_READ_ONLY;
 501         if (vma_res->bi.lmem)
 502                 pte_flags |= PTE_LM;
 503
 504         vm->insert_entries(vm, vma_res, pat_index, pte_flags);
 505         vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
 506 }
 507
 508 void intel_ggtt_unbind_vma(struct i915_address_space *vm,
 509                            struct i915_vma_resource *vma_res)
 510 {
 511         vm->clear_range(vm, vma_res->start, vma_res->vma_size);
 512 }
 513
 514 static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt)
 515 {
 516         u64 size;
 517         int ret;
 518
 519         if (!intel_uc_uses_guc(&ggtt->vm.gt->uc))
 520                 return 0;
 521
 522         GEM_BUG_ON(ggtt->vm.total <= GUC_GGTT_TOP);
 523         size = ggtt->vm.total - GUC_GGTT_TOP;
 524
 525         ret = i915_gem_gtt_reserve(&ggtt->vm, NULL, &ggtt->uc_fw, size,
 526                                    GUC_GGTT_TOP, I915_COLOR_UNEVICTABLE,
 527                                    PIN_NOEVICT);
 528         if (ret)
 529                 drm_dbg(&ggtt->vm.i915->drm,
 530                         "Failed to reserve top of GGTT for GuC\n");
 531
 532         return ret;
 533 }
 534
 535 static void ggtt_release_guc_top(struct i915_ggtt *ggtt)
 536 {
 537         if (drm_mm_node_allocated(&ggtt->uc_fw))
 538                 drm_mm_remove_node(&ggtt->uc_fw);
 539 }
 540
 541 static void cleanup_init_ggtt(struct i915_ggtt *ggtt)
 542 {
 543         ggtt_release_guc_top(ggtt);
 544         if (drm_mm_node_allocated(&ggtt->error_capture))
 545                 drm_mm_remove_node(&ggtt->error_capture);
 546         mutex_destroy(&ggtt->error_mutex);
 547 }
 548
 549 static int init_ggtt(struct i915_ggtt *ggtt)
 550 {
 551         /*
 552          * Let GEM Manage all of the aperture.
 553          *
 554          * However, leave one page at the end still bound to the scratch page.
 555          * There are a number of places where the hardware apparently prefetches
 556          * past the end of the object, and we've seen multiple hangs with the
 557          * GPU head pointer stuck in a batchbuffer bound at the last page of the
 558          * aperture.  One page should be enough to keep any prefetching inside
 559          * of the aperture.
 560          */
 561         unsigned long hole_start, hole_end;
 562         struct drm_mm_node *entry;
 563         int ret;
 564
 565         /*
 566          * GuC requires all resources that we're sharing with it to be placed in
 567          * non-WOPCM memory. If GuC is not present or not in use we still need a
 568          * small bias as ring wraparound at offset 0 sometimes hangs. No idea
 569          * why.
 570          */
 571         ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE,
 572                                intel_wopcm_guc_size(&ggtt->vm.gt->wopcm));
 573
 574         ret = intel_vgt_balloon(ggtt);
 575         if (ret)
 576                 return ret;
 577
 578         mutex_init(&ggtt->error_mutex);
 579         if (ggtt->mappable_end) {
 580                 /*
 581                  * Reserve a mappable slot for our lockless error capture.
 582                  *
 583                  * We strongly prefer taking address 0x0 in order to protect
 584                  * other critical buffers against accidental overwrites,
 585                  * as writing to address 0 is a very common mistake.
 586                  *
 587                  * Since 0 may already be in use by the system (e.g. the BIOS
 588                  * framebuffer), we let the reservation fail quietly and hope
 589                  * 0 remains reserved always.
 590                  *
 591                  * If we fail to reserve 0, and then fail to find any space
 592                  * for an error-capture, remain silent. We can afford not
 593                  * to reserve an error_capture node as we have fallback
 594                  * paths, and we trust that 0 will remain reserved. However,
 595                  * the only likely reason for failure to insert is a driver
 596                  * bug, which we expect to cause other failures...
 597                  *
 598                  * Since CPU can perform speculative reads on error capture
 599                  * (write-combining allows it) add scratch page after error
 600                  * capture to avoid DMAR errors.
 601                  */
 602                 ggtt->error_capture.size = 2 * I915_GTT_PAGE_SIZE;
 603                 ggtt->error_capture.color = I915_COLOR_UNEVICTABLE;
 604                 if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture))
 605                         drm_mm_insert_node_in_range(&ggtt->vm.mm,
 606                                                     &ggtt->error_capture,
 607                                                     ggtt->error_capture.size, 0,
 608                                                     ggtt->error_capture.color,
 609                                                     0, ggtt->mappable_end,
 610                                                     DRM_MM_INSERT_LOW);
 611         }
 612         if (drm_mm_node_allocated(&ggtt->error_capture)) {
 613                 u64 start = ggtt->error_capture.start;
 614                 u64 size = ggtt->error_capture.size;
 615
 616                 ggtt->vm.scratch_range(&ggtt->vm, start, size);
 617                 drm_dbg(&ggtt->vm.i915->drm,
 618                         "Reserved GGTT:[%llx, %llx] for use by error capture\n",
 619                         start, start + size);
 620         }
 621
 622         /*
 623          * The upper portion of the GuC address space has a sizeable hole
 624          * (several MB) that is inaccessible by GuC. Reserve this range within
 625          * GGTT as it can comfortably hold GuC/HuC firmware images.
 626          */
 627         ret = ggtt_reserve_guc_top(ggtt);
 628         if (ret)
 629                 goto err;
 630
 631         /* Clear any non-preallocated blocks */
 632         drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) {
 633                 drm_dbg(&ggtt->vm.i915->drm,
 634                         "clearing unused GTT space: [%lx, %lx]\n",
 635                         hole_start, hole_end);
 636                 ggtt->vm.clear_range(&ggtt->vm, hole_start,
 637                                      hole_end - hole_start);
 638         }
 639
 640         /* And finally clear the reserved guard page */
 641         ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE);
 642
 643         return 0;
 644
 645 err:
 646         cleanup_init_ggtt(ggtt);
 647         return ret;
 648 }
 649
 650 static void aliasing_gtt_bind_vma(struct i915_address_space *vm,
 651                                   struct i915_vm_pt_stash *stash,
 652                                   struct i915_vma_resource *vma_res,
 653                                   unsigned int pat_index,
 654                                   u32 flags)
 655 {
 656         u32 pte_flags;
 657
 658         /* Currently applicable only to VLV */
 659         pte_flags = 0;
 660         if (vma_res->bi.readonly)
 661                 pte_flags |= PTE_READ_ONLY;
 662
 663         if (flags & I915_VMA_LOCAL_BIND)
 664                 ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm,
 665                                stash, vma_res, pat_index, flags);
 666
 667         if (flags & I915_VMA_GLOBAL_BIND)
 668                 vm->insert_entries(vm, vma_res, pat_index, pte_flags);
 669
 670         vma_res->bound_flags |= flags;
 671 }
 672
 673 static void aliasing_gtt_unbind_vma(struct i915_address_space *vm,
 674                                     struct i915_vma_resource *vma_res)
 675 {
 676         if (vma_res->bound_flags & I915_VMA_GLOBAL_BIND)
 677                 vm->clear_range(vm, vma_res->start, vma_res->vma_size);
 678
 679         if (vma_res->bound_flags & I915_VMA_LOCAL_BIND)
 680                 ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma_res);
 681 }
 682
 683 static int init_aliasing_ppgtt(struct i915_ggtt *ggtt)
 684 {
 685         struct i915_vm_pt_stash stash = {};
 686         struct i915_ppgtt *ppgtt;
 687         int err;
 688
 689         ppgtt = i915_ppgtt_create(ggtt->vm.gt, 0);
 690         if (IS_ERR(ppgtt))
 691                 return PTR_ERR(ppgtt);
 692
 693         if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) {
 694                 err = -ENODEV;
 695                 goto err_ppgtt;
 696         }
 697
 698         err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total);
 699         if (err)
 700                 goto err_ppgtt;
 701
 702         i915_gem_object_lock(ppgtt->vm.scratch[0], NULL);
 703         err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
 704         i915_gem_object_unlock(ppgtt->vm.scratch[0]);
 705         if (err)
 706                 goto err_stash;
 707
 708         /*
 709          * Note we only pre-allocate as far as the end of the global
 710          * GTT. On 48b / 4-level page-tables, the difference is very,
 711          * very significant! We have to preallocate as GVT/vgpu does
 712          * not like the page directory disappearing.
 713          */
 714         ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total);
 715
 716         ggtt->alias = ppgtt;
 717         ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags;
 718
 719         GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != intel_ggtt_bind_vma);
 720         ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma;
 721
 722         GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != intel_ggtt_unbind_vma);
 723         ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma;
 724
 725         i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 726         return 0;
 727
 728 err_stash:
 729         i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 730 err_ppgtt:
 731         i915_vm_put(&ppgtt->vm);
 732         return err;
 733 }
 734
 735 static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt)
 736 {
 737         struct i915_ppgtt *ppgtt;
 738
 739         ppgtt = fetch_and_zero(&ggtt->alias);
 740         if (!ppgtt)
 741                 return;
 742
 743         i915_vm_put(&ppgtt->vm);
 744
 745         ggtt->vm.vma_ops.bind_vma   = intel_ggtt_bind_vma;
 746         ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma;
 747 }
 748
 749 int i915_init_ggtt(struct drm_i915_private *i915)
 750 {
 751         int ret;
 752
 753         ret = init_ggtt(to_gt(i915)->ggtt);
 754         if (ret)
 755                 return ret;
 756
 757         if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) {
 758                 ret = init_aliasing_ppgtt(to_gt(i915)->ggtt);
 759                 if (ret)
 760                         cleanup_init_ggtt(to_gt(i915)->ggtt);
 761         }
 762
 763         return 0;
 764 }
 765
 766 static void ggtt_cleanup_hw(struct i915_ggtt *ggtt)
 767 {
 768         struct i915_vma *vma, *vn;
 769
 770         flush_workqueue(ggtt->vm.i915->wq);
 771         i915_gem_drain_freed_objects(ggtt->vm.i915);
 772
 773         mutex_lock(&ggtt->vm.mutex);
 774
 775         ggtt->vm.skip_pte_rewrite = true;
 776
 777         list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
 778                 struct drm_i915_gem_object *obj = vma->obj;
 779                 bool trylock;
 780
 781                 trylock = i915_gem_object_trylock(obj, NULL);
 782                 WARN_ON(!trylock);
 783
 784                 WARN_ON(__i915_vma_unbind(vma));
 785                 if (trylock)
 786                         i915_gem_object_unlock(obj);
 787         }
 788
 789         if (drm_mm_node_allocated(&ggtt->error_capture))
 790                 drm_mm_remove_node(&ggtt->error_capture);
 791         mutex_destroy(&ggtt->error_mutex);
 792
 793         ggtt_release_guc_top(ggtt);
 794         intel_vgt_deballoon(ggtt);
 795
 796         ggtt->vm.cleanup(&ggtt->vm);
 797
 798         mutex_unlock(&ggtt->vm.mutex);
 799         i915_address_space_fini(&ggtt->vm);
 800
 801         arch_phys_wc_del(ggtt->mtrr);
 802
 803         if (ggtt->iomap.size)
 804                 io_mapping_fini(&ggtt->iomap);
 805 }
 806
 807 /**
 808  * i915_ggtt_driver_release - Clean up GGTT hardware initialization
 809  * @i915: i915 device
 810  */
 811 void i915_ggtt_driver_release(struct drm_i915_private *i915)
 812 {
 813         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 814
 815         fini_aliasing_ppgtt(ggtt);
 816
 817         intel_ggtt_fini_fences(ggtt);
 818         ggtt_cleanup_hw(ggtt);
 819 }
 820
 821 /**
 822  * i915_ggtt_driver_late_release - Cleanup of GGTT that needs to be done after
 823  * all free objects have been drained.
 824  * @i915: i915 device
 825  */
 826 void i915_ggtt_driver_late_release(struct drm_i915_private *i915)
 827 {
 828         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 829
 830         GEM_WARN_ON(kref_read(&ggtt->vm.resv_ref) != 1);
 831         dma_resv_fini(&ggtt->vm._resv);
 832 }
 833
 834 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
 835 {
 836         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
 837         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
 838         return snb_gmch_ctl << 20;
 839 }
 840
 841 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
 842 {
 843         bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
 844         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
 845         if (bdw_gmch_ctl)
 846                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
 847
 848 #ifdef CONFIG_X86_32
 849         /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */
 850         if (bdw_gmch_ctl > 4)
 851                 bdw_gmch_ctl = 4;
 852 #endif
 853
 854         return bdw_gmch_ctl << 20;
 855 }
 856
 857 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
 858 {
 859         gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
 860         gmch_ctrl &= SNB_GMCH_GGMS_MASK;
 861
 862         if (gmch_ctrl)
 863                 return 1 << (20 + gmch_ctrl);
 864
 865         return 0;
 866 }
 867
 868 static unsigned int gen6_gttmmadr_size(struct drm_i915_private *i915)
 869 {
 870         /*
 871          * GEN6: GTTMMADR size is 4MB and GTTADR starts at 2MB offset
 872          * GEN8: GTTMMADR size is 16MB and GTTADR starts at 8MB offset
 873          */
 874         GEM_BUG_ON(GRAPHICS_VER(i915) < 6);
 875         return (GRAPHICS_VER(i915) < 8) ? SZ_4M : SZ_16M;
 876 }
 877
 878 static unsigned int gen6_gttadr_offset(struct drm_i915_private *i915)
 879 {
 880         return gen6_gttmmadr_size(i915) / 2;
 881 }
 882
 883 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
 884 {
 885         struct drm_i915_private *i915 = ggtt->vm.i915;
 886         struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
 887         phys_addr_t phys_addr;
 888         u32 pte_flags;
 889         int ret;
 890
 891         GEM_WARN_ON(pci_resource_len(pdev, GEN4_GTTMMADR_BAR) != gen6_gttmmadr_size(i915));
 892         phys_addr = pci_resource_start(pdev, GEN4_GTTMMADR_BAR) + gen6_gttadr_offset(i915);
 893
 894         /*
 895          * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range
 896          * will be dropped. For WC mappings in general we have 64 byte burst
 897          * writes when the WC buffer is flushed, so we can't use it, but have to
 898          * resort to an uncached mapping. The WC issue is easily caught by the
 899          * readback check when writing GTT PTE entries.
 900          */
 901         if (IS_GEN9_LP(i915) || GRAPHICS_VER(i915) >= 11)
 902                 ggtt->gsm = ioremap(phys_addr, size);
 903         else
 904                 ggtt->gsm = ioremap_wc(phys_addr, size);
 905         if (!ggtt->gsm) {
 906                 drm_err(&i915->drm, "Failed to map the ggtt page table\n");
 907                 return -ENOMEM;
 908         }
 909
 910         kref_init(&ggtt->vm.resv_ref);
 911         ret = setup_scratch_page(&ggtt->vm);
 912         if (ret) {
 913                 drm_err(&i915->drm, "Scratch setup failed\n");
 914                 /* iounmap will also get called at remove, but meh */
 915                 iounmap(ggtt->gsm);
 916                 return ret;
 917         }
 918
 919         pte_flags = 0;
 920         if (i915_gem_object_is_lmem(ggtt->vm.scratch[0]))
 921                 pte_flags |= PTE_LM;
 922
 923         ggtt->vm.scratch[0]->encode =
 924                 ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]),
 925                                     i915_gem_get_pat_index(i915,
 926                                                            I915_CACHE_NONE),
 927                                     pte_flags);
 928
 929         return 0;
 930 }
 931
 932 static void gen6_gmch_remove(struct i915_address_space *vm)
 933 {
 934         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 935
 936         iounmap(ggtt->gsm);
 937         free_scratch(vm);
 938 }
 939
 940 static struct resource pci_resource(struct pci_dev *pdev, int bar)
 941 {
 942         return DEFINE_RES_MEM(pci_resource_start(pdev, bar),
 943                               pci_resource_len(pdev, bar));
 944 }
 945
 946 static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 947 {
 948         struct drm_i915_private *i915 = ggtt->vm.i915;
 949         struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
 950         unsigned int size;
 951         u16 snb_gmch_ctl;
 952
 953         if (!HAS_LMEM(i915) && !HAS_LMEMBAR_SMEM_STOLEN(i915)) {
 954                 if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR))
 955                         return -ENXIO;
 956
 957                 ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR);
 958                 ggtt->mappable_end = resource_size(&ggtt->gmadr);
 959         }
 960
 961         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 962         if (IS_CHERRYVIEW(i915))
 963                 size = chv_get_total_gtt_size(snb_gmch_ctl);
 964         else
 965                 size = gen8_get_total_gtt_size(snb_gmch_ctl);
 966
 967         ggtt->vm.alloc_pt_dma = alloc_pt_dma;
 968         ggtt->vm.alloc_scratch_dma = alloc_pt_dma;
 969         ggtt->vm.lmem_pt_obj_flags = I915_BO_ALLOC_PM_EARLY;
 970
 971         ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
 972         ggtt->vm.cleanup = gen6_gmch_remove;
 973         ggtt->vm.insert_page = gen8_ggtt_insert_page;
 974         ggtt->vm.clear_range = nop_clear_range;
 975         ggtt->vm.scratch_range = gen8_ggtt_clear_range;
 976
 977         ggtt->vm.insert_entries = gen8_ggtt_insert_entries;
 978
 979         /*
 980          * Serialize GTT updates with aperture access on BXT if VT-d is on,
 981          * and always on CHV.
 982          */
 983         if (intel_vm_no_concurrent_access_wa(i915)) {
 984                 ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL;
 985                 ggtt->vm.insert_page    = bxt_vtd_ggtt_insert_page__BKL;
 986
 987                 /*
 988                  * Calling stop_machine() version of GGTT update function
 989                  * at error capture/reset path will raise lockdep warning.
 990                  * Allow calling gen8_ggtt_insert_* directly at reset path
 991                  * which is safe from parallel GGTT updates.
 992                  */
 993                 ggtt->vm.raw_insert_page = gen8_ggtt_insert_page;
 994                 ggtt->vm.raw_insert_entries = gen8_ggtt_insert_entries;
 995
 996                 ggtt->vm.bind_async_flags =
 997                         I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
 998         }
 999
1000         if (intel_uc_wants_guc(&ggtt->vm.gt->uc))
1001                 ggtt->invalidate = guc_ggtt_invalidate;
1002         else
1003                 ggtt->invalidate = gen8_ggtt_invalidate;
1004
1005         ggtt->vm.vma_ops.bind_vma    = intel_ggtt_bind_vma;
1006         ggtt->vm.vma_ops.unbind_vma  = intel_ggtt_unbind_vma;
1007
1008         if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
1009                 ggtt->vm.pte_encode = mtl_ggtt_pte_encode;
1010         else
1011                 ggtt->vm.pte_encode = gen8_ggtt_pte_encode;
1012
1013         return ggtt_probe_common(ggtt, size);
1014 }
1015
1016 /*
1017  * For pre-gen8 platforms pat_index is the same as enum i915_cache_level,
1018  * so the switch-case statements in these PTE encode functions are still valid.
1019  * See translation table LEGACY_CACHELEVEL.
1020  */
1021 static u64 snb_pte_encode(dma_addr_t addr,
1022                           unsigned int pat_index,
1023                           u32 flags)
1024 {
1025         gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1026
1027         switch (pat_index) {
1028         case I915_CACHE_L3_LLC:
1029         case I915_CACHE_LLC:
1030                 pte |= GEN6_PTE_CACHE_LLC;
1031                 break;
1032         case I915_CACHE_NONE:
1033                 pte |= GEN6_PTE_UNCACHED;
1034                 break;
1035         default:
1036                 MISSING_CASE(pat_index);
1037         }
1038
1039         return pte;
1040 }
1041
1042 static u64 ivb_pte_encode(dma_addr_t addr,
1043                           unsigned int pat_index,
1044                           u32 flags)
1045 {
1046         gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1047
1048         switch (pat_index) {
1049         case I915_CACHE_L3_LLC:
1050                 pte |= GEN7_PTE_CACHE_L3_LLC;
1051                 break;
1052         case I915_CACHE_LLC:
1053                 pte |= GEN6_PTE_CACHE_LLC;
1054                 break;
1055         case I915_CACHE_NONE:
1056                 pte |= GEN6_PTE_UNCACHED;
1057                 break;
1058         default:
1059                 MISSING_CASE(pat_index);
1060         }
1061
1062         return pte;
1063 }
1064
1065 static u64 byt_pte_encode(dma_addr_t addr,
1066                           unsigned int pat_index,
1067                           u32 flags)
1068 {
1069         gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1070
1071         if (!(flags & PTE_READ_ONLY))
1072                 pte |= BYT_PTE_WRITEABLE;
1073
1074         if (pat_index != I915_CACHE_NONE)
1075                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
1076
1077         return pte;
1078 }
1079
1080 static u64 hsw_pte_encode(dma_addr_t addr,
1081                           unsigned int pat_index,
1082                           u32 flags)
1083 {
1084         gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1085
1086         if (pat_index != I915_CACHE_NONE)
1087                 pte |= HSW_WB_LLC_AGE3;
1088
1089         return pte;
1090 }
1091
1092 static u64 iris_pte_encode(dma_addr_t addr,
1093                            unsigned int pat_index,
1094                            u32 flags)
1095 {
1096         gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1097
1098         switch (pat_index) {
1099         case I915_CACHE_NONE:
1100                 break;
1101         case I915_CACHE_WT:
1102                 pte |= HSW_WT_ELLC_LLC_AGE3;
1103                 break;
1104         default:
1105                 pte |= HSW_WB_ELLC_LLC_AGE3;
1106                 break;
1107         }
1108
1109         return pte;
1110 }
1111
1112 static int gen6_gmch_probe(struct i915_ggtt *ggtt)
1113 {
1114         struct drm_i915_private *i915 = ggtt->vm.i915;
1115         struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
1116         unsigned int size;
1117         u16 snb_gmch_ctl;
1118
1119         if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR))
1120                 return -ENXIO;
1121
1122         ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR);
1123         ggtt->mappable_end = resource_size(&ggtt->gmadr);
1124
1125         /*
1126          * 64/512MB is the current min/max we actually know of, but this is
1127          * just a coarse sanity check.
1128          */
1129         if (ggtt->mappable_end < (64 << 20) ||
1130             ggtt->mappable_end > (512 << 20)) {
1131                 drm_err(&i915->drm, "Unknown GMADR size (%pa)\n",
1132                         &ggtt->mappable_end);
1133                 return -ENXIO;
1134         }
1135
1136         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
1137
1138         size = gen6_get_total_gtt_size(snb_gmch_ctl);
1139         ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE;
1140
1141         ggtt->vm.alloc_pt_dma = alloc_pt_dma;
1142         ggtt->vm.alloc_scratch_dma = alloc_pt_dma;
1143
1144         ggtt->vm.clear_range = nop_clear_range;
1145         if (!HAS_FULL_PPGTT(i915))
1146                 ggtt->vm.clear_range = gen6_ggtt_clear_range;
1147         ggtt->vm.scratch_range = gen6_ggtt_clear_range;
1148         ggtt->vm.insert_page = gen6_ggtt_insert_page;
1149         ggtt->vm.insert_entries = gen6_ggtt_insert_entries;
1150         ggtt->vm.cleanup = gen6_gmch_remove;
1151
1152         ggtt->invalidate = gen6_ggtt_invalidate;
1153
1154         if (HAS_EDRAM(i915))
1155                 ggtt->vm.pte_encode = iris_pte_encode;
1156         else if (IS_HASWELL(i915))
1157                 ggtt->vm.pte_encode = hsw_pte_encode;
1158         else if (IS_VALLEYVIEW(i915))
1159                 ggtt->vm.pte_encode = byt_pte_encode;
1160         else if (GRAPHICS_VER(i915) >= 7)
1161                 ggtt->vm.pte_encode = ivb_pte_encode;
1162         else
1163                 ggtt->vm.pte_encode = snb_pte_encode;
1164
1165         ggtt->vm.vma_ops.bind_vma    = intel_ggtt_bind_vma;
1166         ggtt->vm.vma_ops.unbind_vma  = intel_ggtt_unbind_vma;
1167
1168         return ggtt_probe_common(ggtt, size);
1169 }
1170
1171 static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt)
1172 {
1173         struct drm_i915_private *i915 = gt->i915;
1174         int ret;
1175
1176         ggtt->vm.gt = gt;
1177         ggtt->vm.i915 = i915;
1178         ggtt->vm.dma = i915->drm.dev;
1179         dma_resv_init(&ggtt->vm._resv);
1180
1181         if (GRAPHICS_VER(i915) >= 8)
1182                 ret = gen8_gmch_probe(ggtt);
1183         else if (GRAPHICS_VER(i915) >= 6)
1184                 ret = gen6_gmch_probe(ggtt);
1185         else
1186                 ret = intel_ggtt_gmch_probe(ggtt);
1187
1188         if (ret) {
1189                 dma_resv_fini(&ggtt->vm._resv);
1190                 return ret;
1191         }
1192
1193         if ((ggtt->vm.total - 1) >> 32) {
1194                 drm_err(&i915->drm,
1195                         "We never expected a Global GTT with more than 32bits"
1196                         " of address space! Found %lldM!\n",
1197                         ggtt->vm.total >> 20);
1198                 ggtt->vm.total = 1ULL << 32;
1199                 ggtt->mappable_end =
1200                         min_t(u64, ggtt->mappable_end, ggtt->vm.total);
1201         }
1202
1203         if (ggtt->mappable_end > ggtt->vm.total) {
1204                 drm_err(&i915->drm,
1205                         "mappable aperture extends past end of GGTT,"
1206                         " aperture=%pa, total=%llx\n",
1207                         &ggtt->mappable_end, ggtt->vm.total);
1208                 ggtt->mappable_end = ggtt->vm.total;
1209         }
1210
1211         /* GMADR is the PCI mmio aperture into the global GTT. */
1212         drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20);
1213         drm_dbg(&i915->drm, "GMADR size = %lluM\n",
1214                 (u64)ggtt->mappable_end >> 20);
1215         drm_dbg(&i915->drm, "DSM size = %lluM\n",
1216                 (u64)resource_size(&intel_graphics_stolen_res) >> 20);
1217
1218         return 0;
1219 }
1220
1221 /**
1222  * i915_ggtt_probe_hw - Probe GGTT hardware location
1223  * @i915: i915 device
1224  */
1225 int i915_ggtt_probe_hw(struct drm_i915_private *i915)
1226 {
1227         struct intel_gt *gt;
1228         int ret, i;
1229
1230         for_each_gt(gt, i915, i) {
1231                 ret = intel_gt_assign_ggtt(gt);
1232                 if (ret)
1233                         return ret;
1234         }
1235
1236         ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915));
1237         if (ret)
1238                 return ret;
1239
1240         if (i915_vtd_active(i915))
1241                 drm_info(&i915->drm, "VT-d active for gfx access\n");
1242
1243         return 0;
1244 }
1245
1246 struct i915_ggtt *i915_ggtt_create(struct drm_i915_private *i915)
1247 {
1248         struct i915_ggtt *ggtt;
1249
1250         ggtt = drmm_kzalloc(&i915->drm, sizeof(*ggtt), GFP_KERNEL);
1251         if (!ggtt)
1252                 return ERR_PTR(-ENOMEM);
1253
1254         INIT_LIST_HEAD(&ggtt->gt_list);
1255
1256         return ggtt;
1257 }
1258
1259 int i915_ggtt_enable_hw(struct drm_i915_private *i915)
1260 {
1261         if (GRAPHICS_VER(i915) < 6)
1262                 return intel_ggtt_gmch_enable_hw(i915);
1263
1264         return 0;
1265 }
1266
1267 /**
1268  * i915_ggtt_resume_vm - Restore the memory mappings for a GGTT or DPT VM
1269  * @vm: The VM to restore the mappings for
1270  *
1271  * Restore the memory mappings for all objects mapped to HW via the GGTT or a
1272  * DPT page table.
1273  *
1274  * Returns %true if restoring the mapping for any object that was in a write
1275  * domain before suspend.
1276  */
1277 bool i915_ggtt_resume_vm(struct i915_address_space *vm)
1278 {
1279         struct i915_vma *vma;
1280         bool write_domain_objs = false;
1281
1282         drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt);
1283
1284         /* First fill our portion of the GTT with scratch pages */
1285         vm->clear_range(vm, 0, vm->total);
1286
1287         /* clflush objects bound into the GGTT and rebind them. */
1288         list_for_each_entry(vma, &vm->bound_list, vm_link) {
1289                 struct drm_i915_gem_object *obj = vma->obj;
1290                 unsigned int was_bound =
1291                         atomic_read(&vma->flags) & I915_VMA_BIND_MASK;
1292
1293                 GEM_BUG_ON(!was_bound);
1294
1295                 /*
1296                  * Clear the bound flags of the vma resource to allow
1297                  * ptes to be repopulated.
1298                  */
1299                 vma->resource->bound_flags = 0;
1300                 vma->ops->bind_vma(vm, NULL, vma->resource,
1301                                    obj ? obj->pat_index :
1302                                          i915_gem_get_pat_index(vm->i915,
1303                                                                 I915_CACHE_NONE),
1304                                    was_bound);
1305
1306                 if (obj) { /* only used during resume => exclusive access */
1307                         write_domain_objs |= fetch_and_zero(&obj->write_domain);
1308                         obj->read_domains |= I915_GEM_DOMAIN_GTT;
1309                 }
1310         }
1311
1312         return write_domain_objs;
1313 }
1314
1315 void i915_ggtt_resume(struct i915_ggtt *ggtt)
1316 {
1317         struct intel_gt *gt;
1318         bool flush;
1319
1320         list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
1321                 intel_gt_check_and_clear_faults(gt);
1322
1323         flush = i915_ggtt_resume_vm(&ggtt->vm);
1324
1325         if (drm_mm_node_allocated(&ggtt->error_capture))
1326                 ggtt->vm.scratch_range(&ggtt->vm, ggtt->error_capture.start,
1327                                        ggtt->error_capture.size);
1328
1329         list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
1330                 intel_uc_resume_mappings(&gt->uc);
1331
1332         ggtt->invalidate(ggtt);
1333
1334         if (flush)
1335                 wbinvd_on_all_cpus();
1336
1337         intel_ggtt_restore_fences(ggtt);
1338 }