drivers/gpu/drm/i915/selftests/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/list_sort.h>
  26 #include <linux/prime_numbers.h>
  27
  28 #include "gem/i915_gem_context.h"
  29 #include "gem/selftests/mock_context.h"
  30 #include "gt/intel_context.h"
  31 #include "gt/intel_gpu_commands.h"
  32
  33 #include "i915_random.h"
  34 #include "i915_selftest.h"
  35
  36 #include "mock_drm.h"
  37 #include "mock_gem_device.h"
  38 #include "mock_gtt.h"
  39 #include "igt_flush_test.h"
  40
  41 static void cleanup_freed_objects(struct drm_i915_private *i915)
  42 {
  43         i915_gem_drain_freed_objects(i915);
  44 }
  45
  46 static void fake_free_pages(struct drm_i915_gem_object *obj,
  47                             struct sg_table *pages)
  48 {
  49         sg_free_table(pages);
  50         kfree(pages);
  51 }
  52
  53 static int fake_get_pages(struct drm_i915_gem_object *obj)
  54 {
  55 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
  56 #define PFN_BIAS 0x1000
  57         struct sg_table *pages;
  58         struct scatterlist *sg;
  59         unsigned int sg_page_sizes;
  60         typeof(obj->base.size) rem;
  61
  62         pages = kmalloc(sizeof(*pages), GFP);
  63         if (!pages)
  64                 return -ENOMEM;
  65
  66         rem = round_up(obj->base.size, BIT(31)) >> 31;
  67         if (sg_alloc_table(pages, rem, GFP)) {
  68                 kfree(pages);
  69                 return -ENOMEM;
  70         }
  71
  72         sg_page_sizes = 0;
  73         rem = obj->base.size;
  74         for (sg = pages->sgl; sg; sg = sg_next(sg)) {
  75                 unsigned long len = min_t(typeof(rem), rem, BIT(31));
  76
  77                 GEM_BUG_ON(!len);
  78                 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0);
  79                 sg_dma_address(sg) = page_to_phys(sg_page(sg));
  80                 sg_dma_len(sg) = len;
  81                 sg_page_sizes |= len;
  82
  83                 rem -= len;
  84         }
  85         GEM_BUG_ON(rem);
  86
  87         __i915_gem_object_set_pages(obj, pages, sg_page_sizes);
  88
  89         return 0;
  90 #undef GFP
  91 }
  92
  93 static void fake_put_pages(struct drm_i915_gem_object *obj,
  94                            struct sg_table *pages)
  95 {
  96         fake_free_pages(obj, pages);
  97         obj->mm.dirty = false;
  98 }
  99
 100 static const struct drm_i915_gem_object_ops fake_ops = {
 101         .name = "fake-gem",
 102         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 103         .get_pages = fake_get_pages,
 104         .put_pages = fake_put_pages,
 105 };
 106
 107 static struct drm_i915_gem_object *
 108 fake_dma_object(struct drm_i915_private *i915, u64 size)
 109 {
 110         static struct lock_class_key lock_class;
 111         struct drm_i915_gem_object *obj;
 112
 113         GEM_BUG_ON(!size);
 114         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 115
 116         if (overflows_type(size, obj->base.size))
 117                 return ERR_PTR(-E2BIG);
 118
 119         obj = i915_gem_object_alloc();
 120         if (!obj)
 121                 goto err;
 122
 123         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 124         i915_gem_object_init(obj, &fake_ops, &lock_class, 0);
 125
 126         i915_gem_object_set_volatile(obj);
 127
 128         obj->write_domain = I915_GEM_DOMAIN_CPU;
 129         obj->read_domains = I915_GEM_DOMAIN_CPU;
 130         obj->cache_level = I915_CACHE_NONE;
 131
 132         /* Preallocate the "backing storage" */
 133         if (i915_gem_object_pin_pages_unlocked(obj))
 134                 goto err_obj;
 135
 136         i915_gem_object_unpin_pages(obj);
 137         return obj;
 138
 139 err_obj:
 140         i915_gem_object_put(obj);
 141 err:
 142         return ERR_PTR(-ENOMEM);
 143 }
 144
 145 static int igt_ppgtt_alloc(void *arg)
 146 {
 147         struct drm_i915_private *dev_priv = arg;
 148         struct i915_ppgtt *ppgtt;
 149         struct i915_gem_ww_ctx ww;
 150         u64 size, last, limit;
 151         int err = 0;
 152
 153         /* Allocate a ppggt and try to fill the entire range */
 154
 155         if (!HAS_PPGTT(dev_priv))
 156                 return 0;
 157
 158         ppgtt = i915_ppgtt_create(&dev_priv->gt);
 159         if (IS_ERR(ppgtt))
 160                 return PTR_ERR(ppgtt);
 161
 162         if (!ppgtt->vm.allocate_va_range)
 163                 goto err_ppgtt_cleanup;
 164
 165         /*
 166          * While we only allocate the page tables here and so we could
 167          * address a much larger GTT than we could actually fit into
 168          * RAM, a practical limit is the amount of physical pages in the system.
 169          * This should ensure that we do not run into the oomkiller during
 170          * the test and take down the machine wilfully.
 171          */
 172         limit = totalram_pages() << PAGE_SHIFT;
 173         limit = min(ppgtt->vm.total, limit);
 174
 175         i915_gem_ww_ctx_init(&ww, false);
 176 retry:
 177         err = i915_vm_lock_objects(&ppgtt->vm, &ww);
 178         if (err)
 179                 goto err_ppgtt_cleanup;
 180
 181         /* Check we can allocate the entire range */
 182         for (size = 4096; size <= limit; size <<= 2) {
 183                 struct i915_vm_pt_stash stash = {};
 184
 185                 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size);
 186                 if (err)
 187                         goto err_ppgtt_cleanup;
 188
 189                 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
 190                 if (err) {
 191                         i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 192                         goto err_ppgtt_cleanup;
 193                 }
 194
 195                 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, size);
 196                 cond_resched();
 197
 198                 ppgtt->vm.clear_range(&ppgtt->vm, 0, size);
 199
 200                 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 201         }
 202
 203         /* Check we can incrementally allocate the entire range */
 204         for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) {
 205                 struct i915_vm_pt_stash stash = {};
 206
 207                 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size - last);
 208                 if (err)
 209                         goto err_ppgtt_cleanup;
 210
 211                 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
 212                 if (err) {
 213                         i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 214                         goto err_ppgtt_cleanup;
 215                 }
 216
 217                 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash,
 218                                             last, size - last);
 219                 cond_resched();
 220
 221                 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 222         }
 223
 224 err_ppgtt_cleanup:
 225         if (err == -EDEADLK) {
 226                 err = i915_gem_ww_ctx_backoff(&ww);
 227                 if (!err)
 228                         goto retry;
 229         }
 230         i915_gem_ww_ctx_fini(&ww);
 231
 232         i915_vm_put(&ppgtt->vm);
 233         return err;
 234 }
 235
 236 static int lowlevel_hole(struct i915_address_space *vm,
 237                          u64 hole_start, u64 hole_end,
 238                          unsigned long end_time)
 239 {
 240         I915_RND_STATE(seed_prng);
 241         struct i915_vma *mock_vma;
 242         unsigned int size;
 243
 244         mock_vma = kzalloc(sizeof(*mock_vma), GFP_KERNEL);
 245         if (!mock_vma)
 246                 return -ENOMEM;
 247
 248         /* Keep creating larger objects until one cannot fit into the hole */
 249         for (size = 12; (hole_end - hole_start) >> size; size++) {
 250                 I915_RND_SUBSTATE(prng, seed_prng);
 251                 struct drm_i915_gem_object *obj;
 252                 unsigned int *order, count, n;
 253                 u64 hole_size;
 254
 255                 hole_size = (hole_end - hole_start) >> size;
 256                 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
 257                         hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
 258                 count = hole_size >> 1;
 259                 if (!count) {
 260                         pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
 261                                  __func__, hole_start, hole_end, size, hole_size);
 262                         break;
 263                 }
 264
 265                 do {
 266                         order = i915_random_order(count, &prng);
 267                         if (order)
 268                                 break;
 269                 } while (count >>= 1);
 270                 if (!count) {
 271                         kfree(mock_vma);
 272                         return -ENOMEM;
 273                 }
 274                 GEM_BUG_ON(!order);
 275
 276                 GEM_BUG_ON(count * BIT_ULL(size) > vm->total);
 277                 GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end);
 278
 279                 /* Ignore allocation failures (i.e. don't report them as
 280                  * a test failure) as we are purposefully allocating very
 281                  * large objects without checking that we have sufficient
 282                  * memory. We expect to hit -ENOMEM.
 283                  */
 284
 285                 obj = fake_dma_object(vm->i915, BIT_ULL(size));
 286                 if (IS_ERR(obj)) {
 287                         kfree(order);
 288                         break;
 289                 }
 290
 291                 GEM_BUG_ON(obj->base.size != BIT_ULL(size));
 292
 293                 if (i915_gem_object_pin_pages_unlocked(obj)) {
 294                         i915_gem_object_put(obj);
 295                         kfree(order);
 296                         break;
 297                 }
 298
 299                 for (n = 0; n < count; n++) {
 300                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 301                         intel_wakeref_t wakeref;
 302
 303                         GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
 304
 305                         if (igt_timeout(end_time,
 306                                         "%s timed out before %d/%d\n",
 307                                         __func__, n, count)) {
 308                                 hole_end = hole_start; /* quit */
 309                                 break;
 310                         }
 311
 312                         if (vm->allocate_va_range) {
 313                                 struct i915_vm_pt_stash stash = {};
 314                                 struct i915_gem_ww_ctx ww;
 315                                 int err;
 316
 317                                 i915_gem_ww_ctx_init(&ww, false);
 318 retry:
 319                                 err = i915_vm_lock_objects(vm, &ww);
 320                                 if (err)
 321                                         goto alloc_vm_end;
 322
 323                                 err = -ENOMEM;
 324                                 if (i915_vm_alloc_pt_stash(vm, &stash,
 325                                                            BIT_ULL(size)))
 326                                         goto alloc_vm_end;
 327
 328                                 err = i915_vm_map_pt_stash(vm, &stash);
 329                                 if (!err)
 330                                         vm->allocate_va_range(vm, &stash,
 331                                                               addr, BIT_ULL(size));
 332                                 i915_vm_free_pt_stash(vm, &stash);
 333 alloc_vm_end:
 334                                 if (err == -EDEADLK) {
 335                                         err = i915_gem_ww_ctx_backoff(&ww);
 336                                         if (!err)
 337                                                 goto retry;
 338                                 }
 339                                 i915_gem_ww_ctx_fini(&ww);
 340
 341                                 if (err)
 342                                         break;
 343                         }
 344
 345                         mock_vma->pages = obj->mm.pages;
 346                         mock_vma->node.size = BIT_ULL(size);
 347                         mock_vma->node.start = addr;
 348
 349                         with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
 350                                 vm->insert_entries(vm, mock_vma,
 351                                                    I915_CACHE_NONE, 0);
 352                 }
 353                 count = n;
 354
 355                 i915_random_reorder(order, count, &prng);
 356                 for (n = 0; n < count; n++) {
 357                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 358                         intel_wakeref_t wakeref;
 359
 360                         GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
 361                         with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
 362                                 vm->clear_range(vm, addr, BIT_ULL(size));
 363                 }
 364
 365                 i915_gem_object_unpin_pages(obj);
 366                 i915_gem_object_put(obj);
 367
 368                 kfree(order);
 369
 370                 cleanup_freed_objects(vm->i915);
 371         }
 372
 373         kfree(mock_vma);
 374         return 0;
 375 }
 376
 377 static void close_object_list(struct list_head *objects,
 378                               struct i915_address_space *vm)
 379 {
 380         struct drm_i915_gem_object *obj, *on;
 381         int ignored;
 382
 383         list_for_each_entry_safe(obj, on, objects, st_link) {
 384                 struct i915_vma *vma;
 385
 386                 vma = i915_vma_instance(obj, vm, NULL);
 387                 if (!IS_ERR(vma))
 388                         ignored = i915_vma_unbind(vma);
 389
 390                 list_del(&obj->st_link);
 391                 i915_gem_object_put(obj);
 392         }
 393 }
 394
 395 static int fill_hole(struct i915_address_space *vm,
 396                      u64 hole_start, u64 hole_end,
 397                      unsigned long end_time)
 398 {
 399         const u64 hole_size = hole_end - hole_start;
 400         struct drm_i915_gem_object *obj;
 401         const unsigned long max_pages =
 402                 min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT);
 403         const unsigned long max_step = max(int_sqrt(max_pages), 2UL);
 404         unsigned long npages, prime, flags;
 405         struct i915_vma *vma;
 406         LIST_HEAD(objects);
 407         int err;
 408
 409         /* Try binding many VMA working inwards from either edge */
 410
 411         flags = PIN_OFFSET_FIXED | PIN_USER;
 412         if (i915_is_ggtt(vm))
 413                 flags |= PIN_GLOBAL;
 414
 415         for_each_prime_number_from(prime, 2, max_step) {
 416                 for (npages = 1; npages <= max_pages; npages *= prime) {
 417                         const u64 full_size = npages << PAGE_SHIFT;
 418                         const struct {
 419                                 const char *name;
 420                                 u64 offset;
 421                                 int step;
 422                         } phases[] = {
 423                                 { "top-down", hole_end, -1, },
 424                                 { "bottom-up", hole_start, 1, },
 425                                 { }
 426                         }, *p;
 427
 428                         obj = fake_dma_object(vm->i915, full_size);
 429                         if (IS_ERR(obj))
 430                                 break;
 431
 432                         list_add(&obj->st_link, &objects);
 433
 434                         /* Align differing sized objects against the edges, and
 435                          * check we don't walk off into the void when binding
 436                          * them into the GTT.
 437                          */
 438                         for (p = phases; p->name; p++) {
 439                                 u64 offset;
 440
 441                                 offset = p->offset;
 442                                 list_for_each_entry(obj, &objects, st_link) {
 443                                         vma = i915_vma_instance(obj, vm, NULL);
 444                                         if (IS_ERR(vma))
 445                                                 continue;
 446
 447                                         if (p->step < 0) {
 448                                                 if (offset < hole_start + obj->base.size)
 449                                                         break;
 450                                                 offset -= obj->base.size;
 451                                         }
 452
 453                                         err = i915_vma_pin(vma, 0, 0, offset | flags);
 454                                         if (err) {
 455                                                 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
 456                                                        __func__, p->name, err, npages, prime, offset);
 457                                                 goto err;
 458                                         }
 459
 460                                         if (!drm_mm_node_allocated(&vma->node) ||
 461                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 462                                                 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 463                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 464                                                        offset);
 465                                                 err = -EINVAL;
 466                                                 goto err;
 467                                         }
 468
 469                                         i915_vma_unpin(vma);
 470
 471                                         if (p->step > 0) {
 472                                                 if (offset + obj->base.size > hole_end)
 473                                                         break;
 474                                                 offset += obj->base.size;
 475                                         }
 476                                 }
 477
 478                                 offset = p->offset;
 479                                 list_for_each_entry(obj, &objects, st_link) {
 480                                         vma = i915_vma_instance(obj, vm, NULL);
 481                                         if (IS_ERR(vma))
 482                                                 continue;
 483
 484                                         if (p->step < 0) {
 485                                                 if (offset < hole_start + obj->base.size)
 486                                                         break;
 487                                                 offset -= obj->base.size;
 488                                         }
 489
 490                                         if (!drm_mm_node_allocated(&vma->node) ||
 491                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 492                                                 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n",
 493                                                        __func__, p->name, vma->node.start, vma->node.size,
 494                                                        offset);
 495                                                 err = -EINVAL;
 496                                                 goto err;
 497                                         }
 498
 499                                         err = i915_vma_unbind(vma);
 500                                         if (err) {
 501                                                 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n",
 502                                                        __func__, p->name, vma->node.start, vma->node.size,
 503                                                        err);
 504                                                 goto err;
 505                                         }
 506
 507                                         if (p->step > 0) {
 508                                                 if (offset + obj->base.size > hole_end)
 509                                                         break;
 510                                                 offset += obj->base.size;
 511                                         }
 512                                 }
 513
 514                                 offset = p->offset;
 515                                 list_for_each_entry_reverse(obj, &objects, st_link) {
 516                                         vma = i915_vma_instance(obj, vm, NULL);
 517                                         if (IS_ERR(vma))
 518                                                 continue;
 519
 520                                         if (p->step < 0) {
 521                                                 if (offset < hole_start + obj->base.size)
 522                                                         break;
 523                                                 offset -= obj->base.size;
 524                                         }
 525
 526                                         err = i915_vma_pin(vma, 0, 0, offset | flags);
 527                                         if (err) {
 528                                                 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
 529                                                        __func__, p->name, err, npages, prime, offset);
 530                                                 goto err;
 531                                         }
 532
 533                                         if (!drm_mm_node_allocated(&vma->node) ||
 534                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 535                                                 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 536                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 537                                                        offset);
 538                                                 err = -EINVAL;
 539                                                 goto err;
 540                                         }
 541
 542                                         i915_vma_unpin(vma);
 543
 544                                         if (p->step > 0) {
 545                                                 if (offset + obj->base.size > hole_end)
 546                                                         break;
 547                                                 offset += obj->base.size;
 548                                         }
 549                                 }
 550
 551                                 offset = p->offset;
 552                                 list_for_each_entry_reverse(obj, &objects, st_link) {
 553                                         vma = i915_vma_instance(obj, vm, NULL);
 554                                         if (IS_ERR(vma))
 555                                                 continue;
 556
 557                                         if (p->step < 0) {
 558                                                 if (offset < hole_start + obj->base.size)
 559                                                         break;
 560                                                 offset -= obj->base.size;
 561                                         }
 562
 563                                         if (!drm_mm_node_allocated(&vma->node) ||
 564                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 565                                                 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 566                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 567                                                        offset);
 568                                                 err = -EINVAL;
 569                                                 goto err;
 570                                         }
 571
 572                                         err = i915_vma_unbind(vma);
 573                                         if (err) {
 574                                                 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n",
 575                                                        __func__, p->name, vma->node.start, vma->node.size,
 576                                                        err);
 577                                                 goto err;
 578                                         }
 579
 580                                         if (p->step > 0) {
 581                                                 if (offset + obj->base.size > hole_end)
 582                                                         break;
 583                                                 offset += obj->base.size;
 584                                         }
 585                                 }
 586                         }
 587
 588                         if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n",
 589                                         __func__, npages, prime)) {
 590                                 err = -EINTR;
 591                                 goto err;
 592                         }
 593                 }
 594
 595                 close_object_list(&objects, vm);
 596                 cleanup_freed_objects(vm->i915);
 597         }
 598
 599         return 0;
 600
 601 err:
 602         close_object_list(&objects, vm);
 603         return err;
 604 }
 605
 606 static int walk_hole(struct i915_address_space *vm,
 607                      u64 hole_start, u64 hole_end,
 608                      unsigned long end_time)
 609 {
 610         const u64 hole_size = hole_end - hole_start;
 611         const unsigned long max_pages =
 612                 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT);
 613         unsigned long flags;
 614         u64 size;
 615
 616         /* Try binding a single VMA in different positions within the hole */
 617
 618         flags = PIN_OFFSET_FIXED | PIN_USER;
 619         if (i915_is_ggtt(vm))
 620                 flags |= PIN_GLOBAL;
 621
 622         for_each_prime_number_from(size, 1, max_pages) {
 623                 struct drm_i915_gem_object *obj;
 624                 struct i915_vma *vma;
 625                 u64 addr;
 626                 int err = 0;
 627
 628                 obj = fake_dma_object(vm->i915, size << PAGE_SHIFT);
 629                 if (IS_ERR(obj))
 630                         break;
 631
 632                 vma = i915_vma_instance(obj, vm, NULL);
 633                 if (IS_ERR(vma)) {
 634                         err = PTR_ERR(vma);
 635                         goto err_put;
 636                 }
 637
 638                 for (addr = hole_start;
 639                      addr + obj->base.size < hole_end;
 640                      addr += obj->base.size) {
 641                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 642                         if (err) {
 643                                 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n",
 644                                        __func__, addr, vma->size,
 645                                        hole_start, hole_end, err);
 646                                 goto err_put;
 647                         }
 648                         i915_vma_unpin(vma);
 649
 650                         if (!drm_mm_node_allocated(&vma->node) ||
 651                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 652                                 pr_err("%s incorrect at %llx + %llx\n",
 653                                        __func__, addr, vma->size);
 654                                 err = -EINVAL;
 655                                 goto err_put;
 656                         }
 657
 658                         err = i915_vma_unbind(vma);
 659                         if (err) {
 660                                 pr_err("%s unbind failed at %llx + %llx  with err=%d\n",
 661                                        __func__, addr, vma->size, err);
 662                                 goto err_put;
 663                         }
 664
 665                         GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
 666
 667                         if (igt_timeout(end_time,
 668                                         "%s timed out at %llx\n",
 669                                         __func__, addr)) {
 670                                 err = -EINTR;
 671                                 goto err_put;
 672                         }
 673                 }
 674
 675 err_put:
 676                 i915_gem_object_put(obj);
 677                 if (err)
 678                         return err;
 679
 680                 cleanup_freed_objects(vm->i915);
 681         }
 682
 683         return 0;
 684 }
 685
 686 static int pot_hole(struct i915_address_space *vm,
 687                     u64 hole_start, u64 hole_end,
 688                     unsigned long end_time)
 689 {
 690         struct drm_i915_gem_object *obj;
 691         struct i915_vma *vma;
 692         unsigned long flags;
 693         unsigned int pot;
 694         int err = 0;
 695
 696         flags = PIN_OFFSET_FIXED | PIN_USER;
 697         if (i915_is_ggtt(vm))
 698                 flags |= PIN_GLOBAL;
 699
 700         obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE);
 701         if (IS_ERR(obj))
 702                 return PTR_ERR(obj);
 703
 704         vma = i915_vma_instance(obj, vm, NULL);
 705         if (IS_ERR(vma)) {
 706                 err = PTR_ERR(vma);
 707                 goto err_obj;
 708         }
 709
 710         /* Insert a pair of pages across every pot boundary within the hole */
 711         for (pot = fls64(hole_end - 1) - 1;
 712              pot > ilog2(2 * I915_GTT_PAGE_SIZE);
 713              pot--) {
 714                 u64 step = BIT_ULL(pot);
 715                 u64 addr;
 716
 717                 for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
 718                      addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
 719                      addr += step) {
 720                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 721                         if (err) {
 722                                 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n",
 723                                        __func__,
 724                                        addr,
 725                                        hole_start, hole_end,
 726                                        err);
 727                                 goto err_obj;
 728                         }
 729
 730                         if (!drm_mm_node_allocated(&vma->node) ||
 731                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 732                                 pr_err("%s incorrect at %llx + %llx\n",
 733                                        __func__, addr, vma->size);
 734                                 i915_vma_unpin(vma);
 735                                 err = i915_vma_unbind(vma);
 736                                 err = -EINVAL;
 737                                 goto err_obj;
 738                         }
 739
 740                         i915_vma_unpin(vma);
 741                         err = i915_vma_unbind(vma);
 742                         GEM_BUG_ON(err);
 743                 }
 744
 745                 if (igt_timeout(end_time,
 746                                 "%s timed out after %d/%d\n",
 747                                 __func__, pot, fls64(hole_end - 1) - 1)) {
 748                         err = -EINTR;
 749                         goto err_obj;
 750                 }
 751         }
 752
 753 err_obj:
 754         i915_gem_object_put(obj);
 755         return err;
 756 }
 757
 758 static int drunk_hole(struct i915_address_space *vm,
 759                       u64 hole_start, u64 hole_end,
 760                       unsigned long end_time)
 761 {
 762         I915_RND_STATE(prng);
 763         unsigned int size;
 764         unsigned long flags;
 765
 766         flags = PIN_OFFSET_FIXED | PIN_USER;
 767         if (i915_is_ggtt(vm))
 768                 flags |= PIN_GLOBAL;
 769
 770         /* Keep creating larger objects until one cannot fit into the hole */
 771         for (size = 12; (hole_end - hole_start) >> size; size++) {
 772                 struct drm_i915_gem_object *obj;
 773                 unsigned int *order, count, n;
 774                 struct i915_vma *vma;
 775                 u64 hole_size;
 776                 int err = -ENODEV;
 777
 778                 hole_size = (hole_end - hole_start) >> size;
 779                 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
 780                         hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
 781                 count = hole_size >> 1;
 782                 if (!count) {
 783                         pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
 784                                  __func__, hole_start, hole_end, size, hole_size);
 785                         break;
 786                 }
 787
 788                 do {
 789                         order = i915_random_order(count, &prng);
 790                         if (order)
 791                                 break;
 792                 } while (count >>= 1);
 793                 if (!count)
 794                         return -ENOMEM;
 795                 GEM_BUG_ON(!order);
 796
 797                 /* Ignore allocation failures (i.e. don't report them as
 798                  * a test failure) as we are purposefully allocating very
 799                  * large objects without checking that we have sufficient
 800                  * memory. We expect to hit -ENOMEM.
 801                  */
 802
 803                 obj = fake_dma_object(vm->i915, BIT_ULL(size));
 804                 if (IS_ERR(obj)) {
 805                         kfree(order);
 806                         break;
 807                 }
 808
 809                 vma = i915_vma_instance(obj, vm, NULL);
 810                 if (IS_ERR(vma)) {
 811                         err = PTR_ERR(vma);
 812                         goto err_obj;
 813                 }
 814
 815                 GEM_BUG_ON(vma->size != BIT_ULL(size));
 816
 817                 for (n = 0; n < count; n++) {
 818                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 819
 820                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 821                         if (err) {
 822                                 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
 823                                        __func__,
 824                                        addr, BIT_ULL(size),
 825                                        hole_start, hole_end,
 826                                        err);
 827                                 goto err_obj;
 828                         }
 829
 830                         if (!drm_mm_node_allocated(&vma->node) ||
 831                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 832                                 pr_err("%s incorrect at %llx + %llx\n",
 833                                        __func__, addr, BIT_ULL(size));
 834                                 i915_vma_unpin(vma);
 835                                 err = i915_vma_unbind(vma);
 836                                 err = -EINVAL;
 837                                 goto err_obj;
 838                         }
 839
 840                         i915_vma_unpin(vma);
 841                         err = i915_vma_unbind(vma);
 842                         GEM_BUG_ON(err);
 843
 844                         if (igt_timeout(end_time,
 845                                         "%s timed out after %d/%d\n",
 846                                         __func__, n, count)) {
 847                                 err = -EINTR;
 848                                 goto err_obj;
 849                         }
 850                 }
 851
 852 err_obj:
 853                 i915_gem_object_put(obj);
 854                 kfree(order);
 855                 if (err)
 856                         return err;
 857
 858                 cleanup_freed_objects(vm->i915);
 859         }
 860
 861         return 0;
 862 }
 863
 864 static int __shrink_hole(struct i915_address_space *vm,
 865                          u64 hole_start, u64 hole_end,
 866                          unsigned long end_time)
 867 {
 868         struct drm_i915_gem_object *obj;
 869         unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
 870         unsigned int order = 12;
 871         LIST_HEAD(objects);
 872         int err = 0;
 873         u64 addr;
 874
 875         /* Keep creating larger objects until one cannot fit into the hole */
 876         for (addr = hole_start; addr < hole_end; ) {
 877                 struct i915_vma *vma;
 878                 u64 size = BIT_ULL(order++);
 879
 880                 size = min(size, hole_end - addr);
 881                 obj = fake_dma_object(vm->i915, size);
 882                 if (IS_ERR(obj)) {
 883                         err = PTR_ERR(obj);
 884                         break;
 885                 }
 886
 887                 list_add(&obj->st_link, &objects);
 888
 889                 vma = i915_vma_instance(obj, vm, NULL);
 890                 if (IS_ERR(vma)) {
 891                         err = PTR_ERR(vma);
 892                         break;
 893                 }
 894
 895                 GEM_BUG_ON(vma->size != size);
 896
 897                 err = i915_vma_pin(vma, 0, 0, addr | flags);
 898                 if (err) {
 899                         pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
 900                                __func__, addr, size, hole_start, hole_end, err);
 901                         break;
 902                 }
 903
 904                 if (!drm_mm_node_allocated(&vma->node) ||
 905                     i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 906                         pr_err("%s incorrect at %llx + %llx\n",
 907                                __func__, addr, size);
 908                         i915_vma_unpin(vma);
 909                         err = i915_vma_unbind(vma);
 910                         err = -EINVAL;
 911                         break;
 912                 }
 913
 914                 i915_vma_unpin(vma);
 915                 addr += size;
 916
 917                 /*
 918                  * Since we are injecting allocation faults at random intervals,
 919                  * wait for this allocation to complete before we change the
 920                  * faultinjection.
 921                  */
 922                 err = i915_vma_sync(vma);
 923                 if (err)
 924                         break;
 925
 926                 if (igt_timeout(end_time,
 927                                 "%s timed out at ofset %llx [%llx - %llx]\n",
 928                                 __func__, addr, hole_start, hole_end)) {
 929                         err = -EINTR;
 930                         break;
 931                 }
 932         }
 933
 934         close_object_list(&objects, vm);
 935         cleanup_freed_objects(vm->i915);
 936         return err;
 937 }
 938
 939 static int shrink_hole(struct i915_address_space *vm,
 940                        u64 hole_start, u64 hole_end,
 941                        unsigned long end_time)
 942 {
 943         unsigned long prime;
 944         int err;
 945
 946         vm->fault_attr.probability = 999;
 947         atomic_set(&vm->fault_attr.times, -1);
 948
 949         for_each_prime_number_from(prime, 0, ULONG_MAX - 1) {
 950                 vm->fault_attr.interval = prime;
 951                 err = __shrink_hole(vm, hole_start, hole_end, end_time);
 952                 if (err)
 953                         break;
 954         }
 955
 956         memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
 957
 958         return err;
 959 }
 960
 961 static int shrink_boom(struct i915_address_space *vm,
 962                        u64 hole_start, u64 hole_end,
 963                        unsigned long end_time)
 964 {
 965         unsigned int sizes[] = { SZ_2M, SZ_1G };
 966         struct drm_i915_gem_object *purge;
 967         struct drm_i915_gem_object *explode;
 968         int err;
 969         int i;
 970
 971         /*
 972          * Catch the case which shrink_hole seems to miss. The setup here
 973          * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while
 974          * ensuring that all vma assiocated with the respective pd/pdp are
 975          * unpinned at the time.
 976          */
 977
 978         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
 979                 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
 980                 unsigned int size = sizes[i];
 981                 struct i915_vma *vma;
 982
 983                 purge = fake_dma_object(vm->i915, size);
 984                 if (IS_ERR(purge))
 985                         return PTR_ERR(purge);
 986
 987                 vma = i915_vma_instance(purge, vm, NULL);
 988                 if (IS_ERR(vma)) {
 989                         err = PTR_ERR(vma);
 990                         goto err_purge;
 991                 }
 992
 993                 err = i915_vma_pin(vma, 0, 0, flags);
 994                 if (err)
 995                         goto err_purge;
 996
 997                 /* Should now be ripe for purging */
 998                 i915_vma_unpin(vma);
 999
1000                 explode = fake_dma_object(vm->i915, size);
1001                 if (IS_ERR(explode)) {
1002                         err = PTR_ERR(explode);
1003                         goto err_purge;
1004                 }
1005
1006                 vm->fault_attr.probability = 100;
1007                 vm->fault_attr.interval = 1;
1008                 atomic_set(&vm->fault_attr.times, -1);
1009
1010                 vma = i915_vma_instance(explode, vm, NULL);
1011                 if (IS_ERR(vma)) {
1012                         err = PTR_ERR(vma);
1013                         goto err_explode;
1014                 }
1015
1016                 err = i915_vma_pin(vma, 0, 0, flags | size);
1017                 if (err)
1018                         goto err_explode;
1019
1020                 i915_vma_unpin(vma);
1021
1022                 i915_gem_object_put(purge);
1023                 i915_gem_object_put(explode);
1024
1025                 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
1026                 cleanup_freed_objects(vm->i915);
1027         }
1028
1029         return 0;
1030
1031 err_explode:
1032         i915_gem_object_put(explode);
1033 err_purge:
1034         i915_gem_object_put(purge);
1035         memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
1036         return err;
1037 }
1038
1039 static int exercise_ppgtt(struct drm_i915_private *dev_priv,
1040                           int (*func)(struct i915_address_space *vm,
1041                                       u64 hole_start, u64 hole_end,
1042                                       unsigned long end_time))
1043 {
1044         struct i915_ppgtt *ppgtt;
1045         IGT_TIMEOUT(end_time);
1046         struct file *file;
1047         int err;
1048
1049         if (!HAS_FULL_PPGTT(dev_priv))
1050                 return 0;
1051
1052         file = mock_file(dev_priv);
1053         if (IS_ERR(file))
1054                 return PTR_ERR(file);
1055
1056         ppgtt = i915_ppgtt_create(&dev_priv->gt);
1057         if (IS_ERR(ppgtt)) {
1058                 err = PTR_ERR(ppgtt);
1059                 goto out_free;
1060         }
1061         GEM_BUG_ON(offset_in_page(ppgtt->vm.total));
1062         GEM_BUG_ON(!atomic_read(&ppgtt->vm.open));
1063
1064         err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time);
1065
1066         i915_vm_put(&ppgtt->vm);
1067
1068 out_free:
1069         fput(file);
1070         return err;
1071 }
1072
1073 static int igt_ppgtt_fill(void *arg)
1074 {
1075         return exercise_ppgtt(arg, fill_hole);
1076 }
1077
1078 static int igt_ppgtt_walk(void *arg)
1079 {
1080         return exercise_ppgtt(arg, walk_hole);
1081 }
1082
1083 static int igt_ppgtt_pot(void *arg)
1084 {
1085         return exercise_ppgtt(arg, pot_hole);
1086 }
1087
1088 static int igt_ppgtt_drunk(void *arg)
1089 {
1090         return exercise_ppgtt(arg, drunk_hole);
1091 }
1092
1093 static int igt_ppgtt_lowlevel(void *arg)
1094 {
1095         return exercise_ppgtt(arg, lowlevel_hole);
1096 }
1097
1098 static int igt_ppgtt_shrink(void *arg)
1099 {
1100         return exercise_ppgtt(arg, shrink_hole);
1101 }
1102
1103 static int igt_ppgtt_shrink_boom(void *arg)
1104 {
1105         return exercise_ppgtt(arg, shrink_boom);
1106 }
1107
1108 static int sort_holes(void *priv, const struct list_head *A,
1109                       const struct list_head *B)
1110 {
1111         struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack);
1112         struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack);
1113
1114         if (a->start < b->start)
1115                 return -1;
1116         else
1117                 return 1;
1118 }
1119
1120 static int exercise_ggtt(struct drm_i915_private *i915,
1121                          int (*func)(struct i915_address_space *vm,
1122                                      u64 hole_start, u64 hole_end,
1123                                      unsigned long end_time))
1124 {
1125         struct i915_ggtt *ggtt = &i915->ggtt;
1126         u64 hole_start, hole_end, last = 0;
1127         struct drm_mm_node *node;
1128         IGT_TIMEOUT(end_time);
1129         int err = 0;
1130
1131 restart:
1132         list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes);
1133         drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) {
1134                 if (hole_start < last)
1135                         continue;
1136
1137                 if (ggtt->vm.mm.color_adjust)
1138                         ggtt->vm.mm.color_adjust(node, 0,
1139                                                  &hole_start, &hole_end);
1140                 if (hole_start >= hole_end)
1141                         continue;
1142
1143                 err = func(&ggtt->vm, hole_start, hole_end, end_time);
1144                 if (err)
1145                         break;
1146
1147                 /* As we have manipulated the drm_mm, the list may be corrupt */
1148                 last = hole_end;
1149                 goto restart;
1150         }
1151
1152         return err;
1153 }
1154
1155 static int igt_ggtt_fill(void *arg)
1156 {
1157         return exercise_ggtt(arg, fill_hole);
1158 }
1159
1160 static int igt_ggtt_walk(void *arg)
1161 {
1162         return exercise_ggtt(arg, walk_hole);
1163 }
1164
1165 static int igt_ggtt_pot(void *arg)
1166 {
1167         return exercise_ggtt(arg, pot_hole);
1168 }
1169
1170 static int igt_ggtt_drunk(void *arg)
1171 {
1172         return exercise_ggtt(arg, drunk_hole);
1173 }
1174
1175 static int igt_ggtt_lowlevel(void *arg)
1176 {
1177         return exercise_ggtt(arg, lowlevel_hole);
1178 }
1179
1180 static int igt_ggtt_page(void *arg)
1181 {
1182         const unsigned int count = PAGE_SIZE/sizeof(u32);
1183         I915_RND_STATE(prng);
1184         struct drm_i915_private *i915 = arg;
1185         struct i915_ggtt *ggtt = &i915->ggtt;
1186         struct drm_i915_gem_object *obj;
1187         intel_wakeref_t wakeref;
1188         struct drm_mm_node tmp;
1189         unsigned int *order, n;
1190         int err;
1191
1192         if (!i915_ggtt_has_aperture(ggtt))
1193                 return 0;
1194
1195         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1196         if (IS_ERR(obj))
1197                 return PTR_ERR(obj);
1198
1199         err = i915_gem_object_pin_pages_unlocked(obj);
1200         if (err)
1201                 goto out_free;
1202
1203         memset(&tmp, 0, sizeof(tmp));
1204         mutex_lock(&ggtt->vm.mutex);
1205         err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp,
1206                                           count * PAGE_SIZE, 0,
1207                                           I915_COLOR_UNEVICTABLE,
1208                                           0, ggtt->mappable_end,
1209                                           DRM_MM_INSERT_LOW);
1210         mutex_unlock(&ggtt->vm.mutex);
1211         if (err)
1212                 goto out_unpin;
1213
1214         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1215
1216         for (n = 0; n < count; n++) {
1217                 u64 offset = tmp.start + n * PAGE_SIZE;
1218
1219                 ggtt->vm.insert_page(&ggtt->vm,
1220                                      i915_gem_object_get_dma_address(obj, 0),
1221                                      offset, I915_CACHE_NONE, 0);
1222         }
1223
1224         order = i915_random_order(count, &prng);
1225         if (!order) {
1226                 err = -ENOMEM;
1227                 goto out_remove;
1228         }
1229
1230         for (n = 0; n < count; n++) {
1231                 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1232                 u32 __iomem *vaddr;
1233
1234                 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1235                 iowrite32(n, vaddr + n);
1236                 io_mapping_unmap_atomic(vaddr);
1237         }
1238         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1239
1240         i915_random_reorder(order, count, &prng);
1241         for (n = 0; n < count; n++) {
1242                 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1243                 u32 __iomem *vaddr;
1244                 u32 val;
1245
1246                 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1247                 val = ioread32(vaddr + n);
1248                 io_mapping_unmap_atomic(vaddr);
1249
1250                 if (val != n) {
1251                         pr_err("insert page failed: found %d, expected %d\n",
1252                                val, n);
1253                         err = -EINVAL;
1254                         break;
1255                 }
1256         }
1257
1258         kfree(order);
1259 out_remove:
1260         ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size);
1261         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1262         mutex_lock(&ggtt->vm.mutex);
1263         drm_mm_remove_node(&tmp);
1264         mutex_unlock(&ggtt->vm.mutex);
1265 out_unpin:
1266         i915_gem_object_unpin_pages(obj);
1267 out_free:
1268         i915_gem_object_put(obj);
1269         return err;
1270 }
1271
1272 static void track_vma_bind(struct i915_vma *vma)
1273 {
1274         struct drm_i915_gem_object *obj = vma->obj;
1275
1276         __i915_gem_object_pin_pages(obj);
1277
1278         GEM_BUG_ON(vma->pages);
1279         atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE);
1280         __i915_gem_object_pin_pages(obj);
1281         vma->pages = obj->mm.pages;
1282
1283         mutex_lock(&vma->vm->mutex);
1284         list_add_tail(&vma->vm_link, &vma->vm->bound_list);
1285         mutex_unlock(&vma->vm->mutex);
1286 }
1287
1288 static int exercise_mock(struct drm_i915_private *i915,
1289                          int (*func)(struct i915_address_space *vm,
1290                                      u64 hole_start, u64 hole_end,
1291                                      unsigned long end_time))
1292 {
1293         const u64 limit = totalram_pages() << PAGE_SHIFT;
1294         struct i915_address_space *vm;
1295         struct i915_gem_context *ctx;
1296         IGT_TIMEOUT(end_time);
1297         int err;
1298
1299         ctx = mock_context(i915, "mock");
1300         if (!ctx)
1301                 return -ENOMEM;
1302
1303         vm = i915_gem_context_get_vm_rcu(ctx);
1304         err = func(vm, 0, min(vm->total, limit), end_time);
1305         i915_vm_put(vm);
1306
1307         mock_context_close(ctx);
1308         return err;
1309 }
1310
1311 static int igt_mock_fill(void *arg)
1312 {
1313         struct i915_ggtt *ggtt = arg;
1314
1315         return exercise_mock(ggtt->vm.i915, fill_hole);
1316 }
1317
1318 static int igt_mock_walk(void *arg)
1319 {
1320         struct i915_ggtt *ggtt = arg;
1321
1322         return exercise_mock(ggtt->vm.i915, walk_hole);
1323 }
1324
1325 static int igt_mock_pot(void *arg)
1326 {
1327         struct i915_ggtt *ggtt = arg;
1328
1329         return exercise_mock(ggtt->vm.i915, pot_hole);
1330 }
1331
1332 static int igt_mock_drunk(void *arg)
1333 {
1334         struct i915_ggtt *ggtt = arg;
1335
1336         return exercise_mock(ggtt->vm.i915, drunk_hole);
1337 }
1338
1339 static int igt_gtt_reserve(void *arg)
1340 {
1341         struct i915_ggtt *ggtt = arg;
1342         struct drm_i915_gem_object *obj, *on;
1343         I915_RND_STATE(prng);
1344         LIST_HEAD(objects);
1345         u64 total;
1346         int err = -ENODEV;
1347
1348         /* i915_gem_gtt_reserve() tries to reserve the precise range
1349          * for the node, and evicts if it has to. So our test checks that
1350          * it can give us the requsted space and prevent overlaps.
1351          */
1352
1353         /* Start by filling the GGTT */
1354         for (total = 0;
1355              total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1356              total += 2 * I915_GTT_PAGE_SIZE) {
1357                 struct i915_vma *vma;
1358
1359                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1360                                                       2 * PAGE_SIZE);
1361                 if (IS_ERR(obj)) {
1362                         err = PTR_ERR(obj);
1363                         goto out;
1364                 }
1365
1366                 err = i915_gem_object_pin_pages_unlocked(obj);
1367                 if (err) {
1368                         i915_gem_object_put(obj);
1369                         goto out;
1370                 }
1371
1372                 list_add(&obj->st_link, &objects);
1373
1374                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1375                 if (IS_ERR(vma)) {
1376                         err = PTR_ERR(vma);
1377                         goto out;
1378                 }
1379
1380                 mutex_lock(&ggtt->vm.mutex);
1381                 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1382                                            obj->base.size,
1383                                            total,
1384                                            obj->cache_level,
1385                                            0);
1386                 mutex_unlock(&ggtt->vm.mutex);
1387                 if (err) {
1388                         pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n",
1389                                total, ggtt->vm.total, err);
1390                         goto out;
1391                 }
1392                 track_vma_bind(vma);
1393
1394                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1395                 if (vma->node.start != total ||
1396                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1397                         pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1398                                vma->node.start, vma->node.size,
1399                                total, 2*I915_GTT_PAGE_SIZE);
1400                         err = -EINVAL;
1401                         goto out;
1402                 }
1403         }
1404
1405         /* Now we start forcing evictions */
1406         for (total = I915_GTT_PAGE_SIZE;
1407              total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1408              total += 2 * I915_GTT_PAGE_SIZE) {
1409                 struct i915_vma *vma;
1410
1411                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1412                                                       2 * PAGE_SIZE);
1413                 if (IS_ERR(obj)) {
1414                         err = PTR_ERR(obj);
1415                         goto out;
1416                 }
1417
1418                 err = i915_gem_object_pin_pages_unlocked(obj);
1419                 if (err) {
1420                         i915_gem_object_put(obj);
1421                         goto out;
1422                 }
1423
1424                 list_add(&obj->st_link, &objects);
1425
1426                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1427                 if (IS_ERR(vma)) {
1428                         err = PTR_ERR(vma);
1429                         goto out;
1430                 }
1431
1432                 mutex_lock(&ggtt->vm.mutex);
1433                 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1434                                            obj->base.size,
1435                                            total,
1436                                            obj->cache_level,
1437                                            0);
1438                 mutex_unlock(&ggtt->vm.mutex);
1439                 if (err) {
1440                         pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n",
1441                                total, ggtt->vm.total, err);
1442                         goto out;
1443                 }
1444                 track_vma_bind(vma);
1445
1446                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1447                 if (vma->node.start != total ||
1448                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1449                         pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1450                                vma->node.start, vma->node.size,
1451                                total, 2*I915_GTT_PAGE_SIZE);
1452                         err = -EINVAL;
1453                         goto out;
1454                 }
1455         }
1456
1457         /* And then try at random */
1458         list_for_each_entry_safe(obj, on, &objects, st_link) {
1459                 struct i915_vma *vma;
1460                 u64 offset;
1461
1462                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1463                 if (IS_ERR(vma)) {
1464                         err = PTR_ERR(vma);
1465                         goto out;
1466                 }
1467
1468                 err = i915_vma_unbind(vma);
1469                 if (err) {
1470                         pr_err("i915_vma_unbind failed with err=%d!\n", err);
1471                         goto out;
1472                 }
1473
1474                 offset = igt_random_offset(&prng,
1475                                            0, ggtt->vm.total,
1476                                            2 * I915_GTT_PAGE_SIZE,
1477                                            I915_GTT_MIN_ALIGNMENT);
1478
1479                 mutex_lock(&ggtt->vm.mutex);
1480                 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1481                                            obj->base.size,
1482                                            offset,
1483                                            obj->cache_level,
1484                                            0);
1485                 mutex_unlock(&ggtt->vm.mutex);
1486                 if (err) {
1487                         pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n",
1488                                total, ggtt->vm.total, err);
1489                         goto out;
1490                 }
1491                 track_vma_bind(vma);
1492
1493                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1494                 if (vma->node.start != offset ||
1495                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1496                         pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1497                                vma->node.start, vma->node.size,
1498                                offset, 2*I915_GTT_PAGE_SIZE);
1499                         err = -EINVAL;
1500                         goto out;
1501                 }
1502         }
1503
1504 out:
1505         list_for_each_entry_safe(obj, on, &objects, st_link) {
1506                 i915_gem_object_unpin_pages(obj);
1507                 i915_gem_object_put(obj);
1508         }
1509         return err;
1510 }
1511
1512 static int igt_gtt_insert(void *arg)
1513 {
1514         struct i915_ggtt *ggtt = arg;
1515         struct drm_i915_gem_object *obj, *on;
1516         struct drm_mm_node tmp = {};
1517         const struct invalid_insert {
1518                 u64 size;
1519                 u64 alignment;
1520                 u64 start, end;
1521         } invalid_insert[] = {
1522                 {
1523                         ggtt->vm.total + I915_GTT_PAGE_SIZE, 0,
1524                         0, ggtt->vm.total,
1525                 },
1526                 {
1527                         2*I915_GTT_PAGE_SIZE, 0,
1528                         0, I915_GTT_PAGE_SIZE,
1529                 },
1530                 {
1531                         -(u64)I915_GTT_PAGE_SIZE, 0,
1532                         0, 4*I915_GTT_PAGE_SIZE,
1533                 },
1534                 {
1535                         -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE,
1536                         0, 4*I915_GTT_PAGE_SIZE,
1537                 },
1538                 {
1539                         I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1,
1540                         I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1,
1541                 },
1542                 {}
1543         }, *ii;
1544         LIST_HEAD(objects);
1545         u64 total;
1546         int err = -ENODEV;
1547
1548         /* i915_gem_gtt_insert() tries to allocate some free space in the GTT
1549          * to the node, evicting if required.
1550          */
1551
1552         /* Check a couple of obviously invalid requests */
1553         for (ii = invalid_insert; ii->size; ii++) {
1554                 mutex_lock(&ggtt->vm.mutex);
1555                 err = i915_gem_gtt_insert(&ggtt->vm, &tmp,
1556                                           ii->size, ii->alignment,
1557                                           I915_COLOR_UNEVICTABLE,
1558                                           ii->start, ii->end,
1559                                           0);
1560                 mutex_unlock(&ggtt->vm.mutex);
1561                 if (err != -ENOSPC) {
1562                         pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n",
1563                                ii->size, ii->alignment, ii->start, ii->end,
1564                                err);
1565                         return -EINVAL;
1566                 }
1567         }
1568
1569         /* Start by filling the GGTT */
1570         for (total = 0;
1571              total + I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1572              total += I915_GTT_PAGE_SIZE) {
1573                 struct i915_vma *vma;
1574
1575                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1576                                                       I915_GTT_PAGE_SIZE);
1577                 if (IS_ERR(obj)) {
1578                         err = PTR_ERR(obj);
1579                         goto out;
1580                 }
1581
1582                 err = i915_gem_object_pin_pages_unlocked(obj);
1583                 if (err) {
1584                         i915_gem_object_put(obj);
1585                         goto out;
1586                 }
1587
1588                 list_add(&obj->st_link, &objects);
1589
1590                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1591                 if (IS_ERR(vma)) {
1592                         err = PTR_ERR(vma);
1593                         goto out;
1594                 }
1595
1596                 mutex_lock(&ggtt->vm.mutex);
1597                 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1598                                           obj->base.size, 0, obj->cache_level,
1599                                           0, ggtt->vm.total,
1600                                           0);
1601                 mutex_unlock(&ggtt->vm.mutex);
1602                 if (err == -ENOSPC) {
1603                         /* maxed out the GGTT space */
1604                         i915_gem_object_put(obj);
1605                         break;
1606                 }
1607                 if (err) {
1608                         pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n",
1609                                total, ggtt->vm.total, err);
1610                         goto out;
1611                 }
1612                 track_vma_bind(vma);
1613                 __i915_vma_pin(vma);
1614
1615                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1616         }
1617
1618         list_for_each_entry(obj, &objects, st_link) {
1619                 struct i915_vma *vma;
1620
1621                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1622                 if (IS_ERR(vma)) {
1623                         err = PTR_ERR(vma);
1624                         goto out;
1625                 }
1626
1627                 if (!drm_mm_node_allocated(&vma->node)) {
1628                         pr_err("VMA was unexpectedly evicted!\n");
1629                         err = -EINVAL;
1630                         goto out;
1631                 }
1632
1633                 __i915_vma_unpin(vma);
1634         }
1635
1636         /* If we then reinsert, we should find the same hole */
1637         list_for_each_entry_safe(obj, on, &objects, st_link) {
1638                 struct i915_vma *vma;
1639                 u64 offset;
1640
1641                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1642                 if (IS_ERR(vma)) {
1643                         err = PTR_ERR(vma);
1644                         goto out;
1645                 }
1646
1647                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1648                 offset = vma->node.start;
1649
1650                 err = i915_vma_unbind(vma);
1651                 if (err) {
1652                         pr_err("i915_vma_unbind failed with err=%d!\n", err);
1653                         goto out;
1654                 }
1655
1656                 mutex_lock(&ggtt->vm.mutex);
1657                 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1658                                           obj->base.size, 0, obj->cache_level,
1659                                           0, ggtt->vm.total,
1660                                           0);
1661                 mutex_unlock(&ggtt->vm.mutex);
1662                 if (err) {
1663                         pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n",
1664                                total, ggtt->vm.total, err);
1665                         goto out;
1666                 }
1667                 track_vma_bind(vma);
1668
1669                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1670                 if (vma->node.start != offset) {
1671                         pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n",
1672                                offset, vma->node.start);
1673                         err = -EINVAL;
1674                         goto out;
1675                 }
1676         }
1677
1678         /* And then force evictions */
1679         for (total = 0;
1680              total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1681              total += 2 * I915_GTT_PAGE_SIZE) {
1682                 struct i915_vma *vma;
1683
1684                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1685                                                       2 * I915_GTT_PAGE_SIZE);
1686                 if (IS_ERR(obj)) {
1687                         err = PTR_ERR(obj);
1688                         goto out;
1689                 }
1690
1691                 err = i915_gem_object_pin_pages_unlocked(obj);
1692                 if (err) {
1693                         i915_gem_object_put(obj);
1694                         goto out;
1695                 }
1696
1697                 list_add(&obj->st_link, &objects);
1698
1699                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1700                 if (IS_ERR(vma)) {
1701                         err = PTR_ERR(vma);
1702                         goto out;
1703                 }
1704
1705                 mutex_lock(&ggtt->vm.mutex);
1706                 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1707                                           obj->base.size, 0, obj->cache_level,
1708                                           0, ggtt->vm.total,
1709                                           0);
1710                 mutex_unlock(&ggtt->vm.mutex);
1711                 if (err) {
1712                         pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n",
1713                                total, ggtt->vm.total, err);
1714                         goto out;
1715                 }
1716                 track_vma_bind(vma);
1717
1718                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1719         }
1720
1721 out:
1722         list_for_each_entry_safe(obj, on, &objects, st_link) {
1723                 i915_gem_object_unpin_pages(obj);
1724                 i915_gem_object_put(obj);
1725         }
1726         return err;
1727 }
1728
1729 int i915_gem_gtt_mock_selftests(void)
1730 {
1731         static const struct i915_subtest tests[] = {
1732                 SUBTEST(igt_mock_drunk),
1733                 SUBTEST(igt_mock_walk),
1734                 SUBTEST(igt_mock_pot),
1735                 SUBTEST(igt_mock_fill),
1736                 SUBTEST(igt_gtt_reserve),
1737                 SUBTEST(igt_gtt_insert),
1738         };
1739         struct drm_i915_private *i915;
1740         struct i915_ggtt *ggtt;
1741         int err;
1742
1743         i915 = mock_gem_device();
1744         if (!i915)
1745                 return -ENOMEM;
1746
1747         ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL);
1748         if (!ggtt) {
1749                 err = -ENOMEM;
1750                 goto out_put;
1751         }
1752         mock_init_ggtt(i915, ggtt);
1753
1754         err = i915_subtests(tests, ggtt);
1755
1756         mock_device_flush(i915);
1757         i915_gem_drain_freed_objects(i915);
1758         mock_fini_ggtt(ggtt);
1759         kfree(ggtt);
1760 out_put:
1761         mock_destroy_device(i915);
1762         return err;
1763 }
1764
1765 static int context_sync(struct intel_context *ce)
1766 {
1767         struct i915_request *rq;
1768         long timeout;
1769
1770         rq = intel_context_create_request(ce);
1771         if (IS_ERR(rq))
1772                 return PTR_ERR(rq);
1773
1774         i915_request_get(rq);
1775         i915_request_add(rq);
1776
1777         timeout = i915_request_wait(rq, 0, HZ / 5);
1778         i915_request_put(rq);
1779
1780         return timeout < 0 ? -EIO : 0;
1781 }
1782
1783 static struct i915_request *
1784 submit_batch(struct intel_context *ce, u64 addr)
1785 {
1786         struct i915_request *rq;
1787         int err;
1788
1789         rq = intel_context_create_request(ce);
1790         if (IS_ERR(rq))
1791                 return rq;
1792
1793         err = 0;
1794         if (rq->engine->emit_init_breadcrumb) /* detect a hang */
1795                 err = rq->engine->emit_init_breadcrumb(rq);
1796         if (err == 0)
1797                 err = rq->engine->emit_bb_start(rq, addr, 0, 0);
1798
1799         if (err == 0)
1800                 i915_request_get(rq);
1801         i915_request_add(rq);
1802
1803         return err ? ERR_PTR(err) : rq;
1804 }
1805
1806 static u32 *spinner(u32 *batch, int i)
1807 {
1808         return batch + i * 64 / sizeof(*batch) + 4;
1809 }
1810
1811 static void end_spin(u32 *batch, int i)
1812 {
1813         *spinner(batch, i) = MI_BATCH_BUFFER_END;
1814         wmb();
1815 }
1816
1817 static int igt_cs_tlb(void *arg)
1818 {
1819         const unsigned int count = PAGE_SIZE / 64;
1820         const unsigned int chunk_size = count * PAGE_SIZE;
1821         struct drm_i915_private *i915 = arg;
1822         struct drm_i915_gem_object *bbe, *act, *out;
1823         struct i915_gem_engines_iter it;
1824         struct i915_address_space *vm;
1825         struct i915_gem_context *ctx;
1826         struct intel_context *ce;
1827         struct i915_vma *vma;
1828         I915_RND_STATE(prng);
1829         struct file *file;
1830         unsigned int i;
1831         u32 *result;
1832         u32 *batch;
1833         int err = 0;
1834
1835         /*
1836          * Our mission here is to fool the hardware to execute something
1837          * from scratch as it has not seen the batch move (due to missing
1838          * the TLB invalidate).
1839          */
1840
1841         file = mock_file(i915);
1842         if (IS_ERR(file))
1843                 return PTR_ERR(file);
1844
1845         ctx = live_context(i915, file);
1846         if (IS_ERR(ctx)) {
1847                 err = PTR_ERR(ctx);
1848                 goto out_unlock;
1849         }
1850
1851         vm = i915_gem_context_get_vm_rcu(ctx);
1852         if (i915_is_ggtt(vm))
1853                 goto out_vm;
1854
1855         /* Create two pages; dummy we prefill the TLB, and intended */
1856         bbe = i915_gem_object_create_internal(i915, PAGE_SIZE);
1857         if (IS_ERR(bbe)) {
1858                 err = PTR_ERR(bbe);
1859                 goto out_vm;
1860         }
1861
1862         batch = i915_gem_object_pin_map_unlocked(bbe, I915_MAP_WC);
1863         if (IS_ERR(batch)) {
1864                 err = PTR_ERR(batch);
1865                 goto out_put_bbe;
1866         }
1867         memset32(batch, MI_BATCH_BUFFER_END, PAGE_SIZE / sizeof(u32));
1868         i915_gem_object_flush_map(bbe);
1869         i915_gem_object_unpin_map(bbe);
1870
1871         act = i915_gem_object_create_internal(i915, PAGE_SIZE);
1872         if (IS_ERR(act)) {
1873                 err = PTR_ERR(act);
1874                 goto out_put_bbe;
1875         }
1876
1877         /* Track the execution of each request by writing into different slot */
1878         batch = i915_gem_object_pin_map_unlocked(act, I915_MAP_WC);
1879         if (IS_ERR(batch)) {
1880                 err = PTR_ERR(batch);
1881                 goto out_put_act;
1882         }
1883         for (i = 0; i < count; i++) {
1884                 u32 *cs = batch + i * 64 / sizeof(*cs);
1885                 u64 addr = (vm->total - PAGE_SIZE) + i * sizeof(u32);
1886
1887                 GEM_BUG_ON(INTEL_GEN(i915) < 6);
1888                 cs[0] = MI_STORE_DWORD_IMM_GEN4;
1889                 if (INTEL_GEN(i915) >= 8) {
1890                         cs[1] = lower_32_bits(addr);
1891                         cs[2] = upper_32_bits(addr);
1892                         cs[3] = i;
1893                         cs[4] = MI_NOOP;
1894                         cs[5] = MI_BATCH_BUFFER_START_GEN8;
1895                 } else {
1896                         cs[1] = 0;
1897                         cs[2] = lower_32_bits(addr);
1898                         cs[3] = i;
1899                         cs[4] = MI_NOOP;
1900                         cs[5] = MI_BATCH_BUFFER_START;
1901                 }
1902         }
1903
1904         out = i915_gem_object_create_internal(i915, PAGE_SIZE);
1905         if (IS_ERR(out)) {
1906                 err = PTR_ERR(out);
1907                 goto out_put_batch;
1908         }
1909         i915_gem_object_set_cache_coherency(out, I915_CACHING_CACHED);
1910
1911         vma = i915_vma_instance(out, vm, NULL);
1912         if (IS_ERR(vma)) {
1913                 err = PTR_ERR(vma);
1914                 goto out_put_out;
1915         }
1916
1917         err = i915_vma_pin(vma, 0, 0,
1918                            PIN_USER |
1919                            PIN_OFFSET_FIXED |
1920                            (vm->total - PAGE_SIZE));
1921         if (err)
1922                 goto out_put_out;
1923         GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE);
1924
1925         result = i915_gem_object_pin_map_unlocked(out, I915_MAP_WB);
1926         if (IS_ERR(result)) {
1927                 err = PTR_ERR(result);
1928                 goto out_put_out;
1929         }
1930
1931         for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1932                 IGT_TIMEOUT(end_time);
1933                 unsigned long pass = 0;
1934
1935                 if (!intel_engine_can_store_dword(ce->engine))
1936                         continue;
1937
1938                 while (!__igt_timeout(end_time, NULL)) {
1939                         struct i915_vm_pt_stash stash = {};
1940                         struct i915_request *rq;
1941                         struct i915_gem_ww_ctx ww;
1942                         u64 offset;
1943
1944                         offset = igt_random_offset(&prng,
1945                                                    0, vm->total - PAGE_SIZE,
1946                                                    chunk_size, PAGE_SIZE);
1947
1948                         memset32(result, STACK_MAGIC, PAGE_SIZE / sizeof(u32));
1949
1950                         vma = i915_vma_instance(bbe, vm, NULL);
1951                         if (IS_ERR(vma)) {
1952                                 err = PTR_ERR(vma);
1953                                 goto end;
1954                         }
1955
1956                         err = vma->ops->set_pages(vma);
1957                         if (err)
1958                                 goto end;
1959
1960                         i915_gem_ww_ctx_init(&ww, false);
1961 retry:
1962                         err = i915_vm_lock_objects(vm, &ww);
1963                         if (err)
1964                                 goto end_ww;
1965
1966                         err = i915_vm_alloc_pt_stash(vm, &stash, chunk_size);
1967                         if (err)
1968                                 goto end_ww;
1969
1970                         err = i915_vm_map_pt_stash(vm, &stash);
1971                         if (!err)
1972                                 vm->allocate_va_range(vm, &stash, offset, chunk_size);
1973                         i915_vm_free_pt_stash(vm, &stash);
1974 end_ww:
1975                         if (err == -EDEADLK) {
1976                                 err = i915_gem_ww_ctx_backoff(&ww);
1977                                 if (!err)
1978                                         goto retry;
1979                         }
1980                         i915_gem_ww_ctx_fini(&ww);
1981                         if (err)
1982                                 goto end;
1983
1984                         /* Prime the TLB with the dummy pages */
1985                         for (i = 0; i < count; i++) {
1986                                 vma->node.start = offset + i * PAGE_SIZE;
1987                                 vm->insert_entries(vm, vma, I915_CACHE_NONE, 0);
1988
1989                                 rq = submit_batch(ce, vma->node.start);
1990                                 if (IS_ERR(rq)) {
1991                                         err = PTR_ERR(rq);
1992                                         goto end;
1993                                 }
1994                                 i915_request_put(rq);
1995                         }
1996
1997                         vma->ops->clear_pages(vma);
1998
1999                         err = context_sync(ce);
2000                         if (err) {
2001                                 pr_err("%s: dummy setup timed out\n",
2002                                        ce->engine->name);
2003                                 goto end;
2004                         }
2005
2006                         vma = i915_vma_instance(act, vm, NULL);
2007                         if (IS_ERR(vma)) {
2008                                 err = PTR_ERR(vma);
2009                                 goto end;
2010                         }
2011
2012                         err = vma->ops->set_pages(vma);
2013                         if (err)
2014                                 goto end;
2015
2016                         /* Replace the TLB with target batches */
2017                         for (i = 0; i < count; i++) {
2018                                 struct i915_request *rq;
2019                                 u32 *cs = batch + i * 64 / sizeof(*cs);
2020                                 u64 addr;
2021
2022                                 vma->node.start = offset + i * PAGE_SIZE;
2023                                 vm->insert_entries(vm, vma, I915_CACHE_NONE, 0);
2024
2025                                 addr = vma->node.start + i * 64;
2026                                 cs[4] = MI_NOOP;
2027                                 cs[6] = lower_32_bits(addr);
2028                                 cs[7] = upper_32_bits(addr);
2029                                 wmb();
2030
2031                                 rq = submit_batch(ce, addr);
2032                                 if (IS_ERR(rq)) {
2033                                         err = PTR_ERR(rq);
2034                                         goto end;
2035                                 }
2036
2037                                 /* Wait until the context chain has started */
2038                                 if (i == 0) {
2039                                         while (READ_ONCE(result[i]) &&
2040                                                !i915_request_completed(rq))
2041                                                 cond_resched();
2042                                 } else {
2043                                         end_spin(batch, i - 1);
2044                                 }
2045
2046                                 i915_request_put(rq);
2047                         }
2048                         end_spin(batch, count - 1);
2049
2050                         vma->ops->clear_pages(vma);
2051
2052                         err = context_sync(ce);
2053                         if (err) {
2054                                 pr_err("%s: writes timed out\n",
2055                                        ce->engine->name);
2056                                 goto end;
2057                         }
2058
2059                         for (i = 0; i < count; i++) {
2060                                 if (result[i] != i) {
2061                                         pr_err("%s: Write lost on pass %lu, at offset %llx, index %d, found %x, expected %x\n",
2062                                                ce->engine->name, pass,
2063                                                offset, i, result[i], i);
2064                                         err = -EINVAL;
2065                                         goto end;
2066                                 }
2067                         }
2068
2069                         vm->clear_range(vm, offset, chunk_size);
2070                         pass++;
2071                 }
2072         }
2073 end:
2074         if (igt_flush_test(i915))
2075                 err = -EIO;
2076         i915_gem_context_unlock_engines(ctx);
2077         i915_gem_object_unpin_map(out);
2078 out_put_out:
2079         i915_gem_object_put(out);
2080 out_put_batch:
2081         i915_gem_object_unpin_map(act);
2082 out_put_act:
2083         i915_gem_object_put(act);
2084 out_put_bbe:
2085         i915_gem_object_put(bbe);
2086 out_vm:
2087         i915_vm_put(vm);
2088 out_unlock:
2089         fput(file);
2090         return err;
2091 }
2092
2093 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915)
2094 {
2095         static const struct i915_subtest tests[] = {
2096                 SUBTEST(igt_ppgtt_alloc),
2097                 SUBTEST(igt_ppgtt_lowlevel),
2098                 SUBTEST(igt_ppgtt_drunk),
2099                 SUBTEST(igt_ppgtt_walk),
2100                 SUBTEST(igt_ppgtt_pot),
2101                 SUBTEST(igt_ppgtt_fill),
2102                 SUBTEST(igt_ppgtt_shrink),
2103                 SUBTEST(igt_ppgtt_shrink_boom),
2104                 SUBTEST(igt_ggtt_lowlevel),
2105                 SUBTEST(igt_ggtt_drunk),
2106                 SUBTEST(igt_ggtt_walk),
2107                 SUBTEST(igt_ggtt_pot),
2108                 SUBTEST(igt_ggtt_fill),
2109                 SUBTEST(igt_ggtt_page),
2110                 SUBTEST(igt_cs_tlb),
2111         };
2112
2113         GEM_BUG_ON(offset_in_page(i915->ggtt.vm.total));
2114
2115         return i915_subtests(tests, i915);
2116 }