drivers/gpu/drm/i915/selftests/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/list_sort.h>
  26 #include <linux/prime_numbers.h>
  27
  28 #include "gem/i915_gem_context.h"
  29 #include "gem/selftests/mock_context.h"
  30 #include "gt/intel_context.h"
  31 #include "gt/intel_gpu_commands.h"
  32
  33 #include "i915_random.h"
  34 #include "i915_selftest.h"
  35
  36 #include "mock_drm.h"
  37 #include "mock_gem_device.h"
  38 #include "mock_gtt.h"
  39 #include "igt_flush_test.h"
  40
  41 static void cleanup_freed_objects(struct drm_i915_private *i915)
  42 {
  43         i915_gem_drain_freed_objects(i915);
  44 }
  45
  46 static void fake_free_pages(struct drm_i915_gem_object *obj,
  47                             struct sg_table *pages)
  48 {
  49         sg_free_table(pages);
  50         kfree(pages);
  51 }
  52
  53 static int fake_get_pages(struct drm_i915_gem_object *obj)
  54 {
  55 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
  56 #define PFN_BIAS 0x1000
  57         struct sg_table *pages;
  58         struct scatterlist *sg;
  59         unsigned int sg_page_sizes;
  60         typeof(obj->base.size) rem;
  61
  62         pages = kmalloc(sizeof(*pages), GFP);
  63         if (!pages)
  64                 return -ENOMEM;
  65
  66         rem = round_up(obj->base.size, BIT(31)) >> 31;
  67         if (sg_alloc_table(pages, rem, GFP)) {
  68                 kfree(pages);
  69                 return -ENOMEM;
  70         }
  71
  72         sg_page_sizes = 0;
  73         rem = obj->base.size;
  74         for (sg = pages->sgl; sg; sg = sg_next(sg)) {
  75                 unsigned long len = min_t(typeof(rem), rem, BIT(31));
  76
  77                 GEM_BUG_ON(!len);
  78                 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0);
  79                 sg_dma_address(sg) = page_to_phys(sg_page(sg));
  80                 sg_dma_len(sg) = len;
  81                 sg_page_sizes |= len;
  82
  83                 rem -= len;
  84         }
  85         GEM_BUG_ON(rem);
  86
  87         __i915_gem_object_set_pages(obj, pages, sg_page_sizes);
  88
  89         return 0;
  90 #undef GFP
  91 }
  92
  93 static void fake_put_pages(struct drm_i915_gem_object *obj,
  94                            struct sg_table *pages)
  95 {
  96         fake_free_pages(obj, pages);
  97         obj->mm.dirty = false;
  98 }
  99
 100 static const struct drm_i915_gem_object_ops fake_ops = {
 101         .name = "fake-gem",
 102         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 103         .get_pages = fake_get_pages,
 104         .put_pages = fake_put_pages,
 105 };
 106
 107 static struct drm_i915_gem_object *
 108 fake_dma_object(struct drm_i915_private *i915, u64 size)
 109 {
 110         static struct lock_class_key lock_class;
 111         struct drm_i915_gem_object *obj;
 112
 113         GEM_BUG_ON(!size);
 114         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 115
 116         if (overflows_type(size, obj->base.size))
 117                 return ERR_PTR(-E2BIG);
 118
 119         obj = i915_gem_object_alloc();
 120         if (!obj)
 121                 goto err;
 122
 123         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 124         i915_gem_object_init(obj, &fake_ops, &lock_class);
 125
 126         i915_gem_object_set_volatile(obj);
 127
 128         obj->write_domain = I915_GEM_DOMAIN_CPU;
 129         obj->read_domains = I915_GEM_DOMAIN_CPU;
 130         obj->cache_level = I915_CACHE_NONE;
 131
 132         /* Preallocate the "backing storage" */
 133         if (i915_gem_object_pin_pages(obj))
 134                 goto err_obj;
 135
 136         i915_gem_object_unpin_pages(obj);
 137         return obj;
 138
 139 err_obj:
 140         i915_gem_object_put(obj);
 141 err:
 142         return ERR_PTR(-ENOMEM);
 143 }
 144
 145 static int igt_ppgtt_alloc(void *arg)
 146 {
 147         struct drm_i915_private *dev_priv = arg;
 148         struct i915_ppgtt *ppgtt;
 149         u64 size, last, limit;
 150         int err = 0;
 151
 152         /* Allocate a ppggt and try to fill the entire range */
 153
 154         if (!HAS_PPGTT(dev_priv))
 155                 return 0;
 156
 157         ppgtt = i915_ppgtt_create(&dev_priv->gt);
 158         if (IS_ERR(ppgtt))
 159                 return PTR_ERR(ppgtt);
 160
 161         if (!ppgtt->vm.allocate_va_range)
 162                 goto err_ppgtt_cleanup;
 163
 164         /*
 165          * While we only allocate the page tables here and so we could
 166          * address a much larger GTT than we could actually fit into
 167          * RAM, a practical limit is the amount of physical pages in the system.
 168          * This should ensure that we do not run into the oomkiller during
 169          * the test and take down the machine wilfully.
 170          */
 171         limit = totalram_pages() << PAGE_SHIFT;
 172         limit = min(ppgtt->vm.total, limit);
 173
 174         /* Check we can allocate the entire range */
 175         for (size = 4096; size <= limit; size <<= 2) {
 176                 struct i915_vm_pt_stash stash = {};
 177
 178                 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size);
 179                 if (err)
 180                         goto err_ppgtt_cleanup;
 181
 182                 err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash);
 183                 if (err) {
 184                         i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 185                         goto err_ppgtt_cleanup;
 186                 }
 187
 188                 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, size);
 189                 cond_resched();
 190
 191                 ppgtt->vm.clear_range(&ppgtt->vm, 0, size);
 192
 193                 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 194         }
 195
 196         /* Check we can incrementally allocate the entire range */
 197         for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) {
 198                 struct i915_vm_pt_stash stash = {};
 199
 200                 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size - last);
 201                 if (err)
 202                         goto err_ppgtt_cleanup;
 203
 204                 err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash);
 205                 if (err) {
 206                         i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 207                         goto err_ppgtt_cleanup;
 208                 }
 209
 210                 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash,
 211                                             last, size - last);
 212                 cond_resched();
 213
 214                 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 215         }
 216
 217 err_ppgtt_cleanup:
 218         i915_vm_put(&ppgtt->vm);
 219         return err;
 220 }
 221
 222 static int lowlevel_hole(struct i915_address_space *vm,
 223                          u64 hole_start, u64 hole_end,
 224                          unsigned long end_time)
 225 {
 226         I915_RND_STATE(seed_prng);
 227         struct i915_vma *mock_vma;
 228         unsigned int size;
 229
 230         mock_vma = kzalloc(sizeof(*mock_vma), GFP_KERNEL);
 231         if (!mock_vma)
 232                 return -ENOMEM;
 233
 234         /* Keep creating larger objects until one cannot fit into the hole */
 235         for (size = 12; (hole_end - hole_start) >> size; size++) {
 236                 I915_RND_SUBSTATE(prng, seed_prng);
 237                 struct drm_i915_gem_object *obj;
 238                 unsigned int *order, count, n;
 239                 u64 hole_size;
 240
 241                 hole_size = (hole_end - hole_start) >> size;
 242                 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
 243                         hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
 244                 count = hole_size >> 1;
 245                 if (!count) {
 246                         pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
 247                                  __func__, hole_start, hole_end, size, hole_size);
 248                         break;
 249                 }
 250
 251                 do {
 252                         order = i915_random_order(count, &prng);
 253                         if (order)
 254                                 break;
 255                 } while (count >>= 1);
 256                 if (!count) {
 257                         kfree(mock_vma);
 258                         return -ENOMEM;
 259                 }
 260                 GEM_BUG_ON(!order);
 261
 262                 GEM_BUG_ON(count * BIT_ULL(size) > vm->total);
 263                 GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end);
 264
 265                 /* Ignore allocation failures (i.e. don't report them as
 266                  * a test failure) as we are purposefully allocating very
 267                  * large objects without checking that we have sufficient
 268                  * memory. We expect to hit -ENOMEM.
 269                  */
 270
 271                 obj = fake_dma_object(vm->i915, BIT_ULL(size));
 272                 if (IS_ERR(obj)) {
 273                         kfree(order);
 274                         break;
 275                 }
 276
 277                 GEM_BUG_ON(obj->base.size != BIT_ULL(size));
 278
 279                 if (i915_gem_object_pin_pages(obj)) {
 280                         i915_gem_object_put(obj);
 281                         kfree(order);
 282                         break;
 283                 }
 284
 285                 for (n = 0; n < count; n++) {
 286                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 287                         intel_wakeref_t wakeref;
 288
 289                         GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
 290
 291                         if (igt_timeout(end_time,
 292                                         "%s timed out before %d/%d\n",
 293                                         __func__, n, count)) {
 294                                 hole_end = hole_start; /* quit */
 295                                 break;
 296                         }
 297
 298                         if (vm->allocate_va_range) {
 299                                 struct i915_vm_pt_stash stash = {};
 300
 301                                 if (i915_vm_alloc_pt_stash(vm, &stash,
 302                                                            BIT_ULL(size)))
 303                                         break;
 304
 305                                 if (i915_vm_pin_pt_stash(vm, &stash)) {
 306                                         i915_vm_free_pt_stash(vm, &stash);
 307                                         break;
 308                                 }
 309
 310                                 vm->allocate_va_range(vm, &stash,
 311                                                       addr, BIT_ULL(size));
 312
 313                                 i915_vm_free_pt_stash(vm, &stash);
 314                         }
 315
 316                         mock_vma->pages = obj->mm.pages;
 317                         mock_vma->node.size = BIT_ULL(size);
 318                         mock_vma->node.start = addr;
 319
 320                         with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
 321                                 vm->insert_entries(vm, mock_vma,
 322                                                    I915_CACHE_NONE, 0);
 323                 }
 324                 count = n;
 325
 326                 i915_random_reorder(order, count, &prng);
 327                 for (n = 0; n < count; n++) {
 328                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 329                         intel_wakeref_t wakeref;
 330
 331                         GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
 332                         with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
 333                                 vm->clear_range(vm, addr, BIT_ULL(size));
 334                 }
 335
 336                 i915_gem_object_unpin_pages(obj);
 337                 i915_gem_object_put(obj);
 338
 339                 kfree(order);
 340
 341                 cleanup_freed_objects(vm->i915);
 342         }
 343
 344         kfree(mock_vma);
 345         return 0;
 346 }
 347
 348 static void close_object_list(struct list_head *objects,
 349                               struct i915_address_space *vm)
 350 {
 351         struct drm_i915_gem_object *obj, *on;
 352         int ignored;
 353
 354         list_for_each_entry_safe(obj, on, objects, st_link) {
 355                 struct i915_vma *vma;
 356
 357                 vma = i915_vma_instance(obj, vm, NULL);
 358                 if (!IS_ERR(vma))
 359                         ignored = i915_vma_unbind(vma);
 360
 361                 list_del(&obj->st_link);
 362                 i915_gem_object_put(obj);
 363         }
 364 }
 365
 366 static int fill_hole(struct i915_address_space *vm,
 367                      u64 hole_start, u64 hole_end,
 368                      unsigned long end_time)
 369 {
 370         const u64 hole_size = hole_end - hole_start;
 371         struct drm_i915_gem_object *obj;
 372         const unsigned long max_pages =
 373                 min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT);
 374         const unsigned long max_step = max(int_sqrt(max_pages), 2UL);
 375         unsigned long npages, prime, flags;
 376         struct i915_vma *vma;
 377         LIST_HEAD(objects);
 378         int err;
 379
 380         /* Try binding many VMA working inwards from either edge */
 381
 382         flags = PIN_OFFSET_FIXED | PIN_USER;
 383         if (i915_is_ggtt(vm))
 384                 flags |= PIN_GLOBAL;
 385
 386         for_each_prime_number_from(prime, 2, max_step) {
 387                 for (npages = 1; npages <= max_pages; npages *= prime) {
 388                         const u64 full_size = npages << PAGE_SHIFT;
 389                         const struct {
 390                                 const char *name;
 391                                 u64 offset;
 392                                 int step;
 393                         } phases[] = {
 394                                 { "top-down", hole_end, -1, },
 395                                 { "bottom-up", hole_start, 1, },
 396                                 { }
 397                         }, *p;
 398
 399                         obj = fake_dma_object(vm->i915, full_size);
 400                         if (IS_ERR(obj))
 401                                 break;
 402
 403                         list_add(&obj->st_link, &objects);
 404
 405                         /* Align differing sized objects against the edges, and
 406                          * check we don't walk off into the void when binding
 407                          * them into the GTT.
 408                          */
 409                         for (p = phases; p->name; p++) {
 410                                 u64 offset;
 411
 412                                 offset = p->offset;
 413                                 list_for_each_entry(obj, &objects, st_link) {
 414                                         vma = i915_vma_instance(obj, vm, NULL);
 415                                         if (IS_ERR(vma))
 416                                                 continue;
 417
 418                                         if (p->step < 0) {
 419                                                 if (offset < hole_start + obj->base.size)
 420                                                         break;
 421                                                 offset -= obj->base.size;
 422                                         }
 423
 424                                         err = i915_vma_pin(vma, 0, 0, offset | flags);
 425                                         if (err) {
 426                                                 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
 427                                                        __func__, p->name, err, npages, prime, offset);
 428                                                 goto err;
 429                                         }
 430
 431                                         if (!drm_mm_node_allocated(&vma->node) ||
 432                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 433                                                 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 434                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 435                                                        offset);
 436                                                 err = -EINVAL;
 437                                                 goto err;
 438                                         }
 439
 440                                         i915_vma_unpin(vma);
 441
 442                                         if (p->step > 0) {
 443                                                 if (offset + obj->base.size > hole_end)
 444                                                         break;
 445                                                 offset += obj->base.size;
 446                                         }
 447                                 }
 448
 449                                 offset = p->offset;
 450                                 list_for_each_entry(obj, &objects, st_link) {
 451                                         vma = i915_vma_instance(obj, vm, NULL);
 452                                         if (IS_ERR(vma))
 453                                                 continue;
 454
 455                                         if (p->step < 0) {
 456                                                 if (offset < hole_start + obj->base.size)
 457                                                         break;
 458                                                 offset -= obj->base.size;
 459                                         }
 460
 461                                         if (!drm_mm_node_allocated(&vma->node) ||
 462                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 463                                                 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n",
 464                                                        __func__, p->name, vma->node.start, vma->node.size,
 465                                                        offset);
 466                                                 err = -EINVAL;
 467                                                 goto err;
 468                                         }
 469
 470                                         err = i915_vma_unbind(vma);
 471                                         if (err) {
 472                                                 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n",
 473                                                        __func__, p->name, vma->node.start, vma->node.size,
 474                                                        err);
 475                                                 goto err;
 476                                         }
 477
 478                                         if (p->step > 0) {
 479                                                 if (offset + obj->base.size > hole_end)
 480                                                         break;
 481                                                 offset += obj->base.size;
 482                                         }
 483                                 }
 484
 485                                 offset = p->offset;
 486                                 list_for_each_entry_reverse(obj, &objects, st_link) {
 487                                         vma = i915_vma_instance(obj, vm, NULL);
 488                                         if (IS_ERR(vma))
 489                                                 continue;
 490
 491                                         if (p->step < 0) {
 492                                                 if (offset < hole_start + obj->base.size)
 493                                                         break;
 494                                                 offset -= obj->base.size;
 495                                         }
 496
 497                                         err = i915_vma_pin(vma, 0, 0, offset | flags);
 498                                         if (err) {
 499                                                 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
 500                                                        __func__, p->name, err, npages, prime, offset);
 501                                                 goto err;
 502                                         }
 503
 504                                         if (!drm_mm_node_allocated(&vma->node) ||
 505                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 506                                                 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 507                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 508                                                        offset);
 509                                                 err = -EINVAL;
 510                                                 goto err;
 511                                         }
 512
 513                                         i915_vma_unpin(vma);
 514
 515                                         if (p->step > 0) {
 516                                                 if (offset + obj->base.size > hole_end)
 517                                                         break;
 518                                                 offset += obj->base.size;
 519                                         }
 520                                 }
 521
 522                                 offset = p->offset;
 523                                 list_for_each_entry_reverse(obj, &objects, st_link) {
 524                                         vma = i915_vma_instance(obj, vm, NULL);
 525                                         if (IS_ERR(vma))
 526                                                 continue;
 527
 528                                         if (p->step < 0) {
 529                                                 if (offset < hole_start + obj->base.size)
 530                                                         break;
 531                                                 offset -= obj->base.size;
 532                                         }
 533
 534                                         if (!drm_mm_node_allocated(&vma->node) ||
 535                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 536                                                 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 537                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 538                                                        offset);
 539                                                 err = -EINVAL;
 540                                                 goto err;
 541                                         }
 542
 543                                         err = i915_vma_unbind(vma);
 544                                         if (err) {
 545                                                 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n",
 546                                                        __func__, p->name, vma->node.start, vma->node.size,
 547                                                        err);
 548                                                 goto err;
 549                                         }
 550
 551                                         if (p->step > 0) {
 552                                                 if (offset + obj->base.size > hole_end)
 553                                                         break;
 554                                                 offset += obj->base.size;
 555                                         }
 556                                 }
 557                         }
 558
 559                         if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n",
 560                                         __func__, npages, prime)) {
 561                                 err = -EINTR;
 562                                 goto err;
 563                         }
 564                 }
 565
 566                 close_object_list(&objects, vm);
 567                 cleanup_freed_objects(vm->i915);
 568         }
 569
 570         return 0;
 571
 572 err:
 573         close_object_list(&objects, vm);
 574         return err;
 575 }
 576
 577 static int walk_hole(struct i915_address_space *vm,
 578                      u64 hole_start, u64 hole_end,
 579                      unsigned long end_time)
 580 {
 581         const u64 hole_size = hole_end - hole_start;
 582         const unsigned long max_pages =
 583                 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT);
 584         unsigned long flags;
 585         u64 size;
 586
 587         /* Try binding a single VMA in different positions within the hole */
 588
 589         flags = PIN_OFFSET_FIXED | PIN_USER;
 590         if (i915_is_ggtt(vm))
 591                 flags |= PIN_GLOBAL;
 592
 593         for_each_prime_number_from(size, 1, max_pages) {
 594                 struct drm_i915_gem_object *obj;
 595                 struct i915_vma *vma;
 596                 u64 addr;
 597                 int err = 0;
 598
 599                 obj = fake_dma_object(vm->i915, size << PAGE_SHIFT);
 600                 if (IS_ERR(obj))
 601                         break;
 602
 603                 vma = i915_vma_instance(obj, vm, NULL);
 604                 if (IS_ERR(vma)) {
 605                         err = PTR_ERR(vma);
 606                         goto err_put;
 607                 }
 608
 609                 for (addr = hole_start;
 610                      addr + obj->base.size < hole_end;
 611                      addr += obj->base.size) {
 612                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 613                         if (err) {
 614                                 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n",
 615                                        __func__, addr, vma->size,
 616                                        hole_start, hole_end, err);
 617                                 goto err_put;
 618                         }
 619                         i915_vma_unpin(vma);
 620
 621                         if (!drm_mm_node_allocated(&vma->node) ||
 622                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 623                                 pr_err("%s incorrect at %llx + %llx\n",
 624                                        __func__, addr, vma->size);
 625                                 err = -EINVAL;
 626                                 goto err_put;
 627                         }
 628
 629                         err = i915_vma_unbind(vma);
 630                         if (err) {
 631                                 pr_err("%s unbind failed at %llx + %llx  with err=%d\n",
 632                                        __func__, addr, vma->size, err);
 633                                 goto err_put;
 634                         }
 635
 636                         GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
 637
 638                         if (igt_timeout(end_time,
 639                                         "%s timed out at %llx\n",
 640                                         __func__, addr)) {
 641                                 err = -EINTR;
 642                                 goto err_put;
 643                         }
 644                 }
 645
 646 err_put:
 647                 i915_gem_object_put(obj);
 648                 if (err)
 649                         return err;
 650
 651                 cleanup_freed_objects(vm->i915);
 652         }
 653
 654         return 0;
 655 }
 656
 657 static int pot_hole(struct i915_address_space *vm,
 658                     u64 hole_start, u64 hole_end,
 659                     unsigned long end_time)
 660 {
 661         struct drm_i915_gem_object *obj;
 662         struct i915_vma *vma;
 663         unsigned long flags;
 664         unsigned int pot;
 665         int err = 0;
 666
 667         flags = PIN_OFFSET_FIXED | PIN_USER;
 668         if (i915_is_ggtt(vm))
 669                 flags |= PIN_GLOBAL;
 670
 671         obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE);
 672         if (IS_ERR(obj))
 673                 return PTR_ERR(obj);
 674
 675         vma = i915_vma_instance(obj, vm, NULL);
 676         if (IS_ERR(vma)) {
 677                 err = PTR_ERR(vma);
 678                 goto err_obj;
 679         }
 680
 681         /* Insert a pair of pages across every pot boundary within the hole */
 682         for (pot = fls64(hole_end - 1) - 1;
 683              pot > ilog2(2 * I915_GTT_PAGE_SIZE);
 684              pot--) {
 685                 u64 step = BIT_ULL(pot);
 686                 u64 addr;
 687
 688                 for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
 689                      addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
 690                      addr += step) {
 691                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 692                         if (err) {
 693                                 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n",
 694                                        __func__,
 695                                        addr,
 696                                        hole_start, hole_end,
 697                                        err);
 698                                 goto err_obj;
 699                         }
 700
 701                         if (!drm_mm_node_allocated(&vma->node) ||
 702                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 703                                 pr_err("%s incorrect at %llx + %llx\n",
 704                                        __func__, addr, vma->size);
 705                                 i915_vma_unpin(vma);
 706                                 err = i915_vma_unbind(vma);
 707                                 err = -EINVAL;
 708                                 goto err_obj;
 709                         }
 710
 711                         i915_vma_unpin(vma);
 712                         err = i915_vma_unbind(vma);
 713                         GEM_BUG_ON(err);
 714                 }
 715
 716                 if (igt_timeout(end_time,
 717                                 "%s timed out after %d/%d\n",
 718                                 __func__, pot, fls64(hole_end - 1) - 1)) {
 719                         err = -EINTR;
 720                         goto err_obj;
 721                 }
 722         }
 723
 724 err_obj:
 725         i915_gem_object_put(obj);
 726         return err;
 727 }
 728
 729 static int drunk_hole(struct i915_address_space *vm,
 730                       u64 hole_start, u64 hole_end,
 731                       unsigned long end_time)
 732 {
 733         I915_RND_STATE(prng);
 734         unsigned int size;
 735         unsigned long flags;
 736
 737         flags = PIN_OFFSET_FIXED | PIN_USER;
 738         if (i915_is_ggtt(vm))
 739                 flags |= PIN_GLOBAL;
 740
 741         /* Keep creating larger objects until one cannot fit into the hole */
 742         for (size = 12; (hole_end - hole_start) >> size; size++) {
 743                 struct drm_i915_gem_object *obj;
 744                 unsigned int *order, count, n;
 745                 struct i915_vma *vma;
 746                 u64 hole_size;
 747                 int err = -ENODEV;
 748
 749                 hole_size = (hole_end - hole_start) >> size;
 750                 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
 751                         hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
 752                 count = hole_size >> 1;
 753                 if (!count) {
 754                         pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
 755                                  __func__, hole_start, hole_end, size, hole_size);
 756                         break;
 757                 }
 758
 759                 do {
 760                         order = i915_random_order(count, &prng);
 761                         if (order)
 762                                 break;
 763                 } while (count >>= 1);
 764                 if (!count)
 765                         return -ENOMEM;
 766                 GEM_BUG_ON(!order);
 767
 768                 /* Ignore allocation failures (i.e. don't report them as
 769                  * a test failure) as we are purposefully allocating very
 770                  * large objects without checking that we have sufficient
 771                  * memory. We expect to hit -ENOMEM.
 772                  */
 773
 774                 obj = fake_dma_object(vm->i915, BIT_ULL(size));
 775                 if (IS_ERR(obj)) {
 776                         kfree(order);
 777                         break;
 778                 }
 779
 780                 vma = i915_vma_instance(obj, vm, NULL);
 781                 if (IS_ERR(vma)) {
 782                         err = PTR_ERR(vma);
 783                         goto err_obj;
 784                 }
 785
 786                 GEM_BUG_ON(vma->size != BIT_ULL(size));
 787
 788                 for (n = 0; n < count; n++) {
 789                         u64 addr = hole_start + order[n] * BIT_ULL(size);
 790
 791                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 792                         if (err) {
 793                                 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
 794                                        __func__,
 795                                        addr, BIT_ULL(size),
 796                                        hole_start, hole_end,
 797                                        err);
 798                                 goto err_obj;
 799                         }
 800
 801                         if (!drm_mm_node_allocated(&vma->node) ||
 802                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 803                                 pr_err("%s incorrect at %llx + %llx\n",
 804                                        __func__, addr, BIT_ULL(size));
 805                                 i915_vma_unpin(vma);
 806                                 err = i915_vma_unbind(vma);
 807                                 err = -EINVAL;
 808                                 goto err_obj;
 809                         }
 810
 811                         i915_vma_unpin(vma);
 812                         err = i915_vma_unbind(vma);
 813                         GEM_BUG_ON(err);
 814
 815                         if (igt_timeout(end_time,
 816                                         "%s timed out after %d/%d\n",
 817                                         __func__, n, count)) {
 818                                 err = -EINTR;
 819                                 goto err_obj;
 820                         }
 821                 }
 822
 823 err_obj:
 824                 i915_gem_object_put(obj);
 825                 kfree(order);
 826                 if (err)
 827                         return err;
 828
 829                 cleanup_freed_objects(vm->i915);
 830         }
 831
 832         return 0;
 833 }
 834
 835 static int __shrink_hole(struct i915_address_space *vm,
 836                          u64 hole_start, u64 hole_end,
 837                          unsigned long end_time)
 838 {
 839         struct drm_i915_gem_object *obj;
 840         unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
 841         unsigned int order = 12;
 842         LIST_HEAD(objects);
 843         int err = 0;
 844         u64 addr;
 845
 846         /* Keep creating larger objects until one cannot fit into the hole */
 847         for (addr = hole_start; addr < hole_end; ) {
 848                 struct i915_vma *vma;
 849                 u64 size = BIT_ULL(order++);
 850
 851                 size = min(size, hole_end - addr);
 852                 obj = fake_dma_object(vm->i915, size);
 853                 if (IS_ERR(obj)) {
 854                         err = PTR_ERR(obj);
 855                         break;
 856                 }
 857
 858                 list_add(&obj->st_link, &objects);
 859
 860                 vma = i915_vma_instance(obj, vm, NULL);
 861                 if (IS_ERR(vma)) {
 862                         err = PTR_ERR(vma);
 863                         break;
 864                 }
 865
 866                 GEM_BUG_ON(vma->size != size);
 867
 868                 err = i915_vma_pin(vma, 0, 0, addr | flags);
 869                 if (err) {
 870                         pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
 871                                __func__, addr, size, hole_start, hole_end, err);
 872                         break;
 873                 }
 874
 875                 if (!drm_mm_node_allocated(&vma->node) ||
 876                     i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 877                         pr_err("%s incorrect at %llx + %llx\n",
 878                                __func__, addr, size);
 879                         i915_vma_unpin(vma);
 880                         err = i915_vma_unbind(vma);
 881                         err = -EINVAL;
 882                         break;
 883                 }
 884
 885                 i915_vma_unpin(vma);
 886                 addr += size;
 887
 888                 /*
 889                  * Since we are injecting allocation faults at random intervals,
 890                  * wait for this allocation to complete before we change the
 891                  * faultinjection.
 892                  */
 893                 err = i915_vma_sync(vma);
 894                 if (err)
 895                         break;
 896
 897                 if (igt_timeout(end_time,
 898                                 "%s timed out at ofset %llx [%llx - %llx]\n",
 899                                 __func__, addr, hole_start, hole_end)) {
 900                         err = -EINTR;
 901                         break;
 902                 }
 903         }
 904
 905         close_object_list(&objects, vm);
 906         cleanup_freed_objects(vm->i915);
 907         return err;
 908 }
 909
 910 static int shrink_hole(struct i915_address_space *vm,
 911                        u64 hole_start, u64 hole_end,
 912                        unsigned long end_time)
 913 {
 914         unsigned long prime;
 915         int err;
 916
 917         vm->fault_attr.probability = 999;
 918         atomic_set(&vm->fault_attr.times, -1);
 919
 920         for_each_prime_number_from(prime, 0, ULONG_MAX - 1) {
 921                 vm->fault_attr.interval = prime;
 922                 err = __shrink_hole(vm, hole_start, hole_end, end_time);
 923                 if (err)
 924                         break;
 925         }
 926
 927         memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
 928
 929         return err;
 930 }
 931
 932 static int shrink_boom(struct i915_address_space *vm,
 933                        u64 hole_start, u64 hole_end,
 934                        unsigned long end_time)
 935 {
 936         unsigned int sizes[] = { SZ_2M, SZ_1G };
 937         struct drm_i915_gem_object *purge;
 938         struct drm_i915_gem_object *explode;
 939         int err;
 940         int i;
 941
 942         /*
 943          * Catch the case which shrink_hole seems to miss. The setup here
 944          * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while
 945          * ensuring that all vma assiocated with the respective pd/pdp are
 946          * unpinned at the time.
 947          */
 948
 949         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
 950                 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
 951                 unsigned int size = sizes[i];
 952                 struct i915_vma *vma;
 953
 954                 purge = fake_dma_object(vm->i915, size);
 955                 if (IS_ERR(purge))
 956                         return PTR_ERR(purge);
 957
 958                 vma = i915_vma_instance(purge, vm, NULL);
 959                 if (IS_ERR(vma)) {
 960                         err = PTR_ERR(vma);
 961                         goto err_purge;
 962                 }
 963
 964                 err = i915_vma_pin(vma, 0, 0, flags);
 965                 if (err)
 966                         goto err_purge;
 967
 968                 /* Should now be ripe for purging */
 969                 i915_vma_unpin(vma);
 970
 971                 explode = fake_dma_object(vm->i915, size);
 972                 if (IS_ERR(explode)) {
 973                         err = PTR_ERR(explode);
 974                         goto err_purge;
 975                 }
 976
 977                 vm->fault_attr.probability = 100;
 978                 vm->fault_attr.interval = 1;
 979                 atomic_set(&vm->fault_attr.times, -1);
 980
 981                 vma = i915_vma_instance(explode, vm, NULL);
 982                 if (IS_ERR(vma)) {
 983                         err = PTR_ERR(vma);
 984                         goto err_explode;
 985                 }
 986
 987                 err = i915_vma_pin(vma, 0, 0, flags | size);
 988                 if (err)
 989                         goto err_explode;
 990
 991                 i915_vma_unpin(vma);
 992
 993                 i915_gem_object_put(purge);
 994                 i915_gem_object_put(explode);
 995
 996                 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
 997                 cleanup_freed_objects(vm->i915);
 998         }
 999
1000         return 0;
1001
1002 err_explode:
1003         i915_gem_object_put(explode);
1004 err_purge:
1005         i915_gem_object_put(purge);
1006         memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
1007         return err;
1008 }
1009
1010 static int exercise_ppgtt(struct drm_i915_private *dev_priv,
1011                           int (*func)(struct i915_address_space *vm,
1012                                       u64 hole_start, u64 hole_end,
1013                                       unsigned long end_time))
1014 {
1015         struct i915_ppgtt *ppgtt;
1016         IGT_TIMEOUT(end_time);
1017         struct file *file;
1018         int err;
1019
1020         if (!HAS_FULL_PPGTT(dev_priv))
1021                 return 0;
1022
1023         file = mock_file(dev_priv);
1024         if (IS_ERR(file))
1025                 return PTR_ERR(file);
1026
1027         ppgtt = i915_ppgtt_create(&dev_priv->gt);
1028         if (IS_ERR(ppgtt)) {
1029                 err = PTR_ERR(ppgtt);
1030                 goto out_free;
1031         }
1032         GEM_BUG_ON(offset_in_page(ppgtt->vm.total));
1033         GEM_BUG_ON(!atomic_read(&ppgtt->vm.open));
1034
1035         err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time);
1036
1037         i915_vm_put(&ppgtt->vm);
1038
1039 out_free:
1040         fput(file);
1041         return err;
1042 }
1043
1044 static int igt_ppgtt_fill(void *arg)
1045 {
1046         return exercise_ppgtt(arg, fill_hole);
1047 }
1048
1049 static int igt_ppgtt_walk(void *arg)
1050 {
1051         return exercise_ppgtt(arg, walk_hole);
1052 }
1053
1054 static int igt_ppgtt_pot(void *arg)
1055 {
1056         return exercise_ppgtt(arg, pot_hole);
1057 }
1058
1059 static int igt_ppgtt_drunk(void *arg)
1060 {
1061         return exercise_ppgtt(arg, drunk_hole);
1062 }
1063
1064 static int igt_ppgtt_lowlevel(void *arg)
1065 {
1066         return exercise_ppgtt(arg, lowlevel_hole);
1067 }
1068
1069 static int igt_ppgtt_shrink(void *arg)
1070 {
1071         return exercise_ppgtt(arg, shrink_hole);
1072 }
1073
1074 static int igt_ppgtt_shrink_boom(void *arg)
1075 {
1076         return exercise_ppgtt(arg, shrink_boom);
1077 }
1078
1079 static int sort_holes(void *priv, struct list_head *A, struct list_head *B)
1080 {
1081         struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack);
1082         struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack);
1083
1084         if (a->start < b->start)
1085                 return -1;
1086         else
1087                 return 1;
1088 }
1089
1090 static int exercise_ggtt(struct drm_i915_private *i915,
1091                          int (*func)(struct i915_address_space *vm,
1092                                      u64 hole_start, u64 hole_end,
1093                                      unsigned long end_time))
1094 {
1095         struct i915_ggtt *ggtt = &i915->ggtt;
1096         u64 hole_start, hole_end, last = 0;
1097         struct drm_mm_node *node;
1098         IGT_TIMEOUT(end_time);
1099         int err = 0;
1100
1101 restart:
1102         list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes);
1103         drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) {
1104                 if (hole_start < last)
1105                         continue;
1106
1107                 if (ggtt->vm.mm.color_adjust)
1108                         ggtt->vm.mm.color_adjust(node, 0,
1109                                                  &hole_start, &hole_end);
1110                 if (hole_start >= hole_end)
1111                         continue;
1112
1113                 err = func(&ggtt->vm, hole_start, hole_end, end_time);
1114                 if (err)
1115                         break;
1116
1117                 /* As we have manipulated the drm_mm, the list may be corrupt */
1118                 last = hole_end;
1119                 goto restart;
1120         }
1121
1122         return err;
1123 }
1124
1125 static int igt_ggtt_fill(void *arg)
1126 {
1127         return exercise_ggtt(arg, fill_hole);
1128 }
1129
1130 static int igt_ggtt_walk(void *arg)
1131 {
1132         return exercise_ggtt(arg, walk_hole);
1133 }
1134
1135 static int igt_ggtt_pot(void *arg)
1136 {
1137         return exercise_ggtt(arg, pot_hole);
1138 }
1139
1140 static int igt_ggtt_drunk(void *arg)
1141 {
1142         return exercise_ggtt(arg, drunk_hole);
1143 }
1144
1145 static int igt_ggtt_lowlevel(void *arg)
1146 {
1147         return exercise_ggtt(arg, lowlevel_hole);
1148 }
1149
1150 static int igt_ggtt_page(void *arg)
1151 {
1152         const unsigned int count = PAGE_SIZE/sizeof(u32);
1153         I915_RND_STATE(prng);
1154         struct drm_i915_private *i915 = arg;
1155         struct i915_ggtt *ggtt = &i915->ggtt;
1156         struct drm_i915_gem_object *obj;
1157         intel_wakeref_t wakeref;
1158         struct drm_mm_node tmp;
1159         unsigned int *order, n;
1160         int err;
1161
1162         if (!i915_ggtt_has_aperture(ggtt))
1163                 return 0;
1164
1165         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1166         if (IS_ERR(obj))
1167                 return PTR_ERR(obj);
1168
1169         err = i915_gem_object_pin_pages(obj);
1170         if (err)
1171                 goto out_free;
1172
1173         memset(&tmp, 0, sizeof(tmp));
1174         mutex_lock(&ggtt->vm.mutex);
1175         err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp,
1176                                           count * PAGE_SIZE, 0,
1177                                           I915_COLOR_UNEVICTABLE,
1178                                           0, ggtt->mappable_end,
1179                                           DRM_MM_INSERT_LOW);
1180         mutex_unlock(&ggtt->vm.mutex);
1181         if (err)
1182                 goto out_unpin;
1183
1184         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1185
1186         for (n = 0; n < count; n++) {
1187                 u64 offset = tmp.start + n * PAGE_SIZE;
1188
1189                 ggtt->vm.insert_page(&ggtt->vm,
1190                                      i915_gem_object_get_dma_address(obj, 0),
1191                                      offset, I915_CACHE_NONE, 0);
1192         }
1193
1194         order = i915_random_order(count, &prng);
1195         if (!order) {
1196                 err = -ENOMEM;
1197                 goto out_remove;
1198         }
1199
1200         for (n = 0; n < count; n++) {
1201                 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1202                 u32 __iomem *vaddr;
1203
1204                 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1205                 iowrite32(n, vaddr + n);
1206                 io_mapping_unmap_atomic(vaddr);
1207         }
1208         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1209
1210         i915_random_reorder(order, count, &prng);
1211         for (n = 0; n < count; n++) {
1212                 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1213                 u32 __iomem *vaddr;
1214                 u32 val;
1215
1216                 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1217                 val = ioread32(vaddr + n);
1218                 io_mapping_unmap_atomic(vaddr);
1219
1220                 if (val != n) {
1221                         pr_err("insert page failed: found %d, expected %d\n",
1222                                val, n);
1223                         err = -EINVAL;
1224                         break;
1225                 }
1226         }
1227
1228         kfree(order);
1229 out_remove:
1230         ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size);
1231         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1232         mutex_lock(&ggtt->vm.mutex);
1233         drm_mm_remove_node(&tmp);
1234         mutex_unlock(&ggtt->vm.mutex);
1235 out_unpin:
1236         i915_gem_object_unpin_pages(obj);
1237 out_free:
1238         i915_gem_object_put(obj);
1239         return err;
1240 }
1241
1242 static void track_vma_bind(struct i915_vma *vma)
1243 {
1244         struct drm_i915_gem_object *obj = vma->obj;
1245
1246         __i915_gem_object_pin_pages(obj);
1247
1248         GEM_BUG_ON(vma->pages);
1249         atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE);
1250         __i915_gem_object_pin_pages(obj);
1251         vma->pages = obj->mm.pages;
1252
1253         mutex_lock(&vma->vm->mutex);
1254         list_add_tail(&vma->vm_link, &vma->vm->bound_list);
1255         mutex_unlock(&vma->vm->mutex);
1256 }
1257
1258 static int exercise_mock(struct drm_i915_private *i915,
1259                          int (*func)(struct i915_address_space *vm,
1260                                      u64 hole_start, u64 hole_end,
1261                                      unsigned long end_time))
1262 {
1263         const u64 limit = totalram_pages() << PAGE_SHIFT;
1264         struct i915_address_space *vm;
1265         struct i915_gem_context *ctx;
1266         IGT_TIMEOUT(end_time);
1267         int err;
1268
1269         ctx = mock_context(i915, "mock");
1270         if (!ctx)
1271                 return -ENOMEM;
1272
1273         vm = i915_gem_context_get_vm_rcu(ctx);
1274         err = func(vm, 0, min(vm->total, limit), end_time);
1275         i915_vm_put(vm);
1276
1277         mock_context_close(ctx);
1278         return err;
1279 }
1280
1281 static int igt_mock_fill(void *arg)
1282 {
1283         struct i915_ggtt *ggtt = arg;
1284
1285         return exercise_mock(ggtt->vm.i915, fill_hole);
1286 }
1287
1288 static int igt_mock_walk(void *arg)
1289 {
1290         struct i915_ggtt *ggtt = arg;
1291
1292         return exercise_mock(ggtt->vm.i915, walk_hole);
1293 }
1294
1295 static int igt_mock_pot(void *arg)
1296 {
1297         struct i915_ggtt *ggtt = arg;
1298
1299         return exercise_mock(ggtt->vm.i915, pot_hole);
1300 }
1301
1302 static int igt_mock_drunk(void *arg)
1303 {
1304         struct i915_ggtt *ggtt = arg;
1305
1306         return exercise_mock(ggtt->vm.i915, drunk_hole);
1307 }
1308
1309 static int igt_gtt_reserve(void *arg)
1310 {
1311         struct i915_ggtt *ggtt = arg;
1312         struct drm_i915_gem_object *obj, *on;
1313         I915_RND_STATE(prng);
1314         LIST_HEAD(objects);
1315         u64 total;
1316         int err = -ENODEV;
1317
1318         /* i915_gem_gtt_reserve() tries to reserve the precise range
1319          * for the node, and evicts if it has to. So our test checks that
1320          * it can give us the requsted space and prevent overlaps.
1321          */
1322
1323         /* Start by filling the GGTT */
1324         for (total = 0;
1325              total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1326              total += 2 * I915_GTT_PAGE_SIZE) {
1327                 struct i915_vma *vma;
1328
1329                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1330                                                       2 * PAGE_SIZE);
1331                 if (IS_ERR(obj)) {
1332                         err = PTR_ERR(obj);
1333                         goto out;
1334                 }
1335
1336                 err = i915_gem_object_pin_pages(obj);
1337                 if (err) {
1338                         i915_gem_object_put(obj);
1339                         goto out;
1340                 }
1341
1342                 list_add(&obj->st_link, &objects);
1343
1344                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1345                 if (IS_ERR(vma)) {
1346                         err = PTR_ERR(vma);
1347                         goto out;
1348                 }
1349
1350                 mutex_lock(&ggtt->vm.mutex);
1351                 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1352                                            obj->base.size,
1353                                            total,
1354                                            obj->cache_level,
1355                                            0);
1356                 mutex_unlock(&ggtt->vm.mutex);
1357                 if (err) {
1358                         pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n",
1359                                total, ggtt->vm.total, err);
1360                         goto out;
1361                 }
1362                 track_vma_bind(vma);
1363
1364                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1365                 if (vma->node.start != total ||
1366                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1367                         pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1368                                vma->node.start, vma->node.size,
1369                                total, 2*I915_GTT_PAGE_SIZE);
1370                         err = -EINVAL;
1371                         goto out;
1372                 }
1373         }
1374
1375         /* Now we start forcing evictions */
1376         for (total = I915_GTT_PAGE_SIZE;
1377              total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1378              total += 2 * I915_GTT_PAGE_SIZE) {
1379                 struct i915_vma *vma;
1380
1381                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1382                                                       2 * PAGE_SIZE);
1383                 if (IS_ERR(obj)) {
1384                         err = PTR_ERR(obj);
1385                         goto out;
1386                 }
1387
1388                 err = i915_gem_object_pin_pages(obj);
1389                 if (err) {
1390                         i915_gem_object_put(obj);
1391                         goto out;
1392                 }
1393
1394                 list_add(&obj->st_link, &objects);
1395
1396                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1397                 if (IS_ERR(vma)) {
1398                         err = PTR_ERR(vma);
1399                         goto out;
1400                 }
1401
1402                 mutex_lock(&ggtt->vm.mutex);
1403                 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1404                                            obj->base.size,
1405                                            total,
1406                                            obj->cache_level,
1407                                            0);
1408                 mutex_unlock(&ggtt->vm.mutex);
1409                 if (err) {
1410                         pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n",
1411                                total, ggtt->vm.total, err);
1412                         goto out;
1413                 }
1414                 track_vma_bind(vma);
1415
1416                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1417                 if (vma->node.start != total ||
1418                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1419                         pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1420                                vma->node.start, vma->node.size,
1421                                total, 2*I915_GTT_PAGE_SIZE);
1422                         err = -EINVAL;
1423                         goto out;
1424                 }
1425         }
1426
1427         /* And then try at random */
1428         list_for_each_entry_safe(obj, on, &objects, st_link) {
1429                 struct i915_vma *vma;
1430                 u64 offset;
1431
1432                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1433                 if (IS_ERR(vma)) {
1434                         err = PTR_ERR(vma);
1435                         goto out;
1436                 }
1437
1438                 err = i915_vma_unbind(vma);
1439                 if (err) {
1440                         pr_err("i915_vma_unbind failed with err=%d!\n", err);
1441                         goto out;
1442                 }
1443
1444                 offset = igt_random_offset(&prng,
1445                                            0, ggtt->vm.total,
1446                                            2 * I915_GTT_PAGE_SIZE,
1447                                            I915_GTT_MIN_ALIGNMENT);
1448
1449                 mutex_lock(&ggtt->vm.mutex);
1450                 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1451                                            obj->base.size,
1452                                            offset,
1453                                            obj->cache_level,
1454                                            0);
1455                 mutex_unlock(&ggtt->vm.mutex);
1456                 if (err) {
1457                         pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n",
1458                                total, ggtt->vm.total, err);
1459                         goto out;
1460                 }
1461                 track_vma_bind(vma);
1462
1463                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1464                 if (vma->node.start != offset ||
1465                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1466                         pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1467                                vma->node.start, vma->node.size,
1468                                offset, 2*I915_GTT_PAGE_SIZE);
1469                         err = -EINVAL;
1470                         goto out;
1471                 }
1472         }
1473
1474 out:
1475         list_for_each_entry_safe(obj, on, &objects, st_link) {
1476                 i915_gem_object_unpin_pages(obj);
1477                 i915_gem_object_put(obj);
1478         }
1479         return err;
1480 }
1481
1482 static int igt_gtt_insert(void *arg)
1483 {
1484         struct i915_ggtt *ggtt = arg;
1485         struct drm_i915_gem_object *obj, *on;
1486         struct drm_mm_node tmp = {};
1487         const struct invalid_insert {
1488                 u64 size;
1489                 u64 alignment;
1490                 u64 start, end;
1491         } invalid_insert[] = {
1492                 {
1493                         ggtt->vm.total + I915_GTT_PAGE_SIZE, 0,
1494                         0, ggtt->vm.total,
1495                 },
1496                 {
1497                         2*I915_GTT_PAGE_SIZE, 0,
1498                         0, I915_GTT_PAGE_SIZE,
1499                 },
1500                 {
1501                         -(u64)I915_GTT_PAGE_SIZE, 0,
1502                         0, 4*I915_GTT_PAGE_SIZE,
1503                 },
1504                 {
1505                         -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE,
1506                         0, 4*I915_GTT_PAGE_SIZE,
1507                 },
1508                 {
1509                         I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1,
1510                         I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1,
1511                 },
1512                 {}
1513         }, *ii;
1514         LIST_HEAD(objects);
1515         u64 total;
1516         int err = -ENODEV;
1517
1518         /* i915_gem_gtt_insert() tries to allocate some free space in the GTT
1519          * to the node, evicting if required.
1520          */
1521
1522         /* Check a couple of obviously invalid requests */
1523         for (ii = invalid_insert; ii->size; ii++) {
1524                 mutex_lock(&ggtt->vm.mutex);
1525                 err = i915_gem_gtt_insert(&ggtt->vm, &tmp,
1526                                           ii->size, ii->alignment,
1527                                           I915_COLOR_UNEVICTABLE,
1528                                           ii->start, ii->end,
1529                                           0);
1530                 mutex_unlock(&ggtt->vm.mutex);
1531                 if (err != -ENOSPC) {
1532                         pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n",
1533                                ii->size, ii->alignment, ii->start, ii->end,
1534                                err);
1535                         return -EINVAL;
1536                 }
1537         }
1538
1539         /* Start by filling the GGTT */
1540         for (total = 0;
1541              total + I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1542              total += I915_GTT_PAGE_SIZE) {
1543                 struct i915_vma *vma;
1544
1545                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1546                                                       I915_GTT_PAGE_SIZE);
1547                 if (IS_ERR(obj)) {
1548                         err = PTR_ERR(obj);
1549                         goto out;
1550                 }
1551
1552                 err = i915_gem_object_pin_pages(obj);
1553                 if (err) {
1554                         i915_gem_object_put(obj);
1555                         goto out;
1556                 }
1557
1558                 list_add(&obj->st_link, &objects);
1559
1560                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1561                 if (IS_ERR(vma)) {
1562                         err = PTR_ERR(vma);
1563                         goto out;
1564                 }
1565
1566                 mutex_lock(&ggtt->vm.mutex);
1567                 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1568                                           obj->base.size, 0, obj->cache_level,
1569                                           0, ggtt->vm.total,
1570                                           0);
1571                 mutex_unlock(&ggtt->vm.mutex);
1572                 if (err == -ENOSPC) {
1573                         /* maxed out the GGTT space */
1574                         i915_gem_object_put(obj);
1575                         break;
1576                 }
1577                 if (err) {
1578                         pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n",
1579                                total, ggtt->vm.total, err);
1580                         goto out;
1581                 }
1582                 track_vma_bind(vma);
1583                 __i915_vma_pin(vma);
1584
1585                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1586         }
1587
1588         list_for_each_entry(obj, &objects, st_link) {
1589                 struct i915_vma *vma;
1590
1591                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1592                 if (IS_ERR(vma)) {
1593                         err = PTR_ERR(vma);
1594                         goto out;
1595                 }
1596
1597                 if (!drm_mm_node_allocated(&vma->node)) {
1598                         pr_err("VMA was unexpectedly evicted!\n");
1599                         err = -EINVAL;
1600                         goto out;
1601                 }
1602
1603                 __i915_vma_unpin(vma);
1604         }
1605
1606         /* If we then reinsert, we should find the same hole */
1607         list_for_each_entry_safe(obj, on, &objects, st_link) {
1608                 struct i915_vma *vma;
1609                 u64 offset;
1610
1611                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1612                 if (IS_ERR(vma)) {
1613                         err = PTR_ERR(vma);
1614                         goto out;
1615                 }
1616
1617                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1618                 offset = vma->node.start;
1619
1620                 err = i915_vma_unbind(vma);
1621                 if (err) {
1622                         pr_err("i915_vma_unbind failed with err=%d!\n", err);
1623                         goto out;
1624                 }
1625
1626                 mutex_lock(&ggtt->vm.mutex);
1627                 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1628                                           obj->base.size, 0, obj->cache_level,
1629                                           0, ggtt->vm.total,
1630                                           0);
1631                 mutex_unlock(&ggtt->vm.mutex);
1632                 if (err) {
1633                         pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n",
1634                                total, ggtt->vm.total, err);
1635                         goto out;
1636                 }
1637                 track_vma_bind(vma);
1638
1639                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1640                 if (vma->node.start != offset) {
1641                         pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n",
1642                                offset, vma->node.start);
1643                         err = -EINVAL;
1644                         goto out;
1645                 }
1646         }
1647
1648         /* And then force evictions */
1649         for (total = 0;
1650              total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1651              total += 2 * I915_GTT_PAGE_SIZE) {
1652                 struct i915_vma *vma;
1653
1654                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1655                                                       2 * I915_GTT_PAGE_SIZE);
1656                 if (IS_ERR(obj)) {
1657                         err = PTR_ERR(obj);
1658                         goto out;
1659                 }
1660
1661                 err = i915_gem_object_pin_pages(obj);
1662                 if (err) {
1663                         i915_gem_object_put(obj);
1664                         goto out;
1665                 }
1666
1667                 list_add(&obj->st_link, &objects);
1668
1669                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1670                 if (IS_ERR(vma)) {
1671                         err = PTR_ERR(vma);
1672                         goto out;
1673                 }
1674
1675                 mutex_lock(&ggtt->vm.mutex);
1676                 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1677                                           obj->base.size, 0, obj->cache_level,
1678                                           0, ggtt->vm.total,
1679                                           0);
1680                 mutex_unlock(&ggtt->vm.mutex);
1681                 if (err) {
1682                         pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n",
1683                                total, ggtt->vm.total, err);
1684                         goto out;
1685                 }
1686                 track_vma_bind(vma);
1687
1688                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1689         }
1690
1691 out:
1692         list_for_each_entry_safe(obj, on, &objects, st_link) {
1693                 i915_gem_object_unpin_pages(obj);
1694                 i915_gem_object_put(obj);
1695         }
1696         return err;
1697 }
1698
1699 int i915_gem_gtt_mock_selftests(void)
1700 {
1701         static const struct i915_subtest tests[] = {
1702                 SUBTEST(igt_mock_drunk),
1703                 SUBTEST(igt_mock_walk),
1704                 SUBTEST(igt_mock_pot),
1705                 SUBTEST(igt_mock_fill),
1706                 SUBTEST(igt_gtt_reserve),
1707                 SUBTEST(igt_gtt_insert),
1708         };
1709         struct drm_i915_private *i915;
1710         struct i915_ggtt *ggtt;
1711         int err;
1712
1713         i915 = mock_gem_device();
1714         if (!i915)
1715                 return -ENOMEM;
1716
1717         ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL);
1718         if (!ggtt) {
1719                 err = -ENOMEM;
1720                 goto out_put;
1721         }
1722         mock_init_ggtt(i915, ggtt);
1723
1724         err = i915_subtests(tests, ggtt);
1725
1726         mock_device_flush(i915);
1727         i915_gem_drain_freed_objects(i915);
1728         mock_fini_ggtt(ggtt);
1729         kfree(ggtt);
1730 out_put:
1731         mock_destroy_device(i915);
1732         return err;
1733 }
1734
1735 static int context_sync(struct intel_context *ce)
1736 {
1737         struct i915_request *rq;
1738         long timeout;
1739
1740         rq = intel_context_create_request(ce);
1741         if (IS_ERR(rq))
1742                 return PTR_ERR(rq);
1743
1744         i915_request_get(rq);
1745         i915_request_add(rq);
1746
1747         timeout = i915_request_wait(rq, 0, HZ / 5);
1748         i915_request_put(rq);
1749
1750         return timeout < 0 ? -EIO : 0;
1751 }
1752
1753 static struct i915_request *
1754 submit_batch(struct intel_context *ce, u64 addr)
1755 {
1756         struct i915_request *rq;
1757         int err;
1758
1759         rq = intel_context_create_request(ce);
1760         if (IS_ERR(rq))
1761                 return rq;
1762
1763         err = 0;
1764         if (rq->engine->emit_init_breadcrumb) /* detect a hang */
1765                 err = rq->engine->emit_init_breadcrumb(rq);
1766         if (err == 0)
1767                 err = rq->engine->emit_bb_start(rq, addr, 0, 0);
1768
1769         if (err == 0)
1770                 i915_request_get(rq);
1771         i915_request_add(rq);
1772
1773         return err ? ERR_PTR(err) : rq;
1774 }
1775
1776 static u32 *spinner(u32 *batch, int i)
1777 {
1778         return batch + i * 64 / sizeof(*batch) + 4;
1779 }
1780
1781 static void end_spin(u32 *batch, int i)
1782 {
1783         *spinner(batch, i) = MI_BATCH_BUFFER_END;
1784         wmb();
1785 }
1786
1787 static int igt_cs_tlb(void *arg)
1788 {
1789         const unsigned int count = PAGE_SIZE / 64;
1790         const unsigned int chunk_size = count * PAGE_SIZE;
1791         struct drm_i915_private *i915 = arg;
1792         struct drm_i915_gem_object *bbe, *act, *out;
1793         struct i915_gem_engines_iter it;
1794         struct i915_address_space *vm;
1795         struct i915_gem_context *ctx;
1796         struct intel_context *ce;
1797         struct i915_vma *vma;
1798         I915_RND_STATE(prng);
1799         struct file *file;
1800         unsigned int i;
1801         u32 *result;
1802         u32 *batch;
1803         int err = 0;
1804
1805         /*
1806          * Our mission here is to fool the hardware to execute something
1807          * from scratch as it has not seen the batch move (due to missing
1808          * the TLB invalidate).
1809          */
1810
1811         file = mock_file(i915);
1812         if (IS_ERR(file))
1813                 return PTR_ERR(file);
1814
1815         ctx = live_context(i915, file);
1816         if (IS_ERR(ctx)) {
1817                 err = PTR_ERR(ctx);
1818                 goto out_unlock;
1819         }
1820
1821         vm = i915_gem_context_get_vm_rcu(ctx);
1822         if (i915_is_ggtt(vm))
1823                 goto out_vm;
1824
1825         /* Create two pages; dummy we prefill the TLB, and intended */
1826         bbe = i915_gem_object_create_internal(i915, PAGE_SIZE);
1827         if (IS_ERR(bbe)) {
1828                 err = PTR_ERR(bbe);
1829                 goto out_vm;
1830         }
1831
1832         batch = i915_gem_object_pin_map(bbe, I915_MAP_WC);
1833         if (IS_ERR(batch)) {
1834                 err = PTR_ERR(batch);
1835                 goto out_put_bbe;
1836         }
1837         memset32(batch, MI_BATCH_BUFFER_END, PAGE_SIZE / sizeof(u32));
1838         i915_gem_object_flush_map(bbe);
1839         i915_gem_object_unpin_map(bbe);
1840
1841         act = i915_gem_object_create_internal(i915, PAGE_SIZE);
1842         if (IS_ERR(act)) {
1843                 err = PTR_ERR(act);
1844                 goto out_put_bbe;
1845         }
1846
1847         /* Track the execution of each request by writing into different slot */
1848         batch = i915_gem_object_pin_map(act, I915_MAP_WC);
1849         if (IS_ERR(batch)) {
1850                 err = PTR_ERR(batch);
1851                 goto out_put_act;
1852         }
1853         for (i = 0; i < count; i++) {
1854                 u32 *cs = batch + i * 64 / sizeof(*cs);
1855                 u64 addr = (vm->total - PAGE_SIZE) + i * sizeof(u32);
1856
1857                 GEM_BUG_ON(INTEL_GEN(i915) < 6);
1858                 cs[0] = MI_STORE_DWORD_IMM_GEN4;
1859                 if (INTEL_GEN(i915) >= 8) {
1860                         cs[1] = lower_32_bits(addr);
1861                         cs[2] = upper_32_bits(addr);
1862                         cs[3] = i;
1863                         cs[4] = MI_NOOP;
1864                         cs[5] = MI_BATCH_BUFFER_START_GEN8;
1865                 } else {
1866                         cs[1] = 0;
1867                         cs[2] = lower_32_bits(addr);
1868                         cs[3] = i;
1869                         cs[4] = MI_NOOP;
1870                         cs[5] = MI_BATCH_BUFFER_START;
1871                 }
1872         }
1873
1874         out = i915_gem_object_create_internal(i915, PAGE_SIZE);
1875         if (IS_ERR(out)) {
1876                 err = PTR_ERR(out);
1877                 goto out_put_batch;
1878         }
1879         i915_gem_object_set_cache_coherency(out, I915_CACHING_CACHED);
1880
1881         vma = i915_vma_instance(out, vm, NULL);
1882         if (IS_ERR(vma)) {
1883                 err = PTR_ERR(vma);
1884                 goto out_put_out;
1885         }
1886
1887         err = i915_vma_pin(vma, 0, 0,
1888                            PIN_USER |
1889                            PIN_OFFSET_FIXED |
1890                            (vm->total - PAGE_SIZE));
1891         if (err)
1892                 goto out_put_out;
1893         GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE);
1894
1895         result = i915_gem_object_pin_map(out, I915_MAP_WB);
1896         if (IS_ERR(result)) {
1897                 err = PTR_ERR(result);
1898                 goto out_put_out;
1899         }
1900
1901         for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1902                 IGT_TIMEOUT(end_time);
1903                 unsigned long pass = 0;
1904
1905                 if (!intel_engine_can_store_dword(ce->engine))
1906                         continue;
1907
1908                 while (!__igt_timeout(end_time, NULL)) {
1909                         struct i915_vm_pt_stash stash = {};
1910                         struct i915_request *rq;
1911                         u64 offset;
1912
1913                         offset = igt_random_offset(&prng,
1914                                                    0, vm->total - PAGE_SIZE,
1915                                                    chunk_size, PAGE_SIZE);
1916
1917                         memset32(result, STACK_MAGIC, PAGE_SIZE / sizeof(u32));
1918
1919                         vma = i915_vma_instance(bbe, vm, NULL);
1920                         if (IS_ERR(vma)) {
1921                                 err = PTR_ERR(vma);
1922                                 goto end;
1923                         }
1924
1925                         err = vma->ops->set_pages(vma);
1926                         if (err)
1927                                 goto end;
1928
1929                         err = i915_vm_alloc_pt_stash(vm, &stash, chunk_size);
1930                         if (err)
1931                                 goto end;
1932
1933                         err = i915_vm_pin_pt_stash(vm, &stash);
1934                         if (err) {
1935                                 i915_vm_free_pt_stash(vm, &stash);
1936                                 goto end;
1937                         }
1938
1939                         vm->allocate_va_range(vm, &stash, offset, chunk_size);
1940
1941                         i915_vm_free_pt_stash(vm, &stash);
1942
1943                         /* Prime the TLB with the dummy pages */
1944                         for (i = 0; i < count; i++) {
1945                                 vma->node.start = offset + i * PAGE_SIZE;
1946                                 vm->insert_entries(vm, vma, I915_CACHE_NONE, 0);
1947
1948                                 rq = submit_batch(ce, vma->node.start);
1949                                 if (IS_ERR(rq)) {
1950                                         err = PTR_ERR(rq);
1951                                         goto end;
1952                                 }
1953                                 i915_request_put(rq);
1954                         }
1955
1956                         vma->ops->clear_pages(vma);
1957
1958                         err = context_sync(ce);
1959                         if (err) {
1960                                 pr_err("%s: dummy setup timed out\n",
1961                                        ce->engine->name);
1962                                 goto end;
1963                         }
1964
1965                         vma = i915_vma_instance(act, vm, NULL);
1966                         if (IS_ERR(vma)) {
1967                                 err = PTR_ERR(vma);
1968                                 goto end;
1969                         }
1970
1971                         err = vma->ops->set_pages(vma);
1972                         if (err)
1973                                 goto end;
1974
1975                         /* Replace the TLB with target batches */
1976                         for (i = 0; i < count; i++) {
1977                                 struct i915_request *rq;
1978                                 u32 *cs = batch + i * 64 / sizeof(*cs);
1979                                 u64 addr;
1980
1981                                 vma->node.start = offset + i * PAGE_SIZE;
1982                                 vm->insert_entries(vm, vma, I915_CACHE_NONE, 0);
1983
1984                                 addr = vma->node.start + i * 64;
1985                                 cs[4] = MI_NOOP;
1986                                 cs[6] = lower_32_bits(addr);
1987                                 cs[7] = upper_32_bits(addr);
1988                                 wmb();
1989
1990                                 rq = submit_batch(ce, addr);
1991                                 if (IS_ERR(rq)) {
1992                                         err = PTR_ERR(rq);
1993                                         goto end;
1994                                 }
1995
1996                                 /* Wait until the context chain has started */
1997                                 if (i == 0) {
1998                                         while (READ_ONCE(result[i]) &&
1999                                                !i915_request_completed(rq))
2000                                                 cond_resched();
2001                                 } else {
2002                                         end_spin(batch, i - 1);
2003                                 }
2004
2005                                 i915_request_put(rq);
2006                         }
2007                         end_spin(batch, count - 1);
2008
2009                         vma->ops->clear_pages(vma);
2010
2011                         err = context_sync(ce);
2012                         if (err) {
2013                                 pr_err("%s: writes timed out\n",
2014                                        ce->engine->name);
2015                                 goto end;
2016                         }
2017
2018                         for (i = 0; i < count; i++) {
2019                                 if (result[i] != i) {
2020                                         pr_err("%s: Write lost on pass %lu, at offset %llx, index %d, found %x, expected %x\n",
2021                                                ce->engine->name, pass,
2022                                                offset, i, result[i], i);
2023                                         err = -EINVAL;
2024                                         goto end;
2025                                 }
2026                         }
2027
2028                         vm->clear_range(vm, offset, chunk_size);
2029                         pass++;
2030                 }
2031         }
2032 end:
2033         if (igt_flush_test(i915))
2034                 err = -EIO;
2035         i915_gem_context_unlock_engines(ctx);
2036         i915_gem_object_unpin_map(out);
2037 out_put_out:
2038         i915_gem_object_put(out);
2039 out_put_batch:
2040         i915_gem_object_unpin_map(act);
2041 out_put_act:
2042         i915_gem_object_put(act);
2043 out_put_bbe:
2044         i915_gem_object_put(bbe);
2045 out_vm:
2046         i915_vm_put(vm);
2047 out_unlock:
2048         fput(file);
2049         return err;
2050 }
2051
2052 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915)
2053 {
2054         static const struct i915_subtest tests[] = {
2055                 SUBTEST(igt_ppgtt_alloc),
2056                 SUBTEST(igt_ppgtt_lowlevel),
2057                 SUBTEST(igt_ppgtt_drunk),
2058                 SUBTEST(igt_ppgtt_walk),
2059                 SUBTEST(igt_ppgtt_pot),
2060                 SUBTEST(igt_ppgtt_fill),
2061                 SUBTEST(igt_ppgtt_shrink),
2062                 SUBTEST(igt_ppgtt_shrink_boom),
2063                 SUBTEST(igt_ggtt_lowlevel),
2064                 SUBTEST(igt_ggtt_drunk),
2065                 SUBTEST(igt_ggtt_walk),
2066                 SUBTEST(igt_ggtt_pot),
2067                 SUBTEST(igt_ggtt_fill),
2068                 SUBTEST(igt_ggtt_page),
2069                 SUBTEST(igt_cs_tlb),
2070         };
2071
2072         GEM_BUG_ON(offset_in_page(i915->ggtt.vm.total));
2073
2074         return i915_subtests(tests, i915);
2075 }