drivers/gpu/drm/i915/selftests/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/list_sort.h>
  26 #include <linux/prime_numbers.h>
  27
  28 #include "gem/i915_gem_context.h"
  29 #include "gem/i915_gem_internal.h"
  30 #include "gem/i915_gem_region.h"
  31 #include "gem/selftests/mock_context.h"
  32 #include "gt/intel_context.h"
  33 #include "gt/intel_gpu_commands.h"
  34 #include "gt/intel_gtt.h"
  35
  36 #include "i915_random.h"
  37 #include "i915_selftest.h"
  38 #include "i915_vma_resource.h"
  39
  40 #include "mock_drm.h"
  41 #include "mock_gem_device.h"
  42 #include "mock_gtt.h"
  43 #include "igt_flush_test.h"
  44
  45 static void cleanup_freed_objects(struct drm_i915_private *i915)
  46 {
  47         i915_gem_drain_freed_objects(i915);
  48 }
  49
  50 static void fake_free_pages(struct drm_i915_gem_object *obj,
  51                             struct sg_table *pages)
  52 {
  53         sg_free_table(pages);
  54         kfree(pages);
  55 }
  56
  57 static int fake_get_pages(struct drm_i915_gem_object *obj)
  58 {
  59 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
  60 #define PFN_BIAS 0x1000
  61         struct sg_table *pages;
  62         struct scatterlist *sg;
  63         unsigned int sg_page_sizes;
  64         typeof(obj->base.size) rem;
  65
  66         pages = kmalloc(sizeof(*pages), GFP);
  67         if (!pages)
  68                 return -ENOMEM;
  69
  70         rem = round_up(obj->base.size, BIT(31)) >> 31;
  71         if (sg_alloc_table(pages, rem, GFP)) {
  72                 kfree(pages);
  73                 return -ENOMEM;
  74         }
  75
  76         sg_page_sizes = 0;
  77         rem = obj->base.size;
  78         for (sg = pages->sgl; sg; sg = sg_next(sg)) {
  79                 unsigned long len = min_t(typeof(rem), rem, BIT(31));
  80
  81                 GEM_BUG_ON(!len);
  82                 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0);
  83                 sg_dma_address(sg) = page_to_phys(sg_page(sg));
  84                 sg_dma_len(sg) = len;
  85                 sg_page_sizes |= len;
  86
  87                 rem -= len;
  88         }
  89         GEM_BUG_ON(rem);
  90
  91         __i915_gem_object_set_pages(obj, pages, sg_page_sizes);
  92
  93         return 0;
  94 #undef GFP
  95 }
  96
  97 static void fake_put_pages(struct drm_i915_gem_object *obj,
  98                            struct sg_table *pages)
  99 {
 100         fake_free_pages(obj, pages);
 101         obj->mm.dirty = false;
 102 }
 103
 104 static const struct drm_i915_gem_object_ops fake_ops = {
 105         .name = "fake-gem",
 106         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 107         .get_pages = fake_get_pages,
 108         .put_pages = fake_put_pages,
 109 };
 110
 111 static struct drm_i915_gem_object *
 112 fake_dma_object(struct drm_i915_private *i915, u64 size)
 113 {
 114         static struct lock_class_key lock_class;
 115         struct drm_i915_gem_object *obj;
 116
 117         GEM_BUG_ON(!size);
 118         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 119
 120         if (overflows_type(size, obj->base.size))
 121                 return ERR_PTR(-E2BIG);
 122
 123         obj = i915_gem_object_alloc();
 124         if (!obj)
 125                 goto err;
 126
 127         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 128         i915_gem_object_init(obj, &fake_ops, &lock_class, 0);
 129
 130         i915_gem_object_set_volatile(obj);
 131
 132         obj->write_domain = I915_GEM_DOMAIN_CPU;
 133         obj->read_domains = I915_GEM_DOMAIN_CPU;
 134         obj->cache_level = I915_CACHE_NONE;
 135
 136         /* Preallocate the "backing storage" */
 137         if (i915_gem_object_pin_pages_unlocked(obj))
 138                 goto err_obj;
 139
 140         i915_gem_object_unpin_pages(obj);
 141         return obj;
 142
 143 err_obj:
 144         i915_gem_object_put(obj);
 145 err:
 146         return ERR_PTR(-ENOMEM);
 147 }
 148
 149 static int igt_ppgtt_alloc(void *arg)
 150 {
 151         struct drm_i915_private *dev_priv = arg;
 152         struct i915_ppgtt *ppgtt;
 153         struct i915_gem_ww_ctx ww;
 154         u64 size, last, limit;
 155         int err = 0;
 156
 157         /* Allocate a ppggt and try to fill the entire range */
 158
 159         if (!HAS_PPGTT(dev_priv))
 160                 return 0;
 161
 162         ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0);
 163         if (IS_ERR(ppgtt))
 164                 return PTR_ERR(ppgtt);
 165
 166         if (!ppgtt->vm.allocate_va_range)
 167                 goto err_ppgtt_cleanup;
 168
 169         /*
 170          * While we only allocate the page tables here and so we could
 171          * address a much larger GTT than we could actually fit into
 172          * RAM, a practical limit is the amount of physical pages in the system.
 173          * This should ensure that we do not run into the oomkiller during
 174          * the test and take down the machine wilfully.
 175          */
 176         limit = totalram_pages() << PAGE_SHIFT;
 177         limit = min(ppgtt->vm.total, limit);
 178
 179         i915_gem_ww_ctx_init(&ww, false);
 180 retry:
 181         err = i915_vm_lock_objects(&ppgtt->vm, &ww);
 182         if (err)
 183                 goto err_ppgtt_cleanup;
 184
 185         /* Check we can allocate the entire range */
 186         for (size = 4096; size <= limit; size <<= 2) {
 187                 struct i915_vm_pt_stash stash = {};
 188
 189                 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size);
 190                 if (err)
 191                         goto err_ppgtt_cleanup;
 192
 193                 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
 194                 if (err) {
 195                         i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 196                         goto err_ppgtt_cleanup;
 197                 }
 198
 199                 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, size);
 200                 cond_resched();
 201
 202                 ppgtt->vm.clear_range(&ppgtt->vm, 0, size);
 203
 204                 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 205         }
 206
 207         /* Check we can incrementally allocate the entire range */
 208         for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) {
 209                 struct i915_vm_pt_stash stash = {};
 210
 211                 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size - last);
 212                 if (err)
 213                         goto err_ppgtt_cleanup;
 214
 215                 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
 216                 if (err) {
 217                         i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 218                         goto err_ppgtt_cleanup;
 219                 }
 220
 221                 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash,
 222                                             last, size - last);
 223                 cond_resched();
 224
 225                 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 226         }
 227
 228 err_ppgtt_cleanup:
 229         if (err == -EDEADLK) {
 230                 err = i915_gem_ww_ctx_backoff(&ww);
 231                 if (!err)
 232                         goto retry;
 233         }
 234         i915_gem_ww_ctx_fini(&ww);
 235
 236         i915_vm_put(&ppgtt->vm);
 237         return err;
 238 }
 239
 240 static int lowlevel_hole(struct i915_address_space *vm,
 241                          u64 hole_start, u64 hole_end,
 242                          unsigned long end_time)
 243 {
 244         const unsigned int min_alignment =
 245                 i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
 246         I915_RND_STATE(seed_prng);
 247         struct i915_vma_resource *mock_vma_res;
 248         unsigned int size;
 249
 250         mock_vma_res = kzalloc(sizeof(*mock_vma_res), GFP_KERNEL);
 251         if (!mock_vma_res)
 252                 return -ENOMEM;
 253
 254         /* Keep creating larger objects until one cannot fit into the hole */
 255         for (size = 12; (hole_end - hole_start) >> size; size++) {
 256                 I915_RND_SUBSTATE(prng, seed_prng);
 257                 struct drm_i915_gem_object *obj;
 258                 unsigned int *order, count, n;
 259                 u64 hole_size, aligned_size;
 260
 261                 aligned_size = max_t(u32, ilog2(min_alignment), size);
 262                 hole_size = (hole_end - hole_start) >> aligned_size;
 263                 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
 264                         hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
 265                 count = hole_size >> 1;
 266                 if (!count) {
 267                         pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
 268                                  __func__, hole_start, hole_end, size, hole_size);
 269                         break;
 270                 }
 271
 272                 do {
 273                         order = i915_random_order(count, &prng);
 274                         if (order)
 275                                 break;
 276                 } while (count >>= 1);
 277                 if (!count) {
 278                         kfree(mock_vma_res);
 279                         return -ENOMEM;
 280                 }
 281                 GEM_BUG_ON(!order);
 282
 283                 GEM_BUG_ON(count * BIT_ULL(aligned_size) > vm->total);
 284                 GEM_BUG_ON(hole_start + count * BIT_ULL(aligned_size) > hole_end);
 285
 286                 /* Ignore allocation failures (i.e. don't report them as
 287                  * a test failure) as we are purposefully allocating very
 288                  * large objects without checking that we have sufficient
 289                  * memory. We expect to hit -ENOMEM.
 290                  */
 291
 292                 obj = fake_dma_object(vm->i915, BIT_ULL(size));
 293                 if (IS_ERR(obj)) {
 294                         kfree(order);
 295                         break;
 296                 }
 297
 298                 GEM_BUG_ON(obj->base.size != BIT_ULL(size));
 299
 300                 if (i915_gem_object_pin_pages_unlocked(obj)) {
 301                         i915_gem_object_put(obj);
 302                         kfree(order);
 303                         break;
 304                 }
 305
 306                 for (n = 0; n < count; n++) {
 307                         u64 addr = hole_start + order[n] * BIT_ULL(aligned_size);
 308                         intel_wakeref_t wakeref;
 309
 310                         GEM_BUG_ON(addr + BIT_ULL(aligned_size) > vm->total);
 311
 312                         if (igt_timeout(end_time,
 313                                         "%s timed out before %d/%d\n",
 314                                         __func__, n, count)) {
 315                                 hole_end = hole_start; /* quit */
 316                                 break;
 317                         }
 318
 319                         if (vm->allocate_va_range) {
 320                                 struct i915_vm_pt_stash stash = {};
 321                                 struct i915_gem_ww_ctx ww;
 322                                 int err;
 323
 324                                 i915_gem_ww_ctx_init(&ww, false);
 325 retry:
 326                                 err = i915_vm_lock_objects(vm, &ww);
 327                                 if (err)
 328                                         goto alloc_vm_end;
 329
 330                                 err = -ENOMEM;
 331                                 if (i915_vm_alloc_pt_stash(vm, &stash,
 332                                                            BIT_ULL(size)))
 333                                         goto alloc_vm_end;
 334
 335                                 err = i915_vm_map_pt_stash(vm, &stash);
 336                                 if (!err)
 337                                         vm->allocate_va_range(vm, &stash,
 338                                                               addr, BIT_ULL(size));
 339                                 i915_vm_free_pt_stash(vm, &stash);
 340 alloc_vm_end:
 341                                 if (err == -EDEADLK) {
 342                                         err = i915_gem_ww_ctx_backoff(&ww);
 343                                         if (!err)
 344                                                 goto retry;
 345                                 }
 346                                 i915_gem_ww_ctx_fini(&ww);
 347
 348                                 if (err)
 349                                         break;
 350                         }
 351
 352                         mock_vma_res->bi.pages = obj->mm.pages;
 353                         mock_vma_res->node_size = BIT_ULL(aligned_size);
 354                         mock_vma_res->start = addr;
 355
 356                         with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
 357                           vm->insert_entries(vm, mock_vma_res,
 358                                                    I915_CACHE_NONE, 0);
 359                 }
 360                 count = n;
 361
 362                 i915_random_reorder(order, count, &prng);
 363                 for (n = 0; n < count; n++) {
 364                         u64 addr = hole_start + order[n] * BIT_ULL(aligned_size);
 365                         intel_wakeref_t wakeref;
 366
 367                         GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
 368                         with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
 369                                 vm->clear_range(vm, addr, BIT_ULL(size));
 370                 }
 371
 372                 i915_gem_object_unpin_pages(obj);
 373                 i915_gem_object_put(obj);
 374
 375                 kfree(order);
 376
 377                 cleanup_freed_objects(vm->i915);
 378         }
 379
 380         kfree(mock_vma_res);
 381         return 0;
 382 }
 383
 384 static void close_object_list(struct list_head *objects,
 385                               struct i915_address_space *vm)
 386 {
 387         struct drm_i915_gem_object *obj, *on;
 388         int ignored;
 389
 390         list_for_each_entry_safe(obj, on, objects, st_link) {
 391                 struct i915_vma *vma;
 392
 393                 vma = i915_vma_instance(obj, vm, NULL);
 394                 if (!IS_ERR(vma))
 395                         ignored = i915_vma_unbind_unlocked(vma);
 396
 397                 list_del(&obj->st_link);
 398                 i915_gem_object_put(obj);
 399         }
 400 }
 401
 402 static int fill_hole(struct i915_address_space *vm,
 403                      u64 hole_start, u64 hole_end,
 404                      unsigned long end_time)
 405 {
 406         const u64 hole_size = hole_end - hole_start;
 407         struct drm_i915_gem_object *obj;
 408         const unsigned int min_alignment =
 409                 i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
 410         const unsigned long max_pages =
 411                 min_t(u64, ULONG_MAX - 1, (hole_size / 2) >> ilog2(min_alignment));
 412         const unsigned long max_step = max(int_sqrt(max_pages), 2UL);
 413         unsigned long npages, prime, flags;
 414         struct i915_vma *vma;
 415         LIST_HEAD(objects);
 416         int err;
 417
 418         /* Try binding many VMA working inwards from either edge */
 419
 420         flags = PIN_OFFSET_FIXED | PIN_USER;
 421         if (i915_is_ggtt(vm))
 422                 flags |= PIN_GLOBAL;
 423
 424         for_each_prime_number_from(prime, 2, max_step) {
 425                 for (npages = 1; npages <= max_pages; npages *= prime) {
 426                         const u64 full_size = npages << PAGE_SHIFT;
 427                         const struct {
 428                                 const char *name;
 429                                 u64 offset;
 430                                 int step;
 431                         } phases[] = {
 432                                 { "top-down", hole_end, -1, },
 433                                 { "bottom-up", hole_start, 1, },
 434                                 { }
 435                         }, *p;
 436
 437                         obj = fake_dma_object(vm->i915, full_size);
 438                         if (IS_ERR(obj))
 439                                 break;
 440
 441                         list_add(&obj->st_link, &objects);
 442
 443                         /* Align differing sized objects against the edges, and
 444                          * check we don't walk off into the void when binding
 445                          * them into the GTT.
 446                          */
 447                         for (p = phases; p->name; p++) {
 448                                 u64 offset;
 449
 450                                 offset = p->offset;
 451                                 list_for_each_entry(obj, &objects, st_link) {
 452                                         u64 aligned_size = round_up(obj->base.size,
 453                                                                     min_alignment);
 454
 455                                         vma = i915_vma_instance(obj, vm, NULL);
 456                                         if (IS_ERR(vma))
 457                                                 continue;
 458
 459                                         if (p->step < 0) {
 460                                                 if (offset < hole_start + aligned_size)
 461                                                         break;
 462                                                 offset -= aligned_size;
 463                                         }
 464
 465                                         err = i915_vma_pin(vma, 0, 0, offset | flags);
 466                                         if (err) {
 467                                                 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
 468                                                        __func__, p->name, err, npages, prime, offset);
 469                                                 goto err;
 470                                         }
 471
 472                                         if (!drm_mm_node_allocated(&vma->node) ||
 473                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 474                                                 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 475                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 476                                                        offset);
 477                                                 err = -EINVAL;
 478                                                 goto err;
 479                                         }
 480
 481                                         i915_vma_unpin(vma);
 482
 483                                         if (p->step > 0) {
 484                                                 if (offset + aligned_size > hole_end)
 485                                                         break;
 486                                                 offset += aligned_size;
 487                                         }
 488                                 }
 489
 490                                 offset = p->offset;
 491                                 list_for_each_entry(obj, &objects, st_link) {
 492                                         u64 aligned_size = round_up(obj->base.size,
 493                                                                     min_alignment);
 494
 495                                         vma = i915_vma_instance(obj, vm, NULL);
 496                                         if (IS_ERR(vma))
 497                                                 continue;
 498
 499                                         if (p->step < 0) {
 500                                                 if (offset < hole_start + aligned_size)
 501                                                         break;
 502                                                 offset -= aligned_size;
 503                                         }
 504
 505                                         if (!drm_mm_node_allocated(&vma->node) ||
 506                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 507                                                 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n",
 508                                                        __func__, p->name, vma->node.start, vma->node.size,
 509                                                        offset);
 510                                                 err = -EINVAL;
 511                                                 goto err;
 512                                         }
 513
 514                                         err = i915_vma_unbind_unlocked(vma);
 515                                         if (err) {
 516                                                 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n",
 517                                                        __func__, p->name, vma->node.start, vma->node.size,
 518                                                        err);
 519                                                 goto err;
 520                                         }
 521
 522                                         if (p->step > 0) {
 523                                                 if (offset + aligned_size > hole_end)
 524                                                         break;
 525                                                 offset += aligned_size;
 526                                         }
 527                                 }
 528
 529                                 offset = p->offset;
 530                                 list_for_each_entry_reverse(obj, &objects, st_link) {
 531                                         u64 aligned_size = round_up(obj->base.size,
 532                                                                     min_alignment);
 533
 534                                         vma = i915_vma_instance(obj, vm, NULL);
 535                                         if (IS_ERR(vma))
 536                                                 continue;
 537
 538                                         if (p->step < 0) {
 539                                                 if (offset < hole_start + aligned_size)
 540                                                         break;
 541                                                 offset -= aligned_size;
 542                                         }
 543
 544                                         err = i915_vma_pin(vma, 0, 0, offset | flags);
 545                                         if (err) {
 546                                                 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
 547                                                        __func__, p->name, err, npages, prime, offset);
 548                                                 goto err;
 549                                         }
 550
 551                                         if (!drm_mm_node_allocated(&vma->node) ||
 552                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 553                                                 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 554                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 555                                                        offset);
 556                                                 err = -EINVAL;
 557                                                 goto err;
 558                                         }
 559
 560                                         i915_vma_unpin(vma);
 561
 562                                         if (p->step > 0) {
 563                                                 if (offset + aligned_size > hole_end)
 564                                                         break;
 565                                                 offset += aligned_size;
 566                                         }
 567                                 }
 568
 569                                 offset = p->offset;
 570                                 list_for_each_entry_reverse(obj, &objects, st_link) {
 571                                         u64 aligned_size = round_up(obj->base.size,
 572                                                                     min_alignment);
 573
 574                                         vma = i915_vma_instance(obj, vm, NULL);
 575                                         if (IS_ERR(vma))
 576                                                 continue;
 577
 578                                         if (p->step < 0) {
 579                                                 if (offset < hole_start + aligned_size)
 580                                                         break;
 581                                                 offset -= aligned_size;
 582                                         }
 583
 584                                         if (!drm_mm_node_allocated(&vma->node) ||
 585                                             i915_vma_misplaced(vma, 0, 0, offset | flags)) {
 586                                                 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 587                                                        __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 588                                                        offset);
 589                                                 err = -EINVAL;
 590                                                 goto err;
 591                                         }
 592
 593                                         err = i915_vma_unbind_unlocked(vma);
 594                                         if (err) {
 595                                                 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n",
 596                                                        __func__, p->name, vma->node.start, vma->node.size,
 597                                                        err);
 598                                                 goto err;
 599                                         }
 600
 601                                         if (p->step > 0) {
 602                                                 if (offset + aligned_size > hole_end)
 603                                                         break;
 604                                                 offset += aligned_size;
 605                                         }
 606                                 }
 607                         }
 608
 609                         if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n",
 610                                         __func__, npages, prime)) {
 611                                 err = -EINTR;
 612                                 goto err;
 613                         }
 614                 }
 615
 616                 close_object_list(&objects, vm);
 617                 cleanup_freed_objects(vm->i915);
 618         }
 619
 620         return 0;
 621
 622 err:
 623         close_object_list(&objects, vm);
 624         return err;
 625 }
 626
 627 static int walk_hole(struct i915_address_space *vm,
 628                      u64 hole_start, u64 hole_end,
 629                      unsigned long end_time)
 630 {
 631         const u64 hole_size = hole_end - hole_start;
 632         const unsigned long max_pages =
 633                 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT);
 634         unsigned long min_alignment;
 635         unsigned long flags;
 636         u64 size;
 637
 638         /* Try binding a single VMA in different positions within the hole */
 639
 640         flags = PIN_OFFSET_FIXED | PIN_USER;
 641         if (i915_is_ggtt(vm))
 642                 flags |= PIN_GLOBAL;
 643
 644         min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
 645
 646         for_each_prime_number_from(size, 1, max_pages) {
 647                 struct drm_i915_gem_object *obj;
 648                 struct i915_vma *vma;
 649                 u64 addr;
 650                 int err = 0;
 651
 652                 obj = fake_dma_object(vm->i915, size << PAGE_SHIFT);
 653                 if (IS_ERR(obj))
 654                         break;
 655
 656                 vma = i915_vma_instance(obj, vm, NULL);
 657                 if (IS_ERR(vma)) {
 658                         err = PTR_ERR(vma);
 659                         goto err_put;
 660                 }
 661
 662                 for (addr = hole_start;
 663                      addr + obj->base.size < hole_end;
 664                      addr += round_up(obj->base.size, min_alignment)) {
 665                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 666                         if (err) {
 667                                 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n",
 668                                        __func__, addr, vma->size,
 669                                        hole_start, hole_end, err);
 670                                 goto err_put;
 671                         }
 672                         i915_vma_unpin(vma);
 673
 674                         if (!drm_mm_node_allocated(&vma->node) ||
 675                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 676                                 pr_err("%s incorrect at %llx + %llx\n",
 677                                        __func__, addr, vma->size);
 678                                 err = -EINVAL;
 679                                 goto err_put;
 680                         }
 681
 682                         err = i915_vma_unbind_unlocked(vma);
 683                         if (err) {
 684                                 pr_err("%s unbind failed at %llx + %llx  with err=%d\n",
 685                                        __func__, addr, vma->size, err);
 686                                 goto err_put;
 687                         }
 688
 689                         GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
 690
 691                         if (igt_timeout(end_time,
 692                                         "%s timed out at %llx\n",
 693                                         __func__, addr)) {
 694                                 err = -EINTR;
 695                                 goto err_put;
 696                         }
 697                 }
 698
 699 err_put:
 700                 i915_gem_object_put(obj);
 701                 if (err)
 702                         return err;
 703
 704                 cleanup_freed_objects(vm->i915);
 705         }
 706
 707         return 0;
 708 }
 709
 710 static int pot_hole(struct i915_address_space *vm,
 711                     u64 hole_start, u64 hole_end,
 712                     unsigned long end_time)
 713 {
 714         struct drm_i915_gem_object *obj;
 715         struct i915_vma *vma;
 716         unsigned int min_alignment;
 717         unsigned long flags;
 718         unsigned int pot;
 719         int err = 0;
 720
 721         flags = PIN_OFFSET_FIXED | PIN_USER;
 722         if (i915_is_ggtt(vm))
 723                 flags |= PIN_GLOBAL;
 724
 725         min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
 726
 727         obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE);
 728         if (IS_ERR(obj))
 729                 return PTR_ERR(obj);
 730
 731         vma = i915_vma_instance(obj, vm, NULL);
 732         if (IS_ERR(vma)) {
 733                 err = PTR_ERR(vma);
 734                 goto err_obj;
 735         }
 736
 737         /* Insert a pair of pages across every pot boundary within the hole */
 738         for (pot = fls64(hole_end - 1) - 1;
 739              pot > ilog2(2 * min_alignment);
 740              pot--) {
 741                 u64 step = BIT_ULL(pot);
 742                 u64 addr;
 743
 744                 for (addr = round_up(hole_start + min_alignment, step) - min_alignment;
 745                      hole_end > addr && hole_end - addr >= 2 * min_alignment;
 746                      addr += step) {
 747                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 748                         if (err) {
 749                                 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n",
 750                                        __func__,
 751                                        addr,
 752                                        hole_start, hole_end,
 753                                        err);
 754                                 goto err_obj;
 755                         }
 756
 757                         if (!drm_mm_node_allocated(&vma->node) ||
 758                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 759                                 pr_err("%s incorrect at %llx + %llx\n",
 760                                        __func__, addr, vma->size);
 761                                 i915_vma_unpin(vma);
 762                                 err = i915_vma_unbind_unlocked(vma);
 763                                 err = -EINVAL;
 764                                 goto err_obj;
 765                         }
 766
 767                         i915_vma_unpin(vma);
 768                         err = i915_vma_unbind_unlocked(vma);
 769                         GEM_BUG_ON(err);
 770                 }
 771
 772                 if (igt_timeout(end_time,
 773                                 "%s timed out after %d/%d\n",
 774                                 __func__, pot, fls64(hole_end - 1) - 1)) {
 775                         err = -EINTR;
 776                         goto err_obj;
 777                 }
 778         }
 779
 780 err_obj:
 781         i915_gem_object_put(obj);
 782         return err;
 783 }
 784
 785 static int drunk_hole(struct i915_address_space *vm,
 786                       u64 hole_start, u64 hole_end,
 787                       unsigned long end_time)
 788 {
 789         I915_RND_STATE(prng);
 790         unsigned int min_alignment;
 791         unsigned int size;
 792         unsigned long flags;
 793
 794         flags = PIN_OFFSET_FIXED | PIN_USER;
 795         if (i915_is_ggtt(vm))
 796                 flags |= PIN_GLOBAL;
 797
 798         min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
 799
 800         /* Keep creating larger objects until one cannot fit into the hole */
 801         for (size = 12; (hole_end - hole_start) >> size; size++) {
 802                 struct drm_i915_gem_object *obj;
 803                 unsigned int *order, count, n;
 804                 struct i915_vma *vma;
 805                 u64 hole_size, aligned_size;
 806                 int err = -ENODEV;
 807
 808                 aligned_size = max_t(u32, ilog2(min_alignment), size);
 809                 hole_size = (hole_end - hole_start) >> aligned_size;
 810                 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
 811                         hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
 812                 count = hole_size >> 1;
 813                 if (!count) {
 814                         pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
 815                                  __func__, hole_start, hole_end, size, hole_size);
 816                         break;
 817                 }
 818
 819                 do {
 820                         order = i915_random_order(count, &prng);
 821                         if (order)
 822                                 break;
 823                 } while (count >>= 1);
 824                 if (!count)
 825                         return -ENOMEM;
 826                 GEM_BUG_ON(!order);
 827
 828                 /* Ignore allocation failures (i.e. don't report them as
 829                  * a test failure) as we are purposefully allocating very
 830                  * large objects without checking that we have sufficient
 831                  * memory. We expect to hit -ENOMEM.
 832                  */
 833
 834                 obj = fake_dma_object(vm->i915, BIT_ULL(size));
 835                 if (IS_ERR(obj)) {
 836                         kfree(order);
 837                         break;
 838                 }
 839
 840                 vma = i915_vma_instance(obj, vm, NULL);
 841                 if (IS_ERR(vma)) {
 842                         err = PTR_ERR(vma);
 843                         goto err_obj;
 844                 }
 845
 846                 GEM_BUG_ON(vma->size != BIT_ULL(size));
 847
 848                 for (n = 0; n < count; n++) {
 849                         u64 addr = hole_start + order[n] * BIT_ULL(aligned_size);
 850
 851                         err = i915_vma_pin(vma, 0, 0, addr | flags);
 852                         if (err) {
 853                                 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
 854                                        __func__,
 855                                        addr, BIT_ULL(size),
 856                                        hole_start, hole_end,
 857                                        err);
 858                                 goto err_obj;
 859                         }
 860
 861                         if (!drm_mm_node_allocated(&vma->node) ||
 862                             i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 863                                 pr_err("%s incorrect at %llx + %llx\n",
 864                                        __func__, addr, BIT_ULL(size));
 865                                 i915_vma_unpin(vma);
 866                                 err = i915_vma_unbind_unlocked(vma);
 867                                 err = -EINVAL;
 868                                 goto err_obj;
 869                         }
 870
 871                         i915_vma_unpin(vma);
 872                         err = i915_vma_unbind_unlocked(vma);
 873                         GEM_BUG_ON(err);
 874
 875                         if (igt_timeout(end_time,
 876                                         "%s timed out after %d/%d\n",
 877                                         __func__, n, count)) {
 878                                 err = -EINTR;
 879                                 goto err_obj;
 880                         }
 881                 }
 882
 883 err_obj:
 884                 i915_gem_object_put(obj);
 885                 kfree(order);
 886                 if (err)
 887                         return err;
 888
 889                 cleanup_freed_objects(vm->i915);
 890         }
 891
 892         return 0;
 893 }
 894
 895 static int __shrink_hole(struct i915_address_space *vm,
 896                          u64 hole_start, u64 hole_end,
 897                          unsigned long end_time)
 898 {
 899         struct drm_i915_gem_object *obj;
 900         unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
 901         unsigned int min_alignment;
 902         unsigned int order = 12;
 903         LIST_HEAD(objects);
 904         int err = 0;
 905         u64 addr;
 906
 907         min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
 908
 909         /* Keep creating larger objects until one cannot fit into the hole */
 910         for (addr = hole_start; addr < hole_end; ) {
 911                 struct i915_vma *vma;
 912                 u64 size = BIT_ULL(order++);
 913
 914                 size = min(size, hole_end - addr);
 915                 obj = fake_dma_object(vm->i915, size);
 916                 if (IS_ERR(obj)) {
 917                         err = PTR_ERR(obj);
 918                         break;
 919                 }
 920
 921                 list_add(&obj->st_link, &objects);
 922
 923                 vma = i915_vma_instance(obj, vm, NULL);
 924                 if (IS_ERR(vma)) {
 925                         err = PTR_ERR(vma);
 926                         break;
 927                 }
 928
 929                 GEM_BUG_ON(vma->size != size);
 930
 931                 err = i915_vma_pin(vma, 0, 0, addr | flags);
 932                 if (err) {
 933                         pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
 934                                __func__, addr, size, hole_start, hole_end, err);
 935                         break;
 936                 }
 937
 938                 if (!drm_mm_node_allocated(&vma->node) ||
 939                     i915_vma_misplaced(vma, 0, 0, addr | flags)) {
 940                         pr_err("%s incorrect at %llx + %llx\n",
 941                                __func__, addr, size);
 942                         i915_vma_unpin(vma);
 943                         err = i915_vma_unbind_unlocked(vma);
 944                         err = -EINVAL;
 945                         break;
 946                 }
 947
 948                 i915_vma_unpin(vma);
 949                 addr += round_up(size, min_alignment);
 950
 951                 /*
 952                  * Since we are injecting allocation faults at random intervals,
 953                  * wait for this allocation to complete before we change the
 954                  * faultinjection.
 955                  */
 956                 err = i915_vma_sync(vma);
 957                 if (err)
 958                         break;
 959
 960                 if (igt_timeout(end_time,
 961                                 "%s timed out at ofset %llx [%llx - %llx]\n",
 962                                 __func__, addr, hole_start, hole_end)) {
 963                         err = -EINTR;
 964                         break;
 965                 }
 966         }
 967
 968         close_object_list(&objects, vm);
 969         cleanup_freed_objects(vm->i915);
 970         return err;
 971 }
 972
 973 static int shrink_hole(struct i915_address_space *vm,
 974                        u64 hole_start, u64 hole_end,
 975                        unsigned long end_time)
 976 {
 977         unsigned long prime;
 978         int err;
 979
 980         vm->fault_attr.probability = 999;
 981         atomic_set(&vm->fault_attr.times, -1);
 982
 983         for_each_prime_number_from(prime, 0, ULONG_MAX - 1) {
 984                 vm->fault_attr.interval = prime;
 985                 err = __shrink_hole(vm, hole_start, hole_end, end_time);
 986                 if (err)
 987                         break;
 988         }
 989
 990         memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
 991
 992         return err;
 993 }
 994
 995 static int shrink_boom(struct i915_address_space *vm,
 996                        u64 hole_start, u64 hole_end,
 997                        unsigned long end_time)
 998 {
 999         unsigned int sizes[] = { SZ_2M, SZ_1G };
1000         struct drm_i915_gem_object *purge;
1001         struct drm_i915_gem_object *explode;
1002         int err;
1003         int i;
1004
1005         /*
1006          * Catch the case which shrink_hole seems to miss. The setup here
1007          * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while
1008          * ensuring that all vma assiocated with the respective pd/pdp are
1009          * unpinned at the time.
1010          */
1011
1012         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
1013                 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1014                 unsigned int size = sizes[i];
1015                 struct i915_vma *vma;
1016
1017                 purge = fake_dma_object(vm->i915, size);
1018                 if (IS_ERR(purge))
1019                         return PTR_ERR(purge);
1020
1021                 vma = i915_vma_instance(purge, vm, NULL);
1022                 if (IS_ERR(vma)) {
1023                         err = PTR_ERR(vma);
1024                         goto err_purge;
1025                 }
1026
1027                 err = i915_vma_pin(vma, 0, 0, flags);
1028                 if (err)
1029                         goto err_purge;
1030
1031                 /* Should now be ripe for purging */
1032                 i915_vma_unpin(vma);
1033
1034                 explode = fake_dma_object(vm->i915, size);
1035                 if (IS_ERR(explode)) {
1036                         err = PTR_ERR(explode);
1037                         goto err_purge;
1038                 }
1039
1040                 vm->fault_attr.probability = 100;
1041                 vm->fault_attr.interval = 1;
1042                 atomic_set(&vm->fault_attr.times, -1);
1043
1044                 vma = i915_vma_instance(explode, vm, NULL);
1045                 if (IS_ERR(vma)) {
1046                         err = PTR_ERR(vma);
1047                         goto err_explode;
1048                 }
1049
1050                 err = i915_vma_pin(vma, 0, 0, flags | size);
1051                 if (err)
1052                         goto err_explode;
1053
1054                 i915_vma_unpin(vma);
1055
1056                 i915_gem_object_put(purge);
1057                 i915_gem_object_put(explode);
1058
1059                 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
1060                 cleanup_freed_objects(vm->i915);
1061         }
1062
1063         return 0;
1064
1065 err_explode:
1066         i915_gem_object_put(explode);
1067 err_purge:
1068         i915_gem_object_put(purge);
1069         memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
1070         return err;
1071 }
1072
1073 static int misaligned_case(struct i915_address_space *vm, struct intel_memory_region *mr,
1074                            u64 addr, u64 size, unsigned long flags)
1075 {
1076         struct drm_i915_gem_object *obj;
1077         struct i915_vma *vma;
1078         int err = 0;
1079         u64 expected_vma_size, expected_node_size;
1080         bool is_stolen = mr->type == INTEL_MEMORY_STOLEN_SYSTEM ||
1081                          mr->type == INTEL_MEMORY_STOLEN_LOCAL;
1082
1083         obj = i915_gem_object_create_region(mr, size, 0, 0);
1084         if (IS_ERR(obj)) {
1085                 /* if iGVT-g or DMAR is active, stolen mem will be uninitialized */
1086                 if (PTR_ERR(obj) == -ENODEV && is_stolen)
1087                         return 0;
1088                 return PTR_ERR(obj);
1089         }
1090
1091         vma = i915_vma_instance(obj, vm, NULL);
1092         if (IS_ERR(vma)) {
1093                 err = PTR_ERR(vma);
1094                 goto err_put;
1095         }
1096
1097         err = i915_vma_pin(vma, 0, 0, addr | flags);
1098         if (err)
1099                 goto err_put;
1100         i915_vma_unpin(vma);
1101
1102         if (!drm_mm_node_allocated(&vma->node)) {
1103                 err = -EINVAL;
1104                 goto err_put;
1105         }
1106
1107         if (i915_vma_misplaced(vma, 0, 0, addr | flags)) {
1108                 err = -EINVAL;
1109                 goto err_put;
1110         }
1111
1112         expected_vma_size = round_up(size, 1 << (ffs(vma->resource->page_sizes_gtt) - 1));
1113         expected_node_size = expected_vma_size;
1114
1115         if (HAS_64K_PAGES(vm->i915) && i915_gem_object_is_lmem(obj)) {
1116                 /*
1117                  * The compact-pt should expand lmem node to 2MB for the ppGTT,
1118                  * for all other cases we should only expect 64K.
1119                  */
1120                 expected_vma_size = round_up(size, I915_GTT_PAGE_SIZE_64K);
1121                 if (NEEDS_COMPACT_PT(vm->i915) && !i915_is_ggtt(vm))
1122                         expected_node_size = round_up(size, I915_GTT_PAGE_SIZE_2M);
1123                 else
1124                         expected_node_size = round_up(size, I915_GTT_PAGE_SIZE_64K);
1125         }
1126
1127         if (vma->size != expected_vma_size || vma->node.size != expected_node_size) {
1128                 err = i915_vma_unbind_unlocked(vma);
1129                 err = -EBADSLT;
1130                 goto err_put;
1131         }
1132
1133         err = i915_vma_unbind_unlocked(vma);
1134         if (err)
1135                 goto err_put;
1136
1137         GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
1138
1139 err_put:
1140         i915_gem_object_put(obj);
1141         cleanup_freed_objects(vm->i915);
1142         return err;
1143 }
1144
1145 static int misaligned_pin(struct i915_address_space *vm,
1146                           u64 hole_start, u64 hole_end,
1147                           unsigned long end_time)
1148 {
1149         struct intel_memory_region *mr;
1150         enum intel_region_id id;
1151         unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
1152         int err = 0;
1153         u64 hole_size = hole_end - hole_start;
1154
1155         if (i915_is_ggtt(vm))
1156                 flags |= PIN_GLOBAL;
1157
1158         for_each_memory_region(mr, vm->i915, id) {
1159                 u64 min_alignment = i915_vm_min_alignment(vm, mr->type);
1160                 u64 size = min_alignment;
1161                 u64 addr = round_down(hole_start + (hole_size / 2), min_alignment);
1162
1163                 /* avoid -ENOSPC on very small hole setups */
1164                 if (hole_size < 3 * min_alignment)
1165                         continue;
1166
1167                 /* we can't test < 4k alignment due to flags being encoded in lower bits */
1168                 if (min_alignment != I915_GTT_PAGE_SIZE_4K) {
1169                         err = misaligned_case(vm, mr, addr + (min_alignment / 2), size, flags);
1170                         /* misaligned should error with -EINVAL*/
1171                         if (!err)
1172                                 err = -EBADSLT;
1173                         if (err != -EINVAL)
1174                                 return err;
1175                 }
1176
1177                 /* test for vma->size expansion to min page size */
1178                 err = misaligned_case(vm, mr, addr, PAGE_SIZE, flags);
1179                 if (err)
1180                         return err;
1181
1182                 /* test for intermediate size not expanding vma->size for large alignments */
1183                 err = misaligned_case(vm, mr, addr, size / 2, flags);
1184                 if (err)
1185                         return err;
1186         }
1187
1188         return 0;
1189 }
1190
1191 static int exercise_ppgtt(struct drm_i915_private *dev_priv,
1192                           int (*func)(struct i915_address_space *vm,
1193                                       u64 hole_start, u64 hole_end,
1194                                       unsigned long end_time))
1195 {
1196         struct i915_ppgtt *ppgtt;
1197         IGT_TIMEOUT(end_time);
1198         struct file *file;
1199         int err;
1200
1201         if (!HAS_FULL_PPGTT(dev_priv))
1202                 return 0;
1203
1204         file = mock_file(dev_priv);
1205         if (IS_ERR(file))
1206                 return PTR_ERR(file);
1207
1208         ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0);
1209         if (IS_ERR(ppgtt)) {
1210                 err = PTR_ERR(ppgtt);
1211                 goto out_free;
1212         }
1213         GEM_BUG_ON(offset_in_page(ppgtt->vm.total));
1214         assert_vm_alive(&ppgtt->vm);
1215
1216         err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time);
1217
1218         i915_vm_put(&ppgtt->vm);
1219
1220 out_free:
1221         fput(file);
1222         return err;
1223 }
1224
1225 static int igt_ppgtt_fill(void *arg)
1226 {
1227         return exercise_ppgtt(arg, fill_hole);
1228 }
1229
1230 static int igt_ppgtt_walk(void *arg)
1231 {
1232         return exercise_ppgtt(arg, walk_hole);
1233 }
1234
1235 static int igt_ppgtt_pot(void *arg)
1236 {
1237         return exercise_ppgtt(arg, pot_hole);
1238 }
1239
1240 static int igt_ppgtt_drunk(void *arg)
1241 {
1242         return exercise_ppgtt(arg, drunk_hole);
1243 }
1244
1245 static int igt_ppgtt_lowlevel(void *arg)
1246 {
1247         return exercise_ppgtt(arg, lowlevel_hole);
1248 }
1249
1250 static int igt_ppgtt_shrink(void *arg)
1251 {
1252         return exercise_ppgtt(arg, shrink_hole);
1253 }
1254
1255 static int igt_ppgtt_shrink_boom(void *arg)
1256 {
1257         return exercise_ppgtt(arg, shrink_boom);
1258 }
1259
1260 static int igt_ppgtt_misaligned_pin(void *arg)
1261 {
1262         return exercise_ppgtt(arg, misaligned_pin);
1263 }
1264
1265 static int sort_holes(void *priv, const struct list_head *A,
1266                       const struct list_head *B)
1267 {
1268         struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack);
1269         struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack);
1270
1271         if (a->start < b->start)
1272                 return -1;
1273         else
1274                 return 1;
1275 }
1276
1277 static int exercise_ggtt(struct drm_i915_private *i915,
1278                          int (*func)(struct i915_address_space *vm,
1279                                      u64 hole_start, u64 hole_end,
1280                                      unsigned long end_time))
1281 {
1282         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
1283         u64 hole_start, hole_end, last = 0;
1284         struct drm_mm_node *node;
1285         IGT_TIMEOUT(end_time);
1286         int err = 0;
1287
1288 restart:
1289         list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes);
1290         drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) {
1291                 if (hole_start < last)
1292                         continue;
1293
1294                 if (ggtt->vm.mm.color_adjust)
1295                         ggtt->vm.mm.color_adjust(node, 0,
1296                                                  &hole_start, &hole_end);
1297                 if (hole_start >= hole_end)
1298                         continue;
1299
1300                 err = func(&ggtt->vm, hole_start, hole_end, end_time);
1301                 if (err)
1302                         break;
1303
1304                 /* As we have manipulated the drm_mm, the list may be corrupt */
1305                 last = hole_end;
1306                 goto restart;
1307         }
1308
1309         return err;
1310 }
1311
1312 static int igt_ggtt_fill(void *arg)
1313 {
1314         return exercise_ggtt(arg, fill_hole);
1315 }
1316
1317 static int igt_ggtt_walk(void *arg)
1318 {
1319         return exercise_ggtt(arg, walk_hole);
1320 }
1321
1322 static int igt_ggtt_pot(void *arg)
1323 {
1324         return exercise_ggtt(arg, pot_hole);
1325 }
1326
1327 static int igt_ggtt_drunk(void *arg)
1328 {
1329         return exercise_ggtt(arg, drunk_hole);
1330 }
1331
1332 static int igt_ggtt_lowlevel(void *arg)
1333 {
1334         return exercise_ggtt(arg, lowlevel_hole);
1335 }
1336
1337 static int igt_ggtt_misaligned_pin(void *arg)
1338 {
1339         return exercise_ggtt(arg, misaligned_pin);
1340 }
1341
1342 static int igt_ggtt_page(void *arg)
1343 {
1344         const unsigned int count = PAGE_SIZE/sizeof(u32);
1345         I915_RND_STATE(prng);
1346         struct drm_i915_private *i915 = arg;
1347         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
1348         struct drm_i915_gem_object *obj;
1349         intel_wakeref_t wakeref;
1350         struct drm_mm_node tmp;
1351         unsigned int *order, n;
1352         int err;
1353
1354         if (!i915_ggtt_has_aperture(ggtt))
1355                 return 0;
1356
1357         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1358         if (IS_ERR(obj))
1359                 return PTR_ERR(obj);
1360
1361         err = i915_gem_object_pin_pages_unlocked(obj);
1362         if (err)
1363                 goto out_free;
1364
1365         memset(&tmp, 0, sizeof(tmp));
1366         mutex_lock(&ggtt->vm.mutex);
1367         err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp,
1368                                           count * PAGE_SIZE, 0,
1369                                           I915_COLOR_UNEVICTABLE,
1370                                           0, ggtt->mappable_end,
1371                                           DRM_MM_INSERT_LOW);
1372         mutex_unlock(&ggtt->vm.mutex);
1373         if (err)
1374                 goto out_unpin;
1375
1376         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1377
1378         for (n = 0; n < count; n++) {
1379                 u64 offset = tmp.start + n * PAGE_SIZE;
1380
1381                 ggtt->vm.insert_page(&ggtt->vm,
1382                                      i915_gem_object_get_dma_address(obj, 0),
1383                                      offset, I915_CACHE_NONE, 0);
1384         }
1385
1386         order = i915_random_order(count, &prng);
1387         if (!order) {
1388                 err = -ENOMEM;
1389                 goto out_remove;
1390         }
1391
1392         for (n = 0; n < count; n++) {
1393                 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1394                 u32 __iomem *vaddr;
1395
1396                 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1397                 iowrite32(n, vaddr + n);
1398                 io_mapping_unmap_atomic(vaddr);
1399         }
1400         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1401
1402         i915_random_reorder(order, count, &prng);
1403         for (n = 0; n < count; n++) {
1404                 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1405                 u32 __iomem *vaddr;
1406                 u32 val;
1407
1408                 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1409                 val = ioread32(vaddr + n);
1410                 io_mapping_unmap_atomic(vaddr);
1411
1412                 if (val != n) {
1413                         pr_err("insert page failed: found %d, expected %d\n",
1414                                val, n);
1415                         err = -EINVAL;
1416                         break;
1417                 }
1418         }
1419
1420         kfree(order);
1421 out_remove:
1422         ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size);
1423         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1424         mutex_lock(&ggtt->vm.mutex);
1425         drm_mm_remove_node(&tmp);
1426         mutex_unlock(&ggtt->vm.mutex);
1427 out_unpin:
1428         i915_gem_object_unpin_pages(obj);
1429 out_free:
1430         i915_gem_object_put(obj);
1431         return err;
1432 }
1433
1434 static void track_vma_bind(struct i915_vma *vma)
1435 {
1436         struct drm_i915_gem_object *obj = vma->obj;
1437
1438         __i915_gem_object_pin_pages(obj);
1439
1440         GEM_BUG_ON(atomic_read(&vma->pages_count));
1441         atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE);
1442         __i915_gem_object_pin_pages(obj);
1443         vma->pages = obj->mm.pages;
1444         vma->resource->bi.pages = vma->pages;
1445
1446         mutex_lock(&vma->vm->mutex);
1447         list_move_tail(&vma->vm_link, &vma->vm->bound_list);
1448         mutex_unlock(&vma->vm->mutex);
1449 }
1450
1451 static int exercise_mock(struct drm_i915_private *i915,
1452                          int (*func)(struct i915_address_space *vm,
1453                                      u64 hole_start, u64 hole_end,
1454                                      unsigned long end_time))
1455 {
1456         const u64 limit = totalram_pages() << PAGE_SHIFT;
1457         struct i915_address_space *vm;
1458         struct i915_gem_context *ctx;
1459         IGT_TIMEOUT(end_time);
1460         int err;
1461
1462         ctx = mock_context(i915, "mock");
1463         if (!ctx)
1464                 return -ENOMEM;
1465
1466         vm = i915_gem_context_get_eb_vm(ctx);
1467         err = func(vm, 0, min(vm->total, limit), end_time);
1468         i915_vm_put(vm);
1469
1470         mock_context_close(ctx);
1471         return err;
1472 }
1473
1474 static int igt_mock_fill(void *arg)
1475 {
1476         struct i915_ggtt *ggtt = arg;
1477
1478         return exercise_mock(ggtt->vm.i915, fill_hole);
1479 }
1480
1481 static int igt_mock_walk(void *arg)
1482 {
1483         struct i915_ggtt *ggtt = arg;
1484
1485         return exercise_mock(ggtt->vm.i915, walk_hole);
1486 }
1487
1488 static int igt_mock_pot(void *arg)
1489 {
1490         struct i915_ggtt *ggtt = arg;
1491
1492         return exercise_mock(ggtt->vm.i915, pot_hole);
1493 }
1494
1495 static int igt_mock_drunk(void *arg)
1496 {
1497         struct i915_ggtt *ggtt = arg;
1498
1499         return exercise_mock(ggtt->vm.i915, drunk_hole);
1500 }
1501
1502 static int reserve_gtt_with_resource(struct i915_vma *vma, u64 offset)
1503 {
1504         struct i915_address_space *vm = vma->vm;
1505         struct i915_vma_resource *vma_res;
1506         struct drm_i915_gem_object *obj = vma->obj;
1507         int err;
1508
1509         vma_res = i915_vma_resource_alloc();
1510         if (IS_ERR(vma_res))
1511                 return PTR_ERR(vma_res);
1512
1513         mutex_lock(&vm->mutex);
1514         err = i915_gem_gtt_reserve(vm, NULL, &vma->node, obj->base.size,
1515                                    offset,
1516                                    obj->cache_level,
1517                                    0);
1518         if (!err) {
1519                 i915_vma_resource_init_from_vma(vma_res, vma);
1520                 vma->resource = vma_res;
1521         } else {
1522                 kfree(vma_res);
1523         }
1524         mutex_unlock(&vm->mutex);
1525
1526         return err;
1527 }
1528
1529 static int igt_gtt_reserve(void *arg)
1530 {
1531         struct i915_ggtt *ggtt = arg;
1532         struct drm_i915_gem_object *obj, *on;
1533         I915_RND_STATE(prng);
1534         LIST_HEAD(objects);
1535         u64 total;
1536         int err = -ENODEV;
1537
1538         /* i915_gem_gtt_reserve() tries to reserve the precise range
1539          * for the node, and evicts if it has to. So our test checks that
1540          * it can give us the requsted space and prevent overlaps.
1541          */
1542
1543         /* Start by filling the GGTT */
1544         for (total = 0;
1545              total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1546              total += 2 * I915_GTT_PAGE_SIZE) {
1547                 struct i915_vma *vma;
1548
1549                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1550                                                       2 * PAGE_SIZE);
1551                 if (IS_ERR(obj)) {
1552                         err = PTR_ERR(obj);
1553                         goto out;
1554                 }
1555
1556                 err = i915_gem_object_pin_pages_unlocked(obj);
1557                 if (err) {
1558                         i915_gem_object_put(obj);
1559                         goto out;
1560                 }
1561
1562                 list_add(&obj->st_link, &objects);
1563                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1564                 if (IS_ERR(vma)) {
1565                         err = PTR_ERR(vma);
1566                         goto out;
1567                 }
1568
1569                 err = reserve_gtt_with_resource(vma, total);
1570                 if (err) {
1571                         pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n",
1572                                total, ggtt->vm.total, err);
1573                         goto out;
1574                 }
1575                 track_vma_bind(vma);
1576
1577                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1578                 if (vma->node.start != total ||
1579                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1580                         pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1581                                vma->node.start, vma->node.size,
1582                                total, 2*I915_GTT_PAGE_SIZE);
1583                         err = -EINVAL;
1584                         goto out;
1585                 }
1586         }
1587
1588         /* Now we start forcing evictions */
1589         for (total = I915_GTT_PAGE_SIZE;
1590              total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1591              total += 2 * I915_GTT_PAGE_SIZE) {
1592                 struct i915_vma *vma;
1593
1594                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1595                                                       2 * PAGE_SIZE);
1596                 if (IS_ERR(obj)) {
1597                         err = PTR_ERR(obj);
1598                         goto out;
1599                 }
1600
1601                 err = i915_gem_object_pin_pages_unlocked(obj);
1602                 if (err) {
1603                         i915_gem_object_put(obj);
1604                         goto out;
1605                 }
1606
1607                 list_add(&obj->st_link, &objects);
1608
1609                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1610                 if (IS_ERR(vma)) {
1611                         err = PTR_ERR(vma);
1612                         goto out;
1613                 }
1614
1615                 err = reserve_gtt_with_resource(vma, total);
1616                 if (err) {
1617                         pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n",
1618                                total, ggtt->vm.total, err);
1619                         goto out;
1620                 }
1621                 track_vma_bind(vma);
1622
1623                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1624                 if (vma->node.start != total ||
1625                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1626                         pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1627                                vma->node.start, vma->node.size,
1628                                total, 2*I915_GTT_PAGE_SIZE);
1629                         err = -EINVAL;
1630                         goto out;
1631                 }
1632         }
1633
1634         /* And then try at random */
1635         list_for_each_entry_safe(obj, on, &objects, st_link) {
1636                 struct i915_vma *vma;
1637                 u64 offset;
1638
1639                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1640                 if (IS_ERR(vma)) {
1641                         err = PTR_ERR(vma);
1642                         goto out;
1643                 }
1644
1645                 err = i915_vma_unbind_unlocked(vma);
1646                 if (err) {
1647                         pr_err("i915_vma_unbind failed with err=%d!\n", err);
1648                         goto out;
1649                 }
1650
1651                 offset = igt_random_offset(&prng,
1652                                            0, ggtt->vm.total,
1653                                            2 * I915_GTT_PAGE_SIZE,
1654                                            I915_GTT_MIN_ALIGNMENT);
1655
1656                 err = reserve_gtt_with_resource(vma, offset);
1657                 if (err) {
1658                         pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n",
1659                                total, ggtt->vm.total, err);
1660                         goto out;
1661                 }
1662                 track_vma_bind(vma);
1663
1664                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1665                 if (vma->node.start != offset ||
1666                     vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1667                         pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1668                                vma->node.start, vma->node.size,
1669                                offset, 2*I915_GTT_PAGE_SIZE);
1670                         err = -EINVAL;
1671                         goto out;
1672                 }
1673         }
1674
1675 out:
1676         list_for_each_entry_safe(obj, on, &objects, st_link) {
1677                 i915_gem_object_unpin_pages(obj);
1678                 i915_gem_object_put(obj);
1679         }
1680         return err;
1681 }
1682
1683 static int insert_gtt_with_resource(struct i915_vma *vma)
1684 {
1685         struct i915_address_space *vm = vma->vm;
1686         struct i915_vma_resource *vma_res;
1687         struct drm_i915_gem_object *obj = vma->obj;
1688         int err;
1689
1690         vma_res = i915_vma_resource_alloc();
1691         if (IS_ERR(vma_res))
1692                 return PTR_ERR(vma_res);
1693
1694         mutex_lock(&vm->mutex);
1695         err = i915_gem_gtt_insert(vm, NULL, &vma->node, obj->base.size, 0,
1696                                   obj->cache_level, 0, vm->total, 0);
1697         if (!err) {
1698                 i915_vma_resource_init_from_vma(vma_res, vma);
1699                 vma->resource = vma_res;
1700         } else {
1701                 kfree(vma_res);
1702         }
1703         mutex_unlock(&vm->mutex);
1704
1705         return err;
1706 }
1707
1708 static int igt_gtt_insert(void *arg)
1709 {
1710         struct i915_ggtt *ggtt = arg;
1711         struct drm_i915_gem_object *obj, *on;
1712         struct drm_mm_node tmp = {};
1713         const struct invalid_insert {
1714                 u64 size;
1715                 u64 alignment;
1716                 u64 start, end;
1717         } invalid_insert[] = {
1718                 {
1719                         ggtt->vm.total + I915_GTT_PAGE_SIZE, 0,
1720                         0, ggtt->vm.total,
1721                 },
1722                 {
1723                         2*I915_GTT_PAGE_SIZE, 0,
1724                         0, I915_GTT_PAGE_SIZE,
1725                 },
1726                 {
1727                         -(u64)I915_GTT_PAGE_SIZE, 0,
1728                         0, 4*I915_GTT_PAGE_SIZE,
1729                 },
1730                 {
1731                         -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE,
1732                         0, 4*I915_GTT_PAGE_SIZE,
1733                 },
1734                 {
1735                         I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1,
1736                         I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1,
1737                 },
1738                 {}
1739         }, *ii;
1740         LIST_HEAD(objects);
1741         u64 total;
1742         int err = -ENODEV;
1743
1744         /* i915_gem_gtt_insert() tries to allocate some free space in the GTT
1745          * to the node, evicting if required.
1746          */
1747
1748         /* Check a couple of obviously invalid requests */
1749         for (ii = invalid_insert; ii->size; ii++) {
1750                 mutex_lock(&ggtt->vm.mutex);
1751                 err = i915_gem_gtt_insert(&ggtt->vm, NULL, &tmp,
1752                                           ii->size, ii->alignment,
1753                                           I915_COLOR_UNEVICTABLE,
1754                                           ii->start, ii->end,
1755                                           0);
1756                 mutex_unlock(&ggtt->vm.mutex);
1757                 if (err != -ENOSPC) {
1758                         pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n",
1759                                ii->size, ii->alignment, ii->start, ii->end,
1760                                err);
1761                         return -EINVAL;
1762                 }
1763         }
1764
1765         /* Start by filling the GGTT */
1766         for (total = 0;
1767              total + I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1768              total += I915_GTT_PAGE_SIZE) {
1769                 struct i915_vma *vma;
1770
1771                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1772                                                       I915_GTT_PAGE_SIZE);
1773                 if (IS_ERR(obj)) {
1774                         err = PTR_ERR(obj);
1775                         goto out;
1776                 }
1777
1778                 err = i915_gem_object_pin_pages_unlocked(obj);
1779                 if (err) {
1780                         i915_gem_object_put(obj);
1781                         goto out;
1782                 }
1783
1784                 list_add(&obj->st_link, &objects);
1785
1786                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1787                 if (IS_ERR(vma)) {
1788                         err = PTR_ERR(vma);
1789                         goto out;
1790                 }
1791
1792                 err = insert_gtt_with_resource(vma);
1793                 if (err == -ENOSPC) {
1794                         /* maxed out the GGTT space */
1795                         i915_gem_object_put(obj);
1796                         break;
1797                 }
1798                 if (err) {
1799                         pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n",
1800                                total, ggtt->vm.total, err);
1801                         goto out;
1802                 }
1803                 track_vma_bind(vma);
1804                 __i915_vma_pin(vma);
1805
1806                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1807         }
1808
1809         list_for_each_entry(obj, &objects, st_link) {
1810                 struct i915_vma *vma;
1811
1812                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1813                 if (IS_ERR(vma)) {
1814                         err = PTR_ERR(vma);
1815                         goto out;
1816                 }
1817
1818                 if (!drm_mm_node_allocated(&vma->node)) {
1819                         pr_err("VMA was unexpectedly evicted!\n");
1820                         err = -EINVAL;
1821                         goto out;
1822                 }
1823
1824                 __i915_vma_unpin(vma);
1825         }
1826
1827         /* If we then reinsert, we should find the same hole */
1828         list_for_each_entry_safe(obj, on, &objects, st_link) {
1829                 struct i915_vma *vma;
1830                 u64 offset;
1831
1832                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1833                 if (IS_ERR(vma)) {
1834                         err = PTR_ERR(vma);
1835                         goto out;
1836                 }
1837
1838                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1839                 offset = vma->node.start;
1840
1841                 err = i915_vma_unbind_unlocked(vma);
1842                 if (err) {
1843                         pr_err("i915_vma_unbind failed with err=%d!\n", err);
1844                         goto out;
1845                 }
1846
1847                 err = insert_gtt_with_resource(vma);
1848                 if (err) {
1849                         pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n",
1850                                total, ggtt->vm.total, err);
1851                         goto out;
1852                 }
1853                 track_vma_bind(vma);
1854
1855                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1856                 if (vma->node.start != offset) {
1857                         pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n",
1858                                offset, vma->node.start);
1859                         err = -EINVAL;
1860                         goto out;
1861                 }
1862         }
1863
1864         /* And then force evictions */
1865         for (total = 0;
1866              total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1867              total += 2 * I915_GTT_PAGE_SIZE) {
1868                 struct i915_vma *vma;
1869
1870                 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1871                                                       2 * I915_GTT_PAGE_SIZE);
1872                 if (IS_ERR(obj)) {
1873                         err = PTR_ERR(obj);
1874                         goto out;
1875                 }
1876
1877                 err = i915_gem_object_pin_pages_unlocked(obj);
1878                 if (err) {
1879                         i915_gem_object_put(obj);
1880                         goto out;
1881                 }
1882
1883                 list_add(&obj->st_link, &objects);
1884
1885                 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1886                 if (IS_ERR(vma)) {
1887                         err = PTR_ERR(vma);
1888                         goto out;
1889                 }
1890
1891                 err = insert_gtt_with_resource(vma);
1892                 if (err) {
1893                         pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n",
1894                                total, ggtt->vm.total, err);
1895                         goto out;
1896                 }
1897                 track_vma_bind(vma);
1898
1899                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1900         }
1901
1902 out:
1903         list_for_each_entry_safe(obj, on, &objects, st_link) {
1904                 i915_gem_object_unpin_pages(obj);
1905                 i915_gem_object_put(obj);
1906         }
1907         return err;
1908 }
1909
1910 int i915_gem_gtt_mock_selftests(void)
1911 {
1912         static const struct i915_subtest tests[] = {
1913                 SUBTEST(igt_mock_drunk),
1914                 SUBTEST(igt_mock_walk),
1915                 SUBTEST(igt_mock_pot),
1916                 SUBTEST(igt_mock_fill),
1917                 SUBTEST(igt_gtt_reserve),
1918                 SUBTEST(igt_gtt_insert),
1919         };
1920         struct drm_i915_private *i915;
1921         struct intel_gt *gt;
1922         int err;
1923
1924         i915 = mock_gem_device();
1925         if (!i915)
1926                 return -ENOMEM;
1927
1928         /* allocate the ggtt */
1929         err = intel_gt_assign_ggtt(to_gt(i915));
1930         if (err)
1931                 goto out_put;
1932
1933         gt = to_gt(i915);
1934
1935         mock_init_ggtt(gt);
1936
1937         err = i915_subtests(tests, gt->ggtt);
1938
1939         mock_device_flush(i915);
1940         i915_gem_drain_freed_objects(i915);
1941         mock_fini_ggtt(gt->ggtt);
1942
1943 out_put:
1944         mock_destroy_device(i915);
1945         return err;
1946 }
1947
1948 static int context_sync(struct intel_context *ce)
1949 {
1950         struct i915_request *rq;
1951         long timeout;
1952
1953         rq = intel_context_create_request(ce);
1954         if (IS_ERR(rq))
1955                 return PTR_ERR(rq);
1956
1957         i915_request_get(rq);
1958         i915_request_add(rq);
1959
1960         timeout = i915_request_wait(rq, 0, HZ / 5);
1961         i915_request_put(rq);
1962
1963         return timeout < 0 ? -EIO : 0;
1964 }
1965
1966 static struct i915_request *
1967 submit_batch(struct intel_context *ce, u64 addr)
1968 {
1969         struct i915_request *rq;
1970         int err;
1971
1972         rq = intel_context_create_request(ce);
1973         if (IS_ERR(rq))
1974                 return rq;
1975
1976         err = 0;
1977         if (rq->engine->emit_init_breadcrumb) /* detect a hang */
1978                 err = rq->engine->emit_init_breadcrumb(rq);
1979         if (err == 0)
1980                 err = rq->engine->emit_bb_start(rq, addr, 0, 0);
1981
1982         if (err == 0)
1983                 i915_request_get(rq);
1984         i915_request_add(rq);
1985
1986         return err ? ERR_PTR(err) : rq;
1987 }
1988
1989 static u32 *spinner(u32 *batch, int i)
1990 {
1991         return batch + i * 64 / sizeof(*batch) + 4;
1992 }
1993
1994 static void end_spin(u32 *batch, int i)
1995 {
1996         *spinner(batch, i) = MI_BATCH_BUFFER_END;
1997         wmb();
1998 }
1999
2000 static int igt_cs_tlb(void *arg)
2001 {
2002         const unsigned int count = PAGE_SIZE / 64;
2003         const unsigned int chunk_size = count * PAGE_SIZE;
2004         struct drm_i915_private *i915 = arg;
2005         struct drm_i915_gem_object *bbe, *act, *out;
2006         struct i915_gem_engines_iter it;
2007         struct i915_address_space *vm;
2008         struct i915_gem_context *ctx;
2009         struct intel_context *ce;
2010         struct i915_vma *vma;
2011         I915_RND_STATE(prng);
2012         struct file *file;
2013         unsigned int i;
2014         u32 *result;
2015         u32 *batch;
2016         int err = 0;
2017
2018         /*
2019          * Our mission here is to fool the hardware to execute something
2020          * from scratch as it has not seen the batch move (due to missing
2021          * the TLB invalidate).
2022          */
2023
2024         file = mock_file(i915);
2025         if (IS_ERR(file))
2026                 return PTR_ERR(file);
2027
2028         ctx = live_context(i915, file);
2029         if (IS_ERR(ctx)) {
2030                 err = PTR_ERR(ctx);
2031                 goto out_unlock;
2032         }
2033
2034         vm = i915_gem_context_get_eb_vm(ctx);
2035         if (i915_is_ggtt(vm))
2036                 goto out_vm;
2037
2038         /* Create two pages; dummy we prefill the TLB, and intended */
2039         bbe = i915_gem_object_create_internal(i915, PAGE_SIZE);
2040         if (IS_ERR(bbe)) {
2041                 err = PTR_ERR(bbe);
2042                 goto out_vm;
2043         }
2044
2045         batch = i915_gem_object_pin_map_unlocked(bbe, I915_MAP_WC);
2046         if (IS_ERR(batch)) {
2047                 err = PTR_ERR(batch);
2048                 goto out_put_bbe;
2049         }
2050         memset32(batch, MI_BATCH_BUFFER_END, PAGE_SIZE / sizeof(u32));
2051         i915_gem_object_flush_map(bbe);
2052         i915_gem_object_unpin_map(bbe);
2053
2054         act = i915_gem_object_create_internal(i915, PAGE_SIZE);
2055         if (IS_ERR(act)) {
2056                 err = PTR_ERR(act);
2057                 goto out_put_bbe;
2058         }
2059
2060         /* Track the execution of each request by writing into different slot */
2061         batch = i915_gem_object_pin_map_unlocked(act, I915_MAP_WC);
2062         if (IS_ERR(batch)) {
2063                 err = PTR_ERR(batch);
2064                 goto out_put_act;
2065         }
2066         for (i = 0; i < count; i++) {
2067                 u32 *cs = batch + i * 64 / sizeof(*cs);
2068                 u64 addr = (vm->total - PAGE_SIZE) + i * sizeof(u32);
2069
2070                 GEM_BUG_ON(GRAPHICS_VER(i915) < 6);
2071                 cs[0] = MI_STORE_DWORD_IMM_GEN4;
2072                 if (GRAPHICS_VER(i915) >= 8) {
2073                         cs[1] = lower_32_bits(addr);
2074                         cs[2] = upper_32_bits(addr);
2075                         cs[3] = i;
2076                         cs[4] = MI_NOOP;
2077                         cs[5] = MI_BATCH_BUFFER_START_GEN8;
2078                 } else {
2079                         cs[1] = 0;
2080                         cs[2] = lower_32_bits(addr);
2081                         cs[3] = i;
2082                         cs[4] = MI_NOOP;
2083                         cs[5] = MI_BATCH_BUFFER_START;
2084                 }
2085         }
2086
2087         out = i915_gem_object_create_internal(i915, PAGE_SIZE);
2088         if (IS_ERR(out)) {
2089                 err = PTR_ERR(out);
2090                 goto out_put_batch;
2091         }
2092         i915_gem_object_set_cache_coherency(out, I915_CACHING_CACHED);
2093
2094         vma = i915_vma_instance(out, vm, NULL);
2095         if (IS_ERR(vma)) {
2096                 err = PTR_ERR(vma);
2097                 goto out_put_out;
2098         }
2099
2100         err = i915_vma_pin(vma, 0, 0,
2101                            PIN_USER |
2102                            PIN_OFFSET_FIXED |
2103                            (vm->total - PAGE_SIZE));
2104         if (err)
2105                 goto out_put_out;
2106         GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE);
2107
2108         result = i915_gem_object_pin_map_unlocked(out, I915_MAP_WB);
2109         if (IS_ERR(result)) {
2110                 err = PTR_ERR(result);
2111                 goto out_put_out;
2112         }
2113
2114         for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
2115                 IGT_TIMEOUT(end_time);
2116                 unsigned long pass = 0;
2117
2118                 if (!intel_engine_can_store_dword(ce->engine))
2119                         continue;
2120
2121                 while (!__igt_timeout(end_time, NULL)) {
2122                         struct i915_vm_pt_stash stash = {};
2123                         struct i915_request *rq;
2124                         struct i915_gem_ww_ctx ww;
2125                         struct i915_vma_resource *vma_res;
2126                         u64 offset;
2127
2128                         offset = igt_random_offset(&prng,
2129                                                    0, vm->total - PAGE_SIZE,
2130                                                    chunk_size, PAGE_SIZE);
2131
2132                         memset32(result, STACK_MAGIC, PAGE_SIZE / sizeof(u32));
2133
2134                         vma = i915_vma_instance(bbe, vm, NULL);
2135                         if (IS_ERR(vma)) {
2136                                 err = PTR_ERR(vma);
2137                                 goto end;
2138                         }
2139
2140                         i915_gem_object_lock(bbe, NULL);
2141                         err = i915_vma_get_pages(vma);
2142                         i915_gem_object_unlock(bbe);
2143                         if (err)
2144                                 goto end;
2145
2146                         vma_res = i915_vma_resource_alloc();
2147                         if (IS_ERR(vma_res)) {
2148                                 i915_vma_put_pages(vma);
2149                                 err = PTR_ERR(vma_res);
2150                                 goto end;
2151                         }
2152
2153                         i915_gem_ww_ctx_init(&ww, false);
2154 retry:
2155                         err = i915_vm_lock_objects(vm, &ww);
2156                         if (err)
2157                                 goto end_ww;
2158
2159                         err = i915_vm_alloc_pt_stash(vm, &stash, chunk_size);
2160                         if (err)
2161                                 goto end_ww;
2162
2163                         err = i915_vm_map_pt_stash(vm, &stash);
2164                         if (!err)
2165                                 vm->allocate_va_range(vm, &stash, offset, chunk_size);
2166                         i915_vm_free_pt_stash(vm, &stash);
2167 end_ww:
2168                         if (err == -EDEADLK) {
2169                                 err = i915_gem_ww_ctx_backoff(&ww);
2170                                 if (!err)
2171                                         goto retry;
2172                         }
2173                         i915_gem_ww_ctx_fini(&ww);
2174                         if (err) {
2175                                 kfree(vma_res);
2176                                 goto end;
2177                         }
2178
2179                         i915_vma_resource_init_from_vma(vma_res, vma);
2180                         /* Prime the TLB with the dummy pages */
2181                         for (i = 0; i < count; i++) {
2182                                 vma_res->start = offset + i * PAGE_SIZE;
2183                                 vm->insert_entries(vm, vma_res, I915_CACHE_NONE,
2184                                                    0);
2185
2186                                 rq = submit_batch(ce, vma_res->start);
2187                                 if (IS_ERR(rq)) {
2188                                         err = PTR_ERR(rq);
2189                                         i915_vma_resource_fini(vma_res);
2190                                         kfree(vma_res);
2191                                         goto end;
2192                                 }
2193                                 i915_request_put(rq);
2194                         }
2195                         i915_vma_resource_fini(vma_res);
2196                         i915_vma_put_pages(vma);
2197
2198                         err = context_sync(ce);
2199                         if (err) {
2200                                 pr_err("%s: dummy setup timed out\n",
2201                                        ce->engine->name);
2202                                 kfree(vma_res);
2203                                 goto end;
2204                         }
2205
2206                         vma = i915_vma_instance(act, vm, NULL);
2207                         if (IS_ERR(vma)) {
2208                                 kfree(vma_res);
2209                                 err = PTR_ERR(vma);
2210                                 goto end;
2211                         }
2212
2213                         i915_gem_object_lock(act, NULL);
2214                         err = i915_vma_get_pages(vma);
2215                         i915_gem_object_unlock(act);
2216                         if (err) {
2217                                 kfree(vma_res);
2218                                 goto end;
2219                         }
2220
2221                         i915_vma_resource_init_from_vma(vma_res, vma);
2222                         /* Replace the TLB with target batches */
2223                         for (i = 0; i < count; i++) {
2224                                 struct i915_request *rq;
2225                                 u32 *cs = batch + i * 64 / sizeof(*cs);
2226                                 u64 addr;
2227
2228                                 vma_res->start = offset + i * PAGE_SIZE;
2229                                 vm->insert_entries(vm, vma_res, I915_CACHE_NONE, 0);
2230
2231                                 addr = vma_res->start + i * 64;
2232                                 cs[4] = MI_NOOP;
2233                                 cs[6] = lower_32_bits(addr);
2234                                 cs[7] = upper_32_bits(addr);
2235                                 wmb();
2236
2237                                 rq = submit_batch(ce, addr);
2238                                 if (IS_ERR(rq)) {
2239                                         err = PTR_ERR(rq);
2240                                         i915_vma_resource_fini(vma_res);
2241                                         kfree(vma_res);
2242                                         goto end;
2243                                 }
2244
2245                                 /* Wait until the context chain has started */
2246                                 if (i == 0) {
2247                                         while (READ_ONCE(result[i]) &&
2248                                                !i915_request_completed(rq))
2249                                                 cond_resched();
2250                                 } else {
2251                                         end_spin(batch, i - 1);
2252                                 }
2253
2254                                 i915_request_put(rq);
2255                         }
2256                         end_spin(batch, count - 1);
2257
2258                         i915_vma_resource_fini(vma_res);
2259                         kfree(vma_res);
2260                         i915_vma_put_pages(vma);
2261
2262                         err = context_sync(ce);
2263                         if (err) {
2264                                 pr_err("%s: writes timed out\n",
2265                                        ce->engine->name);
2266                                 goto end;
2267                         }
2268
2269                         for (i = 0; i < count; i++) {
2270                                 if (result[i] != i) {
2271                                         pr_err("%s: Write lost on pass %lu, at offset %llx, index %d, found %x, expected %x\n",
2272                                                ce->engine->name, pass,
2273                                                offset, i, result[i], i);
2274                                         err = -EINVAL;
2275                                         goto end;
2276                                 }
2277                         }
2278
2279                         vm->clear_range(vm, offset, chunk_size);
2280                         pass++;
2281                 }
2282         }
2283 end:
2284         if (igt_flush_test(i915))
2285                 err = -EIO;
2286         i915_gem_context_unlock_engines(ctx);
2287         i915_gem_object_unpin_map(out);
2288 out_put_out:
2289         i915_gem_object_put(out);
2290 out_put_batch:
2291         i915_gem_object_unpin_map(act);
2292 out_put_act:
2293         i915_gem_object_put(act);
2294 out_put_bbe:
2295         i915_gem_object_put(bbe);
2296 out_vm:
2297         i915_vm_put(vm);
2298 out_unlock:
2299         fput(file);
2300         return err;
2301 }
2302
2303 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915)
2304 {
2305         static const struct i915_subtest tests[] = {
2306                 SUBTEST(igt_ppgtt_alloc),
2307                 SUBTEST(igt_ppgtt_lowlevel),
2308                 SUBTEST(igt_ppgtt_drunk),
2309                 SUBTEST(igt_ppgtt_walk),
2310                 SUBTEST(igt_ppgtt_pot),
2311                 SUBTEST(igt_ppgtt_fill),
2312                 SUBTEST(igt_ppgtt_shrink),
2313                 SUBTEST(igt_ppgtt_shrink_boom),
2314                 SUBTEST(igt_ppgtt_misaligned_pin),
2315                 SUBTEST(igt_ggtt_lowlevel),
2316                 SUBTEST(igt_ggtt_drunk),
2317                 SUBTEST(igt_ggtt_walk),
2318                 SUBTEST(igt_ggtt_pot),
2319                 SUBTEST(igt_ggtt_fill),
2320                 SUBTEST(igt_ggtt_page),
2321                 SUBTEST(igt_ggtt_misaligned_pin),
2322                 SUBTEST(igt_cs_tlb),
2323         };
2324
2325         GEM_BUG_ON(offset_in_page(to_gt(i915)->ggtt->vm.total));
2326
2327         return i915_subtests(tests, i915);
2328 }