drivers/gpu/drm/i915/gem/selftests/huge_pages.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2017 Intel Corporation
   5  */
   6
   7 #include <linux/prime_numbers.h>
   8 #include <linux/string_helpers.h>
   9 #include <linux/swap.h>
  10
  11 #include "i915_selftest.h"
  12
  13 #include "gem/i915_gem_internal.h"
  14 #include "gem/i915_gem_lmem.h"
  15 #include "gem/i915_gem_pm.h"
  16 #include "gem/i915_gem_region.h"
  17
  18 #include "gt/intel_gt.h"
  19
  20 #include "igt_gem_utils.h"
  21 #include "mock_context.h"
  22
  23 #include "selftests/mock_drm.h"
  24 #include "selftests/mock_gem_device.h"
  25 #include "selftests/mock_region.h"
  26 #include "selftests/i915_random.h"
  27
  28 static struct i915_gem_context *hugepage_ctx(struct drm_i915_private *i915,
  29                                              struct file *file)
  30 {
  31         struct i915_gem_context *ctx = live_context(i915, file);
  32         struct i915_address_space *vm;
  33
  34         if (IS_ERR(ctx))
  35                 return ctx;
  36
  37         vm = ctx->vm;
  38         if (vm)
  39                 WRITE_ONCE(vm->scrub_64K, true);
  40
  41         return ctx;
  42 }
  43
  44 static const unsigned int page_sizes[] = {
  45         I915_GTT_PAGE_SIZE_2M,
  46         I915_GTT_PAGE_SIZE_64K,
  47         I915_GTT_PAGE_SIZE_4K,
  48 };
  49
  50 static unsigned int get_largest_page_size(struct drm_i915_private *i915,
  51                                           u64 rem)
  52 {
  53         int i;
  54
  55         for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
  56                 unsigned int page_size = page_sizes[i];
  57
  58                 if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size)
  59                         return page_size;
  60         }
  61
  62         return 0;
  63 }
  64
  65 static void huge_pages_free_pages(struct sg_table *st)
  66 {
  67         struct scatterlist *sg;
  68
  69         for (sg = st->sgl; sg; sg = __sg_next(sg)) {
  70                 if (sg_page(sg))
  71                         __free_pages(sg_page(sg), get_order(sg->length));
  72         }
  73
  74         sg_free_table(st);
  75         kfree(st);
  76 }
  77
  78 static int get_huge_pages(struct drm_i915_gem_object *obj)
  79 {
  80 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
  81         unsigned int page_mask = obj->mm.page_mask;
  82         struct sg_table *st;
  83         struct scatterlist *sg;
  84         unsigned int sg_page_sizes;
  85         u64 rem;
  86
  87         /* restricted by sg_alloc_table */
  88         if (overflows_type(obj->base.size >> PAGE_SHIFT, unsigned int))
  89                 return -E2BIG;
  90
  91         st = kmalloc(sizeof(*st), GFP);
  92         if (!st)
  93                 return -ENOMEM;
  94
  95         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
  96                 kfree(st);
  97                 return -ENOMEM;
  98         }
  99
 100         rem = obj->base.size;
 101         sg = st->sgl;
 102         st->nents = 0;
 103         sg_page_sizes = 0;
 104
 105         /*
 106          * Our goal here is simple, we want to greedily fill the object from
 107          * largest to smallest page-size, while ensuring that we use *every*
 108          * page-size as per the given page-mask.
 109          */
 110         do {
 111                 unsigned int bit = ilog2(page_mask);
 112                 unsigned int page_size = BIT(bit);
 113                 int order = get_order(page_size);
 114
 115                 do {
 116                         struct page *page;
 117
 118                         GEM_BUG_ON(order > MAX_ORDER);
 119                         page = alloc_pages(GFP | __GFP_ZERO, order);
 120                         if (!page)
 121                                 goto err;
 122
 123                         sg_set_page(sg, page, page_size, 0);
 124                         sg_page_sizes |= page_size;
 125                         st->nents++;
 126
 127                         rem -= page_size;
 128                         if (!rem) {
 129                                 sg_mark_end(sg);
 130                                 break;
 131                         }
 132
 133                         sg = __sg_next(sg);
 134                 } while ((rem - ((page_size-1) & page_mask)) >= page_size);
 135
 136                 page_mask &= (page_size-1);
 137         } while (page_mask);
 138
 139         if (i915_gem_gtt_prepare_pages(obj, st))
 140                 goto err;
 141
 142         GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask);
 143         __i915_gem_object_set_pages(obj, st);
 144
 145         return 0;
 146
 147 err:
 148         sg_set_page(sg, NULL, 0, 0);
 149         sg_mark_end(sg);
 150         huge_pages_free_pages(st);
 151
 152         return -ENOMEM;
 153 }
 154
 155 static void put_huge_pages(struct drm_i915_gem_object *obj,
 156                            struct sg_table *pages)
 157 {
 158         i915_gem_gtt_finish_pages(obj, pages);
 159         huge_pages_free_pages(pages);
 160
 161         obj->mm.dirty = false;
 162
 163         __start_cpu_write(obj);
 164 }
 165
 166 static const struct drm_i915_gem_object_ops huge_page_ops = {
 167         .name = "huge-gem",
 168         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 169         .get_pages = get_huge_pages,
 170         .put_pages = put_huge_pages,
 171 };
 172
 173 static struct drm_i915_gem_object *
 174 huge_pages_object(struct drm_i915_private *i915,
 175                   u64 size,
 176                   unsigned int page_mask)
 177 {
 178         static struct lock_class_key lock_class;
 179         struct drm_i915_gem_object *obj;
 180         unsigned int cache_level;
 181
 182         GEM_BUG_ON(!size);
 183         GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask))));
 184
 185         if (size >> PAGE_SHIFT > INT_MAX)
 186                 return ERR_PTR(-E2BIG);
 187
 188         if (overflows_type(size, obj->base.size))
 189                 return ERR_PTR(-E2BIG);
 190
 191         obj = i915_gem_object_alloc();
 192         if (!obj)
 193                 return ERR_PTR(-ENOMEM);
 194
 195         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 196         i915_gem_object_init(obj, &huge_page_ops, &lock_class, 0);
 197         obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE;
 198         i915_gem_object_set_volatile(obj);
 199
 200         obj->write_domain = I915_GEM_DOMAIN_CPU;
 201         obj->read_domains = I915_GEM_DOMAIN_CPU;
 202
 203         cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
 204         i915_gem_object_set_cache_coherency(obj, cache_level);
 205
 206         obj->mm.page_mask = page_mask;
 207
 208         return obj;
 209 }
 210
 211 static int fake_get_huge_pages(struct drm_i915_gem_object *obj)
 212 {
 213         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 214         const u64 max_len = rounddown_pow_of_two(UINT_MAX);
 215         struct sg_table *st;
 216         struct scatterlist *sg;
 217         u64 rem;
 218
 219         /* restricted by sg_alloc_table */
 220         if (overflows_type(obj->base.size >> PAGE_SHIFT, unsigned int))
 221                 return -E2BIG;
 222
 223         st = kmalloc(sizeof(*st), GFP);
 224         if (!st)
 225                 return -ENOMEM;
 226
 227         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
 228                 kfree(st);
 229                 return -ENOMEM;
 230         }
 231
 232         /* Use optimal page sized chunks to fill in the sg table */
 233         rem = obj->base.size;
 234         sg = st->sgl;
 235         st->nents = 0;
 236         do {
 237                 unsigned int page_size = get_largest_page_size(i915, rem);
 238                 unsigned int len = min(page_size * div_u64(rem, page_size),
 239                                        max_len);
 240
 241                 GEM_BUG_ON(!page_size);
 242
 243                 sg->offset = 0;
 244                 sg->length = len;
 245                 sg_dma_len(sg) = len;
 246                 sg_dma_address(sg) = page_size;
 247
 248                 st->nents++;
 249
 250                 rem -= len;
 251                 if (!rem) {
 252                         sg_mark_end(sg);
 253                         break;
 254                 }
 255
 256                 sg = sg_next(sg);
 257         } while (1);
 258
 259         i915_sg_trim(st);
 260
 261         __i915_gem_object_set_pages(obj, st);
 262
 263         return 0;
 264 }
 265
 266 static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj)
 267 {
 268         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 269         struct sg_table *st;
 270         struct scatterlist *sg;
 271         unsigned int page_size;
 272
 273         st = kmalloc(sizeof(*st), GFP);
 274         if (!st)
 275                 return -ENOMEM;
 276
 277         if (sg_alloc_table(st, 1, GFP)) {
 278                 kfree(st);
 279                 return -ENOMEM;
 280         }
 281
 282         sg = st->sgl;
 283         st->nents = 1;
 284
 285         page_size = get_largest_page_size(i915, obj->base.size);
 286         GEM_BUG_ON(!page_size);
 287
 288         sg->offset = 0;
 289         sg->length = obj->base.size;
 290         sg_dma_len(sg) = obj->base.size;
 291         sg_dma_address(sg) = page_size;
 292
 293         __i915_gem_object_set_pages(obj, st);
 294
 295         return 0;
 296 #undef GFP
 297 }
 298
 299 static void fake_free_huge_pages(struct drm_i915_gem_object *obj,
 300                                  struct sg_table *pages)
 301 {
 302         sg_free_table(pages);
 303         kfree(pages);
 304 }
 305
 306 static void fake_put_huge_pages(struct drm_i915_gem_object *obj,
 307                                 struct sg_table *pages)
 308 {
 309         fake_free_huge_pages(obj, pages);
 310         obj->mm.dirty = false;
 311 }
 312
 313 static const struct drm_i915_gem_object_ops fake_ops = {
 314         .name = "fake-gem",
 315         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 316         .get_pages = fake_get_huge_pages,
 317         .put_pages = fake_put_huge_pages,
 318 };
 319
 320 static const struct drm_i915_gem_object_ops fake_ops_single = {
 321         .name = "fake-gem",
 322         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 323         .get_pages = fake_get_huge_pages_single,
 324         .put_pages = fake_put_huge_pages,
 325 };
 326
 327 static struct drm_i915_gem_object *
 328 fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single)
 329 {
 330         static struct lock_class_key lock_class;
 331         struct drm_i915_gem_object *obj;
 332
 333         GEM_BUG_ON(!size);
 334         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 335
 336         if (size >> PAGE_SHIFT > UINT_MAX)
 337                 return ERR_PTR(-E2BIG);
 338
 339         if (overflows_type(size, obj->base.size))
 340                 return ERR_PTR(-E2BIG);
 341
 342         obj = i915_gem_object_alloc();
 343         if (!obj)
 344                 return ERR_PTR(-ENOMEM);
 345
 346         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 347
 348         if (single)
 349                 i915_gem_object_init(obj, &fake_ops_single, &lock_class, 0);
 350         else
 351                 i915_gem_object_init(obj, &fake_ops, &lock_class, 0);
 352
 353         i915_gem_object_set_volatile(obj);
 354
 355         obj->write_domain = I915_GEM_DOMAIN_CPU;
 356         obj->read_domains = I915_GEM_DOMAIN_CPU;
 357         obj->pat_index = i915_gem_get_pat_index(i915, I915_CACHE_NONE);
 358
 359         return obj;
 360 }
 361
 362 static int igt_check_page_sizes(struct i915_vma *vma)
 363 {
 364         struct drm_i915_private *i915 = vma->vm->i915;
 365         unsigned int supported = RUNTIME_INFO(i915)->page_sizes;
 366         struct drm_i915_gem_object *obj = vma->obj;
 367         int err;
 368
 369         /* We have to wait for the async bind to complete before our asserts */
 370         err = i915_vma_sync(vma);
 371         if (err)
 372                 return err;
 373
 374         if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) {
 375                 pr_err("unsupported page_sizes.sg=%u, supported=%u\n",
 376                        vma->page_sizes.sg & ~supported, supported);
 377                 err = -EINVAL;
 378         }
 379
 380         if (!HAS_PAGE_SIZES(i915, vma->resource->page_sizes_gtt)) {
 381                 pr_err("unsupported page_sizes.gtt=%u, supported=%u\n",
 382                        vma->resource->page_sizes_gtt & ~supported, supported);
 383                 err = -EINVAL;
 384         }
 385
 386         if (vma->page_sizes.phys != obj->mm.page_sizes.phys) {
 387                 pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n",
 388                        vma->page_sizes.phys, obj->mm.page_sizes.phys);
 389                 err = -EINVAL;
 390         }
 391
 392         if (vma->page_sizes.sg != obj->mm.page_sizes.sg) {
 393                 pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n",
 394                        vma->page_sizes.sg, obj->mm.page_sizes.sg);
 395                 err = -EINVAL;
 396         }
 397
 398         /*
 399          * The dma-api is like a box of chocolates when it comes to the
 400          * alignment of dma addresses, however for LMEM we have total control
 401          * and so can guarantee alignment, likewise when we allocate our blocks
 402          * they should appear in descending order, and if we know that we align
 403          * to the largest page size for the GTT address, we should be able to
 404          * assert that if we see 2M physical pages then we should also get 2M
 405          * GTT pages. If we don't then something might be wrong in our
 406          * construction of the backing pages.
 407          *
 408          * Maintaining alignment is required to utilise huge pages in the ppGGT.
 409          */
 410         if (i915_gem_object_is_lmem(obj) &&
 411             IS_ALIGNED(i915_vma_offset(vma), SZ_2M) &&
 412             vma->page_sizes.sg & SZ_2M &&
 413             vma->resource->page_sizes_gtt < SZ_2M) {
 414                 pr_err("gtt pages mismatch for LMEM, expected 2M GTT pages, sg(%u), gtt(%u)\n",
 415                        vma->page_sizes.sg, vma->resource->page_sizes_gtt);
 416                 err = -EINVAL;
 417         }
 418
 419         return err;
 420 }
 421
 422 static int igt_mock_exhaust_device_supported_pages(void *arg)
 423 {
 424         struct i915_ppgtt *ppgtt = arg;
 425         struct drm_i915_private *i915 = ppgtt->vm.i915;
 426         unsigned int saved_mask = RUNTIME_INFO(i915)->page_sizes;
 427         struct drm_i915_gem_object *obj;
 428         struct i915_vma *vma;
 429         int i, j, single;
 430         int err;
 431
 432         /*
 433          * Sanity check creating objects with every valid page support
 434          * combination for our mock device.
 435          */
 436
 437         for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) {
 438                 unsigned int combination = SZ_4K; /* Required for ppGTT */
 439
 440                 for (j = 0; j < ARRAY_SIZE(page_sizes); j++) {
 441                         if (i & BIT(j))
 442                                 combination |= page_sizes[j];
 443                 }
 444
 445                 RUNTIME_INFO(i915)->page_sizes = combination;
 446
 447                 for (single = 0; single <= 1; ++single) {
 448                         obj = fake_huge_pages_object(i915, combination, !!single);
 449                         if (IS_ERR(obj)) {
 450                                 err = PTR_ERR(obj);
 451                                 goto out_device;
 452                         }
 453
 454                         if (obj->base.size != combination) {
 455                                 pr_err("obj->base.size=%zu, expected=%u\n",
 456                                        obj->base.size, combination);
 457                                 err = -EINVAL;
 458                                 goto out_put;
 459                         }
 460
 461                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 462                         if (IS_ERR(vma)) {
 463                                 err = PTR_ERR(vma);
 464                                 goto out_put;
 465                         }
 466
 467                         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 468                         if (err)
 469                                 goto out_put;
 470
 471                         err = igt_check_page_sizes(vma);
 472
 473                         if (vma->page_sizes.sg != combination) {
 474                                 pr_err("page_sizes.sg=%u, expected=%u\n",
 475                                        vma->page_sizes.sg, combination);
 476                                 err = -EINVAL;
 477                         }
 478
 479                         i915_vma_unpin(vma);
 480                         i915_gem_object_put(obj);
 481
 482                         if (err)
 483                                 goto out_device;
 484                 }
 485         }
 486
 487         goto out_device;
 488
 489 out_put:
 490         i915_gem_object_put(obj);
 491 out_device:
 492         RUNTIME_INFO(i915)->page_sizes = saved_mask;
 493
 494         return err;
 495 }
 496
 497 static int igt_mock_memory_region_huge_pages(void *arg)
 498 {
 499         const unsigned int flags[] = { 0, I915_BO_ALLOC_CONTIGUOUS };
 500         struct i915_ppgtt *ppgtt = arg;
 501         struct drm_i915_private *i915 = ppgtt->vm.i915;
 502         unsigned long supported = RUNTIME_INFO(i915)->page_sizes;
 503         struct intel_memory_region *mem;
 504         struct drm_i915_gem_object *obj;
 505         struct i915_vma *vma;
 506         int bit;
 507         int err = 0;
 508
 509         mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0, 0);
 510         if (IS_ERR(mem)) {
 511                 pr_err("%s failed to create memory region\n", __func__);
 512                 return PTR_ERR(mem);
 513         }
 514
 515         for_each_set_bit(bit, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
 516                 unsigned int page_size = BIT(bit);
 517                 resource_size_t phys;
 518                 int i;
 519
 520                 for (i = 0; i < ARRAY_SIZE(flags); ++i) {
 521                         obj = i915_gem_object_create_region(mem,
 522                                                             page_size, page_size,
 523                                                             flags[i]);
 524                         if (IS_ERR(obj)) {
 525                                 err = PTR_ERR(obj);
 526                                 goto out_region;
 527                         }
 528
 529                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 530                         if (IS_ERR(vma)) {
 531                                 err = PTR_ERR(vma);
 532                                 goto out_put;
 533                         }
 534
 535                         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 536                         if (err)
 537                                 goto out_put;
 538
 539                         err = igt_check_page_sizes(vma);
 540                         if (err)
 541                                 goto out_unpin;
 542
 543                         phys = i915_gem_object_get_dma_address(obj, 0);
 544                         if (!IS_ALIGNED(phys, page_size)) {
 545                                 pr_err("%s addr misaligned(%pa) page_size=%u\n",
 546                                        __func__, &phys, page_size);
 547                                 err = -EINVAL;
 548                                 goto out_unpin;
 549                         }
 550
 551                         if (vma->resource->page_sizes_gtt != page_size) {
 552                                 pr_err("%s page_sizes.gtt=%u, expected=%u\n",
 553                                        __func__, vma->resource->page_sizes_gtt,
 554                                        page_size);
 555                                 err = -EINVAL;
 556                                 goto out_unpin;
 557                         }
 558
 559                         i915_vma_unpin(vma);
 560                         __i915_gem_object_put_pages(obj);
 561                         i915_gem_object_put(obj);
 562                 }
 563         }
 564
 565         goto out_region;
 566
 567 out_unpin:
 568         i915_vma_unpin(vma);
 569 out_put:
 570         i915_gem_object_put(obj);
 571 out_region:
 572         intel_memory_region_destroy(mem);
 573         return err;
 574 }
 575
 576 static int igt_mock_ppgtt_misaligned_dma(void *arg)
 577 {
 578         struct i915_ppgtt *ppgtt = arg;
 579         struct drm_i915_private *i915 = ppgtt->vm.i915;
 580         unsigned long supported = RUNTIME_INFO(i915)->page_sizes;
 581         struct drm_i915_gem_object *obj;
 582         int bit;
 583         int err;
 584
 585         /*
 586          * Sanity check dma misalignment for huge pages -- the dma addresses we
 587          * insert into the paging structures need to always respect the page
 588          * size alignment.
 589          */
 590
 591         bit = ilog2(I915_GTT_PAGE_SIZE_64K);
 592
 593         for_each_set_bit_from(bit, &supported,
 594                               ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
 595                 IGT_TIMEOUT(end_time);
 596                 unsigned int page_size = BIT(bit);
 597                 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
 598                 unsigned int offset;
 599                 unsigned int size =
 600                         round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1;
 601                 struct i915_vma *vma;
 602
 603                 obj = fake_huge_pages_object(i915, size, true);
 604                 if (IS_ERR(obj))
 605                         return PTR_ERR(obj);
 606
 607                 if (obj->base.size != size) {
 608                         pr_err("obj->base.size=%zu, expected=%u\n",
 609                                obj->base.size, size);
 610                         err = -EINVAL;
 611                         goto out_put;
 612                 }
 613
 614                 err = i915_gem_object_pin_pages_unlocked(obj);
 615                 if (err)
 616                         goto out_put;
 617
 618                 /* Force the page size for this object */
 619                 obj->mm.page_sizes.sg = page_size;
 620
 621                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 622                 if (IS_ERR(vma)) {
 623                         err = PTR_ERR(vma);
 624                         goto out_unpin;
 625                 }
 626
 627                 err = i915_vma_pin(vma, 0, 0, flags);
 628                 if (err)
 629                         goto out_unpin;
 630
 631
 632                 err = igt_check_page_sizes(vma);
 633
 634                 if (vma->resource->page_sizes_gtt != page_size) {
 635                         pr_err("page_sizes.gtt=%u, expected %u\n",
 636                                vma->resource->page_sizes_gtt, page_size);
 637                         err = -EINVAL;
 638                 }
 639
 640                 i915_vma_unpin(vma);
 641
 642                 if (err)
 643                         goto out_unpin;
 644
 645                 /*
 646                  * Try all the other valid offsets until the next
 647                  * boundary -- should always fall back to using 4K
 648                  * pages.
 649                  */
 650                 for (offset = 4096; offset < page_size; offset += 4096) {
 651                         err = i915_vma_unbind_unlocked(vma);
 652                         if (err)
 653                                 goto out_unpin;
 654
 655                         err = i915_vma_pin(vma, 0, 0, flags | offset);
 656                         if (err)
 657                                 goto out_unpin;
 658
 659                         err = igt_check_page_sizes(vma);
 660
 661                         if (vma->resource->page_sizes_gtt != I915_GTT_PAGE_SIZE_4K) {
 662                                 pr_err("page_sizes.gtt=%u, expected %llu\n",
 663                                        vma->resource->page_sizes_gtt,
 664                                        I915_GTT_PAGE_SIZE_4K);
 665                                 err = -EINVAL;
 666                         }
 667
 668                         i915_vma_unpin(vma);
 669
 670                         if (err)
 671                                 goto out_unpin;
 672
 673                         if (igt_timeout(end_time,
 674                                         "%s timed out at offset %x with page-size %x\n",
 675                                         __func__, offset, page_size))
 676                                 break;
 677                 }
 678
 679                 i915_gem_object_lock(obj, NULL);
 680                 i915_gem_object_unpin_pages(obj);
 681                 __i915_gem_object_put_pages(obj);
 682                 i915_gem_object_unlock(obj);
 683                 i915_gem_object_put(obj);
 684         }
 685
 686         return 0;
 687
 688 out_unpin:
 689         i915_gem_object_lock(obj, NULL);
 690         i915_gem_object_unpin_pages(obj);
 691         i915_gem_object_unlock(obj);
 692 out_put:
 693         i915_gem_object_put(obj);
 694
 695         return err;
 696 }
 697
 698 static void close_object_list(struct list_head *objects)
 699 {
 700         struct drm_i915_gem_object *obj, *on;
 701
 702         list_for_each_entry_safe(obj, on, objects, st_link) {
 703                 list_del(&obj->st_link);
 704                 i915_gem_object_lock(obj, NULL);
 705                 i915_gem_object_unpin_pages(obj);
 706                 __i915_gem_object_put_pages(obj);
 707                 i915_gem_object_unlock(obj);
 708                 i915_gem_object_put(obj);
 709         }
 710 }
 711
 712 static int igt_ppgtt_huge_fill(void *arg)
 713 {
 714         struct drm_i915_private *i915 = arg;
 715         unsigned int supported = RUNTIME_INFO(i915)->page_sizes;
 716         bool has_pte64 = GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50);
 717         struct i915_address_space *vm;
 718         struct i915_gem_context *ctx;
 719         unsigned long max_pages;
 720         unsigned long page_num;
 721         struct file *file;
 722         bool single = false;
 723         LIST_HEAD(objects);
 724         IGT_TIMEOUT(end_time);
 725         int err = -ENODEV;
 726
 727         if (supported == I915_GTT_PAGE_SIZE_4K)
 728                 return 0;
 729
 730         file = mock_file(i915);
 731         if (IS_ERR(file))
 732                 return PTR_ERR(file);
 733
 734         ctx = hugepage_ctx(i915, file);
 735         if (IS_ERR(ctx)) {
 736                 err = PTR_ERR(ctx);
 737                 goto out;
 738         }
 739         vm = i915_gem_context_get_eb_vm(ctx);
 740         max_pages = vm->total >> PAGE_SHIFT;
 741
 742         for_each_prime_number_from(page_num, 1, max_pages) {
 743                 struct drm_i915_gem_object *obj;
 744                 u64 size = page_num << PAGE_SHIFT;
 745                 struct i915_vma *vma;
 746                 unsigned int expected_gtt = 0;
 747                 int i;
 748
 749                 obj = fake_huge_pages_object(i915, size, single);
 750                 if (IS_ERR(obj)) {
 751                         err = PTR_ERR(obj);
 752                         break;
 753                 }
 754
 755                 if (obj->base.size != size) {
 756                         pr_err("obj->base.size=%zd, expected=%llu\n",
 757                                obj->base.size, size);
 758                         i915_gem_object_put(obj);
 759                         err = -EINVAL;
 760                         break;
 761                 }
 762
 763                 err = i915_gem_object_pin_pages_unlocked(obj);
 764                 if (err) {
 765                         i915_gem_object_put(obj);
 766                         break;
 767                 }
 768
 769                 list_add(&obj->st_link, &objects);
 770
 771                 vma = i915_vma_instance(obj, vm, NULL);
 772                 if (IS_ERR(vma)) {
 773                         err = PTR_ERR(vma);
 774                         break;
 775                 }
 776
 777                 /* vma start must be aligned to BIT(21) to allow 2M PTEs */
 778                 err = i915_vma_pin(vma, 0, BIT(21), PIN_USER);
 779                 if (err)
 780                         break;
 781
 782                 err = igt_check_page_sizes(vma);
 783                 if (err) {
 784                         i915_vma_unpin(vma);
 785                         break;
 786                 }
 787
 788                 /*
 789                  * Figure out the expected gtt page size knowing that we go from
 790                  * largest to smallest page size sg chunks, and that we align to
 791                  * the largest page size.
 792                  */
 793                 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
 794                         unsigned int page_size = page_sizes[i];
 795
 796                         if (HAS_PAGE_SIZES(i915, page_size) &&
 797                             size >= page_size) {
 798                                 expected_gtt |= page_size;
 799                                 size &= page_size-1;
 800                         }
 801                 }
 802
 803                 GEM_BUG_ON(!expected_gtt);
 804                 GEM_BUG_ON(size);
 805
 806                 if (!has_pte64 && (obj->base.size < I915_GTT_PAGE_SIZE_2M ||
 807                                    expected_gtt & I915_GTT_PAGE_SIZE_2M))
 808                         expected_gtt &= ~I915_GTT_PAGE_SIZE_64K;
 809
 810                 i915_vma_unpin(vma);
 811
 812                 if (!has_pte64 && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 813                         if (!IS_ALIGNED(vma->node.start,
 814                                         I915_GTT_PAGE_SIZE_2M)) {
 815                                 pr_err("node.start(%llx) not aligned to 2M\n",
 816                                        vma->node.start);
 817                                 err = -EINVAL;
 818                                 break;
 819                         }
 820
 821                         if (!IS_ALIGNED(vma->node.size,
 822                                         I915_GTT_PAGE_SIZE_2M)) {
 823                                 pr_err("node.size(%llx) not aligned to 2M\n",
 824                                        vma->node.size);
 825                                 err = -EINVAL;
 826                                 break;
 827                         }
 828                 }
 829
 830                 if (vma->resource->page_sizes_gtt != expected_gtt) {
 831                         pr_err("gtt=%#x, expected=%#x, size=0x%zx, single=%s\n",
 832                                vma->resource->page_sizes_gtt, expected_gtt,
 833                                obj->base.size, str_yes_no(!!single));
 834                         err = -EINVAL;
 835                         break;
 836                 }
 837
 838                 if (igt_timeout(end_time,
 839                                 "%s timed out at size %zd\n",
 840                                 __func__, obj->base.size))
 841                         break;
 842
 843                 single = !single;
 844         }
 845
 846         close_object_list(&objects);
 847
 848         if (err == -ENOMEM || err == -ENOSPC)
 849                 err = 0;
 850
 851         i915_vm_put(vm);
 852 out:
 853         fput(file);
 854         return err;
 855 }
 856
 857 static int igt_ppgtt_64K(void *arg)
 858 {
 859         struct drm_i915_private *i915 = arg;
 860         bool has_pte64 = GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50);
 861         struct drm_i915_gem_object *obj;
 862         struct i915_address_space *vm;
 863         struct i915_gem_context *ctx;
 864         struct file *file;
 865         const struct object_info {
 866                 unsigned int size;
 867                 unsigned int gtt;
 868                 unsigned int offset;
 869         } objects[] = {
 870                 /* Cases with forced padding/alignment */
 871                 {
 872                         .size = SZ_64K,
 873                         .gtt = I915_GTT_PAGE_SIZE_64K,
 874                         .offset = 0,
 875                 },
 876                 {
 877                         .size = SZ_64K + SZ_4K,
 878                         .gtt = I915_GTT_PAGE_SIZE_4K,
 879                         .offset = 0,
 880                 },
 881                 {
 882                         .size = SZ_64K - SZ_4K,
 883                         .gtt = I915_GTT_PAGE_SIZE_4K,
 884                         .offset = 0,
 885                 },
 886                 {
 887                         .size = SZ_2M,
 888                         .gtt = I915_GTT_PAGE_SIZE_64K,
 889                         .offset = 0,
 890                 },
 891                 {
 892                         .size = SZ_2M - SZ_4K,
 893                         .gtt = I915_GTT_PAGE_SIZE_4K,
 894                         .offset = 0,
 895                 },
 896                 {
 897                         .size = SZ_2M + SZ_4K,
 898                         .gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K,
 899                         .offset = 0,
 900                 },
 901                 {
 902                         .size = SZ_2M + SZ_64K,
 903                         .gtt = I915_GTT_PAGE_SIZE_64K,
 904                         .offset = 0,
 905                 },
 906                 {
 907                         .size = SZ_2M - SZ_64K,
 908                         .gtt = I915_GTT_PAGE_SIZE_64K,
 909                         .offset = 0,
 910                 },
 911                 /* Try without any forced padding/alignment */
 912                 {
 913                         .size = SZ_64K,
 914                         .offset = SZ_2M,
 915                         .gtt = I915_GTT_PAGE_SIZE_4K,
 916                 },
 917                 {
 918                         .size = SZ_128K,
 919                         .offset = SZ_2M - SZ_64K,
 920                         .gtt = I915_GTT_PAGE_SIZE_4K,
 921                 },
 922         };
 923         struct i915_vma *vma;
 924         int i, single;
 925         int err;
 926
 927         /*
 928          * Sanity check some of the trickiness with 64K pages -- either we can
 929          * safely mark the whole page-table(2M block) as 64K, or we have to
 930          * always fallback to 4K.
 931          */
 932
 933         if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K))
 934                 return 0;
 935
 936         file = mock_file(i915);
 937         if (IS_ERR(file))
 938                 return PTR_ERR(file);
 939
 940         ctx = hugepage_ctx(i915, file);
 941         if (IS_ERR(ctx)) {
 942                 err = PTR_ERR(ctx);
 943                 goto out;
 944         }
 945         vm = i915_gem_context_get_eb_vm(ctx);
 946
 947         for (i = 0; i < ARRAY_SIZE(objects); ++i) {
 948                 unsigned int size = objects[i].size;
 949                 unsigned int expected_gtt = objects[i].gtt;
 950                 unsigned int offset = objects[i].offset;
 951                 unsigned int flags = PIN_USER;
 952
 953                 /*
 954                  * For modern GTT models, the requirements for marking a page-table
 955                  * as 64K have been relaxed.  Account for this.
 956                  */
 957                 if (has_pte64) {
 958                         expected_gtt = 0;
 959                         if (size >= SZ_64K)
 960                                 expected_gtt |= I915_GTT_PAGE_SIZE_64K;
 961                         if (size & (SZ_64K - 1))
 962                                 expected_gtt |= I915_GTT_PAGE_SIZE_4K;
 963                 }
 964
 965                 for (single = 0; single <= 1; single++) {
 966                         obj = fake_huge_pages_object(i915, size, !!single);
 967                         if (IS_ERR(obj)) {
 968                                 err = PTR_ERR(obj);
 969                                 goto out_vm;
 970                         }
 971
 972                         err = i915_gem_object_pin_pages_unlocked(obj);
 973                         if (err)
 974                                 goto out_object_put;
 975
 976                         /*
 977                          * Disable 2M pages -- We only want to use 64K/4K pages
 978                          * for this test.
 979                          */
 980                         obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M;
 981
 982                         vma = i915_vma_instance(obj, vm, NULL);
 983                         if (IS_ERR(vma)) {
 984                                 err = PTR_ERR(vma);
 985                                 goto out_object_unpin;
 986                         }
 987
 988                         if (offset)
 989                                 flags |= PIN_OFFSET_FIXED | offset;
 990
 991                         err = i915_vma_pin(vma, 0, 0, flags);
 992                         if (err)
 993                                 goto out_object_unpin;
 994
 995                         err = igt_check_page_sizes(vma);
 996                         if (err)
 997                                 goto out_vma_unpin;
 998
 999                         if (!has_pte64 && !offset &&
1000                             vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
1001                                 if (!IS_ALIGNED(vma->node.start,
1002                                                 I915_GTT_PAGE_SIZE_2M)) {
1003                                         pr_err("node.start(%llx) not aligned to 2M\n",
1004                                                vma->node.start);
1005                                         err = -EINVAL;
1006                                         goto out_vma_unpin;
1007                                 }
1008
1009                                 if (!IS_ALIGNED(vma->node.size,
1010                                                 I915_GTT_PAGE_SIZE_2M)) {
1011                                         pr_err("node.size(%llx) not aligned to 2M\n",
1012                                                vma->node.size);
1013                                         err = -EINVAL;
1014                                         goto out_vma_unpin;
1015                                 }
1016                         }
1017
1018                         if (vma->resource->page_sizes_gtt != expected_gtt) {
1019                                 pr_err("gtt=%#x, expected=%#x, i=%d, single=%s offset=%#x size=%#x\n",
1020                                        vma->resource->page_sizes_gtt,
1021                                        expected_gtt, i, str_yes_no(!!single),
1022                                        offset, size);
1023                                 err = -EINVAL;
1024                                 goto out_vma_unpin;
1025                         }
1026
1027                         i915_vma_unpin(vma);
1028                         i915_gem_object_lock(obj, NULL);
1029                         i915_gem_object_unpin_pages(obj);
1030                         __i915_gem_object_put_pages(obj);
1031                         i915_gem_object_unlock(obj);
1032                         i915_gem_object_put(obj);
1033
1034                         i915_gem_drain_freed_objects(i915);
1035                 }
1036         }
1037
1038         goto out_vm;
1039
1040 out_vma_unpin:
1041         i915_vma_unpin(vma);
1042 out_object_unpin:
1043         i915_gem_object_lock(obj, NULL);
1044         i915_gem_object_unpin_pages(obj);
1045         i915_gem_object_unlock(obj);
1046 out_object_put:
1047         i915_gem_object_put(obj);
1048 out_vm:
1049         i915_vm_put(vm);
1050 out:
1051         fput(file);
1052         return err;
1053 }
1054
1055 static int gpu_write(struct intel_context *ce,
1056                      struct i915_vma *vma,
1057                      u32 dw,
1058                      u32 val)
1059 {
1060         int err;
1061
1062         i915_gem_object_lock(vma->obj, NULL);
1063         err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
1064         i915_gem_object_unlock(vma->obj);
1065         if (err)
1066                 return err;
1067
1068         return igt_gpu_fill_dw(ce, vma, dw * sizeof(u32),
1069                                vma->size >> PAGE_SHIFT, val);
1070 }
1071
1072 static int
1073 __cpu_check_shmem(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1074 {
1075         unsigned int needs_flush;
1076         unsigned long n;
1077         int err;
1078
1079         i915_gem_object_lock(obj, NULL);
1080         err = i915_gem_object_prepare_read(obj, &needs_flush);
1081         if (err)
1082                 goto err_unlock;
1083
1084         for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) {
1085                 u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n));
1086
1087                 if (needs_flush & CLFLUSH_BEFORE)
1088                         drm_clflush_virt_range(ptr, PAGE_SIZE);
1089
1090                 if (ptr[dword] != val) {
1091                         pr_err("n=%lu ptr[%u]=%u, val=%u\n",
1092                                n, dword, ptr[dword], val);
1093                         kunmap_atomic(ptr);
1094                         err = -EINVAL;
1095                         break;
1096                 }
1097
1098                 kunmap_atomic(ptr);
1099         }
1100
1101         i915_gem_object_finish_access(obj);
1102 err_unlock:
1103         i915_gem_object_unlock(obj);
1104
1105         return err;
1106 }
1107
1108 static int __cpu_check_vmap(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1109 {
1110         unsigned long n = obj->base.size >> PAGE_SHIFT;
1111         u32 *ptr;
1112         int err;
1113
1114         err = i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT);
1115         if (err)
1116                 return err;
1117
1118         ptr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1119         if (IS_ERR(ptr))
1120                 return PTR_ERR(ptr);
1121
1122         ptr += dword;
1123         while (n--) {
1124                 if (*ptr != val) {
1125                         pr_err("base[%u]=%08x, val=%08x\n",
1126                                dword, *ptr, val);
1127                         err = -EINVAL;
1128                         break;
1129                 }
1130
1131                 ptr += PAGE_SIZE / sizeof(*ptr);
1132         }
1133
1134         i915_gem_object_unpin_map(obj);
1135         return err;
1136 }
1137
1138 static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1139 {
1140         if (i915_gem_object_has_struct_page(obj))
1141                 return __cpu_check_shmem(obj, dword, val);
1142         else
1143                 return __cpu_check_vmap(obj, dword, val);
1144 }
1145
1146 static int __igt_write_huge(struct intel_context *ce,
1147                             struct drm_i915_gem_object *obj,
1148                             u64 size, u64 offset,
1149                             u32 dword, u32 val)
1150 {
1151         unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1152         struct i915_vma *vma;
1153         int err;
1154
1155         vma = i915_vma_instance(obj, ce->vm, NULL);
1156         if (IS_ERR(vma))
1157                 return PTR_ERR(vma);
1158
1159         err = i915_vma_pin(vma, size, 0, flags | offset);
1160         if (err) {
1161                 /*
1162                  * The ggtt may have some pages reserved so
1163                  * refrain from erroring out.
1164                  */
1165                 if (err == -ENOSPC && i915_is_ggtt(ce->vm))
1166                         err = 0;
1167
1168                 return err;
1169         }
1170
1171         err = igt_check_page_sizes(vma);
1172         if (err)
1173                 goto out_vma_unpin;
1174
1175         err = gpu_write(ce, vma, dword, val);
1176         if (err) {
1177                 pr_err("gpu-write failed at offset=%llx\n", offset);
1178                 goto out_vma_unpin;
1179         }
1180
1181         err = cpu_check(obj, dword, val);
1182         if (err) {
1183                 pr_err("cpu-check failed at offset=%llx\n", offset);
1184                 goto out_vma_unpin;
1185         }
1186
1187 out_vma_unpin:
1188         i915_vma_unpin(vma);
1189         return err;
1190 }
1191
1192 static int igt_write_huge(struct drm_i915_private *i915,
1193                           struct drm_i915_gem_object *obj)
1194 {
1195         struct i915_gem_engines *engines;
1196         struct i915_gem_engines_iter it;
1197         struct intel_context *ce;
1198         I915_RND_STATE(prng);
1199         IGT_TIMEOUT(end_time);
1200         unsigned int max_page_size;
1201         unsigned int count;
1202         struct i915_gem_context *ctx;
1203         struct file *file;
1204         u64 max;
1205         u64 num;
1206         u64 size;
1207         int *order;
1208         int i, n;
1209         int err = 0;
1210
1211         file = mock_file(i915);
1212         if (IS_ERR(file))
1213                 return PTR_ERR(file);
1214
1215         ctx = hugepage_ctx(i915, file);
1216         if (IS_ERR(ctx)) {
1217                 err = PTR_ERR(ctx);
1218                 goto out;
1219         }
1220
1221         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
1222
1223         size = obj->base.size;
1224         if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K &&
1225             !HAS_64K_PAGES(i915))
1226                 size = round_up(size, I915_GTT_PAGE_SIZE_2M);
1227
1228         n = 0;
1229         count = 0;
1230         max = U64_MAX;
1231         for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1232                 count++;
1233                 if (!intel_engine_can_store_dword(ce->engine))
1234                         continue;
1235
1236                 max = min(max, ce->vm->total);
1237                 n++;
1238         }
1239         i915_gem_context_unlock_engines(ctx);
1240         if (!n)
1241                 goto out;
1242
1243         /*
1244          * To keep things interesting when alternating between engines in our
1245          * randomized order, lets also make feeding to the same engine a few
1246          * times in succession a possibility by enlarging the permutation array.
1247          */
1248         order = i915_random_order(count * count, &prng);
1249         if (!order) {
1250                 err = -ENOMEM;
1251                 goto out;
1252         }
1253
1254         max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg);
1255         max = div_u64(max - size, max_page_size);
1256
1257         /*
1258          * Try various offsets in an ascending/descending fashion until we
1259          * timeout -- we want to avoid issues hidden by effectively always using
1260          * offset = 0.
1261          */
1262         i = 0;
1263         engines = i915_gem_context_lock_engines(ctx);
1264         for_each_prime_number_from(num, 0, max) {
1265                 u64 offset_low = num * max_page_size;
1266                 u64 offset_high = (max - num) * max_page_size;
1267                 u32 dword = offset_in_page(num) / 4;
1268                 struct intel_context *ce;
1269
1270                 ce = engines->engines[order[i] % engines->num_engines];
1271                 i = (i + 1) % (count * count);
1272                 if (!ce || !intel_engine_can_store_dword(ce->engine))
1273                         continue;
1274
1275                 /*
1276                  * In order to utilize 64K pages we need to both pad the vma
1277                  * size and ensure the vma offset is at the start of the pt
1278                  * boundary, however to improve coverage we opt for testing both
1279                  * aligned and unaligned offsets.
1280                  *
1281                  * With PS64 this is no longer the case, but to ensure we
1282                  * sometimes get the compact layout for smaller objects, apply
1283                  * the round_up anyway.
1284                  */
1285                 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
1286                         offset_low = round_down(offset_low,
1287                                                 I915_GTT_PAGE_SIZE_2M);
1288
1289                 err = __igt_write_huge(ce, obj, size, offset_low,
1290                                        dword, num + 1);
1291                 if (err)
1292                         break;
1293
1294                 err = __igt_write_huge(ce, obj, size, offset_high,
1295                                        dword, num + 1);
1296                 if (err)
1297                         break;
1298
1299                 if (igt_timeout(end_time,
1300                                 "%s timed out on %s, offset_low=%llx offset_high=%llx, max_page_size=%x\n",
1301                                 __func__, ce->engine->name, offset_low, offset_high,
1302                                 max_page_size))
1303                         break;
1304         }
1305         i915_gem_context_unlock_engines(ctx);
1306
1307         kfree(order);
1308
1309 out:
1310         fput(file);
1311         return err;
1312 }
1313
1314 typedef struct drm_i915_gem_object *
1315 (*igt_create_fn)(struct drm_i915_private *i915, u32 size, u32 flags);
1316
1317 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915)
1318 {
1319         return i915->mm.gemfs && has_transparent_hugepage();
1320 }
1321
1322 static struct drm_i915_gem_object *
1323 igt_create_shmem(struct drm_i915_private *i915, u32 size, u32 flags)
1324 {
1325         if (!igt_can_allocate_thp(i915)) {
1326                 pr_info("%s missing THP support, skipping\n", __func__);
1327                 return ERR_PTR(-ENODEV);
1328         }
1329
1330         return i915_gem_object_create_shmem(i915, size);
1331 }
1332
1333 static struct drm_i915_gem_object *
1334 igt_create_internal(struct drm_i915_private *i915, u32 size, u32 flags)
1335 {
1336         return i915_gem_object_create_internal(i915, size);
1337 }
1338
1339 static struct drm_i915_gem_object *
1340 igt_create_system(struct drm_i915_private *i915, u32 size, u32 flags)
1341 {
1342         return huge_pages_object(i915, size, size);
1343 }
1344
1345 static struct drm_i915_gem_object *
1346 igt_create_local(struct drm_i915_private *i915, u32 size, u32 flags)
1347 {
1348         return i915_gem_object_create_lmem(i915, size, flags);
1349 }
1350
1351 static u32 igt_random_size(struct rnd_state *prng,
1352                            u32 min_page_size,
1353                            u32 max_page_size)
1354 {
1355         u64 mask;
1356         u32 size;
1357
1358         GEM_BUG_ON(!is_power_of_2(min_page_size));
1359         GEM_BUG_ON(!is_power_of_2(max_page_size));
1360         GEM_BUG_ON(min_page_size < PAGE_SIZE);
1361         GEM_BUG_ON(min_page_size > max_page_size);
1362
1363         mask = ((max_page_size << 1ULL) - 1) & PAGE_MASK;
1364         size = prandom_u32_state(prng) & mask;
1365         if (size < min_page_size)
1366                 size |= min_page_size;
1367
1368         return size;
1369 }
1370
1371 static int igt_ppgtt_smoke_huge(void *arg)
1372 {
1373         struct drm_i915_private *i915 = arg;
1374         struct drm_i915_gem_object *obj;
1375         I915_RND_STATE(prng);
1376         struct {
1377                 igt_create_fn fn;
1378                 u32 min;
1379                 u32 max;
1380         } backends[] = {
1381                 { igt_create_internal, SZ_64K, SZ_2M,  },
1382                 { igt_create_shmem,    SZ_64K, SZ_32M, },
1383                 { igt_create_local,    SZ_64K, SZ_1G,  },
1384         };
1385         int err;
1386         int i;
1387
1388         /*
1389          * Sanity check that the HW uses huge pages correctly through our
1390          * various backends -- ensure that our writes land in the right place.
1391          */
1392
1393         for (i = 0; i < ARRAY_SIZE(backends); ++i) {
1394                 u32 min = backends[i].min;
1395                 u32 max = backends[i].max;
1396                 u32 size = max;
1397
1398 try_again:
1399                 size = igt_random_size(&prng, min, rounddown_pow_of_two(size));
1400
1401                 obj = backends[i].fn(i915, size, 0);
1402                 if (IS_ERR(obj)) {
1403                         err = PTR_ERR(obj);
1404                         if (err == -E2BIG) {
1405                                 size >>= 1;
1406                                 goto try_again;
1407                         } else if (err == -ENODEV) {
1408                                 err = 0;
1409                                 continue;
1410                         }
1411
1412                         return err;
1413                 }
1414
1415                 err = i915_gem_object_pin_pages_unlocked(obj);
1416                 if (err) {
1417                         if (err == -ENXIO || err == -E2BIG || err == -ENOMEM) {
1418                                 i915_gem_object_put(obj);
1419                                 size >>= 1;
1420                                 goto try_again;
1421                         }
1422                         goto out_put;
1423                 }
1424
1425                 if (obj->mm.page_sizes.phys < min) {
1426                         pr_info("%s unable to allocate huge-page(s) with size=%u, i=%d\n",
1427                                 __func__, size, i);
1428                         err = -ENOMEM;
1429                         goto out_unpin;
1430                 }
1431
1432                 err = igt_write_huge(i915, obj);
1433                 if (err) {
1434                         pr_err("%s write-huge failed with size=%u, i=%d\n",
1435                                __func__, size, i);
1436                 }
1437 out_unpin:
1438                 i915_gem_object_lock(obj, NULL);
1439                 i915_gem_object_unpin_pages(obj);
1440                 __i915_gem_object_put_pages(obj);
1441                 i915_gem_object_unlock(obj);
1442 out_put:
1443                 i915_gem_object_put(obj);
1444
1445                 if (err == -ENOMEM || err == -ENXIO)
1446                         err = 0;
1447
1448                 if (err)
1449                         break;
1450
1451                 cond_resched();
1452         }
1453
1454         return err;
1455 }
1456
1457 static int igt_ppgtt_sanity_check(void *arg)
1458 {
1459         struct drm_i915_private *i915 = arg;
1460         unsigned int supported = RUNTIME_INFO(i915)->page_sizes;
1461         struct {
1462                 igt_create_fn fn;
1463                 unsigned int flags;
1464         } backends[] = {
1465                 { igt_create_system, 0,                        },
1466                 { igt_create_local,  0,                        },
1467                 { igt_create_local,  I915_BO_ALLOC_CONTIGUOUS, },
1468         };
1469         struct {
1470                 u32 size;
1471                 u32 pages;
1472         } combos[] = {
1473                 { SZ_64K,               SZ_64K          },
1474                 { SZ_2M,                SZ_2M           },
1475                 { SZ_2M,                SZ_64K          },
1476                 { SZ_2M - SZ_64K,       SZ_64K          },
1477                 { SZ_2M - SZ_4K,        SZ_64K | SZ_4K  },
1478                 { SZ_2M + SZ_4K,        SZ_64K | SZ_4K  },
1479                 { SZ_2M + SZ_4K,        SZ_2M  | SZ_4K  },
1480                 { SZ_2M + SZ_64K,       SZ_2M  | SZ_64K },
1481                 { SZ_2M + SZ_64K,       SZ_64K          },
1482         };
1483         int i, j;
1484         int err;
1485
1486         if (supported == I915_GTT_PAGE_SIZE_4K)
1487                 return 0;
1488
1489         /*
1490          * Sanity check that the HW behaves with a limited set of combinations.
1491          * We already have a bunch of randomised testing, which should give us
1492          * a decent amount of variation between runs, however we should keep
1493          * this to limit the chances of introducing a temporary regression, by
1494          * testing the most obvious cases that might make something blow up.
1495          */
1496
1497         for (i = 0; i < ARRAY_SIZE(backends); ++i) {
1498                 for (j = 0; j < ARRAY_SIZE(combos); ++j) {
1499                         struct drm_i915_gem_object *obj;
1500                         u32 size = combos[j].size;
1501                         u32 pages = combos[j].pages;
1502
1503                         obj = backends[i].fn(i915, size, backends[i].flags);
1504                         if (IS_ERR(obj)) {
1505                                 err = PTR_ERR(obj);
1506                                 if (err == -ENODEV) {
1507                                         pr_info("Device lacks local memory, skipping\n");
1508                                         err = 0;
1509                                         break;
1510                                 }
1511
1512                                 return err;
1513                         }
1514
1515                         err = i915_gem_object_pin_pages_unlocked(obj);
1516                         if (err) {
1517                                 i915_gem_object_put(obj);
1518                                 goto out;
1519                         }
1520
1521                         GEM_BUG_ON(pages > obj->base.size);
1522                         pages = pages & supported;
1523
1524                         if (pages)
1525                                 obj->mm.page_sizes.sg = pages;
1526
1527                         err = igt_write_huge(i915, obj);
1528
1529                         i915_gem_object_lock(obj, NULL);
1530                         i915_gem_object_unpin_pages(obj);
1531                         __i915_gem_object_put_pages(obj);
1532                         i915_gem_object_unlock(obj);
1533                         i915_gem_object_put(obj);
1534
1535                         if (err) {
1536                                 pr_err("%s write-huge failed with size=%u pages=%u i=%d, j=%d\n",
1537                                        __func__, size, pages, i, j);
1538                                 goto out;
1539                         }
1540                 }
1541
1542                 cond_resched();
1543         }
1544
1545 out:
1546         if (err == -ENOMEM)
1547                 err = 0;
1548
1549         return err;
1550 }
1551
1552 static int igt_ppgtt_compact(void *arg)
1553 {
1554         struct drm_i915_private *i915 = arg;
1555         struct drm_i915_gem_object *obj;
1556         int err;
1557
1558         /*
1559          * Simple test to catch issues with compact 64K pages -- since the pt is
1560          * compacted to 256B that gives us 32 entries per pt, however since the
1561          * backing page for the pt is 4K, any extra entries we might incorrectly
1562          * write out should be ignored by the HW. If ever hit such a case this
1563          * test should catch it since some of our writes would land in scratch.
1564          */
1565
1566         if (!HAS_64K_PAGES(i915)) {
1567                 pr_info("device lacks compact 64K page support, skipping\n");
1568                 return 0;
1569         }
1570
1571         if (!HAS_LMEM(i915)) {
1572                 pr_info("device lacks LMEM support, skipping\n");
1573                 return 0;
1574         }
1575
1576         /* We want the range to cover multiple page-table boundaries. */
1577         obj = i915_gem_object_create_lmem(i915, SZ_4M, 0);
1578         if (IS_ERR(obj))
1579                 return PTR_ERR(obj);
1580
1581         err = i915_gem_object_pin_pages_unlocked(obj);
1582         if (err)
1583                 goto out_put;
1584
1585         if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_64K) {
1586                 pr_info("LMEM compact unable to allocate huge-page(s)\n");
1587                 goto out_unpin;
1588         }
1589
1590         /*
1591          * Disable 2M GTT pages by forcing the page-size to 64K for the GTT
1592          * insertion.
1593          */
1594         obj->mm.page_sizes.sg = I915_GTT_PAGE_SIZE_64K;
1595
1596         err = igt_write_huge(i915, obj);
1597         if (err)
1598                 pr_err("LMEM compact write-huge failed\n");
1599
1600 out_unpin:
1601         i915_gem_object_unpin_pages(obj);
1602 out_put:
1603         i915_gem_object_put(obj);
1604
1605         if (err == -ENOMEM)
1606                 err = 0;
1607
1608         return err;
1609 }
1610
1611 static int igt_ppgtt_mixed(void *arg)
1612 {
1613         struct drm_i915_private *i915 = arg;
1614         const unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
1615         struct drm_i915_gem_object *obj, *on;
1616         struct i915_gem_engines *engines;
1617         struct i915_gem_engines_iter it;
1618         struct i915_address_space *vm;
1619         struct i915_gem_context *ctx;
1620         struct intel_context *ce;
1621         struct file *file;
1622         I915_RND_STATE(prng);
1623         LIST_HEAD(objects);
1624         struct intel_memory_region *mr;
1625         struct i915_vma *vma;
1626         unsigned int count;
1627         u32 i, addr;
1628         int *order;
1629         int n, err;
1630
1631         /*
1632          * Sanity check mixing 4K and 64K pages within the same page-table via
1633          * the new PS64 TLB hint.
1634          */
1635
1636         if (!HAS_64K_PAGES(i915)) {
1637                 pr_info("device lacks PS64, skipping\n");
1638                 return 0;
1639         }
1640
1641         file = mock_file(i915);
1642         if (IS_ERR(file))
1643                 return PTR_ERR(file);
1644
1645         ctx = hugepage_ctx(i915, file);
1646         if (IS_ERR(ctx)) {
1647                 err = PTR_ERR(ctx);
1648                 goto out;
1649         }
1650         vm = i915_gem_context_get_eb_vm(ctx);
1651
1652         i = 0;
1653         addr = 0;
1654         do {
1655                 u32 sz;
1656
1657                 sz = i915_prandom_u32_max_state(SZ_4M, &prng);
1658                 sz = max_t(u32, sz, SZ_4K);
1659
1660                 mr = i915->mm.regions[INTEL_REGION_LMEM_0];
1661                 if (i & 1)
1662                         mr = i915->mm.regions[INTEL_REGION_SMEM];
1663
1664                 obj = i915_gem_object_create_region(mr, sz, 0, 0);
1665                 if (IS_ERR(obj)) {
1666                         err = PTR_ERR(obj);
1667                         goto out_vm;
1668                 }
1669
1670                 list_add_tail(&obj->st_link, &objects);
1671
1672                 vma = i915_vma_instance(obj, vm, NULL);
1673                 if (IS_ERR(vma)) {
1674                         err = PTR_ERR(vma);
1675                         goto err_put;
1676                 }
1677
1678                 addr = round_up(addr, mr->min_page_size);
1679                 err = i915_vma_pin(vma, 0, 0, addr | flags);
1680                 if (err)
1681                         goto err_put;
1682
1683                 if (mr->type == INTEL_MEMORY_LOCAL &&
1684                     (vma->resource->page_sizes_gtt & I915_GTT_PAGE_SIZE_4K)) {
1685                         err = -EINVAL;
1686                         goto err_put;
1687                 }
1688
1689                 addr += obj->base.size;
1690                 i++;
1691         } while (addr <= SZ_16M);
1692
1693         n = 0;
1694         count = 0;
1695         for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1696                 count++;
1697                 if (!intel_engine_can_store_dword(ce->engine))
1698                         continue;
1699
1700                 n++;
1701         }
1702         i915_gem_context_unlock_engines(ctx);
1703         if (!n)
1704                 goto err_put;
1705
1706         order = i915_random_order(count * count, &prng);
1707         if (!order) {
1708                 err = -ENOMEM;
1709                 goto err_put;
1710         }
1711
1712         i = 0;
1713         addr = 0;
1714         engines = i915_gem_context_lock_engines(ctx);
1715         list_for_each_entry(obj, &objects, st_link) {
1716                 u32 rnd = i915_prandom_u32_max_state(UINT_MAX, &prng);
1717
1718                 addr = round_up(addr, obj->mm.region->min_page_size);
1719
1720                 ce = engines->engines[order[i] % engines->num_engines];
1721                 i = (i + 1) % (count * count);
1722                 if (!ce || !intel_engine_can_store_dword(ce->engine))
1723                         continue;
1724
1725                 err = __igt_write_huge(ce, obj, obj->base.size, addr, 0, rnd);
1726                 if (err)
1727                         break;
1728
1729                 err = __igt_write_huge(ce, obj, obj->base.size, addr,
1730                                        offset_in_page(rnd) / sizeof(u32), rnd + 1);
1731                 if (err)
1732                         break;
1733
1734                 err = __igt_write_huge(ce, obj, obj->base.size, addr,
1735                                        (PAGE_SIZE / sizeof(u32)) - 1,
1736                                        rnd + 2);
1737                 if (err)
1738                         break;
1739
1740                 addr += obj->base.size;
1741
1742                 cond_resched();
1743         }
1744
1745         i915_gem_context_unlock_engines(ctx);
1746         kfree(order);
1747 err_put:
1748         list_for_each_entry_safe(obj, on, &objects, st_link) {
1749                 list_del(&obj->st_link);
1750                 i915_gem_object_put(obj);
1751         }
1752 out_vm:
1753         i915_vm_put(vm);
1754 out:
1755         fput(file);
1756         return err;
1757 }
1758
1759 static int igt_tmpfs_fallback(void *arg)
1760 {
1761         struct drm_i915_private *i915 = arg;
1762         struct i915_address_space *vm;
1763         struct i915_gem_context *ctx;
1764         struct vfsmount *gemfs = i915->mm.gemfs;
1765         struct drm_i915_gem_object *obj;
1766         struct i915_vma *vma;
1767         struct file *file;
1768         u32 *vaddr;
1769         int err = 0;
1770
1771         file = mock_file(i915);
1772         if (IS_ERR(file))
1773                 return PTR_ERR(file);
1774
1775         ctx = hugepage_ctx(i915, file);
1776         if (IS_ERR(ctx)) {
1777                 err = PTR_ERR(ctx);
1778                 goto out;
1779         }
1780         vm = i915_gem_context_get_eb_vm(ctx);
1781
1782         /*
1783          * Make sure that we don't burst into a ball of flames upon falling back
1784          * to tmpfs, which we rely on if on the off-chance we encouter a failure
1785          * when setting up gemfs.
1786          */
1787
1788         i915->mm.gemfs = NULL;
1789
1790         obj = i915_gem_object_create_shmem(i915, PAGE_SIZE);
1791         if (IS_ERR(obj)) {
1792                 err = PTR_ERR(obj);
1793                 goto out_restore;
1794         }
1795
1796         vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1797         if (IS_ERR(vaddr)) {
1798                 err = PTR_ERR(vaddr);
1799                 goto out_put;
1800         }
1801         *vaddr = 0xdeadbeaf;
1802
1803         __i915_gem_object_flush_map(obj, 0, 64);
1804         i915_gem_object_unpin_map(obj);
1805
1806         vma = i915_vma_instance(obj, vm, NULL);
1807         if (IS_ERR(vma)) {
1808                 err = PTR_ERR(vma);
1809                 goto out_put;
1810         }
1811
1812         err = i915_vma_pin(vma, 0, 0, PIN_USER);
1813         if (err)
1814                 goto out_put;
1815
1816         err = igt_check_page_sizes(vma);
1817
1818         i915_vma_unpin(vma);
1819 out_put:
1820         i915_gem_object_put(obj);
1821 out_restore:
1822         i915->mm.gemfs = gemfs;
1823
1824         i915_vm_put(vm);
1825 out:
1826         fput(file);
1827         return err;
1828 }
1829
1830 static int igt_shrink_thp(void *arg)
1831 {
1832         struct drm_i915_private *i915 = arg;
1833         struct i915_address_space *vm;
1834         struct i915_gem_context *ctx;
1835         struct drm_i915_gem_object *obj;
1836         struct i915_gem_engines_iter it;
1837         struct intel_context *ce;
1838         struct i915_vma *vma;
1839         struct file *file;
1840         unsigned int flags = PIN_USER;
1841         unsigned int n;
1842         intel_wakeref_t wf;
1843         bool should_swap;
1844         int err;
1845
1846         if (!igt_can_allocate_thp(i915)) {
1847                 pr_info("missing THP support, skipping\n");
1848                 return 0;
1849         }
1850
1851         file = mock_file(i915);
1852         if (IS_ERR(file))
1853                 return PTR_ERR(file);
1854
1855         ctx = hugepage_ctx(i915, file);
1856         if (IS_ERR(ctx)) {
1857                 err = PTR_ERR(ctx);
1858                 goto out;
1859         }
1860         vm = i915_gem_context_get_eb_vm(ctx);
1861
1862         /*
1863          * Sanity check shrinking huge-paged object -- make sure nothing blows
1864          * up.
1865          */
1866
1867         obj = i915_gem_object_create_shmem(i915, SZ_2M);
1868         if (IS_ERR(obj)) {
1869                 err = PTR_ERR(obj);
1870                 goto out_vm;
1871         }
1872
1873         vma = i915_vma_instance(obj, vm, NULL);
1874         if (IS_ERR(vma)) {
1875                 err = PTR_ERR(vma);
1876                 goto out_put;
1877         }
1878
1879         wf = intel_runtime_pm_get(&i915->runtime_pm); /* active shrink */
1880
1881         err = i915_vma_pin(vma, 0, 0, flags);
1882         if (err)
1883                 goto out_wf;
1884
1885         if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) {
1886                 pr_info("failed to allocate THP, finishing test early\n");
1887                 goto out_unpin;
1888         }
1889
1890         err = igt_check_page_sizes(vma);
1891         if (err)
1892                 goto out_unpin;
1893
1894         n = 0;
1895
1896         for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
1897                 if (!intel_engine_can_store_dword(ce->engine))
1898                         continue;
1899
1900                 err = gpu_write(ce, vma, n++, 0xdeadbeaf);
1901                 if (err)
1902                         break;
1903         }
1904         i915_gem_context_unlock_engines(ctx);
1905         /*
1906          * Nuke everything *before* we unpin the pages so we can be reasonably
1907          * sure that when later checking get_nr_swap_pages() that some random
1908          * leftover object doesn't steal the remaining swap space.
1909          */
1910         i915_gem_shrink(NULL, i915, -1UL, NULL,
1911                         I915_SHRINK_BOUND |
1912                         I915_SHRINK_UNBOUND |
1913                         I915_SHRINK_ACTIVE);
1914         i915_vma_unpin(vma);
1915         if (err)
1916                 goto out_wf;
1917
1918         /*
1919          * Now that the pages are *unpinned* shrinking should invoke
1920          * shmem to truncate our pages, if we have available swap.
1921          */
1922         should_swap = get_nr_swap_pages() > 0;
1923         i915_gem_shrink(NULL, i915, -1UL, NULL,
1924                         I915_SHRINK_BOUND |
1925                         I915_SHRINK_UNBOUND |
1926                         I915_SHRINK_ACTIVE |
1927                         I915_SHRINK_WRITEBACK);
1928         if (should_swap == i915_gem_object_has_pages(obj)) {
1929                 pr_err("unexpected pages mismatch, should_swap=%s\n",
1930                        str_yes_no(should_swap));
1931                 err = -EINVAL;
1932                 goto out_wf;
1933         }
1934
1935         if (should_swap == (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys)) {
1936                 pr_err("unexpected residual page-size bits, should_swap=%s\n",
1937                        str_yes_no(should_swap));
1938                 err = -EINVAL;
1939                 goto out_wf;
1940         }
1941
1942         err = i915_vma_pin(vma, 0, 0, flags);
1943         if (err)
1944                 goto out_wf;
1945
1946         while (n--) {
1947                 err = cpu_check(obj, n, 0xdeadbeaf);
1948                 if (err)
1949                         break;
1950         }
1951
1952 out_unpin:
1953         i915_vma_unpin(vma);
1954 out_wf:
1955         intel_runtime_pm_put(&i915->runtime_pm, wf);
1956 out_put:
1957         i915_gem_object_put(obj);
1958 out_vm:
1959         i915_vm_put(vm);
1960 out:
1961         fput(file);
1962         return err;
1963 }
1964
1965 int i915_gem_huge_page_mock_selftests(void)
1966 {
1967         static const struct i915_subtest tests[] = {
1968                 SUBTEST(igt_mock_exhaust_device_supported_pages),
1969                 SUBTEST(igt_mock_memory_region_huge_pages),
1970                 SUBTEST(igt_mock_ppgtt_misaligned_dma),
1971         };
1972         struct drm_i915_private *dev_priv;
1973         struct i915_ppgtt *ppgtt;
1974         int err;
1975
1976         dev_priv = mock_gem_device();
1977         if (!dev_priv)
1978                 return -ENOMEM;
1979
1980         /* Pretend to be a device which supports the 48b PPGTT */
1981         RUNTIME_INFO(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL;
1982         RUNTIME_INFO(dev_priv)->ppgtt_size = 48;
1983
1984         ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0);
1985         if (IS_ERR(ppgtt)) {
1986                 err = PTR_ERR(ppgtt);
1987                 goto out_unlock;
1988         }
1989
1990         if (!i915_vm_is_4lvl(&ppgtt->vm)) {
1991                 pr_err("failed to create 48b PPGTT\n");
1992                 err = -EINVAL;
1993                 goto out_put;
1994         }
1995
1996         /* If we were ever hit this then it's time to mock the 64K scratch */
1997         if (!i915_vm_has_scratch_64K(&ppgtt->vm)) {
1998                 pr_err("PPGTT missing 64K scratch page\n");
1999                 err = -EINVAL;
2000                 goto out_put;
2001         }
2002
2003         err = i915_subtests(tests, ppgtt);
2004
2005 out_put:
2006         i915_vm_put(&ppgtt->vm);
2007 out_unlock:
2008         mock_destroy_device(dev_priv);
2009         return err;
2010 }
2011
2012 int i915_gem_huge_page_live_selftests(struct drm_i915_private *i915)
2013 {
2014         static const struct i915_subtest tests[] = {
2015                 SUBTEST(igt_shrink_thp),
2016                 SUBTEST(igt_tmpfs_fallback),
2017                 SUBTEST(igt_ppgtt_smoke_huge),
2018                 SUBTEST(igt_ppgtt_sanity_check),
2019                 SUBTEST(igt_ppgtt_compact),
2020                 SUBTEST(igt_ppgtt_mixed),
2021                 SUBTEST(igt_ppgtt_huge_fill),
2022                 SUBTEST(igt_ppgtt_64K),
2023         };
2024
2025         if (!HAS_PPGTT(i915)) {
2026                 pr_info("PPGTT not supported, skipping live-selftests\n");
2027                 return 0;
2028         }
2029
2030         if (intel_gt_is_wedged(to_gt(i915)))
2031                 return 0;
2032
2033         return i915_live_subtests(tests, i915);
2034 }