drivers/gpu/drm/i915/selftests/huge_pages.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include "../i915_selftest.h"
  26
  27 #include <linux/prime_numbers.h>
  28
  29 #include "mock_drm.h"
  30 #include "i915_random.h"
  31
  32 static const unsigned int page_sizes[] = {
  33         I915_GTT_PAGE_SIZE_2M,
  34         I915_GTT_PAGE_SIZE_64K,
  35         I915_GTT_PAGE_SIZE_4K,
  36 };
  37
  38 static unsigned int get_largest_page_size(struct drm_i915_private *i915,
  39                                           u64 rem)
  40 {
  41         int i;
  42
  43         for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
  44                 unsigned int page_size = page_sizes[i];
  45
  46                 if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size)
  47                         return page_size;
  48         }
  49
  50         return 0;
  51 }
  52
  53 static void huge_pages_free_pages(struct sg_table *st)
  54 {
  55         struct scatterlist *sg;
  56
  57         for (sg = st->sgl; sg; sg = __sg_next(sg)) {
  58                 if (sg_page(sg))
  59                         __free_pages(sg_page(sg), get_order(sg->length));
  60         }
  61
  62         sg_free_table(st);
  63         kfree(st);
  64 }
  65
  66 static int get_huge_pages(struct drm_i915_gem_object *obj)
  67 {
  68 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
  69         unsigned int page_mask = obj->mm.page_mask;
  70         struct sg_table *st;
  71         struct scatterlist *sg;
  72         unsigned int sg_page_sizes;
  73         u64 rem;
  74
  75         st = kmalloc(sizeof(*st), GFP);
  76         if (!st)
  77                 return -ENOMEM;
  78
  79         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
  80                 kfree(st);
  81                 return -ENOMEM;
  82         }
  83
  84         rem = obj->base.size;
  85         sg = st->sgl;
  86         st->nents = 0;
  87         sg_page_sizes = 0;
  88
  89         /*
  90          * Our goal here is simple, we want to greedily fill the object from
  91          * largest to smallest page-size, while ensuring that we use *every*
  92          * page-size as per the given page-mask.
  93          */
  94         do {
  95                 unsigned int bit = ilog2(page_mask);
  96                 unsigned int page_size = BIT(bit);
  97                 int order = get_order(page_size);
  98
  99                 do {
 100                         struct page *page;
 101
 102                         GEM_BUG_ON(order >= MAX_ORDER);
 103                         page = alloc_pages(GFP | __GFP_ZERO, order);
 104                         if (!page)
 105                                 goto err;
 106
 107                         sg_set_page(sg, page, page_size, 0);
 108                         sg_page_sizes |= page_size;
 109                         st->nents++;
 110
 111                         rem -= page_size;
 112                         if (!rem) {
 113                                 sg_mark_end(sg);
 114                                 break;
 115                         }
 116
 117                         sg = __sg_next(sg);
 118                 } while ((rem - ((page_size-1) & page_mask)) >= page_size);
 119
 120                 page_mask &= (page_size-1);
 121         } while (page_mask);
 122
 123         if (i915_gem_gtt_prepare_pages(obj, st))
 124                 goto err;
 125
 126         obj->mm.madv = I915_MADV_DONTNEED;
 127
 128         GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask);
 129         __i915_gem_object_set_pages(obj, st, sg_page_sizes);
 130
 131         return 0;
 132
 133 err:
 134         sg_set_page(sg, NULL, 0, 0);
 135         sg_mark_end(sg);
 136         huge_pages_free_pages(st);
 137
 138         return -ENOMEM;
 139 }
 140
 141 static void put_huge_pages(struct drm_i915_gem_object *obj,
 142                            struct sg_table *pages)
 143 {
 144         i915_gem_gtt_finish_pages(obj, pages);
 145         huge_pages_free_pages(pages);
 146
 147         obj->mm.dirty = false;
 148         obj->mm.madv = I915_MADV_WILLNEED;
 149 }
 150
 151 static const struct drm_i915_gem_object_ops huge_page_ops = {
 152         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
 153                  I915_GEM_OBJECT_IS_SHRINKABLE,
 154         .get_pages = get_huge_pages,
 155         .put_pages = put_huge_pages,
 156 };
 157
 158 static struct drm_i915_gem_object *
 159 huge_pages_object(struct drm_i915_private *i915,
 160                   u64 size,
 161                   unsigned int page_mask)
 162 {
 163         struct drm_i915_gem_object *obj;
 164
 165         GEM_BUG_ON(!size);
 166         GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask))));
 167
 168         if (size >> PAGE_SHIFT > INT_MAX)
 169                 return ERR_PTR(-E2BIG);
 170
 171         if (overflows_type(size, obj->base.size))
 172                 return ERR_PTR(-E2BIG);
 173
 174         obj = i915_gem_object_alloc(i915);
 175         if (!obj)
 176                 return ERR_PTR(-ENOMEM);
 177
 178         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 179         i915_gem_object_init(obj, &huge_page_ops);
 180
 181         obj->write_domain = I915_GEM_DOMAIN_CPU;
 182         obj->read_domains = I915_GEM_DOMAIN_CPU;
 183         obj->cache_level = I915_CACHE_NONE;
 184
 185         obj->mm.page_mask = page_mask;
 186
 187         return obj;
 188 }
 189
 190 static int fake_get_huge_pages(struct drm_i915_gem_object *obj)
 191 {
 192         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 193         const u64 max_len = rounddown_pow_of_two(UINT_MAX);
 194         struct sg_table *st;
 195         struct scatterlist *sg;
 196         unsigned int sg_page_sizes;
 197         u64 rem;
 198
 199         st = kmalloc(sizeof(*st), GFP);
 200         if (!st)
 201                 return -ENOMEM;
 202
 203         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
 204                 kfree(st);
 205                 return -ENOMEM;
 206         }
 207
 208         /* Use optimal page sized chunks to fill in the sg table */
 209         rem = obj->base.size;
 210         sg = st->sgl;
 211         st->nents = 0;
 212         sg_page_sizes = 0;
 213         do {
 214                 unsigned int page_size = get_largest_page_size(i915, rem);
 215                 unsigned int len = min(page_size * div_u64(rem, page_size),
 216                                        max_len);
 217
 218                 GEM_BUG_ON(!page_size);
 219
 220                 sg->offset = 0;
 221                 sg->length = len;
 222                 sg_dma_len(sg) = len;
 223                 sg_dma_address(sg) = page_size;
 224
 225                 sg_page_sizes |= len;
 226
 227                 st->nents++;
 228
 229                 rem -= len;
 230                 if (!rem) {
 231                         sg_mark_end(sg);
 232                         break;
 233                 }
 234
 235                 sg = sg_next(sg);
 236         } while (1);
 237
 238         obj->mm.madv = I915_MADV_DONTNEED;
 239
 240         __i915_gem_object_set_pages(obj, st, sg_page_sizes);
 241
 242         return 0;
 243 }
 244
 245 static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj)
 246 {
 247         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 248         struct sg_table *st;
 249         struct scatterlist *sg;
 250         unsigned int page_size;
 251
 252         st = kmalloc(sizeof(*st), GFP);
 253         if (!st)
 254                 return -ENOMEM;
 255
 256         if (sg_alloc_table(st, 1, GFP)) {
 257                 kfree(st);
 258                 return -ENOMEM;
 259         }
 260
 261         sg = st->sgl;
 262         st->nents = 1;
 263
 264         page_size = get_largest_page_size(i915, obj->base.size);
 265         GEM_BUG_ON(!page_size);
 266
 267         sg->offset = 0;
 268         sg->length = obj->base.size;
 269         sg_dma_len(sg) = obj->base.size;
 270         sg_dma_address(sg) = page_size;
 271
 272         obj->mm.madv = I915_MADV_DONTNEED;
 273
 274         __i915_gem_object_set_pages(obj, st, sg->length);
 275
 276         return 0;
 277 #undef GFP
 278 }
 279
 280 static void fake_free_huge_pages(struct drm_i915_gem_object *obj,
 281                                  struct sg_table *pages)
 282 {
 283         sg_free_table(pages);
 284         kfree(pages);
 285 }
 286
 287 static void fake_put_huge_pages(struct drm_i915_gem_object *obj,
 288                                 struct sg_table *pages)
 289 {
 290         fake_free_huge_pages(obj, pages);
 291         obj->mm.dirty = false;
 292         obj->mm.madv = I915_MADV_WILLNEED;
 293 }
 294
 295 static const struct drm_i915_gem_object_ops fake_ops = {
 296         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 297         .get_pages = fake_get_huge_pages,
 298         .put_pages = fake_put_huge_pages,
 299 };
 300
 301 static const struct drm_i915_gem_object_ops fake_ops_single = {
 302         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 303         .get_pages = fake_get_huge_pages_single,
 304         .put_pages = fake_put_huge_pages,
 305 };
 306
 307 static struct drm_i915_gem_object *
 308 fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single)
 309 {
 310         struct drm_i915_gem_object *obj;
 311
 312         GEM_BUG_ON(!size);
 313         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 314
 315         if (size >> PAGE_SHIFT > UINT_MAX)
 316                 return ERR_PTR(-E2BIG);
 317
 318         if (overflows_type(size, obj->base.size))
 319                 return ERR_PTR(-E2BIG);
 320
 321         obj = i915_gem_object_alloc(i915);
 322         if (!obj)
 323                 return ERR_PTR(-ENOMEM);
 324
 325         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 326
 327         if (single)
 328                 i915_gem_object_init(obj, &fake_ops_single);
 329         else
 330                 i915_gem_object_init(obj, &fake_ops);
 331
 332         obj->write_domain = I915_GEM_DOMAIN_CPU;
 333         obj->read_domains = I915_GEM_DOMAIN_CPU;
 334         obj->cache_level = I915_CACHE_NONE;
 335
 336         return obj;
 337 }
 338
 339 static int igt_check_page_sizes(struct i915_vma *vma)
 340 {
 341         struct drm_i915_private *i915 = vma->vm->i915;
 342         unsigned int supported = INTEL_INFO(i915)->page_sizes;
 343         struct drm_i915_gem_object *obj = vma->obj;
 344         int err = 0;
 345
 346         if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) {
 347                 pr_err("unsupported page_sizes.sg=%u, supported=%u\n",
 348                        vma->page_sizes.sg & ~supported, supported);
 349                 err = -EINVAL;
 350         }
 351
 352         if (!HAS_PAGE_SIZES(i915, vma->page_sizes.gtt)) {
 353                 pr_err("unsupported page_sizes.gtt=%u, supported=%u\n",
 354                        vma->page_sizes.gtt & ~supported, supported);
 355                 err = -EINVAL;
 356         }
 357
 358         if (vma->page_sizes.phys != obj->mm.page_sizes.phys) {
 359                 pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n",
 360                        vma->page_sizes.phys, obj->mm.page_sizes.phys);
 361                 err = -EINVAL;
 362         }
 363
 364         if (vma->page_sizes.sg != obj->mm.page_sizes.sg) {
 365                 pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n",
 366                        vma->page_sizes.sg, obj->mm.page_sizes.sg);
 367                 err = -EINVAL;
 368         }
 369
 370         if (obj->mm.page_sizes.gtt) {
 371                 pr_err("obj->page_sizes.gtt(%u) should never be set\n",
 372                        obj->mm.page_sizes.gtt);
 373                 err = -EINVAL;
 374         }
 375
 376         return err;
 377 }
 378
 379 static int igt_mock_exhaust_device_supported_pages(void *arg)
 380 {
 381         struct i915_hw_ppgtt *ppgtt = arg;
 382         struct drm_i915_private *i915 = ppgtt->vm.i915;
 383         unsigned int saved_mask = INTEL_INFO(i915)->page_sizes;
 384         struct drm_i915_gem_object *obj;
 385         struct i915_vma *vma;
 386         int i, j, single;
 387         int err;
 388
 389         /*
 390          * Sanity check creating objects with every valid page support
 391          * combination for our mock device.
 392          */
 393
 394         for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) {
 395                 unsigned int combination = 0;
 396
 397                 for (j = 0; j < ARRAY_SIZE(page_sizes); j++) {
 398                         if (i & BIT(j))
 399                                 combination |= page_sizes[j];
 400                 }
 401
 402                 mkwrite_device_info(i915)->page_sizes = combination;
 403
 404                 for (single = 0; single <= 1; ++single) {
 405                         obj = fake_huge_pages_object(i915, combination, !!single);
 406                         if (IS_ERR(obj)) {
 407                                 err = PTR_ERR(obj);
 408                                 goto out_device;
 409                         }
 410
 411                         if (obj->base.size != combination) {
 412                                 pr_err("obj->base.size=%zu, expected=%u\n",
 413                                        obj->base.size, combination);
 414                                 err = -EINVAL;
 415                                 goto out_put;
 416                         }
 417
 418                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 419                         if (IS_ERR(vma)) {
 420                                 err = PTR_ERR(vma);
 421                                 goto out_put;
 422                         }
 423
 424                         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 425                         if (err)
 426                                 goto out_close;
 427
 428                         err = igt_check_page_sizes(vma);
 429
 430                         if (vma->page_sizes.sg != combination) {
 431                                 pr_err("page_sizes.sg=%u, expected=%u\n",
 432                                        vma->page_sizes.sg, combination);
 433                                 err = -EINVAL;
 434                         }
 435
 436                         i915_vma_unpin(vma);
 437                         i915_vma_close(vma);
 438
 439                         i915_gem_object_put(obj);
 440
 441                         if (err)
 442                                 goto out_device;
 443                 }
 444         }
 445
 446         goto out_device;
 447
 448 out_close:
 449         i915_vma_close(vma);
 450 out_put:
 451         i915_gem_object_put(obj);
 452 out_device:
 453         mkwrite_device_info(i915)->page_sizes = saved_mask;
 454
 455         return err;
 456 }
 457
 458 static int igt_mock_ppgtt_misaligned_dma(void *arg)
 459 {
 460         struct i915_hw_ppgtt *ppgtt = arg;
 461         struct drm_i915_private *i915 = ppgtt->vm.i915;
 462         unsigned long supported = INTEL_INFO(i915)->page_sizes;
 463         struct drm_i915_gem_object *obj;
 464         int bit;
 465         int err;
 466
 467         /*
 468          * Sanity check dma misalignment for huge pages -- the dma addresses we
 469          * insert into the paging structures need to always respect the page
 470          * size alignment.
 471          */
 472
 473         bit = ilog2(I915_GTT_PAGE_SIZE_64K);
 474
 475         for_each_set_bit_from(bit, &supported,
 476                               ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
 477                 IGT_TIMEOUT(end_time);
 478                 unsigned int page_size = BIT(bit);
 479                 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
 480                 unsigned int offset;
 481                 unsigned int size =
 482                         round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1;
 483                 struct i915_vma *vma;
 484
 485                 obj = fake_huge_pages_object(i915, size, true);
 486                 if (IS_ERR(obj))
 487                         return PTR_ERR(obj);
 488
 489                 if (obj->base.size != size) {
 490                         pr_err("obj->base.size=%zu, expected=%u\n",
 491                                obj->base.size, size);
 492                         err = -EINVAL;
 493                         goto out_put;
 494                 }
 495
 496                 err = i915_gem_object_pin_pages(obj);
 497                 if (err)
 498                         goto out_put;
 499
 500                 /* Force the page size for this object */
 501                 obj->mm.page_sizes.sg = page_size;
 502
 503                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 504                 if (IS_ERR(vma)) {
 505                         err = PTR_ERR(vma);
 506                         goto out_unpin;
 507                 }
 508
 509                 err = i915_vma_pin(vma, 0, 0, flags);
 510                 if (err) {
 511                         i915_vma_close(vma);
 512                         goto out_unpin;
 513                 }
 514
 515
 516                 err = igt_check_page_sizes(vma);
 517
 518                 if (vma->page_sizes.gtt != page_size) {
 519                         pr_err("page_sizes.gtt=%u, expected %u\n",
 520                                vma->page_sizes.gtt, page_size);
 521                         err = -EINVAL;
 522                 }
 523
 524                 i915_vma_unpin(vma);
 525
 526                 if (err) {
 527                         i915_vma_close(vma);
 528                         goto out_unpin;
 529                 }
 530
 531                 /*
 532                  * Try all the other valid offsets until the next
 533                  * boundary -- should always fall back to using 4K
 534                  * pages.
 535                  */
 536                 for (offset = 4096; offset < page_size; offset += 4096) {
 537                         err = i915_vma_unbind(vma);
 538                         if (err) {
 539                                 i915_vma_close(vma);
 540                                 goto out_unpin;
 541                         }
 542
 543                         err = i915_vma_pin(vma, 0, 0, flags | offset);
 544                         if (err) {
 545                                 i915_vma_close(vma);
 546                                 goto out_unpin;
 547                         }
 548
 549                         err = igt_check_page_sizes(vma);
 550
 551                         if (vma->page_sizes.gtt != I915_GTT_PAGE_SIZE_4K) {
 552                                 pr_err("page_sizes.gtt=%u, expected %lu\n",
 553                                        vma->page_sizes.gtt, I915_GTT_PAGE_SIZE_4K);
 554                                 err = -EINVAL;
 555                         }
 556
 557                         i915_vma_unpin(vma);
 558
 559                         if (err) {
 560                                 i915_vma_close(vma);
 561                                 goto out_unpin;
 562                         }
 563
 564                         if (igt_timeout(end_time,
 565                                         "%s timed out at offset %x with page-size %x\n",
 566                                         __func__, offset, page_size))
 567                                 break;
 568                 }
 569
 570                 i915_vma_close(vma);
 571
 572                 i915_gem_object_unpin_pages(obj);
 573                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 574                 i915_gem_object_put(obj);
 575         }
 576
 577         return 0;
 578
 579 out_unpin:
 580         i915_gem_object_unpin_pages(obj);
 581 out_put:
 582         i915_gem_object_put(obj);
 583
 584         return err;
 585 }
 586
 587 static void close_object_list(struct list_head *objects,
 588                               struct i915_hw_ppgtt *ppgtt)
 589 {
 590         struct drm_i915_gem_object *obj, *on;
 591
 592         list_for_each_entry_safe(obj, on, objects, st_link) {
 593                 struct i915_vma *vma;
 594
 595                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 596                 if (!IS_ERR(vma))
 597                         i915_vma_close(vma);
 598
 599                 list_del(&obj->st_link);
 600                 i915_gem_object_unpin_pages(obj);
 601                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 602                 i915_gem_object_put(obj);
 603         }
 604 }
 605
 606 static int igt_mock_ppgtt_huge_fill(void *arg)
 607 {
 608         struct i915_hw_ppgtt *ppgtt = arg;
 609         struct drm_i915_private *i915 = ppgtt->vm.i915;
 610         unsigned long max_pages = ppgtt->vm.total >> PAGE_SHIFT;
 611         unsigned long page_num;
 612         bool single = false;
 613         LIST_HEAD(objects);
 614         IGT_TIMEOUT(end_time);
 615         int err = -ENODEV;
 616
 617         for_each_prime_number_from(page_num, 1, max_pages) {
 618                 struct drm_i915_gem_object *obj;
 619                 u64 size = page_num << PAGE_SHIFT;
 620                 struct i915_vma *vma;
 621                 unsigned int expected_gtt = 0;
 622                 int i;
 623
 624                 obj = fake_huge_pages_object(i915, size, single);
 625                 if (IS_ERR(obj)) {
 626                         err = PTR_ERR(obj);
 627                         break;
 628                 }
 629
 630                 if (obj->base.size != size) {
 631                         pr_err("obj->base.size=%zd, expected=%llu\n",
 632                                obj->base.size, size);
 633                         i915_gem_object_put(obj);
 634                         err = -EINVAL;
 635                         break;
 636                 }
 637
 638                 err = i915_gem_object_pin_pages(obj);
 639                 if (err) {
 640                         i915_gem_object_put(obj);
 641                         break;
 642                 }
 643
 644                 list_add(&obj->st_link, &objects);
 645
 646                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 647                 if (IS_ERR(vma)) {
 648                         err = PTR_ERR(vma);
 649                         break;
 650                 }
 651
 652                 err = i915_vma_pin(vma, 0, 0, PIN_USER);
 653                 if (err)
 654                         break;
 655
 656                 err = igt_check_page_sizes(vma);
 657                 if (err) {
 658                         i915_vma_unpin(vma);
 659                         break;
 660                 }
 661
 662                 /*
 663                  * Figure out the expected gtt page size knowing that we go from
 664                  * largest to smallest page size sg chunks, and that we align to
 665                  * the largest page size.
 666                  */
 667                 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
 668                         unsigned int page_size = page_sizes[i];
 669
 670                         if (HAS_PAGE_SIZES(i915, page_size) &&
 671                             size >= page_size) {
 672                                 expected_gtt |= page_size;
 673                                 size &= page_size-1;
 674                         }
 675                 }
 676
 677                 GEM_BUG_ON(!expected_gtt);
 678                 GEM_BUG_ON(size);
 679
 680                 if (expected_gtt & I915_GTT_PAGE_SIZE_4K)
 681                         expected_gtt &= ~I915_GTT_PAGE_SIZE_64K;
 682
 683                 i915_vma_unpin(vma);
 684
 685                 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 686                         if (!IS_ALIGNED(vma->node.start,
 687                                         I915_GTT_PAGE_SIZE_2M)) {
 688                                 pr_err("node.start(%llx) not aligned to 2M\n",
 689                                        vma->node.start);
 690                                 err = -EINVAL;
 691                                 break;
 692                         }
 693
 694                         if (!IS_ALIGNED(vma->node.size,
 695                                         I915_GTT_PAGE_SIZE_2M)) {
 696                                 pr_err("node.size(%llx) not aligned to 2M\n",
 697                                        vma->node.size);
 698                                 err = -EINVAL;
 699                                 break;
 700                         }
 701                 }
 702
 703                 if (vma->page_sizes.gtt != expected_gtt) {
 704                         pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n",
 705                                vma->page_sizes.gtt, expected_gtt,
 706                                obj->base.size, yesno(!!single));
 707                         err = -EINVAL;
 708                         break;
 709                 }
 710
 711                 if (igt_timeout(end_time,
 712                                 "%s timed out at size %zd\n",
 713                                 __func__, obj->base.size))
 714                         break;
 715
 716                 single = !single;
 717         }
 718
 719         close_object_list(&objects, ppgtt);
 720
 721         if (err == -ENOMEM || err == -ENOSPC)
 722                 err = 0;
 723
 724         return err;
 725 }
 726
 727 static int igt_mock_ppgtt_64K(void *arg)
 728 {
 729         struct i915_hw_ppgtt *ppgtt = arg;
 730         struct drm_i915_private *i915 = ppgtt->vm.i915;
 731         struct drm_i915_gem_object *obj;
 732         const struct object_info {
 733                 unsigned int size;
 734                 unsigned int gtt;
 735                 unsigned int offset;
 736         } objects[] = {
 737                 /* Cases with forced padding/alignment */
 738                 {
 739                         .size = SZ_64K,
 740                         .gtt = I915_GTT_PAGE_SIZE_64K,
 741                         .offset = 0,
 742                 },
 743                 {
 744                         .size = SZ_64K + SZ_4K,
 745                         .gtt = I915_GTT_PAGE_SIZE_4K,
 746                         .offset = 0,
 747                 },
 748                 {
 749                         .size = SZ_64K - SZ_4K,
 750                         .gtt = I915_GTT_PAGE_SIZE_4K,
 751                         .offset = 0,
 752                 },
 753                 {
 754                         .size = SZ_2M,
 755                         .gtt = I915_GTT_PAGE_SIZE_64K,
 756                         .offset = 0,
 757                 },
 758                 {
 759                         .size = SZ_2M - SZ_4K,
 760                         .gtt = I915_GTT_PAGE_SIZE_4K,
 761                         .offset = 0,
 762                 },
 763                 {
 764                         .size = SZ_2M + SZ_4K,
 765                         .gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K,
 766                         .offset = 0,
 767                 },
 768                 {
 769                         .size = SZ_2M + SZ_64K,
 770                         .gtt = I915_GTT_PAGE_SIZE_64K,
 771                         .offset = 0,
 772                 },
 773                 {
 774                         .size = SZ_2M - SZ_64K,
 775                         .gtt = I915_GTT_PAGE_SIZE_64K,
 776                         .offset = 0,
 777                 },
 778                 /* Try without any forced padding/alignment */
 779                 {
 780                         .size = SZ_64K,
 781                         .offset = SZ_2M,
 782                         .gtt = I915_GTT_PAGE_SIZE_4K,
 783                 },
 784                 {
 785                         .size = SZ_128K,
 786                         .offset = SZ_2M - SZ_64K,
 787                         .gtt = I915_GTT_PAGE_SIZE_4K,
 788                 },
 789         };
 790         struct i915_vma *vma;
 791         int i, single;
 792         int err;
 793
 794         /*
 795          * Sanity check some of the trickiness with 64K pages -- either we can
 796          * safely mark the whole page-table(2M block) as 64K, or we have to
 797          * always fallback to 4K.
 798          */
 799
 800         if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K))
 801                 return 0;
 802
 803         for (i = 0; i < ARRAY_SIZE(objects); ++i) {
 804                 unsigned int size = objects[i].size;
 805                 unsigned int expected_gtt = objects[i].gtt;
 806                 unsigned int offset = objects[i].offset;
 807                 unsigned int flags = PIN_USER;
 808
 809                 for (single = 0; single <= 1; single++) {
 810                         obj = fake_huge_pages_object(i915, size, !!single);
 811                         if (IS_ERR(obj))
 812                                 return PTR_ERR(obj);
 813
 814                         err = i915_gem_object_pin_pages(obj);
 815                         if (err)
 816                                 goto out_object_put;
 817
 818                         /*
 819                          * Disable 2M pages -- We only want to use 64K/4K pages
 820                          * for this test.
 821                          */
 822                         obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M;
 823
 824                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 825                         if (IS_ERR(vma)) {
 826                                 err = PTR_ERR(vma);
 827                                 goto out_object_unpin;
 828                         }
 829
 830                         if (offset)
 831                                 flags |= PIN_OFFSET_FIXED | offset;
 832
 833                         err = i915_vma_pin(vma, 0, 0, flags);
 834                         if (err)
 835                                 goto out_vma_close;
 836
 837                         err = igt_check_page_sizes(vma);
 838                         if (err)
 839                                 goto out_vma_unpin;
 840
 841                         if (!offset && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 842                                 if (!IS_ALIGNED(vma->node.start,
 843                                                 I915_GTT_PAGE_SIZE_2M)) {
 844                                         pr_err("node.start(%llx) not aligned to 2M\n",
 845                                                vma->node.start);
 846                                         err = -EINVAL;
 847                                         goto out_vma_unpin;
 848                                 }
 849
 850                                 if (!IS_ALIGNED(vma->node.size,
 851                                                 I915_GTT_PAGE_SIZE_2M)) {
 852                                         pr_err("node.size(%llx) not aligned to 2M\n",
 853                                                vma->node.size);
 854                                         err = -EINVAL;
 855                                         goto out_vma_unpin;
 856                                 }
 857                         }
 858
 859                         if (vma->page_sizes.gtt != expected_gtt) {
 860                                 pr_err("gtt=%u, expected=%u, i=%d, single=%s\n",
 861                                        vma->page_sizes.gtt, expected_gtt, i,
 862                                        yesno(!!single));
 863                                 err = -EINVAL;
 864                                 goto out_vma_unpin;
 865                         }
 866
 867                         i915_vma_unpin(vma);
 868                         i915_vma_close(vma);
 869
 870                         i915_gem_object_unpin_pages(obj);
 871                         __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 872                         i915_gem_object_put(obj);
 873                 }
 874         }
 875
 876         return 0;
 877
 878 out_vma_unpin:
 879         i915_vma_unpin(vma);
 880 out_vma_close:
 881         i915_vma_close(vma);
 882 out_object_unpin:
 883         i915_gem_object_unpin_pages(obj);
 884 out_object_put:
 885         i915_gem_object_put(obj);
 886
 887         return err;
 888 }
 889
 890 static struct i915_vma *
 891 gpu_write_dw(struct i915_vma *vma, u64 offset, u32 val)
 892 {
 893         struct drm_i915_private *i915 = vma->vm->i915;
 894         const int gen = INTEL_GEN(i915);
 895         unsigned int count = vma->size >> PAGE_SHIFT;
 896         struct drm_i915_gem_object *obj;
 897         struct i915_vma *batch;
 898         unsigned int size;
 899         u32 *cmd;
 900         int n;
 901         int err;
 902
 903         size = (1 + 4 * count) * sizeof(u32);
 904         size = round_up(size, PAGE_SIZE);
 905         obj = i915_gem_object_create_internal(i915, size);
 906         if (IS_ERR(obj))
 907                 return ERR_CAST(obj);
 908
 909         err = i915_gem_object_set_to_wc_domain(obj, true);
 910         if (err)
 911                 goto err;
 912
 913         cmd = i915_gem_object_pin_map(obj, I915_MAP_WC);
 914         if (IS_ERR(cmd)) {
 915                 err = PTR_ERR(cmd);
 916                 goto err;
 917         }
 918
 919         offset += vma->node.start;
 920
 921         for (n = 0; n < count; n++) {
 922                 if (gen >= 8) {
 923                         *cmd++ = MI_STORE_DWORD_IMM_GEN4;
 924                         *cmd++ = lower_32_bits(offset);
 925                         *cmd++ = upper_32_bits(offset);
 926                         *cmd++ = val;
 927                 } else if (gen >= 4) {
 928                         *cmd++ = MI_STORE_DWORD_IMM_GEN4 |
 929                                 (gen < 6 ? MI_USE_GGTT : 0);
 930                         *cmd++ = 0;
 931                         *cmd++ = offset;
 932                         *cmd++ = val;
 933                 } else {
 934                         *cmd++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
 935                         *cmd++ = offset;
 936                         *cmd++ = val;
 937                 }
 938
 939                 offset += PAGE_SIZE;
 940         }
 941
 942         *cmd = MI_BATCH_BUFFER_END;
 943         i915_gem_chipset_flush(i915);
 944
 945         i915_gem_object_unpin_map(obj);
 946
 947         batch = i915_vma_instance(obj, vma->vm, NULL);
 948         if (IS_ERR(batch)) {
 949                 err = PTR_ERR(batch);
 950                 goto err;
 951         }
 952
 953         err = i915_vma_pin(batch, 0, 0, PIN_USER);
 954         if (err)
 955                 goto err;
 956
 957         return batch;
 958
 959 err:
 960         i915_gem_object_put(obj);
 961
 962         return ERR_PTR(err);
 963 }
 964
 965 static int gpu_write(struct i915_vma *vma,
 966                      struct i915_gem_context *ctx,
 967                      struct intel_engine_cs *engine,
 968                      u32 dword,
 969                      u32 value)
 970 {
 971         struct i915_request *rq;
 972         struct i915_vma *batch;
 973         int flags = 0;
 974         int err;
 975
 976         GEM_BUG_ON(!intel_engine_can_store_dword(engine));
 977
 978         err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
 979         if (err)
 980                 return err;
 981
 982         rq = i915_request_alloc(engine, ctx);
 983         if (IS_ERR(rq))
 984                 return PTR_ERR(rq);
 985
 986         batch = gpu_write_dw(vma, dword * sizeof(u32), value);
 987         if (IS_ERR(batch)) {
 988                 err = PTR_ERR(batch);
 989                 goto err_request;
 990         }
 991
 992         err = i915_vma_move_to_active(batch, rq, 0);
 993         if (err)
 994                 goto err_request;
 995
 996         i915_gem_object_set_active_reference(batch->obj);
 997         i915_vma_unpin(batch);
 998         i915_vma_close(batch);
 999
1000         err = engine->emit_bb_start(rq,
1001                                     batch->node.start, batch->node.size,
1002                                     flags);
1003         if (err)
1004                 goto err_request;
1005
1006         err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1007         if (err)
1008                 i915_request_skip(rq, err);
1009
1010 err_request:
1011         i915_request_add(rq);
1012
1013         return err;
1014 }
1015
1016 static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1017 {
1018         unsigned int needs_flush;
1019         unsigned long n;
1020         int err;
1021
1022         err = i915_gem_obj_prepare_shmem_read(obj, &needs_flush);
1023         if (err)
1024                 return err;
1025
1026         for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) {
1027                 u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n));
1028
1029                 if (needs_flush & CLFLUSH_BEFORE)
1030                         drm_clflush_virt_range(ptr, PAGE_SIZE);
1031
1032                 if (ptr[dword] != val) {
1033                         pr_err("n=%lu ptr[%u]=%u, val=%u\n",
1034                                n, dword, ptr[dword], val);
1035                         kunmap_atomic(ptr);
1036                         err = -EINVAL;
1037                         break;
1038                 }
1039
1040                 kunmap_atomic(ptr);
1041         }
1042
1043         i915_gem_obj_finish_shmem_access(obj);
1044
1045         return err;
1046 }
1047
1048 static int __igt_write_huge(struct i915_gem_context *ctx,
1049                             struct intel_engine_cs *engine,
1050                             struct drm_i915_gem_object *obj,
1051                             u64 size, u64 offset,
1052                             u32 dword, u32 val)
1053 {
1054         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1055         struct i915_address_space *vm =
1056                 ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm;
1057         unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1058         struct i915_vma *vma;
1059         int err;
1060
1061         vma = i915_vma_instance(obj, vm, NULL);
1062         if (IS_ERR(vma))
1063                 return PTR_ERR(vma);
1064
1065         err = i915_vma_unbind(vma);
1066         if (err)
1067                 goto out_vma_close;
1068
1069         err = i915_vma_pin(vma, size, 0, flags | offset);
1070         if (err) {
1071                 /*
1072                  * The ggtt may have some pages reserved so
1073                  * refrain from erroring out.
1074                  */
1075                 if (err == -ENOSPC && i915_is_ggtt(vm))
1076                         err = 0;
1077
1078                 goto out_vma_close;
1079         }
1080
1081         err = igt_check_page_sizes(vma);
1082         if (err)
1083                 goto out_vma_unpin;
1084
1085         err = gpu_write(vma, ctx, engine, dword, val);
1086         if (err) {
1087                 pr_err("gpu-write failed at offset=%llx\n", offset);
1088                 goto out_vma_unpin;
1089         }
1090
1091         err = cpu_check(obj, dword, val);
1092         if (err) {
1093                 pr_err("cpu-check failed at offset=%llx\n", offset);
1094                 goto out_vma_unpin;
1095         }
1096
1097 out_vma_unpin:
1098         i915_vma_unpin(vma);
1099 out_vma_close:
1100         i915_vma_destroy(vma);
1101
1102         return err;
1103 }
1104
1105 static int igt_write_huge(struct i915_gem_context *ctx,
1106                           struct drm_i915_gem_object *obj)
1107 {
1108         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1109         struct i915_address_space *vm =
1110                 ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm;
1111         static struct intel_engine_cs *engines[I915_NUM_ENGINES];
1112         struct intel_engine_cs *engine;
1113         I915_RND_STATE(prng);
1114         IGT_TIMEOUT(end_time);
1115         unsigned int max_page_size;
1116         unsigned int id;
1117         u64 max;
1118         u64 num;
1119         u64 size;
1120         int *order;
1121         int i, n;
1122         int err = 0;
1123
1124         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
1125
1126         size = obj->base.size;
1127         if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
1128                 size = round_up(size, I915_GTT_PAGE_SIZE_2M);
1129
1130         max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg);
1131         max = div_u64((vm->total - size), max_page_size);
1132
1133         n = 0;
1134         for_each_engine(engine, i915, id) {
1135                 if (!intel_engine_can_store_dword(engine)) {
1136                         pr_info("store-dword-imm not supported on engine=%u\n", id);
1137                         continue;
1138                 }
1139                 engines[n++] = engine;
1140         }
1141
1142         if (!n)
1143                 return 0;
1144
1145         /*
1146          * To keep things interesting when alternating between engines in our
1147          * randomized order, lets also make feeding to the same engine a few
1148          * times in succession a possibility by enlarging the permutation array.
1149          */
1150         order = i915_random_order(n * I915_NUM_ENGINES, &prng);
1151         if (!order)
1152                 return -ENOMEM;
1153
1154         /*
1155          * Try various offsets in an ascending/descending fashion until we
1156          * timeout -- we want to avoid issues hidden by effectively always using
1157          * offset = 0.
1158          */
1159         i = 0;
1160         for_each_prime_number_from(num, 0, max) {
1161                 u64 offset_low = num * max_page_size;
1162                 u64 offset_high = (max - num) * max_page_size;
1163                 u32 dword = offset_in_page(num) / 4;
1164
1165                 engine = engines[order[i] % n];
1166                 i = (i + 1) % (n * I915_NUM_ENGINES);
1167
1168                 err = __igt_write_huge(ctx, engine, obj, size, offset_low, dword, num + 1);
1169                 if (err)
1170                         break;
1171
1172                 err = __igt_write_huge(ctx, engine, obj, size, offset_high, dword, num + 1);
1173                 if (err)
1174                         break;
1175
1176                 if (igt_timeout(end_time,
1177                                 "%s timed out on engine=%u, offset_low=%llx offset_high=%llx, max_page_size=%x\n",
1178                                 __func__, engine->id, offset_low, offset_high, max_page_size))
1179                         break;
1180         }
1181
1182         kfree(order);
1183
1184         return err;
1185 }
1186
1187 static int igt_ppgtt_exhaust_huge(void *arg)
1188 {
1189         struct i915_gem_context *ctx = arg;
1190         struct drm_i915_private *i915 = ctx->i915;
1191         unsigned long supported = INTEL_INFO(i915)->page_sizes;
1192         static unsigned int pages[ARRAY_SIZE(page_sizes)];
1193         struct drm_i915_gem_object *obj;
1194         unsigned int size_mask;
1195         unsigned int page_mask;
1196         int n, i;
1197         int err = -ENODEV;
1198
1199         if (supported == I915_GTT_PAGE_SIZE_4K)
1200                 return 0;
1201
1202         /*
1203          * Sanity check creating objects with a varying mix of page sizes --
1204          * ensuring that our writes lands in the right place.
1205          */
1206
1207         n = 0;
1208         for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1)
1209                 pages[n++] = BIT(i);
1210
1211         for (size_mask = 2; size_mask < BIT(n); size_mask++) {
1212                 unsigned int size = 0;
1213
1214                 for (i = 0; i < n; i++) {
1215                         if (size_mask & BIT(i))
1216                                 size |= pages[i];
1217                 }
1218
1219                 /*
1220                  * For our page mask we want to enumerate all the page-size
1221                  * combinations which will fit into our chosen object size.
1222                  */
1223                 for (page_mask = 2; page_mask <= size_mask; page_mask++) {
1224                         unsigned int page_sizes = 0;
1225
1226                         for (i = 0; i < n; i++) {
1227                                 if (page_mask & BIT(i))
1228                                         page_sizes |= pages[i];
1229                         }
1230
1231                         /*
1232                          * Ensure that we can actually fill the given object
1233                          * with our chosen page mask.
1234                          */
1235                         if (!IS_ALIGNED(size, BIT(__ffs(page_sizes))))
1236                                 continue;
1237
1238                         obj = huge_pages_object(i915, size, page_sizes);
1239                         if (IS_ERR(obj)) {
1240                                 err = PTR_ERR(obj);
1241                                 goto out_device;
1242                         }
1243
1244                         err = i915_gem_object_pin_pages(obj);
1245                         if (err) {
1246                                 i915_gem_object_put(obj);
1247
1248                                 if (err == -ENOMEM) {
1249                                         pr_info("unable to get pages, size=%u, pages=%u\n",
1250                                                 size, page_sizes);
1251                                         err = 0;
1252                                         break;
1253                                 }
1254
1255                                 pr_err("pin_pages failed, size=%u, pages=%u\n",
1256                                        size_mask, page_mask);
1257
1258                                 goto out_device;
1259                         }
1260
1261                         /* Force the page-size for the gtt insertion */
1262                         obj->mm.page_sizes.sg = page_sizes;
1263
1264                         err = igt_write_huge(ctx, obj);
1265                         if (err) {
1266                                 pr_err("exhaust write-huge failed with size=%u\n",
1267                                        size);
1268                                 goto out_unpin;
1269                         }
1270
1271                         i915_gem_object_unpin_pages(obj);
1272                         __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
1273                         i915_gem_object_put(obj);
1274                 }
1275         }
1276
1277         goto out_device;
1278
1279 out_unpin:
1280         i915_gem_object_unpin_pages(obj);
1281         i915_gem_object_put(obj);
1282 out_device:
1283         mkwrite_device_info(i915)->page_sizes = supported;
1284
1285         return err;
1286 }
1287
1288 static int igt_ppgtt_internal_huge(void *arg)
1289 {
1290         struct i915_gem_context *ctx = arg;
1291         struct drm_i915_private *i915 = ctx->i915;
1292         struct drm_i915_gem_object *obj;
1293         static const unsigned int sizes[] = {
1294                 SZ_64K,
1295                 SZ_128K,
1296                 SZ_256K,
1297                 SZ_512K,
1298                 SZ_1M,
1299                 SZ_2M,
1300         };
1301         int i;
1302         int err;
1303
1304         /*
1305          * Sanity check that the HW uses huge pages correctly through internal
1306          * -- ensure that our writes land in the right place.
1307          */
1308
1309         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
1310                 unsigned int size = sizes[i];
1311
1312                 obj = i915_gem_object_create_internal(i915, size);
1313                 if (IS_ERR(obj))
1314                         return PTR_ERR(obj);
1315
1316                 err = i915_gem_object_pin_pages(obj);
1317                 if (err)
1318                         goto out_put;
1319
1320                 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_64K) {
1321                         pr_info("internal unable to allocate huge-page(s) with size=%u\n",
1322                                 size);
1323                         goto out_unpin;
1324                 }
1325
1326                 err = igt_write_huge(ctx, obj);
1327                 if (err) {
1328                         pr_err("internal write-huge failed with size=%u\n",
1329                                size);
1330                         goto out_unpin;
1331                 }
1332
1333                 i915_gem_object_unpin_pages(obj);
1334                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
1335                 i915_gem_object_put(obj);
1336         }
1337
1338         return 0;
1339
1340 out_unpin:
1341         i915_gem_object_unpin_pages(obj);
1342 out_put:
1343         i915_gem_object_put(obj);
1344
1345         return err;
1346 }
1347
1348 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915)
1349 {
1350         return i915->mm.gemfs && has_transparent_hugepage();
1351 }
1352
1353 static int igt_ppgtt_gemfs_huge(void *arg)
1354 {
1355         struct i915_gem_context *ctx = arg;
1356         struct drm_i915_private *i915 = ctx->i915;
1357         struct drm_i915_gem_object *obj;
1358         static const unsigned int sizes[] = {
1359                 SZ_2M,
1360                 SZ_4M,
1361                 SZ_8M,
1362                 SZ_16M,
1363                 SZ_32M,
1364         };
1365         int i;
1366         int err;
1367
1368         /*
1369          * Sanity check that the HW uses huge pages correctly through gemfs --
1370          * ensure that our writes land in the right place.
1371          */
1372
1373         if (!igt_can_allocate_thp(i915)) {
1374                 pr_info("missing THP support, skipping\n");
1375                 return 0;
1376         }
1377
1378         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
1379                 unsigned int size = sizes[i];
1380
1381                 obj = i915_gem_object_create(i915, size);
1382                 if (IS_ERR(obj))
1383                         return PTR_ERR(obj);
1384
1385                 err = i915_gem_object_pin_pages(obj);
1386                 if (err)
1387                         goto out_put;
1388
1389                 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) {
1390                         pr_info("finishing test early, gemfs unable to allocate huge-page(s) with size=%u\n",
1391                                 size);
1392                         goto out_unpin;
1393                 }
1394
1395                 err = igt_write_huge(ctx, obj);
1396                 if (err) {
1397                         pr_err("gemfs write-huge failed with size=%u\n",
1398                                size);
1399                         goto out_unpin;
1400                 }
1401
1402                 i915_gem_object_unpin_pages(obj);
1403                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
1404                 i915_gem_object_put(obj);
1405         }
1406
1407         return 0;
1408
1409 out_unpin:
1410         i915_gem_object_unpin_pages(obj);
1411 out_put:
1412         i915_gem_object_put(obj);
1413
1414         return err;
1415 }
1416
1417 static int igt_ppgtt_pin_update(void *arg)
1418 {
1419         struct i915_gem_context *ctx = arg;
1420         struct drm_i915_private *dev_priv = ctx->i915;
1421         unsigned long supported = INTEL_INFO(dev_priv)->page_sizes;
1422         struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
1423         struct drm_i915_gem_object *obj;
1424         struct i915_vma *vma;
1425         unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1426         int first, last;
1427         int err;
1428
1429         /*
1430          * Make sure there's no funny business when doing a PIN_UPDATE -- in the
1431          * past we had a subtle issue with being able to incorrectly do multiple
1432          * alloc va ranges on the same object when doing a PIN_UPDATE, which
1433          * resulted in some pretty nasty bugs, though only when using
1434          * huge-gtt-pages.
1435          */
1436
1437         if (!USES_FULL_48BIT_PPGTT(dev_priv)) {
1438                 pr_info("48b PPGTT not supported, skipping\n");
1439                 return 0;
1440         }
1441
1442         first = ilog2(I915_GTT_PAGE_SIZE_64K);
1443         last = ilog2(I915_GTT_PAGE_SIZE_2M);
1444
1445         for_each_set_bit_from(first, &supported, last + 1) {
1446                 unsigned int page_size = BIT(first);
1447
1448                 obj = i915_gem_object_create_internal(dev_priv, page_size);
1449                 if (IS_ERR(obj))
1450                         return PTR_ERR(obj);
1451
1452                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
1453                 if (IS_ERR(vma)) {
1454                         err = PTR_ERR(vma);
1455                         goto out_put;
1456                 }
1457
1458                 err = i915_vma_pin(vma, SZ_2M, 0, flags);
1459                 if (err)
1460                         goto out_close;
1461
1462                 if (vma->page_sizes.sg < page_size) {
1463                         pr_info("Unable to allocate page-size %x, finishing test early\n",
1464                                 page_size);
1465                         goto out_unpin;
1466                 }
1467
1468                 err = igt_check_page_sizes(vma);
1469                 if (err)
1470                         goto out_unpin;
1471
1472                 if (vma->page_sizes.gtt != page_size) {
1473                         dma_addr_t addr = i915_gem_object_get_dma_address(obj, 0);
1474
1475                         /*
1476                          * The only valid reason for this to ever fail would be
1477                          * if the dma-mapper screwed us over when we did the
1478                          * dma_map_sg(), since it has the final say over the dma
1479                          * address.
1480                          */
1481                         if (IS_ALIGNED(addr, page_size)) {
1482                                 pr_err("page_sizes.gtt=%u, expected=%u\n",
1483                                        vma->page_sizes.gtt, page_size);
1484                                 err = -EINVAL;
1485                         } else {
1486                                 pr_info("dma address misaligned, finishing test early\n");
1487                         }
1488
1489                         goto out_unpin;
1490                 }
1491
1492                 err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE);
1493                 if (err)
1494                         goto out_unpin;
1495
1496                 i915_vma_unpin(vma);
1497                 i915_vma_close(vma);
1498
1499                 i915_gem_object_put(obj);
1500         }
1501
1502         obj = i915_gem_object_create_internal(dev_priv, PAGE_SIZE);
1503         if (IS_ERR(obj))
1504                 return PTR_ERR(obj);
1505
1506         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
1507         if (IS_ERR(vma)) {
1508                 err = PTR_ERR(vma);
1509                 goto out_put;
1510         }
1511
1512         err = i915_vma_pin(vma, 0, 0, flags);
1513         if (err)
1514                 goto out_close;
1515
1516         /*
1517          * Make sure we don't end up with something like where the pde is still
1518          * pointing to the 2M page, and the pt we just filled-in is dangling --
1519          * we can check this by writing to the first page where it would then
1520          * land in the now stale 2M page.
1521          */
1522
1523         err = gpu_write(vma, ctx, dev_priv->engine[RCS], 0, 0xdeadbeaf);
1524         if (err)
1525                 goto out_unpin;
1526
1527         err = cpu_check(obj, 0, 0xdeadbeaf);
1528
1529 out_unpin:
1530         i915_vma_unpin(vma);
1531 out_close:
1532         i915_vma_close(vma);
1533 out_put:
1534         i915_gem_object_put(obj);
1535
1536         return err;
1537 }
1538
1539 static int igt_tmpfs_fallback(void *arg)
1540 {
1541         struct i915_gem_context *ctx = arg;
1542         struct drm_i915_private *i915 = ctx->i915;
1543         struct vfsmount *gemfs = i915->mm.gemfs;
1544         struct i915_address_space *vm =
1545                 ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm;
1546         struct drm_i915_gem_object *obj;
1547         struct i915_vma *vma;
1548         u32 *vaddr;
1549         int err = 0;
1550
1551         /*
1552          * Make sure that we don't burst into a ball of flames upon falling back
1553          * to tmpfs, which we rely on if on the off-chance we encouter a failure
1554          * when setting up gemfs.
1555          */
1556
1557         i915->mm.gemfs = NULL;
1558
1559         obj = i915_gem_object_create(i915, PAGE_SIZE);
1560         if (IS_ERR(obj)) {
1561                 err = PTR_ERR(obj);
1562                 goto out_restore;
1563         }
1564
1565         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
1566         if (IS_ERR(vaddr)) {
1567                 err = PTR_ERR(vaddr);
1568                 goto out_put;
1569         }
1570         *vaddr = 0xdeadbeaf;
1571
1572         i915_gem_object_unpin_map(obj);
1573
1574         vma = i915_vma_instance(obj, vm, NULL);
1575         if (IS_ERR(vma)) {
1576                 err = PTR_ERR(vma);
1577                 goto out_put;
1578         }
1579
1580         err = i915_vma_pin(vma, 0, 0, PIN_USER);
1581         if (err)
1582                 goto out_close;
1583
1584         err = igt_check_page_sizes(vma);
1585
1586         i915_vma_unpin(vma);
1587 out_close:
1588         i915_vma_close(vma);
1589 out_put:
1590         i915_gem_object_put(obj);
1591 out_restore:
1592         i915->mm.gemfs = gemfs;
1593
1594         return err;
1595 }
1596
1597 static int igt_shrink_thp(void *arg)
1598 {
1599         struct i915_gem_context *ctx = arg;
1600         struct drm_i915_private *i915 = ctx->i915;
1601         struct i915_address_space *vm =
1602                 ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm;
1603         struct drm_i915_gem_object *obj;
1604         struct i915_vma *vma;
1605         unsigned int flags = PIN_USER;
1606         int err;
1607
1608         /*
1609          * Sanity check shrinking huge-paged object -- make sure nothing blows
1610          * up.
1611          */
1612
1613         if (!igt_can_allocate_thp(i915)) {
1614                 pr_info("missing THP support, skipping\n");
1615                 return 0;
1616         }
1617
1618         obj = i915_gem_object_create(i915, SZ_2M);
1619         if (IS_ERR(obj))
1620                 return PTR_ERR(obj);
1621
1622         vma = i915_vma_instance(obj, vm, NULL);
1623         if (IS_ERR(vma)) {
1624                 err = PTR_ERR(vma);
1625                 goto out_put;
1626         }
1627
1628         err = i915_vma_pin(vma, 0, 0, flags);
1629         if (err)
1630                 goto out_close;
1631
1632         if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) {
1633                 pr_info("failed to allocate THP, finishing test early\n");
1634                 goto out_unpin;
1635         }
1636
1637         err = igt_check_page_sizes(vma);
1638         if (err)
1639                 goto out_unpin;
1640
1641         err = gpu_write(vma, ctx, i915->engine[RCS], 0, 0xdeadbeaf);
1642         if (err)
1643                 goto out_unpin;
1644
1645         i915_vma_unpin(vma);
1646
1647         /*
1648          * Now that the pages are *unpinned* shrink-all should invoke
1649          * shmem to truncate our pages.
1650          */
1651         i915_gem_shrink_all(i915);
1652         if (i915_gem_object_has_pages(obj)) {
1653                 pr_err("shrink-all didn't truncate the pages\n");
1654                 err = -EINVAL;
1655                 goto out_close;
1656         }
1657
1658         if (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys) {
1659                 pr_err("residual page-size bits left\n");
1660                 err = -EINVAL;
1661                 goto out_close;
1662         }
1663
1664         err = i915_vma_pin(vma, 0, 0, flags);
1665         if (err)
1666                 goto out_close;
1667
1668         err = cpu_check(obj, 0, 0xdeadbeaf);
1669
1670 out_unpin:
1671         i915_vma_unpin(vma);
1672 out_close:
1673         i915_vma_close(vma);
1674 out_put:
1675         i915_gem_object_put(obj);
1676
1677         return err;
1678 }
1679
1680 int i915_gem_huge_page_mock_selftests(void)
1681 {
1682         static const struct i915_subtest tests[] = {
1683                 SUBTEST(igt_mock_exhaust_device_supported_pages),
1684                 SUBTEST(igt_mock_ppgtt_misaligned_dma),
1685                 SUBTEST(igt_mock_ppgtt_huge_fill),
1686                 SUBTEST(igt_mock_ppgtt_64K),
1687         };
1688         int saved_ppgtt = i915_modparams.enable_ppgtt;
1689         struct drm_i915_private *dev_priv;
1690         struct pci_dev *pdev;
1691         struct i915_hw_ppgtt *ppgtt;
1692         int err;
1693
1694         dev_priv = mock_gem_device();
1695         if (!dev_priv)
1696                 return -ENOMEM;
1697
1698         /* Pretend to be a device which supports the 48b PPGTT */
1699         i915_modparams.enable_ppgtt = 3;
1700
1701         pdev = dev_priv->drm.pdev;
1702         dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(39));
1703
1704         mutex_lock(&dev_priv->drm.struct_mutex);
1705         ppgtt = i915_ppgtt_create(dev_priv, ERR_PTR(-ENODEV));
1706         if (IS_ERR(ppgtt)) {
1707                 err = PTR_ERR(ppgtt);
1708                 goto out_unlock;
1709         }
1710
1711         if (!i915_vm_is_48bit(&ppgtt->vm)) {
1712                 pr_err("failed to create 48b PPGTT\n");
1713                 err = -EINVAL;
1714                 goto out_close;
1715         }
1716
1717         /* If we were ever hit this then it's time to mock the 64K scratch */
1718         if (!i915_vm_has_scratch_64K(&ppgtt->vm)) {
1719                 pr_err("PPGTT missing 64K scratch page\n");
1720                 err = -EINVAL;
1721                 goto out_close;
1722         }
1723
1724         err = i915_subtests(tests, ppgtt);
1725
1726 out_close:
1727         i915_ppgtt_close(&ppgtt->vm);
1728         i915_ppgtt_put(ppgtt);
1729
1730 out_unlock:
1731         mutex_unlock(&dev_priv->drm.struct_mutex);
1732
1733         i915_modparams.enable_ppgtt = saved_ppgtt;
1734
1735         drm_dev_put(&dev_priv->drm);
1736
1737         return err;
1738 }
1739
1740 int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv)
1741 {
1742         static const struct i915_subtest tests[] = {
1743                 SUBTEST(igt_shrink_thp),
1744                 SUBTEST(igt_ppgtt_pin_update),
1745                 SUBTEST(igt_tmpfs_fallback),
1746                 SUBTEST(igt_ppgtt_exhaust_huge),
1747                 SUBTEST(igt_ppgtt_gemfs_huge),
1748                 SUBTEST(igt_ppgtt_internal_huge),
1749         };
1750         struct drm_file *file;
1751         struct i915_gem_context *ctx;
1752         int err;
1753
1754         if (!USES_PPGTT(dev_priv)) {
1755                 pr_info("PPGTT not supported, skipping live-selftests\n");
1756                 return 0;
1757         }
1758
1759         if (i915_terminally_wedged(&dev_priv->gpu_error))
1760                 return 0;
1761
1762         file = mock_file(dev_priv);
1763         if (IS_ERR(file))
1764                 return PTR_ERR(file);
1765
1766         mutex_lock(&dev_priv->drm.struct_mutex);
1767         intel_runtime_pm_get(dev_priv);
1768
1769         ctx = live_context(dev_priv, file);
1770         if (IS_ERR(ctx)) {
1771                 err = PTR_ERR(ctx);
1772                 goto out_unlock;
1773         }
1774
1775         if (ctx->ppgtt)
1776                 ctx->ppgtt->vm.scrub_64K = true;
1777
1778         err = i915_subtests(tests, ctx);
1779
1780 out_unlock:
1781         intel_runtime_pm_put(dev_priv);
1782         mutex_unlock(&dev_priv->drm.struct_mutex);
1783
1784         mock_file_free(dev_priv, file);
1785
1786         return err;
1787 }