lib/test_hmm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This is a module to test the HMM (Heterogeneous Memory Management)
   4  * mirror and zone device private memory migration APIs of the kernel.
   5  * Userspace programs can register with the driver to mirror their own address
   6  * space and can use the device to read/write any valid virtual address.
   7  */
   8 #include <linux/init.h>
   9 #include <linux/fs.h>
  10 #include <linux/mm.h>
  11 #include <linux/module.h>
  12 #include <linux/kernel.h>
  13 #include <linux/cdev.h>
  14 #include <linux/device.h>
  15 #include <linux/mutex.h>
  16 #include <linux/rwsem.h>
  17 #include <linux/sched.h>
  18 #include <linux/slab.h>
  19 #include <linux/highmem.h>
  20 #include <linux/delay.h>
  21 #include <linux/pagemap.h>
  22 #include <linux/hmm.h>
  23 #include <linux/vmalloc.h>
  24 #include <linux/swap.h>
  25 #include <linux/swapops.h>
  26 #include <linux/sched/mm.h>
  27 #include <linux/platform_device.h>
  28 #include <linux/rmap.h>
  29
  30 #include "test_hmm_uapi.h"
  31
  32 #define DMIRROR_NDEVICES                2
  33 #define DMIRROR_RANGE_FAULT_TIMEOUT     1000
  34 #define DEVMEM_CHUNK_SIZE               (256 * 1024 * 1024U)
  35 #define DEVMEM_CHUNKS_RESERVE           16
  36
  37 static const struct dev_pagemap_ops dmirror_devmem_ops;
  38 static const struct mmu_interval_notifier_ops dmirror_min_ops;
  39 static dev_t dmirror_dev;
  40
  41 struct dmirror_device;
  42
  43 struct dmirror_bounce {
  44         void                    *ptr;
  45         unsigned long           size;
  46         unsigned long           addr;
  47         unsigned long           cpages;
  48 };
  49
  50 #define DPT_XA_TAG_ATOMIC 1UL
  51 #define DPT_XA_TAG_WRITE 3UL
  52
  53 /*
  54  * Data structure to track address ranges and register for mmu interval
  55  * notifier updates.
  56  */
  57 struct dmirror_interval {
  58         struct mmu_interval_notifier    notifier;
  59         struct dmirror                  *dmirror;
  60 };
  61
  62 /*
  63  * Data attached to the open device file.
  64  * Note that it might be shared after a fork().
  65  */
  66 struct dmirror {
  67         struct dmirror_device           *mdevice;
  68         struct xarray                   pt;
  69         struct mmu_interval_notifier    notifier;
  70         struct mutex                    mutex;
  71 };
  72
  73 /*
  74  * ZONE_DEVICE pages for migration and simulating device memory.
  75  */
  76 struct dmirror_chunk {
  77         struct dev_pagemap      pagemap;
  78         struct dmirror_device   *mdevice;
  79 };
  80
  81 /*
  82  * Per device data.
  83  */
  84 struct dmirror_device {
  85         struct cdev             cdevice;
  86         struct hmm_devmem       *devmem;
  87
  88         unsigned int            devmem_capacity;
  89         unsigned int            devmem_count;
  90         struct dmirror_chunk    **devmem_chunks;
  91         struct mutex            devmem_lock;    /* protects the above */
  92
  93         unsigned long           calloc;
  94         unsigned long           cfree;
  95         struct page             *free_pages;
  96         spinlock_t              lock;           /* protects the above */
  97 };
  98
  99 static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES];
 100
 101 static int dmirror_bounce_init(struct dmirror_bounce *bounce,
 102                                unsigned long addr,
 103                                unsigned long size)
 104 {
 105         bounce->addr = addr;
 106         bounce->size = size;
 107         bounce->cpages = 0;
 108         bounce->ptr = vmalloc(size);
 109         if (!bounce->ptr)
 110                 return -ENOMEM;
 111         return 0;
 112 }
 113
 114 static void dmirror_bounce_fini(struct dmirror_bounce *bounce)
 115 {
 116         vfree(bounce->ptr);
 117 }
 118
 119 static int dmirror_fops_open(struct inode *inode, struct file *filp)
 120 {
 121         struct cdev *cdev = inode->i_cdev;
 122         struct dmirror *dmirror;
 123         int ret;
 124
 125         /* Mirror this process address space */
 126         dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL);
 127         if (dmirror == NULL)
 128                 return -ENOMEM;
 129
 130         dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);
 131         mutex_init(&dmirror->mutex);
 132         xa_init(&dmirror->pt);
 133
 134         ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,
 135                                 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops);
 136         if (ret) {
 137                 kfree(dmirror);
 138                 return ret;
 139         }
 140
 141         filp->private_data = dmirror;
 142         return 0;
 143 }
 144
 145 static int dmirror_fops_release(struct inode *inode, struct file *filp)
 146 {
 147         struct dmirror *dmirror = filp->private_data;
 148
 149         mmu_interval_notifier_remove(&dmirror->notifier);
 150         xa_destroy(&dmirror->pt);
 151         kfree(dmirror);
 152         return 0;
 153 }
 154
 155 static struct dmirror_device *dmirror_page_to_device(struct page *page)
 156
 157 {
 158         return container_of(page->pgmap, struct dmirror_chunk,
 159                             pagemap)->mdevice;
 160 }
 161
 162 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)
 163 {
 164         unsigned long *pfns = range->hmm_pfns;
 165         unsigned long pfn;
 166
 167         for (pfn = (range->start >> PAGE_SHIFT);
 168              pfn < (range->end >> PAGE_SHIFT);
 169              pfn++, pfns++) {
 170                 struct page *page;
 171                 void *entry;
 172
 173                 /*
 174                  * Since we asked for hmm_range_fault() to populate pages,
 175                  * it shouldn't return an error entry on success.
 176                  */
 177                 WARN_ON(*pfns & HMM_PFN_ERROR);
 178                 WARN_ON(!(*pfns & HMM_PFN_VALID));
 179
 180                 page = hmm_pfn_to_page(*pfns);
 181                 WARN_ON(!page);
 182
 183                 entry = page;
 184                 if (*pfns & HMM_PFN_WRITE)
 185                         entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
 186                 else if (WARN_ON(range->default_flags & HMM_PFN_WRITE))
 187                         return -EFAULT;
 188                 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
 189                 if (xa_is_err(entry))
 190                         return xa_err(entry);
 191         }
 192
 193         return 0;
 194 }
 195
 196 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,
 197                               unsigned long end)
 198 {
 199         unsigned long pfn;
 200         void *entry;
 201
 202         /*
 203          * The XArray doesn't hold references to pages since it relies on
 204          * the mmu notifier to clear page pointers when they become stale.
 205          * Therefore, it is OK to just clear the entry.
 206          */
 207         xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,
 208                           end >> PAGE_SHIFT)
 209                 xa_erase(&dmirror->pt, pfn);
 210 }
 211
 212 static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni,
 213                                 const struct mmu_notifier_range *range,
 214                                 unsigned long cur_seq)
 215 {
 216         struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);
 217
 218         /*
 219          * Ignore invalidation callbacks for device private pages since
 220          * the invalidation is handled as part of the migration process.
 221          */
 222         if (range->event == MMU_NOTIFY_MIGRATE &&
 223             range->owner == dmirror->mdevice)
 224                 return true;
 225
 226         if (mmu_notifier_range_blockable(range))
 227                 mutex_lock(&dmirror->mutex);
 228         else if (!mutex_trylock(&dmirror->mutex))
 229                 return false;
 230
 231         mmu_interval_set_seq(mni, cur_seq);
 232         dmirror_do_update(dmirror, range->start, range->end);
 233
 234         mutex_unlock(&dmirror->mutex);
 235         return true;
 236 }
 237
 238 static const struct mmu_interval_notifier_ops dmirror_min_ops = {
 239         .invalidate = dmirror_interval_invalidate,
 240 };
 241
 242 static int dmirror_range_fault(struct dmirror *dmirror,
 243                                 struct hmm_range *range)
 244 {
 245         struct mm_struct *mm = dmirror->notifier.mm;
 246         unsigned long timeout =
 247                 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
 248         int ret;
 249
 250         while (true) {
 251                 if (time_after(jiffies, timeout)) {
 252                         ret = -EBUSY;
 253                         goto out;
 254                 }
 255
 256                 range->notifier_seq = mmu_interval_read_begin(range->notifier);
 257                 mmap_read_lock(mm);
 258                 ret = hmm_range_fault(range);
 259                 mmap_read_unlock(mm);
 260                 if (ret) {
 261                         if (ret == -EBUSY)
 262                                 continue;
 263                         goto out;
 264                 }
 265
 266                 mutex_lock(&dmirror->mutex);
 267                 if (mmu_interval_read_retry(range->notifier,
 268                                             range->notifier_seq)) {
 269                         mutex_unlock(&dmirror->mutex);
 270                         continue;
 271                 }
 272                 break;
 273         }
 274
 275         ret = dmirror_do_fault(dmirror, range);
 276
 277         mutex_unlock(&dmirror->mutex);
 278 out:
 279         return ret;
 280 }
 281
 282 static int dmirror_fault(struct dmirror *dmirror, unsigned long start,
 283                          unsigned long end, bool write)
 284 {
 285         struct mm_struct *mm = dmirror->notifier.mm;
 286         unsigned long addr;
 287         unsigned long pfns[64];
 288         struct hmm_range range = {
 289                 .notifier = &dmirror->notifier,
 290                 .hmm_pfns = pfns,
 291                 .pfn_flags_mask = 0,
 292                 .default_flags =
 293                         HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0),
 294                 .dev_private_owner = dmirror->mdevice,
 295         };
 296         int ret = 0;
 297
 298         /* Since the mm is for the mirrored process, get a reference first. */
 299         if (!mmget_not_zero(mm))
 300                 return 0;
 301
 302         for (addr = start; addr < end; addr = range.end) {
 303                 range.start = addr;
 304                 range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
 305
 306                 ret = dmirror_range_fault(dmirror, &range);
 307                 if (ret)
 308                         break;
 309         }
 310
 311         mmput(mm);
 312         return ret;
 313 }
 314
 315 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,
 316                            unsigned long end, struct dmirror_bounce *bounce)
 317 {
 318         unsigned long pfn;
 319         void *ptr;
 320
 321         ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
 322
 323         for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
 324                 void *entry;
 325                 struct page *page;
 326                 void *tmp;
 327
 328                 entry = xa_load(&dmirror->pt, pfn);
 329                 page = xa_untag_pointer(entry);
 330                 if (!page)
 331                         return -ENOENT;
 332
 333                 tmp = kmap(page);
 334                 memcpy(ptr, tmp, PAGE_SIZE);
 335                 kunmap(page);
 336
 337                 ptr += PAGE_SIZE;
 338                 bounce->cpages++;
 339         }
 340
 341         return 0;
 342 }
 343
 344 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
 345 {
 346         struct dmirror_bounce bounce;
 347         unsigned long start, end;
 348         unsigned long size = cmd->npages << PAGE_SHIFT;
 349         int ret;
 350
 351         start = cmd->addr;
 352         end = start + size;
 353         if (end < start)
 354                 return -EINVAL;
 355
 356         ret = dmirror_bounce_init(&bounce, start, size);
 357         if (ret)
 358                 return ret;
 359
 360         while (1) {
 361                 mutex_lock(&dmirror->mutex);
 362                 ret = dmirror_do_read(dmirror, start, end, &bounce);
 363                 mutex_unlock(&dmirror->mutex);
 364                 if (ret != -ENOENT)
 365                         break;
 366
 367                 start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
 368                 ret = dmirror_fault(dmirror, start, end, false);
 369                 if (ret)
 370                         break;
 371                 cmd->faults++;
 372         }
 373
 374         if (ret == 0) {
 375                 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
 376                                  bounce.size))
 377                         ret = -EFAULT;
 378         }
 379         cmd->cpages = bounce.cpages;
 380         dmirror_bounce_fini(&bounce);
 381         return ret;
 382 }
 383
 384 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,
 385                             unsigned long end, struct dmirror_bounce *bounce)
 386 {
 387         unsigned long pfn;
 388         void *ptr;
 389
 390         ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
 391
 392         for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
 393                 void *entry;
 394                 struct page *page;
 395                 void *tmp;
 396
 397                 entry = xa_load(&dmirror->pt, pfn);
 398                 page = xa_untag_pointer(entry);
 399                 if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE)
 400                         return -ENOENT;
 401
 402                 tmp = kmap(page);
 403                 memcpy(tmp, ptr, PAGE_SIZE);
 404                 kunmap(page);
 405
 406                 ptr += PAGE_SIZE;
 407                 bounce->cpages++;
 408         }
 409
 410         return 0;
 411 }
 412
 413 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
 414 {
 415         struct dmirror_bounce bounce;
 416         unsigned long start, end;
 417         unsigned long size = cmd->npages << PAGE_SHIFT;
 418         int ret;
 419
 420         start = cmd->addr;
 421         end = start + size;
 422         if (end < start)
 423                 return -EINVAL;
 424
 425         ret = dmirror_bounce_init(&bounce, start, size);
 426         if (ret)
 427                 return ret;
 428         if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr),
 429                            bounce.size)) {
 430                 ret = -EFAULT;
 431                 goto fini;
 432         }
 433
 434         while (1) {
 435                 mutex_lock(&dmirror->mutex);
 436                 ret = dmirror_do_write(dmirror, start, end, &bounce);
 437                 mutex_unlock(&dmirror->mutex);
 438                 if (ret != -ENOENT)
 439                         break;
 440
 441                 start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
 442                 ret = dmirror_fault(dmirror, start, end, true);
 443                 if (ret)
 444                         break;
 445                 cmd->faults++;
 446         }
 447
 448 fini:
 449         cmd->cpages = bounce.cpages;
 450         dmirror_bounce_fini(&bounce);
 451         return ret;
 452 }
 453
 454 static bool dmirror_allocate_chunk(struct dmirror_device *mdevice,
 455                                    struct page **ppage)
 456 {
 457         struct dmirror_chunk *devmem;
 458         struct resource *res;
 459         unsigned long pfn;
 460         unsigned long pfn_first;
 461         unsigned long pfn_last;
 462         void *ptr;
 463
 464         devmem = kzalloc(sizeof(*devmem), GFP_KERNEL);
 465         if (!devmem)
 466                 return false;
 467
 468         res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE,
 469                                       "hmm_dmirror");
 470         if (IS_ERR(res))
 471                 goto err_devmem;
 472
 473         devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
 474         devmem->pagemap.range.start = res->start;
 475         devmem->pagemap.range.end = res->end;
 476         devmem->pagemap.nr_range = 1;
 477         devmem->pagemap.ops = &dmirror_devmem_ops;
 478         devmem->pagemap.owner = mdevice;
 479
 480         mutex_lock(&mdevice->devmem_lock);
 481
 482         if (mdevice->devmem_count == mdevice->devmem_capacity) {
 483                 struct dmirror_chunk **new_chunks;
 484                 unsigned int new_capacity;
 485
 486                 new_capacity = mdevice->devmem_capacity +
 487                                 DEVMEM_CHUNKS_RESERVE;
 488                 new_chunks = krealloc(mdevice->devmem_chunks,
 489                                 sizeof(new_chunks[0]) * new_capacity,
 490                                 GFP_KERNEL);
 491                 if (!new_chunks)
 492                         goto err_release;
 493                 mdevice->devmem_capacity = new_capacity;
 494                 mdevice->devmem_chunks = new_chunks;
 495         }
 496
 497         ptr = memremap_pages(&devmem->pagemap, numa_node_id());
 498         if (IS_ERR(ptr))
 499                 goto err_release;
 500
 501         devmem->mdevice = mdevice;
 502         pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT;
 503         pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT);
 504         mdevice->devmem_chunks[mdevice->devmem_count++] = devmem;
 505
 506         mutex_unlock(&mdevice->devmem_lock);
 507
 508         pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n",
 509                 DEVMEM_CHUNK_SIZE / (1024 * 1024),
 510                 mdevice->devmem_count,
 511                 mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)),
 512                 pfn_first, pfn_last);
 513
 514         spin_lock(&mdevice->lock);
 515         for (pfn = pfn_first; pfn < pfn_last; pfn++) {
 516                 struct page *page = pfn_to_page(pfn);
 517
 518                 page->zone_device_data = mdevice->free_pages;
 519                 mdevice->free_pages = page;
 520         }
 521         if (ppage) {
 522                 *ppage = mdevice->free_pages;
 523                 mdevice->free_pages = (*ppage)->zone_device_data;
 524                 mdevice->calloc++;
 525         }
 526         spin_unlock(&mdevice->lock);
 527
 528         return true;
 529
 530 err_release:
 531         mutex_unlock(&mdevice->devmem_lock);
 532         release_mem_region(devmem->pagemap.range.start, range_len(&devmem->pagemap.range));
 533 err_devmem:
 534         kfree(devmem);
 535
 536         return false;
 537 }
 538
 539 static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice)
 540 {
 541         struct page *dpage = NULL;
 542         struct page *rpage;
 543
 544         /*
 545          * This is a fake device so we alloc real system memory to store
 546          * our device memory.
 547          */
 548         rpage = alloc_page(GFP_HIGHUSER);
 549         if (!rpage)
 550                 return NULL;
 551
 552         spin_lock(&mdevice->lock);
 553
 554         if (mdevice->free_pages) {
 555                 dpage = mdevice->free_pages;
 556                 mdevice->free_pages = dpage->zone_device_data;
 557                 mdevice->calloc++;
 558                 spin_unlock(&mdevice->lock);
 559         } else {
 560                 spin_unlock(&mdevice->lock);
 561                 if (!dmirror_allocate_chunk(mdevice, &dpage))
 562                         goto error;
 563         }
 564
 565         dpage->zone_device_data = rpage;
 566         get_page(dpage);
 567         lock_page(dpage);
 568         return dpage;
 569
 570 error:
 571         __free_page(rpage);
 572         return NULL;
 573 }
 574
 575 static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args,
 576                                            struct dmirror *dmirror)
 577 {
 578         struct dmirror_device *mdevice = dmirror->mdevice;
 579         const unsigned long *src = args->src;
 580         unsigned long *dst = args->dst;
 581         unsigned long addr;
 582
 583         for (addr = args->start; addr < args->end; addr += PAGE_SIZE,
 584                                                    src++, dst++) {
 585                 struct page *spage;
 586                 struct page *dpage;
 587                 struct page *rpage;
 588
 589                 if (!(*src & MIGRATE_PFN_MIGRATE))
 590                         continue;
 591
 592                 /*
 593                  * Note that spage might be NULL which is OK since it is an
 594                  * unallocated pte_none() or read-only zero page.
 595                  */
 596                 spage = migrate_pfn_to_page(*src);
 597
 598                 dpage = dmirror_devmem_alloc_page(mdevice);
 599                 if (!dpage)
 600                         continue;
 601
 602                 rpage = dpage->zone_device_data;
 603                 if (spage)
 604                         copy_highpage(rpage, spage);
 605                 else
 606                         clear_highpage(rpage);
 607
 608                 /*
 609                  * Normally, a device would use the page->zone_device_data to
 610                  * point to the mirror but here we use it to hold the page for
 611                  * the simulated device memory and that page holds the pointer
 612                  * to the mirror.
 613                  */
 614                 rpage->zone_device_data = dmirror;
 615
 616                 *dst = migrate_pfn(page_to_pfn(dpage)) |
 617                             MIGRATE_PFN_LOCKED;
 618                 if ((*src & MIGRATE_PFN_WRITE) ||
 619                     (!spage && args->vma->vm_flags & VM_WRITE))
 620                         *dst |= MIGRATE_PFN_WRITE;
 621         }
 622 }
 623
 624 static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start,
 625                              unsigned long end)
 626 {
 627         unsigned long pfn;
 628
 629         for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
 630                 void *entry;
 631
 632                 entry = xa_load(&dmirror->pt, pfn);
 633                 if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC)
 634                         return -EPERM;
 635         }
 636
 637         return 0;
 638 }
 639
 640 static int dmirror_atomic_map(unsigned long start, unsigned long end,
 641                               struct page **pages, struct dmirror *dmirror)
 642 {
 643         unsigned long pfn, mapped = 0;
 644         int i;
 645
 646         /* Map the migrated pages into the device's page tables. */
 647         mutex_lock(&dmirror->mutex);
 648
 649         for (i = 0, pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, i++) {
 650                 void *entry;
 651
 652                 if (!pages[i])
 653                         continue;
 654
 655                 entry = pages[i];
 656                 entry = xa_tag_pointer(entry, DPT_XA_TAG_ATOMIC);
 657                 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
 658                 if (xa_is_err(entry)) {
 659                         mutex_unlock(&dmirror->mutex);
 660                         return xa_err(entry);
 661                 }
 662
 663                 mapped++;
 664         }
 665
 666         mutex_unlock(&dmirror->mutex);
 667         return mapped;
 668 }
 669
 670 static int dmirror_migrate_finalize_and_map(struct migrate_vma *args,
 671                                             struct dmirror *dmirror)
 672 {
 673         unsigned long start = args->start;
 674         unsigned long end = args->end;
 675         const unsigned long *src = args->src;
 676         const unsigned long *dst = args->dst;
 677         unsigned long pfn;
 678
 679         /* Map the migrated pages into the device's page tables. */
 680         mutex_lock(&dmirror->mutex);
 681
 682         for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++,
 683                                                                 src++, dst++) {
 684                 struct page *dpage;
 685                 void *entry;
 686
 687                 if (!(*src & MIGRATE_PFN_MIGRATE))
 688                         continue;
 689
 690                 dpage = migrate_pfn_to_page(*dst);
 691                 if (!dpage)
 692                         continue;
 693
 694                 /*
 695                  * Store the page that holds the data so the page table
 696                  * doesn't have to deal with ZONE_DEVICE private pages.
 697                  */
 698                 entry = dpage->zone_device_data;
 699                 if (*dst & MIGRATE_PFN_WRITE)
 700                         entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
 701                 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
 702                 if (xa_is_err(entry)) {
 703                         mutex_unlock(&dmirror->mutex);
 704                         return xa_err(entry);
 705                 }
 706         }
 707
 708         mutex_unlock(&dmirror->mutex);
 709         return 0;
 710 }
 711
 712 static int dmirror_exclusive(struct dmirror *dmirror,
 713                              struct hmm_dmirror_cmd *cmd)
 714 {
 715         unsigned long start, end, addr;
 716         unsigned long size = cmd->npages << PAGE_SHIFT;
 717         struct mm_struct *mm = dmirror->notifier.mm;
 718         struct page *pages[64];
 719         struct dmirror_bounce bounce;
 720         unsigned long next;
 721         int ret;
 722
 723         start = cmd->addr;
 724         end = start + size;
 725         if (end < start)
 726                 return -EINVAL;
 727
 728         /* Since the mm is for the mirrored process, get a reference first. */
 729         if (!mmget_not_zero(mm))
 730                 return -EINVAL;
 731
 732         mmap_read_lock(mm);
 733         for (addr = start; addr < end; addr = next) {
 734                 unsigned long mapped = 0;
 735                 int i;
 736
 737                 if (end < addr + (ARRAY_SIZE(pages) << PAGE_SHIFT))
 738                         next = end;
 739                 else
 740                         next = addr + (ARRAY_SIZE(pages) << PAGE_SHIFT);
 741
 742                 ret = make_device_exclusive_range(mm, addr, next, pages, NULL);
 743                 /*
 744                  * Do dmirror_atomic_map() iff all pages are marked for
 745                  * exclusive access to avoid accessing uninitialized
 746                  * fields of pages.
 747                  */
 748                 if (ret == (next - addr) >> PAGE_SHIFT)
 749                         mapped = dmirror_atomic_map(addr, next, pages, dmirror);
 750                 for (i = 0; i < ret; i++) {
 751                         if (pages[i]) {
 752                                 unlock_page(pages[i]);
 753                                 put_page(pages[i]);
 754                         }
 755                 }
 756
 757                 if (addr + (mapped << PAGE_SHIFT) < next) {
 758                         mmap_read_unlock(mm);
 759                         mmput(mm);
 760                         return -EBUSY;
 761                 }
 762         }
 763         mmap_read_unlock(mm);
 764         mmput(mm);
 765
 766         /* Return the migrated data for verification. */
 767         ret = dmirror_bounce_init(&bounce, start, size);
 768         if (ret)
 769                 return ret;
 770         mutex_lock(&dmirror->mutex);
 771         ret = dmirror_do_read(dmirror, start, end, &bounce);
 772         mutex_unlock(&dmirror->mutex);
 773         if (ret == 0) {
 774                 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
 775                                  bounce.size))
 776                         ret = -EFAULT;
 777         }
 778
 779         cmd->cpages = bounce.cpages;
 780         dmirror_bounce_fini(&bounce);
 781         return ret;
 782 }
 783
 784 static int dmirror_migrate(struct dmirror *dmirror,
 785                            struct hmm_dmirror_cmd *cmd)
 786 {
 787         unsigned long start, end, addr;
 788         unsigned long size = cmd->npages << PAGE_SHIFT;
 789         struct mm_struct *mm = dmirror->notifier.mm;
 790         struct vm_area_struct *vma;
 791         unsigned long src_pfns[64];
 792         unsigned long dst_pfns[64];
 793         struct dmirror_bounce bounce;
 794         struct migrate_vma args;
 795         unsigned long next;
 796         int ret;
 797
 798         start = cmd->addr;
 799         end = start + size;
 800         if (end < start)
 801                 return -EINVAL;
 802
 803         /* Since the mm is for the mirrored process, get a reference first. */
 804         if (!mmget_not_zero(mm))
 805                 return -EINVAL;
 806
 807         mmap_read_lock(mm);
 808         for (addr = start; addr < end; addr = next) {
 809                 vma = vma_lookup(mm, addr);
 810                 if (!vma || !(vma->vm_flags & VM_READ)) {
 811                         ret = -EINVAL;
 812                         goto out;
 813                 }
 814                 next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT));
 815                 if (next > vma->vm_end)
 816                         next = vma->vm_end;
 817
 818                 args.vma = vma;
 819                 args.src = src_pfns;
 820                 args.dst = dst_pfns;
 821                 args.start = addr;
 822                 args.end = next;
 823                 args.pgmap_owner = dmirror->mdevice;
 824                 args.flags = MIGRATE_VMA_SELECT_SYSTEM;
 825                 ret = migrate_vma_setup(&args);
 826                 if (ret)
 827                         goto out;
 828
 829                 dmirror_migrate_alloc_and_copy(&args, dmirror);
 830                 migrate_vma_pages(&args);
 831                 dmirror_migrate_finalize_and_map(&args, dmirror);
 832                 migrate_vma_finalize(&args);
 833         }
 834         mmap_read_unlock(mm);
 835         mmput(mm);
 836
 837         /* Return the migrated data for verification. */
 838         ret = dmirror_bounce_init(&bounce, start, size);
 839         if (ret)
 840                 return ret;
 841         mutex_lock(&dmirror->mutex);
 842         ret = dmirror_do_read(dmirror, start, end, &bounce);
 843         mutex_unlock(&dmirror->mutex);
 844         if (ret == 0) {
 845                 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
 846                                  bounce.size))
 847                         ret = -EFAULT;
 848         }
 849         cmd->cpages = bounce.cpages;
 850         dmirror_bounce_fini(&bounce);
 851         return ret;
 852
 853 out:
 854         mmap_read_unlock(mm);
 855         mmput(mm);
 856         return ret;
 857 }
 858
 859 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,
 860                             unsigned char *perm, unsigned long entry)
 861 {
 862         struct page *page;
 863
 864         if (entry & HMM_PFN_ERROR) {
 865                 *perm = HMM_DMIRROR_PROT_ERROR;
 866                 return;
 867         }
 868         if (!(entry & HMM_PFN_VALID)) {
 869                 *perm = HMM_DMIRROR_PROT_NONE;
 870                 return;
 871         }
 872
 873         page = hmm_pfn_to_page(entry);
 874         if (is_device_private_page(page)) {
 875                 /* Is the page migrated to this device or some other? */
 876                 if (dmirror->mdevice == dmirror_page_to_device(page))
 877                         *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL;
 878                 else
 879                         *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE;
 880         } else if (is_zero_pfn(page_to_pfn(page)))
 881                 *perm = HMM_DMIRROR_PROT_ZERO;
 882         else
 883                 *perm = HMM_DMIRROR_PROT_NONE;
 884         if (entry & HMM_PFN_WRITE)
 885                 *perm |= HMM_DMIRROR_PROT_WRITE;
 886         else
 887                 *perm |= HMM_DMIRROR_PROT_READ;
 888         if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT)
 889                 *perm |= HMM_DMIRROR_PROT_PMD;
 890         else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT)
 891                 *perm |= HMM_DMIRROR_PROT_PUD;
 892 }
 893
 894 static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni,
 895                                 const struct mmu_notifier_range *range,
 896                                 unsigned long cur_seq)
 897 {
 898         struct dmirror_interval *dmi =
 899                 container_of(mni, struct dmirror_interval, notifier);
 900         struct dmirror *dmirror = dmi->dmirror;
 901
 902         if (mmu_notifier_range_blockable(range))
 903                 mutex_lock(&dmirror->mutex);
 904         else if (!mutex_trylock(&dmirror->mutex))
 905                 return false;
 906
 907         /*
 908          * Snapshots only need to set the sequence number since any
 909          * invalidation in the interval invalidates the whole snapshot.
 910          */
 911         mmu_interval_set_seq(mni, cur_seq);
 912
 913         mutex_unlock(&dmirror->mutex);
 914         return true;
 915 }
 916
 917 static const struct mmu_interval_notifier_ops dmirror_mrn_ops = {
 918         .invalidate = dmirror_snapshot_invalidate,
 919 };
 920
 921 static int dmirror_range_snapshot(struct dmirror *dmirror,
 922                                   struct hmm_range *range,
 923                                   unsigned char *perm)
 924 {
 925         struct mm_struct *mm = dmirror->notifier.mm;
 926         struct dmirror_interval notifier;
 927         unsigned long timeout =
 928                 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
 929         unsigned long i;
 930         unsigned long n;
 931         int ret = 0;
 932
 933         notifier.dmirror = dmirror;
 934         range->notifier = &notifier.notifier;
 935
 936         ret = mmu_interval_notifier_insert(range->notifier, mm,
 937                         range->start, range->end - range->start,
 938                         &dmirror_mrn_ops);
 939         if (ret)
 940                 return ret;
 941
 942         while (true) {
 943                 if (time_after(jiffies, timeout)) {
 944                         ret = -EBUSY;
 945                         goto out;
 946                 }
 947
 948                 range->notifier_seq = mmu_interval_read_begin(range->notifier);
 949
 950                 mmap_read_lock(mm);
 951                 ret = hmm_range_fault(range);
 952                 mmap_read_unlock(mm);
 953                 if (ret) {
 954                         if (ret == -EBUSY)
 955                                 continue;
 956                         goto out;
 957                 }
 958
 959                 mutex_lock(&dmirror->mutex);
 960                 if (mmu_interval_read_retry(range->notifier,
 961                                             range->notifier_seq)) {
 962                         mutex_unlock(&dmirror->mutex);
 963                         continue;
 964                 }
 965                 break;
 966         }
 967
 968         n = (range->end - range->start) >> PAGE_SHIFT;
 969         for (i = 0; i < n; i++)
 970                 dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);
 971
 972         mutex_unlock(&dmirror->mutex);
 973 out:
 974         mmu_interval_notifier_remove(range->notifier);
 975         return ret;
 976 }
 977
 978 static int dmirror_snapshot(struct dmirror *dmirror,
 979                             struct hmm_dmirror_cmd *cmd)
 980 {
 981         struct mm_struct *mm = dmirror->notifier.mm;
 982         unsigned long start, end;
 983         unsigned long size = cmd->npages << PAGE_SHIFT;
 984         unsigned long addr;
 985         unsigned long next;
 986         unsigned long pfns[64];
 987         unsigned char perm[64];
 988         char __user *uptr;
 989         struct hmm_range range = {
 990                 .hmm_pfns = pfns,
 991                 .dev_private_owner = dmirror->mdevice,
 992         };
 993         int ret = 0;
 994
 995         start = cmd->addr;
 996         end = start + size;
 997         if (end < start)
 998                 return -EINVAL;
 999
1000         /* Since the mm is for the mirrored process, get a reference first. */
1001         if (!mmget_not_zero(mm))
1002                 return -EINVAL;
1003
1004         /*
1005          * Register a temporary notifier to detect invalidations even if it
1006          * overlaps with other mmu_interval_notifiers.
1007          */
1008         uptr = u64_to_user_ptr(cmd->ptr);
1009         for (addr = start; addr < end; addr = next) {
1010                 unsigned long n;
1011
1012                 next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
1013                 range.start = addr;
1014                 range.end = next;
1015
1016                 ret = dmirror_range_snapshot(dmirror, &range, perm);
1017                 if (ret)
1018                         break;
1019
1020                 n = (range.end - range.start) >> PAGE_SHIFT;
1021                 if (copy_to_user(uptr, perm, n)) {
1022                         ret = -EFAULT;
1023                         break;
1024                 }
1025
1026                 cmd->cpages += n;
1027                 uptr += n;
1028         }
1029         mmput(mm);
1030
1031         return ret;
1032 }
1033
1034 static long dmirror_fops_unlocked_ioctl(struct file *filp,
1035                                         unsigned int command,
1036                                         unsigned long arg)
1037 {
1038         void __user *uarg = (void __user *)arg;
1039         struct hmm_dmirror_cmd cmd;
1040         struct dmirror *dmirror;
1041         int ret;
1042
1043         dmirror = filp->private_data;
1044         if (!dmirror)
1045                 return -EINVAL;
1046
1047         if (copy_from_user(&cmd, uarg, sizeof(cmd)))
1048                 return -EFAULT;
1049
1050         if (cmd.addr & ~PAGE_MASK)
1051                 return -EINVAL;
1052         if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT)))
1053                 return -EINVAL;
1054
1055         cmd.cpages = 0;
1056         cmd.faults = 0;
1057
1058         switch (command) {
1059         case HMM_DMIRROR_READ:
1060                 ret = dmirror_read(dmirror, &cmd);
1061                 break;
1062
1063         case HMM_DMIRROR_WRITE:
1064                 ret = dmirror_write(dmirror, &cmd);
1065                 break;
1066
1067         case HMM_DMIRROR_MIGRATE:
1068                 ret = dmirror_migrate(dmirror, &cmd);
1069                 break;
1070
1071         case HMM_DMIRROR_EXCLUSIVE:
1072                 ret = dmirror_exclusive(dmirror, &cmd);
1073                 break;
1074
1075         case HMM_DMIRROR_CHECK_EXCLUSIVE:
1076                 ret = dmirror_check_atomic(dmirror, cmd.addr,
1077                                         cmd.addr + (cmd.npages << PAGE_SHIFT));
1078                 break;
1079
1080         case HMM_DMIRROR_SNAPSHOT:
1081                 ret = dmirror_snapshot(dmirror, &cmd);
1082                 break;
1083
1084         default:
1085                 return -EINVAL;
1086         }
1087         if (ret)
1088                 return ret;
1089
1090         if (copy_to_user(uarg, &cmd, sizeof(cmd)))
1091                 return -EFAULT;
1092
1093         return 0;
1094 }
1095
1096 static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma)
1097 {
1098         unsigned long addr;
1099
1100         for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) {
1101                 struct page *page;
1102                 int ret;
1103
1104                 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
1105                 if (!page)
1106                         return -ENOMEM;
1107
1108                 ret = vm_insert_page(vma, addr, page);
1109                 if (ret) {
1110                         __free_page(page);
1111                         return ret;
1112                 }
1113                 put_page(page);
1114         }
1115
1116         return 0;
1117 }
1118
1119 static const struct file_operations dmirror_fops = {
1120         .open           = dmirror_fops_open,
1121         .release        = dmirror_fops_release,
1122         .mmap           = dmirror_fops_mmap,
1123         .unlocked_ioctl = dmirror_fops_unlocked_ioctl,
1124         .llseek         = default_llseek,
1125         .owner          = THIS_MODULE,
1126 };
1127
1128 static void dmirror_devmem_free(struct page *page)
1129 {
1130         struct page *rpage = page->zone_device_data;
1131         struct dmirror_device *mdevice;
1132
1133         if (rpage)
1134                 __free_page(rpage);
1135
1136         mdevice = dmirror_page_to_device(page);
1137
1138         spin_lock(&mdevice->lock);
1139         mdevice->cfree++;
1140         page->zone_device_data = mdevice->free_pages;
1141         mdevice->free_pages = page;
1142         spin_unlock(&mdevice->lock);
1143 }
1144
1145 static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args,
1146                                                       struct dmirror *dmirror)
1147 {
1148         const unsigned long *src = args->src;
1149         unsigned long *dst = args->dst;
1150         unsigned long start = args->start;
1151         unsigned long end = args->end;
1152         unsigned long addr;
1153
1154         for (addr = start; addr < end; addr += PAGE_SIZE,
1155                                        src++, dst++) {
1156                 struct page *dpage, *spage;
1157
1158                 spage = migrate_pfn_to_page(*src);
1159                 if (!spage || !(*src & MIGRATE_PFN_MIGRATE))
1160                         continue;
1161                 spage = spage->zone_device_data;
1162
1163                 dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);
1164                 if (!dpage)
1165                         continue;
1166
1167                 lock_page(dpage);
1168                 xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
1169                 copy_highpage(dpage, spage);
1170                 *dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
1171                 if (*src & MIGRATE_PFN_WRITE)
1172                         *dst |= MIGRATE_PFN_WRITE;
1173         }
1174         return 0;
1175 }
1176
1177 static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
1178 {
1179         struct migrate_vma args;
1180         unsigned long src_pfns;
1181         unsigned long dst_pfns;
1182         struct page *rpage;
1183         struct dmirror *dmirror;
1184         vm_fault_t ret;
1185
1186         /*
1187          * Normally, a device would use the page->zone_device_data to point to
1188          * the mirror but here we use it to hold the page for the simulated
1189          * device memory and that page holds the pointer to the mirror.
1190          */
1191         rpage = vmf->page->zone_device_data;
1192         dmirror = rpage->zone_device_data;
1193
1194         /* FIXME demonstrate how we can adjust migrate range */
1195         args.vma = vmf->vma;
1196         args.start = vmf->address;
1197         args.end = args.start + PAGE_SIZE;
1198         args.src = &src_pfns;
1199         args.dst = &dst_pfns;
1200         args.pgmap_owner = dmirror->mdevice;
1201         args.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
1202
1203         if (migrate_vma_setup(&args))
1204                 return VM_FAULT_SIGBUS;
1205
1206         ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);
1207         if (ret)
1208                 return ret;
1209         migrate_vma_pages(&args);
1210         /*
1211          * No device finalize step is needed since
1212          * dmirror_devmem_fault_alloc_and_copy() will have already
1213          * invalidated the device page table.
1214          */
1215         migrate_vma_finalize(&args);
1216         return 0;
1217 }
1218
1219 static const struct dev_pagemap_ops dmirror_devmem_ops = {
1220         .page_free      = dmirror_devmem_free,
1221         .migrate_to_ram = dmirror_devmem_fault,
1222 };
1223
1224 static int dmirror_device_init(struct dmirror_device *mdevice, int id)
1225 {
1226         dev_t dev;
1227         int ret;
1228
1229         dev = MKDEV(MAJOR(dmirror_dev), id);
1230         mutex_init(&mdevice->devmem_lock);
1231         spin_lock_init(&mdevice->lock);
1232
1233         cdev_init(&mdevice->cdevice, &dmirror_fops);
1234         mdevice->cdevice.owner = THIS_MODULE;
1235         ret = cdev_add(&mdevice->cdevice, dev, 1);
1236         if (ret)
1237                 return ret;
1238
1239         /* Build a list of free ZONE_DEVICE private struct pages */
1240         dmirror_allocate_chunk(mdevice, NULL);
1241
1242         return 0;
1243 }
1244
1245 static void dmirror_device_remove(struct dmirror_device *mdevice)
1246 {
1247         unsigned int i;
1248
1249         if (mdevice->devmem_chunks) {
1250                 for (i = 0; i < mdevice->devmem_count; i++) {
1251                         struct dmirror_chunk *devmem =
1252                                 mdevice->devmem_chunks[i];
1253
1254                         memunmap_pages(&devmem->pagemap);
1255                         release_mem_region(devmem->pagemap.range.start,
1256                                            range_len(&devmem->pagemap.range));
1257                         kfree(devmem);
1258                 }
1259                 kfree(mdevice->devmem_chunks);
1260         }
1261
1262         cdev_del(&mdevice->cdevice);
1263 }
1264
1265 static int __init hmm_dmirror_init(void)
1266 {
1267         int ret;
1268         int id;
1269
1270         ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES,
1271                                   "HMM_DMIRROR");
1272         if (ret)
1273                 goto err_unreg;
1274
1275         for (id = 0; id < DMIRROR_NDEVICES; id++) {
1276                 ret = dmirror_device_init(dmirror_devices + id, id);
1277                 if (ret)
1278                         goto err_chrdev;
1279         }
1280
1281         pr_info("HMM test module loaded. This is only for testing HMM.\n");
1282         return 0;
1283
1284 err_chrdev:
1285         while (--id >= 0)
1286                 dmirror_device_remove(dmirror_devices + id);
1287         unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);
1288 err_unreg:
1289         return ret;
1290 }
1291
1292 static void __exit hmm_dmirror_exit(void)
1293 {
1294         int id;
1295
1296         for (id = 0; id < DMIRROR_NDEVICES; id++)
1297                 dmirror_device_remove(dmirror_devices + id);
1298         unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);
1299 }
1300
1301 module_init(hmm_dmirror_init);
1302 module_exit(hmm_dmirror_exit);
1303 MODULE_LICENSE("GPL");