tools/testing/selftests/vm/hmm-tests.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * HMM stands for Heterogeneous Memory Management, it is a helper layer inside
   4  * the linux kernel to help device drivers mirror a process address space in
   5  * the device. This allows the device to use the same address space which
   6  * makes communication and data exchange a lot easier.
   7  *
   8  * This framework's sole purpose is to exercise various code paths inside
   9  * the kernel to make sure that HMM performs as expected and to flush out any
  10  * bugs.
  11  */
  12
  13 #include "../kselftest_harness.h"
  14
  15 #include <errno.h>
  16 #include <fcntl.h>
  17 #include <stdio.h>
  18 #include <stdlib.h>
  19 #include <stdint.h>
  20 #include <unistd.h>
  21 #include <strings.h>
  22 #include <time.h>
  23 #include <pthread.h>
  24 #include <sys/types.h>
  25 #include <sys/stat.h>
  26 #include <sys/mman.h>
  27 #include <sys/ioctl.h>
  28
  29 #include "./local_config.h"
  30 #ifdef LOCAL_CONFIG_HAVE_LIBHUGETLBFS
  31 #include <hugetlbfs.h>
  32 #endif
  33
  34 /*
  35  * This is a private UAPI to the kernel test module so it isn't exported
  36  * in the usual include/uapi/... directory.
  37  */
  38 #include "../../../../lib/test_hmm_uapi.h"
  39 #include "../../../../mm/gup_test.h"
  40
  41 struct hmm_buffer {
  42         void            *ptr;
  43         void            *mirror;
  44         unsigned long   size;
  45         int             fd;
  46         uint64_t        cpages;
  47         uint64_t        faults;
  48 };
  49
  50 enum {
  51         HMM_PRIVATE_DEVICE_ONE,
  52         HMM_PRIVATE_DEVICE_TWO,
  53         HMM_COHERENCE_DEVICE_ONE,
  54         HMM_COHERENCE_DEVICE_TWO,
  55 };
  56
  57 #define TWOMEG          (1 << 21)
  58 #define HMM_BUFFER_SIZE (1024 << 12)
  59 #define HMM_PATH_MAX    64
  60 #define NTIMES          10
  61
  62 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1)))
  63 /* Just the flags we need, copied from mm.h: */
  64 #define FOLL_WRITE      0x01    /* check pte is writable */
  65 #define FOLL_LONGTERM   0x10000 /* mapping lifetime is indefinite */
  66
  67 FIXTURE(hmm)
  68 {
  69         int             fd;
  70         unsigned int    page_size;
  71         unsigned int    page_shift;
  72 };
  73
  74 FIXTURE_VARIANT(hmm)
  75 {
  76         int     device_number;
  77 };
  78
  79 FIXTURE_VARIANT_ADD(hmm, hmm_device_private)
  80 {
  81         .device_number = HMM_PRIVATE_DEVICE_ONE,
  82 };
  83
  84 FIXTURE_VARIANT_ADD(hmm, hmm_device_coherent)
  85 {
  86         .device_number = HMM_COHERENCE_DEVICE_ONE,
  87 };
  88
  89 FIXTURE(hmm2)
  90 {
  91         int             fd0;
  92         int             fd1;
  93         unsigned int    page_size;
  94         unsigned int    page_shift;
  95 };
  96
  97 FIXTURE_VARIANT(hmm2)
  98 {
  99         int     device_number0;
 100         int     device_number1;
 101 };
 102
 103 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_private)
 104 {
 105         .device_number0 = HMM_PRIVATE_DEVICE_ONE,
 106         .device_number1 = HMM_PRIVATE_DEVICE_TWO,
 107 };
 108
 109 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_coherent)
 110 {
 111         .device_number0 = HMM_COHERENCE_DEVICE_ONE,
 112         .device_number1 = HMM_COHERENCE_DEVICE_TWO,
 113 };
 114
 115 static int hmm_open(int unit)
 116 {
 117         char pathname[HMM_PATH_MAX];
 118         int fd;
 119
 120         snprintf(pathname, sizeof(pathname), "/dev/hmm_dmirror%d", unit);
 121         fd = open(pathname, O_RDWR, 0);
 122         if (fd < 0)
 123                 fprintf(stderr, "could not open hmm dmirror driver (%s)\n",
 124                         pathname);
 125         return fd;
 126 }
 127
 128 static bool hmm_is_coherent_type(int dev_num)
 129 {
 130         return (dev_num >= HMM_COHERENCE_DEVICE_ONE);
 131 }
 132
 133 FIXTURE_SETUP(hmm)
 134 {
 135         self->page_size = sysconf(_SC_PAGE_SIZE);
 136         self->page_shift = ffs(self->page_size) - 1;
 137
 138         self->fd = hmm_open(variant->device_number);
 139         if (self->fd < 0 && hmm_is_coherent_type(variant->device_number))
 140                 SKIP(exit(0), "DEVICE_COHERENT not available");
 141         ASSERT_GE(self->fd, 0);
 142 }
 143
 144 FIXTURE_SETUP(hmm2)
 145 {
 146         self->page_size = sysconf(_SC_PAGE_SIZE);
 147         self->page_shift = ffs(self->page_size) - 1;
 148
 149         self->fd0 = hmm_open(variant->device_number0);
 150         if (self->fd0 < 0 && hmm_is_coherent_type(variant->device_number0))
 151                 SKIP(exit(0), "DEVICE_COHERENT not available");
 152         ASSERT_GE(self->fd0, 0);
 153         self->fd1 = hmm_open(variant->device_number1);
 154         ASSERT_GE(self->fd1, 0);
 155 }
 156
 157 FIXTURE_TEARDOWN(hmm)
 158 {
 159         int ret = close(self->fd);
 160
 161         ASSERT_EQ(ret, 0);
 162         self->fd = -1;
 163 }
 164
 165 FIXTURE_TEARDOWN(hmm2)
 166 {
 167         int ret = close(self->fd0);
 168
 169         ASSERT_EQ(ret, 0);
 170         self->fd0 = -1;
 171
 172         ret = close(self->fd1);
 173         ASSERT_EQ(ret, 0);
 174         self->fd1 = -1;
 175 }
 176
 177 static int hmm_dmirror_cmd(int fd,
 178                            unsigned long request,
 179                            struct hmm_buffer *buffer,
 180                            unsigned long npages)
 181 {
 182         struct hmm_dmirror_cmd cmd;
 183         int ret;
 184
 185         /* Simulate a device reading system memory. */
 186         cmd.addr = (__u64)buffer->ptr;
 187         cmd.ptr = (__u64)buffer->mirror;
 188         cmd.npages = npages;
 189
 190         for (;;) {
 191                 ret = ioctl(fd, request, &cmd);
 192                 if (ret == 0)
 193                         break;
 194                 if (errno == EINTR)
 195                         continue;
 196                 return -errno;
 197         }
 198         buffer->cpages = cmd.cpages;
 199         buffer->faults = cmd.faults;
 200
 201         return 0;
 202 }
 203
 204 static void hmm_buffer_free(struct hmm_buffer *buffer)
 205 {
 206         if (buffer == NULL)
 207                 return;
 208
 209         if (buffer->ptr)
 210                 munmap(buffer->ptr, buffer->size);
 211         free(buffer->mirror);
 212         free(buffer);
 213 }
 214
 215 /*
 216  * Create a temporary file that will be deleted on close.
 217  */
 218 static int hmm_create_file(unsigned long size)
 219 {
 220         char path[HMM_PATH_MAX];
 221         int fd;
 222
 223         strcpy(path, "/tmp");
 224         fd = open(path, O_TMPFILE | O_EXCL | O_RDWR, 0600);
 225         if (fd >= 0) {
 226                 int r;
 227
 228                 do {
 229                         r = ftruncate(fd, size);
 230                 } while (r == -1 && errno == EINTR);
 231                 if (!r)
 232                         return fd;
 233                 close(fd);
 234         }
 235         return -1;
 236 }
 237
 238 /*
 239  * Return a random unsigned number.
 240  */
 241 static unsigned int hmm_random(void)
 242 {
 243         static int fd = -1;
 244         unsigned int r;
 245
 246         if (fd < 0) {
 247                 fd = open("/dev/urandom", O_RDONLY);
 248                 if (fd < 0) {
 249                         fprintf(stderr, "%s:%d failed to open /dev/urandom\n",
 250                                         __FILE__, __LINE__);
 251                         return ~0U;
 252                 }
 253         }
 254         read(fd, &r, sizeof(r));
 255         return r;
 256 }
 257
 258 static void hmm_nanosleep(unsigned int n)
 259 {
 260         struct timespec t;
 261
 262         t.tv_sec = 0;
 263         t.tv_nsec = n;
 264         nanosleep(&t, NULL);
 265 }
 266
 267 static int hmm_migrate_sys_to_dev(int fd,
 268                                    struct hmm_buffer *buffer,
 269                                    unsigned long npages)
 270 {
 271         return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_DEV, buffer, npages);
 272 }
 273
 274 static int hmm_migrate_dev_to_sys(int fd,
 275                                    struct hmm_buffer *buffer,
 276                                    unsigned long npages)
 277 {
 278         return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_SYS, buffer, npages);
 279 }
 280
 281 /*
 282  * Simple NULL test of device open/close.
 283  */
 284 TEST_F(hmm, open_close)
 285 {
 286 }
 287
 288 /*
 289  * Read private anonymous memory.
 290  */
 291 TEST_F(hmm, anon_read)
 292 {
 293         struct hmm_buffer *buffer;
 294         unsigned long npages;
 295         unsigned long size;
 296         unsigned long i;
 297         int *ptr;
 298         int ret;
 299         int val;
 300
 301         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 302         ASSERT_NE(npages, 0);
 303         size = npages << self->page_shift;
 304
 305         buffer = malloc(sizeof(*buffer));
 306         ASSERT_NE(buffer, NULL);
 307
 308         buffer->fd = -1;
 309         buffer->size = size;
 310         buffer->mirror = malloc(size);
 311         ASSERT_NE(buffer->mirror, NULL);
 312
 313         buffer->ptr = mmap(NULL, size,
 314                            PROT_READ | PROT_WRITE,
 315                            MAP_PRIVATE | MAP_ANONYMOUS,
 316                            buffer->fd, 0);
 317         ASSERT_NE(buffer->ptr, MAP_FAILED);
 318
 319         /*
 320          * Initialize buffer in system memory but leave the first two pages
 321          * zero (pte_none and pfn_zero).
 322          */
 323         i = 2 * self->page_size / sizeof(*ptr);
 324         for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 325                 ptr[i] = i;
 326
 327         /* Set buffer permission to read-only. */
 328         ret = mprotect(buffer->ptr, size, PROT_READ);
 329         ASSERT_EQ(ret, 0);
 330
 331         /* Populate the CPU page table with a special zero page. */
 332         val = *(int *)(buffer->ptr + self->page_size);
 333         ASSERT_EQ(val, 0);
 334
 335         /* Simulate a device reading system memory. */
 336         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
 337         ASSERT_EQ(ret, 0);
 338         ASSERT_EQ(buffer->cpages, npages);
 339         ASSERT_EQ(buffer->faults, 1);
 340
 341         /* Check what the device read. */
 342         ptr = buffer->mirror;
 343         for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i)
 344                 ASSERT_EQ(ptr[i], 0);
 345         for (; i < size / sizeof(*ptr); ++i)
 346                 ASSERT_EQ(ptr[i], i);
 347
 348         hmm_buffer_free(buffer);
 349 }
 350
 351 /*
 352  * Read private anonymous memory which has been protected with
 353  * mprotect() PROT_NONE.
 354  */
 355 TEST_F(hmm, anon_read_prot)
 356 {
 357         struct hmm_buffer *buffer;
 358         unsigned long npages;
 359         unsigned long size;
 360         unsigned long i;
 361         int *ptr;
 362         int ret;
 363
 364         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 365         ASSERT_NE(npages, 0);
 366         size = npages << self->page_shift;
 367
 368         buffer = malloc(sizeof(*buffer));
 369         ASSERT_NE(buffer, NULL);
 370
 371         buffer->fd = -1;
 372         buffer->size = size;
 373         buffer->mirror = malloc(size);
 374         ASSERT_NE(buffer->mirror, NULL);
 375
 376         buffer->ptr = mmap(NULL, size,
 377                            PROT_READ | PROT_WRITE,
 378                            MAP_PRIVATE | MAP_ANONYMOUS,
 379                            buffer->fd, 0);
 380         ASSERT_NE(buffer->ptr, MAP_FAILED);
 381
 382         /* Initialize buffer in system memory. */
 383         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 384                 ptr[i] = i;
 385
 386         /* Initialize mirror buffer so we can verify it isn't written. */
 387         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 388                 ptr[i] = -i;
 389
 390         /* Protect buffer from reading. */
 391         ret = mprotect(buffer->ptr, size, PROT_NONE);
 392         ASSERT_EQ(ret, 0);
 393
 394         /* Simulate a device reading system memory. */
 395         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
 396         ASSERT_EQ(ret, -EFAULT);
 397
 398         /* Allow CPU to read the buffer so we can check it. */
 399         ret = mprotect(buffer->ptr, size, PROT_READ);
 400         ASSERT_EQ(ret, 0);
 401         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 402                 ASSERT_EQ(ptr[i], i);
 403
 404         /* Check what the device read. */
 405         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 406                 ASSERT_EQ(ptr[i], -i);
 407
 408         hmm_buffer_free(buffer);
 409 }
 410
 411 /*
 412  * Write private anonymous memory.
 413  */
 414 TEST_F(hmm, anon_write)
 415 {
 416         struct hmm_buffer *buffer;
 417         unsigned long npages;
 418         unsigned long size;
 419         unsigned long i;
 420         int *ptr;
 421         int ret;
 422
 423         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 424         ASSERT_NE(npages, 0);
 425         size = npages << self->page_shift;
 426
 427         buffer = malloc(sizeof(*buffer));
 428         ASSERT_NE(buffer, NULL);
 429
 430         buffer->fd = -1;
 431         buffer->size = size;
 432         buffer->mirror = malloc(size);
 433         ASSERT_NE(buffer->mirror, NULL);
 434
 435         buffer->ptr = mmap(NULL, size,
 436                            PROT_READ | PROT_WRITE,
 437                            MAP_PRIVATE | MAP_ANONYMOUS,
 438                            buffer->fd, 0);
 439         ASSERT_NE(buffer->ptr, MAP_FAILED);
 440
 441         /* Initialize data that the device will write to buffer->ptr. */
 442         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 443                 ptr[i] = i;
 444
 445         /* Simulate a device writing system memory. */
 446         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 447         ASSERT_EQ(ret, 0);
 448         ASSERT_EQ(buffer->cpages, npages);
 449         ASSERT_EQ(buffer->faults, 1);
 450
 451         /* Check what the device wrote. */
 452         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 453                 ASSERT_EQ(ptr[i], i);
 454
 455         hmm_buffer_free(buffer);
 456 }
 457
 458 /*
 459  * Write private anonymous memory which has been protected with
 460  * mprotect() PROT_READ.
 461  */
 462 TEST_F(hmm, anon_write_prot)
 463 {
 464         struct hmm_buffer *buffer;
 465         unsigned long npages;
 466         unsigned long size;
 467         unsigned long i;
 468         int *ptr;
 469         int ret;
 470
 471         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 472         ASSERT_NE(npages, 0);
 473         size = npages << self->page_shift;
 474
 475         buffer = malloc(sizeof(*buffer));
 476         ASSERT_NE(buffer, NULL);
 477
 478         buffer->fd = -1;
 479         buffer->size = size;
 480         buffer->mirror = malloc(size);
 481         ASSERT_NE(buffer->mirror, NULL);
 482
 483         buffer->ptr = mmap(NULL, size,
 484                            PROT_READ,
 485                            MAP_PRIVATE | MAP_ANONYMOUS,
 486                            buffer->fd, 0);
 487         ASSERT_NE(buffer->ptr, MAP_FAILED);
 488
 489         /* Simulate a device reading a zero page of memory. */
 490         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1);
 491         ASSERT_EQ(ret, 0);
 492         ASSERT_EQ(buffer->cpages, 1);
 493         ASSERT_EQ(buffer->faults, 1);
 494
 495         /* Initialize data that the device will write to buffer->ptr. */
 496         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 497                 ptr[i] = i;
 498
 499         /* Simulate a device writing system memory. */
 500         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 501         ASSERT_EQ(ret, -EPERM);
 502
 503         /* Check what the device wrote. */
 504         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 505                 ASSERT_EQ(ptr[i], 0);
 506
 507         /* Now allow writing and see that the zero page is replaced. */
 508         ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ);
 509         ASSERT_EQ(ret, 0);
 510
 511         /* Simulate a device writing system memory. */
 512         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 513         ASSERT_EQ(ret, 0);
 514         ASSERT_EQ(buffer->cpages, npages);
 515         ASSERT_EQ(buffer->faults, 1);
 516
 517         /* Check what the device wrote. */
 518         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 519                 ASSERT_EQ(ptr[i], i);
 520
 521         hmm_buffer_free(buffer);
 522 }
 523
 524 /*
 525  * Check that a device writing an anonymous private mapping
 526  * will copy-on-write if a child process inherits the mapping.
 527  */
 528 TEST_F(hmm, anon_write_child)
 529 {
 530         struct hmm_buffer *buffer;
 531         unsigned long npages;
 532         unsigned long size;
 533         unsigned long i;
 534         int *ptr;
 535         pid_t pid;
 536         int child_fd;
 537         int ret;
 538
 539         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 540         ASSERT_NE(npages, 0);
 541         size = npages << self->page_shift;
 542
 543         buffer = malloc(sizeof(*buffer));
 544         ASSERT_NE(buffer, NULL);
 545
 546         buffer->fd = -1;
 547         buffer->size = size;
 548         buffer->mirror = malloc(size);
 549         ASSERT_NE(buffer->mirror, NULL);
 550
 551         buffer->ptr = mmap(NULL, size,
 552                            PROT_READ | PROT_WRITE,
 553                            MAP_PRIVATE | MAP_ANONYMOUS,
 554                            buffer->fd, 0);
 555         ASSERT_NE(buffer->ptr, MAP_FAILED);
 556
 557         /* Initialize buffer->ptr so we can tell if it is written. */
 558         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 559                 ptr[i] = i;
 560
 561         /* Initialize data that the device will write to buffer->ptr. */
 562         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 563                 ptr[i] = -i;
 564
 565         pid = fork();
 566         if (pid == -1)
 567                 ASSERT_EQ(pid, 0);
 568         if (pid != 0) {
 569                 waitpid(pid, &ret, 0);
 570                 ASSERT_EQ(WIFEXITED(ret), 1);
 571
 572                 /* Check that the parent's buffer did not change. */
 573                 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 574                         ASSERT_EQ(ptr[i], i);
 575                 return;
 576         }
 577
 578         /* Check that we see the parent's values. */
 579         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 580                 ASSERT_EQ(ptr[i], i);
 581         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 582                 ASSERT_EQ(ptr[i], -i);
 583
 584         /* The child process needs its own mirror to its own mm. */
 585         child_fd = hmm_open(0);
 586         ASSERT_GE(child_fd, 0);
 587
 588         /* Simulate a device writing system memory. */
 589         ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages);
 590         ASSERT_EQ(ret, 0);
 591         ASSERT_EQ(buffer->cpages, npages);
 592         ASSERT_EQ(buffer->faults, 1);
 593
 594         /* Check what the device wrote. */
 595         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 596                 ASSERT_EQ(ptr[i], -i);
 597
 598         close(child_fd);
 599         exit(0);
 600 }
 601
 602 /*
 603  * Check that a device writing an anonymous shared mapping
 604  * will not copy-on-write if a child process inherits the mapping.
 605  */
 606 TEST_F(hmm, anon_write_child_shared)
 607 {
 608         struct hmm_buffer *buffer;
 609         unsigned long npages;
 610         unsigned long size;
 611         unsigned long i;
 612         int *ptr;
 613         pid_t pid;
 614         int child_fd;
 615         int ret;
 616
 617         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 618         ASSERT_NE(npages, 0);
 619         size = npages << self->page_shift;
 620
 621         buffer = malloc(sizeof(*buffer));
 622         ASSERT_NE(buffer, NULL);
 623
 624         buffer->fd = -1;
 625         buffer->size = size;
 626         buffer->mirror = malloc(size);
 627         ASSERT_NE(buffer->mirror, NULL);
 628
 629         buffer->ptr = mmap(NULL, size,
 630                            PROT_READ | PROT_WRITE,
 631                            MAP_SHARED | MAP_ANONYMOUS,
 632                            buffer->fd, 0);
 633         ASSERT_NE(buffer->ptr, MAP_FAILED);
 634
 635         /* Initialize buffer->ptr so we can tell if it is written. */
 636         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 637                 ptr[i] = i;
 638
 639         /* Initialize data that the device will write to buffer->ptr. */
 640         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 641                 ptr[i] = -i;
 642
 643         pid = fork();
 644         if (pid == -1)
 645                 ASSERT_EQ(pid, 0);
 646         if (pid != 0) {
 647                 waitpid(pid, &ret, 0);
 648                 ASSERT_EQ(WIFEXITED(ret), 1);
 649
 650                 /* Check that the parent's buffer did change. */
 651                 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 652                         ASSERT_EQ(ptr[i], -i);
 653                 return;
 654         }
 655
 656         /* Check that we see the parent's values. */
 657         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 658                 ASSERT_EQ(ptr[i], i);
 659         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 660                 ASSERT_EQ(ptr[i], -i);
 661
 662         /* The child process needs its own mirror to its own mm. */
 663         child_fd = hmm_open(0);
 664         ASSERT_GE(child_fd, 0);
 665
 666         /* Simulate a device writing system memory. */
 667         ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages);
 668         ASSERT_EQ(ret, 0);
 669         ASSERT_EQ(buffer->cpages, npages);
 670         ASSERT_EQ(buffer->faults, 1);
 671
 672         /* Check what the device wrote. */
 673         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 674                 ASSERT_EQ(ptr[i], -i);
 675
 676         close(child_fd);
 677         exit(0);
 678 }
 679
 680 /*
 681  * Write private anonymous huge page.
 682  */
 683 TEST_F(hmm, anon_write_huge)
 684 {
 685         struct hmm_buffer *buffer;
 686         unsigned long npages;
 687         unsigned long size;
 688         unsigned long i;
 689         void *old_ptr;
 690         void *map;
 691         int *ptr;
 692         int ret;
 693
 694         size = 2 * TWOMEG;
 695
 696         buffer = malloc(sizeof(*buffer));
 697         ASSERT_NE(buffer, NULL);
 698
 699         buffer->fd = -1;
 700         buffer->size = size;
 701         buffer->mirror = malloc(size);
 702         ASSERT_NE(buffer->mirror, NULL);
 703
 704         buffer->ptr = mmap(NULL, size,
 705                            PROT_READ | PROT_WRITE,
 706                            MAP_PRIVATE | MAP_ANONYMOUS,
 707                            buffer->fd, 0);
 708         ASSERT_NE(buffer->ptr, MAP_FAILED);
 709
 710         size = TWOMEG;
 711         npages = size >> self->page_shift;
 712         map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
 713         ret = madvise(map, size, MADV_HUGEPAGE);
 714         ASSERT_EQ(ret, 0);
 715         old_ptr = buffer->ptr;
 716         buffer->ptr = map;
 717
 718         /* Initialize data that the device will write to buffer->ptr. */
 719         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 720                 ptr[i] = i;
 721
 722         /* Simulate a device writing system memory. */
 723         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 724         ASSERT_EQ(ret, 0);
 725         ASSERT_EQ(buffer->cpages, npages);
 726         ASSERT_EQ(buffer->faults, 1);
 727
 728         /* Check what the device wrote. */
 729         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 730                 ASSERT_EQ(ptr[i], i);
 731
 732         buffer->ptr = old_ptr;
 733         hmm_buffer_free(buffer);
 734 }
 735
 736 #ifdef LOCAL_CONFIG_HAVE_LIBHUGETLBFS
 737 /*
 738  * Write huge TLBFS page.
 739  */
 740 TEST_F(hmm, anon_write_hugetlbfs)
 741 {
 742         struct hmm_buffer *buffer;
 743         unsigned long npages;
 744         unsigned long size;
 745         unsigned long i;
 746         int *ptr;
 747         int ret;
 748         long pagesizes[4];
 749         int n, idx;
 750
 751         /* Skip test if we can't allocate a hugetlbfs page. */
 752
 753         n = gethugepagesizes(pagesizes, 4);
 754         if (n <= 0)
 755                 SKIP(return, "Huge page size could not be determined");
 756         for (idx = 0; --n > 0; ) {
 757                 if (pagesizes[n] < pagesizes[idx])
 758                         idx = n;
 759         }
 760         size = ALIGN(TWOMEG, pagesizes[idx]);
 761         npages = size >> self->page_shift;
 762
 763         buffer = malloc(sizeof(*buffer));
 764         ASSERT_NE(buffer, NULL);
 765
 766         buffer->ptr = get_hugepage_region(size, GHR_STRICT);
 767         if (buffer->ptr == NULL) {
 768                 free(buffer);
 769                 SKIP(return, "Huge page could not be allocated");
 770         }
 771
 772         buffer->fd = -1;
 773         buffer->size = size;
 774         buffer->mirror = malloc(size);
 775         ASSERT_NE(buffer->mirror, NULL);
 776
 777         /* Initialize data that the device will write to buffer->ptr. */
 778         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 779                 ptr[i] = i;
 780
 781         /* Simulate a device writing system memory. */
 782         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 783         ASSERT_EQ(ret, 0);
 784         ASSERT_EQ(buffer->cpages, npages);
 785         ASSERT_EQ(buffer->faults, 1);
 786
 787         /* Check what the device wrote. */
 788         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 789                 ASSERT_EQ(ptr[i], i);
 790
 791         free_hugepage_region(buffer->ptr);
 792         buffer->ptr = NULL;
 793         hmm_buffer_free(buffer);
 794 }
 795 #endif /* LOCAL_CONFIG_HAVE_LIBHUGETLBFS */
 796
 797 /*
 798  * Read mmap'ed file memory.
 799  */
 800 TEST_F(hmm, file_read)
 801 {
 802         struct hmm_buffer *buffer;
 803         unsigned long npages;
 804         unsigned long size;
 805         unsigned long i;
 806         int *ptr;
 807         int ret;
 808         int fd;
 809         ssize_t len;
 810
 811         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 812         ASSERT_NE(npages, 0);
 813         size = npages << self->page_shift;
 814
 815         fd = hmm_create_file(size);
 816         ASSERT_GE(fd, 0);
 817
 818         buffer = malloc(sizeof(*buffer));
 819         ASSERT_NE(buffer, NULL);
 820
 821         buffer->fd = fd;
 822         buffer->size = size;
 823         buffer->mirror = malloc(size);
 824         ASSERT_NE(buffer->mirror, NULL);
 825
 826         /* Write initial contents of the file. */
 827         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 828                 ptr[i] = i;
 829         len = pwrite(fd, buffer->mirror, size, 0);
 830         ASSERT_EQ(len, size);
 831         memset(buffer->mirror, 0, size);
 832
 833         buffer->ptr = mmap(NULL, size,
 834                            PROT_READ,
 835                            MAP_SHARED,
 836                            buffer->fd, 0);
 837         ASSERT_NE(buffer->ptr, MAP_FAILED);
 838
 839         /* Simulate a device reading system memory. */
 840         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
 841         ASSERT_EQ(ret, 0);
 842         ASSERT_EQ(buffer->cpages, npages);
 843         ASSERT_EQ(buffer->faults, 1);
 844
 845         /* Check what the device read. */
 846         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 847                 ASSERT_EQ(ptr[i], i);
 848
 849         hmm_buffer_free(buffer);
 850 }
 851
 852 /*
 853  * Write mmap'ed file memory.
 854  */
 855 TEST_F(hmm, file_write)
 856 {
 857         struct hmm_buffer *buffer;
 858         unsigned long npages;
 859         unsigned long size;
 860         unsigned long i;
 861         int *ptr;
 862         int ret;
 863         int fd;
 864         ssize_t len;
 865
 866         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 867         ASSERT_NE(npages, 0);
 868         size = npages << self->page_shift;
 869
 870         fd = hmm_create_file(size);
 871         ASSERT_GE(fd, 0);
 872
 873         buffer = malloc(sizeof(*buffer));
 874         ASSERT_NE(buffer, NULL);
 875
 876         buffer->fd = fd;
 877         buffer->size = size;
 878         buffer->mirror = malloc(size);
 879         ASSERT_NE(buffer->mirror, NULL);
 880
 881         buffer->ptr = mmap(NULL, size,
 882                            PROT_READ | PROT_WRITE,
 883                            MAP_SHARED,
 884                            buffer->fd, 0);
 885         ASSERT_NE(buffer->ptr, MAP_FAILED);
 886
 887         /* Initialize data that the device will write to buffer->ptr. */
 888         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 889                 ptr[i] = i;
 890
 891         /* Simulate a device writing system memory. */
 892         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 893         ASSERT_EQ(ret, 0);
 894         ASSERT_EQ(buffer->cpages, npages);
 895         ASSERT_EQ(buffer->faults, 1);
 896
 897         /* Check what the device wrote. */
 898         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 899                 ASSERT_EQ(ptr[i], i);
 900
 901         /* Check that the device also wrote the file. */
 902         len = pread(fd, buffer->mirror, size, 0);
 903         ASSERT_EQ(len, size);
 904         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 905                 ASSERT_EQ(ptr[i], i);
 906
 907         hmm_buffer_free(buffer);
 908 }
 909
 910 /*
 911  * Migrate anonymous memory to device private memory.
 912  */
 913 TEST_F(hmm, migrate)
 914 {
 915         struct hmm_buffer *buffer;
 916         unsigned long npages;
 917         unsigned long size;
 918         unsigned long i;
 919         int *ptr;
 920         int ret;
 921
 922         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 923         ASSERT_NE(npages, 0);
 924         size = npages << self->page_shift;
 925
 926         buffer = malloc(sizeof(*buffer));
 927         ASSERT_NE(buffer, NULL);
 928
 929         buffer->fd = -1;
 930         buffer->size = size;
 931         buffer->mirror = malloc(size);
 932         ASSERT_NE(buffer->mirror, NULL);
 933
 934         buffer->ptr = mmap(NULL, size,
 935                            PROT_READ | PROT_WRITE,
 936                            MAP_PRIVATE | MAP_ANONYMOUS,
 937                            buffer->fd, 0);
 938         ASSERT_NE(buffer->ptr, MAP_FAILED);
 939
 940         /* Initialize buffer in system memory. */
 941         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 942                 ptr[i] = i;
 943
 944         /* Migrate memory to device. */
 945         ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
 946         ASSERT_EQ(ret, 0);
 947         ASSERT_EQ(buffer->cpages, npages);
 948
 949         /* Check what the device read. */
 950         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 951                 ASSERT_EQ(ptr[i], i);
 952
 953         hmm_buffer_free(buffer);
 954 }
 955
 956 /*
 957  * Migrate anonymous memory to device private memory and fault some of it back
 958  * to system memory, then try migrating the resulting mix of system and device
 959  * private memory to the device.
 960  */
 961 TEST_F(hmm, migrate_fault)
 962 {
 963         struct hmm_buffer *buffer;
 964         unsigned long npages;
 965         unsigned long size;
 966         unsigned long i;
 967         int *ptr;
 968         int ret;
 969
 970         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 971         ASSERT_NE(npages, 0);
 972         size = npages << self->page_shift;
 973
 974         buffer = malloc(sizeof(*buffer));
 975         ASSERT_NE(buffer, NULL);
 976
 977         buffer->fd = -1;
 978         buffer->size = size;
 979         buffer->mirror = malloc(size);
 980         ASSERT_NE(buffer->mirror, NULL);
 981
 982         buffer->ptr = mmap(NULL, size,
 983                            PROT_READ | PROT_WRITE,
 984                            MAP_PRIVATE | MAP_ANONYMOUS,
 985                            buffer->fd, 0);
 986         ASSERT_NE(buffer->ptr, MAP_FAILED);
 987
 988         /* Initialize buffer in system memory. */
 989         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 990                 ptr[i] = i;
 991
 992         /* Migrate memory to device. */
 993         ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
 994         ASSERT_EQ(ret, 0);
 995         ASSERT_EQ(buffer->cpages, npages);
 996
 997         /* Check what the device read. */
 998         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 999                 ASSERT_EQ(ptr[i], i);
1000
1001         /* Fault half the pages back to system memory and check them. */
1002         for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)
1003                 ASSERT_EQ(ptr[i], i);
1004
1005         /* Migrate memory to the device again. */
1006         ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1007         ASSERT_EQ(ret, 0);
1008         ASSERT_EQ(buffer->cpages, npages);
1009
1010         /* Check what the device read. */
1011         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1012                 ASSERT_EQ(ptr[i], i);
1013
1014         hmm_buffer_free(buffer);
1015 }
1016
1017 /*
1018  * Migrate anonymous shared memory to device private memory.
1019  */
1020 TEST_F(hmm, migrate_shared)
1021 {
1022         struct hmm_buffer *buffer;
1023         unsigned long npages;
1024         unsigned long size;
1025         int ret;
1026
1027         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1028         ASSERT_NE(npages, 0);
1029         size = npages << self->page_shift;
1030
1031         buffer = malloc(sizeof(*buffer));
1032         ASSERT_NE(buffer, NULL);
1033
1034         buffer->fd = -1;
1035         buffer->size = size;
1036         buffer->mirror = malloc(size);
1037         ASSERT_NE(buffer->mirror, NULL);
1038
1039         buffer->ptr = mmap(NULL, size,
1040                            PROT_READ | PROT_WRITE,
1041                            MAP_SHARED | MAP_ANONYMOUS,
1042                            buffer->fd, 0);
1043         ASSERT_NE(buffer->ptr, MAP_FAILED);
1044
1045         /* Migrate memory to device. */
1046         ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1047         ASSERT_EQ(ret, -ENOENT);
1048
1049         hmm_buffer_free(buffer);
1050 }
1051
1052 /*
1053  * Try to migrate various memory types to device private memory.
1054  */
1055 TEST_F(hmm2, migrate_mixed)
1056 {
1057         struct hmm_buffer *buffer;
1058         unsigned long npages;
1059         unsigned long size;
1060         int *ptr;
1061         unsigned char *p;
1062         int ret;
1063         int val;
1064
1065         npages = 6;
1066         size = npages << self->page_shift;
1067
1068         buffer = malloc(sizeof(*buffer));
1069         ASSERT_NE(buffer, NULL);
1070
1071         buffer->fd = -1;
1072         buffer->size = size;
1073         buffer->mirror = malloc(size);
1074         ASSERT_NE(buffer->mirror, NULL);
1075
1076         /* Reserve a range of addresses. */
1077         buffer->ptr = mmap(NULL, size,
1078                            PROT_NONE,
1079                            MAP_PRIVATE | MAP_ANONYMOUS,
1080                            buffer->fd, 0);
1081         ASSERT_NE(buffer->ptr, MAP_FAILED);
1082         p = buffer->ptr;
1083
1084         /* Migrating a protected area should be an error. */
1085         ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages);
1086         ASSERT_EQ(ret, -EINVAL);
1087
1088         /* Punch a hole after the first page address. */
1089         ret = munmap(buffer->ptr + self->page_size, self->page_size);
1090         ASSERT_EQ(ret, 0);
1091
1092         /* We expect an error if the vma doesn't cover the range. */
1093         ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 3);
1094         ASSERT_EQ(ret, -EINVAL);
1095
1096         /* Page 2 will be a read-only zero page. */
1097         ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size,
1098                                 PROT_READ);
1099         ASSERT_EQ(ret, 0);
1100         ptr = (int *)(buffer->ptr + 2 * self->page_size);
1101         val = *ptr + 3;
1102         ASSERT_EQ(val, 3);
1103
1104         /* Page 3 will be read-only. */
1105         ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1106                                 PROT_READ | PROT_WRITE);
1107         ASSERT_EQ(ret, 0);
1108         ptr = (int *)(buffer->ptr + 3 * self->page_size);
1109         *ptr = val;
1110         ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1111                                 PROT_READ);
1112         ASSERT_EQ(ret, 0);
1113
1114         /* Page 4-5 will be read-write. */
1115         ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size,
1116                                 PROT_READ | PROT_WRITE);
1117         ASSERT_EQ(ret, 0);
1118         ptr = (int *)(buffer->ptr + 4 * self->page_size);
1119         *ptr = val;
1120         ptr = (int *)(buffer->ptr + 5 * self->page_size);
1121         *ptr = val;
1122
1123         /* Now try to migrate pages 2-5 to device 1. */
1124         buffer->ptr = p + 2 * self->page_size;
1125         ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 4);
1126         ASSERT_EQ(ret, 0);
1127         ASSERT_EQ(buffer->cpages, 4);
1128
1129         /* Page 5 won't be migrated to device 0 because it's on device 1. */
1130         buffer->ptr = p + 5 * self->page_size;
1131         ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1);
1132         ASSERT_EQ(ret, -ENOENT);
1133         buffer->ptr = p;
1134
1135         buffer->ptr = p;
1136         hmm_buffer_free(buffer);
1137 }
1138
1139 /*
1140  * Migrate anonymous memory to device memory and back to system memory
1141  * multiple times. In case of private zone configuration, this is done
1142  * through fault pages accessed by CPU. In case of coherent zone configuration,
1143  * the pages from the device should be explicitly migrated back to system memory.
1144  * The reason is Coherent device zone has coherent access by CPU, therefore
1145  * it will not generate any page fault.
1146  */
1147 TEST_F(hmm, migrate_multiple)
1148 {
1149         struct hmm_buffer *buffer;
1150         unsigned long npages;
1151         unsigned long size;
1152         unsigned long i;
1153         unsigned long c;
1154         int *ptr;
1155         int ret;
1156
1157         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1158         ASSERT_NE(npages, 0);
1159         size = npages << self->page_shift;
1160
1161         for (c = 0; c < NTIMES; c++) {
1162                 buffer = malloc(sizeof(*buffer));
1163                 ASSERT_NE(buffer, NULL);
1164
1165                 buffer->fd = -1;
1166                 buffer->size = size;
1167                 buffer->mirror = malloc(size);
1168                 ASSERT_NE(buffer->mirror, NULL);
1169
1170                 buffer->ptr = mmap(NULL, size,
1171                                    PROT_READ | PROT_WRITE,
1172                                    MAP_PRIVATE | MAP_ANONYMOUS,
1173                                    buffer->fd, 0);
1174                 ASSERT_NE(buffer->ptr, MAP_FAILED);
1175
1176                 /* Initialize buffer in system memory. */
1177                 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1178                         ptr[i] = i;
1179
1180                 /* Migrate memory to device. */
1181                 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1182                 ASSERT_EQ(ret, 0);
1183                 ASSERT_EQ(buffer->cpages, npages);
1184
1185                 /* Check what the device read. */
1186                 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1187                         ASSERT_EQ(ptr[i], i);
1188
1189                 /* Migrate back to system memory and check them. */
1190                 if (hmm_is_coherent_type(variant->device_number)) {
1191                         ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages);
1192                         ASSERT_EQ(ret, 0);
1193                         ASSERT_EQ(buffer->cpages, npages);
1194                 }
1195
1196                 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1197                         ASSERT_EQ(ptr[i], i);
1198
1199                 hmm_buffer_free(buffer);
1200         }
1201 }
1202
1203 /*
1204  * Read anonymous memory multiple times.
1205  */
1206 TEST_F(hmm, anon_read_multiple)
1207 {
1208         struct hmm_buffer *buffer;
1209         unsigned long npages;
1210         unsigned long size;
1211         unsigned long i;
1212         unsigned long c;
1213         int *ptr;
1214         int ret;
1215
1216         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1217         ASSERT_NE(npages, 0);
1218         size = npages << self->page_shift;
1219
1220         for (c = 0; c < NTIMES; c++) {
1221                 buffer = malloc(sizeof(*buffer));
1222                 ASSERT_NE(buffer, NULL);
1223
1224                 buffer->fd = -1;
1225                 buffer->size = size;
1226                 buffer->mirror = malloc(size);
1227                 ASSERT_NE(buffer->mirror, NULL);
1228
1229                 buffer->ptr = mmap(NULL, size,
1230                                    PROT_READ | PROT_WRITE,
1231                                    MAP_PRIVATE | MAP_ANONYMOUS,
1232                                    buffer->fd, 0);
1233                 ASSERT_NE(buffer->ptr, MAP_FAILED);
1234
1235                 /* Initialize buffer in system memory. */
1236                 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1237                         ptr[i] = i + c;
1238
1239                 /* Simulate a device reading system memory. */
1240                 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer,
1241                                       npages);
1242                 ASSERT_EQ(ret, 0);
1243                 ASSERT_EQ(buffer->cpages, npages);
1244                 ASSERT_EQ(buffer->faults, 1);
1245
1246                 /* Check what the device read. */
1247                 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1248                         ASSERT_EQ(ptr[i], i + c);
1249
1250                 hmm_buffer_free(buffer);
1251         }
1252 }
1253
1254 void *unmap_buffer(void *p)
1255 {
1256         struct hmm_buffer *buffer = p;
1257
1258         /* Delay for a bit and then unmap buffer while it is being read. */
1259         hmm_nanosleep(hmm_random() % 32000);
1260         munmap(buffer->ptr + buffer->size / 2, buffer->size / 2);
1261         buffer->ptr = NULL;
1262
1263         return NULL;
1264 }
1265
1266 /*
1267  * Try reading anonymous memory while it is being unmapped.
1268  */
1269 TEST_F(hmm, anon_teardown)
1270 {
1271         unsigned long npages;
1272         unsigned long size;
1273         unsigned long c;
1274         void *ret;
1275
1276         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1277         ASSERT_NE(npages, 0);
1278         size = npages << self->page_shift;
1279
1280         for (c = 0; c < NTIMES; ++c) {
1281                 pthread_t thread;
1282                 struct hmm_buffer *buffer;
1283                 unsigned long i;
1284                 int *ptr;
1285                 int rc;
1286
1287                 buffer = malloc(sizeof(*buffer));
1288                 ASSERT_NE(buffer, NULL);
1289
1290                 buffer->fd = -1;
1291                 buffer->size = size;
1292                 buffer->mirror = malloc(size);
1293                 ASSERT_NE(buffer->mirror, NULL);
1294
1295                 buffer->ptr = mmap(NULL, size,
1296                                    PROT_READ | PROT_WRITE,
1297                                    MAP_PRIVATE | MAP_ANONYMOUS,
1298                                    buffer->fd, 0);
1299                 ASSERT_NE(buffer->ptr, MAP_FAILED);
1300
1301                 /* Initialize buffer in system memory. */
1302                 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1303                         ptr[i] = i + c;
1304
1305                 rc = pthread_create(&thread, NULL, unmap_buffer, buffer);
1306                 ASSERT_EQ(rc, 0);
1307
1308                 /* Simulate a device reading system memory. */
1309                 rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer,
1310                                      npages);
1311                 if (rc == 0) {
1312                         ASSERT_EQ(buffer->cpages, npages);
1313                         ASSERT_EQ(buffer->faults, 1);
1314
1315                         /* Check what the device read. */
1316                         for (i = 0, ptr = buffer->mirror;
1317                              i < size / sizeof(*ptr);
1318                              ++i)
1319                                 ASSERT_EQ(ptr[i], i + c);
1320                 }
1321
1322                 pthread_join(thread, &ret);
1323                 hmm_buffer_free(buffer);
1324         }
1325 }
1326
1327 /*
1328  * Test memory snapshot without faulting in pages accessed by the device.
1329  */
1330 TEST_F(hmm, mixedmap)
1331 {
1332         struct hmm_buffer *buffer;
1333         unsigned long npages;
1334         unsigned long size;
1335         unsigned char *m;
1336         int ret;
1337
1338         npages = 1;
1339         size = npages << self->page_shift;
1340
1341         buffer = malloc(sizeof(*buffer));
1342         ASSERT_NE(buffer, NULL);
1343
1344         buffer->fd = -1;
1345         buffer->size = size;
1346         buffer->mirror = malloc(npages);
1347         ASSERT_NE(buffer->mirror, NULL);
1348
1349
1350         /* Reserve a range of addresses. */
1351         buffer->ptr = mmap(NULL, size,
1352                            PROT_READ | PROT_WRITE,
1353                            MAP_PRIVATE,
1354                            self->fd, 0);
1355         ASSERT_NE(buffer->ptr, MAP_FAILED);
1356
1357         /* Simulate a device snapshotting CPU pagetables. */
1358         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1359         ASSERT_EQ(ret, 0);
1360         ASSERT_EQ(buffer->cpages, npages);
1361
1362         /* Check what the device saw. */
1363         m = buffer->mirror;
1364         ASSERT_EQ(m[0], HMM_DMIRROR_PROT_READ);
1365
1366         hmm_buffer_free(buffer);
1367 }
1368
1369 /*
1370  * Test memory snapshot without faulting in pages accessed by the device.
1371  */
1372 TEST_F(hmm2, snapshot)
1373 {
1374         struct hmm_buffer *buffer;
1375         unsigned long npages;
1376         unsigned long size;
1377         int *ptr;
1378         unsigned char *p;
1379         unsigned char *m;
1380         int ret;
1381         int val;
1382
1383         npages = 7;
1384         size = npages << self->page_shift;
1385
1386         buffer = malloc(sizeof(*buffer));
1387         ASSERT_NE(buffer, NULL);
1388
1389         buffer->fd = -1;
1390         buffer->size = size;
1391         buffer->mirror = malloc(npages);
1392         ASSERT_NE(buffer->mirror, NULL);
1393
1394         /* Reserve a range of addresses. */
1395         buffer->ptr = mmap(NULL, size,
1396                            PROT_NONE,
1397                            MAP_PRIVATE | MAP_ANONYMOUS,
1398                            buffer->fd, 0);
1399         ASSERT_NE(buffer->ptr, MAP_FAILED);
1400         p = buffer->ptr;
1401
1402         /* Punch a hole after the first page address. */
1403         ret = munmap(buffer->ptr + self->page_size, self->page_size);
1404         ASSERT_EQ(ret, 0);
1405
1406         /* Page 2 will be read-only zero page. */
1407         ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size,
1408                                 PROT_READ);
1409         ASSERT_EQ(ret, 0);
1410         ptr = (int *)(buffer->ptr + 2 * self->page_size);
1411         val = *ptr + 3;
1412         ASSERT_EQ(val, 3);
1413
1414         /* Page 3 will be read-only. */
1415         ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1416                                 PROT_READ | PROT_WRITE);
1417         ASSERT_EQ(ret, 0);
1418         ptr = (int *)(buffer->ptr + 3 * self->page_size);
1419         *ptr = val;
1420         ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1421                                 PROT_READ);
1422         ASSERT_EQ(ret, 0);
1423
1424         /* Page 4-6 will be read-write. */
1425         ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size,
1426                                 PROT_READ | PROT_WRITE);
1427         ASSERT_EQ(ret, 0);
1428         ptr = (int *)(buffer->ptr + 4 * self->page_size);
1429         *ptr = val;
1430
1431         /* Page 5 will be migrated to device 0. */
1432         buffer->ptr = p + 5 * self->page_size;
1433         ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1);
1434         ASSERT_EQ(ret, 0);
1435         ASSERT_EQ(buffer->cpages, 1);
1436
1437         /* Page 6 will be migrated to device 1. */
1438         buffer->ptr = p + 6 * self->page_size;
1439         ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 1);
1440         ASSERT_EQ(ret, 0);
1441         ASSERT_EQ(buffer->cpages, 1);
1442
1443         /* Simulate a device snapshotting CPU pagetables. */
1444         buffer->ptr = p;
1445         ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1446         ASSERT_EQ(ret, 0);
1447         ASSERT_EQ(buffer->cpages, npages);
1448
1449         /* Check what the device saw. */
1450         m = buffer->mirror;
1451         ASSERT_EQ(m[0], HMM_DMIRROR_PROT_ERROR);
1452         ASSERT_EQ(m[1], HMM_DMIRROR_PROT_ERROR);
1453         ASSERT_EQ(m[2], HMM_DMIRROR_PROT_ZERO | HMM_DMIRROR_PROT_READ);
1454         ASSERT_EQ(m[3], HMM_DMIRROR_PROT_READ);
1455         ASSERT_EQ(m[4], HMM_DMIRROR_PROT_WRITE);
1456         if (!hmm_is_coherent_type(variant->device_number0)) {
1457                 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL |
1458                                 HMM_DMIRROR_PROT_WRITE);
1459                 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_NONE);
1460         } else {
1461                 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL |
1462                                 HMM_DMIRROR_PROT_WRITE);
1463                 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE |
1464                                 HMM_DMIRROR_PROT_WRITE);
1465         }
1466
1467         hmm_buffer_free(buffer);
1468 }
1469
1470 #ifdef LOCAL_CONFIG_HAVE_LIBHUGETLBFS
1471 /*
1472  * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that
1473  * should be mapped by a large page table entry.
1474  */
1475 TEST_F(hmm, compound)
1476 {
1477         struct hmm_buffer *buffer;
1478         unsigned long npages;
1479         unsigned long size;
1480         int *ptr;
1481         unsigned char *m;
1482         int ret;
1483         long pagesizes[4];
1484         int n, idx;
1485         unsigned long i;
1486
1487         /* Skip test if we can't allocate a hugetlbfs page. */
1488
1489         n = gethugepagesizes(pagesizes, 4);
1490         if (n <= 0)
1491                 return;
1492         for (idx = 0; --n > 0; ) {
1493                 if (pagesizes[n] < pagesizes[idx])
1494                         idx = n;
1495         }
1496         size = ALIGN(TWOMEG, pagesizes[idx]);
1497         npages = size >> self->page_shift;
1498
1499         buffer = malloc(sizeof(*buffer));
1500         ASSERT_NE(buffer, NULL);
1501
1502         buffer->ptr = get_hugepage_region(size, GHR_STRICT);
1503         if (buffer->ptr == NULL) {
1504                 free(buffer);
1505                 return;
1506         }
1507
1508         buffer->size = size;
1509         buffer->mirror = malloc(npages);
1510         ASSERT_NE(buffer->mirror, NULL);
1511
1512         /* Initialize the pages the device will snapshot in buffer->ptr. */
1513         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1514                 ptr[i] = i;
1515
1516         /* Simulate a device snapshotting CPU pagetables. */
1517         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1518         ASSERT_EQ(ret, 0);
1519         ASSERT_EQ(buffer->cpages, npages);
1520
1521         /* Check what the device saw. */
1522         m = buffer->mirror;
1523         for (i = 0; i < npages; ++i)
1524                 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE |
1525                                 HMM_DMIRROR_PROT_PMD);
1526
1527         /* Make the region read-only. */
1528         ret = mprotect(buffer->ptr, size, PROT_READ);
1529         ASSERT_EQ(ret, 0);
1530
1531         /* Simulate a device snapshotting CPU pagetables. */
1532         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1533         ASSERT_EQ(ret, 0);
1534         ASSERT_EQ(buffer->cpages, npages);
1535
1536         /* Check what the device saw. */
1537         m = buffer->mirror;
1538         for (i = 0; i < npages; ++i)
1539                 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ |
1540                                 HMM_DMIRROR_PROT_PMD);
1541
1542         free_hugepage_region(buffer->ptr);
1543         buffer->ptr = NULL;
1544         hmm_buffer_free(buffer);
1545 }
1546 #endif /* LOCAL_CONFIG_HAVE_LIBHUGETLBFS */
1547
1548 /*
1549  * Test two devices reading the same memory (double mapped).
1550  */
1551 TEST_F(hmm2, double_map)
1552 {
1553         struct hmm_buffer *buffer;
1554         unsigned long npages;
1555         unsigned long size;
1556         unsigned long i;
1557         int *ptr;
1558         int ret;
1559
1560         npages = 6;
1561         size = npages << self->page_shift;
1562
1563         buffer = malloc(sizeof(*buffer));
1564         ASSERT_NE(buffer, NULL);
1565
1566         buffer->fd = -1;
1567         buffer->size = size;
1568         buffer->mirror = malloc(npages);
1569         ASSERT_NE(buffer->mirror, NULL);
1570
1571         /* Reserve a range of addresses. */
1572         buffer->ptr = mmap(NULL, size,
1573                            PROT_READ | PROT_WRITE,
1574                            MAP_PRIVATE | MAP_ANONYMOUS,
1575                            buffer->fd, 0);
1576         ASSERT_NE(buffer->ptr, MAP_FAILED);
1577
1578         /* Initialize buffer in system memory. */
1579         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1580                 ptr[i] = i;
1581
1582         /* Make region read-only. */
1583         ret = mprotect(buffer->ptr, size, PROT_READ);
1584         ASSERT_EQ(ret, 0);
1585
1586         /* Simulate device 0 reading system memory. */
1587         ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages);
1588         ASSERT_EQ(ret, 0);
1589         ASSERT_EQ(buffer->cpages, npages);
1590         ASSERT_EQ(buffer->faults, 1);
1591
1592         /* Check what the device read. */
1593         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1594                 ASSERT_EQ(ptr[i], i);
1595
1596         /* Simulate device 1 reading system memory. */
1597         ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages);
1598         ASSERT_EQ(ret, 0);
1599         ASSERT_EQ(buffer->cpages, npages);
1600         ASSERT_EQ(buffer->faults, 1);
1601
1602         /* Check what the device read. */
1603         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1604                 ASSERT_EQ(ptr[i], i);
1605
1606         /* Migrate pages to device 1 and try to read from device 0. */
1607         ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages);
1608         ASSERT_EQ(ret, 0);
1609         ASSERT_EQ(buffer->cpages, npages);
1610
1611         ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages);
1612         ASSERT_EQ(ret, 0);
1613         ASSERT_EQ(buffer->cpages, npages);
1614         ASSERT_EQ(buffer->faults, 1);
1615
1616         /* Check what device 0 read. */
1617         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1618                 ASSERT_EQ(ptr[i], i);
1619
1620         hmm_buffer_free(buffer);
1621 }
1622
1623 /*
1624  * Basic check of exclusive faulting.
1625  */
1626 TEST_F(hmm, exclusive)
1627 {
1628         struct hmm_buffer *buffer;
1629         unsigned long npages;
1630         unsigned long size;
1631         unsigned long i;
1632         int *ptr;
1633         int ret;
1634
1635         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1636         ASSERT_NE(npages, 0);
1637         size = npages << self->page_shift;
1638
1639         buffer = malloc(sizeof(*buffer));
1640         ASSERT_NE(buffer, NULL);
1641
1642         buffer->fd = -1;
1643         buffer->size = size;
1644         buffer->mirror = malloc(size);
1645         ASSERT_NE(buffer->mirror, NULL);
1646
1647         buffer->ptr = mmap(NULL, size,
1648                            PROT_READ | PROT_WRITE,
1649                            MAP_PRIVATE | MAP_ANONYMOUS,
1650                            buffer->fd, 0);
1651         ASSERT_NE(buffer->ptr, MAP_FAILED);
1652
1653         /* Initialize buffer in system memory. */
1654         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1655                 ptr[i] = i;
1656
1657         /* Map memory exclusively for device access. */
1658         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1659         ASSERT_EQ(ret, 0);
1660         ASSERT_EQ(buffer->cpages, npages);
1661
1662         /* Check what the device read. */
1663         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1664                 ASSERT_EQ(ptr[i], i);
1665
1666         /* Fault pages back to system memory and check them. */
1667         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1668                 ASSERT_EQ(ptr[i]++, i);
1669
1670         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1671                 ASSERT_EQ(ptr[i], i+1);
1672
1673         /* Check atomic access revoked */
1674         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_CHECK_EXCLUSIVE, buffer, npages);
1675         ASSERT_EQ(ret, 0);
1676
1677         hmm_buffer_free(buffer);
1678 }
1679
1680 TEST_F(hmm, exclusive_mprotect)
1681 {
1682         struct hmm_buffer *buffer;
1683         unsigned long npages;
1684         unsigned long size;
1685         unsigned long i;
1686         int *ptr;
1687         int ret;
1688
1689         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1690         ASSERT_NE(npages, 0);
1691         size = npages << self->page_shift;
1692
1693         buffer = malloc(sizeof(*buffer));
1694         ASSERT_NE(buffer, NULL);
1695
1696         buffer->fd = -1;
1697         buffer->size = size;
1698         buffer->mirror = malloc(size);
1699         ASSERT_NE(buffer->mirror, NULL);
1700
1701         buffer->ptr = mmap(NULL, size,
1702                            PROT_READ | PROT_WRITE,
1703                            MAP_PRIVATE | MAP_ANONYMOUS,
1704                            buffer->fd, 0);
1705         ASSERT_NE(buffer->ptr, MAP_FAILED);
1706
1707         /* Initialize buffer in system memory. */
1708         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1709                 ptr[i] = i;
1710
1711         /* Map memory exclusively for device access. */
1712         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1713         ASSERT_EQ(ret, 0);
1714         ASSERT_EQ(buffer->cpages, npages);
1715
1716         /* Check what the device read. */
1717         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1718                 ASSERT_EQ(ptr[i], i);
1719
1720         ret = mprotect(buffer->ptr, size, PROT_READ);
1721         ASSERT_EQ(ret, 0);
1722
1723         /* Simulate a device writing system memory. */
1724         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
1725         ASSERT_EQ(ret, -EPERM);
1726
1727         hmm_buffer_free(buffer);
1728 }
1729
1730 /*
1731  * Check copy-on-write works.
1732  */
1733 TEST_F(hmm, exclusive_cow)
1734 {
1735         struct hmm_buffer *buffer;
1736         unsigned long npages;
1737         unsigned long size;
1738         unsigned long i;
1739         int *ptr;
1740         int ret;
1741
1742         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1743         ASSERT_NE(npages, 0);
1744         size = npages << self->page_shift;
1745
1746         buffer = malloc(sizeof(*buffer));
1747         ASSERT_NE(buffer, NULL);
1748
1749         buffer->fd = -1;
1750         buffer->size = size;
1751         buffer->mirror = malloc(size);
1752         ASSERT_NE(buffer->mirror, NULL);
1753
1754         buffer->ptr = mmap(NULL, size,
1755                            PROT_READ | PROT_WRITE,
1756                            MAP_PRIVATE | MAP_ANONYMOUS,
1757                            buffer->fd, 0);
1758         ASSERT_NE(buffer->ptr, MAP_FAILED);
1759
1760         /* Initialize buffer in system memory. */
1761         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1762                 ptr[i] = i;
1763
1764         /* Map memory exclusively for device access. */
1765         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1766         ASSERT_EQ(ret, 0);
1767         ASSERT_EQ(buffer->cpages, npages);
1768
1769         fork();
1770
1771         /* Fault pages back to system memory and check them. */
1772         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1773                 ASSERT_EQ(ptr[i]++, i);
1774
1775         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1776                 ASSERT_EQ(ptr[i], i+1);
1777
1778         hmm_buffer_free(buffer);
1779 }
1780
1781 static int gup_test_exec(int gup_fd, unsigned long addr, int cmd,
1782                          int npages, int size, int flags)
1783 {
1784         struct gup_test gup = {
1785                 .nr_pages_per_call      = npages,
1786                 .addr                   = addr,
1787                 .gup_flags              = FOLL_WRITE | flags,
1788                 .size                   = size,
1789         };
1790
1791         if (ioctl(gup_fd, cmd, &gup)) {
1792                 perror("ioctl on error\n");
1793                 return errno;
1794         }
1795
1796         return 0;
1797 }
1798
1799 /*
1800  * Test get user device pages through gup_test. Setting PIN_LONGTERM flag.
1801  * This should trigger a migration back to system memory for both, private
1802  * and coherent type pages.
1803  * This test makes use of gup_test module. Make sure GUP_TEST_CONFIG is added
1804  * to your configuration before you run it.
1805  */
1806 TEST_F(hmm, hmm_gup_test)
1807 {
1808         struct hmm_buffer *buffer;
1809         int gup_fd;
1810         unsigned long npages;
1811         unsigned long size;
1812         unsigned long i;
1813         int *ptr;
1814         int ret;
1815         unsigned char *m;
1816
1817         gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
1818         if (gup_fd == -1)
1819                 SKIP(return, "Skipping test, could not find gup_test driver");
1820
1821         npages = 4;
1822         size = npages << self->page_shift;
1823
1824         buffer = malloc(sizeof(*buffer));
1825         ASSERT_NE(buffer, NULL);
1826
1827         buffer->fd = -1;
1828         buffer->size = size;
1829         buffer->mirror = malloc(size);
1830         ASSERT_NE(buffer->mirror, NULL);
1831
1832         buffer->ptr = mmap(NULL, size,
1833                            PROT_READ | PROT_WRITE,
1834                            MAP_PRIVATE | MAP_ANONYMOUS,
1835                            buffer->fd, 0);
1836         ASSERT_NE(buffer->ptr, MAP_FAILED);
1837
1838         /* Initialize buffer in system memory. */
1839         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1840                 ptr[i] = i;
1841
1842         /* Migrate memory to device. */
1843         ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1844         ASSERT_EQ(ret, 0);
1845         ASSERT_EQ(buffer->cpages, npages);
1846         /* Check what the device read. */
1847         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1848                 ASSERT_EQ(ptr[i], i);
1849
1850         ASSERT_EQ(gup_test_exec(gup_fd,
1851                                 (unsigned long)buffer->ptr,
1852                                 GUP_BASIC_TEST, 1, self->page_size, 0), 0);
1853         ASSERT_EQ(gup_test_exec(gup_fd,
1854                                 (unsigned long)buffer->ptr + 1 * self->page_size,
1855                                 GUP_FAST_BENCHMARK, 1, self->page_size, 0), 0);
1856         ASSERT_EQ(gup_test_exec(gup_fd,
1857                                 (unsigned long)buffer->ptr + 2 * self->page_size,
1858                                 PIN_FAST_BENCHMARK, 1, self->page_size, FOLL_LONGTERM), 0);
1859         ASSERT_EQ(gup_test_exec(gup_fd,
1860                                 (unsigned long)buffer->ptr + 3 * self->page_size,
1861                                 PIN_LONGTERM_BENCHMARK, 1, self->page_size, 0), 0);
1862
1863         /* Take snapshot to CPU pagetables */
1864         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1865         ASSERT_EQ(ret, 0);
1866         ASSERT_EQ(buffer->cpages, npages);
1867         m = buffer->mirror;
1868         if (hmm_is_coherent_type(variant->device_number)) {
1869                 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[0]);
1870                 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[1]);
1871         } else {
1872                 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[0]);
1873                 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[1]);
1874         }
1875         ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[2]);
1876         ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[3]);
1877         /*
1878          * Check again the content on the pages. Make sure there's no
1879          * corrupted data.
1880          */
1881         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1882                 ASSERT_EQ(ptr[i], i);
1883
1884         close(gup_fd);
1885         hmm_buffer_free(buffer);
1886 }
1887
1888 /*
1889  * Test copy-on-write in device pages.
1890  * In case of writing to COW private page(s), a page fault will migrate pages
1891  * back to system memory first. Then, these pages will be duplicated. In case
1892  * of COW device coherent type, pages are duplicated directly from device
1893  * memory.
1894  */
1895 TEST_F(hmm, hmm_cow_in_device)
1896 {
1897         struct hmm_buffer *buffer;
1898         unsigned long npages;
1899         unsigned long size;
1900         unsigned long i;
1901         int *ptr;
1902         int ret;
1903         unsigned char *m;
1904         pid_t pid;
1905         int status;
1906
1907         npages = 4;
1908         size = npages << self->page_shift;
1909
1910         buffer = malloc(sizeof(*buffer));
1911         ASSERT_NE(buffer, NULL);
1912
1913         buffer->fd = -1;
1914         buffer->size = size;
1915         buffer->mirror = malloc(size);
1916         ASSERT_NE(buffer->mirror, NULL);
1917
1918         buffer->ptr = mmap(NULL, size,
1919                            PROT_READ | PROT_WRITE,
1920                            MAP_PRIVATE | MAP_ANONYMOUS,
1921                            buffer->fd, 0);
1922         ASSERT_NE(buffer->ptr, MAP_FAILED);
1923
1924         /* Initialize buffer in system memory. */
1925         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1926                 ptr[i] = i;
1927
1928         /* Migrate memory to device. */
1929
1930         ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1931         ASSERT_EQ(ret, 0);
1932         ASSERT_EQ(buffer->cpages, npages);
1933
1934         pid = fork();
1935         if (pid == -1)
1936                 ASSERT_EQ(pid, 0);
1937         if (!pid) {
1938                 /* Child process waitd for SIGTERM from the parent. */
1939                 while (1) {
1940                 }
1941                 perror("Should not reach this\n");
1942                 exit(0);
1943         }
1944         /* Parent process writes to COW pages(s) and gets a
1945          * new copy in system. In case of device private pages,
1946          * this write causes a migration to system mem first.
1947          */
1948         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1949                 ptr[i] = i;
1950
1951         /* Terminate child and wait */
1952         EXPECT_EQ(0, kill(pid, SIGTERM));
1953         EXPECT_EQ(pid, waitpid(pid, &status, 0));
1954         EXPECT_NE(0, WIFSIGNALED(status));
1955         EXPECT_EQ(SIGTERM, WTERMSIG(status));
1956
1957         /* Take snapshot to CPU pagetables */
1958         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1959         ASSERT_EQ(ret, 0);
1960         ASSERT_EQ(buffer->cpages, npages);
1961         m = buffer->mirror;
1962         for (i = 0; i < npages; i++)
1963                 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[i]);
1964
1965         hmm_buffer_free(buffer);
1966 }
1967 TEST_HARNESS_MAIN