tools/testing/selftests/vm/ksm_tests.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <sys/mman.h>
   4 #include <stdbool.h>
   5 #include <time.h>
   6 #include <string.h>
   7 #include <numa.h>
   8 #include <unistd.h>
   9 #include <fcntl.h>
  10 #include <stdint.h>
  11 #include <err.h>
  12
  13 #include "../kselftest.h"
  14 #include "../../../../include/vdso/time64.h"
  15 #include "util.h"
  16
  17 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/"
  18 #define KSM_FP(s) (KSM_SYSFS_PATH s)
  19 #define KSM_SCAN_LIMIT_SEC_DEFAULT 120
  20 #define KSM_PAGE_COUNT_DEFAULT 10l
  21 #define KSM_PROT_STR_DEFAULT "rw"
  22 #define KSM_USE_ZERO_PAGES_DEFAULT false
  23 #define KSM_MERGE_ACROSS_NODES_DEFAULT true
  24 #define MB (1ul << 20)
  25
  26 struct ksm_sysfs {
  27         unsigned long max_page_sharing;
  28         unsigned long merge_across_nodes;
  29         unsigned long pages_to_scan;
  30         unsigned long run;
  31         unsigned long sleep_millisecs;
  32         unsigned long stable_node_chains_prune_millisecs;
  33         unsigned long use_zero_pages;
  34 };
  35
  36 enum ksm_test_name {
  37         CHECK_KSM_MERGE,
  38         CHECK_KSM_UNMERGE,
  39         CHECK_KSM_ZERO_PAGE_MERGE,
  40         CHECK_KSM_NUMA_MERGE,
  41         KSM_MERGE_TIME,
  42         KSM_MERGE_TIME_HUGE_PAGES,
  43         KSM_COW_TIME
  44 };
  45
  46 static int ksm_write_sysfs(const char *file_path, unsigned long val)
  47 {
  48         FILE *f = fopen(file_path, "w");
  49
  50         if (!f) {
  51                 fprintf(stderr, "f %s\n", file_path);
  52                 perror("fopen");
  53                 return 1;
  54         }
  55         if (fprintf(f, "%lu", val) < 0) {
  56                 perror("fprintf");
  57                 fclose(f);
  58                 return 1;
  59         }
  60         fclose(f);
  61
  62         return 0;
  63 }
  64
  65 static int ksm_read_sysfs(const char *file_path, unsigned long *val)
  66 {
  67         FILE *f = fopen(file_path, "r");
  68
  69         if (!f) {
  70                 fprintf(stderr, "f %s\n", file_path);
  71                 perror("fopen");
  72                 return 1;
  73         }
  74         if (fscanf(f, "%lu", val) != 1) {
  75                 perror("fscanf");
  76                 fclose(f);
  77                 return 1;
  78         }
  79         fclose(f);
  80
  81         return 0;
  82 }
  83
  84 static int str_to_prot(char *prot_str)
  85 {
  86         int prot = 0;
  87
  88         if ((strchr(prot_str, 'r')) != NULL)
  89                 prot |= PROT_READ;
  90         if ((strchr(prot_str, 'w')) != NULL)
  91                 prot |= PROT_WRITE;
  92         if ((strchr(prot_str, 'x')) != NULL)
  93                 prot |= PROT_EXEC;
  94
  95         return prot;
  96 }
  97
  98 static void print_help(void)
  99 {
 100         printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n"
 101                "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n");
 102
 103         printf("Supported <test type>:\n"
 104                " -M (page merging)\n"
 105                " -Z (zero pages merging)\n"
 106                " -N (merging of pages in different NUMA nodes)\n"
 107                " -U (page unmerging)\n"
 108                " -P evaluate merging time and speed.\n"
 109                "    For this test, the size of duplicated memory area (in MiB)\n"
 110                "    must be provided using -s option\n"
 111                                  " -H evaluate merging time and speed of area allocated mostly with huge pages\n"
 112                "    For this test, the size of duplicated memory area (in MiB)\n"
 113                "    must be provided using -s option\n"
 114                " -C evaluate the time required to break COW of merged pages.\n\n");
 115
 116         printf(" -a: specify the access protections of pages.\n"
 117                "     <prot> must be of the form [rwx].\n"
 118                "     Default: %s\n", KSM_PROT_STR_DEFAULT);
 119         printf(" -p: specify the number of pages to test.\n"
 120                "     Default: %ld\n", KSM_PAGE_COUNT_DEFAULT);
 121         printf(" -l: limit the maximum running time (in seconds) for a test.\n"
 122                "     Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT);
 123         printf(" -z: change use_zero_pages tunable\n"
 124                "     Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT);
 125         printf(" -m: change merge_across_nodes tunable\n"
 126                "     Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT);
 127         printf(" -s: the size of duplicated memory area (in MiB)\n");
 128
 129         exit(0);
 130 }
 131
 132 static void  *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size)
 133 {
 134         void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0);
 135
 136         if (!map_ptr) {
 137                 perror("mmap");
 138                 return NULL;
 139         }
 140         memset(map_ptr, data, map_size);
 141         if (mprotect(map_ptr, map_size, prot)) {
 142                 perror("mprotect");
 143                 munmap(map_ptr, map_size);
 144                 return NULL;
 145         }
 146
 147         return map_ptr;
 148 }
 149
 150 static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout)
 151 {
 152         struct timespec cur_time;
 153         unsigned long cur_scan, init_scan;
 154
 155         if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan))
 156                 return 1;
 157         cur_scan = init_scan;
 158
 159         while (cur_scan < init_scan + scan_count) {
 160                 if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan))
 161                         return 1;
 162                 if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) {
 163                         perror("clock_gettime");
 164                         return 1;
 165                 }
 166                 if ((cur_time.tv_sec - start_time.tv_sec) > timeout) {
 167                         printf("Scan time limit exceeded\n");
 168                         return 1;
 169                 }
 170         }
 171
 172         return 0;
 173 }
 174
 175 static int ksm_merge_pages(void *addr, size_t size, struct timespec start_time, int timeout)
 176 {
 177         if (madvise(addr, size, MADV_MERGEABLE)) {
 178                 perror("madvise");
 179                 return 1;
 180         }
 181         if (ksm_write_sysfs(KSM_FP("run"), 1))
 182                 return 1;
 183
 184         /* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */
 185         if (ksm_do_scan(2, start_time, timeout))
 186                 return 1;
 187
 188         return 0;
 189 }
 190
 191 static bool assert_ksm_pages_count(long dupl_page_count)
 192 {
 193         unsigned long max_page_sharing, pages_sharing, pages_shared;
 194
 195         if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
 196             ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
 197             ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing))
 198                 return false;
 199
 200         /*
 201          * Since there must be at least 2 pages for merging and 1 page can be
 202          * shared with the limited number of pages (max_page_sharing), sometimes
 203          * there are 'leftover' pages that cannot be merged. For example, if there
 204          * are 11 pages and max_page_sharing = 10, then only 10 pages will be
 205          * merged and the 11th page won't be affected. As a result, when the number
 206          * of duplicate pages is divided by max_page_sharing and the remainder is 1,
 207          * pages_shared and pages_sharing values will be equal between dupl_page_count
 208          * and dupl_page_count - 1.
 209          */
 210         if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) {
 211                 if (pages_shared == dupl_page_count / max_page_sharing &&
 212                     pages_sharing == pages_shared * (max_page_sharing - 1))
 213                         return true;
 214         } else {
 215                 if (pages_shared == (dupl_page_count / max_page_sharing + 1) &&
 216                     pages_sharing == dupl_page_count - pages_shared)
 217                         return true;
 218         }
 219
 220         return false;
 221 }
 222
 223 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs)
 224 {
 225         if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) ||
 226             numa_available() ? 0 :
 227                 ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) ||
 228             ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) ||
 229             ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) ||
 230             ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) ||
 231             ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
 232                            &ksm_sysfs->stable_node_chains_prune_millisecs) ||
 233             ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages))
 234                 return 1;
 235
 236         return 0;
 237 }
 238
 239 static int ksm_restore(struct ksm_sysfs *ksm_sysfs)
 240 {
 241         if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) ||
 242             numa_available() ? 0 :
 243                 ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) ||
 244             ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) ||
 245             ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) ||
 246             ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) ||
 247             ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
 248                             ksm_sysfs->stable_node_chains_prune_millisecs) ||
 249             ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages))
 250                 return 1;
 251
 252         return 0;
 253 }
 254
 255 static int check_ksm_merge(int mapping, int prot, long page_count, int timeout, size_t page_size)
 256 {
 257         void *map_ptr;
 258         struct timespec start_time;
 259
 260         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 261                 perror("clock_gettime");
 262                 return KSFT_FAIL;
 263         }
 264
 265         /* fill pages with the same data and merge them */
 266         map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
 267         if (!map_ptr)
 268                 return KSFT_FAIL;
 269
 270         if (ksm_merge_pages(map_ptr, page_size * page_count, start_time, timeout))
 271                 goto err_out;
 272
 273         /* verify that the right number of pages are merged */
 274         if (assert_ksm_pages_count(page_count)) {
 275                 printf("OK\n");
 276                 munmap(map_ptr, page_size * page_count);
 277                 return KSFT_PASS;
 278         }
 279
 280 err_out:
 281         printf("Not OK\n");
 282         munmap(map_ptr, page_size * page_count);
 283         return KSFT_FAIL;
 284 }
 285
 286 static int check_ksm_unmerge(int mapping, int prot, int timeout, size_t page_size)
 287 {
 288         void *map_ptr;
 289         struct timespec start_time;
 290         int page_count = 2;
 291
 292         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 293                 perror("clock_gettime");
 294                 return KSFT_FAIL;
 295         }
 296
 297         /* fill pages with the same data and merge them */
 298         map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
 299         if (!map_ptr)
 300                 return KSFT_FAIL;
 301
 302         if (ksm_merge_pages(map_ptr, page_size * page_count, start_time, timeout))
 303                 goto err_out;
 304
 305         /* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */
 306         memset(map_ptr, '-', 1);
 307         memset(map_ptr + page_size, '+', 1);
 308
 309         /* get at least 1 scan, so KSM can detect that the pages were modified */
 310         if (ksm_do_scan(1, start_time, timeout))
 311                 goto err_out;
 312
 313         /* check that unmerging was successful and 0 pages are currently merged */
 314         if (assert_ksm_pages_count(0)) {
 315                 printf("OK\n");
 316                 munmap(map_ptr, page_size * page_count);
 317                 return KSFT_PASS;
 318         }
 319
 320 err_out:
 321         printf("Not OK\n");
 322         munmap(map_ptr, page_size * page_count);
 323         return KSFT_FAIL;
 324 }
 325
 326 static int check_ksm_zero_page_merge(int mapping, int prot, long page_count, int timeout,
 327                                      bool use_zero_pages, size_t page_size)
 328 {
 329         void *map_ptr;
 330         struct timespec start_time;
 331
 332         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 333                 perror("clock_gettime");
 334                 return KSFT_FAIL;
 335         }
 336
 337         if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages))
 338                 return KSFT_FAIL;
 339
 340         /* fill pages with zero and try to merge them */
 341         map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count);
 342         if (!map_ptr)
 343                 return KSFT_FAIL;
 344
 345         if (ksm_merge_pages(map_ptr, page_size * page_count, start_time, timeout))
 346                 goto err_out;
 347
 348        /*
 349         * verify that the right number of pages are merged:
 350         * 1) if use_zero_pages is set to 1, empty pages are merged
 351         *    with the kernel zero page instead of with each other;
 352         * 2) if use_zero_pages is set to 0, empty pages are not treated specially
 353         *    and merged as usual.
 354         */
 355         if (use_zero_pages && !assert_ksm_pages_count(0))
 356                 goto err_out;
 357         else if (!use_zero_pages && !assert_ksm_pages_count(page_count))
 358                 goto err_out;
 359
 360         printf("OK\n");
 361         munmap(map_ptr, page_size * page_count);
 362         return KSFT_PASS;
 363
 364 err_out:
 365         printf("Not OK\n");
 366         munmap(map_ptr, page_size * page_count);
 367         return KSFT_FAIL;
 368 }
 369
 370 static int get_next_mem_node(int node)
 371 {
 372
 373         long node_size;
 374         int mem_node = 0;
 375         int i, max_node = numa_max_node();
 376
 377         for (i = node + 1; i <= max_node + node; i++) {
 378                 mem_node = i % (max_node + 1);
 379                 node_size = numa_node_size(mem_node, NULL);
 380                 if (node_size > 0)
 381                         break;
 382         }
 383         return mem_node;
 384 }
 385
 386 static int get_first_mem_node(void)
 387 {
 388         return get_next_mem_node(numa_max_node());
 389 }
 390
 391 static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_across_nodes,
 392                                 size_t page_size)
 393 {
 394         void *numa1_map_ptr, *numa2_map_ptr;
 395         struct timespec start_time;
 396         int page_count = 2;
 397         int first_node;
 398
 399         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 400                 perror("clock_gettime");
 401                 return KSFT_FAIL;
 402         }
 403
 404         if (numa_available() < 0) {
 405                 perror("NUMA support not enabled");
 406                 return KSFT_SKIP;
 407         }
 408         if (numa_num_configured_nodes() <= 1) {
 409                 printf("At least 2 NUMA nodes must be available\n");
 410                 return KSFT_SKIP;
 411         }
 412         if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes))
 413                 return KSFT_FAIL;
 414
 415         /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
 416         first_node = get_first_mem_node();
 417         numa1_map_ptr = numa_alloc_onnode(page_size, first_node);
 418         numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));
 419         if (!numa1_map_ptr || !numa2_map_ptr) {
 420                 perror("numa_alloc_onnode");
 421                 return KSFT_FAIL;
 422         }
 423
 424         memset(numa1_map_ptr, '*', page_size);
 425         memset(numa2_map_ptr, '*', page_size);
 426
 427         /* try to merge the pages */
 428         if (ksm_merge_pages(numa1_map_ptr, page_size, start_time, timeout) ||
 429             ksm_merge_pages(numa2_map_ptr, page_size, start_time, timeout))
 430                 goto err_out;
 431
 432        /*
 433         * verify that the right number of pages are merged:
 434         * 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged;
 435         * 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is
 436         *    only 1 unique page in each node and they can't be shared.
 437         */
 438         if (merge_across_nodes && !assert_ksm_pages_count(page_count))
 439                 goto err_out;
 440         else if (!merge_across_nodes && !assert_ksm_pages_count(0))
 441                 goto err_out;
 442
 443         numa_free(numa1_map_ptr, page_size);
 444         numa_free(numa2_map_ptr, page_size);
 445         printf("OK\n");
 446         return KSFT_PASS;
 447
 448 err_out:
 449         numa_free(numa1_map_ptr, page_size);
 450         numa_free(numa2_map_ptr, page_size);
 451         printf("Not OK\n");
 452         return KSFT_FAIL;
 453 }
 454
 455 static int ksm_merge_hugepages_time(int mapping, int prot, int timeout, size_t map_size)
 456 {
 457         void *map_ptr, *map_ptr_orig;
 458         struct timespec start_time, end_time;
 459         unsigned long scan_time_ns;
 460         int pagemap_fd, n_normal_pages, n_huge_pages;
 461
 462         map_size *= MB;
 463         size_t len = map_size;
 464
 465         len -= len % HPAGE_SIZE;
 466         map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE,
 467                         MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0);
 468         map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE;
 469
 470         if (map_ptr_orig == MAP_FAILED)
 471                 err(2, "initial mmap");
 472
 473         if (madvise(map_ptr, len + HPAGE_SIZE, MADV_HUGEPAGE))
 474                 err(2, "MADV_HUGEPAGE");
 475
 476         pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
 477         if (pagemap_fd < 0)
 478                 err(2, "open pagemap");
 479
 480         n_normal_pages = 0;
 481         n_huge_pages = 0;
 482         for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) {
 483                 if (allocate_transhuge(p, pagemap_fd) < 0)
 484                         n_normal_pages++;
 485                 else
 486                         n_huge_pages++;
 487         }
 488         printf("Number of normal pages:    %d\n", n_normal_pages);
 489         printf("Number of huge pages:    %d\n", n_huge_pages);
 490
 491         memset(map_ptr, '*', len);
 492
 493         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 494                 perror("clock_gettime");
 495                 goto err_out;
 496         }
 497         if (ksm_merge_pages(map_ptr, map_size, start_time, timeout))
 498                 goto err_out;
 499         if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
 500                 perror("clock_gettime");
 501                 goto err_out;
 502         }
 503
 504         scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
 505                        (end_time.tv_nsec - start_time.tv_nsec);
 506
 507         printf("Total size:    %lu MiB\n", map_size / MB);
 508         printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
 509                scan_time_ns % NSEC_PER_SEC);
 510         printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
 511                                                ((double)scan_time_ns / NSEC_PER_SEC));
 512
 513         munmap(map_ptr_orig, len + HPAGE_SIZE);
 514         return KSFT_PASS;
 515
 516 err_out:
 517         printf("Not OK\n");
 518         munmap(map_ptr_orig, len + HPAGE_SIZE);
 519         return KSFT_FAIL;
 520 }
 521
 522 static int ksm_merge_time(int mapping, int prot, int timeout, size_t map_size)
 523 {
 524         void *map_ptr;
 525         struct timespec start_time, end_time;
 526         unsigned long scan_time_ns;
 527
 528         map_size *= MB;
 529
 530         map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
 531         if (!map_ptr)
 532                 return KSFT_FAIL;
 533
 534         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 535                 perror("clock_gettime");
 536                 goto err_out;
 537         }
 538         if (ksm_merge_pages(map_ptr, map_size, start_time, timeout))
 539                 goto err_out;
 540         if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
 541                 perror("clock_gettime");
 542                 goto err_out;
 543         }
 544
 545         scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
 546                        (end_time.tv_nsec - start_time.tv_nsec);
 547
 548         printf("Total size:    %lu MiB\n", map_size / MB);
 549         printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
 550                scan_time_ns % NSEC_PER_SEC);
 551         printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
 552                                                ((double)scan_time_ns / NSEC_PER_SEC));
 553
 554         munmap(map_ptr, map_size);
 555         return KSFT_PASS;
 556
 557 err_out:
 558         printf("Not OK\n");
 559         munmap(map_ptr, map_size);
 560         return KSFT_FAIL;
 561 }
 562
 563 static int ksm_cow_time(int mapping, int prot, int timeout, size_t page_size)
 564 {
 565         void *map_ptr;
 566         struct timespec start_time, end_time;
 567         unsigned long cow_time_ns;
 568
 569         /* page_count must be less than 2*page_size */
 570         size_t page_count = 4000;
 571
 572         map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
 573         if (!map_ptr)
 574                 return KSFT_FAIL;
 575
 576         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 577                 perror("clock_gettime");
 578                 return KSFT_FAIL;
 579         }
 580         for (size_t i = 0; i < page_count - 1; i = i + 2)
 581                 memset(map_ptr + page_size * i, '-', 1);
 582         if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
 583                 perror("clock_gettime");
 584                 return KSFT_FAIL;
 585         }
 586
 587         cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
 588                        (end_time.tv_nsec - start_time.tv_nsec);
 589
 590         printf("Total size:    %lu MiB\n\n", (page_size * page_count) / MB);
 591         printf("Not merged pages:\n");
 592         printf("Total time:     %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
 593                cow_time_ns % NSEC_PER_SEC);
 594         printf("Average speed:  %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) /
 595                                                ((double)cow_time_ns / NSEC_PER_SEC));
 596
 597         /* Create 2000 pairs of duplicate pages */
 598         for (size_t i = 0; i < page_count - 1; i = i + 2) {
 599                 memset(map_ptr + page_size * i, '+', i / 2 + 1);
 600                 memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1);
 601         }
 602         if (ksm_merge_pages(map_ptr, page_size * page_count, start_time, timeout))
 603                 goto err_out;
 604
 605         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 606                 perror("clock_gettime");
 607                 goto err_out;
 608         }
 609         for (size_t i = 0; i < page_count - 1; i = i + 2)
 610                 memset(map_ptr + page_size * i, '-', 1);
 611         if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
 612                 perror("clock_gettime");
 613                 goto err_out;
 614         }
 615
 616         cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
 617                        (end_time.tv_nsec - start_time.tv_nsec);
 618
 619         printf("Merged pages:\n");
 620         printf("Total time:     %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
 621                cow_time_ns % NSEC_PER_SEC);
 622         printf("Average speed:  %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) /
 623                                                ((double)cow_time_ns / NSEC_PER_SEC));
 624
 625         munmap(map_ptr, page_size * page_count);
 626         return KSFT_PASS;
 627
 628 err_out:
 629         printf("Not OK\n");
 630         munmap(map_ptr, page_size * page_count);
 631         return KSFT_FAIL;
 632 }
 633
 634 int main(int argc, char *argv[])
 635 {
 636         int ret, opt;
 637         int prot = 0;
 638         int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT;
 639         long page_count = KSM_PAGE_COUNT_DEFAULT;
 640         size_t page_size = sysconf(_SC_PAGESIZE);
 641         struct ksm_sysfs ksm_sysfs_old;
 642         int test_name = CHECK_KSM_MERGE;
 643         bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT;
 644         bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT;
 645         long size_MB = 0;
 646
 647         while ((opt = getopt(argc, argv, "ha:p:l:z:m:s:MUZNPCH")) != -1) {
 648                 switch (opt) {
 649                 case 'a':
 650                         prot = str_to_prot(optarg);
 651                         break;
 652                 case 'p':
 653                         page_count = atol(optarg);
 654                         if (page_count <= 0) {
 655                                 printf("The number of pages must be greater than 0\n");
 656                                 return KSFT_FAIL;
 657                         }
 658                         break;
 659                 case 'l':
 660                         ksm_scan_limit_sec = atoi(optarg);
 661                         if (ksm_scan_limit_sec <= 0) {
 662                                 printf("Timeout value must be greater than 0\n");
 663                                 return KSFT_FAIL;
 664                         }
 665                         break;
 666                 case 'h':
 667                         print_help();
 668                         break;
 669                 case 'z':
 670                         if (strcmp(optarg, "0") == 0)
 671                                 use_zero_pages = 0;
 672                         else
 673                                 use_zero_pages = 1;
 674                         break;
 675                 case 'm':
 676                         if (strcmp(optarg, "0") == 0)
 677                                 merge_across_nodes = 0;
 678                         else
 679                                 merge_across_nodes = 1;
 680                         break;
 681                 case 's':
 682                         size_MB = atoi(optarg);
 683                         if (size_MB <= 0) {
 684                                 printf("Size must be greater than 0\n");
 685                                 return KSFT_FAIL;
 686                         }
 687                 case 'M':
 688                         break;
 689                 case 'U':
 690                         test_name = CHECK_KSM_UNMERGE;
 691                         break;
 692                 case 'Z':
 693                         test_name = CHECK_KSM_ZERO_PAGE_MERGE;
 694                         break;
 695                 case 'N':
 696                         test_name = CHECK_KSM_NUMA_MERGE;
 697                         break;
 698                 case 'P':
 699                         test_name = KSM_MERGE_TIME;
 700                         break;
 701                 case 'H':
 702                         test_name = KSM_MERGE_TIME_HUGE_PAGES;
 703                         break;
 704                 case 'C':
 705                         test_name = KSM_COW_TIME;
 706                         break;
 707                 default:
 708                         return KSFT_FAIL;
 709                 }
 710         }
 711
 712         if (prot == 0)
 713                 prot = str_to_prot(KSM_PROT_STR_DEFAULT);
 714
 715         if (access(KSM_SYSFS_PATH, F_OK)) {
 716                 printf("Config KSM not enabled\n");
 717                 return KSFT_SKIP;
 718         }
 719
 720         if (ksm_save_def(&ksm_sysfs_old)) {
 721                 printf("Cannot save default tunables\n");
 722                 return KSFT_FAIL;
 723         }
 724
 725         if (ksm_write_sysfs(KSM_FP("run"), 2) ||
 726             ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) ||
 727             numa_available() ? 0 :
 728                 ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) ||
 729             ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count))
 730                 return KSFT_FAIL;
 731
 732         switch (test_name) {
 733         case CHECK_KSM_MERGE:
 734                 ret = check_ksm_merge(MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count,
 735                                       ksm_scan_limit_sec, page_size);
 736                 break;
 737         case CHECK_KSM_UNMERGE:
 738                 ret = check_ksm_unmerge(MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec,
 739                                         page_size);
 740                 break;
 741         case CHECK_KSM_ZERO_PAGE_MERGE:
 742                 ret = check_ksm_zero_page_merge(MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count,
 743                                                 ksm_scan_limit_sec, use_zero_pages, page_size);
 744                 break;
 745         case CHECK_KSM_NUMA_MERGE:
 746                 ret = check_ksm_numa_merge(MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec,
 747                                            merge_across_nodes, page_size);
 748                 break;
 749         case KSM_MERGE_TIME:
 750                 if (size_MB == 0) {
 751                         printf("Option '-s' is required.\n");
 752                         return KSFT_FAIL;
 753                 }
 754                 ret = ksm_merge_time(MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec,
 755                                      size_MB);
 756                 break;
 757         case KSM_MERGE_TIME_HUGE_PAGES:
 758                 if (size_MB == 0) {
 759                         printf("Option '-s' is required.\n");
 760                         return KSFT_FAIL;
 761                 }
 762                 ret = ksm_merge_hugepages_time(MAP_PRIVATE | MAP_ANONYMOUS, prot,
 763                                 ksm_scan_limit_sec, size_MB);
 764                 break;
 765         case KSM_COW_TIME:
 766                 ret = ksm_cow_time(MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec,
 767                                    page_size);
 768                 break;
 769         }
 770
 771         if (ksm_restore(&ksm_sysfs_old)) {
 772                 printf("Cannot restore default tunables\n");
 773                 return KSFT_FAIL;
 774         }
 775
 776         return ret;
 777 }