tools/testing/selftests/mm/ksm_tests.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <sys/mman.h>
   4 #include <sys/prctl.h>
   5 #include <sys/wait.h>
   6 #include <stdbool.h>
   7 #include <time.h>
   8 #include <string.h>
   9 #include <numa.h>
  10 #include <unistd.h>
  11 #include <fcntl.h>
  12 #include <stdint.h>
  13 #include <err.h>
  14
  15 #include "../kselftest.h"
  16 #include <include/vdso/time64.h>
  17 #include "vm_util.h"
  18
  19 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/"
  20 #define KSM_FP(s) (KSM_SYSFS_PATH s)
  21 #define KSM_SCAN_LIMIT_SEC_DEFAULT 120
  22 #define KSM_PAGE_COUNT_DEFAULT 10l
  23 #define KSM_PROT_STR_DEFAULT "rw"
  24 #define KSM_USE_ZERO_PAGES_DEFAULT false
  25 #define KSM_MERGE_ACROSS_NODES_DEFAULT true
  26 #define KSM_MERGE_TYPE_DEFAULT 0
  27 #define MB (1ul << 20)
  28
  29 struct ksm_sysfs {
  30         unsigned long max_page_sharing;
  31         unsigned long merge_across_nodes;
  32         unsigned long pages_to_scan;
  33         unsigned long run;
  34         unsigned long sleep_millisecs;
  35         unsigned long stable_node_chains_prune_millisecs;
  36         unsigned long use_zero_pages;
  37 };
  38
  39 enum ksm_merge_type {
  40         KSM_MERGE_MADVISE,
  41         KSM_MERGE_PRCTL,
  42         KSM_MERGE_LAST = KSM_MERGE_PRCTL
  43 };
  44
  45 enum ksm_test_name {
  46         CHECK_KSM_MERGE,
  47         CHECK_KSM_UNMERGE,
  48         CHECK_KSM_GET_MERGE_TYPE,
  49         CHECK_KSM_ZERO_PAGE_MERGE,
  50         CHECK_KSM_NUMA_MERGE,
  51         KSM_MERGE_TIME,
  52         KSM_MERGE_TIME_HUGE_PAGES,
  53         KSM_UNMERGE_TIME,
  54         KSM_COW_TIME
  55 };
  56
  57 int debug;
  58
  59 static int ksm_write_sysfs(const char *file_path, unsigned long val)
  60 {
  61         FILE *f = fopen(file_path, "w");
  62
  63         if (!f) {
  64                 fprintf(stderr, "f %s\n", file_path);
  65                 perror("fopen");
  66                 return 1;
  67         }
  68         if (fprintf(f, "%lu", val) < 0) {
  69                 perror("fprintf");
  70                 fclose(f);
  71                 return 1;
  72         }
  73         fclose(f);
  74
  75         return 0;
  76 }
  77
  78 static int ksm_read_sysfs(const char *file_path, unsigned long *val)
  79 {
  80         FILE *f = fopen(file_path, "r");
  81
  82         if (!f) {
  83                 fprintf(stderr, "f %s\n", file_path);
  84                 perror("fopen");
  85                 return 1;
  86         }
  87         if (fscanf(f, "%lu", val) != 1) {
  88                 perror("fscanf");
  89                 fclose(f);
  90                 return 1;
  91         }
  92         fclose(f);
  93
  94         return 0;
  95 }
  96
  97 static void ksm_print_sysfs(void)
  98 {
  99         unsigned long max_page_sharing, pages_sharing, pages_shared;
 100         unsigned long full_scans, pages_unshared, pages_volatile;
 101         unsigned long stable_node_chains, stable_node_dups;
 102         long general_profit;
 103
 104         if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
 105             ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
 106             ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing) ||
 107             ksm_read_sysfs(KSM_FP("full_scans"), &full_scans) ||
 108             ksm_read_sysfs(KSM_FP("pages_unshared"), &pages_unshared) ||
 109             ksm_read_sysfs(KSM_FP("pages_volatile"), &pages_volatile) ||
 110             ksm_read_sysfs(KSM_FP("stable_node_chains"), &stable_node_chains) ||
 111             ksm_read_sysfs(KSM_FP("stable_node_dups"), &stable_node_dups) ||
 112             ksm_read_sysfs(KSM_FP("general_profit"), (unsigned long *)&general_profit))
 113                 return;
 114
 115         printf("pages_shared      : %lu\n", pages_shared);
 116         printf("pages_sharing     : %lu\n", pages_sharing);
 117         printf("max_page_sharing  : %lu\n", max_page_sharing);
 118         printf("full_scans        : %lu\n", full_scans);
 119         printf("pages_unshared    : %lu\n", pages_unshared);
 120         printf("pages_volatile    : %lu\n", pages_volatile);
 121         printf("stable_node_chains: %lu\n", stable_node_chains);
 122         printf("stable_node_dups  : %lu\n", stable_node_dups);
 123         printf("general_profit    : %ld\n", general_profit);
 124 }
 125
 126 static void ksm_print_procfs(void)
 127 {
 128         const char *file_name = "/proc/self/ksm_stat";
 129         char buffer[512];
 130         FILE *f = fopen(file_name, "r");
 131
 132         if (!f) {
 133                 fprintf(stderr, "f %s\n", file_name);
 134                 perror("fopen");
 135                 return;
 136         }
 137
 138         while (fgets(buffer, sizeof(buffer), f))
 139                 printf("%s", buffer);
 140
 141         fclose(f);
 142 }
 143
 144 static int str_to_prot(char *prot_str)
 145 {
 146         int prot = 0;
 147
 148         if ((strchr(prot_str, 'r')) != NULL)
 149                 prot |= PROT_READ;
 150         if ((strchr(prot_str, 'w')) != NULL)
 151                 prot |= PROT_WRITE;
 152         if ((strchr(prot_str, 'x')) != NULL)
 153                 prot |= PROT_EXEC;
 154
 155         return prot;
 156 }
 157
 158 static void print_help(void)
 159 {
 160         printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n"
 161                "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n");
 162
 163         printf("Supported <test type>:\n"
 164                " -M (page merging)\n"
 165                " -Z (zero pages merging)\n"
 166                " -N (merging of pages in different NUMA nodes)\n"
 167                " -U (page unmerging)\n"
 168                " -P evaluate merging time and speed.\n"
 169                "    For this test, the size of duplicated memory area (in MiB)\n"
 170                "    must be provided using -s option\n"
 171                " -H evaluate merging time and speed of area allocated mostly with huge pages\n"
 172                "    For this test, the size of duplicated memory area (in MiB)\n"
 173                "    must be provided using -s option\n"
 174                " -D evaluate unmerging time and speed when disabling KSM.\n"
 175                "    For this test, the size of duplicated memory area (in MiB)\n"
 176                "    must be provided using -s option\n"
 177                " -C evaluate the time required to break COW of merged pages.\n\n");
 178
 179         printf(" -a: specify the access protections of pages.\n"
 180                "     <prot> must be of the form [rwx].\n"
 181                "     Default: %s\n", KSM_PROT_STR_DEFAULT);
 182         printf(" -p: specify the number of pages to test.\n"
 183                "     Default: %ld\n", KSM_PAGE_COUNT_DEFAULT);
 184         printf(" -l: limit the maximum running time (in seconds) for a test.\n"
 185                "     Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT);
 186         printf(" -z: change use_zero_pages tunable\n"
 187                "     Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT);
 188         printf(" -m: change merge_across_nodes tunable\n"
 189                "     Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT);
 190         printf(" -d: turn debugging output on\n");
 191         printf(" -s: the size of duplicated memory area (in MiB)\n");
 192         printf(" -t: KSM merge type\n"
 193                "     Default: 0\n"
 194                "     0: madvise merging\n"
 195                "     1: prctl merging\n");
 196
 197         exit(0);
 198 }
 199
 200 static void  *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size)
 201 {
 202         void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0);
 203
 204         if (!map_ptr) {
 205                 perror("mmap");
 206                 return NULL;
 207         }
 208         memset(map_ptr, data, map_size);
 209         if (mprotect(map_ptr, map_size, prot)) {
 210                 perror("mprotect");
 211                 munmap(map_ptr, map_size);
 212                 return NULL;
 213         }
 214
 215         return map_ptr;
 216 }
 217
 218 static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout)
 219 {
 220         struct timespec cur_time;
 221         unsigned long cur_scan, init_scan;
 222
 223         if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan))
 224                 return 1;
 225         cur_scan = init_scan;
 226
 227         while (cur_scan < init_scan + scan_count) {
 228                 if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan))
 229                         return 1;
 230                 if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) {
 231                         perror("clock_gettime");
 232                         return 1;
 233                 }
 234                 if ((cur_time.tv_sec - start_time.tv_sec) > timeout) {
 235                         printf("Scan time limit exceeded\n");
 236                         return 1;
 237                 }
 238         }
 239
 240         return 0;
 241 }
 242
 243 static int ksm_merge_pages(int merge_type, void *addr, size_t size,
 244                         struct timespec start_time, int timeout)
 245 {
 246         if (merge_type == KSM_MERGE_MADVISE) {
 247                 if (madvise(addr, size, MADV_MERGEABLE)) {
 248                         perror("madvise");
 249                         return 1;
 250                 }
 251         } else if (merge_type == KSM_MERGE_PRCTL) {
 252                 if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) {
 253                         perror("prctl");
 254                         return 1;
 255                 }
 256         }
 257
 258         if (ksm_write_sysfs(KSM_FP("run"), 1))
 259                 return 1;
 260
 261         /* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */
 262         if (ksm_do_scan(2, start_time, timeout))
 263                 return 1;
 264
 265         return 0;
 266 }
 267
 268 static int ksm_unmerge_pages(void *addr, size_t size,
 269                              struct timespec start_time, int timeout)
 270 {
 271         if (madvise(addr, size, MADV_UNMERGEABLE)) {
 272                 perror("madvise");
 273                 return 1;
 274         }
 275         return 0;
 276 }
 277
 278 static bool assert_ksm_pages_count(long dupl_page_count)
 279 {
 280         unsigned long max_page_sharing, pages_sharing, pages_shared;
 281
 282         if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
 283             ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
 284             ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing))
 285                 return false;
 286
 287         if (debug) {
 288                 ksm_print_sysfs();
 289                 ksm_print_procfs();
 290         }
 291
 292         /*
 293          * Since there must be at least 2 pages for merging and 1 page can be
 294          * shared with the limited number of pages (max_page_sharing), sometimes
 295          * there are 'leftover' pages that cannot be merged. For example, if there
 296          * are 11 pages and max_page_sharing = 10, then only 10 pages will be
 297          * merged and the 11th page won't be affected. As a result, when the number
 298          * of duplicate pages is divided by max_page_sharing and the remainder is 1,
 299          * pages_shared and pages_sharing values will be equal between dupl_page_count
 300          * and dupl_page_count - 1.
 301          */
 302         if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) {
 303                 if (pages_shared == dupl_page_count / max_page_sharing &&
 304                     pages_sharing == pages_shared * (max_page_sharing - 1))
 305                         return true;
 306         } else {
 307                 if (pages_shared == (dupl_page_count / max_page_sharing + 1) &&
 308                     pages_sharing == dupl_page_count - pages_shared)
 309                         return true;
 310         }
 311
 312         return false;
 313 }
 314
 315 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs)
 316 {
 317         if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) ||
 318             numa_available() ? 0 :
 319                 ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) ||
 320             ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) ||
 321             ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) ||
 322             ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) ||
 323             ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
 324                            &ksm_sysfs->stable_node_chains_prune_millisecs) ||
 325             ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages))
 326                 return 1;
 327
 328         return 0;
 329 }
 330
 331 static int ksm_restore(struct ksm_sysfs *ksm_sysfs)
 332 {
 333         if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) ||
 334             numa_available() ? 0 :
 335                 ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) ||
 336             ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) ||
 337             ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) ||
 338             ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) ||
 339             ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
 340                             ksm_sysfs->stable_node_chains_prune_millisecs) ||
 341             ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages))
 342                 return 1;
 343
 344         return 0;
 345 }
 346
 347 static int check_ksm_merge(int merge_type, int mapping, int prot,
 348                         long page_count, int timeout, size_t page_size)
 349 {
 350         void *map_ptr;
 351         struct timespec start_time;
 352
 353         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 354                 perror("clock_gettime");
 355                 return KSFT_FAIL;
 356         }
 357
 358         /* fill pages with the same data and merge them */
 359         map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
 360         if (!map_ptr)
 361                 return KSFT_FAIL;
 362
 363         if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
 364                 goto err_out;
 365
 366         /* verify that the right number of pages are merged */
 367         if (assert_ksm_pages_count(page_count)) {
 368                 printf("OK\n");
 369                 munmap(map_ptr, page_size * page_count);
 370                 if (merge_type == KSM_MERGE_PRCTL)
 371                         prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0);
 372                 return KSFT_PASS;
 373         }
 374
 375 err_out:
 376         printf("Not OK\n");
 377         munmap(map_ptr, page_size * page_count);
 378         return KSFT_FAIL;
 379 }
 380
 381 static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, size_t page_size)
 382 {
 383         void *map_ptr;
 384         struct timespec start_time;
 385         int page_count = 2;
 386
 387         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 388                 perror("clock_gettime");
 389                 return KSFT_FAIL;
 390         }
 391
 392         /* fill pages with the same data and merge them */
 393         map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
 394         if (!map_ptr)
 395                 return KSFT_FAIL;
 396
 397         if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
 398                 goto err_out;
 399
 400         /* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */
 401         memset(map_ptr, '-', 1);
 402         memset(map_ptr + page_size, '+', 1);
 403
 404         /* get at least 1 scan, so KSM can detect that the pages were modified */
 405         if (ksm_do_scan(1, start_time, timeout))
 406                 goto err_out;
 407
 408         /* check that unmerging was successful and 0 pages are currently merged */
 409         if (assert_ksm_pages_count(0)) {
 410                 printf("OK\n");
 411                 munmap(map_ptr, page_size * page_count);
 412                 return KSFT_PASS;
 413         }
 414
 415 err_out:
 416         printf("Not OK\n");
 417         munmap(map_ptr, page_size * page_count);
 418         return KSFT_FAIL;
 419 }
 420
 421 static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long page_count,
 422                                 int timeout, bool use_zero_pages, size_t page_size)
 423 {
 424         void *map_ptr;
 425         struct timespec start_time;
 426
 427         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 428                 perror("clock_gettime");
 429                 return KSFT_FAIL;
 430         }
 431
 432         if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages))
 433                 return KSFT_FAIL;
 434
 435         /* fill pages with zero and try to merge them */
 436         map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count);
 437         if (!map_ptr)
 438                 return KSFT_FAIL;
 439
 440         if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
 441                 goto err_out;
 442
 443        /*
 444         * verify that the right number of pages are merged:
 445         * 1) if use_zero_pages is set to 1, empty pages are merged
 446         *    with the kernel zero page instead of with each other;
 447         * 2) if use_zero_pages is set to 0, empty pages are not treated specially
 448         *    and merged as usual.
 449         */
 450         if (use_zero_pages && !assert_ksm_pages_count(0))
 451                 goto err_out;
 452         else if (!use_zero_pages && !assert_ksm_pages_count(page_count))
 453                 goto err_out;
 454
 455         printf("OK\n");
 456         munmap(map_ptr, page_size * page_count);
 457         return KSFT_PASS;
 458
 459 err_out:
 460         printf("Not OK\n");
 461         munmap(map_ptr, page_size * page_count);
 462         return KSFT_FAIL;
 463 }
 464
 465 static int get_next_mem_node(int node)
 466 {
 467
 468         long node_size;
 469         int mem_node = 0;
 470         int i, max_node = numa_max_node();
 471
 472         for (i = node + 1; i <= max_node + node; i++) {
 473                 mem_node = i % (max_node + 1);
 474                 node_size = numa_node_size(mem_node, NULL);
 475                 if (node_size > 0)
 476                         break;
 477         }
 478         return mem_node;
 479 }
 480
 481 static int get_first_mem_node(void)
 482 {
 483         return get_next_mem_node(numa_max_node());
 484 }
 485
 486 static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeout,
 487                                 bool merge_across_nodes, size_t page_size)
 488 {
 489         void *numa1_map_ptr, *numa2_map_ptr;
 490         struct timespec start_time;
 491         int page_count = 2;
 492         int first_node;
 493
 494         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 495                 perror("clock_gettime");
 496                 return KSFT_FAIL;
 497         }
 498
 499         if (numa_available() < 0) {
 500                 perror("NUMA support not enabled");
 501                 return KSFT_SKIP;
 502         }
 503         if (numa_num_configured_nodes() <= 1) {
 504                 printf("At least 2 NUMA nodes must be available\n");
 505                 return KSFT_SKIP;
 506         }
 507         if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes))
 508                 return KSFT_FAIL;
 509
 510         /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
 511         first_node = get_first_mem_node();
 512         numa1_map_ptr = numa_alloc_onnode(page_size, first_node);
 513         numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));
 514         if (!numa1_map_ptr || !numa2_map_ptr) {
 515                 perror("numa_alloc_onnode");
 516                 return KSFT_FAIL;
 517         }
 518
 519         memset(numa1_map_ptr, '*', page_size);
 520         memset(numa2_map_ptr, '*', page_size);
 521
 522         /* try to merge the pages */
 523         if (ksm_merge_pages(merge_type, numa1_map_ptr, page_size, start_time, timeout) ||
 524             ksm_merge_pages(merge_type, numa2_map_ptr, page_size, start_time, timeout))
 525                 goto err_out;
 526
 527        /*
 528         * verify that the right number of pages are merged:
 529         * 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged;
 530         * 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is
 531         *    only 1 unique page in each node and they can't be shared.
 532         */
 533         if (merge_across_nodes && !assert_ksm_pages_count(page_count))
 534                 goto err_out;
 535         else if (!merge_across_nodes && !assert_ksm_pages_count(0))
 536                 goto err_out;
 537
 538         numa_free(numa1_map_ptr, page_size);
 539         numa_free(numa2_map_ptr, page_size);
 540         printf("OK\n");
 541         return KSFT_PASS;
 542
 543 err_out:
 544         numa_free(numa1_map_ptr, page_size);
 545         numa_free(numa2_map_ptr, page_size);
 546         printf("Not OK\n");
 547         return KSFT_FAIL;
 548 }
 549
 550 static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot,
 551                                 int timeout, size_t map_size)
 552 {
 553         void *map_ptr, *map_ptr_orig;
 554         struct timespec start_time, end_time;
 555         unsigned long scan_time_ns;
 556         int pagemap_fd, n_normal_pages, n_huge_pages;
 557
 558         map_size *= MB;
 559         size_t len = map_size;
 560
 561         len -= len % HPAGE_SIZE;
 562         map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE,
 563                         MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0);
 564         map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE;
 565
 566         if (map_ptr_orig == MAP_FAILED)
 567                 err(2, "initial mmap");
 568
 569         if (madvise(map_ptr, len + HPAGE_SIZE, MADV_HUGEPAGE))
 570                 err(2, "MADV_HUGEPAGE");
 571
 572         pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
 573         if (pagemap_fd < 0)
 574                 err(2, "open pagemap");
 575
 576         n_normal_pages = 0;
 577         n_huge_pages = 0;
 578         for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) {
 579                 if (allocate_transhuge(p, pagemap_fd) < 0)
 580                         n_normal_pages++;
 581                 else
 582                         n_huge_pages++;
 583         }
 584         printf("Number of normal pages:    %d\n", n_normal_pages);
 585         printf("Number of huge pages:    %d\n", n_huge_pages);
 586
 587         memset(map_ptr, '*', len);
 588
 589         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 590                 perror("clock_gettime");
 591                 goto err_out;
 592         }
 593         if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
 594                 goto err_out;
 595         if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
 596                 perror("clock_gettime");
 597                 goto err_out;
 598         }
 599
 600         scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
 601                        (end_time.tv_nsec - start_time.tv_nsec);
 602
 603         printf("Total size:    %lu MiB\n", map_size / MB);
 604         printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
 605                scan_time_ns % NSEC_PER_SEC);
 606         printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
 607                                                ((double)scan_time_ns / NSEC_PER_SEC));
 608
 609         munmap(map_ptr_orig, len + HPAGE_SIZE);
 610         return KSFT_PASS;
 611
 612 err_out:
 613         printf("Not OK\n");
 614         munmap(map_ptr_orig, len + HPAGE_SIZE);
 615         return KSFT_FAIL;
 616 }
 617
 618 static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size)
 619 {
 620         void *map_ptr;
 621         struct timespec start_time, end_time;
 622         unsigned long scan_time_ns;
 623
 624         map_size *= MB;
 625
 626         map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
 627         if (!map_ptr)
 628                 return KSFT_FAIL;
 629
 630         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 631                 perror("clock_gettime");
 632                 goto err_out;
 633         }
 634         if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
 635                 goto err_out;
 636         if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
 637                 perror("clock_gettime");
 638                 goto err_out;
 639         }
 640
 641         scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
 642                        (end_time.tv_nsec - start_time.tv_nsec);
 643
 644         printf("Total size:    %lu MiB\n", map_size / MB);
 645         printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
 646                scan_time_ns % NSEC_PER_SEC);
 647         printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
 648                                                ((double)scan_time_ns / NSEC_PER_SEC));
 649
 650         munmap(map_ptr, map_size);
 651         return KSFT_PASS;
 652
 653 err_out:
 654         printf("Not OK\n");
 655         munmap(map_ptr, map_size);
 656         return KSFT_FAIL;
 657 }
 658
 659 static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size)
 660 {
 661         void *map_ptr;
 662         struct timespec start_time, end_time;
 663         unsigned long scan_time_ns;
 664
 665         map_size *= MB;
 666
 667         map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
 668         if (!map_ptr)
 669                 return KSFT_FAIL;
 670         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 671                 perror("clock_gettime");
 672                 goto err_out;
 673         }
 674         if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))
 675                 goto err_out;
 676
 677         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 678                 perror("clock_gettime");
 679                 goto err_out;
 680         }
 681         if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout))
 682                 goto err_out;
 683         if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
 684                 perror("clock_gettime");
 685                 goto err_out;
 686         }
 687
 688         scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
 689                        (end_time.tv_nsec - start_time.tv_nsec);
 690
 691         printf("Total size:    %lu MiB\n", map_size / MB);
 692         printf("Total time:    %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
 693                scan_time_ns % NSEC_PER_SEC);
 694         printf("Average speed:  %.3f MiB/s\n", (map_size / MB) /
 695                                                ((double)scan_time_ns / NSEC_PER_SEC));
 696
 697         munmap(map_ptr, map_size);
 698         return KSFT_PASS;
 699
 700 err_out:
 701         printf("Not OK\n");
 702         munmap(map_ptr, map_size);
 703         return KSFT_FAIL;
 704 }
 705
 706 static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size_t page_size)
 707 {
 708         void *map_ptr;
 709         struct timespec start_time, end_time;
 710         unsigned long cow_time_ns;
 711
 712         /* page_count must be less than 2*page_size */
 713         size_t page_count = 4000;
 714
 715         map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
 716         if (!map_ptr)
 717                 return KSFT_FAIL;
 718
 719         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 720                 perror("clock_gettime");
 721                 return KSFT_FAIL;
 722         }
 723         for (size_t i = 0; i < page_count - 1; i = i + 2)
 724                 memset(map_ptr + page_size * i, '-', 1);
 725         if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
 726                 perror("clock_gettime");
 727                 return KSFT_FAIL;
 728         }
 729
 730         cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
 731                        (end_time.tv_nsec - start_time.tv_nsec);
 732
 733         printf("Total size:    %lu MiB\n\n", (page_size * page_count) / MB);
 734         printf("Not merged pages:\n");
 735         printf("Total time:     %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
 736                cow_time_ns % NSEC_PER_SEC);
 737         printf("Average speed:  %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) /
 738                                                ((double)cow_time_ns / NSEC_PER_SEC));
 739
 740         /* Create 2000 pairs of duplicate pages */
 741         for (size_t i = 0; i < page_count - 1; i = i + 2) {
 742                 memset(map_ptr + page_size * i, '+', i / 2 + 1);
 743                 memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1);
 744         }
 745         if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))
 746                 goto err_out;
 747
 748         if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 749                 perror("clock_gettime");
 750                 goto err_out;
 751         }
 752         for (size_t i = 0; i < page_count - 1; i = i + 2)
 753                 memset(map_ptr + page_size * i, '-', 1);
 754         if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
 755                 perror("clock_gettime");
 756                 goto err_out;
 757         }
 758
 759         cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
 760                        (end_time.tv_nsec - start_time.tv_nsec);
 761
 762         printf("Merged pages:\n");
 763         printf("Total time:     %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
 764                cow_time_ns % NSEC_PER_SEC);
 765         printf("Average speed:  %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) /
 766                                                ((double)cow_time_ns / NSEC_PER_SEC));
 767
 768         munmap(map_ptr, page_size * page_count);
 769         return KSFT_PASS;
 770
 771 err_out:
 772         printf("Not OK\n");
 773         munmap(map_ptr, page_size * page_count);
 774         return KSFT_FAIL;
 775 }
 776
 777 int main(int argc, char *argv[])
 778 {
 779         int ret, opt;
 780         int prot = 0;
 781         int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT;
 782         int merge_type = KSM_MERGE_TYPE_DEFAULT;
 783         long page_count = KSM_PAGE_COUNT_DEFAULT;
 784         size_t page_size = sysconf(_SC_PAGESIZE);
 785         struct ksm_sysfs ksm_sysfs_old;
 786         int test_name = CHECK_KSM_MERGE;
 787         bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT;
 788         bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT;
 789         long size_MB = 0;
 790
 791         while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) {
 792                 switch (opt) {
 793                 case 'a':
 794                         prot = str_to_prot(optarg);
 795                         break;
 796                 case 'p':
 797                         page_count = atol(optarg);
 798                         if (page_count <= 0) {
 799                                 printf("The number of pages must be greater than 0\n");
 800                                 return KSFT_FAIL;
 801                         }
 802                         break;
 803                 case 'l':
 804                         ksm_scan_limit_sec = atoi(optarg);
 805                         if (ksm_scan_limit_sec <= 0) {
 806                                 printf("Timeout value must be greater than 0\n");
 807                                 return KSFT_FAIL;
 808                         }
 809                         break;
 810                 case 'h':
 811                         print_help();
 812                         break;
 813                 case 'z':
 814                         if (strcmp(optarg, "0") == 0)
 815                                 use_zero_pages = 0;
 816                         else
 817                                 use_zero_pages = 1;
 818                         break;
 819                 case 'm':
 820                         if (strcmp(optarg, "0") == 0)
 821                                 merge_across_nodes = 0;
 822                         else
 823                                 merge_across_nodes = 1;
 824                         break;
 825                 case 'd':
 826                         debug = 1;
 827                         break;
 828                 case 's':
 829                         size_MB = atoi(optarg);
 830                         if (size_MB <= 0) {
 831                                 printf("Size must be greater than 0\n");
 832                                 return KSFT_FAIL;
 833                         }
 834                 case 't':
 835                         {
 836                                 int tmp = atoi(optarg);
 837
 838                                 if (tmp < 0 || tmp > KSM_MERGE_LAST) {
 839                                         printf("Invalid merge type\n");
 840                                         return KSFT_FAIL;
 841                                 }
 842                                 merge_type = tmp;
 843                         }
 844                         break;
 845                 case 'M':
 846                         break;
 847                 case 'U':
 848                         test_name = CHECK_KSM_UNMERGE;
 849                         break;
 850                 case 'Z':
 851                         test_name = CHECK_KSM_ZERO_PAGE_MERGE;
 852                         break;
 853                 case 'N':
 854                         test_name = CHECK_KSM_NUMA_MERGE;
 855                         break;
 856                 case 'P':
 857                         test_name = KSM_MERGE_TIME;
 858                         break;
 859                 case 'H':
 860                         test_name = KSM_MERGE_TIME_HUGE_PAGES;
 861                         break;
 862                 case 'D':
 863                         test_name = KSM_UNMERGE_TIME;
 864                         break;
 865                 case 'C':
 866                         test_name = KSM_COW_TIME;
 867                         break;
 868                 default:
 869                         return KSFT_FAIL;
 870                 }
 871         }
 872
 873         if (prot == 0)
 874                 prot = str_to_prot(KSM_PROT_STR_DEFAULT);
 875
 876         if (access(KSM_SYSFS_PATH, F_OK)) {
 877                 printf("Config KSM not enabled\n");
 878                 return KSFT_SKIP;
 879         }
 880
 881         if (ksm_save_def(&ksm_sysfs_old)) {
 882                 printf("Cannot save default tunables\n");
 883                 return KSFT_FAIL;
 884         }
 885
 886         if (ksm_write_sysfs(KSM_FP("run"), 2) ||
 887             ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) ||
 888             numa_available() ? 0 :
 889                 ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) ||
 890             ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count))
 891                 return KSFT_FAIL;
 892
 893         switch (test_name) {
 894         case CHECK_KSM_MERGE:
 895                 ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count,
 896                                       ksm_scan_limit_sec, page_size);
 897                 break;
 898         case CHECK_KSM_UNMERGE:
 899                 ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
 900                                         ksm_scan_limit_sec, page_size);
 901                 break;
 902         case CHECK_KSM_ZERO_PAGE_MERGE:
 903                 ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
 904                                                 page_count, ksm_scan_limit_sec, use_zero_pages,
 905                                                 page_size);
 906                 break;
 907         case CHECK_KSM_NUMA_MERGE:
 908                 ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
 909                                         ksm_scan_limit_sec, merge_across_nodes, page_size);
 910                 break;
 911         case KSM_MERGE_TIME:
 912                 if (size_MB == 0) {
 913                         printf("Option '-s' is required.\n");
 914                         return KSFT_FAIL;
 915                 }
 916                 ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
 917                                 ksm_scan_limit_sec, size_MB);
 918                 break;
 919         case KSM_MERGE_TIME_HUGE_PAGES:
 920                 if (size_MB == 0) {
 921                         printf("Option '-s' is required.\n");
 922                         return KSFT_FAIL;
 923                 }
 924                 ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
 925                                 ksm_scan_limit_sec, size_MB);
 926                 break;
 927         case KSM_UNMERGE_TIME:
 928                 if (size_MB == 0) {
 929                         printf("Option '-s' is required.\n");
 930                         return KSFT_FAIL;
 931                 }
 932                 ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
 933                                        ksm_scan_limit_sec, size_MB);
 934                 break;
 935         case KSM_COW_TIME:
 936                 ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,
 937                                 ksm_scan_limit_sec, page_size);
 938                 break;
 939         }
 940
 941         if (ksm_restore(&ksm_sysfs_old)) {
 942                 printf("Cannot restore default tunables\n");
 943                 return KSFT_FAIL;
 944         }
 945
 946         return ret;
 947 }