mm/damon/sysfs-schemes.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * DAMON sysfs Interface
   4  *
   5  * Copyright (c) 2022 SeongJae Park <sj@kernel.org>
   6  */
   7
   8 #include <linux/slab.h>
   9
  10 #include "sysfs-common.h"
  11
  12 /*
  13  * scheme region directory
  14  */
  15
  16 struct damon_sysfs_scheme_region {
  17         struct kobject kobj;
  18         struct damon_addr_range ar;
  19         unsigned int nr_accesses;
  20         unsigned int age;
  21         struct list_head list;
  22 };
  23
  24 static struct damon_sysfs_scheme_region *damon_sysfs_scheme_region_alloc(
  25                 struct damon_region *region)
  26 {
  27         struct damon_sysfs_scheme_region *sysfs_region = kmalloc(
  28                         sizeof(*sysfs_region), GFP_KERNEL);
  29
  30         if (!sysfs_region)
  31                 return NULL;
  32         sysfs_region->kobj = (struct kobject){};
  33         sysfs_region->ar = region->ar;
  34         sysfs_region->nr_accesses = region->nr_accesses;
  35         sysfs_region->age = region->age;
  36         INIT_LIST_HEAD(&sysfs_region->list);
  37         return sysfs_region;
  38 }
  39
  40 static ssize_t start_show(struct kobject *kobj, struct kobj_attribute *attr,
  41                 char *buf)
  42 {
  43         struct damon_sysfs_scheme_region *region = container_of(kobj,
  44                         struct damon_sysfs_scheme_region, kobj);
  45
  46         return sysfs_emit(buf, "%lu\n", region->ar.start);
  47 }
  48
  49 static ssize_t end_show(struct kobject *kobj, struct kobj_attribute *attr,
  50                 char *buf)
  51 {
  52         struct damon_sysfs_scheme_region *region = container_of(kobj,
  53                         struct damon_sysfs_scheme_region, kobj);
  54
  55         return sysfs_emit(buf, "%lu\n", region->ar.end);
  56 }
  57
  58 static ssize_t nr_accesses_show(struct kobject *kobj,
  59                 struct kobj_attribute *attr, char *buf)
  60 {
  61         struct damon_sysfs_scheme_region *region = container_of(kobj,
  62                         struct damon_sysfs_scheme_region, kobj);
  63
  64         return sysfs_emit(buf, "%u\n", region->nr_accesses);
  65 }
  66
  67 static ssize_t age_show(struct kobject *kobj, struct kobj_attribute *attr,
  68                 char *buf)
  69 {
  70         struct damon_sysfs_scheme_region *region = container_of(kobj,
  71                         struct damon_sysfs_scheme_region, kobj);
  72
  73         return sysfs_emit(buf, "%u\n", region->age);
  74 }
  75
  76 static void damon_sysfs_scheme_region_release(struct kobject *kobj)
  77 {
  78         struct damon_sysfs_scheme_region *region = container_of(kobj,
  79                         struct damon_sysfs_scheme_region, kobj);
  80
  81         list_del(&region->list);
  82         kfree(region);
  83 }
  84
  85 static struct kobj_attribute damon_sysfs_scheme_region_start_attr =
  86                 __ATTR_RO_MODE(start, 0400);
  87
  88 static struct kobj_attribute damon_sysfs_scheme_region_end_attr =
  89                 __ATTR_RO_MODE(end, 0400);
  90
  91 static struct kobj_attribute damon_sysfs_scheme_region_nr_accesses_attr =
  92                 __ATTR_RO_MODE(nr_accesses, 0400);
  93
  94 static struct kobj_attribute damon_sysfs_scheme_region_age_attr =
  95                 __ATTR_RO_MODE(age, 0400);
  96
  97 static struct attribute *damon_sysfs_scheme_region_attrs[] = {
  98         &damon_sysfs_scheme_region_start_attr.attr,
  99         &damon_sysfs_scheme_region_end_attr.attr,
 100         &damon_sysfs_scheme_region_nr_accesses_attr.attr,
 101         &damon_sysfs_scheme_region_age_attr.attr,
 102         NULL,
 103 };
 104 ATTRIBUTE_GROUPS(damon_sysfs_scheme_region);
 105
 106 static const struct kobj_type damon_sysfs_scheme_region_ktype = {
 107         .release = damon_sysfs_scheme_region_release,
 108         .sysfs_ops = &kobj_sysfs_ops,
 109         .default_groups = damon_sysfs_scheme_region_groups,
 110 };
 111
 112 /*
 113  * scheme regions directory
 114  */
 115
 116 struct damon_sysfs_scheme_regions {
 117         struct kobject kobj;
 118         struct list_head regions_list;
 119         int nr_regions;
 120         unsigned long total_bytes;
 121 };
 122
 123 static struct damon_sysfs_scheme_regions *
 124 damon_sysfs_scheme_regions_alloc(void)
 125 {
 126         struct damon_sysfs_scheme_regions *regions = kmalloc(sizeof(*regions),
 127                         GFP_KERNEL);
 128
 129         if (!regions)
 130                 return NULL;
 131
 132         regions->kobj = (struct kobject){};
 133         INIT_LIST_HEAD(&regions->regions_list);
 134         regions->nr_regions = 0;
 135         regions->total_bytes = 0;
 136         return regions;
 137 }
 138
 139 static ssize_t total_bytes_show(struct kobject *kobj,
 140                 struct kobj_attribute *attr, char *buf)
 141 {
 142         struct damon_sysfs_scheme_regions *regions = container_of(kobj,
 143                         struct damon_sysfs_scheme_regions, kobj);
 144
 145         return sysfs_emit(buf, "%lu\n", regions->total_bytes);
 146 }
 147
 148 static void damon_sysfs_scheme_regions_rm_dirs(
 149                 struct damon_sysfs_scheme_regions *regions)
 150 {
 151         struct damon_sysfs_scheme_region *r, *next;
 152
 153         list_for_each_entry_safe(r, next, &regions->regions_list, list) {
 154                 /* release function deletes it from the list */
 155                 kobject_put(&r->kobj);
 156                 regions->nr_regions--;
 157         }
 158 }
 159
 160 static void damon_sysfs_scheme_regions_release(struct kobject *kobj)
 161 {
 162         kfree(container_of(kobj, struct damon_sysfs_scheme_regions, kobj));
 163 }
 164
 165 static struct kobj_attribute damon_sysfs_scheme_regions_total_bytes_attr =
 166                 __ATTR_RO_MODE(total_bytes, 0400);
 167
 168 static struct attribute *damon_sysfs_scheme_regions_attrs[] = {
 169         &damon_sysfs_scheme_regions_total_bytes_attr.attr,
 170         NULL,
 171 };
 172 ATTRIBUTE_GROUPS(damon_sysfs_scheme_regions);
 173
 174 static const struct kobj_type damon_sysfs_scheme_regions_ktype = {
 175         .release = damon_sysfs_scheme_regions_release,
 176         .sysfs_ops = &kobj_sysfs_ops,
 177         .default_groups = damon_sysfs_scheme_regions_groups,
 178 };
 179
 180 /*
 181  * schemes/stats directory
 182  */
 183
 184 struct damon_sysfs_stats {
 185         struct kobject kobj;
 186         unsigned long nr_tried;
 187         unsigned long sz_tried;
 188         unsigned long nr_applied;
 189         unsigned long sz_applied;
 190         unsigned long qt_exceeds;
 191 };
 192
 193 static struct damon_sysfs_stats *damon_sysfs_stats_alloc(void)
 194 {
 195         return kzalloc(sizeof(struct damon_sysfs_stats), GFP_KERNEL);
 196 }
 197
 198 static ssize_t nr_tried_show(struct kobject *kobj, struct kobj_attribute *attr,
 199                 char *buf)
 200 {
 201         struct damon_sysfs_stats *stats = container_of(kobj,
 202                         struct damon_sysfs_stats, kobj);
 203
 204         return sysfs_emit(buf, "%lu\n", stats->nr_tried);
 205 }
 206
 207 static ssize_t sz_tried_show(struct kobject *kobj, struct kobj_attribute *attr,
 208                 char *buf)
 209 {
 210         struct damon_sysfs_stats *stats = container_of(kobj,
 211                         struct damon_sysfs_stats, kobj);
 212
 213         return sysfs_emit(buf, "%lu\n", stats->sz_tried);
 214 }
 215
 216 static ssize_t nr_applied_show(struct kobject *kobj,
 217                 struct kobj_attribute *attr, char *buf)
 218 {
 219         struct damon_sysfs_stats *stats = container_of(kobj,
 220                         struct damon_sysfs_stats, kobj);
 221
 222         return sysfs_emit(buf, "%lu\n", stats->nr_applied);
 223 }
 224
 225 static ssize_t sz_applied_show(struct kobject *kobj,
 226                 struct kobj_attribute *attr, char *buf)
 227 {
 228         struct damon_sysfs_stats *stats = container_of(kobj,
 229                         struct damon_sysfs_stats, kobj);
 230
 231         return sysfs_emit(buf, "%lu\n", stats->sz_applied);
 232 }
 233
 234 static ssize_t qt_exceeds_show(struct kobject *kobj,
 235                 struct kobj_attribute *attr, char *buf)
 236 {
 237         struct damon_sysfs_stats *stats = container_of(kobj,
 238                         struct damon_sysfs_stats, kobj);
 239
 240         return sysfs_emit(buf, "%lu\n", stats->qt_exceeds);
 241 }
 242
 243 static void damon_sysfs_stats_release(struct kobject *kobj)
 244 {
 245         kfree(container_of(kobj, struct damon_sysfs_stats, kobj));
 246 }
 247
 248 static struct kobj_attribute damon_sysfs_stats_nr_tried_attr =
 249                 __ATTR_RO_MODE(nr_tried, 0400);
 250
 251 static struct kobj_attribute damon_sysfs_stats_sz_tried_attr =
 252                 __ATTR_RO_MODE(sz_tried, 0400);
 253
 254 static struct kobj_attribute damon_sysfs_stats_nr_applied_attr =
 255                 __ATTR_RO_MODE(nr_applied, 0400);
 256
 257 static struct kobj_attribute damon_sysfs_stats_sz_applied_attr =
 258                 __ATTR_RO_MODE(sz_applied, 0400);
 259
 260 static struct kobj_attribute damon_sysfs_stats_qt_exceeds_attr =
 261                 __ATTR_RO_MODE(qt_exceeds, 0400);
 262
 263 static struct attribute *damon_sysfs_stats_attrs[] = {
 264         &damon_sysfs_stats_nr_tried_attr.attr,
 265         &damon_sysfs_stats_sz_tried_attr.attr,
 266         &damon_sysfs_stats_nr_applied_attr.attr,
 267         &damon_sysfs_stats_sz_applied_attr.attr,
 268         &damon_sysfs_stats_qt_exceeds_attr.attr,
 269         NULL,
 270 };
 271 ATTRIBUTE_GROUPS(damon_sysfs_stats);
 272
 273 static const struct kobj_type damon_sysfs_stats_ktype = {
 274         .release = damon_sysfs_stats_release,
 275         .sysfs_ops = &kobj_sysfs_ops,
 276         .default_groups = damon_sysfs_stats_groups,
 277 };
 278
 279 /*
 280  * filter directory
 281  */
 282
 283 struct damon_sysfs_scheme_filter {
 284         struct kobject kobj;
 285         enum damos_filter_type type;
 286         bool matching;
 287         char *memcg_path;
 288         struct damon_addr_range addr_range;
 289         int target_idx;
 290 };
 291
 292 static struct damon_sysfs_scheme_filter *damon_sysfs_scheme_filter_alloc(void)
 293 {
 294         return kzalloc(sizeof(struct damon_sysfs_scheme_filter), GFP_KERNEL);
 295 }
 296
 297 /* Should match with enum damos_filter_type */
 298 static const char * const damon_sysfs_scheme_filter_type_strs[] = {
 299         "anon",
 300         "memcg",
 301         "addr",
 302         "target",
 303 };
 304
 305 static ssize_t type_show(struct kobject *kobj,
 306                 struct kobj_attribute *attr, char *buf)
 307 {
 308         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 309                         struct damon_sysfs_scheme_filter, kobj);
 310
 311         return sysfs_emit(buf, "%s\n",
 312                         damon_sysfs_scheme_filter_type_strs[filter->type]);
 313 }
 314
 315 static ssize_t type_store(struct kobject *kobj,
 316                 struct kobj_attribute *attr, const char *buf, size_t count)
 317 {
 318         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 319                         struct damon_sysfs_scheme_filter, kobj);
 320         enum damos_filter_type type;
 321         ssize_t ret = -EINVAL;
 322
 323         for (type = 0; type < NR_DAMOS_FILTER_TYPES; type++) {
 324                 if (sysfs_streq(buf, damon_sysfs_scheme_filter_type_strs[
 325                                         type])) {
 326                         filter->type = type;
 327                         ret = count;
 328                         break;
 329                 }
 330         }
 331         return ret;
 332 }
 333
 334 static ssize_t matching_show(struct kobject *kobj,
 335                 struct kobj_attribute *attr, char *buf)
 336 {
 337         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 338                         struct damon_sysfs_scheme_filter, kobj);
 339
 340         return sysfs_emit(buf, "%c\n", filter->matching ? 'Y' : 'N');
 341 }
 342
 343 static ssize_t matching_store(struct kobject *kobj,
 344                 struct kobj_attribute *attr, const char *buf, size_t count)
 345 {
 346         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 347                         struct damon_sysfs_scheme_filter, kobj);
 348         bool matching;
 349         int err = kstrtobool(buf, &matching);
 350
 351         if (err)
 352                 return err;
 353
 354         filter->matching = matching;
 355         return count;
 356 }
 357
 358 static ssize_t memcg_path_show(struct kobject *kobj,
 359                 struct kobj_attribute *attr, char *buf)
 360 {
 361         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 362                         struct damon_sysfs_scheme_filter, kobj);
 363
 364         return sysfs_emit(buf, "%s\n",
 365                         filter->memcg_path ? filter->memcg_path : "");
 366 }
 367
 368 static ssize_t memcg_path_store(struct kobject *kobj,
 369                 struct kobj_attribute *attr, const char *buf, size_t count)
 370 {
 371         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 372                         struct damon_sysfs_scheme_filter, kobj);
 373         char *path = kmalloc(sizeof(*path) * (count + 1), GFP_KERNEL);
 374
 375         if (!path)
 376                 return -ENOMEM;
 377
 378         strscpy(path, buf, count + 1);
 379         filter->memcg_path = path;
 380         return count;
 381 }
 382
 383 static ssize_t addr_start_show(struct kobject *kobj,
 384                 struct kobj_attribute *attr, char *buf)
 385 {
 386         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 387                         struct damon_sysfs_scheme_filter, kobj);
 388
 389         return sysfs_emit(buf, "%lu\n", filter->addr_range.start);
 390 }
 391
 392 static ssize_t addr_start_store(struct kobject *kobj,
 393                 struct kobj_attribute *attr, const char *buf, size_t count)
 394 {
 395         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 396                         struct damon_sysfs_scheme_filter, kobj);
 397         int err = kstrtoul(buf, 0, &filter->addr_range.start);
 398
 399         return err ? err : count;
 400 }
 401
 402 static ssize_t addr_end_show(struct kobject *kobj,
 403                 struct kobj_attribute *attr, char *buf)
 404 {
 405         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 406                         struct damon_sysfs_scheme_filter, kobj);
 407
 408         return sysfs_emit(buf, "%lu\n", filter->addr_range.end);
 409 }
 410
 411 static ssize_t addr_end_store(struct kobject *kobj,
 412                 struct kobj_attribute *attr, const char *buf, size_t count)
 413 {
 414         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 415                         struct damon_sysfs_scheme_filter, kobj);
 416         int err = kstrtoul(buf, 0, &filter->addr_range.end);
 417
 418         return err ? err : count;
 419 }
 420
 421 static ssize_t damon_target_idx_show(struct kobject *kobj,
 422                 struct kobj_attribute *attr, char *buf)
 423 {
 424         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 425                         struct damon_sysfs_scheme_filter, kobj);
 426
 427         return sysfs_emit(buf, "%d\n", filter->target_idx);
 428 }
 429
 430 static ssize_t damon_target_idx_store(struct kobject *kobj,
 431                 struct kobj_attribute *attr, const char *buf, size_t count)
 432 {
 433         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 434                         struct damon_sysfs_scheme_filter, kobj);
 435         int err = kstrtoint(buf, 0, &filter->target_idx);
 436
 437         return err ? err : count;
 438 }
 439
 440 static void damon_sysfs_scheme_filter_release(struct kobject *kobj)
 441 {
 442         struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 443                         struct damon_sysfs_scheme_filter, kobj);
 444
 445         kfree(filter->memcg_path);
 446         kfree(filter);
 447 }
 448
 449 static struct kobj_attribute damon_sysfs_scheme_filter_type_attr =
 450                 __ATTR_RW_MODE(type, 0600);
 451
 452 static struct kobj_attribute damon_sysfs_scheme_filter_matching_attr =
 453                 __ATTR_RW_MODE(matching, 0600);
 454
 455 static struct kobj_attribute damon_sysfs_scheme_filter_memcg_path_attr =
 456                 __ATTR_RW_MODE(memcg_path, 0600);
 457
 458 static struct kobj_attribute damon_sysfs_scheme_filter_addr_start_attr =
 459                 __ATTR_RW_MODE(addr_start, 0600);
 460
 461 static struct kobj_attribute damon_sysfs_scheme_filter_addr_end_attr =
 462                 __ATTR_RW_MODE(addr_end, 0600);
 463
 464 static struct kobj_attribute damon_sysfs_scheme_filter_damon_target_idx_attr =
 465                 __ATTR_RW_MODE(damon_target_idx, 0600);
 466
 467 static struct attribute *damon_sysfs_scheme_filter_attrs[] = {
 468         &damon_sysfs_scheme_filter_type_attr.attr,
 469         &damon_sysfs_scheme_filter_matching_attr.attr,
 470         &damon_sysfs_scheme_filter_memcg_path_attr.attr,
 471         &damon_sysfs_scheme_filter_addr_start_attr.attr,
 472         &damon_sysfs_scheme_filter_addr_end_attr.attr,
 473         &damon_sysfs_scheme_filter_damon_target_idx_attr.attr,
 474         NULL,
 475 };
 476 ATTRIBUTE_GROUPS(damon_sysfs_scheme_filter);
 477
 478 static const struct kobj_type damon_sysfs_scheme_filter_ktype = {
 479         .release = damon_sysfs_scheme_filter_release,
 480         .sysfs_ops = &kobj_sysfs_ops,
 481         .default_groups = damon_sysfs_scheme_filter_groups,
 482 };
 483
 484 /*
 485  * filters directory
 486  */
 487
 488 struct damon_sysfs_scheme_filters {
 489         struct kobject kobj;
 490         struct damon_sysfs_scheme_filter **filters_arr;
 491         int nr;
 492 };
 493
 494 static struct damon_sysfs_scheme_filters *
 495 damon_sysfs_scheme_filters_alloc(void)
 496 {
 497         return kzalloc(sizeof(struct damon_sysfs_scheme_filters), GFP_KERNEL);
 498 }
 499
 500 static void damon_sysfs_scheme_filters_rm_dirs(
 501                 struct damon_sysfs_scheme_filters *filters)
 502 {
 503         struct damon_sysfs_scheme_filter **filters_arr = filters->filters_arr;
 504         int i;
 505
 506         for (i = 0; i < filters->nr; i++)
 507                 kobject_put(&filters_arr[i]->kobj);
 508         filters->nr = 0;
 509         kfree(filters_arr);
 510         filters->filters_arr = NULL;
 511 }
 512
 513 static int damon_sysfs_scheme_filters_add_dirs(
 514                 struct damon_sysfs_scheme_filters *filters, int nr_filters)
 515 {
 516         struct damon_sysfs_scheme_filter **filters_arr, *filter;
 517         int err, i;
 518
 519         damon_sysfs_scheme_filters_rm_dirs(filters);
 520         if (!nr_filters)
 521                 return 0;
 522
 523         filters_arr = kmalloc_array(nr_filters, sizeof(*filters_arr),
 524                         GFP_KERNEL | __GFP_NOWARN);
 525         if (!filters_arr)
 526                 return -ENOMEM;
 527         filters->filters_arr = filters_arr;
 528
 529         for (i = 0; i < nr_filters; i++) {
 530                 filter = damon_sysfs_scheme_filter_alloc();
 531                 if (!filter) {
 532                         damon_sysfs_scheme_filters_rm_dirs(filters);
 533                         return -ENOMEM;
 534                 }
 535
 536                 err = kobject_init_and_add(&filter->kobj,
 537                                 &damon_sysfs_scheme_filter_ktype,
 538                                 &filters->kobj, "%d", i);
 539                 if (err) {
 540                         kobject_put(&filter->kobj);
 541                         damon_sysfs_scheme_filters_rm_dirs(filters);
 542                         return err;
 543                 }
 544
 545                 filters_arr[i] = filter;
 546                 filters->nr++;
 547         }
 548         return 0;
 549 }
 550
 551 static ssize_t nr_filters_show(struct kobject *kobj,
 552                 struct kobj_attribute *attr, char *buf)
 553 {
 554         struct damon_sysfs_scheme_filters *filters = container_of(kobj,
 555                         struct damon_sysfs_scheme_filters, kobj);
 556
 557         return sysfs_emit(buf, "%d\n", filters->nr);
 558 }
 559
 560 static ssize_t nr_filters_store(struct kobject *kobj,
 561                 struct kobj_attribute *attr, const char *buf, size_t count)
 562 {
 563         struct damon_sysfs_scheme_filters *filters;
 564         int nr, err = kstrtoint(buf, 0, &nr);
 565
 566         if (err)
 567                 return err;
 568         if (nr < 0)
 569                 return -EINVAL;
 570
 571         filters = container_of(kobj, struct damon_sysfs_scheme_filters, kobj);
 572
 573         if (!mutex_trylock(&damon_sysfs_lock))
 574                 return -EBUSY;
 575         err = damon_sysfs_scheme_filters_add_dirs(filters, nr);
 576         mutex_unlock(&damon_sysfs_lock);
 577         if (err)
 578                 return err;
 579
 580         return count;
 581 }
 582
 583 static void damon_sysfs_scheme_filters_release(struct kobject *kobj)
 584 {
 585         kfree(container_of(kobj, struct damon_sysfs_scheme_filters, kobj));
 586 }
 587
 588 static struct kobj_attribute damon_sysfs_scheme_filters_nr_attr =
 589                 __ATTR_RW_MODE(nr_filters, 0600);
 590
 591 static struct attribute *damon_sysfs_scheme_filters_attrs[] = {
 592         &damon_sysfs_scheme_filters_nr_attr.attr,
 593         NULL,
 594 };
 595 ATTRIBUTE_GROUPS(damon_sysfs_scheme_filters);
 596
 597 static const struct kobj_type damon_sysfs_scheme_filters_ktype = {
 598         .release = damon_sysfs_scheme_filters_release,
 599         .sysfs_ops = &kobj_sysfs_ops,
 600         .default_groups = damon_sysfs_scheme_filters_groups,
 601 };
 602
 603 /*
 604  * watermarks directory
 605  */
 606
 607 struct damon_sysfs_watermarks {
 608         struct kobject kobj;
 609         enum damos_wmark_metric metric;
 610         unsigned long interval_us;
 611         unsigned long high;
 612         unsigned long mid;
 613         unsigned long low;
 614 };
 615
 616 static struct damon_sysfs_watermarks *damon_sysfs_watermarks_alloc(
 617                 enum damos_wmark_metric metric, unsigned long interval_us,
 618                 unsigned long high, unsigned long mid, unsigned long low)
 619 {
 620         struct damon_sysfs_watermarks *watermarks = kmalloc(
 621                         sizeof(*watermarks), GFP_KERNEL);
 622
 623         if (!watermarks)
 624                 return NULL;
 625         watermarks->kobj = (struct kobject){};
 626         watermarks->metric = metric;
 627         watermarks->interval_us = interval_us;
 628         watermarks->high = high;
 629         watermarks->mid = mid;
 630         watermarks->low = low;
 631         return watermarks;
 632 }
 633
 634 /* Should match with enum damos_wmark_metric */
 635 static const char * const damon_sysfs_wmark_metric_strs[] = {
 636         "none",
 637         "free_mem_rate",
 638 };
 639
 640 static ssize_t metric_show(struct kobject *kobj, struct kobj_attribute *attr,
 641                 char *buf)
 642 {
 643         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 644                         struct damon_sysfs_watermarks, kobj);
 645
 646         return sysfs_emit(buf, "%s\n",
 647                         damon_sysfs_wmark_metric_strs[watermarks->metric]);
 648 }
 649
 650 static ssize_t metric_store(struct kobject *kobj, struct kobj_attribute *attr,
 651                 const char *buf, size_t count)
 652 {
 653         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 654                         struct damon_sysfs_watermarks, kobj);
 655         enum damos_wmark_metric metric;
 656
 657         for (metric = 0; metric < NR_DAMOS_WMARK_METRICS; metric++) {
 658                 if (sysfs_streq(buf, damon_sysfs_wmark_metric_strs[metric])) {
 659                         watermarks->metric = metric;
 660                         return count;
 661                 }
 662         }
 663         return -EINVAL;
 664 }
 665
 666 static ssize_t interval_us_show(struct kobject *kobj,
 667                 struct kobj_attribute *attr, char *buf)
 668 {
 669         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 670                         struct damon_sysfs_watermarks, kobj);
 671
 672         return sysfs_emit(buf, "%lu\n", watermarks->interval_us);
 673 }
 674
 675 static ssize_t interval_us_store(struct kobject *kobj,
 676                 struct kobj_attribute *attr, const char *buf, size_t count)
 677 {
 678         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 679                         struct damon_sysfs_watermarks, kobj);
 680         int err = kstrtoul(buf, 0, &watermarks->interval_us);
 681
 682         return err ? err : count;
 683 }
 684
 685 static ssize_t high_show(struct kobject *kobj,
 686                 struct kobj_attribute *attr, char *buf)
 687 {
 688         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 689                         struct damon_sysfs_watermarks, kobj);
 690
 691         return sysfs_emit(buf, "%lu\n", watermarks->high);
 692 }
 693
 694 static ssize_t high_store(struct kobject *kobj,
 695                 struct kobj_attribute *attr, const char *buf, size_t count)
 696 {
 697         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 698                         struct damon_sysfs_watermarks, kobj);
 699         int err = kstrtoul(buf, 0, &watermarks->high);
 700
 701         return err ? err : count;
 702 }
 703
 704 static ssize_t mid_show(struct kobject *kobj,
 705                 struct kobj_attribute *attr, char *buf)
 706 {
 707         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 708                         struct damon_sysfs_watermarks, kobj);
 709
 710         return sysfs_emit(buf, "%lu\n", watermarks->mid);
 711 }
 712
 713 static ssize_t mid_store(struct kobject *kobj,
 714                 struct kobj_attribute *attr, const char *buf, size_t count)
 715 {
 716         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 717                         struct damon_sysfs_watermarks, kobj);
 718         int err = kstrtoul(buf, 0, &watermarks->mid);
 719
 720         return err ? err : count;
 721 }
 722
 723 static ssize_t low_show(struct kobject *kobj,
 724                 struct kobj_attribute *attr, char *buf)
 725 {
 726         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 727                         struct damon_sysfs_watermarks, kobj);
 728
 729         return sysfs_emit(buf, "%lu\n", watermarks->low);
 730 }
 731
 732 static ssize_t low_store(struct kobject *kobj,
 733                 struct kobj_attribute *attr, const char *buf, size_t count)
 734 {
 735         struct damon_sysfs_watermarks *watermarks = container_of(kobj,
 736                         struct damon_sysfs_watermarks, kobj);
 737         int err = kstrtoul(buf, 0, &watermarks->low);
 738
 739         return err ? err : count;
 740 }
 741
 742 static void damon_sysfs_watermarks_release(struct kobject *kobj)
 743 {
 744         kfree(container_of(kobj, struct damon_sysfs_watermarks, kobj));
 745 }
 746
 747 static struct kobj_attribute damon_sysfs_watermarks_metric_attr =
 748                 __ATTR_RW_MODE(metric, 0600);
 749
 750 static struct kobj_attribute damon_sysfs_watermarks_interval_us_attr =
 751                 __ATTR_RW_MODE(interval_us, 0600);
 752
 753 static struct kobj_attribute damon_sysfs_watermarks_high_attr =
 754                 __ATTR_RW_MODE(high, 0600);
 755
 756 static struct kobj_attribute damon_sysfs_watermarks_mid_attr =
 757                 __ATTR_RW_MODE(mid, 0600);
 758
 759 static struct kobj_attribute damon_sysfs_watermarks_low_attr =
 760                 __ATTR_RW_MODE(low, 0600);
 761
 762 static struct attribute *damon_sysfs_watermarks_attrs[] = {
 763         &damon_sysfs_watermarks_metric_attr.attr,
 764         &damon_sysfs_watermarks_interval_us_attr.attr,
 765         &damon_sysfs_watermarks_high_attr.attr,
 766         &damon_sysfs_watermarks_mid_attr.attr,
 767         &damon_sysfs_watermarks_low_attr.attr,
 768         NULL,
 769 };
 770 ATTRIBUTE_GROUPS(damon_sysfs_watermarks);
 771
 772 static const struct kobj_type damon_sysfs_watermarks_ktype = {
 773         .release = damon_sysfs_watermarks_release,
 774         .sysfs_ops = &kobj_sysfs_ops,
 775         .default_groups = damon_sysfs_watermarks_groups,
 776 };
 777
 778 /*
 779  * scheme/weights directory
 780  */
 781
 782 struct damon_sysfs_weights {
 783         struct kobject kobj;
 784         unsigned int sz;
 785         unsigned int nr_accesses;
 786         unsigned int age;
 787 };
 788
 789 static struct damon_sysfs_weights *damon_sysfs_weights_alloc(unsigned int sz,
 790                 unsigned int nr_accesses, unsigned int age)
 791 {
 792         struct damon_sysfs_weights *weights = kmalloc(sizeof(*weights),
 793                         GFP_KERNEL);
 794
 795         if (!weights)
 796                 return NULL;
 797         weights->kobj = (struct kobject){};
 798         weights->sz = sz;
 799         weights->nr_accesses = nr_accesses;
 800         weights->age = age;
 801         return weights;
 802 }
 803
 804 static ssize_t sz_permil_show(struct kobject *kobj,
 805                 struct kobj_attribute *attr, char *buf)
 806 {
 807         struct damon_sysfs_weights *weights = container_of(kobj,
 808                         struct damon_sysfs_weights, kobj);
 809
 810         return sysfs_emit(buf, "%u\n", weights->sz);
 811 }
 812
 813 static ssize_t sz_permil_store(struct kobject *kobj,
 814                 struct kobj_attribute *attr, const char *buf, size_t count)
 815 {
 816         struct damon_sysfs_weights *weights = container_of(kobj,
 817                         struct damon_sysfs_weights, kobj);
 818         int err = kstrtouint(buf, 0, &weights->sz);
 819
 820         return err ? err : count;
 821 }
 822
 823 static ssize_t nr_accesses_permil_show(struct kobject *kobj,
 824                 struct kobj_attribute *attr, char *buf)
 825 {
 826         struct damon_sysfs_weights *weights = container_of(kobj,
 827                         struct damon_sysfs_weights, kobj);
 828
 829         return sysfs_emit(buf, "%u\n", weights->nr_accesses);
 830 }
 831
 832 static ssize_t nr_accesses_permil_store(struct kobject *kobj,
 833                 struct kobj_attribute *attr, const char *buf, size_t count)
 834 {
 835         struct damon_sysfs_weights *weights = container_of(kobj,
 836                         struct damon_sysfs_weights, kobj);
 837         int err = kstrtouint(buf, 0, &weights->nr_accesses);
 838
 839         return err ? err : count;
 840 }
 841
 842 static ssize_t age_permil_show(struct kobject *kobj,
 843                 struct kobj_attribute *attr, char *buf)
 844 {
 845         struct damon_sysfs_weights *weights = container_of(kobj,
 846                         struct damon_sysfs_weights, kobj);
 847
 848         return sysfs_emit(buf, "%u\n", weights->age);
 849 }
 850
 851 static ssize_t age_permil_store(struct kobject *kobj,
 852                 struct kobj_attribute *attr, const char *buf, size_t count)
 853 {
 854         struct damon_sysfs_weights *weights = container_of(kobj,
 855                         struct damon_sysfs_weights, kobj);
 856         int err = kstrtouint(buf, 0, &weights->age);
 857
 858         return err ? err : count;
 859 }
 860
 861 static void damon_sysfs_weights_release(struct kobject *kobj)
 862 {
 863         kfree(container_of(kobj, struct damon_sysfs_weights, kobj));
 864 }
 865
 866 static struct kobj_attribute damon_sysfs_weights_sz_attr =
 867                 __ATTR_RW_MODE(sz_permil, 0600);
 868
 869 static struct kobj_attribute damon_sysfs_weights_nr_accesses_attr =
 870                 __ATTR_RW_MODE(nr_accesses_permil, 0600);
 871
 872 static struct kobj_attribute damon_sysfs_weights_age_attr =
 873                 __ATTR_RW_MODE(age_permil, 0600);
 874
 875 static struct attribute *damon_sysfs_weights_attrs[] = {
 876         &damon_sysfs_weights_sz_attr.attr,
 877         &damon_sysfs_weights_nr_accesses_attr.attr,
 878         &damon_sysfs_weights_age_attr.attr,
 879         NULL,
 880 };
 881 ATTRIBUTE_GROUPS(damon_sysfs_weights);
 882
 883 static const struct kobj_type damon_sysfs_weights_ktype = {
 884         .release = damon_sysfs_weights_release,
 885         .sysfs_ops = &kobj_sysfs_ops,
 886         .default_groups = damon_sysfs_weights_groups,
 887 };
 888
 889 /*
 890  * quotas directory
 891  */
 892
 893 struct damon_sysfs_quotas {
 894         struct kobject kobj;
 895         struct damon_sysfs_weights *weights;
 896         unsigned long ms;
 897         unsigned long sz;
 898         unsigned long reset_interval_ms;
 899 };
 900
 901 static struct damon_sysfs_quotas *damon_sysfs_quotas_alloc(void)
 902 {
 903         return kzalloc(sizeof(struct damon_sysfs_quotas), GFP_KERNEL);
 904 }
 905
 906 static int damon_sysfs_quotas_add_dirs(struct damon_sysfs_quotas *quotas)
 907 {
 908         struct damon_sysfs_weights *weights;
 909         int err;
 910
 911         weights = damon_sysfs_weights_alloc(0, 0, 0);
 912         if (!weights)
 913                 return -ENOMEM;
 914
 915         err = kobject_init_and_add(&weights->kobj, &damon_sysfs_weights_ktype,
 916                         &quotas->kobj, "weights");
 917         if (err)
 918                 kobject_put(&weights->kobj);
 919         else
 920                 quotas->weights = weights;
 921         return err;
 922 }
 923
 924 static void damon_sysfs_quotas_rm_dirs(struct damon_sysfs_quotas *quotas)
 925 {
 926         kobject_put(&quotas->weights->kobj);
 927 }
 928
 929 static ssize_t ms_show(struct kobject *kobj, struct kobj_attribute *attr,
 930                 char *buf)
 931 {
 932         struct damon_sysfs_quotas *quotas = container_of(kobj,
 933                         struct damon_sysfs_quotas, kobj);
 934
 935         return sysfs_emit(buf, "%lu\n", quotas->ms);
 936 }
 937
 938 static ssize_t ms_store(struct kobject *kobj, struct kobj_attribute *attr,
 939                 const char *buf, size_t count)
 940 {
 941         struct damon_sysfs_quotas *quotas = container_of(kobj,
 942                         struct damon_sysfs_quotas, kobj);
 943         int err = kstrtoul(buf, 0, &quotas->ms);
 944
 945         if (err)
 946                 return -EINVAL;
 947         return count;
 948 }
 949
 950 static ssize_t bytes_show(struct kobject *kobj, struct kobj_attribute *attr,
 951                 char *buf)
 952 {
 953         struct damon_sysfs_quotas *quotas = container_of(kobj,
 954                         struct damon_sysfs_quotas, kobj);
 955
 956         return sysfs_emit(buf, "%lu\n", quotas->sz);
 957 }
 958
 959 static ssize_t bytes_store(struct kobject *kobj,
 960                 struct kobj_attribute *attr, const char *buf, size_t count)
 961 {
 962         struct damon_sysfs_quotas *quotas = container_of(kobj,
 963                         struct damon_sysfs_quotas, kobj);
 964         int err = kstrtoul(buf, 0, &quotas->sz);
 965
 966         if (err)
 967                 return -EINVAL;
 968         return count;
 969 }
 970
 971 static ssize_t reset_interval_ms_show(struct kobject *kobj,
 972                 struct kobj_attribute *attr, char *buf)
 973 {
 974         struct damon_sysfs_quotas *quotas = container_of(kobj,
 975                         struct damon_sysfs_quotas, kobj);
 976
 977         return sysfs_emit(buf, "%lu\n", quotas->reset_interval_ms);
 978 }
 979
 980 static ssize_t reset_interval_ms_store(struct kobject *kobj,
 981                 struct kobj_attribute *attr, const char *buf, size_t count)
 982 {
 983         struct damon_sysfs_quotas *quotas = container_of(kobj,
 984                         struct damon_sysfs_quotas, kobj);
 985         int err = kstrtoul(buf, 0, &quotas->reset_interval_ms);
 986
 987         if (err)
 988                 return -EINVAL;
 989         return count;
 990 }
 991
 992 static void damon_sysfs_quotas_release(struct kobject *kobj)
 993 {
 994         kfree(container_of(kobj, struct damon_sysfs_quotas, kobj));
 995 }
 996
 997 static struct kobj_attribute damon_sysfs_quotas_ms_attr =
 998                 __ATTR_RW_MODE(ms, 0600);
 999
1000 static struct kobj_attribute damon_sysfs_quotas_sz_attr =
1001                 __ATTR_RW_MODE(bytes, 0600);
1002
1003 static struct kobj_attribute damon_sysfs_quotas_reset_interval_ms_attr =
1004                 __ATTR_RW_MODE(reset_interval_ms, 0600);
1005
1006 static struct attribute *damon_sysfs_quotas_attrs[] = {
1007         &damon_sysfs_quotas_ms_attr.attr,
1008         &damon_sysfs_quotas_sz_attr.attr,
1009         &damon_sysfs_quotas_reset_interval_ms_attr.attr,
1010         NULL,
1011 };
1012 ATTRIBUTE_GROUPS(damon_sysfs_quotas);
1013
1014 static const struct kobj_type damon_sysfs_quotas_ktype = {
1015         .release = damon_sysfs_quotas_release,
1016         .sysfs_ops = &kobj_sysfs_ops,
1017         .default_groups = damon_sysfs_quotas_groups,
1018 };
1019
1020 /*
1021  * access_pattern directory
1022  */
1023
1024 struct damon_sysfs_access_pattern {
1025         struct kobject kobj;
1026         struct damon_sysfs_ul_range *sz;
1027         struct damon_sysfs_ul_range *nr_accesses;
1028         struct damon_sysfs_ul_range *age;
1029 };
1030
1031 static
1032 struct damon_sysfs_access_pattern *damon_sysfs_access_pattern_alloc(void)
1033 {
1034         struct damon_sysfs_access_pattern *access_pattern =
1035                 kmalloc(sizeof(*access_pattern), GFP_KERNEL);
1036
1037         if (!access_pattern)
1038                 return NULL;
1039         access_pattern->kobj = (struct kobject){};
1040         return access_pattern;
1041 }
1042
1043 static int damon_sysfs_access_pattern_add_range_dir(
1044                 struct damon_sysfs_access_pattern *access_pattern,
1045                 struct damon_sysfs_ul_range **range_dir_ptr,
1046                 char *name)
1047 {
1048         struct damon_sysfs_ul_range *range = damon_sysfs_ul_range_alloc(0, 0);
1049         int err;
1050
1051         if (!range)
1052                 return -ENOMEM;
1053         err = kobject_init_and_add(&range->kobj, &damon_sysfs_ul_range_ktype,
1054                         &access_pattern->kobj, name);
1055         if (err)
1056                 kobject_put(&range->kobj);
1057         else
1058                 *range_dir_ptr = range;
1059         return err;
1060 }
1061
1062 static int damon_sysfs_access_pattern_add_dirs(
1063                 struct damon_sysfs_access_pattern *access_pattern)
1064 {
1065         int err;
1066
1067         err = damon_sysfs_access_pattern_add_range_dir(access_pattern,
1068                         &access_pattern->sz, "sz");
1069         if (err)
1070                 goto put_sz_out;
1071
1072         err = damon_sysfs_access_pattern_add_range_dir(access_pattern,
1073                         &access_pattern->nr_accesses, "nr_accesses");
1074         if (err)
1075                 goto put_nr_accesses_sz_out;
1076
1077         err = damon_sysfs_access_pattern_add_range_dir(access_pattern,
1078                         &access_pattern->age, "age");
1079         if (err)
1080                 goto put_age_nr_accesses_sz_out;
1081         return 0;
1082
1083 put_age_nr_accesses_sz_out:
1084         kobject_put(&access_pattern->age->kobj);
1085         access_pattern->age = NULL;
1086 put_nr_accesses_sz_out:
1087         kobject_put(&access_pattern->nr_accesses->kobj);
1088         access_pattern->nr_accesses = NULL;
1089 put_sz_out:
1090         kobject_put(&access_pattern->sz->kobj);
1091         access_pattern->sz = NULL;
1092         return err;
1093 }
1094
1095 static void damon_sysfs_access_pattern_rm_dirs(
1096                 struct damon_sysfs_access_pattern *access_pattern)
1097 {
1098         kobject_put(&access_pattern->sz->kobj);
1099         kobject_put(&access_pattern->nr_accesses->kobj);
1100         kobject_put(&access_pattern->age->kobj);
1101 }
1102
1103 static void damon_sysfs_access_pattern_release(struct kobject *kobj)
1104 {
1105         kfree(container_of(kobj, struct damon_sysfs_access_pattern, kobj));
1106 }
1107
1108 static struct attribute *damon_sysfs_access_pattern_attrs[] = {
1109         NULL,
1110 };
1111 ATTRIBUTE_GROUPS(damon_sysfs_access_pattern);
1112
1113 static const struct kobj_type damon_sysfs_access_pattern_ktype = {
1114         .release = damon_sysfs_access_pattern_release,
1115         .sysfs_ops = &kobj_sysfs_ops,
1116         .default_groups = damon_sysfs_access_pattern_groups,
1117 };
1118
1119 /*
1120  * scheme directory
1121  */
1122
1123 struct damon_sysfs_scheme {
1124         struct kobject kobj;
1125         enum damos_action action;
1126         struct damon_sysfs_access_pattern *access_pattern;
1127         struct damon_sysfs_quotas *quotas;
1128         struct damon_sysfs_watermarks *watermarks;
1129         struct damon_sysfs_scheme_filters *filters;
1130         struct damon_sysfs_stats *stats;
1131         struct damon_sysfs_scheme_regions *tried_regions;
1132 };
1133
1134 /* This should match with enum damos_action */
1135 static const char * const damon_sysfs_damos_action_strs[] = {
1136         "willneed",
1137         "cold",
1138         "pageout",
1139         "hugepage",
1140         "nohugepage",
1141         "lru_prio",
1142         "lru_deprio",
1143         "stat",
1144 };
1145
1146 static struct damon_sysfs_scheme *damon_sysfs_scheme_alloc(
1147                 enum damos_action action)
1148 {
1149         struct damon_sysfs_scheme *scheme = kmalloc(sizeof(*scheme),
1150                                 GFP_KERNEL);
1151
1152         if (!scheme)
1153                 return NULL;
1154         scheme->kobj = (struct kobject){};
1155         scheme->action = action;
1156         return scheme;
1157 }
1158
1159 static int damon_sysfs_scheme_set_access_pattern(
1160                 struct damon_sysfs_scheme *scheme)
1161 {
1162         struct damon_sysfs_access_pattern *access_pattern;
1163         int err;
1164
1165         access_pattern = damon_sysfs_access_pattern_alloc();
1166         if (!access_pattern)
1167                 return -ENOMEM;
1168         err = kobject_init_and_add(&access_pattern->kobj,
1169                         &damon_sysfs_access_pattern_ktype, &scheme->kobj,
1170                         "access_pattern");
1171         if (err)
1172                 goto out;
1173         err = damon_sysfs_access_pattern_add_dirs(access_pattern);
1174         if (err)
1175                 goto out;
1176         scheme->access_pattern = access_pattern;
1177         return 0;
1178
1179 out:
1180         kobject_put(&access_pattern->kobj);
1181         return err;
1182 }
1183
1184 static int damon_sysfs_scheme_set_quotas(struct damon_sysfs_scheme *scheme)
1185 {
1186         struct damon_sysfs_quotas *quotas = damon_sysfs_quotas_alloc();
1187         int err;
1188
1189         if (!quotas)
1190                 return -ENOMEM;
1191         err = kobject_init_and_add(&quotas->kobj, &damon_sysfs_quotas_ktype,
1192                         &scheme->kobj, "quotas");
1193         if (err)
1194                 goto out;
1195         err = damon_sysfs_quotas_add_dirs(quotas);
1196         if (err)
1197                 goto out;
1198         scheme->quotas = quotas;
1199         return 0;
1200
1201 out:
1202         kobject_put(&quotas->kobj);
1203         return err;
1204 }
1205
1206 static int damon_sysfs_scheme_set_watermarks(struct damon_sysfs_scheme *scheme)
1207 {
1208         struct damon_sysfs_watermarks *watermarks =
1209                 damon_sysfs_watermarks_alloc(DAMOS_WMARK_NONE, 0, 0, 0, 0);
1210         int err;
1211
1212         if (!watermarks)
1213                 return -ENOMEM;
1214         err = kobject_init_and_add(&watermarks->kobj,
1215                         &damon_sysfs_watermarks_ktype, &scheme->kobj,
1216                         "watermarks");
1217         if (err)
1218                 kobject_put(&watermarks->kobj);
1219         else
1220                 scheme->watermarks = watermarks;
1221         return err;
1222 }
1223
1224 static int damon_sysfs_scheme_set_filters(struct damon_sysfs_scheme *scheme)
1225 {
1226         struct damon_sysfs_scheme_filters *filters =
1227                 damon_sysfs_scheme_filters_alloc();
1228         int err;
1229
1230         if (!filters)
1231                 return -ENOMEM;
1232         err = kobject_init_and_add(&filters->kobj,
1233                         &damon_sysfs_scheme_filters_ktype, &scheme->kobj,
1234                         "filters");
1235         if (err)
1236                 kobject_put(&filters->kobj);
1237         else
1238                 scheme->filters = filters;
1239         return err;
1240 }
1241
1242 static int damon_sysfs_scheme_set_stats(struct damon_sysfs_scheme *scheme)
1243 {
1244         struct damon_sysfs_stats *stats = damon_sysfs_stats_alloc();
1245         int err;
1246
1247         if (!stats)
1248                 return -ENOMEM;
1249         err = kobject_init_and_add(&stats->kobj, &damon_sysfs_stats_ktype,
1250                         &scheme->kobj, "stats");
1251         if (err)
1252                 kobject_put(&stats->kobj);
1253         else
1254                 scheme->stats = stats;
1255         return err;
1256 }
1257
1258 static int damon_sysfs_scheme_set_tried_regions(
1259                 struct damon_sysfs_scheme *scheme)
1260 {
1261         struct damon_sysfs_scheme_regions *tried_regions =
1262                 damon_sysfs_scheme_regions_alloc();
1263         int err;
1264
1265         if (!tried_regions)
1266                 return -ENOMEM;
1267         err = kobject_init_and_add(&tried_regions->kobj,
1268                         &damon_sysfs_scheme_regions_ktype, &scheme->kobj,
1269                         "tried_regions");
1270         if (err)
1271                 kobject_put(&tried_regions->kobj);
1272         else
1273                 scheme->tried_regions = tried_regions;
1274         return err;
1275 }
1276
1277 static int damon_sysfs_scheme_add_dirs(struct damon_sysfs_scheme *scheme)
1278 {
1279         int err;
1280
1281         err = damon_sysfs_scheme_set_access_pattern(scheme);
1282         if (err)
1283                 return err;
1284         err = damon_sysfs_scheme_set_quotas(scheme);
1285         if (err)
1286                 goto put_access_pattern_out;
1287         err = damon_sysfs_scheme_set_watermarks(scheme);
1288         if (err)
1289                 goto put_quotas_access_pattern_out;
1290         err = damon_sysfs_scheme_set_filters(scheme);
1291         if (err)
1292                 goto put_watermarks_quotas_access_pattern_out;
1293         err = damon_sysfs_scheme_set_stats(scheme);
1294         if (err)
1295                 goto put_filters_watermarks_quotas_access_pattern_out;
1296         err = damon_sysfs_scheme_set_tried_regions(scheme);
1297         if (err)
1298                 goto put_tried_regions_out;
1299         return 0;
1300
1301 put_tried_regions_out:
1302         kobject_put(&scheme->tried_regions->kobj);
1303         scheme->tried_regions = NULL;
1304 put_filters_watermarks_quotas_access_pattern_out:
1305         kobject_put(&scheme->filters->kobj);
1306         scheme->filters = NULL;
1307 put_watermarks_quotas_access_pattern_out:
1308         kobject_put(&scheme->watermarks->kobj);
1309         scheme->watermarks = NULL;
1310 put_quotas_access_pattern_out:
1311         kobject_put(&scheme->quotas->kobj);
1312         scheme->quotas = NULL;
1313 put_access_pattern_out:
1314         kobject_put(&scheme->access_pattern->kobj);
1315         scheme->access_pattern = NULL;
1316         return err;
1317 }
1318
1319 static void damon_sysfs_scheme_rm_dirs(struct damon_sysfs_scheme *scheme)
1320 {
1321         damon_sysfs_access_pattern_rm_dirs(scheme->access_pattern);
1322         kobject_put(&scheme->access_pattern->kobj);
1323         damon_sysfs_quotas_rm_dirs(scheme->quotas);
1324         kobject_put(&scheme->quotas->kobj);
1325         kobject_put(&scheme->watermarks->kobj);
1326         damon_sysfs_scheme_filters_rm_dirs(scheme->filters);
1327         kobject_put(&scheme->filters->kobj);
1328         kobject_put(&scheme->stats->kobj);
1329         damon_sysfs_scheme_regions_rm_dirs(scheme->tried_regions);
1330         kobject_put(&scheme->tried_regions->kobj);
1331 }
1332
1333 static ssize_t action_show(struct kobject *kobj, struct kobj_attribute *attr,
1334                 char *buf)
1335 {
1336         struct damon_sysfs_scheme *scheme = container_of(kobj,
1337                         struct damon_sysfs_scheme, kobj);
1338
1339         return sysfs_emit(buf, "%s\n",
1340                         damon_sysfs_damos_action_strs[scheme->action]);
1341 }
1342
1343 static ssize_t action_store(struct kobject *kobj, struct kobj_attribute *attr,
1344                 const char *buf, size_t count)
1345 {
1346         struct damon_sysfs_scheme *scheme = container_of(kobj,
1347                         struct damon_sysfs_scheme, kobj);
1348         enum damos_action action;
1349
1350         for (action = 0; action < NR_DAMOS_ACTIONS; action++) {
1351                 if (sysfs_streq(buf, damon_sysfs_damos_action_strs[action])) {
1352                         scheme->action = action;
1353                         return count;
1354                 }
1355         }
1356         return -EINVAL;
1357 }
1358
1359 static void damon_sysfs_scheme_release(struct kobject *kobj)
1360 {
1361         kfree(container_of(kobj, struct damon_sysfs_scheme, kobj));
1362 }
1363
1364 static struct kobj_attribute damon_sysfs_scheme_action_attr =
1365                 __ATTR_RW_MODE(action, 0600);
1366
1367 static struct attribute *damon_sysfs_scheme_attrs[] = {
1368         &damon_sysfs_scheme_action_attr.attr,
1369         NULL,
1370 };
1371 ATTRIBUTE_GROUPS(damon_sysfs_scheme);
1372
1373 static const struct kobj_type damon_sysfs_scheme_ktype = {
1374         .release = damon_sysfs_scheme_release,
1375         .sysfs_ops = &kobj_sysfs_ops,
1376         .default_groups = damon_sysfs_scheme_groups,
1377 };
1378
1379 /*
1380  * schemes directory
1381  */
1382
1383 struct damon_sysfs_schemes *damon_sysfs_schemes_alloc(void)
1384 {
1385         return kzalloc(sizeof(struct damon_sysfs_schemes), GFP_KERNEL);
1386 }
1387
1388 void damon_sysfs_schemes_rm_dirs(struct damon_sysfs_schemes *schemes)
1389 {
1390         struct damon_sysfs_scheme **schemes_arr = schemes->schemes_arr;
1391         int i;
1392
1393         for (i = 0; i < schemes->nr; i++) {
1394                 damon_sysfs_scheme_rm_dirs(schemes_arr[i]);
1395                 kobject_put(&schemes_arr[i]->kobj);
1396         }
1397         schemes->nr = 0;
1398         kfree(schemes_arr);
1399         schemes->schemes_arr = NULL;
1400 }
1401
1402 static int damon_sysfs_schemes_add_dirs(struct damon_sysfs_schemes *schemes,
1403                 int nr_schemes)
1404 {
1405         struct damon_sysfs_scheme **schemes_arr, *scheme;
1406         int err, i;
1407
1408         damon_sysfs_schemes_rm_dirs(schemes);
1409         if (!nr_schemes)
1410                 return 0;
1411
1412         schemes_arr = kmalloc_array(nr_schemes, sizeof(*schemes_arr),
1413                         GFP_KERNEL | __GFP_NOWARN);
1414         if (!schemes_arr)
1415                 return -ENOMEM;
1416         schemes->schemes_arr = schemes_arr;
1417
1418         for (i = 0; i < nr_schemes; i++) {
1419                 scheme = damon_sysfs_scheme_alloc(DAMOS_STAT);
1420                 if (!scheme) {
1421                         damon_sysfs_schemes_rm_dirs(schemes);
1422                         return -ENOMEM;
1423                 }
1424
1425                 err = kobject_init_and_add(&scheme->kobj,
1426                                 &damon_sysfs_scheme_ktype, &schemes->kobj,
1427                                 "%d", i);
1428                 if (err)
1429                         goto out;
1430                 err = damon_sysfs_scheme_add_dirs(scheme);
1431                 if (err)
1432                         goto out;
1433
1434                 schemes_arr[i] = scheme;
1435                 schemes->nr++;
1436         }
1437         return 0;
1438
1439 out:
1440         damon_sysfs_schemes_rm_dirs(schemes);
1441         kobject_put(&scheme->kobj);
1442         return err;
1443 }
1444
1445 static ssize_t nr_schemes_show(struct kobject *kobj,
1446                 struct kobj_attribute *attr, char *buf)
1447 {
1448         struct damon_sysfs_schemes *schemes = container_of(kobj,
1449                         struct damon_sysfs_schemes, kobj);
1450
1451         return sysfs_emit(buf, "%d\n", schemes->nr);
1452 }
1453
1454 static ssize_t nr_schemes_store(struct kobject *kobj,
1455                 struct kobj_attribute *attr, const char *buf, size_t count)
1456 {
1457         struct damon_sysfs_schemes *schemes;
1458         int nr, err = kstrtoint(buf, 0, &nr);
1459
1460         if (err)
1461                 return err;
1462         if (nr < 0)
1463                 return -EINVAL;
1464
1465         schemes = container_of(kobj, struct damon_sysfs_schemes, kobj);
1466
1467         if (!mutex_trylock(&damon_sysfs_lock))
1468                 return -EBUSY;
1469         err = damon_sysfs_schemes_add_dirs(schemes, nr);
1470         mutex_unlock(&damon_sysfs_lock);
1471         if (err)
1472                 return err;
1473         return count;
1474 }
1475
1476 static void damon_sysfs_schemes_release(struct kobject *kobj)
1477 {
1478         kfree(container_of(kobj, struct damon_sysfs_schemes, kobj));
1479 }
1480
1481 static struct kobj_attribute damon_sysfs_schemes_nr_attr =
1482                 __ATTR_RW_MODE(nr_schemes, 0600);
1483
1484 static struct attribute *damon_sysfs_schemes_attrs[] = {
1485         &damon_sysfs_schemes_nr_attr.attr,
1486         NULL,
1487 };
1488 ATTRIBUTE_GROUPS(damon_sysfs_schemes);
1489
1490 const struct kobj_type damon_sysfs_schemes_ktype = {
1491         .release = damon_sysfs_schemes_release,
1492         .sysfs_ops = &kobj_sysfs_ops,
1493         .default_groups = damon_sysfs_schemes_groups,
1494 };
1495
1496 static bool damon_sysfs_memcg_path_eq(struct mem_cgroup *memcg,
1497                 char *memcg_path_buf, char *path)
1498 {
1499 #ifdef CONFIG_MEMCG
1500         cgroup_path(memcg->css.cgroup, memcg_path_buf, PATH_MAX);
1501         if (sysfs_streq(memcg_path_buf, path))
1502                 return true;
1503 #endif /* CONFIG_MEMCG */
1504         return false;
1505 }
1506
1507 static int damon_sysfs_memcg_path_to_id(char *memcg_path, unsigned short *id)
1508 {
1509         struct mem_cgroup *memcg;
1510         char *path;
1511         bool found = false;
1512
1513         if (!memcg_path)
1514                 return -EINVAL;
1515
1516         path = kmalloc(sizeof(*path) * PATH_MAX, GFP_KERNEL);
1517         if (!path)
1518                 return -ENOMEM;
1519
1520         for (memcg = mem_cgroup_iter(NULL, NULL, NULL); memcg;
1521                         memcg = mem_cgroup_iter(NULL, memcg, NULL)) {
1522                 /* skip removed memcg */
1523                 if (!mem_cgroup_id(memcg))
1524                         continue;
1525                 if (damon_sysfs_memcg_path_eq(memcg, path, memcg_path)) {
1526                         *id = mem_cgroup_id(memcg);
1527                         found = true;
1528                         break;
1529                 }
1530         }
1531
1532         kfree(path);
1533         return found ? 0 : -EINVAL;
1534 }
1535
1536 static int damon_sysfs_set_scheme_filters(struct damos *scheme,
1537                 struct damon_sysfs_scheme_filters *sysfs_filters)
1538 {
1539         int i;
1540         struct damos_filter *filter, *next;
1541
1542         damos_for_each_filter_safe(filter, next, scheme)
1543                 damos_destroy_filter(filter);
1544
1545         for (i = 0; i < sysfs_filters->nr; i++) {
1546                 struct damon_sysfs_scheme_filter *sysfs_filter =
1547                         sysfs_filters->filters_arr[i];
1548                 struct damos_filter *filter =
1549                         damos_new_filter(sysfs_filter->type,
1550                                         sysfs_filter->matching);
1551                 int err;
1552
1553                 if (!filter)
1554                         return -ENOMEM;
1555                 if (filter->type == DAMOS_FILTER_TYPE_MEMCG) {
1556                         err = damon_sysfs_memcg_path_to_id(
1557                                         sysfs_filter->memcg_path,
1558                                         &filter->memcg_id);
1559                         if (err) {
1560                                 damos_destroy_filter(filter);
1561                                 return err;
1562                         }
1563                 } else if (filter->type == DAMOS_FILTER_TYPE_ADDR) {
1564                         if (sysfs_filter->addr_range.end <
1565                                         sysfs_filter->addr_range.start) {
1566                                 damos_destroy_filter(filter);
1567                                 return -EINVAL;
1568                         }
1569                         filter->addr_range = sysfs_filter->addr_range;
1570                 } else if (filter->type == DAMOS_FILTER_TYPE_TARGET) {
1571                         filter->target_idx = sysfs_filter->target_idx;
1572                 }
1573
1574                 damos_add_filter(scheme, filter);
1575         }
1576         return 0;
1577 }
1578
1579 static struct damos *damon_sysfs_mk_scheme(
1580                 struct damon_sysfs_scheme *sysfs_scheme)
1581 {
1582         struct damon_sysfs_access_pattern *access_pattern =
1583                 sysfs_scheme->access_pattern;
1584         struct damon_sysfs_quotas *sysfs_quotas = sysfs_scheme->quotas;
1585         struct damon_sysfs_weights *sysfs_weights = sysfs_quotas->weights;
1586         struct damon_sysfs_watermarks *sysfs_wmarks = sysfs_scheme->watermarks;
1587         struct damon_sysfs_scheme_filters *sysfs_filters =
1588                 sysfs_scheme->filters;
1589         struct damos *scheme;
1590         int err;
1591
1592         struct damos_access_pattern pattern = {
1593                 .min_sz_region = access_pattern->sz->min,
1594                 .max_sz_region = access_pattern->sz->max,
1595                 .min_nr_accesses = access_pattern->nr_accesses->min,
1596                 .max_nr_accesses = access_pattern->nr_accesses->max,
1597                 .min_age_region = access_pattern->age->min,
1598                 .max_age_region = access_pattern->age->max,
1599         };
1600         struct damos_quota quota = {
1601                 .ms = sysfs_quotas->ms,
1602                 .sz = sysfs_quotas->sz,
1603                 .reset_interval = sysfs_quotas->reset_interval_ms,
1604                 .weight_sz = sysfs_weights->sz,
1605                 .weight_nr_accesses = sysfs_weights->nr_accesses,
1606                 .weight_age = sysfs_weights->age,
1607         };
1608         struct damos_watermarks wmarks = {
1609                 .metric = sysfs_wmarks->metric,
1610                 .interval = sysfs_wmarks->interval_us,
1611                 .high = sysfs_wmarks->high,
1612                 .mid = sysfs_wmarks->mid,
1613                 .low = sysfs_wmarks->low,
1614         };
1615
1616         scheme = damon_new_scheme(&pattern, sysfs_scheme->action, &quota,
1617                         &wmarks);
1618         if (!scheme)
1619                 return NULL;
1620
1621         err = damon_sysfs_set_scheme_filters(scheme, sysfs_filters);
1622         if (err) {
1623                 damon_destroy_scheme(scheme);
1624                 return NULL;
1625         }
1626         return scheme;
1627 }
1628
1629 static void damon_sysfs_update_scheme(struct damos *scheme,
1630                 struct damon_sysfs_scheme *sysfs_scheme)
1631 {
1632         struct damon_sysfs_access_pattern *access_pattern =
1633                 sysfs_scheme->access_pattern;
1634         struct damon_sysfs_quotas *sysfs_quotas = sysfs_scheme->quotas;
1635         struct damon_sysfs_weights *sysfs_weights = sysfs_quotas->weights;
1636         struct damon_sysfs_watermarks *sysfs_wmarks = sysfs_scheme->watermarks;
1637         int err;
1638
1639         scheme->pattern.min_sz_region = access_pattern->sz->min;
1640         scheme->pattern.max_sz_region = access_pattern->sz->max;
1641         scheme->pattern.min_nr_accesses = access_pattern->nr_accesses->min;
1642         scheme->pattern.max_nr_accesses = access_pattern->nr_accesses->max;
1643         scheme->pattern.min_age_region = access_pattern->age->min;
1644         scheme->pattern.max_age_region = access_pattern->age->max;
1645
1646         scheme->action = sysfs_scheme->action;
1647
1648         scheme->quota.ms = sysfs_quotas->ms;
1649         scheme->quota.sz = sysfs_quotas->sz;
1650         scheme->quota.reset_interval = sysfs_quotas->reset_interval_ms;
1651         scheme->quota.weight_sz = sysfs_weights->sz;
1652         scheme->quota.weight_nr_accesses = sysfs_weights->nr_accesses;
1653         scheme->quota.weight_age = sysfs_weights->age;
1654
1655         scheme->wmarks.metric = sysfs_wmarks->metric;
1656         scheme->wmarks.interval = sysfs_wmarks->interval_us;
1657         scheme->wmarks.high = sysfs_wmarks->high;
1658         scheme->wmarks.mid = sysfs_wmarks->mid;
1659         scheme->wmarks.low = sysfs_wmarks->low;
1660
1661         err = damon_sysfs_set_scheme_filters(scheme, sysfs_scheme->filters);
1662         if (err)
1663                 damon_destroy_scheme(scheme);
1664 }
1665
1666 int damon_sysfs_set_schemes(struct damon_ctx *ctx,
1667                 struct damon_sysfs_schemes *sysfs_schemes)
1668 {
1669         struct damos *scheme, *next;
1670         int i = 0;
1671
1672         damon_for_each_scheme_safe(scheme, next, ctx) {
1673                 if (i < sysfs_schemes->nr)
1674                         damon_sysfs_update_scheme(scheme,
1675                                         sysfs_schemes->schemes_arr[i]);
1676                 else
1677                         damon_destroy_scheme(scheme);
1678                 i++;
1679         }
1680
1681         for (; i < sysfs_schemes->nr; i++) {
1682                 struct damos *scheme, *next;
1683
1684                 scheme = damon_sysfs_mk_scheme(sysfs_schemes->schemes_arr[i]);
1685                 if (!scheme) {
1686                         damon_for_each_scheme_safe(scheme, next, ctx)
1687                                 damon_destroy_scheme(scheme);
1688                         return -ENOMEM;
1689                 }
1690                 damon_add_scheme(ctx, scheme);
1691         }
1692         return 0;
1693 }
1694
1695 void damon_sysfs_schemes_update_stats(
1696                 struct damon_sysfs_schemes *sysfs_schemes,
1697                 struct damon_ctx *ctx)
1698 {
1699         struct damos *scheme;
1700         int schemes_idx = 0;
1701
1702         damon_for_each_scheme(scheme, ctx) {
1703                 struct damon_sysfs_stats *sysfs_stats;
1704
1705                 /* user could have removed the scheme sysfs dir */
1706                 if (schemes_idx >= sysfs_schemes->nr)
1707                         break;
1708
1709                 sysfs_stats = sysfs_schemes->schemes_arr[schemes_idx++]->stats;
1710                 sysfs_stats->nr_tried = scheme->stat.nr_tried;
1711                 sysfs_stats->sz_tried = scheme->stat.sz_tried;
1712                 sysfs_stats->nr_applied = scheme->stat.nr_applied;
1713                 sysfs_stats->sz_applied = scheme->stat.sz_applied;
1714                 sysfs_stats->qt_exceeds = scheme->stat.qt_exceeds;
1715         }
1716 }
1717
1718 /*
1719  * damon_sysfs_schemes that need to update its schemes regions dir.  Protected
1720  * by damon_sysfs_lock
1721  */
1722 static struct damon_sysfs_schemes *damon_sysfs_schemes_for_damos_callback;
1723 static int damon_sysfs_schemes_region_idx;
1724 static bool damos_regions_upd_total_bytes_only;
1725
1726 /*
1727  * DAMON callback that called before damos apply.  While this callback is
1728  * registered, damon_sysfs_lock should be held to ensure the regions
1729  * directories exist.
1730  */
1731 static int damon_sysfs_before_damos_apply(struct damon_ctx *ctx,
1732                 struct damon_target *t, struct damon_region *r,
1733                 struct damos *s)
1734 {
1735         struct damos *scheme;
1736         struct damon_sysfs_scheme_regions *sysfs_regions;
1737         struct damon_sysfs_scheme_region *region;
1738         struct damon_sysfs_schemes *sysfs_schemes =
1739                 damon_sysfs_schemes_for_damos_callback;
1740         int schemes_idx = 0;
1741
1742         damon_for_each_scheme(scheme, ctx) {
1743                 if (scheme == s)
1744                         break;
1745                 schemes_idx++;
1746         }
1747
1748         /* user could have removed the scheme sysfs dir */
1749         if (schemes_idx >= sysfs_schemes->nr)
1750                 return 0;
1751
1752         sysfs_regions = sysfs_schemes->schemes_arr[schemes_idx]->tried_regions;
1753         sysfs_regions->total_bytes += r->ar.end - r->ar.start;
1754         if (damos_regions_upd_total_bytes_only)
1755                 return 0;
1756
1757         region = damon_sysfs_scheme_region_alloc(r);
1758         if (!region)
1759                 return 0;
1760         list_add_tail(&region->list, &sysfs_regions->regions_list);
1761         sysfs_regions->nr_regions++;
1762         if (kobject_init_and_add(&region->kobj,
1763                                 &damon_sysfs_scheme_region_ktype,
1764                                 &sysfs_regions->kobj, "%d",
1765                                 damon_sysfs_schemes_region_idx++)) {
1766                 kobject_put(&region->kobj);
1767         }
1768         return 0;
1769 }
1770
1771 /* Called from damon_sysfs_cmd_request_callback under damon_sysfs_lock */
1772 int damon_sysfs_schemes_clear_regions(
1773                 struct damon_sysfs_schemes *sysfs_schemes,
1774                 struct damon_ctx *ctx)
1775 {
1776         struct damos *scheme;
1777         int schemes_idx = 0;
1778
1779         damon_for_each_scheme(scheme, ctx) {
1780                 struct damon_sysfs_scheme *sysfs_scheme;
1781
1782                 /* user could have removed the scheme sysfs dir */
1783                 if (schemes_idx >= sysfs_schemes->nr)
1784                         break;
1785
1786                 sysfs_scheme = sysfs_schemes->schemes_arr[schemes_idx++];
1787                 damon_sysfs_scheme_regions_rm_dirs(
1788                                 sysfs_scheme->tried_regions);
1789                 sysfs_scheme->tried_regions->total_bytes = 0;
1790         }
1791         return 0;
1792 }
1793
1794 /* Called from damon_sysfs_cmd_request_callback under damon_sysfs_lock */
1795 int damon_sysfs_schemes_update_regions_start(
1796                 struct damon_sysfs_schemes *sysfs_schemes,
1797                 struct damon_ctx *ctx, bool total_bytes_only)
1798 {
1799         damon_sysfs_schemes_clear_regions(sysfs_schemes, ctx);
1800         damon_sysfs_schemes_for_damos_callback = sysfs_schemes;
1801         damos_regions_upd_total_bytes_only = total_bytes_only;
1802         ctx->callback.before_damos_apply = damon_sysfs_before_damos_apply;
1803         return 0;
1804 }
1805
1806 /*
1807  * Called from damon_sysfs_cmd_request_callback under damon_sysfs_lock.  Caller
1808  * should unlock damon_sysfs_lock which held before
1809  * damon_sysfs_schemes_update_regions_start()
1810  */
1811 int damon_sysfs_schemes_update_regions_stop(struct damon_ctx *ctx)
1812 {
1813         damon_sysfs_schemes_for_damos_callback = NULL;
1814         ctx->callback.before_damos_apply = NULL;
1815         damon_sysfs_schemes_region_idx = 0;
1816         return 0;
1817 }