BACKPORT: mm: multi-gen LRU: minimal implementation

[platform/kernel/linux-rpi.git] / include / linux / mmzone.h

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 2d3a8c9..a7cbcec 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -336,6 +336,28 @@ enum lruvec_flags {
 #define MIN_NR_GENS            2U
 #define MAX_NR_GENS            4U
 
+/*
+ * Each generation is divided into multiple tiers. A page accessed N times
+ * through file descriptors is in tier order_base_2(N). A page in the first tier
+ * (N=0,1) is marked by PG_referenced unless it was faulted in through page
+ * tables or read ahead. A page in any other tier (N>1) is marked by
+ * PG_referenced and PG_workingset. This implies a minimum of two tiers is
+ * supported without using additional bits in page->flags.
+ *
+ * In contrast to moving across generations which requires the LRU lock, moving
+ * across tiers only involves atomic operations on page->flags and therefore
+ * has a negligible cost in the buffered access path. In the eviction path,
+ * comparisons of refaulted/(evicted+protected) from the first tier and the
+ * rest infer whether pages accessed multiple times through file descriptors
+ * are statistically hot and thus worth protecting.
+ *
+ * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the
+ * number of categories of the active/inactive LRU when keeping track of
+ * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in
+ * page->flags.
+ */
+#define MAX_NR_TIERS           4U
+
 #ifndef __GENERATING_BOUNDS_H
 
 struct lruvec;
@@ -350,6 +372,16 @@ enum {
        LRU_GEN_FILE,
 };
 
+#define MIN_LRU_BATCH          BITS_PER_LONG
+#define MAX_LRU_BATCH          (MIN_LRU_BATCH * 64)
+
+/* whether to keep historical stats from evicted generations */
+#ifdef CONFIG_LRU_GEN_STATS
+#define NR_HIST_GENS           MAX_NR_GENS
+#else
+#define NR_HIST_GENS           1U
+#endif
+
 /*
  * The youngest generation number is stored in max_seq for both anon and file
  * types as they are aged on an equal footing. The oldest generation numbers are
@@ -372,6 +404,15 @@ struct lru_gen_struct {
        struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
        /* the multi-gen LRU sizes, eventually consistent */
        long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
+       /* the exponential moving average of refaulted */
+       unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS];
+       /* the exponential moving average of evicted+protected */
+       unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS];
+       /* the first tier doesn't need protection, hence the minus one */
+       unsigned long protected[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS - 1];
+       /* can be modified without holding the LRU lock */
+       atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS];
+       atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS];
 };
 
 void lru_gen_init_lruvec(struct lruvec *lruvec);