mm/mmap_lock.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #define CREATE_TRACE_POINTS
   3 #include <trace/events/mmap_lock.h>
   4
   5 #include <linux/mm.h>
   6 #include <linux/cgroup.h>
   7 #include <linux/memcontrol.h>
   8 #include <linux/mmap_lock.h>
   9 #include <linux/mutex.h>
  10 #include <linux/percpu.h>
  11 #include <linux/rcupdate.h>
  12 #include <linux/smp.h>
  13 #include <linux/trace_events.h>
  14 #include <linux/local_lock.h>
  15
  16 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_start_locking);
  17 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_acquire_returned);
  18 EXPORT_TRACEPOINT_SYMBOL(mmap_lock_released);
  19
  20 #ifdef CONFIG_MEMCG
  21
  22 /*
  23  * Our various events all share the same buffer (because we don't want or need
  24  * to allocate a set of buffers *per event type*), so we need to protect against
  25  * concurrent _reg() and _unreg() calls, and count how many _reg() calls have
  26  * been made.
  27  */
  28 static DEFINE_MUTEX(reg_lock);
  29 static int reg_refcount; /* Protected by reg_lock. */
  30
  31 /*
  32  * Size of the buffer for memcg path names. Ignoring stack trace support,
  33  * trace_events_hist.c uses MAX_FILTER_STR_VAL for this, so we also use it.
  34  */
  35 #define MEMCG_PATH_BUF_SIZE MAX_FILTER_STR_VAL
  36
  37 /*
  38  * How many contexts our trace events might be called in: normal, softirq, irq,
  39  * and NMI.
  40  */
  41 #define CONTEXT_COUNT 4
  42
  43 struct memcg_path {
  44         local_lock_t lock;
  45         char __rcu *buf;
  46         local_t buf_idx;
  47 };
  48 static DEFINE_PER_CPU(struct memcg_path, memcg_paths) = {
  49         .lock = INIT_LOCAL_LOCK(lock),
  50         .buf_idx = LOCAL_INIT(0),
  51 };
  52
  53 static char **tmp_bufs;
  54
  55 /* Called with reg_lock held. */
  56 static void free_memcg_path_bufs(void)
  57 {
  58         struct memcg_path *memcg_path;
  59         int cpu;
  60         char **old = tmp_bufs;
  61
  62         for_each_possible_cpu(cpu) {
  63                 memcg_path = per_cpu_ptr(&memcg_paths, cpu);
  64                 *(old++) = rcu_dereference_protected(memcg_path->buf,
  65                         lockdep_is_held(&reg_lock));
  66                 rcu_assign_pointer(memcg_path->buf, NULL);
  67         }
  68
  69         /* Wait for inflight memcg_path_buf users to finish. */
  70         synchronize_rcu();
  71
  72         old = tmp_bufs;
  73         for_each_possible_cpu(cpu) {
  74                 kfree(*(old++));
  75         }
  76
  77         kfree(tmp_bufs);
  78         tmp_bufs = NULL;
  79 }
  80
  81 int trace_mmap_lock_reg(void)
  82 {
  83         int cpu;
  84         char *new;
  85
  86         mutex_lock(&reg_lock);
  87
  88         /* If the refcount is going 0->1, proceed with allocating buffers. */
  89         if (reg_refcount++)
  90                 goto out;
  91
  92         tmp_bufs = kmalloc_array(num_possible_cpus(), sizeof(*tmp_bufs),
  93                                  GFP_KERNEL);
  94         if (tmp_bufs == NULL)
  95                 goto out_fail;
  96
  97         for_each_possible_cpu(cpu) {
  98                 new = kmalloc(MEMCG_PATH_BUF_SIZE * CONTEXT_COUNT, GFP_KERNEL);
  99                 if (new == NULL)
 100                         goto out_fail_free;
 101                 rcu_assign_pointer(per_cpu_ptr(&memcg_paths, cpu)->buf, new);
 102                 /* Don't need to wait for inflights, they'd have gotten NULL. */
 103         }
 104
 105 out:
 106         mutex_unlock(&reg_lock);
 107         return 0;
 108
 109 out_fail_free:
 110         free_memcg_path_bufs();
 111 out_fail:
 112         /* Since we failed, undo the earlier ref increment. */
 113         --reg_refcount;
 114
 115         mutex_unlock(&reg_lock);
 116         return -ENOMEM;
 117 }
 118
 119 void trace_mmap_lock_unreg(void)
 120 {
 121         mutex_lock(&reg_lock);
 122
 123         /* If the refcount is going 1->0, proceed with freeing buffers. */
 124         if (--reg_refcount)
 125                 goto out;
 126
 127         free_memcg_path_bufs();
 128
 129 out:
 130         mutex_unlock(&reg_lock);
 131 }
 132
 133 static inline char *get_memcg_path_buf(void)
 134 {
 135         struct memcg_path *memcg_path = this_cpu_ptr(&memcg_paths);
 136         char *buf;
 137         int idx;
 138
 139         rcu_read_lock();
 140         buf = rcu_dereference(memcg_path->buf);
 141         if (buf == NULL) {
 142                 rcu_read_unlock();
 143                 return NULL;
 144         }
 145         idx = local_add_return(MEMCG_PATH_BUF_SIZE, &memcg_path->buf_idx) -
 146               MEMCG_PATH_BUF_SIZE;
 147         return &buf[idx];
 148 }
 149
 150 static inline void put_memcg_path_buf(void)
 151 {
 152         local_sub(MEMCG_PATH_BUF_SIZE, &this_cpu_ptr(&memcg_paths)->buf_idx);
 153         rcu_read_unlock();
 154 }
 155
 156 /*
 157  * Write the given mm_struct's memcg path to a percpu buffer, and return a
 158  * pointer to it. If the path cannot be determined, or no buffer was available
 159  * (because the trace event is being unregistered), NULL is returned.
 160  *
 161  * Note: buffers are allocated per-cpu to avoid locking, so preemption must be
 162  * disabled by the caller before calling us, and re-enabled only after the
 163  * caller is done with the pointer.
 164  *
 165  * The caller must call put_memcg_path_buf() once the buffer is no longer
 166  * needed. This must be done while preemption is still disabled.
 167  */
 168 static const char *get_mm_memcg_path(struct mm_struct *mm)
 169 {
 170         char *buf = NULL;
 171         struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm);
 172
 173         if (memcg == NULL)
 174                 goto out;
 175         if (unlikely(memcg->css.cgroup == NULL))
 176                 goto out_put;
 177
 178         buf = get_memcg_path_buf();
 179         if (buf == NULL)
 180                 goto out_put;
 181
 182         cgroup_path(memcg->css.cgroup, buf, MEMCG_PATH_BUF_SIZE);
 183
 184 out_put:
 185         css_put(&memcg->css);
 186 out:
 187         return buf;
 188 }
 189
 190 #define TRACE_MMAP_LOCK_EVENT(type, mm, ...)                                   \
 191         do {                                                                   \
 192                 const char *memcg_path;                                        \
 193                 local_lock(&memcg_paths.lock);                                 \
 194                 memcg_path = get_mm_memcg_path(mm);                            \
 195                 trace_mmap_lock_##type(mm,                                     \
 196                                        memcg_path != NULL ? memcg_path : "",   \
 197                                        ##__VA_ARGS__);                         \
 198                 if (likely(memcg_path != NULL))                                \
 199                         put_memcg_path_buf();                                  \
 200                 local_unlock(&memcg_paths.lock);                               \
 201         } while (0)
 202
 203 #else /* !CONFIG_MEMCG */
 204
 205 int trace_mmap_lock_reg(void)
 206 {
 207         return 0;
 208 }
 209
 210 void trace_mmap_lock_unreg(void)
 211 {
 212 }
 213
 214 #define TRACE_MMAP_LOCK_EVENT(type, mm, ...)                                   \
 215         trace_mmap_lock_##type(mm, "", ##__VA_ARGS__)
 216
 217 #endif /* CONFIG_MEMCG */
 218
 219 /*
 220  * Trace calls must be in a separate file, as otherwise there's a circular
 221  * dependency between linux/mmap_lock.h and trace/events/mmap_lock.h.
 222  */
 223
 224 void __mmap_lock_do_trace_start_locking(struct mm_struct *mm, bool write)
 225 {
 226         TRACE_MMAP_LOCK_EVENT(start_locking, mm, write);
 227 }
 228 EXPORT_SYMBOL(__mmap_lock_do_trace_start_locking);
 229
 230 void __mmap_lock_do_trace_acquire_returned(struct mm_struct *mm, bool write,
 231                                            bool success)
 232 {
 233         TRACE_MMAP_LOCK_EVENT(acquire_returned, mm, write, success);
 234 }
 235 EXPORT_SYMBOL(__mmap_lock_do_trace_acquire_returned);
 236
 237 void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write)
 238 {
 239         TRACE_MMAP_LOCK_EVENT(released, mm, write);
 240 }
 241 EXPORT_SYMBOL(__mmap_lock_do_trace_released);