tools/perf/util/bpf_skel/lock_contention.bpf.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 // Copyright (c) 2022 Google
   3 #include "vmlinux.h"
   4 #include <bpf/bpf_helpers.h>
   5 #include <bpf/bpf_tracing.h>
   6 #include <bpf/bpf_core_read.h>
   7
   8 #include "lock_data.h"
   9
  10 /* default buffer size */
  11 #define MAX_ENTRIES  10240
  12
  13 /* lock contention flags from include/trace/events/lock.h */
  14 #define LCB_F_SPIN      (1U << 0)
  15 #define LCB_F_READ      (1U << 1)
  16 #define LCB_F_WRITE     (1U << 2)
  17 #define LCB_F_RT        (1U << 3)
  18 #define LCB_F_PERCPU    (1U << 4)
  19 #define LCB_F_MUTEX     (1U << 5)
  20
  21 struct tstamp_data {
  22         __u64 timestamp;
  23         __u64 lock;
  24         __u32 flags;
  25         __s32 stack_id;
  26 };
  27
  28 /* callstack storage  */
  29 struct {
  30         __uint(type, BPF_MAP_TYPE_STACK_TRACE);
  31         __uint(key_size, sizeof(__u32));
  32         __uint(value_size, sizeof(__u64));
  33         __uint(max_entries, MAX_ENTRIES);
  34 } stacks SEC(".maps");
  35
  36 /* maintain timestamp at the beginning of contention */
  37 struct {
  38         __uint(type, BPF_MAP_TYPE_HASH);
  39         __type(key, int);
  40         __type(value, struct tstamp_data);
  41         __uint(max_entries, MAX_ENTRIES);
  42 } tstamp SEC(".maps");
  43
  44 /* actual lock contention statistics */
  45 struct {
  46         __uint(type, BPF_MAP_TYPE_HASH);
  47         __uint(key_size, sizeof(struct contention_key));
  48         __uint(value_size, sizeof(struct contention_data));
  49         __uint(max_entries, MAX_ENTRIES);
  50 } lock_stat SEC(".maps");
  51
  52 struct {
  53         __uint(type, BPF_MAP_TYPE_HASH);
  54         __uint(key_size, sizeof(__u32));
  55         __uint(value_size, sizeof(struct contention_task_data));
  56         __uint(max_entries, MAX_ENTRIES);
  57 } task_data SEC(".maps");
  58
  59 struct {
  60         __uint(type, BPF_MAP_TYPE_HASH);
  61         __uint(key_size, sizeof(__u32));
  62         __uint(value_size, sizeof(__u8));
  63         __uint(max_entries, 1);
  64 } cpu_filter SEC(".maps");
  65
  66 struct {
  67         __uint(type, BPF_MAP_TYPE_HASH);
  68         __uint(key_size, sizeof(__u32));
  69         __uint(value_size, sizeof(__u8));
  70         __uint(max_entries, 1);
  71 } task_filter SEC(".maps");
  72
  73 struct {
  74         __uint(type, BPF_MAP_TYPE_HASH);
  75         __uint(key_size, sizeof(__u32));
  76         __uint(value_size, sizeof(__u8));
  77         __uint(max_entries, 1);
  78 } type_filter SEC(".maps");
  79
  80 struct {
  81         __uint(type, BPF_MAP_TYPE_HASH);
  82         __uint(key_size, sizeof(__u64));
  83         __uint(value_size, sizeof(__u8));
  84         __uint(max_entries, 1);
  85 } addr_filter SEC(".maps");
  86
  87 struct rw_semaphore___old {
  88         struct task_struct *owner;
  89 } __attribute__((preserve_access_index));
  90
  91 struct rw_semaphore___new {
  92         atomic_long_t owner;
  93 } __attribute__((preserve_access_index));
  94
  95 struct mm_struct___old {
  96         struct rw_semaphore mmap_sem;
  97 } __attribute__((preserve_access_index));
  98
  99 struct mm_struct___new {
 100         struct rw_semaphore mmap_lock;
 101 } __attribute__((preserve_access_index));
 102
 103 /* control flags */
 104 int enabled;
 105 int has_cpu;
 106 int has_task;
 107 int has_type;
 108 int has_addr;
 109 int needs_callstack;
 110 int stack_skip;
 111 int lock_owner;
 112
 113 /* determine the key of lock stat */
 114 int aggr_mode;
 115
 116 /* error stat */
 117 int lost;
 118
 119 static inline int can_record(u64 *ctx)
 120 {
 121         if (has_cpu) {
 122                 __u32 cpu = bpf_get_smp_processor_id();
 123                 __u8 *ok;
 124
 125                 ok = bpf_map_lookup_elem(&cpu_filter, &cpu);
 126                 if (!ok)
 127                         return 0;
 128         }
 129
 130         if (has_task) {
 131                 __u8 *ok;
 132                 __u32 pid = bpf_get_current_pid_tgid();
 133
 134                 ok = bpf_map_lookup_elem(&task_filter, &pid);
 135                 if (!ok)
 136                         return 0;
 137         }
 138
 139         if (has_type) {
 140                 __u8 *ok;
 141                 __u32 flags = (__u32)ctx[1];
 142
 143                 ok = bpf_map_lookup_elem(&type_filter, &flags);
 144                 if (!ok)
 145                         return 0;
 146         }
 147
 148         if (has_addr) {
 149                 __u8 *ok;
 150                 __u64 addr = ctx[0];
 151
 152                 ok = bpf_map_lookup_elem(&addr_filter, &addr);
 153                 if (!ok)
 154                         return 0;
 155         }
 156
 157         return 1;
 158 }
 159
 160 static inline int update_task_data(struct task_struct *task)
 161 {
 162         struct contention_task_data *p;
 163         int pid, err;
 164
 165         err = bpf_core_read(&pid, sizeof(pid), &task->pid);
 166         if (err)
 167                 return -1;
 168
 169         p = bpf_map_lookup_elem(&task_data, &pid);
 170         if (p == NULL) {
 171                 struct contention_task_data data = {};
 172
 173                 BPF_CORE_READ_STR_INTO(&data.comm, task, comm);
 174                 bpf_map_update_elem(&task_data, &pid, &data, BPF_NOEXIST);
 175         }
 176
 177         return 0;
 178 }
 179
 180 #ifndef __has_builtin
 181 # define __has_builtin(x) 0
 182 #endif
 183
 184 static inline struct task_struct *get_lock_owner(__u64 lock, __u32 flags)
 185 {
 186         struct task_struct *task;
 187         __u64 owner = 0;
 188
 189         if (flags & LCB_F_MUTEX) {
 190                 struct mutex *mutex = (void *)lock;
 191                 owner = BPF_CORE_READ(mutex, owner.counter);
 192         } else if (flags == LCB_F_READ || flags == LCB_F_WRITE) {
 193         /*
 194          * Support for the BPF_TYPE_MATCHES argument to the
 195          * __builtin_preserve_type_info builtin was added at some point during
 196          * development of clang 15 and it's what is needed for
 197          * bpf_core_type_matches.
 198          */
 199 #if __has_builtin(__builtin_preserve_type_info) && __clang_major__ >= 15
 200                 if (bpf_core_type_matches(struct rw_semaphore___old)) {
 201                         struct rw_semaphore___old *rwsem = (void *)lock;
 202                         owner = (unsigned long)BPF_CORE_READ(rwsem, owner);
 203                 } else if (bpf_core_type_matches(struct rw_semaphore___new)) {
 204                         struct rw_semaphore___new *rwsem = (void *)lock;
 205                         owner = BPF_CORE_READ(rwsem, owner.counter);
 206                 }
 207 #else
 208                 /* assume new struct */
 209                 struct rw_semaphore *rwsem = (void *)lock;
 210                 owner = BPF_CORE_READ(rwsem, owner.counter);
 211 #endif
 212         }
 213
 214         if (!owner)
 215                 return NULL;
 216
 217         task = (void *)(owner & ~7UL);
 218         return task;
 219 }
 220
 221 static inline __u32 check_lock_type(__u64 lock, __u32 flags)
 222 {
 223         struct task_struct *curr;
 224         struct mm_struct___old *mm_old;
 225         struct mm_struct___new *mm_new;
 226
 227         switch (flags) {
 228         case LCB_F_READ:  /* rwsem */
 229         case LCB_F_WRITE:
 230                 curr = bpf_get_current_task_btf();
 231                 if (curr->mm == NULL)
 232                         break;
 233                 mm_new = (void *)curr->mm;
 234                 if (bpf_core_field_exists(mm_new->mmap_lock)) {
 235                         if (&mm_new->mmap_lock == (void *)lock)
 236                                 return LCD_F_MMAP_LOCK;
 237                         break;
 238                 }
 239                 mm_old = (void *)curr->mm;
 240                 if (bpf_core_field_exists(mm_old->mmap_sem)) {
 241                         if (&mm_old->mmap_sem == (void *)lock)
 242                                 return LCD_F_MMAP_LOCK;
 243                 }
 244                 break;
 245         case LCB_F_SPIN:  /* spinlock */
 246                 curr = bpf_get_current_task_btf();
 247                 if (&curr->sighand->siglock == (void *)lock)
 248                         return LCD_F_SIGHAND_LOCK;
 249                 break;
 250         default:
 251                 break;
 252         }
 253         return 0;
 254 }
 255
 256 SEC("tp_btf/contention_begin")
 257 int contention_begin(u64 *ctx)
 258 {
 259         __u32 pid;
 260         struct tstamp_data *pelem;
 261
 262         if (!enabled || !can_record(ctx))
 263                 return 0;
 264
 265         pid = bpf_get_current_pid_tgid();
 266         pelem = bpf_map_lookup_elem(&tstamp, &pid);
 267         if (pelem && pelem->lock)
 268                 return 0;
 269
 270         if (pelem == NULL) {
 271                 struct tstamp_data zero = {};
 272
 273                 bpf_map_update_elem(&tstamp, &pid, &zero, BPF_ANY);
 274                 pelem = bpf_map_lookup_elem(&tstamp, &pid);
 275                 if (pelem == NULL) {
 276                         lost++;
 277                         return 0;
 278                 }
 279         }
 280
 281         pelem->timestamp = bpf_ktime_get_ns();
 282         pelem->lock = (__u64)ctx[0];
 283         pelem->flags = (__u32)ctx[1];
 284
 285         if (needs_callstack) {
 286                 pelem->stack_id = bpf_get_stackid(ctx, &stacks,
 287                                                   BPF_F_FAST_STACK_CMP | stack_skip);
 288                 if (pelem->stack_id < 0)
 289                         lost++;
 290         } else if (aggr_mode == LOCK_AGGR_TASK) {
 291                 struct task_struct *task;
 292
 293                 if (lock_owner) {
 294                         task = get_lock_owner(pelem->lock, pelem->flags);
 295
 296                         /* The flags is not used anymore.  Pass the owner pid. */
 297                         if (task)
 298                                 pelem->flags = BPF_CORE_READ(task, pid);
 299                         else
 300                                 pelem->flags = -1U;
 301
 302                 } else {
 303                         task = bpf_get_current_task_btf();
 304                 }
 305
 306                 if (task) {
 307                         if (update_task_data(task) < 0 && lock_owner)
 308                                 pelem->flags = -1U;
 309                 }
 310         }
 311
 312         return 0;
 313 }
 314
 315 SEC("tp_btf/contention_end")
 316 int contention_end(u64 *ctx)
 317 {
 318         __u32 pid;
 319         struct tstamp_data *pelem;
 320         struct contention_key key = {};
 321         struct contention_data *data;
 322         __u64 duration;
 323
 324         if (!enabled)
 325                 return 0;
 326
 327         pid = bpf_get_current_pid_tgid();
 328         pelem = bpf_map_lookup_elem(&tstamp, &pid);
 329         if (!pelem || pelem->lock != ctx[0])
 330                 return 0;
 331
 332         duration = bpf_ktime_get_ns() - pelem->timestamp;
 333
 334         switch (aggr_mode) {
 335         case LOCK_AGGR_CALLER:
 336                 key.stack_id = pelem->stack_id;
 337                 break;
 338         case LOCK_AGGR_TASK:
 339                 if (lock_owner)
 340                         key.pid = pelem->flags;
 341                 else
 342                         key.pid = pid;
 343                 if (needs_callstack)
 344                         key.stack_id = pelem->stack_id;
 345                 break;
 346         case LOCK_AGGR_ADDR:
 347                 key.lock_addr = pelem->lock;
 348                 if (needs_callstack)
 349                         key.stack_id = pelem->stack_id;
 350                 break;
 351         default:
 352                 /* should not happen */
 353                 return 0;
 354         }
 355
 356         data = bpf_map_lookup_elem(&lock_stat, &key);
 357         if (!data) {
 358                 struct contention_data first = {
 359                         .total_time = duration,
 360                         .max_time = duration,
 361                         .min_time = duration,
 362                         .count = 1,
 363                         .flags = pelem->flags,
 364                 };
 365
 366                 if (aggr_mode == LOCK_AGGR_ADDR)
 367                         first.flags |= check_lock_type(pelem->lock, pelem->flags);
 368
 369                 bpf_map_update_elem(&lock_stat, &key, &first, BPF_NOEXIST);
 370                 bpf_map_delete_elem(&tstamp, &pid);
 371                 return 0;
 372         }
 373
 374         __sync_fetch_and_add(&data->total_time, duration);
 375         __sync_fetch_and_add(&data->count, 1);
 376
 377         /* FIXME: need atomic operations */
 378         if (data->max_time < duration)
 379                 data->max_time = duration;
 380         if (data->min_time > duration)
 381                 data->min_time = duration;
 382
 383         bpf_map_delete_elem(&tstamp, &pid);
 384         return 0;
 385 }
 386
 387 char LICENSE[] SEC("license") = "Dual BSD/GPL";