tools/perf/util/bpf_skel/lock_contention.bpf.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 // Copyright (c) 2022 Google
   3 #include "vmlinux.h"
   4 #include <bpf/bpf_helpers.h>
   5 #include <bpf/bpf_tracing.h>
   6 #include <bpf/bpf_core_read.h>
   7
   8 #include "lock_data.h"
   9
  10 /* for collect_lock_syms().  4096 was rejected by the verifier */
  11 #define MAX_CPUS  1024
  12
  13 /* lock contention flags from include/trace/events/lock.h */
  14 #define LCB_F_SPIN      (1U << 0)
  15 #define LCB_F_READ      (1U << 1)
  16 #define LCB_F_WRITE     (1U << 2)
  17 #define LCB_F_RT        (1U << 3)
  18 #define LCB_F_PERCPU    (1U << 4)
  19 #define LCB_F_MUTEX     (1U << 5)
  20
  21 struct tstamp_data {
  22         __u64 timestamp;
  23         __u64 lock;
  24         __u32 flags;
  25         __s32 stack_id;
  26 };
  27
  28 /* callstack storage  */
  29 struct {
  30         __uint(type, BPF_MAP_TYPE_STACK_TRACE);
  31         __uint(key_size, sizeof(__u32));
  32         __uint(value_size, sizeof(__u64));
  33         __uint(max_entries, MAX_ENTRIES);
  34 } stacks SEC(".maps");
  35
  36 /* maintain timestamp at the beginning of contention */
  37 struct {
  38         __uint(type, BPF_MAP_TYPE_HASH);
  39         __type(key, int);
  40         __type(value, struct tstamp_data);
  41         __uint(max_entries, MAX_ENTRIES);
  42 } tstamp SEC(".maps");
  43
  44 /* actual lock contention statistics */
  45 struct {
  46         __uint(type, BPF_MAP_TYPE_HASH);
  47         __uint(key_size, sizeof(struct contention_key));
  48         __uint(value_size, sizeof(struct contention_data));
  49         __uint(max_entries, MAX_ENTRIES);
  50 } lock_stat SEC(".maps");
  51
  52 struct {
  53         __uint(type, BPF_MAP_TYPE_HASH);
  54         __uint(key_size, sizeof(__u32));
  55         __uint(value_size, sizeof(struct contention_task_data));
  56         __uint(max_entries, MAX_ENTRIES);
  57 } task_data SEC(".maps");
  58
  59 struct {
  60         __uint(type, BPF_MAP_TYPE_HASH);
  61         __uint(key_size, sizeof(__u64));
  62         __uint(value_size, sizeof(__u32));
  63         __uint(max_entries, MAX_ENTRIES);
  64 } lock_syms SEC(".maps");
  65
  66 struct {
  67         __uint(type, BPF_MAP_TYPE_HASH);
  68         __uint(key_size, sizeof(__u32));
  69         __uint(value_size, sizeof(__u8));
  70         __uint(max_entries, 1);
  71 } cpu_filter SEC(".maps");
  72
  73 struct {
  74         __uint(type, BPF_MAP_TYPE_HASH);
  75         __uint(key_size, sizeof(__u32));
  76         __uint(value_size, sizeof(__u8));
  77         __uint(max_entries, 1);
  78 } task_filter SEC(".maps");
  79
  80 struct {
  81         __uint(type, BPF_MAP_TYPE_HASH);
  82         __uint(key_size, sizeof(__u32));
  83         __uint(value_size, sizeof(__u8));
  84         __uint(max_entries, 1);
  85 } type_filter SEC(".maps");
  86
  87 struct {
  88         __uint(type, BPF_MAP_TYPE_HASH);
  89         __uint(key_size, sizeof(__u64));
  90         __uint(value_size, sizeof(__u8));
  91         __uint(max_entries, 1);
  92 } addr_filter SEC(".maps");
  93
  94 struct rw_semaphore___old {
  95         struct task_struct *owner;
  96 } __attribute__((preserve_access_index));
  97
  98 struct rw_semaphore___new {
  99         atomic_long_t owner;
 100 } __attribute__((preserve_access_index));
 101
 102 struct mm_struct___old {
 103         struct rw_semaphore mmap_sem;
 104 } __attribute__((preserve_access_index));
 105
 106 struct mm_struct___new {
 107         struct rw_semaphore mmap_lock;
 108 } __attribute__((preserve_access_index));
 109
 110 /* control flags */
 111 int enabled;
 112 int has_cpu;
 113 int has_task;
 114 int has_type;
 115 int has_addr;
 116 int needs_callstack;
 117 int stack_skip;
 118 int lock_owner;
 119
 120 /* determine the key of lock stat */
 121 int aggr_mode;
 122
 123 /* error stat */
 124 int task_fail;
 125 int stack_fail;
 126 int time_fail;
 127
 128 static inline int can_record(u64 *ctx)
 129 {
 130         if (has_cpu) {
 131                 __u32 cpu = bpf_get_smp_processor_id();
 132                 __u8 *ok;
 133
 134                 ok = bpf_map_lookup_elem(&cpu_filter, &cpu);
 135                 if (!ok)
 136                         return 0;
 137         }
 138
 139         if (has_task) {
 140                 __u8 *ok;
 141                 __u32 pid = bpf_get_current_pid_tgid();
 142
 143                 ok = bpf_map_lookup_elem(&task_filter, &pid);
 144                 if (!ok)
 145                         return 0;
 146         }
 147
 148         if (has_type) {
 149                 __u8 *ok;
 150                 __u32 flags = (__u32)ctx[1];
 151
 152                 ok = bpf_map_lookup_elem(&type_filter, &flags);
 153                 if (!ok)
 154                         return 0;
 155         }
 156
 157         if (has_addr) {
 158                 __u8 *ok;
 159                 __u64 addr = ctx[0];
 160
 161                 ok = bpf_map_lookup_elem(&addr_filter, &addr);
 162                 if (!ok)
 163                         return 0;
 164         }
 165
 166         return 1;
 167 }
 168
 169 static inline int update_task_data(struct task_struct *task)
 170 {
 171         struct contention_task_data *p;
 172         int pid, err;
 173
 174         err = bpf_core_read(&pid, sizeof(pid), &task->pid);
 175         if (err)
 176                 return -1;
 177
 178         p = bpf_map_lookup_elem(&task_data, &pid);
 179         if (p == NULL) {
 180                 struct contention_task_data data = {};
 181
 182                 BPF_CORE_READ_STR_INTO(&data.comm, task, comm);
 183                 bpf_map_update_elem(&task_data, &pid, &data, BPF_NOEXIST);
 184         }
 185
 186         return 0;
 187 }
 188
 189 #ifndef __has_builtin
 190 # define __has_builtin(x) 0
 191 #endif
 192
 193 static inline struct task_struct *get_lock_owner(__u64 lock, __u32 flags)
 194 {
 195         struct task_struct *task;
 196         __u64 owner = 0;
 197
 198         if (flags & LCB_F_MUTEX) {
 199                 struct mutex *mutex = (void *)lock;
 200                 owner = BPF_CORE_READ(mutex, owner.counter);
 201         } else if (flags == LCB_F_READ || flags == LCB_F_WRITE) {
 202         /*
 203          * Support for the BPF_TYPE_MATCHES argument to the
 204          * __builtin_preserve_type_info builtin was added at some point during
 205          * development of clang 15 and it's what is needed for
 206          * bpf_core_type_matches.
 207          */
 208 #if __has_builtin(__builtin_preserve_type_info) && __clang_major__ >= 15
 209                 if (bpf_core_type_matches(struct rw_semaphore___old)) {
 210                         struct rw_semaphore___old *rwsem = (void *)lock;
 211                         owner = (unsigned long)BPF_CORE_READ(rwsem, owner);
 212                 } else if (bpf_core_type_matches(struct rw_semaphore___new)) {
 213                         struct rw_semaphore___new *rwsem = (void *)lock;
 214                         owner = BPF_CORE_READ(rwsem, owner.counter);
 215                 }
 216 #else
 217                 /* assume new struct */
 218                 struct rw_semaphore *rwsem = (void *)lock;
 219                 owner = BPF_CORE_READ(rwsem, owner.counter);
 220 #endif
 221         }
 222
 223         if (!owner)
 224                 return NULL;
 225
 226         task = (void *)(owner & ~7UL);
 227         return task;
 228 }
 229
 230 static inline __u32 check_lock_type(__u64 lock, __u32 flags)
 231 {
 232         struct task_struct *curr;
 233         struct mm_struct___old *mm_old;
 234         struct mm_struct___new *mm_new;
 235
 236         switch (flags) {
 237         case LCB_F_READ:  /* rwsem */
 238         case LCB_F_WRITE:
 239                 curr = bpf_get_current_task_btf();
 240                 if (curr->mm == NULL)
 241                         break;
 242                 mm_new = (void *)curr->mm;
 243                 if (bpf_core_field_exists(mm_new->mmap_lock)) {
 244                         if (&mm_new->mmap_lock == (void *)lock)
 245                                 return LCD_F_MMAP_LOCK;
 246                         break;
 247                 }
 248                 mm_old = (void *)curr->mm;
 249                 if (bpf_core_field_exists(mm_old->mmap_sem)) {
 250                         if (&mm_old->mmap_sem == (void *)lock)
 251                                 return LCD_F_MMAP_LOCK;
 252                 }
 253                 break;
 254         case LCB_F_SPIN:  /* spinlock */
 255                 curr = bpf_get_current_task_btf();
 256                 if (&curr->sighand->siglock == (void *)lock)
 257                         return LCD_F_SIGHAND_LOCK;
 258                 break;
 259         default:
 260                 break;
 261         }
 262         return 0;
 263 }
 264
 265 SEC("tp_btf/contention_begin")
 266 int contention_begin(u64 *ctx)
 267 {
 268         __u32 pid;
 269         struct tstamp_data *pelem;
 270
 271         if (!enabled || !can_record(ctx))
 272                 return 0;
 273
 274         pid = bpf_get_current_pid_tgid();
 275         pelem = bpf_map_lookup_elem(&tstamp, &pid);
 276         if (pelem && pelem->lock)
 277                 return 0;
 278
 279         if (pelem == NULL) {
 280                 struct tstamp_data zero = {};
 281
 282                 bpf_map_update_elem(&tstamp, &pid, &zero, BPF_ANY);
 283                 pelem = bpf_map_lookup_elem(&tstamp, &pid);
 284                 if (pelem == NULL) {
 285                         __sync_fetch_and_add(&task_fail, 1);
 286                         return 0;
 287                 }
 288         }
 289
 290         pelem->timestamp = bpf_ktime_get_ns();
 291         pelem->lock = (__u64)ctx[0];
 292         pelem->flags = (__u32)ctx[1];
 293
 294         if (needs_callstack) {
 295                 pelem->stack_id = bpf_get_stackid(ctx, &stacks,
 296                                                   BPF_F_FAST_STACK_CMP | stack_skip);
 297                 if (pelem->stack_id < 0)
 298                         __sync_fetch_and_add(&stack_fail, 1);
 299         } else if (aggr_mode == LOCK_AGGR_TASK) {
 300                 struct task_struct *task;
 301
 302                 if (lock_owner) {
 303                         task = get_lock_owner(pelem->lock, pelem->flags);
 304
 305                         /* The flags is not used anymore.  Pass the owner pid. */
 306                         if (task)
 307                                 pelem->flags = BPF_CORE_READ(task, pid);
 308                         else
 309                                 pelem->flags = -1U;
 310
 311                 } else {
 312                         task = bpf_get_current_task_btf();
 313                 }
 314
 315                 if (task) {
 316                         if (update_task_data(task) < 0 && lock_owner)
 317                                 pelem->flags = -1U;
 318                 }
 319         }
 320
 321         return 0;
 322 }
 323
 324 SEC("tp_btf/contention_end")
 325 int contention_end(u64 *ctx)
 326 {
 327         __u32 pid;
 328         struct tstamp_data *pelem;
 329         struct contention_key key = {};
 330         struct contention_data *data;
 331         __u64 duration;
 332
 333         if (!enabled)
 334                 return 0;
 335
 336         pid = bpf_get_current_pid_tgid();
 337         pelem = bpf_map_lookup_elem(&tstamp, &pid);
 338         if (!pelem || pelem->lock != ctx[0])
 339                 return 0;
 340
 341         duration = bpf_ktime_get_ns() - pelem->timestamp;
 342         if ((__s64)duration < 0) {
 343                 bpf_map_delete_elem(&tstamp, &pid);
 344                 __sync_fetch_and_add(&time_fail, 1);
 345                 return 0;
 346         }
 347
 348         switch (aggr_mode) {
 349         case LOCK_AGGR_CALLER:
 350                 key.stack_id = pelem->stack_id;
 351                 break;
 352         case LOCK_AGGR_TASK:
 353                 if (lock_owner)
 354                         key.pid = pelem->flags;
 355                 else
 356                         key.pid = pid;
 357                 if (needs_callstack)
 358                         key.stack_id = pelem->stack_id;
 359                 break;
 360         case LOCK_AGGR_ADDR:
 361                 key.lock_addr = pelem->lock;
 362                 if (needs_callstack)
 363                         key.stack_id = pelem->stack_id;
 364                 break;
 365         default:
 366                 /* should not happen */
 367                 return 0;
 368         }
 369
 370         data = bpf_map_lookup_elem(&lock_stat, &key);
 371         if (!data) {
 372                 struct contention_data first = {
 373                         .total_time = duration,
 374                         .max_time = duration,
 375                         .min_time = duration,
 376                         .count = 1,
 377                         .flags = pelem->flags,
 378                 };
 379
 380                 if (aggr_mode == LOCK_AGGR_ADDR)
 381                         first.flags |= check_lock_type(pelem->lock, pelem->flags);
 382
 383                 bpf_map_update_elem(&lock_stat, &key, &first, BPF_NOEXIST);
 384                 bpf_map_delete_elem(&tstamp, &pid);
 385                 return 0;
 386         }
 387
 388         __sync_fetch_and_add(&data->total_time, duration);
 389         __sync_fetch_and_add(&data->count, 1);
 390
 391         /* FIXME: need atomic operations */
 392         if (data->max_time < duration)
 393                 data->max_time = duration;
 394         if (data->min_time > duration)
 395                 data->min_time = duration;
 396
 397         bpf_map_delete_elem(&tstamp, &pid);
 398         return 0;
 399 }
 400
 401 extern struct rq runqueues __ksym;
 402
 403 SEC("raw_tp/bpf_test_finish")
 404 int BPF_PROG(collect_lock_syms)
 405 {
 406         __u64 lock_addr;
 407         __u32 lock_flag;
 408
 409         for (int i = 0; i < MAX_CPUS; i++) {
 410                 struct rq *rq = bpf_per_cpu_ptr(&runqueues, i);
 411
 412                 if (rq == NULL)
 413                         break;
 414
 415                 lock_addr = (__u64)&rq->__lock;
 416                 lock_flag = LOCK_CLASS_RQLOCK;
 417                 bpf_map_update_elem(&lock_syms, &lock_addr, &lock_flag, BPF_ANY);
 418         }
 419         return 0;
 420 }
 421
 422 char LICENSE[] SEC("license") = "Dual BSD/GPL";