tools/perf/util/bpf_off_cpu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include "util/bpf_counter.h"
   3 #include "util/debug.h"
   4 #include "util/evsel.h"
   5 #include "util/evlist.h"
   6 #include "util/off_cpu.h"
   7 #include "util/perf-hooks.h"
   8 #include "util/record.h"
   9 #include "util/session.h"
  10 #include "util/target.h"
  11 #include "util/cpumap.h"
  12 #include "util/thread_map.h"
  13 #include "util/cgroup.h"
  14 #include "util/strlist.h"
  15 #include <bpf/bpf.h>
  16
  17 #include "bpf_skel/off_cpu.skel.h"
  18
  19 #define MAX_STACKS  32
  20 #define MAX_PROC  4096
  21 /* we don't need actual timestamp, just want to put the samples at last */
  22 #define OFF_CPU_TIMESTAMP  (~0ull << 32)
  23
  24 static struct off_cpu_bpf *skel;
  25
  26 struct off_cpu_key {
  27         u32 pid;
  28         u32 tgid;
  29         u32 stack_id;
  30         u32 state;
  31         u64 cgroup_id;
  32 };
  33
  34 union off_cpu_data {
  35         struct perf_event_header hdr;
  36         u64 array[1024 / sizeof(u64)];
  37 };
  38
  39 static int off_cpu_config(struct evlist *evlist)
  40 {
  41         struct evsel *evsel;
  42         struct perf_event_attr attr = {
  43                 .type   = PERF_TYPE_SOFTWARE,
  44                 .config = PERF_COUNT_SW_BPF_OUTPUT,
  45                 .size   = sizeof(attr), /* to capture ABI version */
  46         };
  47         char *evname = strdup(OFFCPU_EVENT);
  48
  49         if (evname == NULL)
  50                 return -ENOMEM;
  51
  52         evsel = evsel__new(&attr);
  53         if (!evsel) {
  54                 free(evname);
  55                 return -ENOMEM;
  56         }
  57
  58         evsel->core.attr.freq = 1;
  59         evsel->core.attr.sample_period = 1;
  60         /* off-cpu analysis depends on stack trace */
  61         evsel->core.attr.sample_type = PERF_SAMPLE_CALLCHAIN;
  62
  63         evlist__add(evlist, evsel);
  64
  65         free(evsel->name);
  66         evsel->name = evname;
  67
  68         return 0;
  69 }
  70
  71 static void off_cpu_start(void *arg)
  72 {
  73         struct evlist *evlist = arg;
  74
  75         /* update task filter for the given workload */
  76         if (!skel->bss->has_cpu && !skel->bss->has_task &&
  77             perf_thread_map__pid(evlist->core.threads, 0) != -1) {
  78                 int fd;
  79                 u32 pid;
  80                 u8 val = 1;
  81
  82                 skel->bss->has_task = 1;
  83                 skel->bss->uses_tgid = 1;
  84                 fd = bpf_map__fd(skel->maps.task_filter);
  85                 pid = perf_thread_map__pid(evlist->core.threads, 0);
  86                 bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
  87         }
  88
  89         skel->bss->enabled = 1;
  90 }
  91
  92 static void off_cpu_finish(void *arg __maybe_unused)
  93 {
  94         skel->bss->enabled = 0;
  95         off_cpu_bpf__destroy(skel);
  96 }
  97
  98 /* v5.18 kernel added prev_state arg, so it needs to check the signature */
  99 static void check_sched_switch_args(void)
 100 {
 101         const struct btf *btf = bpf_object__btf(skel->obj);
 102         const struct btf_type *t1, *t2, *t3;
 103         u32 type_id;
 104
 105         type_id = btf__find_by_name_kind(btf, "btf_trace_sched_switch",
 106                                          BTF_KIND_TYPEDEF);
 107         if ((s32)type_id < 0)
 108                 return;
 109
 110         t1 = btf__type_by_id(btf, type_id);
 111         if (t1 == NULL)
 112                 return;
 113
 114         t2 = btf__type_by_id(btf, t1->type);
 115         if (t2 == NULL || !btf_is_ptr(t2))
 116                 return;
 117
 118         t3 = btf__type_by_id(btf, t2->type);
 119         if (t3 && btf_is_func_proto(t3) && btf_vlen(t3) == 4) {
 120                 /* new format: pass prev_state as 4th arg */
 121                 skel->rodata->has_prev_state = true;
 122         }
 123 }
 124
 125 int off_cpu_prepare(struct evlist *evlist, struct target *target,
 126                     struct record_opts *opts)
 127 {
 128         int err, fd, i;
 129         int ncpus = 1, ntasks = 1, ncgrps = 1;
 130         struct strlist *pid_slist = NULL;
 131         struct str_node *pos;
 132
 133         if (off_cpu_config(evlist) < 0) {
 134                 pr_err("Failed to config off-cpu BPF event\n");
 135                 return -1;
 136         }
 137
 138         skel = off_cpu_bpf__open();
 139         if (!skel) {
 140                 pr_err("Failed to open off-cpu BPF skeleton\n");
 141                 return -1;
 142         }
 143
 144         /* don't need to set cpu filter for system-wide mode */
 145         if (target->cpu_list) {
 146                 ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus);
 147                 bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus);
 148         }
 149
 150         if (target->pid) {
 151                 pid_slist = strlist__new(target->pid, NULL);
 152                 if (!pid_slist) {
 153                         pr_err("Failed to create a strlist for pid\n");
 154                         return -1;
 155                 }
 156
 157                 ntasks = 0;
 158                 strlist__for_each_entry(pos, pid_slist) {
 159                         char *end_ptr;
 160                         int pid = strtol(pos->s, &end_ptr, 10);
 161
 162                         if (pid == INT_MIN || pid == INT_MAX ||
 163                             (*end_ptr != '\0' && *end_ptr != ','))
 164                                 continue;
 165
 166                         ntasks++;
 167                 }
 168
 169                 if (ntasks < MAX_PROC)
 170                         ntasks = MAX_PROC;
 171
 172                 bpf_map__set_max_entries(skel->maps.task_filter, ntasks);
 173         } else if (target__has_task(target)) {
 174                 ntasks = perf_thread_map__nr(evlist->core.threads);
 175                 bpf_map__set_max_entries(skel->maps.task_filter, ntasks);
 176         } else if (target__none(target)) {
 177                 bpf_map__set_max_entries(skel->maps.task_filter, MAX_PROC);
 178         }
 179
 180         if (evlist__first(evlist)->cgrp) {
 181                 ncgrps = evlist->core.nr_entries - 1; /* excluding a dummy */
 182                 bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps);
 183
 184                 if (!cgroup_is_v2("perf_event"))
 185                         skel->rodata->uses_cgroup_v1 = true;
 186         }
 187
 188         if (opts->record_cgroup) {
 189                 skel->rodata->needs_cgroup = true;
 190
 191                 if (!cgroup_is_v2("perf_event"))
 192                         skel->rodata->uses_cgroup_v1 = true;
 193         }
 194
 195         set_max_rlimit();
 196         check_sched_switch_args();
 197
 198         err = off_cpu_bpf__load(skel);
 199         if (err) {
 200                 pr_err("Failed to load off-cpu skeleton\n");
 201                 goto out;
 202         }
 203
 204         if (target->cpu_list) {
 205                 u32 cpu;
 206                 u8 val = 1;
 207
 208                 skel->bss->has_cpu = 1;
 209                 fd = bpf_map__fd(skel->maps.cpu_filter);
 210
 211                 for (i = 0; i < ncpus; i++) {
 212                         cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu;
 213                         bpf_map_update_elem(fd, &cpu, &val, BPF_ANY);
 214                 }
 215         }
 216
 217         if (target->pid) {
 218                 u8 val = 1;
 219
 220                 skel->bss->has_task = 1;
 221                 skel->bss->uses_tgid = 1;
 222                 fd = bpf_map__fd(skel->maps.task_filter);
 223
 224                 strlist__for_each_entry(pos, pid_slist) {
 225                         char *end_ptr;
 226                         u32 tgid;
 227                         int pid = strtol(pos->s, &end_ptr, 10);
 228
 229                         if (pid == INT_MIN || pid == INT_MAX ||
 230                             (*end_ptr != '\0' && *end_ptr != ','))
 231                                 continue;
 232
 233                         tgid = pid;
 234                         bpf_map_update_elem(fd, &tgid, &val, BPF_ANY);
 235                 }
 236         } else if (target__has_task(target)) {
 237                 u32 pid;
 238                 u8 val = 1;
 239
 240                 skel->bss->has_task = 1;
 241                 fd = bpf_map__fd(skel->maps.task_filter);
 242
 243                 for (i = 0; i < ntasks; i++) {
 244                         pid = perf_thread_map__pid(evlist->core.threads, i);
 245                         bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
 246                 }
 247         }
 248
 249         if (evlist__first(evlist)->cgrp) {
 250                 struct evsel *evsel;
 251                 u8 val = 1;
 252
 253                 skel->bss->has_cgroup = 1;
 254                 fd = bpf_map__fd(skel->maps.cgroup_filter);
 255
 256                 evlist__for_each_entry(evlist, evsel) {
 257                         struct cgroup *cgrp = evsel->cgrp;
 258
 259                         if (cgrp == NULL)
 260                                 continue;
 261
 262                         if (!cgrp->id && read_cgroup_id(cgrp) < 0) {
 263                                 pr_err("Failed to read cgroup id of %s\n",
 264                                        cgrp->name);
 265                                 goto out;
 266                         }
 267
 268                         bpf_map_update_elem(fd, &cgrp->id, &val, BPF_ANY);
 269                 }
 270         }
 271
 272         err = off_cpu_bpf__attach(skel);
 273         if (err) {
 274                 pr_err("Failed to attach off-cpu BPF skeleton\n");
 275                 goto out;
 276         }
 277
 278         if (perf_hooks__set_hook("record_start", off_cpu_start, evlist) ||
 279             perf_hooks__set_hook("record_end", off_cpu_finish, evlist)) {
 280                 pr_err("Failed to attach off-cpu skeleton\n");
 281                 goto out;
 282         }
 283
 284         return 0;
 285
 286 out:
 287         off_cpu_bpf__destroy(skel);
 288         return -1;
 289 }
 290
 291 int off_cpu_write(struct perf_session *session)
 292 {
 293         int bytes = 0, size;
 294         int fd, stack;
 295         u64 sample_type, val, sid = 0;
 296         struct evsel *evsel;
 297         struct perf_data_file *file = &session->data->file;
 298         struct off_cpu_key prev, key;
 299         union off_cpu_data data = {
 300                 .hdr = {
 301                         .type = PERF_RECORD_SAMPLE,
 302                         .misc = PERF_RECORD_MISC_USER,
 303                 },
 304         };
 305         u64 tstamp = OFF_CPU_TIMESTAMP;
 306
 307         skel->bss->enabled = 0;
 308
 309         evsel = evlist__find_evsel_by_str(session->evlist, OFFCPU_EVENT);
 310         if (evsel == NULL) {
 311                 pr_err("%s evsel not found\n", OFFCPU_EVENT);
 312                 return 0;
 313         }
 314
 315         sample_type = evsel->core.attr.sample_type;
 316
 317         if (sample_type & ~OFFCPU_SAMPLE_TYPES) {
 318                 pr_err("not supported sample type: %llx\n",
 319                        (unsigned long long)sample_type);
 320                 return -1;
 321         }
 322
 323         if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) {
 324                 if (evsel->core.id)
 325                         sid = evsel->core.id[0];
 326         }
 327
 328         fd = bpf_map__fd(skel->maps.off_cpu);
 329         stack = bpf_map__fd(skel->maps.stacks);
 330         memset(&prev, 0, sizeof(prev));
 331
 332         while (!bpf_map_get_next_key(fd, &prev, &key)) {
 333                 int n = 1;  /* start from perf_event_header */
 334                 int ip_pos = -1;
 335
 336                 bpf_map_lookup_elem(fd, &key, &val);
 337
 338                 if (sample_type & PERF_SAMPLE_IDENTIFIER)
 339                         data.array[n++] = sid;
 340                 if (sample_type & PERF_SAMPLE_IP) {
 341                         ip_pos = n;
 342                         data.array[n++] = 0;  /* will be updated */
 343                 }
 344                 if (sample_type & PERF_SAMPLE_TID)
 345                         data.array[n++] = (u64)key.pid << 32 | key.tgid;
 346                 if (sample_type & PERF_SAMPLE_TIME)
 347                         data.array[n++] = tstamp;
 348                 if (sample_type & PERF_SAMPLE_ID)
 349                         data.array[n++] = sid;
 350                 if (sample_type & PERF_SAMPLE_CPU)
 351                         data.array[n++] = 0;
 352                 if (sample_type & PERF_SAMPLE_PERIOD)
 353                         data.array[n++] = val;
 354                 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
 355                         int len = 0;
 356
 357                         /* data.array[n] is callchain->nr (updated later) */
 358                         data.array[n + 1] = PERF_CONTEXT_USER;
 359                         data.array[n + 2] = 0;
 360
 361                         bpf_map_lookup_elem(stack, &key.stack_id, &data.array[n + 2]);
 362                         while (data.array[n + 2 + len])
 363                                 len++;
 364
 365                         /* update length of callchain */
 366                         data.array[n] = len + 1;
 367
 368                         /* update sample ip with the first callchain entry */
 369                         if (ip_pos >= 0)
 370                                 data.array[ip_pos] = data.array[n + 2];
 371
 372                         /* calculate sample callchain data array length */
 373                         n += len + 2;
 374                 }
 375                 if (sample_type & PERF_SAMPLE_CGROUP)
 376                         data.array[n++] = key.cgroup_id;
 377
 378                 size = n * sizeof(u64);
 379                 data.hdr.size = size;
 380                 bytes += size;
 381
 382                 if (perf_data_file__write(file, &data, size) < 0) {
 383                         pr_err("failed to write perf data, error: %m\n");
 384                         return bytes;
 385                 }
 386
 387                 prev = key;
 388                 /* increase dummy timestamp to sort later samples */
 389                 tstamp++;
 390         }
 391         return bytes;
 392 }