1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2019 Facebook
3 #include <linux/sched.h>
4 #include <linux/ptrace.h>
9 #include <bpf/bpf_helpers.h>
12 #define FUNCTION_NAME_LEN 64
13 #define FILE_NAME_LEN 128
14 #define TASK_COMM_LEN 16
17 int PyThreadState_frame;
18 int PyThreadState_thread;
19 int PyFrameObject_back;
20 int PyFrameObject_code;
21 int PyFrameObject_lineno;
22 int PyCodeObject_filename;
23 int PyCodeObject_name;
29 uintptr_t current_state_addr;
30 uintptr_t tls_key_addr;
40 char name[FUNCTION_NAME_LEN];
41 char file[FILE_NAME_LEN];
47 char comm[TASK_COMM_LEN];
48 int32_t kernel_stack_id;
49 int32_t user_stack_id;
54 int32_t stack[STACK_MAX_LEN];
65 void* f_back; // PyFrameObject.f_back, previous frame
66 void* f_code; // PyFrameObject.f_code, pointer to PyCodeObject
67 void* co_filename; // PyCodeObject.co_filename
68 void* co_name; // PyCodeObject.co_name
76 static void *get_thread_state(void *tls_base, PidData *pidData)
81 bpf_probe_read_user(&key, sizeof(key), (void*)(long)pidData->tls_key_addr);
82 bpf_probe_read_user(&thread_state, sizeof(thread_state),
83 tls_base + 0x310 + key * 0x10 + 0x08);
87 static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData,
88 FrameData *frame, Symbol *symbol)
90 // read data from PyFrameObject
91 bpf_probe_read_user(&frame->f_back,
92 sizeof(frame->f_back),
93 frame_ptr + pidData->offsets.PyFrameObject_back);
94 bpf_probe_read_user(&frame->f_code,
95 sizeof(frame->f_code),
96 frame_ptr + pidData->offsets.PyFrameObject_code);
98 // read data from PyCodeObject
101 bpf_probe_read_user(&frame->co_filename,
102 sizeof(frame->co_filename),
103 frame->f_code + pidData->offsets.PyCodeObject_filename);
104 bpf_probe_read_user(&frame->co_name,
105 sizeof(frame->co_name),
106 frame->f_code + pidData->offsets.PyCodeObject_name);
107 // read actual names into symbol
108 if (frame->co_filename)
109 bpf_probe_read_user_str(&symbol->file,
110 sizeof(symbol->file),
112 pidData->offsets.String_data);
114 bpf_probe_read_user_str(&symbol->name,
115 sizeof(symbol->name),
117 pidData->offsets.String_data);
122 __uint(type, BPF_MAP_TYPE_HASH);
123 __uint(max_entries, 1);
125 __type(value, PidData);
126 } pidmap SEC(".maps");
129 __uint(type, BPF_MAP_TYPE_HASH);
130 __uint(max_entries, 1);
132 __type(value, Event);
133 } eventmap SEC(".maps");
136 __uint(type, BPF_MAP_TYPE_HASH);
137 __uint(max_entries, 1);
140 } symbolmap SEC(".maps");
143 __uint(type, BPF_MAP_TYPE_ARRAY);
144 __uint(max_entries, 1);
146 __type(value, Stats);
147 } statsmap SEC(".maps");
150 __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
151 __uint(max_entries, 32);
152 __uint(key_size, sizeof(int));
153 __uint(value_size, sizeof(int));
154 } perfmap SEC(".maps");
157 __uint(type, BPF_MAP_TYPE_STACK_TRACE);
158 __uint(max_entries, 1000);
159 __uint(key_size, sizeof(int));
160 __uint(value_size, sizeof(long long) * 127);
161 } stackmap SEC(".maps");
164 struct process_frame_ctx {
166 int32_t *symbol_counter;
175 static int process_frame_callback(__u32 i, struct process_frame_ctx *ctx)
178 void *frame_ptr = ctx->frame_ptr;
179 PidData *pidData = ctx->pidData;
180 FrameData *frame = ctx->frame;
181 int32_t *symbol_counter = ctx->symbol_counter;
182 int cur_cpu = ctx->cur_cpu;
183 Event *event = ctx->event;
184 Symbol *sym = ctx->sym;
186 if (frame_ptr && get_frame_data(frame_ptr, pidData, frame, sym)) {
187 int32_t new_symbol_id = *symbol_counter * 64 + cur_cpu;
188 int32_t *symbol_id = bpf_map_lookup_elem(&symbolmap, sym);
191 bpf_map_update_elem(&symbolmap, sym, &zero, 0);
192 symbol_id = bpf_map_lookup_elem(&symbolmap, sym);
198 if (*symbol_id == new_symbol_id)
202 if (i >= STACK_MAX_LEN)
205 event->stack[i] = *symbol_id;
207 event->stack_len = i + 1;
208 frame_ptr = frame->f_back;
212 #endif /* USE_BPF_LOOP */
216 #elif defined(SUBPROGS)
219 static __always_inline
221 int __on_event(struct bpf_raw_tracepoint_args *ctx)
223 uint64_t pid_tgid = bpf_get_current_pid_tgid();
224 pid_t pid = (pid_t)(pid_tgid >> 32);
225 PidData* pidData = bpf_map_lookup_elem(&pidmap, &pid);
230 Event* event = bpf_map_lookup_elem(&eventmap, &zero);
236 event->tid = (pid_t)pid_tgid;
237 bpf_get_current_comm(&event->comm, sizeof(event->comm));
239 event->user_stack_id = bpf_get_stackid(ctx, &stackmap, BPF_F_USER_STACK);
240 event->kernel_stack_id = bpf_get_stackid(ctx, &stackmap, 0);
242 void* thread_state_current = (void*)0;
243 bpf_probe_read_user(&thread_state_current,
244 sizeof(thread_state_current),
245 (void*)(long)pidData->current_state_addr);
247 struct task_struct* task = (struct task_struct*)bpf_get_current_task();
248 void* tls_base = (void*)task;
250 void* thread_state = pidData->use_tls ? get_thread_state(tls_base, pidData)
251 : thread_state_current;
252 event->thread_current = thread_state == thread_state_current;
254 if (pidData->use_tls) {
255 uint64_t pthread_created;
256 uint64_t pthread_self;
257 bpf_probe_read_user(&pthread_self, sizeof(pthread_self),
260 bpf_probe_read_user(&pthread_created,
261 sizeof(pthread_created),
263 pidData->offsets.PyThreadState_thread);
264 event->pthread_match = pthread_created == pthread_self;
266 event->pthread_match = 1;
269 if (event->pthread_match || !pidData->use_tls) {
273 int cur_cpu = bpf_get_smp_processor_id();
275 bpf_probe_read_user(&frame_ptr,
278 pidData->offsets.PyThreadState_frame);
280 int32_t* symbol_counter = bpf_map_lookup_elem(&symbolmap, &sym);
281 if (symbol_counter == NULL)
284 struct process_frame_ctx ctx = {
286 .symbol_counter = symbol_counter,
287 .frame_ptr = frame_ptr,
294 bpf_loop(STACK_MAX_LEN, process_frame_callback, &ctx, 0);
298 #if defined(USE_ITER)
299 /* no for loop, no unrolling */
300 #elif defined(NO_UNROLL)
301 #pragma clang loop unroll(disable)
302 #elif defined(UNROLL_COUNT)
303 #pragma clang loop unroll_count(UNROLL_COUNT)
305 #pragma clang loop unroll(full)
306 #endif /* NO_UNROLL */
307 /* Unwind python stack */
310 bpf_for(i, 0, STACK_MAX_LEN) {
311 #else /* !USE_ITER */
312 for (int i = 0; i < STACK_MAX_LEN; ++i) {
314 if (frame_ptr && get_frame_data(frame_ptr, pidData, &frame, &sym)) {
315 int32_t new_symbol_id = *symbol_counter * 64 + cur_cpu;
316 int32_t *symbol_id = bpf_map_lookup_elem(&symbolmap, &sym);
318 bpf_map_update_elem(&symbolmap, &sym, &zero, 0);
319 symbol_id = bpf_map_lookup_elem(&symbolmap, &sym);
323 if (*symbol_id == new_symbol_id)
325 event->stack[i] = *symbol_id;
326 event->stack_len = i + 1;
327 frame_ptr = frame.f_back;
330 #endif /* USE_BPF_LOOP */
331 event->stack_complete = frame_ptr == NULL;
333 event->stack_complete = 1;
336 Stats* stats = bpf_map_lookup_elem(&statsmap, &zero);
341 bpf_perf_event_output(ctx, &perfmap, 0, event, offsetof(Event, metadata));
345 SEC("raw_tracepoint/kfree_skb")
346 int on_event(struct bpf_raw_tracepoint_args* ctx)
349 ret |= __on_event(ctx);
350 ret |= __on_event(ctx);
351 ret |= __on_event(ctx);
352 ret |= __on_event(ctx);
353 ret |= __on_event(ctx);
357 char _license[] SEC("license") = "GPL";