1 #ifndef __BPF_HELPERS_H
2 #define __BPF_HELPERS_H
4 /* helper macro to place programs, maps, license in
5 * different sections in elf_bpf file. Section names
6 * are interpreted by elf_bpf loader
8 #define SEC(NAME) __attribute__((section(NAME), used))
10 /* helper functions called from eBPF programs written in C */
11 static void *(*bpf_map_lookup_elem)(void *map, void *key) =
12 (void *) BPF_FUNC_map_lookup_elem;
13 static int (*bpf_map_update_elem)(void *map, void *key, void *value,
14 unsigned long long flags) =
15 (void *) BPF_FUNC_map_update_elem;
16 static int (*bpf_map_delete_elem)(void *map, void *key) =
17 (void *) BPF_FUNC_map_delete_elem;
18 static int (*bpf_probe_read)(void *dst, int size, void *unsafe_ptr) =
19 (void *) BPF_FUNC_probe_read;
20 static unsigned long long (*bpf_ktime_get_ns)(void) =
21 (void *) BPF_FUNC_ktime_get_ns;
22 static int (*bpf_trace_printk)(const char *fmt, int fmt_size, ...) =
23 (void *) BPF_FUNC_trace_printk;
24 static void (*bpf_tail_call)(void *ctx, void *map, int index) =
25 (void *) BPF_FUNC_tail_call;
26 static unsigned long long (*bpf_get_smp_processor_id)(void) =
27 (void *) BPF_FUNC_get_smp_processor_id;
28 static unsigned long long (*bpf_get_current_pid_tgid)(void) =
29 (void *) BPF_FUNC_get_current_pid_tgid;
30 static unsigned long long (*bpf_get_current_uid_gid)(void) =
31 (void *) BPF_FUNC_get_current_uid_gid;
32 static int (*bpf_get_current_comm)(void *buf, int buf_size) =
33 (void *) BPF_FUNC_get_current_comm;
34 static unsigned long long (*bpf_perf_event_read)(void *map,
35 unsigned long long flags) =
36 (void *) BPF_FUNC_perf_event_read;
37 static int (*bpf_clone_redirect)(void *ctx, int ifindex, int flags) =
38 (void *) BPF_FUNC_clone_redirect;
39 static int (*bpf_redirect)(int ifindex, int flags) =
40 (void *) BPF_FUNC_redirect;
41 static int (*bpf_redirect_map)(void *map, int key, int flags) =
42 (void *) BPF_FUNC_redirect_map;
43 static int (*bpf_perf_event_output)(void *ctx, void *map,
44 unsigned long long flags, void *data,
46 (void *) BPF_FUNC_perf_event_output;
47 static int (*bpf_get_stackid)(void *ctx, void *map, int flags) =
48 (void *) BPF_FUNC_get_stackid;
49 static int (*bpf_probe_write_user)(void *dst, void *src, int size) =
50 (void *) BPF_FUNC_probe_write_user;
51 static int (*bpf_current_task_under_cgroup)(void *map, int index) =
52 (void *) BPF_FUNC_current_task_under_cgroup;
53 static int (*bpf_skb_get_tunnel_key)(void *ctx, void *key, int size, int flags) =
54 (void *) BPF_FUNC_skb_get_tunnel_key;
55 static int (*bpf_skb_set_tunnel_key)(void *ctx, void *key, int size, int flags) =
56 (void *) BPF_FUNC_skb_set_tunnel_key;
57 static int (*bpf_skb_get_tunnel_opt)(void *ctx, void *md, int size) =
58 (void *) BPF_FUNC_skb_get_tunnel_opt;
59 static int (*bpf_skb_set_tunnel_opt)(void *ctx, void *md, int size) =
60 (void *) BPF_FUNC_skb_set_tunnel_opt;
61 static unsigned long long (*bpf_get_prandom_u32)(void) =
62 (void *) BPF_FUNC_get_prandom_u32;
63 static int (*bpf_xdp_adjust_head)(void *ctx, int offset) =
64 (void *) BPF_FUNC_xdp_adjust_head;
65 static int (*bpf_xdp_adjust_meta)(void *ctx, int offset) =
66 (void *) BPF_FUNC_xdp_adjust_meta;
67 static int (*bpf_setsockopt)(void *ctx, int level, int optname, void *optval,
69 (void *) BPF_FUNC_setsockopt;
70 static int (*bpf_sk_redirect_map)(void *map, int key, int flags) =
71 (void *) BPF_FUNC_sk_redirect_map;
72 static int (*bpf_sock_map_update)(void *map, void *key, void *value,
73 unsigned long long flags) =
74 (void *) BPF_FUNC_sock_map_update;
75 static int (*bpf_perf_event_read_value)(void *map, unsigned long long flags,
76 void *buf, unsigned int buf_size) =
77 (void *) BPF_FUNC_perf_event_read_value;
80 /* llvm builtin functions that eBPF C program may use to
81 * emit BPF_LD_ABS and BPF_LD_IND instructions
84 unsigned long long load_byte(void *skb,
85 unsigned long long off) asm("llvm.bpf.load.byte");
86 unsigned long long load_half(void *skb,
87 unsigned long long off) asm("llvm.bpf.load.half");
88 unsigned long long load_word(void *skb,
89 unsigned long long off) asm("llvm.bpf.load.word");
91 /* a helper structure used by eBPF C program
92 * to describe map attributes to elf_bpf loader
96 unsigned int key_size;
97 unsigned int value_size;
98 unsigned int max_entries;
99 unsigned int map_flags;
100 unsigned int inner_map_idx;
101 unsigned int numa_node;
104 static int (*bpf_skb_load_bytes)(void *ctx, int off, void *to, int len) =
105 (void *) BPF_FUNC_skb_load_bytes;
106 static int (*bpf_skb_store_bytes)(void *ctx, int off, void *from, int len, int flags) =
107 (void *) BPF_FUNC_skb_store_bytes;
108 static int (*bpf_l3_csum_replace)(void *ctx, int off, int from, int to, int flags) =
109 (void *) BPF_FUNC_l3_csum_replace;
110 static int (*bpf_l4_csum_replace)(void *ctx, int off, int from, int to, int flags) =
111 (void *) BPF_FUNC_l4_csum_replace;
112 static int (*bpf_skb_under_cgroup)(void *ctx, void *map, int index) =
113 (void *) BPF_FUNC_skb_under_cgroup;
114 static int (*bpf_skb_change_head)(void *, int len, int flags) =
115 (void *) BPF_FUNC_skb_change_head;
117 /* Scan the ARCH passed in from ARCH env variable (see Makefile) */
118 #if defined(__TARGET_ARCH_x86)
119 #define bpf_target_x86
120 #define bpf_target_defined
121 #elif defined(__TARGET_ARCH_s930x)
122 #define bpf_target_s930x
123 #define bpf_target_defined
124 #elif defined(__TARGET_ARCH_arm64)
125 #define bpf_target_arm64
126 #define bpf_target_defined
127 #elif defined(__TARGET_ARCH_mips)
128 #define bpf_target_mips
129 #define bpf_target_defined
130 #elif defined(__TARGET_ARCH_powerpc)
131 #define bpf_target_powerpc
132 #define bpf_target_defined
133 #elif defined(__TARGET_ARCH_sparc)
134 #define bpf_target_sparc
135 #define bpf_target_defined
137 #undef bpf_target_defined
140 /* Fall back to what the compiler says */
141 #ifndef bpf_target_defined
142 #if defined(__x86_64__)
143 #define bpf_target_x86
144 #elif defined(__s390x__)
145 #define bpf_target_s930x
146 #elif defined(__aarch64__)
147 #define bpf_target_arm64
148 #elif defined(__mips__)
149 #define bpf_target_mips
150 #elif defined(__powerpc__)
151 #define bpf_target_powerpc
152 #elif defined(__sparc__)
153 #define bpf_target_sparc
157 #if defined(bpf_target_x86)
159 #define PT_REGS_PARM1(x) ((x)->di)
160 #define PT_REGS_PARM2(x) ((x)->si)
161 #define PT_REGS_PARM3(x) ((x)->dx)
162 #define PT_REGS_PARM4(x) ((x)->cx)
163 #define PT_REGS_PARM5(x) ((x)->r8)
164 #define PT_REGS_RET(x) ((x)->sp)
165 #define PT_REGS_FP(x) ((x)->bp)
166 #define PT_REGS_RC(x) ((x)->ax)
167 #define PT_REGS_SP(x) ((x)->sp)
168 #define PT_REGS_IP(x) ((x)->ip)
170 #elif defined(bpf_target_s390x)
172 #define PT_REGS_PARM1(x) ((x)->gprs[2])
173 #define PT_REGS_PARM2(x) ((x)->gprs[3])
174 #define PT_REGS_PARM3(x) ((x)->gprs[4])
175 #define PT_REGS_PARM4(x) ((x)->gprs[5])
176 #define PT_REGS_PARM5(x) ((x)->gprs[6])
177 #define PT_REGS_RET(x) ((x)->gprs[14])
178 #define PT_REGS_FP(x) ((x)->gprs[11]) /* Works only with CONFIG_FRAME_POINTER */
179 #define PT_REGS_RC(x) ((x)->gprs[2])
180 #define PT_REGS_SP(x) ((x)->gprs[15])
181 #define PT_REGS_IP(x) ((x)->psw.addr)
183 #elif defined(bpf_target_arm64)
185 #define PT_REGS_PARM1(x) ((x)->regs[0])
186 #define PT_REGS_PARM2(x) ((x)->regs[1])
187 #define PT_REGS_PARM3(x) ((x)->regs[2])
188 #define PT_REGS_PARM4(x) ((x)->regs[3])
189 #define PT_REGS_PARM5(x) ((x)->regs[4])
190 #define PT_REGS_RET(x) ((x)->regs[30])
191 #define PT_REGS_FP(x) ((x)->regs[29]) /* Works only with CONFIG_FRAME_POINTER */
192 #define PT_REGS_RC(x) ((x)->regs[0])
193 #define PT_REGS_SP(x) ((x)->sp)
194 #define PT_REGS_IP(x) ((x)->pc)
196 #elif defined(bpf_target_mips)
198 #define PT_REGS_PARM1(x) ((x)->regs[4])
199 #define PT_REGS_PARM2(x) ((x)->regs[5])
200 #define PT_REGS_PARM3(x) ((x)->regs[6])
201 #define PT_REGS_PARM4(x) ((x)->regs[7])
202 #define PT_REGS_PARM5(x) ((x)->regs[8])
203 #define PT_REGS_RET(x) ((x)->regs[31])
204 #define PT_REGS_FP(x) ((x)->regs[30]) /* Works only with CONFIG_FRAME_POINTER */
205 #define PT_REGS_RC(x) ((x)->regs[1])
206 #define PT_REGS_SP(x) ((x)->regs[29])
207 #define PT_REGS_IP(x) ((x)->cp0_epc)
209 #elif defined(bpf_target_powerpc)
211 #define PT_REGS_PARM1(x) ((x)->gpr[3])
212 #define PT_REGS_PARM2(x) ((x)->gpr[4])
213 #define PT_REGS_PARM3(x) ((x)->gpr[5])
214 #define PT_REGS_PARM4(x) ((x)->gpr[6])
215 #define PT_REGS_PARM5(x) ((x)->gpr[7])
216 #define PT_REGS_RC(x) ((x)->gpr[3])
217 #define PT_REGS_SP(x) ((x)->sp)
218 #define PT_REGS_IP(x) ((x)->nip)
220 #elif defined(bpf_target_sparc)
222 #define PT_REGS_PARM1(x) ((x)->u_regs[UREG_I0])
223 #define PT_REGS_PARM2(x) ((x)->u_regs[UREG_I1])
224 #define PT_REGS_PARM3(x) ((x)->u_regs[UREG_I2])
225 #define PT_REGS_PARM4(x) ((x)->u_regs[UREG_I3])
226 #define PT_REGS_PARM5(x) ((x)->u_regs[UREG_I4])
227 #define PT_REGS_RET(x) ((x)->u_regs[UREG_I7])
228 #define PT_REGS_RC(x) ((x)->u_regs[UREG_I0])
229 #define PT_REGS_SP(x) ((x)->u_regs[UREG_FP])
231 /* Should this also be a bpf_target check for the sparc case? */
232 #if defined(__arch64__)
233 #define PT_REGS_IP(x) ((x)->tpc)
235 #define PT_REGS_IP(x) ((x)->pc)
240 #ifdef bpf_target_powerpc
241 #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; })
242 #define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP
243 #elif bpf_target_sparc
244 #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); })
245 #define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP
247 #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ \
248 bpf_probe_read(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); })
249 #define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ \
250 bpf_probe_read(&(ip), sizeof(ip), \
251 (void *)(PT_REGS_FP(ctx) + sizeof(ip))); })