patch-5.15.79-rt54.patch
[platform/kernel/linux-rpi.git] / include / linux / skmsg.h
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
3
4 #ifndef _LINUX_SKMSG_H
5 #define _LINUX_SKMSG_H
6
7 #include <linux/bpf.h>
8 #include <linux/filter.h>
9 #include <linux/scatterlist.h>
10 #include <linux/skbuff.h>
11
12 #include <net/sock.h>
13 #include <net/tcp.h>
14 #include <net/strparser.h>
15
16 #define MAX_MSG_FRAGS                   MAX_SKB_FRAGS
17 #define NR_MSG_FRAG_IDS                 (MAX_MSG_FRAGS + 1)
18
19 enum __sk_action {
20         __SK_DROP = 0,
21         __SK_PASS,
22         __SK_REDIRECT,
23         __SK_NONE,
24 };
25
26 struct sk_msg_sg {
27         u32                             start;
28         u32                             curr;
29         u32                             end;
30         u32                             size;
31         u32                             copybreak;
32         unsigned long                   copy;
33         /* The extra two elements:
34          * 1) used for chaining the front and sections when the list becomes
35          *    partitioned (e.g. end < start). The crypto APIs require the
36          *    chaining;
37          * 2) to chain tailer SG entries after the message.
38          */
39         struct scatterlist              data[MAX_MSG_FRAGS + 2];
40 };
41 static_assert(BITS_PER_LONG >= NR_MSG_FRAG_IDS);
42
43 /* UAPI in filter.c depends on struct sk_msg_sg being first element. */
44 struct sk_msg {
45         struct sk_msg_sg                sg;
46         void                            *data;
47         void                            *data_end;
48         u32                             apply_bytes;
49         u32                             cork_bytes;
50         u32                             flags;
51         struct sk_buff                  *skb;
52         struct sock                     *sk_redir;
53         struct sock                     *sk;
54         struct list_head                list;
55 };
56
57 struct sk_psock_progs {
58         struct bpf_prog                 *msg_parser;
59         struct bpf_prog                 *stream_parser;
60         struct bpf_prog                 *stream_verdict;
61         struct bpf_prog                 *skb_verdict;
62 };
63
64 enum sk_psock_state_bits {
65         SK_PSOCK_TX_ENABLED,
66 };
67
68 struct sk_psock_link {
69         struct list_head                list;
70         struct bpf_map                  *map;
71         void                            *link_raw;
72 };
73
74 struct sk_psock_work_state {
75         struct sk_buff                  *skb;
76         u32                             len;
77         u32                             off;
78 };
79
80 struct sk_psock {
81         struct sock                     *sk;
82         struct sock                     *sk_redir;
83         u32                             apply_bytes;
84         u32                             cork_bytes;
85         u32                             eval;
86         struct sk_msg                   *cork;
87         struct sk_psock_progs           progs;
88 #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
89         struct strparser                strp;
90 #endif
91         struct sk_buff_head             ingress_skb;
92         struct list_head                ingress_msg;
93         spinlock_t                      ingress_lock;
94         unsigned long                   state;
95         struct list_head                link;
96         spinlock_t                      link_lock;
97         refcount_t                      refcnt;
98         void (*saved_unhash)(struct sock *sk);
99         void (*saved_destroy)(struct sock *sk);
100         void (*saved_close)(struct sock *sk, long timeout);
101         void (*saved_write_space)(struct sock *sk);
102         void (*saved_data_ready)(struct sock *sk);
103         int  (*psock_update_sk_prot)(struct sock *sk, struct sk_psock *psock,
104                                      bool restore);
105         struct proto                    *sk_proto;
106         struct mutex                    work_mutex;
107         struct sk_psock_work_state      work_state;
108         struct work_struct              work;
109         struct rcu_work                 rwork;
110 };
111
112 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
113                  int elem_first_coalesce);
114 int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src,
115                  u32 off, u32 len);
116 void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len);
117 int sk_msg_free(struct sock *sk, struct sk_msg *msg);
118 int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg);
119 void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes);
120 void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,
121                                   u32 bytes);
122
123 void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes);
124 void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes);
125
126 int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
127                               struct sk_msg *msg, u32 bytes);
128 int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from,
129                              struct sk_msg *msg, u32 bytes);
130 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
131                    int len, int flags);
132 bool sk_msg_is_readable(struct sock *sk);
133
134 static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)
135 {
136         WARN_ON(i == msg->sg.end && bytes);
137 }
138
139 static inline void sk_msg_apply_bytes(struct sk_psock *psock, u32 bytes)
140 {
141         if (psock->apply_bytes) {
142                 if (psock->apply_bytes < bytes)
143                         psock->apply_bytes = 0;
144                 else
145                         psock->apply_bytes -= bytes;
146         }
147 }
148
149 static inline u32 sk_msg_iter_dist(u32 start, u32 end)
150 {
151         return end >= start ? end - start : end + (NR_MSG_FRAG_IDS - start);
152 }
153
154 #define sk_msg_iter_var_prev(var)                       \
155         do {                                            \
156                 if (var == 0)                           \
157                         var = NR_MSG_FRAG_IDS - 1;      \
158                 else                                    \
159                         var--;                          \
160         } while (0)
161
162 #define sk_msg_iter_var_next(var)                       \
163         do {                                            \
164                 var++;                                  \
165                 if (var == NR_MSG_FRAG_IDS)             \
166                         var = 0;                        \
167         } while (0)
168
169 #define sk_msg_iter_prev(msg, which)                    \
170         sk_msg_iter_var_prev(msg->sg.which)
171
172 #define sk_msg_iter_next(msg, which)                    \
173         sk_msg_iter_var_next(msg->sg.which)
174
175 static inline void sk_msg_clear_meta(struct sk_msg *msg)
176 {
177         memset(&msg->sg, 0, offsetofend(struct sk_msg_sg, copy));
178 }
179
180 static inline void sk_msg_init(struct sk_msg *msg)
181 {
182         BUILD_BUG_ON(ARRAY_SIZE(msg->sg.data) - 1 != NR_MSG_FRAG_IDS);
183         memset(msg, 0, sizeof(*msg));
184         sg_init_marker(msg->sg.data, NR_MSG_FRAG_IDS);
185 }
186
187 static inline void sk_msg_xfer(struct sk_msg *dst, struct sk_msg *src,
188                                int which, u32 size)
189 {
190         dst->sg.data[which] = src->sg.data[which];
191         dst->sg.data[which].length  = size;
192         dst->sg.size               += size;
193         src->sg.size               -= size;
194         src->sg.data[which].length -= size;
195         src->sg.data[which].offset += size;
196 }
197
198 static inline void sk_msg_xfer_full(struct sk_msg *dst, struct sk_msg *src)
199 {
200         memcpy(dst, src, sizeof(*src));
201         sk_msg_init(src);
202 }
203
204 static inline bool sk_msg_full(const struct sk_msg *msg)
205 {
206         return sk_msg_iter_dist(msg->sg.start, msg->sg.end) == MAX_MSG_FRAGS;
207 }
208
209 static inline u32 sk_msg_elem_used(const struct sk_msg *msg)
210 {
211         return sk_msg_iter_dist(msg->sg.start, msg->sg.end);
212 }
213
214 static inline struct scatterlist *sk_msg_elem(struct sk_msg *msg, int which)
215 {
216         return &msg->sg.data[which];
217 }
218
219 static inline struct scatterlist sk_msg_elem_cpy(struct sk_msg *msg, int which)
220 {
221         return msg->sg.data[which];
222 }
223
224 static inline struct page *sk_msg_page(struct sk_msg *msg, int which)
225 {
226         return sg_page(sk_msg_elem(msg, which));
227 }
228
229 static inline bool sk_msg_to_ingress(const struct sk_msg *msg)
230 {
231         return msg->flags & BPF_F_INGRESS;
232 }
233
234 static inline void sk_msg_compute_data_pointers(struct sk_msg *msg)
235 {
236         struct scatterlist *sge = sk_msg_elem(msg, msg->sg.start);
237
238         if (test_bit(msg->sg.start, &msg->sg.copy)) {
239                 msg->data = NULL;
240                 msg->data_end = NULL;
241         } else {
242                 msg->data = sg_virt(sge);
243                 msg->data_end = msg->data + sge->length;
244         }
245 }
246
247 static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page,
248                                    u32 len, u32 offset)
249 {
250         struct scatterlist *sge;
251
252         get_page(page);
253         sge = sk_msg_elem(msg, msg->sg.end);
254         sg_set_page(sge, page, len, offset);
255         sg_unmark_end(sge);
256
257         __set_bit(msg->sg.end, &msg->sg.copy);
258         msg->sg.size += len;
259         sk_msg_iter_next(msg, end);
260 }
261
262 static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state)
263 {
264         do {
265                 if (copy_state)
266                         __set_bit(i, &msg->sg.copy);
267                 else
268                         __clear_bit(i, &msg->sg.copy);
269                 sk_msg_iter_var_next(i);
270                 if (i == msg->sg.end)
271                         break;
272         } while (1);
273 }
274
275 static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start)
276 {
277         sk_msg_sg_copy(msg, start, true);
278 }
279
280 static inline void sk_msg_sg_copy_clear(struct sk_msg *msg, u32 start)
281 {
282         sk_msg_sg_copy(msg, start, false);
283 }
284
285 static inline struct sk_psock *sk_psock(const struct sock *sk)
286 {
287         return __rcu_dereference_sk_user_data_with_flags(sk,
288                                                          SK_USER_DATA_PSOCK);
289 }
290
291 static inline void sk_psock_set_state(struct sk_psock *psock,
292                                       enum sk_psock_state_bits bit)
293 {
294         set_bit(bit, &psock->state);
295 }
296
297 static inline void sk_psock_clear_state(struct sk_psock *psock,
298                                         enum sk_psock_state_bits bit)
299 {
300         clear_bit(bit, &psock->state);
301 }
302
303 static inline bool sk_psock_test_state(const struct sk_psock *psock,
304                                        enum sk_psock_state_bits bit)
305 {
306         return test_bit(bit, &psock->state);
307 }
308
309 static inline void sock_drop(struct sock *sk, struct sk_buff *skb)
310 {
311         sk_drops_add(sk, skb);
312         kfree_skb(skb);
313 }
314
315 static inline void sk_psock_queue_msg(struct sk_psock *psock,
316                                       struct sk_msg *msg)
317 {
318         spin_lock_bh(&psock->ingress_lock);
319         if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
320                 list_add_tail(&msg->list, &psock->ingress_msg);
321         else {
322                 sk_msg_free(psock->sk, msg);
323                 kfree(msg);
324         }
325         spin_unlock_bh(&psock->ingress_lock);
326 }
327
328 static inline struct sk_msg *sk_psock_dequeue_msg(struct sk_psock *psock)
329 {
330         struct sk_msg *msg;
331
332         spin_lock_bh(&psock->ingress_lock);
333         msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);
334         if (msg)
335                 list_del(&msg->list);
336         spin_unlock_bh(&psock->ingress_lock);
337         return msg;
338 }
339
340 static inline struct sk_msg *sk_psock_peek_msg(struct sk_psock *psock)
341 {
342         struct sk_msg *msg;
343
344         spin_lock_bh(&psock->ingress_lock);
345         msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);
346         spin_unlock_bh(&psock->ingress_lock);
347         return msg;
348 }
349
350 static inline struct sk_msg *sk_psock_next_msg(struct sk_psock *psock,
351                                                struct sk_msg *msg)
352 {
353         struct sk_msg *ret;
354
355         spin_lock_bh(&psock->ingress_lock);
356         if (list_is_last(&msg->list, &psock->ingress_msg))
357                 ret = NULL;
358         else
359                 ret = list_next_entry(msg, list);
360         spin_unlock_bh(&psock->ingress_lock);
361         return ret;
362 }
363
364 static inline bool sk_psock_queue_empty(const struct sk_psock *psock)
365 {
366         return psock ? list_empty(&psock->ingress_msg) : true;
367 }
368
369 static inline void kfree_sk_msg(struct sk_msg *msg)
370 {
371         if (msg->skb)
372                 consume_skb(msg->skb);
373         kfree(msg);
374 }
375
376 static inline void sk_psock_report_error(struct sk_psock *psock, int err)
377 {
378         struct sock *sk = psock->sk;
379
380         sk->sk_err = err;
381         sk_error_report(sk);
382 }
383
384 struct sk_psock *sk_psock_init(struct sock *sk, int node);
385 void sk_psock_stop(struct sk_psock *psock);
386
387 #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
388 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock);
389 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock);
390 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock);
391 #else
392 static inline int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock)
393 {
394         return -EOPNOTSUPP;
395 }
396
397 static inline void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)
398 {
399 }
400
401 static inline void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)
402 {
403 }
404 #endif
405
406 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock);
407 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock);
408
409 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,
410                          struct sk_msg *msg);
411
412 static inline struct sk_psock_link *sk_psock_init_link(void)
413 {
414         return kzalloc(sizeof(struct sk_psock_link),
415                        GFP_ATOMIC | __GFP_NOWARN);
416 }
417
418 static inline void sk_psock_free_link(struct sk_psock_link *link)
419 {
420         kfree(link);
421 }
422
423 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock);
424
425 static inline void sk_psock_cork_free(struct sk_psock *psock)
426 {
427         if (psock->cork) {
428                 sk_msg_free(psock->sk, psock->cork);
429                 kfree(psock->cork);
430                 psock->cork = NULL;
431         }
432 }
433
434 static inline void sk_psock_restore_proto(struct sock *sk,
435                                           struct sk_psock *psock)
436 {
437         if (psock->psock_update_sk_prot)
438                 psock->psock_update_sk_prot(sk, psock, true);
439 }
440
441 static inline struct sk_psock *sk_psock_get(struct sock *sk)
442 {
443         struct sk_psock *psock;
444
445         rcu_read_lock();
446         psock = sk_psock(sk);
447         if (psock && !refcount_inc_not_zero(&psock->refcnt))
448                 psock = NULL;
449         rcu_read_unlock();
450         return psock;
451 }
452
453 void sk_psock_drop(struct sock *sk, struct sk_psock *psock);
454
455 static inline void sk_psock_put(struct sock *sk, struct sk_psock *psock)
456 {
457         if (refcount_dec_and_test(&psock->refcnt))
458                 sk_psock_drop(sk, psock);
459 }
460
461 static inline void sk_psock_data_ready(struct sock *sk, struct sk_psock *psock)
462 {
463         if (psock->saved_data_ready)
464                 psock->saved_data_ready(sk);
465         else
466                 sk->sk_data_ready(sk);
467 }
468
469 static inline void psock_set_prog(struct bpf_prog **pprog,
470                                   struct bpf_prog *prog)
471 {
472         prog = xchg(pprog, prog);
473         if (prog)
474                 bpf_prog_put(prog);
475 }
476
477 static inline int psock_replace_prog(struct bpf_prog **pprog,
478                                      struct bpf_prog *prog,
479                                      struct bpf_prog *old)
480 {
481         if (cmpxchg(pprog, old, prog) != old)
482                 return -ENOENT;
483
484         if (old)
485                 bpf_prog_put(old);
486
487         return 0;
488 }
489
490 static inline void psock_progs_drop(struct sk_psock_progs *progs)
491 {
492         psock_set_prog(&progs->msg_parser, NULL);
493         psock_set_prog(&progs->stream_parser, NULL);
494         psock_set_prog(&progs->stream_verdict, NULL);
495         psock_set_prog(&progs->skb_verdict, NULL);
496 }
497
498 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb);
499
500 static inline bool sk_psock_strp_enabled(struct sk_psock *psock)
501 {
502         if (!psock)
503                 return false;
504         return !!psock->saved_data_ready;
505 }
506
507 #if IS_ENABLED(CONFIG_NET_SOCK_MSG)
508
509 #define BPF_F_STRPARSER (1UL << 1)
510
511 /* We only have two bits so far. */
512 #define BPF_F_PTR_MASK ~(BPF_F_INGRESS | BPF_F_STRPARSER)
513
514 static inline bool skb_bpf_strparser(const struct sk_buff *skb)
515 {
516         unsigned long sk_redir = skb->_sk_redir;
517
518         return sk_redir & BPF_F_STRPARSER;
519 }
520
521 static inline void skb_bpf_set_strparser(struct sk_buff *skb)
522 {
523         skb->_sk_redir |= BPF_F_STRPARSER;
524 }
525
526 static inline bool skb_bpf_ingress(const struct sk_buff *skb)
527 {
528         unsigned long sk_redir = skb->_sk_redir;
529
530         return sk_redir & BPF_F_INGRESS;
531 }
532
533 static inline void skb_bpf_set_ingress(struct sk_buff *skb)
534 {
535         skb->_sk_redir |= BPF_F_INGRESS;
536 }
537
538 static inline void skb_bpf_set_redir(struct sk_buff *skb, struct sock *sk_redir,
539                                      bool ingress)
540 {
541         skb->_sk_redir = (unsigned long)sk_redir;
542         if (ingress)
543                 skb->_sk_redir |= BPF_F_INGRESS;
544 }
545
546 static inline struct sock *skb_bpf_redirect_fetch(const struct sk_buff *skb)
547 {
548         unsigned long sk_redir = skb->_sk_redir;
549
550         return (struct sock *)(sk_redir & BPF_F_PTR_MASK);
551 }
552
553 static inline void skb_bpf_redirect_clear(struct sk_buff *skb)
554 {
555         skb->_sk_redir = 0;
556 }
557 #endif /* CONFIG_NET_SOCK_MSG */
558 #endif /* _LINUX_SKMSG_H */