bpf: cpumap: Implement XDP_REDIRECT for eBPF programs attached to map entries

Introduce XDP_REDIRECT support for eBPF programs attached to cpumap
entries.
This patch has been tested on Marvell ESPRESSObin using a modified
version of xdp_redirect_cpu sample in order to attach a XDP program
to CPUMAP entries to perform a redirect on the mvneta interface.
In particular the following scenario has been tested:

rq (cpu0) --> mvneta - XDP_REDIRECT (cpu0) --> CPUMAP - XDP_REDIRECT (cpu1) --> mvneta

$./xdp_redirect_cpu -p xdp_cpu_map0 -d eth0 -c 1 -e xdp_redirect \
	-f xdp_redirect_kern.o -m tx_port -r eth0

tx: 285.2 Kpps rx: 285.2 Kpps

Attaching a simple XDP program on eth0 to perform XDP_TX gives
comparable results:

tx: 288.4 Kpps rx: 288.4 Kpps

Co-developed-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Link: https://lore.kernel.org/bpf/2cf8373a731867af302b00c4ff16c122630c4980.1594734381.git.lorenzo@kernel.org

diff --git a/include/net/xdp.h b/include/net/xdp.h
index 83b9e01..5be0d4d 100644 (file)
--- a/include/net/xdp.h
+++ b/include/net/xdp.h
@@ -99,6 +99,7 @@ struct xdp_frame {
 };
 
 struct xdp_cpumap_stats {
+       unsigned int redirect;
        unsigned int pass;
        unsigned int drop;
 };

diff --git a/include/trace/events/xdp.h b/include/trace/events/xdp.h
index e2c99f5..cd24e8a 100644 (file)
--- a/include/trace/events/xdp.h
+++ b/include/trace/events/xdp.h
@@ -190,6 +190,7 @@ TRACE_EVENT(xdp_cpumap_kthread,
                __field(int, sched)
                __field(unsigned int, xdp_pass)
                __field(unsigned int, xdp_drop)
+               __field(unsigned int, xdp_redirect)
        ),
 
        TP_fast_assign(
@@ -201,18 +202,19 @@ TRACE_EVENT(xdp_cpumap_kthread,
                __entry->sched  = sched;
                __entry->xdp_pass       = xdp_stats->pass;
                __entry->xdp_drop       = xdp_stats->drop;
+               __entry->xdp_redirect   = xdp_stats->redirect;
        ),
 
        TP_printk("kthread"
                  " cpu=%d map_id=%d action=%s"
                  " processed=%u drops=%u"
                  " sched=%d"
-                 " xdp_pass=%u xdp_drop=%u",
+                 " xdp_pass=%u xdp_drop=%u xdp_redirect=%u",
                  __entry->cpu, __entry->map_id,
                  __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB),
                  __entry->processed, __entry->drops,
                  __entry->sched,
-                 __entry->xdp_pass, __entry->xdp_drop)
+                 __entry->xdp_pass, __entry->xdp_drop, __entry->xdp_redirect)
 );
 
 TRACE_EVENT(xdp_cpumap_enqueue,

diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index b3a8aea..4c95d06 100644 (file)
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -239,7 +239,7 @@ static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu,
        if (!rcpu->prog)
                return n;
 
-       rcu_read_lock();
+       rcu_read_lock_bh();
 
        xdp_set_return_frame_no_direct();
        xdp.rxq = &rxq;
@@ -267,6 +267,16 @@ static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu,
                                stats->pass++;
                        }
                        break;
+               case XDP_REDIRECT:
+                       err = xdp_do_redirect(xdpf->dev_rx, &xdp,
+                                             rcpu->prog);
+                       if (unlikely(err)) {
+                               xdp_return_frame(xdpf);
+                               stats->drop++;
+                       } else {
+                               stats->redirect++;
+                       }
+                       break;
                default:
                        bpf_warn_invalid_xdp_action(act);
                        /* fallthrough */
@@ -277,9 +287,12 @@ static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu,
                }
        }
 
+       if (stats->redirect)
+               xdp_do_flush_map();
+
        xdp_clear_return_frame_no_direct();
 
-       rcu_read_unlock();
+       rcu_read_unlock_bh(); /* resched point, may call do_softirq() */
 
        return nframes;
 }