1 /* SPDX-License-Identifier: GPL-2.0 */
2 #include <linux/init.h>
3 #include <linux/module.h>
4 #include <linux/netfilter.h>
5 #include <net/flow_offload.h>
6 #include <net/netfilter/nf_tables.h>
7 #include <net/netfilter/nf_tables_offload.h>
8 #include <net/pkt_cls.h>
10 static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
12 struct nft_flow_rule *flow;
14 flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
18 flow->rule = flow_rule_alloc(num_actions);
24 flow->rule->match.dissector = &flow->match.dissector;
25 flow->rule->match.mask = &flow->match.mask;
26 flow->rule->match.key = &flow->match.key;
31 void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow,
32 enum flow_dissector_key_id addr_type)
34 struct nft_flow_match *match = &flow->match;
35 struct nft_flow_key *mask = &match->mask;
36 struct nft_flow_key *key = &match->key;
38 if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL))
41 key->control.addr_type = addr_type;
42 mask->control.addr_type = 0xffff;
43 match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_CONTROL);
44 match->dissector.offset[FLOW_DISSECTOR_KEY_CONTROL] =
45 offsetof(struct nft_flow_key, control);
48 struct nft_offload_ethertype {
53 static void nft_flow_rule_transfer_vlan(struct nft_offload_ctx *ctx,
54 struct nft_flow_rule *flow)
56 struct nft_flow_match *match = &flow->match;
57 struct nft_offload_ethertype ethertype = {
58 .value = match->key.basic.n_proto,
59 .mask = match->mask.basic.n_proto,
62 if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_VLAN) &&
63 (match->key.vlan.vlan_tpid == htons(ETH_P_8021Q) ||
64 match->key.vlan.vlan_tpid == htons(ETH_P_8021AD))) {
65 match->key.basic.n_proto = match->key.cvlan.vlan_tpid;
66 match->mask.basic.n_proto = match->mask.cvlan.vlan_tpid;
67 match->key.cvlan.vlan_tpid = match->key.vlan.vlan_tpid;
68 match->mask.cvlan.vlan_tpid = match->mask.vlan.vlan_tpid;
69 match->key.vlan.vlan_tpid = ethertype.value;
70 match->mask.vlan.vlan_tpid = ethertype.mask;
71 match->dissector.offset[FLOW_DISSECTOR_KEY_CVLAN] =
72 offsetof(struct nft_flow_key, cvlan);
73 match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_CVLAN);
74 } else if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_BASIC) &&
75 (match->key.basic.n_proto == htons(ETH_P_8021Q) ||
76 match->key.basic.n_proto == htons(ETH_P_8021AD))) {
77 match->key.basic.n_proto = match->key.vlan.vlan_tpid;
78 match->mask.basic.n_proto = match->mask.vlan.vlan_tpid;
79 match->key.vlan.vlan_tpid = ethertype.value;
80 match->mask.vlan.vlan_tpid = ethertype.mask;
81 match->dissector.offset[FLOW_DISSECTOR_KEY_VLAN] =
82 offsetof(struct nft_flow_key, vlan);
83 match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_VLAN);
87 struct nft_flow_rule *nft_flow_rule_create(struct net *net,
88 const struct nft_rule *rule)
90 struct nft_offload_ctx *ctx;
91 struct nft_flow_rule *flow;
92 int num_actions = 0, err;
93 struct nft_expr *expr;
95 expr = nft_expr_first(rule);
96 while (nft_expr_more(rule, expr)) {
97 if (expr->ops->offload_action &&
98 expr->ops->offload_action(expr))
101 expr = nft_expr_next(expr);
104 if (num_actions == 0)
105 return ERR_PTR(-EOPNOTSUPP);
107 flow = nft_flow_rule_alloc(num_actions);
109 return ERR_PTR(-ENOMEM);
111 expr = nft_expr_first(rule);
113 ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
119 ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
121 while (nft_expr_more(rule, expr)) {
122 if (!expr->ops->offload) {
126 err = expr->ops->offload(ctx, flow, expr);
130 expr = nft_expr_next(expr);
132 nft_flow_rule_transfer_vlan(ctx, flow);
134 flow->proto = ctx->dep.l3num;
140 nft_flow_rule_destroy(flow);
145 void nft_flow_rule_destroy(struct nft_flow_rule *flow)
147 struct flow_action_entry *entry;
150 flow_action_for_each(i, entry, &flow->rule->action) {
152 case FLOW_ACTION_REDIRECT:
153 case FLOW_ACTION_MIRRED:
164 void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
165 enum nft_offload_dep_type type)
167 ctx->dep.type = type;
170 void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
171 const void *data, u32 len)
173 switch (ctx->dep.type) {
174 case NFT_OFFLOAD_DEP_NETWORK:
175 WARN_ON(len != sizeof(__u16));
176 memcpy(&ctx->dep.l3num, data, sizeof(__u16));
178 case NFT_OFFLOAD_DEP_TRANSPORT:
179 WARN_ON(len != sizeof(__u8));
180 memcpy(&ctx->dep.protonum, data, sizeof(__u8));
185 ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
188 static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
189 __be16 proto, int priority,
190 struct netlink_ext_ack *extack)
192 common->protocol = proto;
193 common->prio = priority;
194 common->extack = extack;
197 static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
198 struct list_head *cb_list)
200 struct flow_block_cb *block_cb;
203 list_for_each_entry(block_cb, cb_list, list) {
204 err = block_cb->cb(type, type_data, block_cb->cb_priv);
211 static int nft_chain_offload_priority(const struct nft_base_chain *basechain)
213 if (basechain->ops.priority <= 0 ||
214 basechain->ops.priority > USHRT_MAX)
220 bool nft_chain_offload_support(const struct nft_base_chain *basechain)
222 struct net_device *dev;
223 struct nft_hook *hook;
225 if (nft_chain_offload_priority(basechain) < 0)
228 list_for_each_entry(hook, &basechain->hook_list, list) {
229 if (hook->ops.pf != NFPROTO_NETDEV ||
230 hook->ops.hooknum != NF_NETDEV_INGRESS)
234 if (!dev->netdev_ops->ndo_setup_tc && !flow_indr_dev_exists())
241 static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
242 const struct nft_base_chain *basechain,
243 const struct nft_rule *rule,
244 const struct nft_flow_rule *flow,
245 struct netlink_ext_ack *extack,
246 enum flow_cls_command command)
248 __be16 proto = ETH_P_ALL;
250 memset(cls_flow, 0, sizeof(*cls_flow));
255 nft_flow_offload_common_init(&cls_flow->common, proto,
256 basechain->ops.priority, extack);
257 cls_flow->command = command;
258 cls_flow->cookie = (unsigned long) rule;
260 cls_flow->rule = flow->rule;
263 static int nft_flow_offload_cmd(const struct nft_chain *chain,
264 const struct nft_rule *rule,
265 struct nft_flow_rule *flow,
266 enum flow_cls_command command,
267 struct flow_cls_offload *cls_flow)
269 struct netlink_ext_ack extack = {};
270 struct nft_base_chain *basechain;
272 if (!nft_is_base_chain(chain))
275 basechain = nft_base_chain(chain);
276 nft_flow_cls_offload_setup(cls_flow, basechain, rule, flow, &extack,
279 return nft_setup_cb_call(TC_SETUP_CLSFLOWER, cls_flow,
280 &basechain->flow_block.cb_list);
283 static int nft_flow_offload_rule(const struct nft_chain *chain,
284 struct nft_rule *rule,
285 struct nft_flow_rule *flow,
286 enum flow_cls_command command)
288 struct flow_cls_offload cls_flow;
290 return nft_flow_offload_cmd(chain, rule, flow, command, &cls_flow);
293 int nft_flow_rule_stats(const struct nft_chain *chain,
294 const struct nft_rule *rule)
296 struct flow_cls_offload cls_flow = {};
297 struct nft_expr *expr, *next;
300 err = nft_flow_offload_cmd(chain, rule, NULL, FLOW_CLS_STATS,
305 nft_rule_for_each_expr(expr, next, rule) {
306 if (expr->ops->offload_stats)
307 expr->ops->offload_stats(expr, &cls_flow.stats);
313 static int nft_flow_offload_bind(struct flow_block_offload *bo,
314 struct nft_base_chain *basechain)
316 list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
320 static int nft_flow_offload_unbind(struct flow_block_offload *bo,
321 struct nft_base_chain *basechain)
323 struct flow_block_cb *block_cb, *next;
324 struct flow_cls_offload cls_flow;
325 struct netlink_ext_ack extack;
326 struct nft_chain *chain;
327 struct nft_rule *rule;
329 chain = &basechain->chain;
330 list_for_each_entry(rule, &chain->rules, list) {
331 memset(&extack, 0, sizeof(extack));
332 nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
333 &extack, FLOW_CLS_DESTROY);
334 nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
337 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
338 list_del(&block_cb->list);
339 flow_block_cb_free(block_cb);
345 static int nft_block_setup(struct nft_base_chain *basechain,
346 struct flow_block_offload *bo,
347 enum flow_block_command cmd)
352 case FLOW_BLOCK_BIND:
353 err = nft_flow_offload_bind(bo, basechain);
355 case FLOW_BLOCK_UNBIND:
356 err = nft_flow_offload_unbind(bo, basechain);
366 static void nft_flow_block_offload_init(struct flow_block_offload *bo,
368 enum flow_block_command cmd,
369 struct nft_base_chain *basechain,
370 struct netlink_ext_ack *extack)
372 memset(bo, 0, sizeof(*bo));
374 bo->block = &basechain->flow_block;
376 bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
378 bo->cb_list_head = &basechain->flow_block.cb_list;
379 INIT_LIST_HEAD(&bo->cb_list);
382 static int nft_block_offload_cmd(struct nft_base_chain *chain,
383 struct net_device *dev,
384 enum flow_block_command cmd)
386 struct netlink_ext_ack extack = {};
387 struct flow_block_offload bo;
390 nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
392 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
396 return nft_block_setup(chain, &bo, cmd);
399 static void nft_indr_block_cleanup(struct flow_block_cb *block_cb)
401 struct nft_base_chain *basechain = block_cb->indr.data;
402 struct net_device *dev = block_cb->indr.dev;
403 struct netlink_ext_ack extack = {};
404 struct nftables_pernet *nft_net;
405 struct net *net = dev_net(dev);
406 struct flow_block_offload bo;
408 nft_flow_block_offload_init(&bo, dev_net(dev), FLOW_BLOCK_UNBIND,
410 nft_net = nft_pernet(net);
411 mutex_lock(&nft_net->commit_mutex);
412 list_del(&block_cb->driver_list);
413 list_move(&block_cb->list, &bo.cb_list);
414 nft_flow_offload_unbind(&bo, basechain);
415 mutex_unlock(&nft_net->commit_mutex);
418 static int nft_indr_block_offload_cmd(struct nft_base_chain *basechain,
419 struct net_device *dev,
420 enum flow_block_command cmd)
422 struct netlink_ext_ack extack = {};
423 struct flow_block_offload bo;
426 nft_flow_block_offload_init(&bo, dev_net(dev), cmd, basechain, &extack);
428 err = flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_BLOCK, basechain, &bo,
429 nft_indr_block_cleanup);
433 if (list_empty(&bo.cb_list))
436 return nft_block_setup(basechain, &bo, cmd);
439 static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
440 struct net_device *dev,
441 enum flow_block_command cmd)
445 if (dev->netdev_ops->ndo_setup_tc)
446 err = nft_block_offload_cmd(basechain, dev, cmd);
448 err = nft_indr_block_offload_cmd(basechain, dev, cmd);
453 static int nft_flow_block_chain(struct nft_base_chain *basechain,
454 const struct net_device *this_dev,
455 enum flow_block_command cmd)
457 struct net_device *dev;
458 struct nft_hook *hook;
461 list_for_each_entry(hook, &basechain->hook_list, list) {
463 if (this_dev && this_dev != dev)
466 err = nft_chain_offload_cmd(basechain, dev, cmd);
467 if (err < 0 && cmd == FLOW_BLOCK_BIND) {
479 list_for_each_entry(hook, &basechain->hook_list, list) {
484 nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
489 static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy,
490 enum flow_block_command cmd)
492 struct nft_base_chain *basechain;
495 if (!nft_is_base_chain(chain))
498 basechain = nft_base_chain(chain);
499 policy = ppolicy ? *ppolicy : basechain->policy;
501 /* Only default policy to accept is supported for now. */
502 if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
505 return nft_flow_block_chain(basechain, NULL, cmd);
508 static void nft_flow_rule_offload_abort(struct net *net,
509 struct nft_trans *trans)
511 struct nftables_pernet *nft_net = nft_pernet(net);
514 list_for_each_entry_continue_reverse(trans, &nft_net->commit_list, list) {
515 if (trans->ctx.family != NFPROTO_NETDEV)
518 switch (trans->msg_type) {
519 case NFT_MSG_NEWCHAIN:
520 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
521 nft_trans_chain_update(trans))
524 err = nft_flow_offload_chain(trans->ctx.chain, NULL,
527 case NFT_MSG_DELCHAIN:
528 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
531 err = nft_flow_offload_chain(trans->ctx.chain, NULL,
534 case NFT_MSG_NEWRULE:
535 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
538 err = nft_flow_offload_rule(trans->ctx.chain,
539 nft_trans_rule(trans),
540 NULL, FLOW_CLS_DESTROY);
542 case NFT_MSG_DELRULE:
543 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
546 err = nft_flow_offload_rule(trans->ctx.chain,
547 nft_trans_rule(trans),
548 nft_trans_flow_rule(trans),
553 if (WARN_ON_ONCE(err))
558 int nft_flow_rule_offload_commit(struct net *net)
560 struct nftables_pernet *nft_net = nft_pernet(net);
561 struct nft_trans *trans;
565 list_for_each_entry(trans, &nft_net->commit_list, list) {
566 if (trans->ctx.family != NFPROTO_NETDEV)
569 switch (trans->msg_type) {
570 case NFT_MSG_NEWCHAIN:
571 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
572 nft_trans_chain_update(trans))
575 policy = nft_trans_chain_policy(trans);
576 err = nft_flow_offload_chain(trans->ctx.chain, &policy,
579 case NFT_MSG_DELCHAIN:
580 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
583 policy = nft_trans_chain_policy(trans);
584 err = nft_flow_offload_chain(trans->ctx.chain, &policy,
587 case NFT_MSG_NEWRULE:
588 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
591 if (trans->ctx.flags & NLM_F_REPLACE ||
592 !(trans->ctx.flags & NLM_F_APPEND)) {
596 err = nft_flow_offload_rule(trans->ctx.chain,
597 nft_trans_rule(trans),
598 nft_trans_flow_rule(trans),
601 case NFT_MSG_DELRULE:
602 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
605 err = nft_flow_offload_rule(trans->ctx.chain,
606 nft_trans_rule(trans),
607 NULL, FLOW_CLS_DESTROY);
612 nft_flow_rule_offload_abort(net, trans);
620 static struct nft_chain *__nft_offload_get_chain(const struct nftables_pernet *nft_net,
621 struct net_device *dev)
623 struct nft_base_chain *basechain;
624 struct nft_hook *hook, *found;
625 const struct nft_table *table;
626 struct nft_chain *chain;
628 list_for_each_entry(table, &nft_net->tables, list) {
629 if (table->family != NFPROTO_NETDEV)
632 list_for_each_entry(chain, &table->chains, list) {
633 if (!nft_is_base_chain(chain) ||
634 !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
638 basechain = nft_base_chain(chain);
639 list_for_each_entry(hook, &basechain->hook_list, list) {
640 if (hook->ops.dev != dev)
656 static int nft_offload_netdev_event(struct notifier_block *this,
657 unsigned long event, void *ptr)
659 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
660 struct nftables_pernet *nft_net;
661 struct net *net = dev_net(dev);
662 struct nft_chain *chain;
664 if (event != NETDEV_UNREGISTER)
667 nft_net = nft_pernet(net);
668 mutex_lock(&nft_net->commit_mutex);
669 chain = __nft_offload_get_chain(nft_net, dev);
671 nft_flow_block_chain(nft_base_chain(chain), dev,
674 mutex_unlock(&nft_net->commit_mutex);
679 static struct notifier_block nft_offload_netdev_notifier = {
680 .notifier_call = nft_offload_netdev_event,
683 int nft_offload_init(void)
685 return register_netdevice_notifier(&nft_offload_netdev_notifier);
688 void nft_offload_exit(void)
690 unregister_netdevice_notifier(&nft_offload_netdev_notifier);