1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * net/sched/sch_api.c Packet scheduler API.
5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
9 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired.
10 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
11 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support
14 #include <linux/module.h>
15 #include <linux/types.h>
16 #include <linux/kernel.h>
17 #include <linux/string.h>
18 #include <linux/errno.h>
19 #include <linux/skbuff.h>
20 #include <linux/init.h>
21 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h>
23 #include <linux/kmod.h>
24 #include <linux/list.h>
25 #include <linux/hrtimer.h>
26 #include <linux/slab.h>
27 #include <linux/hashtable.h>
29 #include <net/net_namespace.h>
31 #include <net/netlink.h>
32 #include <net/pkt_sched.h>
33 #include <net/pkt_cls.h>
35 #include <trace/events/qdisc.h>
42 This file consists of two interrelated parts:
44 1. queueing disciplines manager frontend.
45 2. traffic classes manager frontend.
47 Generally, queueing discipline ("qdisc") is a black box,
48 which is able to enqueue packets and to dequeue them (when
49 device is ready to send something) in order and at times
50 determined by algorithm hidden in it.
52 qdisc's are divided to two categories:
53 - "queues", which have no internal structure visible from outside.
54 - "schedulers", which split all the packets to "traffic classes",
55 using "packet classifiers" (look at cls_api.c)
57 In turn, classes may have child qdiscs (as rule, queues)
58 attached to them etc. etc. etc.
60 The goal of the routines in this file is to translate
61 information supplied by user in the form of handles
62 to more intelligible for kernel form, to make some sanity
63 checks and part of work, which is common to all qdiscs
64 and to provide rtnetlink notifications.
66 All real intelligent work is done inside qdisc modules.
70 Every discipline has two major routines: enqueue and dequeue.
74 dequeue usually returns a skb to send. It is allowed to return NULL,
75 but it does not mean that queue is empty, it just means that
76 discipline does not want to send anything this time.
77 Queue is really empty if q->q.qlen == 0.
78 For complicated disciplines with multiple queues q->q is not
79 real packet queue, but however q->q.qlen must be valid.
83 enqueue returns 0, if packet was enqueued successfully.
84 If packet (this one or another one) was dropped, it returns
86 NET_XMIT_DROP - this packet dropped
87 Expected action: do not backoff, but wait until queue will clear.
88 NET_XMIT_CN - probably this packet enqueued, but another one dropped.
89 Expected action: backoff or ignore
95 like dequeue but without removing a packet from the queue
99 returns qdisc to initial state: purge all buffers, clear all
100 timers, counters (except for statistics) etc.
104 initializes newly created qdisc.
108 destroys resources allocated by init and during lifetime of qdisc.
112 changes qdisc parameters.
115 /* Protects list of registered TC modules. It is pure SMP lock. */
116 static DEFINE_RWLOCK(qdisc_mod_lock);
119 /************************************************
120 * Queueing disciplines manipulation. *
121 ************************************************/
124 /* The list of all installed queueing disciplines. */
126 static struct Qdisc_ops *qdisc_base;
128 /* Register/unregister queueing discipline */
130 int register_qdisc(struct Qdisc_ops *qops)
132 struct Qdisc_ops *q, **qp;
135 write_lock(&qdisc_mod_lock);
136 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
137 if (!strcmp(qops->id, q->id))
140 if (qops->enqueue == NULL)
141 qops->enqueue = noop_qdisc_ops.enqueue;
142 if (qops->peek == NULL) {
143 if (qops->dequeue == NULL)
144 qops->peek = noop_qdisc_ops.peek;
148 if (qops->dequeue == NULL)
149 qops->dequeue = noop_qdisc_ops.dequeue;
152 const struct Qdisc_class_ops *cops = qops->cl_ops;
154 if (!(cops->find && cops->walk && cops->leaf))
157 if (cops->tcf_block && !(cops->bind_tcf && cops->unbind_tcf))
165 write_unlock(&qdisc_mod_lock);
172 EXPORT_SYMBOL(register_qdisc);
174 void unregister_qdisc(struct Qdisc_ops *qops)
176 struct Qdisc_ops *q, **qp;
179 write_lock(&qdisc_mod_lock);
180 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
188 write_unlock(&qdisc_mod_lock);
190 WARN(err, "unregister qdisc(%s) failed\n", qops->id);
192 EXPORT_SYMBOL(unregister_qdisc);
194 /* Get default qdisc if not otherwise specified */
195 void qdisc_get_default(char *name, size_t len)
197 read_lock(&qdisc_mod_lock);
198 strscpy(name, default_qdisc_ops->id, len);
199 read_unlock(&qdisc_mod_lock);
202 static struct Qdisc_ops *qdisc_lookup_default(const char *name)
204 struct Qdisc_ops *q = NULL;
206 for (q = qdisc_base; q; q = q->next) {
207 if (!strcmp(name, q->id)) {
208 if (!try_module_get(q->owner))
217 /* Set new default qdisc to use */
218 int qdisc_set_default(const char *name)
220 const struct Qdisc_ops *ops;
222 if (!capable(CAP_NET_ADMIN))
225 write_lock(&qdisc_mod_lock);
226 ops = qdisc_lookup_default(name);
228 /* Not found, drop lock and try to load module */
229 write_unlock(&qdisc_mod_lock);
230 request_module("sch_%s", name);
231 write_lock(&qdisc_mod_lock);
233 ops = qdisc_lookup_default(name);
237 /* Set new default */
238 module_put(default_qdisc_ops->owner);
239 default_qdisc_ops = ops;
241 write_unlock(&qdisc_mod_lock);
243 return ops ? 0 : -ENOENT;
246 #ifdef CONFIG_NET_SCH_DEFAULT
247 /* Set default value from kernel config */
248 static int __init sch_default_qdisc(void)
250 return qdisc_set_default(CONFIG_DEFAULT_NET_SCH);
252 late_initcall(sch_default_qdisc);
255 /* We know handle. Find qdisc among all qdisc's attached to device
256 * (root qdisc, all its children, children of children etc.)
257 * Note: caller either uses rtnl or rcu_read_lock()
260 static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle)
264 if (!qdisc_dev(root))
265 return (root->handle == handle ? root : NULL);
267 if (!(root->flags & TCQ_F_BUILTIN) &&
268 root->handle == handle)
271 hash_for_each_possible_rcu(qdisc_dev(root)->qdisc_hash, q, hash, handle,
272 lockdep_rtnl_is_held()) {
273 if (q->handle == handle)
279 void qdisc_hash_add(struct Qdisc *q, bool invisible)
281 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
283 hash_add_rcu(qdisc_dev(q)->qdisc_hash, &q->hash, q->handle);
285 q->flags |= TCQ_F_INVISIBLE;
288 EXPORT_SYMBOL(qdisc_hash_add);
290 void qdisc_hash_del(struct Qdisc *q)
292 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
294 hash_del_rcu(&q->hash);
297 EXPORT_SYMBOL(qdisc_hash_del);
299 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
305 q = qdisc_match_from_root(rtnl_dereference(dev->qdisc), handle);
309 if (dev_ingress_queue(dev))
310 q = qdisc_match_from_root(
311 dev_ingress_queue(dev)->qdisc_sleeping,
317 struct Qdisc *qdisc_lookup_rcu(struct net_device *dev, u32 handle)
319 struct netdev_queue *nq;
324 q = qdisc_match_from_root(rcu_dereference(dev->qdisc), handle);
328 nq = dev_ingress_queue_rcu(dev);
330 q = qdisc_match_from_root(nq->qdisc_sleeping, handle);
335 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
338 const struct Qdisc_class_ops *cops = p->ops->cl_ops;
342 cl = cops->find(p, classid);
346 return cops->leaf(p, cl);
349 /* Find queueing discipline by name */
351 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind)
353 struct Qdisc_ops *q = NULL;
356 read_lock(&qdisc_mod_lock);
357 for (q = qdisc_base; q; q = q->next) {
358 if (nla_strcmp(kind, q->id) == 0) {
359 if (!try_module_get(q->owner))
364 read_unlock(&qdisc_mod_lock);
369 /* The linklayer setting were not transferred from iproute2, in older
370 * versions, and the rate tables lookup systems have been dropped in
371 * the kernel. To keep backward compatible with older iproute2 tc
372 * utils, we detect the linklayer setting by detecting if the rate
373 * table were modified.
375 * For linklayer ATM table entries, the rate table will be aligned to
376 * 48 bytes, thus some table entries will contain the same value. The
377 * mpu (min packet unit) is also encoded into the old rate table, thus
378 * starting from the mpu, we find low and high table entries for
379 * mapping this cell. If these entries contain the same value, when
380 * the rate tables have been modified for linklayer ATM.
382 * This is done by rounding mpu to the nearest 48 bytes cell/entry,
383 * and then roundup to the next cell, calc the table entry one below,
386 static __u8 __detect_linklayer(struct tc_ratespec *r, __u32 *rtab)
388 int low = roundup(r->mpu, 48);
389 int high = roundup(low+1, 48);
390 int cell_low = low >> r->cell_log;
391 int cell_high = (high >> r->cell_log) - 1;
393 /* rtab is too inaccurate at rates > 100Mbit/s */
394 if ((r->rate > (100000000/8)) || (rtab[0] == 0)) {
395 pr_debug("TC linklayer: Giving up ATM detection\n");
396 return TC_LINKLAYER_ETHERNET;
399 if ((cell_high > cell_low) && (cell_high < 256)
400 && (rtab[cell_low] == rtab[cell_high])) {
401 pr_debug("TC linklayer: Detected ATM, low(%d)=high(%d)=%u\n",
402 cell_low, cell_high, rtab[cell_high]);
403 return TC_LINKLAYER_ATM;
405 return TC_LINKLAYER_ETHERNET;
408 static struct qdisc_rate_table *qdisc_rtab_list;
410 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r,
412 struct netlink_ext_ack *extack)
414 struct qdisc_rate_table *rtab;
416 if (tab == NULL || r->rate == 0 ||
417 r->cell_log == 0 || r->cell_log >= 32 ||
418 nla_len(tab) != TC_RTAB_SIZE) {
419 NL_SET_ERR_MSG(extack, "Invalid rate table parameters for searching");
423 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) {
424 if (!memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) &&
425 !memcmp(&rtab->data, nla_data(tab), 1024)) {
431 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL);
435 memcpy(rtab->data, nla_data(tab), 1024);
436 if (r->linklayer == TC_LINKLAYER_UNAWARE)
437 r->linklayer = __detect_linklayer(r, rtab->data);
438 rtab->next = qdisc_rtab_list;
439 qdisc_rtab_list = rtab;
441 NL_SET_ERR_MSG(extack, "Failed to allocate new qdisc rate table");
445 EXPORT_SYMBOL(qdisc_get_rtab);
447 void qdisc_put_rtab(struct qdisc_rate_table *tab)
449 struct qdisc_rate_table *rtab, **rtabp;
451 if (!tab || --tab->refcnt)
454 for (rtabp = &qdisc_rtab_list;
455 (rtab = *rtabp) != NULL;
456 rtabp = &rtab->next) {
464 EXPORT_SYMBOL(qdisc_put_rtab);
466 static LIST_HEAD(qdisc_stab_list);
468 static const struct nla_policy stab_policy[TCA_STAB_MAX + 1] = {
469 [TCA_STAB_BASE] = { .len = sizeof(struct tc_sizespec) },
470 [TCA_STAB_DATA] = { .type = NLA_BINARY },
473 static struct qdisc_size_table *qdisc_get_stab(struct nlattr *opt,
474 struct netlink_ext_ack *extack)
476 struct nlattr *tb[TCA_STAB_MAX + 1];
477 struct qdisc_size_table *stab;
478 struct tc_sizespec *s;
479 unsigned int tsize = 0;
483 err = nla_parse_nested_deprecated(tb, TCA_STAB_MAX, opt, stab_policy,
487 if (!tb[TCA_STAB_BASE]) {
488 NL_SET_ERR_MSG(extack, "Size table base attribute is missing");
489 return ERR_PTR(-EINVAL);
492 s = nla_data(tb[TCA_STAB_BASE]);
495 if (!tb[TCA_STAB_DATA]) {
496 NL_SET_ERR_MSG(extack, "Size table data attribute is missing");
497 return ERR_PTR(-EINVAL);
499 tab = nla_data(tb[TCA_STAB_DATA]);
500 tsize = nla_len(tb[TCA_STAB_DATA]) / sizeof(u16);
503 if (tsize != s->tsize || (!tab && tsize > 0)) {
504 NL_SET_ERR_MSG(extack, "Invalid size of size table");
505 return ERR_PTR(-EINVAL);
508 list_for_each_entry(stab, &qdisc_stab_list, list) {
509 if (memcmp(&stab->szopts, s, sizeof(*s)))
512 memcmp(stab->data, tab, flex_array_size(stab, data, tsize)))
518 if (s->size_log > STAB_SIZE_LOG_MAX ||
519 s->cell_log > STAB_SIZE_LOG_MAX) {
520 NL_SET_ERR_MSG(extack, "Invalid logarithmic size of size table");
521 return ERR_PTR(-EINVAL);
524 stab = kmalloc(struct_size(stab, data, tsize), GFP_KERNEL);
526 return ERR_PTR(-ENOMEM);
531 memcpy(stab->data, tab, flex_array_size(stab, data, tsize));
533 list_add_tail(&stab->list, &qdisc_stab_list);
538 void qdisc_put_stab(struct qdisc_size_table *tab)
543 if (--tab->refcnt == 0) {
544 list_del(&tab->list);
548 EXPORT_SYMBOL(qdisc_put_stab);
550 static int qdisc_dump_stab(struct sk_buff *skb, struct qdisc_size_table *stab)
554 nest = nla_nest_start_noflag(skb, TCA_STAB);
556 goto nla_put_failure;
557 if (nla_put(skb, TCA_STAB_BASE, sizeof(stab->szopts), &stab->szopts))
558 goto nla_put_failure;
559 nla_nest_end(skb, nest);
567 void __qdisc_calculate_pkt_len(struct sk_buff *skb,
568 const struct qdisc_size_table *stab)
572 pkt_len = skb->len + stab->szopts.overhead;
573 if (unlikely(!stab->szopts.tsize))
576 slot = pkt_len + stab->szopts.cell_align;
577 if (unlikely(slot < 0))
580 slot >>= stab->szopts.cell_log;
581 if (likely(slot < stab->szopts.tsize))
582 pkt_len = stab->data[slot];
584 pkt_len = stab->data[stab->szopts.tsize - 1] *
585 (slot / stab->szopts.tsize) +
586 stab->data[slot % stab->szopts.tsize];
588 pkt_len <<= stab->szopts.size_log;
590 if (unlikely(pkt_len < 1))
592 qdisc_skb_cb(skb)->pkt_len = pkt_len;
594 EXPORT_SYMBOL(__qdisc_calculate_pkt_len);
596 void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc)
598 if (!(qdisc->flags & TCQ_F_WARN_NONWC)) {
599 pr_warn("%s: %s qdisc %X: is non-work-conserving?\n",
600 txt, qdisc->ops->id, qdisc->handle >> 16);
601 qdisc->flags |= TCQ_F_WARN_NONWC;
604 EXPORT_SYMBOL(qdisc_warn_nonwc);
606 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
608 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog,
612 __netif_schedule(qdisc_root(wd->qdisc));
615 return HRTIMER_NORESTART;
618 void qdisc_watchdog_init_clockid(struct qdisc_watchdog *wd, struct Qdisc *qdisc,
621 hrtimer_init(&wd->timer, clockid, HRTIMER_MODE_ABS_PINNED);
622 wd->timer.function = qdisc_watchdog;
625 EXPORT_SYMBOL(qdisc_watchdog_init_clockid);
627 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
629 qdisc_watchdog_init_clockid(wd, qdisc, CLOCK_MONOTONIC);
631 EXPORT_SYMBOL(qdisc_watchdog_init);
633 void qdisc_watchdog_schedule_range_ns(struct qdisc_watchdog *wd, u64 expires,
636 if (test_bit(__QDISC_STATE_DEACTIVATED,
637 &qdisc_root_sleeping(wd->qdisc)->state))
640 if (hrtimer_is_queued(&wd->timer)) {
641 /* If timer is already set in [expires, expires + delta_ns],
642 * do not reprogram it.
644 if (wd->last_expires - expires <= delta_ns)
648 wd->last_expires = expires;
649 hrtimer_start_range_ns(&wd->timer,
650 ns_to_ktime(expires),
652 HRTIMER_MODE_ABS_PINNED);
654 EXPORT_SYMBOL(qdisc_watchdog_schedule_range_ns);
656 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd)
658 hrtimer_cancel(&wd->timer);
660 EXPORT_SYMBOL(qdisc_watchdog_cancel);
662 static struct hlist_head *qdisc_class_hash_alloc(unsigned int n)
664 struct hlist_head *h;
667 h = kvmalloc_array(n, sizeof(struct hlist_head), GFP_KERNEL);
670 for (i = 0; i < n; i++)
671 INIT_HLIST_HEAD(&h[i]);
676 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash)
678 struct Qdisc_class_common *cl;
679 struct hlist_node *next;
680 struct hlist_head *nhash, *ohash;
681 unsigned int nsize, nmask, osize;
684 /* Rehash when load factor exceeds 0.75 */
685 if (clhash->hashelems * 4 <= clhash->hashsize * 3)
687 nsize = clhash->hashsize * 2;
689 nhash = qdisc_class_hash_alloc(nsize);
693 ohash = clhash->hash;
694 osize = clhash->hashsize;
697 for (i = 0; i < osize; i++) {
698 hlist_for_each_entry_safe(cl, next, &ohash[i], hnode) {
699 h = qdisc_class_hash(cl->classid, nmask);
700 hlist_add_head(&cl->hnode, &nhash[h]);
703 clhash->hash = nhash;
704 clhash->hashsize = nsize;
705 clhash->hashmask = nmask;
706 sch_tree_unlock(sch);
710 EXPORT_SYMBOL(qdisc_class_hash_grow);
712 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash)
714 unsigned int size = 4;
716 clhash->hash = qdisc_class_hash_alloc(size);
719 clhash->hashsize = size;
720 clhash->hashmask = size - 1;
721 clhash->hashelems = 0;
724 EXPORT_SYMBOL(qdisc_class_hash_init);
726 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash)
728 kvfree(clhash->hash);
730 EXPORT_SYMBOL(qdisc_class_hash_destroy);
732 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash,
733 struct Qdisc_class_common *cl)
737 INIT_HLIST_NODE(&cl->hnode);
738 h = qdisc_class_hash(cl->classid, clhash->hashmask);
739 hlist_add_head(&cl->hnode, &clhash->hash[h]);
742 EXPORT_SYMBOL(qdisc_class_hash_insert);
744 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash,
745 struct Qdisc_class_common *cl)
747 hlist_del(&cl->hnode);
750 EXPORT_SYMBOL(qdisc_class_hash_remove);
752 /* Allocate an unique handle from space managed by kernel
753 * Possible range is [8000-FFFF]:0000 (0x8000 values)
755 static u32 qdisc_alloc_handle(struct net_device *dev)
758 static u32 autohandle = TC_H_MAKE(0x80000000U, 0);
761 autohandle += TC_H_MAKE(0x10000U, 0);
762 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0))
763 autohandle = TC_H_MAKE(0x80000000U, 0);
764 if (!qdisc_lookup(dev, autohandle))
772 void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len)
774 bool qdisc_is_offloaded = sch->flags & TCQ_F_OFFLOADED;
775 const struct Qdisc_class_ops *cops;
781 if (n == 0 && len == 0)
783 drops = max_t(int, n, 0);
785 while ((parentid = sch->parent)) {
786 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS))
789 if (sch->flags & TCQ_F_NOPARENT)
791 /* Notify parent qdisc only if child qdisc becomes empty.
793 * If child was empty even before update then backlog
794 * counter is screwed and we skip notification because
795 * parent class is already passive.
797 * If the original child was offloaded then it is allowed
798 * to be seem as empty, so the parent is notified anyway.
800 notify = !sch->q.qlen && !WARN_ON_ONCE(!n &&
801 !qdisc_is_offloaded);
802 /* TODO: perform the search on a per txq basis */
803 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid));
805 WARN_ON_ONCE(parentid != TC_H_ROOT);
808 cops = sch->ops->cl_ops;
809 if (notify && cops->qlen_notify) {
810 cl = cops->find(sch, parentid);
811 cops->qlen_notify(sch, cl);
814 sch->qstats.backlog -= len;
815 __qdisc_qstats_drop(sch, drops);
819 EXPORT_SYMBOL(qdisc_tree_reduce_backlog);
821 int qdisc_offload_dump_helper(struct Qdisc *sch, enum tc_setup_type type,
824 struct net_device *dev = qdisc_dev(sch);
827 sch->flags &= ~TCQ_F_OFFLOADED;
828 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
831 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data);
832 if (err == -EOPNOTSUPP)
836 sch->flags |= TCQ_F_OFFLOADED;
840 EXPORT_SYMBOL(qdisc_offload_dump_helper);
842 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
843 struct Qdisc *new, struct Qdisc *old,
844 enum tc_setup_type type, void *type_data,
845 struct netlink_ext_ack *extack)
847 bool any_qdisc_is_offloaded;
850 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
853 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data);
855 /* Don't report error if the graft is part of destroy operation. */
856 if (!err || !new || new == &noop_qdisc)
859 /* Don't report error if the parent, the old child and the new
860 * one are not offloaded.
862 any_qdisc_is_offloaded = new->flags & TCQ_F_OFFLOADED;
863 any_qdisc_is_offloaded |= sch && sch->flags & TCQ_F_OFFLOADED;
864 any_qdisc_is_offloaded |= old && old->flags & TCQ_F_OFFLOADED;
866 if (any_qdisc_is_offloaded)
867 NL_SET_ERR_MSG(extack, "Offloading graft operation failed.");
869 EXPORT_SYMBOL(qdisc_offload_graft_helper);
871 void qdisc_offload_query_caps(struct net_device *dev,
872 enum tc_setup_type type,
873 void *caps, size_t caps_len)
875 const struct net_device_ops *ops = dev->netdev_ops;
876 struct tc_query_caps_base base = {
881 memset(caps, 0, caps_len);
883 if (ops->ndo_setup_tc)
884 ops->ndo_setup_tc(dev, TC_QUERY_CAPS, &base);
886 EXPORT_SYMBOL(qdisc_offload_query_caps);
888 static void qdisc_offload_graft_root(struct net_device *dev,
889 struct Qdisc *new, struct Qdisc *old,
890 struct netlink_ext_ack *extack)
892 struct tc_root_qopt_offload graft_offload = {
893 .command = TC_ROOT_GRAFT,
894 .handle = new ? new->handle : 0,
895 .ingress = (new && new->flags & TCQ_F_INGRESS) ||
896 (old && old->flags & TCQ_F_INGRESS),
899 qdisc_offload_graft_helper(dev, NULL, new, old,
900 TC_SETUP_ROOT_QDISC, &graft_offload, extack);
903 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
904 u32 portid, u32 seq, u16 flags, int event)
906 struct gnet_stats_basic_sync __percpu *cpu_bstats = NULL;
907 struct gnet_stats_queue __percpu *cpu_qstats = NULL;
909 struct nlmsghdr *nlh;
910 unsigned char *b = skb_tail_pointer(skb);
912 struct qdisc_size_table *stab;
917 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
920 tcm = nlmsg_data(nlh);
921 tcm->tcm_family = AF_UNSPEC;
924 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
925 tcm->tcm_parent = clid;
926 tcm->tcm_handle = q->handle;
927 tcm->tcm_info = refcount_read(&q->refcnt);
928 if (nla_put_string(skb, TCA_KIND, q->ops->id))
929 goto nla_put_failure;
930 if (q->ops->ingress_block_get) {
931 block_index = q->ops->ingress_block_get(q);
933 nla_put_u32(skb, TCA_INGRESS_BLOCK, block_index))
934 goto nla_put_failure;
936 if (q->ops->egress_block_get) {
937 block_index = q->ops->egress_block_get(q);
939 nla_put_u32(skb, TCA_EGRESS_BLOCK, block_index))
940 goto nla_put_failure;
942 if (q->ops->dump && q->ops->dump(q, skb) < 0)
943 goto nla_put_failure;
944 if (nla_put_u8(skb, TCA_HW_OFFLOAD, !!(q->flags & TCQ_F_OFFLOADED)))
945 goto nla_put_failure;
946 qlen = qdisc_qlen_sum(q);
948 stab = rtnl_dereference(q->stab);
949 if (stab && qdisc_dump_stab(skb, stab) < 0)
950 goto nla_put_failure;
952 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
953 NULL, &d, TCA_PAD) < 0)
954 goto nla_put_failure;
956 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
957 goto nla_put_failure;
959 if (qdisc_is_percpu_stats(q)) {
960 cpu_bstats = q->cpu_bstats;
961 cpu_qstats = q->cpu_qstats;
964 if (gnet_stats_copy_basic(&d, cpu_bstats, &q->bstats, true) < 0 ||
965 gnet_stats_copy_rate_est(&d, &q->rate_est) < 0 ||
966 gnet_stats_copy_queue(&d, cpu_qstats, &q->qstats, qlen) < 0)
967 goto nla_put_failure;
969 if (gnet_stats_finish_copy(&d) < 0)
970 goto nla_put_failure;
972 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
981 static bool tc_qdisc_dump_ignore(struct Qdisc *q, bool dump_invisible)
983 if (q->flags & TCQ_F_BUILTIN)
985 if ((q->flags & TCQ_F_INVISIBLE) && !dump_invisible)
991 static int qdisc_notify(struct net *net, struct sk_buff *oskb,
992 struct nlmsghdr *n, u32 clid,
993 struct Qdisc *old, struct Qdisc *new)
996 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
998 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1002 if (old && !tc_qdisc_dump_ignore(old, false)) {
1003 if (tc_fill_qdisc(skb, old, clid, portid, n->nlmsg_seq,
1004 0, RTM_DELQDISC) < 0)
1007 if (new && !tc_qdisc_dump_ignore(new, false)) {
1008 if (tc_fill_qdisc(skb, new, clid, portid, n->nlmsg_seq,
1009 old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0)
1014 return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1015 n->nlmsg_flags & NLM_F_ECHO);
1022 static void notify_and_destroy(struct net *net, struct sk_buff *skb,
1023 struct nlmsghdr *n, u32 clid,
1024 struct Qdisc *old, struct Qdisc *new)
1027 qdisc_notify(net, skb, n, clid, old, new);
1033 static void qdisc_clear_nolock(struct Qdisc *sch)
1035 sch->flags &= ~TCQ_F_NOLOCK;
1036 if (!(sch->flags & TCQ_F_CPUSTATS))
1039 free_percpu(sch->cpu_bstats);
1040 free_percpu(sch->cpu_qstats);
1041 sch->cpu_bstats = NULL;
1042 sch->cpu_qstats = NULL;
1043 sch->flags &= ~TCQ_F_CPUSTATS;
1046 /* Graft qdisc "new" to class "classid" of qdisc "parent" or
1049 * When appropriate send a netlink notification using 'skb'
1052 * On success, destroy old qdisc.
1055 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
1056 struct sk_buff *skb, struct nlmsghdr *n, u32 classid,
1057 struct Qdisc *new, struct Qdisc *old,
1058 struct netlink_ext_ack *extack)
1060 struct Qdisc *q = old;
1061 struct net *net = dev_net(dev);
1063 if (parent == NULL) {
1064 unsigned int i, num_q, ingress;
1067 num_q = dev->num_tx_queues;
1068 if ((q && q->flags & TCQ_F_INGRESS) ||
1069 (new && new->flags & TCQ_F_INGRESS)) {
1072 if (!dev_ingress_queue(dev)) {
1073 NL_SET_ERR_MSG(extack, "Device does not have an ingress queue");
1078 if (dev->flags & IFF_UP)
1079 dev_deactivate(dev);
1081 qdisc_offload_graft_root(dev, new, old, extack);
1083 if (new && new->ops->attach && !ingress)
1086 for (i = 0; i < num_q; i++) {
1087 struct netdev_queue *dev_queue = dev_ingress_queue(dev);
1090 dev_queue = netdev_get_tx_queue(dev, i);
1092 old = dev_graft_qdisc(dev_queue, new);
1094 qdisc_refcount_inc(new);
1102 old = rtnl_dereference(dev->qdisc);
1103 if (new && !new->ops->attach)
1104 qdisc_refcount_inc(new);
1105 rcu_assign_pointer(dev->qdisc, new ? : &noop_qdisc);
1107 notify_and_destroy(net, skb, n, classid, old, new);
1109 if (new && new->ops->attach)
1110 new->ops->attach(new);
1112 notify_and_destroy(net, skb, n, classid, old, new);
1115 if (dev->flags & IFF_UP)
1118 const struct Qdisc_class_ops *cops = parent->ops->cl_ops;
1122 /* Only support running class lockless if parent is lockless */
1123 if (new && (new->flags & TCQ_F_NOLOCK) && !(parent->flags & TCQ_F_NOLOCK))
1124 qdisc_clear_nolock(new);
1126 if (!cops || !cops->graft)
1129 cl = cops->find(parent, classid);
1131 NL_SET_ERR_MSG(extack, "Specified class not found");
1135 if (new && new->ops == &noqueue_qdisc_ops) {
1136 NL_SET_ERR_MSG(extack, "Cannot assign noqueue to a class");
1140 err = cops->graft(parent, cl, new, &old, extack);
1143 notify_and_destroy(net, skb, n, classid, old, new);
1148 static int qdisc_block_indexes_set(struct Qdisc *sch, struct nlattr **tca,
1149 struct netlink_ext_ack *extack)
1153 if (tca[TCA_INGRESS_BLOCK]) {
1154 block_index = nla_get_u32(tca[TCA_INGRESS_BLOCK]);
1157 NL_SET_ERR_MSG(extack, "Ingress block index cannot be 0");
1160 if (!sch->ops->ingress_block_set) {
1161 NL_SET_ERR_MSG(extack, "Ingress block sharing is not supported");
1164 sch->ops->ingress_block_set(sch, block_index);
1166 if (tca[TCA_EGRESS_BLOCK]) {
1167 block_index = nla_get_u32(tca[TCA_EGRESS_BLOCK]);
1170 NL_SET_ERR_MSG(extack, "Egress block index cannot be 0");
1173 if (!sch->ops->egress_block_set) {
1174 NL_SET_ERR_MSG(extack, "Egress block sharing is not supported");
1177 sch->ops->egress_block_set(sch, block_index);
1183 Allocate and initialize new qdisc.
1185 Parameters are passed via opt.
1188 static struct Qdisc *qdisc_create(struct net_device *dev,
1189 struct netdev_queue *dev_queue,
1190 u32 parent, u32 handle,
1191 struct nlattr **tca, int *errp,
1192 struct netlink_ext_ack *extack)
1195 struct nlattr *kind = tca[TCA_KIND];
1197 struct Qdisc_ops *ops;
1198 struct qdisc_size_table *stab;
1200 ops = qdisc_lookup_ops(kind);
1201 #ifdef CONFIG_MODULES
1202 if (ops == NULL && kind != NULL) {
1203 char name[IFNAMSIZ];
1204 if (nla_strscpy(name, kind, IFNAMSIZ) >= 0) {
1205 /* We dropped the RTNL semaphore in order to
1206 * perform the module load. So, even if we
1207 * succeeded in loading the module we have to
1208 * tell the caller to replay the request. We
1209 * indicate this using -EAGAIN.
1210 * We replay the request because the device may
1211 * go away in the mean time.
1214 request_module("sch_%s", name);
1216 ops = qdisc_lookup_ops(kind);
1218 /* We will try again qdisc_lookup_ops,
1219 * so don't keep a reference.
1221 module_put(ops->owner);
1231 NL_SET_ERR_MSG(extack, "Specified qdisc kind is unknown");
1235 sch = qdisc_alloc(dev_queue, ops, extack);
1241 sch->parent = parent;
1243 if (handle == TC_H_INGRESS) {
1244 if (!(sch->flags & TCQ_F_INGRESS)) {
1245 NL_SET_ERR_MSG(extack,
1246 "Specified parent ID is reserved for ingress and clsact Qdiscs");
1250 handle = TC_H_MAKE(TC_H_INGRESS, 0);
1253 handle = qdisc_alloc_handle(dev);
1255 NL_SET_ERR_MSG(extack, "Maximum number of qdisc handles was exceeded");
1260 if (!netif_is_multiqueue(dev))
1261 sch->flags |= TCQ_F_ONETXQUEUE;
1264 sch->handle = handle;
1266 /* This exist to keep backward compatible with a userspace
1267 * loophole, what allowed userspace to get IFF_NO_QUEUE
1268 * facility on older kernels by setting tx_queue_len=0 (prior
1269 * to qdisc init), and then forgot to reinit tx_queue_len
1270 * before again attaching a qdisc.
1272 if ((dev->priv_flags & IFF_NO_QUEUE) && (dev->tx_queue_len == 0)) {
1273 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
1274 netdev_info(dev, "Caught tx_queue_len zero misconfig\n");
1277 err = qdisc_block_indexes_set(sch, tca, extack);
1282 err = ops->init(sch, tca[TCA_OPTIONS], extack);
1287 if (tca[TCA_STAB]) {
1288 stab = qdisc_get_stab(tca[TCA_STAB], extack);
1290 err = PTR_ERR(stab);
1293 rcu_assign_pointer(sch->stab, stab);
1295 if (tca[TCA_RATE]) {
1297 if (sch->flags & TCQ_F_MQROOT) {
1298 NL_SET_ERR_MSG(extack, "Cannot attach rate estimator to a multi-queue root qdisc");
1302 err = gen_new_estimator(&sch->bstats,
1309 NL_SET_ERR_MSG(extack, "Failed to generate new estimator");
1314 qdisc_hash_add(sch, false);
1315 trace_qdisc_create(ops, dev, parent);
1320 /* ops->init() failed, we call ->destroy() like qdisc_create_dflt() */
1324 netdev_put(dev, &sch->dev_tracker);
1327 module_put(ops->owner);
1334 * Any broken qdiscs that would require a ops->reset() here?
1335 * The qdisc was never in action so it shouldn't be necessary.
1337 qdisc_put_stab(rtnl_dereference(sch->stab));
1343 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca,
1344 struct netlink_ext_ack *extack)
1346 struct qdisc_size_table *ostab, *stab = NULL;
1349 if (tca[TCA_OPTIONS]) {
1350 if (!sch->ops->change) {
1351 NL_SET_ERR_MSG(extack, "Change operation not supported by specified qdisc");
1354 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) {
1355 NL_SET_ERR_MSG(extack, "Change of blocks is not supported");
1358 err = sch->ops->change(sch, tca[TCA_OPTIONS], extack);
1363 if (tca[TCA_STAB]) {
1364 stab = qdisc_get_stab(tca[TCA_STAB], extack);
1366 return PTR_ERR(stab);
1369 ostab = rtnl_dereference(sch->stab);
1370 rcu_assign_pointer(sch->stab, stab);
1371 qdisc_put_stab(ostab);
1373 if (tca[TCA_RATE]) {
1374 /* NB: ignores errors from replace_estimator
1375 because change can't be undone. */
1376 if (sch->flags & TCQ_F_MQROOT)
1378 gen_replace_estimator(&sch->bstats,
1389 struct check_loop_arg {
1390 struct qdisc_walker w;
1395 static int check_loop_fn(struct Qdisc *q, unsigned long cl,
1396 struct qdisc_walker *w);
1398 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth)
1400 struct check_loop_arg arg;
1402 if (q->ops->cl_ops == NULL)
1405 arg.w.stop = arg.w.skip = arg.w.count = 0;
1406 arg.w.fn = check_loop_fn;
1409 q->ops->cl_ops->walk(q, &arg.w);
1410 return arg.w.stop ? -ELOOP : 0;
1414 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w)
1417 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1418 struct check_loop_arg *arg = (struct check_loop_arg *)w;
1420 leaf = cops->leaf(q, cl);
1422 if (leaf == arg->p || arg->depth > 7)
1424 return check_loop(leaf, arg->p, arg->depth + 1);
1429 const struct nla_policy rtm_tca_policy[TCA_MAX + 1] = {
1430 [TCA_KIND] = { .type = NLA_STRING },
1431 [TCA_RATE] = { .type = NLA_BINARY,
1432 .len = sizeof(struct tc_estimator) },
1433 [TCA_STAB] = { .type = NLA_NESTED },
1434 [TCA_DUMP_INVISIBLE] = { .type = NLA_FLAG },
1435 [TCA_CHAIN] = { .type = NLA_U32 },
1436 [TCA_INGRESS_BLOCK] = { .type = NLA_U32 },
1437 [TCA_EGRESS_BLOCK] = { .type = NLA_U32 },
1444 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
1445 struct netlink_ext_ack *extack)
1447 struct net *net = sock_net(skb->sk);
1448 struct tcmsg *tcm = nlmsg_data(n);
1449 struct nlattr *tca[TCA_MAX + 1];
1450 struct net_device *dev;
1452 struct Qdisc *q = NULL;
1453 struct Qdisc *p = NULL;
1456 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
1457 rtm_tca_policy, extack);
1461 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1465 clid = tcm->tcm_parent;
1467 if (clid != TC_H_ROOT) {
1468 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) {
1469 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1471 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified classid");
1474 q = qdisc_leaf(p, clid);
1475 } else if (dev_ingress_queue(dev)) {
1476 q = dev_ingress_queue(dev)->qdisc_sleeping;
1479 q = rtnl_dereference(dev->qdisc);
1482 NL_SET_ERR_MSG(extack, "Cannot find specified qdisc on specified device");
1486 if (tcm->tcm_handle && q->handle != tcm->tcm_handle) {
1487 NL_SET_ERR_MSG(extack, "Invalid handle");
1491 q = qdisc_lookup(dev, tcm->tcm_handle);
1493 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified handle");
1498 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1499 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1503 if (n->nlmsg_type == RTM_DELQDISC) {
1505 NL_SET_ERR_MSG(extack, "Classid cannot be zero");
1508 if (q->handle == 0) {
1509 NL_SET_ERR_MSG(extack, "Cannot delete qdisc with handle of zero");
1512 err = qdisc_graft(dev, p, skb, n, clid, NULL, q, extack);
1516 qdisc_notify(net, skb, n, clid, NULL, q);
1522 * Create/change qdisc.
1525 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
1526 struct netlink_ext_ack *extack)
1528 struct net *net = sock_net(skb->sk);
1530 struct nlattr *tca[TCA_MAX + 1];
1531 struct net_device *dev;
1533 struct Qdisc *q, *p;
1537 /* Reinit, just in case something touches this. */
1538 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
1539 rtm_tca_policy, extack);
1543 tcm = nlmsg_data(n);
1544 clid = tcm->tcm_parent;
1547 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1553 if (clid != TC_H_ROOT) {
1554 if (clid != TC_H_INGRESS) {
1555 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1557 NL_SET_ERR_MSG(extack, "Failed to find specified qdisc");
1560 q = qdisc_leaf(p, clid);
1561 } else if (dev_ingress_queue_create(dev)) {
1562 q = dev_ingress_queue(dev)->qdisc_sleeping;
1565 q = rtnl_dereference(dev->qdisc);
1568 /* It may be default qdisc, ignore it */
1569 if (q && q->handle == 0)
1572 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) {
1573 if (tcm->tcm_handle) {
1574 if (q && !(n->nlmsg_flags & NLM_F_REPLACE)) {
1575 NL_SET_ERR_MSG(extack, "NLM_F_REPLACE needed to override");
1578 if (TC_H_MIN(tcm->tcm_handle)) {
1579 NL_SET_ERR_MSG(extack, "Invalid minor handle");
1582 q = qdisc_lookup(dev, tcm->tcm_handle);
1584 goto create_n_graft;
1585 if (n->nlmsg_flags & NLM_F_EXCL) {
1586 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot override");
1589 if (tca[TCA_KIND] &&
1590 nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1591 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1595 (p && check_loop(q, p, 0))) {
1596 NL_SET_ERR_MSG(extack, "Qdisc parent/child loop detected");
1599 qdisc_refcount_inc(q);
1603 goto create_n_graft;
1605 /* This magic test requires explanation.
1607 * We know, that some child q is already
1608 * attached to this parent and have choice:
1609 * either to change it or to create/graft new one.
1611 * 1. We are allowed to create/graft only
1612 * if CREATE and REPLACE flags are set.
1614 * 2. If EXCL is set, requestor wanted to say,
1615 * that qdisc tcm_handle is not expected
1616 * to exist, so that we choose create/graft too.
1618 * 3. The last case is when no flags are set.
1619 * Alas, it is sort of hole in API, we
1620 * cannot decide what to do unambiguously.
1621 * For now we select create/graft, if
1622 * user gave KIND, which does not match existing.
1624 if ((n->nlmsg_flags & NLM_F_CREATE) &&
1625 (n->nlmsg_flags & NLM_F_REPLACE) &&
1626 ((n->nlmsg_flags & NLM_F_EXCL) ||
1628 nla_strcmp(tca[TCA_KIND], q->ops->id))))
1629 goto create_n_graft;
1633 if (!tcm->tcm_handle) {
1634 NL_SET_ERR_MSG(extack, "Handle cannot be zero");
1637 q = qdisc_lookup(dev, tcm->tcm_handle);
1640 /* Change qdisc parameters */
1642 NL_SET_ERR_MSG(extack, "Specified qdisc not found");
1645 if (n->nlmsg_flags & NLM_F_EXCL) {
1646 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot modify");
1649 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1650 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1653 err = qdisc_change(q, tca, extack);
1655 qdisc_notify(net, skb, n, clid, NULL, q);
1659 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
1660 NL_SET_ERR_MSG(extack, "Qdisc not found. To create specify NLM_F_CREATE flag");
1663 if (clid == TC_H_INGRESS) {
1664 if (dev_ingress_queue(dev)) {
1665 q = qdisc_create(dev, dev_ingress_queue(dev),
1666 tcm->tcm_parent, tcm->tcm_parent,
1669 NL_SET_ERR_MSG(extack, "Cannot find ingress queue for specified device");
1673 struct netdev_queue *dev_queue;
1675 if (p && p->ops->cl_ops && p->ops->cl_ops->select_queue)
1676 dev_queue = p->ops->cl_ops->select_queue(p, tcm);
1678 dev_queue = p->dev_queue;
1680 dev_queue = netdev_get_tx_queue(dev, 0);
1682 q = qdisc_create(dev, dev_queue,
1683 tcm->tcm_parent, tcm->tcm_handle,
1693 err = qdisc_graft(dev, p, skb, n, clid, q, NULL, extack);
1703 static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
1704 struct netlink_callback *cb,
1705 int *q_idx_p, int s_q_idx, bool recur,
1706 bool dump_invisible)
1708 int ret = 0, q_idx = *q_idx_p;
1716 if (q_idx < s_q_idx) {
1719 if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
1720 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
1721 cb->nlh->nlmsg_seq, NLM_F_MULTI,
1727 /* If dumping singletons, there is no qdisc_dev(root) and the singleton
1728 * itself has already been dumped.
1730 * If we've already dumped the top-level (ingress) qdisc above and the global
1731 * qdisc hashtable, we don't want to hit it again
1733 if (!qdisc_dev(root) || !recur)
1736 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
1737 if (q_idx < s_q_idx) {
1741 if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
1742 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
1743 cb->nlh->nlmsg_seq, NLM_F_MULTI,
1757 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
1759 struct net *net = sock_net(skb->sk);
1762 struct net_device *dev;
1763 const struct nlmsghdr *nlh = cb->nlh;
1764 struct nlattr *tca[TCA_MAX + 1];
1767 s_idx = cb->args[0];
1768 s_q_idx = q_idx = cb->args[1];
1773 err = nlmsg_parse_deprecated(nlh, sizeof(struct tcmsg), tca, TCA_MAX,
1774 rtm_tca_policy, cb->extack);
1778 for_each_netdev(net, dev) {
1779 struct netdev_queue *dev_queue;
1787 if (tc_dump_qdisc_root(rtnl_dereference(dev->qdisc),
1788 skb, cb, &q_idx, s_q_idx,
1789 true, tca[TCA_DUMP_INVISIBLE]) < 0)
1792 dev_queue = dev_ingress_queue(dev);
1794 tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb,
1795 &q_idx, s_q_idx, false,
1796 tca[TCA_DUMP_INVISIBLE]) < 0)
1805 cb->args[1] = q_idx;
1812 /************************************************
1813 * Traffic classes manipulation. *
1814 ************************************************/
1816 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
1818 u32 portid, u32 seq, u16 flags, int event)
1821 struct nlmsghdr *nlh;
1822 unsigned char *b = skb_tail_pointer(skb);
1824 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops;
1827 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1829 goto out_nlmsg_trim;
1830 tcm = nlmsg_data(nlh);
1831 tcm->tcm_family = AF_UNSPEC;
1834 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1835 tcm->tcm_parent = q->handle;
1836 tcm->tcm_handle = q->handle;
1838 if (nla_put_string(skb, TCA_KIND, q->ops->id))
1839 goto nla_put_failure;
1840 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0)
1841 goto nla_put_failure;
1843 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
1844 NULL, &d, TCA_PAD) < 0)
1845 goto nla_put_failure;
1847 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)
1848 goto nla_put_failure;
1850 if (gnet_stats_finish_copy(&d) < 0)
1851 goto nla_put_failure;
1853 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1862 static int tclass_notify(struct net *net, struct sk_buff *oskb,
1863 struct nlmsghdr *n, struct Qdisc *q,
1864 unsigned long cl, int event)
1866 struct sk_buff *skb;
1867 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1869 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1873 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, event) < 0) {
1878 return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1879 n->nlmsg_flags & NLM_F_ECHO);
1882 static int tclass_del_notify(struct net *net,
1883 const struct Qdisc_class_ops *cops,
1884 struct sk_buff *oskb, struct nlmsghdr *n,
1885 struct Qdisc *q, unsigned long cl,
1886 struct netlink_ext_ack *extack)
1888 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1889 struct sk_buff *skb;
1895 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1899 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0,
1900 RTM_DELTCLASS) < 0) {
1905 err = cops->delete(q, cl, extack);
1911 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1912 n->nlmsg_flags & NLM_F_ECHO);
1916 #ifdef CONFIG_NET_CLS
1918 struct tcf_bind_args {
1919 struct tcf_walker w;
1925 static int tcf_node_bind(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
1927 struct tcf_bind_args *a = (void *)arg;
1929 if (n && tp->ops->bind_class) {
1930 struct Qdisc *q = tcf_block_q(tp->chain->block);
1933 tp->ops->bind_class(n, a->classid, a->cl, q, a->base);
1939 struct tc_bind_class_args {
1940 struct qdisc_walker w;
1941 unsigned long new_cl;
1946 static int tc_bind_class_walker(struct Qdisc *q, unsigned long cl,
1947 struct qdisc_walker *w)
1949 struct tc_bind_class_args *a = (struct tc_bind_class_args *)w;
1950 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1951 struct tcf_block *block;
1952 struct tcf_chain *chain;
1954 block = cops->tcf_block(q, cl, NULL);
1957 for (chain = tcf_get_next_chain(block, NULL);
1959 chain = tcf_get_next_chain(block, chain)) {
1960 struct tcf_proto *tp;
1962 for (tp = tcf_get_next_proto(chain, NULL);
1963 tp; tp = tcf_get_next_proto(chain, tp)) {
1964 struct tcf_bind_args arg = {};
1966 arg.w.fn = tcf_node_bind;
1967 arg.classid = a->clid;
1970 tp->ops->walk(tp, &arg.w, true);
1977 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid,
1978 unsigned long new_cl)
1980 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1981 struct tc_bind_class_args args = {};
1983 if (!cops->tcf_block)
1985 args.portid = portid;
1987 args.new_cl = new_cl;
1988 args.w.fn = tc_bind_class_walker;
1989 q->ops->cl_ops->walk(q, &args.w);
1994 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid,
1995 unsigned long new_cl)
2001 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n,
2002 struct netlink_ext_ack *extack)
2004 struct net *net = sock_net(skb->sk);
2005 struct tcmsg *tcm = nlmsg_data(n);
2006 struct nlattr *tca[TCA_MAX + 1];
2007 struct net_device *dev;
2008 struct Qdisc *q = NULL;
2009 const struct Qdisc_class_ops *cops;
2010 unsigned long cl = 0;
2011 unsigned long new_cl;
2017 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
2018 rtm_tca_policy, extack);
2022 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2027 parent == TC_H_UNSPEC - unspecified parent.
2028 parent == TC_H_ROOT - class is root, which has no parent.
2029 parent == X:0 - parent is root class.
2030 parent == X:Y - parent is a node in hierarchy.
2031 parent == 0:Y - parent is X:Y, where X:0 is qdisc.
2033 handle == 0:0 - generate handle from kernel pool.
2034 handle == 0:Y - class is X:Y, where X:0 is qdisc.
2035 handle == X:Y - clear.
2036 handle == X:0 - root class.
2039 /* Step 1. Determine qdisc handle X:0 */
2041 portid = tcm->tcm_parent;
2042 clid = tcm->tcm_handle;
2043 qid = TC_H_MAJ(clid);
2045 if (portid != TC_H_ROOT) {
2046 u32 qid1 = TC_H_MAJ(portid);
2049 /* If both majors are known, they must be identical. */
2054 } else if (qid == 0)
2055 qid = rtnl_dereference(dev->qdisc)->handle;
2057 /* Now qid is genuine qdisc handle consistent
2058 * both with parent and child.
2060 * TC_H_MAJ(portid) still may be unspecified, complete it now.
2063 portid = TC_H_MAKE(qid, portid);
2066 qid = rtnl_dereference(dev->qdisc)->handle;
2069 /* OK. Locate qdisc */
2070 q = qdisc_lookup(dev, qid);
2074 /* An check that it supports classes */
2075 cops = q->ops->cl_ops;
2079 /* Now try to get class */
2081 if (portid == TC_H_ROOT)
2084 clid = TC_H_MAKE(qid, clid);
2087 cl = cops->find(q, clid);
2091 if (n->nlmsg_type != RTM_NEWTCLASS ||
2092 !(n->nlmsg_flags & NLM_F_CREATE))
2095 switch (n->nlmsg_type) {
2098 if (n->nlmsg_flags & NLM_F_EXCL)
2102 err = tclass_del_notify(net, cops, skb, n, q, cl, extack);
2103 /* Unbind the class with flilters with 0 */
2104 tc_bind_tclass(q, portid, clid, 0);
2107 err = tclass_notify(net, skb, n, q, cl, RTM_NEWTCLASS);
2115 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) {
2116 NL_SET_ERR_MSG(extack, "Shared blocks are not supported for classes");
2123 err = cops->change(q, clid, portid, tca, &new_cl, extack);
2125 tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS);
2126 /* We just create a new class, need to do reverse binding. */
2128 tc_bind_tclass(q, portid, clid, new_cl);
2134 struct qdisc_dump_args {
2135 struct qdisc_walker w;
2136 struct sk_buff *skb;
2137 struct netlink_callback *cb;
2140 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl,
2141 struct qdisc_walker *arg)
2143 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg;
2145 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).portid,
2146 a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
2150 static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb,
2151 struct tcmsg *tcm, struct netlink_callback *cb,
2154 struct qdisc_dump_args arg;
2156 if (tc_qdisc_dump_ignore(q, false) ||
2157 *t_p < s_t || !q->ops->cl_ops ||
2159 TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
2164 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
2165 arg.w.fn = qdisc_class_dump;
2169 arg.w.skip = cb->args[1];
2171 q->ops->cl_ops->walk(q, &arg.w);
2172 cb->args[1] = arg.w.count;
2179 static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb,
2180 struct tcmsg *tcm, struct netlink_callback *cb,
2181 int *t_p, int s_t, bool recur)
2189 if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0)
2192 if (!qdisc_dev(root) || !recur)
2195 if (tcm->tcm_parent) {
2196 q = qdisc_match_from_root(root, TC_H_MAJ(tcm->tcm_parent));
2197 if (q && q != root &&
2198 tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
2202 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
2203 if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
2210 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb)
2212 struct tcmsg *tcm = nlmsg_data(cb->nlh);
2213 struct net *net = sock_net(skb->sk);
2214 struct netdev_queue *dev_queue;
2215 struct net_device *dev;
2218 if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2220 dev = dev_get_by_index(net, tcm->tcm_ifindex);
2227 if (tc_dump_tclass_root(rtnl_dereference(dev->qdisc),
2228 skb, tcm, cb, &t, s_t, true) < 0)
2231 dev_queue = dev_ingress_queue(dev);
2233 tc_dump_tclass_root(dev_queue->qdisc_sleeping, skb, tcm, cb,
2234 &t, s_t, false) < 0)
2244 #ifdef CONFIG_PROC_FS
2245 static int psched_show(struct seq_file *seq, void *v)
2247 seq_printf(seq, "%08x %08x %08x %08x\n",
2248 (u32)NSEC_PER_USEC, (u32)PSCHED_TICKS2NS(1),
2250 (u32)NSEC_PER_SEC / hrtimer_resolution);
2255 static int __net_init psched_net_init(struct net *net)
2257 struct proc_dir_entry *e;
2259 e = proc_create_single("psched", 0, net->proc_net, psched_show);
2266 static void __net_exit psched_net_exit(struct net *net)
2268 remove_proc_entry("psched", net->proc_net);
2271 static int __net_init psched_net_init(struct net *net)
2276 static void __net_exit psched_net_exit(struct net *net)
2281 static struct pernet_operations psched_net_ops = {
2282 .init = psched_net_init,
2283 .exit = psched_net_exit,
2286 static int __init pktsched_init(void)
2290 err = register_pernet_subsys(&psched_net_ops);
2292 pr_err("pktsched_init: "
2293 "cannot initialize per netns operations\n");
2297 register_qdisc(&pfifo_fast_ops);
2298 register_qdisc(&pfifo_qdisc_ops);
2299 register_qdisc(&bfifo_qdisc_ops);
2300 register_qdisc(&pfifo_head_drop_qdisc_ops);
2301 register_qdisc(&mq_qdisc_ops);
2302 register_qdisc(&noqueue_qdisc_ops);
2304 rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL, 0);
2305 rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL, 0);
2306 rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc,
2308 rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL, 0);
2309 rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL, 0);
2310 rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass,
2316 subsys_initcall(pktsched_init);