1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * net/sched/sch_api.c Packet scheduler API.
5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
9 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired.
10 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
11 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support
14 #include <linux/module.h>
15 #include <linux/types.h>
16 #include <linux/kernel.h>
17 #include <linux/string.h>
18 #include <linux/errno.h>
19 #include <linux/skbuff.h>
20 #include <linux/init.h>
21 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h>
23 #include <linux/kmod.h>
24 #include <linux/list.h>
25 #include <linux/hrtimer.h>
26 #include <linux/slab.h>
27 #include <linux/hashtable.h>
29 #include <net/net_namespace.h>
31 #include <net/netlink.h>
32 #include <net/pkt_sched.h>
33 #include <net/pkt_cls.h>
35 #include <trace/events/qdisc.h>
42 This file consists of two interrelated parts:
44 1. queueing disciplines manager frontend.
45 2. traffic classes manager frontend.
47 Generally, queueing discipline ("qdisc") is a black box,
48 which is able to enqueue packets and to dequeue them (when
49 device is ready to send something) in order and at times
50 determined by algorithm hidden in it.
52 qdisc's are divided to two categories:
53 - "queues", which have no internal structure visible from outside.
54 - "schedulers", which split all the packets to "traffic classes",
55 using "packet classifiers" (look at cls_api.c)
57 In turn, classes may have child qdiscs (as rule, queues)
58 attached to them etc. etc. etc.
60 The goal of the routines in this file is to translate
61 information supplied by user in the form of handles
62 to more intelligible for kernel form, to make some sanity
63 checks and part of work, which is common to all qdiscs
64 and to provide rtnetlink notifications.
66 All real intelligent work is done inside qdisc modules.
70 Every discipline has two major routines: enqueue and dequeue.
74 dequeue usually returns a skb to send. It is allowed to return NULL,
75 but it does not mean that queue is empty, it just means that
76 discipline does not want to send anything this time.
77 Queue is really empty if q->q.qlen == 0.
78 For complicated disciplines with multiple queues q->q is not
79 real packet queue, but however q->q.qlen must be valid.
83 enqueue returns 0, if packet was enqueued successfully.
84 If packet (this one or another one) was dropped, it returns
86 NET_XMIT_DROP - this packet dropped
87 Expected action: do not backoff, but wait until queue will clear.
88 NET_XMIT_CN - probably this packet enqueued, but another one dropped.
89 Expected action: backoff or ignore
95 like dequeue but without removing a packet from the queue
99 returns qdisc to initial state: purge all buffers, clear all
100 timers, counters (except for statistics) etc.
104 initializes newly created qdisc.
108 destroys resources allocated by init and during lifetime of qdisc.
112 changes qdisc parameters.
115 /* Protects list of registered TC modules. It is pure SMP lock. */
116 static DEFINE_RWLOCK(qdisc_mod_lock);
119 /************************************************
120 * Queueing disciplines manipulation. *
121 ************************************************/
124 /* The list of all installed queueing disciplines. */
126 static struct Qdisc_ops *qdisc_base;
128 /* Register/unregister queueing discipline */
130 int register_qdisc(struct Qdisc_ops *qops)
132 struct Qdisc_ops *q, **qp;
135 write_lock(&qdisc_mod_lock);
136 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
137 if (!strcmp(qops->id, q->id))
140 if (qops->enqueue == NULL)
141 qops->enqueue = noop_qdisc_ops.enqueue;
142 if (qops->peek == NULL) {
143 if (qops->dequeue == NULL)
144 qops->peek = noop_qdisc_ops.peek;
148 if (qops->dequeue == NULL)
149 qops->dequeue = noop_qdisc_ops.dequeue;
152 const struct Qdisc_class_ops *cops = qops->cl_ops;
154 if (!(cops->find && cops->walk && cops->leaf))
157 if (cops->tcf_block && !(cops->bind_tcf && cops->unbind_tcf))
165 write_unlock(&qdisc_mod_lock);
172 EXPORT_SYMBOL(register_qdisc);
174 void unregister_qdisc(struct Qdisc_ops *qops)
176 struct Qdisc_ops *q, **qp;
179 write_lock(&qdisc_mod_lock);
180 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
188 write_unlock(&qdisc_mod_lock);
190 WARN(err, "unregister qdisc(%s) failed\n", qops->id);
192 EXPORT_SYMBOL(unregister_qdisc);
194 /* Get default qdisc if not otherwise specified */
195 void qdisc_get_default(char *name, size_t len)
197 read_lock(&qdisc_mod_lock);
198 strscpy(name, default_qdisc_ops->id, len);
199 read_unlock(&qdisc_mod_lock);
202 static struct Qdisc_ops *qdisc_lookup_default(const char *name)
204 struct Qdisc_ops *q = NULL;
206 for (q = qdisc_base; q; q = q->next) {
207 if (!strcmp(name, q->id)) {
208 if (!try_module_get(q->owner))
217 /* Set new default qdisc to use */
218 int qdisc_set_default(const char *name)
220 const struct Qdisc_ops *ops;
222 if (!capable(CAP_NET_ADMIN))
225 write_lock(&qdisc_mod_lock);
226 ops = qdisc_lookup_default(name);
228 /* Not found, drop lock and try to load module */
229 write_unlock(&qdisc_mod_lock);
230 request_module("sch_%s", name);
231 write_lock(&qdisc_mod_lock);
233 ops = qdisc_lookup_default(name);
237 /* Set new default */
238 module_put(default_qdisc_ops->owner);
239 default_qdisc_ops = ops;
241 write_unlock(&qdisc_mod_lock);
243 return ops ? 0 : -ENOENT;
246 #ifdef CONFIG_NET_SCH_DEFAULT
247 /* Set default value from kernel config */
248 static int __init sch_default_qdisc(void)
250 return qdisc_set_default(CONFIG_DEFAULT_NET_SCH);
252 late_initcall(sch_default_qdisc);
255 /* We know handle. Find qdisc among all qdisc's attached to device
256 * (root qdisc, all its children, children of children etc.)
257 * Note: caller either uses rtnl or rcu_read_lock()
260 static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle)
264 if (!qdisc_dev(root))
265 return (root->handle == handle ? root : NULL);
267 if (!(root->flags & TCQ_F_BUILTIN) &&
268 root->handle == handle)
271 hash_for_each_possible_rcu(qdisc_dev(root)->qdisc_hash, q, hash, handle,
272 lockdep_rtnl_is_held()) {
273 if (q->handle == handle)
279 void qdisc_hash_add(struct Qdisc *q, bool invisible)
281 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
283 hash_add_rcu(qdisc_dev(q)->qdisc_hash, &q->hash, q->handle);
285 q->flags |= TCQ_F_INVISIBLE;
288 EXPORT_SYMBOL(qdisc_hash_add);
290 void qdisc_hash_del(struct Qdisc *q)
292 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
294 hash_del_rcu(&q->hash);
297 EXPORT_SYMBOL(qdisc_hash_del);
299 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
305 q = qdisc_match_from_root(rtnl_dereference(dev->qdisc), handle);
309 if (dev_ingress_queue(dev))
310 q = qdisc_match_from_root(
311 dev_ingress_queue(dev)->qdisc_sleeping,
317 struct Qdisc *qdisc_lookup_rcu(struct net_device *dev, u32 handle)
319 struct netdev_queue *nq;
324 q = qdisc_match_from_root(rcu_dereference(dev->qdisc), handle);
328 nq = dev_ingress_queue_rcu(dev);
330 q = qdisc_match_from_root(nq->qdisc_sleeping, handle);
335 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
338 const struct Qdisc_class_ops *cops = p->ops->cl_ops;
342 cl = cops->find(p, classid);
346 return cops->leaf(p, cl);
349 /* Find queueing discipline by name */
351 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind)
353 struct Qdisc_ops *q = NULL;
356 read_lock(&qdisc_mod_lock);
357 for (q = qdisc_base; q; q = q->next) {
358 if (nla_strcmp(kind, q->id) == 0) {
359 if (!try_module_get(q->owner))
364 read_unlock(&qdisc_mod_lock);
369 /* The linklayer setting were not transferred from iproute2, in older
370 * versions, and the rate tables lookup systems have been dropped in
371 * the kernel. To keep backward compatible with older iproute2 tc
372 * utils, we detect the linklayer setting by detecting if the rate
373 * table were modified.
375 * For linklayer ATM table entries, the rate table will be aligned to
376 * 48 bytes, thus some table entries will contain the same value. The
377 * mpu (min packet unit) is also encoded into the old rate table, thus
378 * starting from the mpu, we find low and high table entries for
379 * mapping this cell. If these entries contain the same value, when
380 * the rate tables have been modified for linklayer ATM.
382 * This is done by rounding mpu to the nearest 48 bytes cell/entry,
383 * and then roundup to the next cell, calc the table entry one below,
386 static __u8 __detect_linklayer(struct tc_ratespec *r, __u32 *rtab)
388 int low = roundup(r->mpu, 48);
389 int high = roundup(low+1, 48);
390 int cell_low = low >> r->cell_log;
391 int cell_high = (high >> r->cell_log) - 1;
393 /* rtab is too inaccurate at rates > 100Mbit/s */
394 if ((r->rate > (100000000/8)) || (rtab[0] == 0)) {
395 pr_debug("TC linklayer: Giving up ATM detection\n");
396 return TC_LINKLAYER_ETHERNET;
399 if ((cell_high > cell_low) && (cell_high < 256)
400 && (rtab[cell_low] == rtab[cell_high])) {
401 pr_debug("TC linklayer: Detected ATM, low(%d)=high(%d)=%u\n",
402 cell_low, cell_high, rtab[cell_high]);
403 return TC_LINKLAYER_ATM;
405 return TC_LINKLAYER_ETHERNET;
408 static struct qdisc_rate_table *qdisc_rtab_list;
410 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r,
412 struct netlink_ext_ack *extack)
414 struct qdisc_rate_table *rtab;
416 if (tab == NULL || r->rate == 0 ||
417 r->cell_log == 0 || r->cell_log >= 32 ||
418 nla_len(tab) != TC_RTAB_SIZE) {
419 NL_SET_ERR_MSG(extack, "Invalid rate table parameters for searching");
423 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) {
424 if (!memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) &&
425 !memcmp(&rtab->data, nla_data(tab), 1024)) {
431 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL);
435 memcpy(rtab->data, nla_data(tab), 1024);
436 if (r->linklayer == TC_LINKLAYER_UNAWARE)
437 r->linklayer = __detect_linklayer(r, rtab->data);
438 rtab->next = qdisc_rtab_list;
439 qdisc_rtab_list = rtab;
441 NL_SET_ERR_MSG(extack, "Failed to allocate new qdisc rate table");
445 EXPORT_SYMBOL(qdisc_get_rtab);
447 void qdisc_put_rtab(struct qdisc_rate_table *tab)
449 struct qdisc_rate_table *rtab, **rtabp;
451 if (!tab || --tab->refcnt)
454 for (rtabp = &qdisc_rtab_list;
455 (rtab = *rtabp) != NULL;
456 rtabp = &rtab->next) {
464 EXPORT_SYMBOL(qdisc_put_rtab);
466 static LIST_HEAD(qdisc_stab_list);
468 static const struct nla_policy stab_policy[TCA_STAB_MAX + 1] = {
469 [TCA_STAB_BASE] = { .len = sizeof(struct tc_sizespec) },
470 [TCA_STAB_DATA] = { .type = NLA_BINARY },
473 static struct qdisc_size_table *qdisc_get_stab(struct nlattr *opt,
474 struct netlink_ext_ack *extack)
476 struct nlattr *tb[TCA_STAB_MAX + 1];
477 struct qdisc_size_table *stab;
478 struct tc_sizespec *s;
479 unsigned int tsize = 0;
483 err = nla_parse_nested_deprecated(tb, TCA_STAB_MAX, opt, stab_policy,
487 if (!tb[TCA_STAB_BASE]) {
488 NL_SET_ERR_MSG(extack, "Size table base attribute is missing");
489 return ERR_PTR(-EINVAL);
492 s = nla_data(tb[TCA_STAB_BASE]);
495 if (!tb[TCA_STAB_DATA]) {
496 NL_SET_ERR_MSG(extack, "Size table data attribute is missing");
497 return ERR_PTR(-EINVAL);
499 tab = nla_data(tb[TCA_STAB_DATA]);
500 tsize = nla_len(tb[TCA_STAB_DATA]) / sizeof(u16);
503 if (tsize != s->tsize || (!tab && tsize > 0)) {
504 NL_SET_ERR_MSG(extack, "Invalid size of size table");
505 return ERR_PTR(-EINVAL);
508 list_for_each_entry(stab, &qdisc_stab_list, list) {
509 if (memcmp(&stab->szopts, s, sizeof(*s)))
512 memcmp(stab->data, tab, flex_array_size(stab, data, tsize)))
518 if (s->size_log > STAB_SIZE_LOG_MAX ||
519 s->cell_log > STAB_SIZE_LOG_MAX) {
520 NL_SET_ERR_MSG(extack, "Invalid logarithmic size of size table");
521 return ERR_PTR(-EINVAL);
524 stab = kmalloc(struct_size(stab, data, tsize), GFP_KERNEL);
526 return ERR_PTR(-ENOMEM);
531 memcpy(stab->data, tab, flex_array_size(stab, data, tsize));
533 list_add_tail(&stab->list, &qdisc_stab_list);
538 void qdisc_put_stab(struct qdisc_size_table *tab)
543 if (--tab->refcnt == 0) {
544 list_del(&tab->list);
548 EXPORT_SYMBOL(qdisc_put_stab);
550 static int qdisc_dump_stab(struct sk_buff *skb, struct qdisc_size_table *stab)
554 nest = nla_nest_start_noflag(skb, TCA_STAB);
556 goto nla_put_failure;
557 if (nla_put(skb, TCA_STAB_BASE, sizeof(stab->szopts), &stab->szopts))
558 goto nla_put_failure;
559 nla_nest_end(skb, nest);
567 void __qdisc_calculate_pkt_len(struct sk_buff *skb,
568 const struct qdisc_size_table *stab)
572 pkt_len = skb->len + stab->szopts.overhead;
573 if (unlikely(!stab->szopts.tsize))
576 slot = pkt_len + stab->szopts.cell_align;
577 if (unlikely(slot < 0))
580 slot >>= stab->szopts.cell_log;
581 if (likely(slot < stab->szopts.tsize))
582 pkt_len = stab->data[slot];
584 pkt_len = stab->data[stab->szopts.tsize - 1] *
585 (slot / stab->szopts.tsize) +
586 stab->data[slot % stab->szopts.tsize];
588 pkt_len <<= stab->szopts.size_log;
590 if (unlikely(pkt_len < 1))
592 qdisc_skb_cb(skb)->pkt_len = pkt_len;
594 EXPORT_SYMBOL(__qdisc_calculate_pkt_len);
596 void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc)
598 if (!(qdisc->flags & TCQ_F_WARN_NONWC)) {
599 pr_warn("%s: %s qdisc %X: is non-work-conserving?\n",
600 txt, qdisc->ops->id, qdisc->handle >> 16);
601 qdisc->flags |= TCQ_F_WARN_NONWC;
604 EXPORT_SYMBOL(qdisc_warn_nonwc);
606 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
608 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog,
612 __netif_schedule(qdisc_root(wd->qdisc));
615 return HRTIMER_NORESTART;
618 void qdisc_watchdog_init_clockid(struct qdisc_watchdog *wd, struct Qdisc *qdisc,
621 hrtimer_init(&wd->timer, clockid, HRTIMER_MODE_ABS_PINNED);
622 wd->timer.function = qdisc_watchdog;
625 EXPORT_SYMBOL(qdisc_watchdog_init_clockid);
627 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
629 qdisc_watchdog_init_clockid(wd, qdisc, CLOCK_MONOTONIC);
631 EXPORT_SYMBOL(qdisc_watchdog_init);
633 void qdisc_watchdog_schedule_range_ns(struct qdisc_watchdog *wd, u64 expires,
636 if (test_bit(__QDISC_STATE_DEACTIVATED,
637 &qdisc_root_sleeping(wd->qdisc)->state))
640 if (hrtimer_is_queued(&wd->timer)) {
641 /* If timer is already set in [expires, expires + delta_ns],
642 * do not reprogram it.
644 if (wd->last_expires - expires <= delta_ns)
648 wd->last_expires = expires;
649 hrtimer_start_range_ns(&wd->timer,
650 ns_to_ktime(expires),
652 HRTIMER_MODE_ABS_PINNED);
654 EXPORT_SYMBOL(qdisc_watchdog_schedule_range_ns);
656 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd)
658 hrtimer_cancel(&wd->timer);
660 EXPORT_SYMBOL(qdisc_watchdog_cancel);
662 static struct hlist_head *qdisc_class_hash_alloc(unsigned int n)
664 struct hlist_head *h;
667 h = kvmalloc_array(n, sizeof(struct hlist_head), GFP_KERNEL);
670 for (i = 0; i < n; i++)
671 INIT_HLIST_HEAD(&h[i]);
676 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash)
678 struct Qdisc_class_common *cl;
679 struct hlist_node *next;
680 struct hlist_head *nhash, *ohash;
681 unsigned int nsize, nmask, osize;
684 /* Rehash when load factor exceeds 0.75 */
685 if (clhash->hashelems * 4 <= clhash->hashsize * 3)
687 nsize = clhash->hashsize * 2;
689 nhash = qdisc_class_hash_alloc(nsize);
693 ohash = clhash->hash;
694 osize = clhash->hashsize;
697 for (i = 0; i < osize; i++) {
698 hlist_for_each_entry_safe(cl, next, &ohash[i], hnode) {
699 h = qdisc_class_hash(cl->classid, nmask);
700 hlist_add_head(&cl->hnode, &nhash[h]);
703 clhash->hash = nhash;
704 clhash->hashsize = nsize;
705 clhash->hashmask = nmask;
706 sch_tree_unlock(sch);
710 EXPORT_SYMBOL(qdisc_class_hash_grow);
712 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash)
714 unsigned int size = 4;
716 clhash->hash = qdisc_class_hash_alloc(size);
719 clhash->hashsize = size;
720 clhash->hashmask = size - 1;
721 clhash->hashelems = 0;
724 EXPORT_SYMBOL(qdisc_class_hash_init);
726 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash)
728 kvfree(clhash->hash);
730 EXPORT_SYMBOL(qdisc_class_hash_destroy);
732 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash,
733 struct Qdisc_class_common *cl)
737 INIT_HLIST_NODE(&cl->hnode);
738 h = qdisc_class_hash(cl->classid, clhash->hashmask);
739 hlist_add_head(&cl->hnode, &clhash->hash[h]);
742 EXPORT_SYMBOL(qdisc_class_hash_insert);
744 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash,
745 struct Qdisc_class_common *cl)
747 hlist_del(&cl->hnode);
750 EXPORT_SYMBOL(qdisc_class_hash_remove);
752 /* Allocate an unique handle from space managed by kernel
753 * Possible range is [8000-FFFF]:0000 (0x8000 values)
755 static u32 qdisc_alloc_handle(struct net_device *dev)
758 static u32 autohandle = TC_H_MAKE(0x80000000U, 0);
761 autohandle += TC_H_MAKE(0x10000U, 0);
762 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0))
763 autohandle = TC_H_MAKE(0x80000000U, 0);
764 if (!qdisc_lookup(dev, autohandle))
772 void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len)
774 bool qdisc_is_offloaded = sch->flags & TCQ_F_OFFLOADED;
775 const struct Qdisc_class_ops *cops;
781 if (n == 0 && len == 0)
783 drops = max_t(int, n, 0);
785 while ((parentid = sch->parent)) {
786 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS))
789 if (sch->flags & TCQ_F_NOPARENT)
791 /* Notify parent qdisc only if child qdisc becomes empty.
793 * If child was empty even before update then backlog
794 * counter is screwed and we skip notification because
795 * parent class is already passive.
797 * If the original child was offloaded then it is allowed
798 * to be seem as empty, so the parent is notified anyway.
800 notify = !sch->q.qlen && !WARN_ON_ONCE(!n &&
801 !qdisc_is_offloaded);
802 /* TODO: perform the search on a per txq basis */
803 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid));
805 WARN_ON_ONCE(parentid != TC_H_ROOT);
808 cops = sch->ops->cl_ops;
809 if (notify && cops->qlen_notify) {
810 cl = cops->find(sch, parentid);
811 cops->qlen_notify(sch, cl);
814 sch->qstats.backlog -= len;
815 __qdisc_qstats_drop(sch, drops);
819 EXPORT_SYMBOL(qdisc_tree_reduce_backlog);
821 int qdisc_offload_dump_helper(struct Qdisc *sch, enum tc_setup_type type,
824 struct net_device *dev = qdisc_dev(sch);
827 sch->flags &= ~TCQ_F_OFFLOADED;
828 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
831 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data);
832 if (err == -EOPNOTSUPP)
836 sch->flags |= TCQ_F_OFFLOADED;
840 EXPORT_SYMBOL(qdisc_offload_dump_helper);
842 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
843 struct Qdisc *new, struct Qdisc *old,
844 enum tc_setup_type type, void *type_data,
845 struct netlink_ext_ack *extack)
847 bool any_qdisc_is_offloaded;
850 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
853 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data);
855 /* Don't report error if the graft is part of destroy operation. */
856 if (!err || !new || new == &noop_qdisc)
859 /* Don't report error if the parent, the old child and the new
860 * one are not offloaded.
862 any_qdisc_is_offloaded = new->flags & TCQ_F_OFFLOADED;
863 any_qdisc_is_offloaded |= sch && sch->flags & TCQ_F_OFFLOADED;
864 any_qdisc_is_offloaded |= old && old->flags & TCQ_F_OFFLOADED;
866 if (any_qdisc_is_offloaded)
867 NL_SET_ERR_MSG(extack, "Offloading graft operation failed.");
869 EXPORT_SYMBOL(qdisc_offload_graft_helper);
871 void qdisc_offload_query_caps(struct net_device *dev,
872 enum tc_setup_type type,
873 void *caps, size_t caps_len)
875 const struct net_device_ops *ops = dev->netdev_ops;
876 struct tc_query_caps_base base = {
881 memset(caps, 0, caps_len);
883 if (ops->ndo_setup_tc)
884 ops->ndo_setup_tc(dev, TC_QUERY_CAPS, &base);
886 EXPORT_SYMBOL(qdisc_offload_query_caps);
888 static void qdisc_offload_graft_root(struct net_device *dev,
889 struct Qdisc *new, struct Qdisc *old,
890 struct netlink_ext_ack *extack)
892 struct tc_root_qopt_offload graft_offload = {
893 .command = TC_ROOT_GRAFT,
894 .handle = new ? new->handle : 0,
895 .ingress = (new && new->flags & TCQ_F_INGRESS) ||
896 (old && old->flags & TCQ_F_INGRESS),
899 qdisc_offload_graft_helper(dev, NULL, new, old,
900 TC_SETUP_ROOT_QDISC, &graft_offload, extack);
903 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
904 u32 portid, u32 seq, u16 flags, int event)
906 struct gnet_stats_basic_sync __percpu *cpu_bstats = NULL;
907 struct gnet_stats_queue __percpu *cpu_qstats = NULL;
909 struct nlmsghdr *nlh;
910 unsigned char *b = skb_tail_pointer(skb);
912 struct qdisc_size_table *stab;
917 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
920 tcm = nlmsg_data(nlh);
921 tcm->tcm_family = AF_UNSPEC;
924 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
925 tcm->tcm_parent = clid;
926 tcm->tcm_handle = q->handle;
927 tcm->tcm_info = refcount_read(&q->refcnt);
928 if (nla_put_string(skb, TCA_KIND, q->ops->id))
929 goto nla_put_failure;
930 if (q->ops->ingress_block_get) {
931 block_index = q->ops->ingress_block_get(q);
933 nla_put_u32(skb, TCA_INGRESS_BLOCK, block_index))
934 goto nla_put_failure;
936 if (q->ops->egress_block_get) {
937 block_index = q->ops->egress_block_get(q);
939 nla_put_u32(skb, TCA_EGRESS_BLOCK, block_index))
940 goto nla_put_failure;
942 if (q->ops->dump && q->ops->dump(q, skb) < 0)
943 goto nla_put_failure;
944 if (nla_put_u8(skb, TCA_HW_OFFLOAD, !!(q->flags & TCQ_F_OFFLOADED)))
945 goto nla_put_failure;
946 qlen = qdisc_qlen_sum(q);
948 stab = rtnl_dereference(q->stab);
949 if (stab && qdisc_dump_stab(skb, stab) < 0)
950 goto nla_put_failure;
952 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
953 NULL, &d, TCA_PAD) < 0)
954 goto nla_put_failure;
956 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
957 goto nla_put_failure;
959 if (qdisc_is_percpu_stats(q)) {
960 cpu_bstats = q->cpu_bstats;
961 cpu_qstats = q->cpu_qstats;
964 if (gnet_stats_copy_basic(&d, cpu_bstats, &q->bstats, true) < 0 ||
965 gnet_stats_copy_rate_est(&d, &q->rate_est) < 0 ||
966 gnet_stats_copy_queue(&d, cpu_qstats, &q->qstats, qlen) < 0)
967 goto nla_put_failure;
969 if (gnet_stats_finish_copy(&d) < 0)
970 goto nla_put_failure;
972 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
981 static bool tc_qdisc_dump_ignore(struct Qdisc *q, bool dump_invisible)
983 if (q->flags & TCQ_F_BUILTIN)
985 if ((q->flags & TCQ_F_INVISIBLE) && !dump_invisible)
991 static int qdisc_notify(struct net *net, struct sk_buff *oskb,
992 struct nlmsghdr *n, u32 clid,
993 struct Qdisc *old, struct Qdisc *new)
996 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
998 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1002 if (old && !tc_qdisc_dump_ignore(old, false)) {
1003 if (tc_fill_qdisc(skb, old, clid, portid, n->nlmsg_seq,
1004 0, RTM_DELQDISC) < 0)
1007 if (new && !tc_qdisc_dump_ignore(new, false)) {
1008 if (tc_fill_qdisc(skb, new, clid, portid, n->nlmsg_seq,
1009 old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0)
1014 return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1015 n->nlmsg_flags & NLM_F_ECHO);
1022 static void notify_and_destroy(struct net *net, struct sk_buff *skb,
1023 struct nlmsghdr *n, u32 clid,
1024 struct Qdisc *old, struct Qdisc *new)
1027 qdisc_notify(net, skb, n, clid, old, new);
1033 static void qdisc_clear_nolock(struct Qdisc *sch)
1035 sch->flags &= ~TCQ_F_NOLOCK;
1036 if (!(sch->flags & TCQ_F_CPUSTATS))
1039 free_percpu(sch->cpu_bstats);
1040 free_percpu(sch->cpu_qstats);
1041 sch->cpu_bstats = NULL;
1042 sch->cpu_qstats = NULL;
1043 sch->flags &= ~TCQ_F_CPUSTATS;
1046 /* Graft qdisc "new" to class "classid" of qdisc "parent" or
1049 * When appropriate send a netlink notification using 'skb'
1052 * On success, destroy old qdisc.
1055 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
1056 struct sk_buff *skb, struct nlmsghdr *n, u32 classid,
1057 struct Qdisc *new, struct Qdisc *old,
1058 struct netlink_ext_ack *extack)
1060 struct Qdisc *q = old;
1061 struct net *net = dev_net(dev);
1063 if (parent == NULL) {
1064 unsigned int i, num_q, ingress;
1067 num_q = dev->num_tx_queues;
1068 if ((q && q->flags & TCQ_F_INGRESS) ||
1069 (new && new->flags & TCQ_F_INGRESS)) {
1072 if (!dev_ingress_queue(dev)) {
1073 NL_SET_ERR_MSG(extack, "Device does not have an ingress queue");
1078 if (dev->flags & IFF_UP)
1079 dev_deactivate(dev);
1081 qdisc_offload_graft_root(dev, new, old, extack);
1083 if (new && new->ops->attach && !ingress)
1086 for (i = 0; i < num_q; i++) {
1087 struct netdev_queue *dev_queue = dev_ingress_queue(dev);
1090 dev_queue = netdev_get_tx_queue(dev, i);
1092 old = dev_graft_qdisc(dev_queue, new);
1094 qdisc_refcount_inc(new);
1102 old = rtnl_dereference(dev->qdisc);
1103 if (new && !new->ops->attach)
1104 qdisc_refcount_inc(new);
1105 rcu_assign_pointer(dev->qdisc, new ? : &noop_qdisc);
1107 notify_and_destroy(net, skb, n, classid, old, new);
1109 if (new && new->ops->attach)
1110 new->ops->attach(new);
1112 notify_and_destroy(net, skb, n, classid, old, new);
1115 if (dev->flags & IFF_UP)
1118 const struct Qdisc_class_ops *cops = parent->ops->cl_ops;
1122 /* Only support running class lockless if parent is lockless */
1123 if (new && (new->flags & TCQ_F_NOLOCK) && !(parent->flags & TCQ_F_NOLOCK))
1124 qdisc_clear_nolock(new);
1126 if (!cops || !cops->graft)
1129 cl = cops->find(parent, classid);
1131 NL_SET_ERR_MSG(extack, "Specified class not found");
1135 if (new && new->ops == &noqueue_qdisc_ops) {
1136 NL_SET_ERR_MSG(extack, "Cannot assign noqueue to a class");
1140 err = cops->graft(parent, cl, new, &old, extack);
1143 notify_and_destroy(net, skb, n, classid, old, new);
1148 static int qdisc_block_indexes_set(struct Qdisc *sch, struct nlattr **tca,
1149 struct netlink_ext_ack *extack)
1153 if (tca[TCA_INGRESS_BLOCK]) {
1154 block_index = nla_get_u32(tca[TCA_INGRESS_BLOCK]);
1157 NL_SET_ERR_MSG(extack, "Ingress block index cannot be 0");
1160 if (!sch->ops->ingress_block_set) {
1161 NL_SET_ERR_MSG(extack, "Ingress block sharing is not supported");
1164 sch->ops->ingress_block_set(sch, block_index);
1166 if (tca[TCA_EGRESS_BLOCK]) {
1167 block_index = nla_get_u32(tca[TCA_EGRESS_BLOCK]);
1170 NL_SET_ERR_MSG(extack, "Egress block index cannot be 0");
1173 if (!sch->ops->egress_block_set) {
1174 NL_SET_ERR_MSG(extack, "Egress block sharing is not supported");
1177 sch->ops->egress_block_set(sch, block_index);
1183 Allocate and initialize new qdisc.
1185 Parameters are passed via opt.
1188 static struct Qdisc *qdisc_create(struct net_device *dev,
1189 struct netdev_queue *dev_queue,
1190 u32 parent, u32 handle,
1191 struct nlattr **tca, int *errp,
1192 struct netlink_ext_ack *extack)
1195 struct nlattr *kind = tca[TCA_KIND];
1197 struct Qdisc_ops *ops;
1198 struct qdisc_size_table *stab;
1200 ops = qdisc_lookup_ops(kind);
1201 #ifdef CONFIG_MODULES
1202 if (ops == NULL && kind != NULL) {
1203 char name[IFNAMSIZ];
1204 if (nla_strscpy(name, kind, IFNAMSIZ) >= 0) {
1205 /* We dropped the RTNL semaphore in order to
1206 * perform the module load. So, even if we
1207 * succeeded in loading the module we have to
1208 * tell the caller to replay the request. We
1209 * indicate this using -EAGAIN.
1210 * We replay the request because the device may
1211 * go away in the mean time.
1214 request_module("sch_%s", name);
1216 ops = qdisc_lookup_ops(kind);
1218 /* We will try again qdisc_lookup_ops,
1219 * so don't keep a reference.
1221 module_put(ops->owner);
1231 NL_SET_ERR_MSG(extack, "Specified qdisc kind is unknown");
1235 sch = qdisc_alloc(dev_queue, ops, extack);
1241 sch->parent = parent;
1243 if (handle == TC_H_INGRESS) {
1244 if (!(sch->flags & TCQ_F_INGRESS)) {
1245 NL_SET_ERR_MSG(extack,
1246 "Specified parent ID is reserved for ingress and clsact Qdiscs");
1250 handle = TC_H_MAKE(TC_H_INGRESS, 0);
1253 handle = qdisc_alloc_handle(dev);
1255 NL_SET_ERR_MSG(extack, "Maximum number of qdisc handles was exceeded");
1260 if (!netif_is_multiqueue(dev))
1261 sch->flags |= TCQ_F_ONETXQUEUE;
1264 sch->handle = handle;
1266 /* This exist to keep backward compatible with a userspace
1267 * loophole, what allowed userspace to get IFF_NO_QUEUE
1268 * facility on older kernels by setting tx_queue_len=0 (prior
1269 * to qdisc init), and then forgot to reinit tx_queue_len
1270 * before again attaching a qdisc.
1272 if ((dev->priv_flags & IFF_NO_QUEUE) && (dev->tx_queue_len == 0)) {
1273 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
1274 netdev_info(dev, "Caught tx_queue_len zero misconfig\n");
1277 err = qdisc_block_indexes_set(sch, tca, extack);
1282 err = ops->init(sch, tca[TCA_OPTIONS], extack);
1287 if (tca[TCA_STAB]) {
1288 stab = qdisc_get_stab(tca[TCA_STAB], extack);
1290 err = PTR_ERR(stab);
1293 rcu_assign_pointer(sch->stab, stab);
1295 if (tca[TCA_RATE]) {
1297 if (sch->flags & TCQ_F_MQROOT) {
1298 NL_SET_ERR_MSG(extack, "Cannot attach rate estimator to a multi-queue root qdisc");
1302 err = gen_new_estimator(&sch->bstats,
1309 NL_SET_ERR_MSG(extack, "Failed to generate new estimator");
1314 qdisc_hash_add(sch, false);
1315 trace_qdisc_create(ops, dev, parent);
1320 /* ops->init() failed, we call ->destroy() like qdisc_create_dflt() */
1324 netdev_put(dev, &sch->dev_tracker);
1327 module_put(ops->owner);
1334 * Any broken qdiscs that would require a ops->reset() here?
1335 * The qdisc was never in action so it shouldn't be necessary.
1337 qdisc_put_stab(rtnl_dereference(sch->stab));
1343 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca,
1344 struct netlink_ext_ack *extack)
1346 struct qdisc_size_table *ostab, *stab = NULL;
1349 if (tca[TCA_OPTIONS]) {
1350 if (!sch->ops->change) {
1351 NL_SET_ERR_MSG(extack, "Change operation not supported by specified qdisc");
1354 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) {
1355 NL_SET_ERR_MSG(extack, "Change of blocks is not supported");
1358 err = sch->ops->change(sch, tca[TCA_OPTIONS], extack);
1363 if (tca[TCA_STAB]) {
1364 stab = qdisc_get_stab(tca[TCA_STAB], extack);
1366 return PTR_ERR(stab);
1369 ostab = rtnl_dereference(sch->stab);
1370 rcu_assign_pointer(sch->stab, stab);
1371 qdisc_put_stab(ostab);
1373 if (tca[TCA_RATE]) {
1374 /* NB: ignores errors from replace_estimator
1375 because change can't be undone. */
1376 if (sch->flags & TCQ_F_MQROOT)
1378 gen_replace_estimator(&sch->bstats,
1389 struct check_loop_arg {
1390 struct qdisc_walker w;
1395 static int check_loop_fn(struct Qdisc *q, unsigned long cl,
1396 struct qdisc_walker *w);
1398 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth)
1400 struct check_loop_arg arg;
1402 if (q->ops->cl_ops == NULL)
1405 arg.w.stop = arg.w.skip = arg.w.count = 0;
1406 arg.w.fn = check_loop_fn;
1409 q->ops->cl_ops->walk(q, &arg.w);
1410 return arg.w.stop ? -ELOOP : 0;
1414 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w)
1417 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1418 struct check_loop_arg *arg = (struct check_loop_arg *)w;
1420 leaf = cops->leaf(q, cl);
1422 if (leaf == arg->p || arg->depth > 7)
1424 return check_loop(leaf, arg->p, arg->depth + 1);
1429 const struct nla_policy rtm_tca_policy[TCA_MAX + 1] = {
1430 [TCA_KIND] = { .type = NLA_STRING },
1431 [TCA_RATE] = { .type = NLA_BINARY,
1432 .len = sizeof(struct tc_estimator) },
1433 [TCA_STAB] = { .type = NLA_NESTED },
1434 [TCA_DUMP_INVISIBLE] = { .type = NLA_FLAG },
1435 [TCA_CHAIN] = { .type = NLA_U32 },
1436 [TCA_INGRESS_BLOCK] = { .type = NLA_U32 },
1437 [TCA_EGRESS_BLOCK] = { .type = NLA_U32 },
1444 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
1445 struct netlink_ext_ack *extack)
1447 struct net *net = sock_net(skb->sk);
1448 struct tcmsg *tcm = nlmsg_data(n);
1449 struct nlattr *tca[TCA_MAX + 1];
1450 struct net_device *dev;
1452 struct Qdisc *q = NULL;
1453 struct Qdisc *p = NULL;
1456 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
1457 rtm_tca_policy, extack);
1461 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1465 clid = tcm->tcm_parent;
1467 if (clid != TC_H_ROOT) {
1468 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) {
1469 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1471 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified classid");
1474 q = qdisc_leaf(p, clid);
1475 } else if (dev_ingress_queue(dev)) {
1476 q = dev_ingress_queue(dev)->qdisc_sleeping;
1479 q = rtnl_dereference(dev->qdisc);
1482 NL_SET_ERR_MSG(extack, "Cannot find specified qdisc on specified device");
1486 if (tcm->tcm_handle && q->handle != tcm->tcm_handle) {
1487 NL_SET_ERR_MSG(extack, "Invalid handle");
1491 q = qdisc_lookup(dev, tcm->tcm_handle);
1493 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified handle");
1498 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1499 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1503 if (n->nlmsg_type == RTM_DELQDISC) {
1505 NL_SET_ERR_MSG(extack, "Classid cannot be zero");
1508 if (q->handle == 0) {
1509 NL_SET_ERR_MSG(extack, "Cannot delete qdisc with handle of zero");
1512 err = qdisc_graft(dev, p, skb, n, clid, NULL, q, extack);
1516 qdisc_notify(net, skb, n, clid, NULL, q);
1522 * Create/change qdisc.
1525 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
1526 struct netlink_ext_ack *extack)
1528 struct net *net = sock_net(skb->sk);
1530 struct nlattr *tca[TCA_MAX + 1];
1531 struct net_device *dev;
1533 struct Qdisc *q, *p;
1537 /* Reinit, just in case something touches this. */
1538 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
1539 rtm_tca_policy, extack);
1543 tcm = nlmsg_data(n);
1544 clid = tcm->tcm_parent;
1547 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1553 if (clid != TC_H_ROOT) {
1554 if (clid != TC_H_INGRESS) {
1555 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1557 NL_SET_ERR_MSG(extack, "Failed to find specified qdisc");
1560 q = qdisc_leaf(p, clid);
1561 } else if (dev_ingress_queue_create(dev)) {
1562 q = dev_ingress_queue(dev)->qdisc_sleeping;
1565 q = rtnl_dereference(dev->qdisc);
1568 /* It may be default qdisc, ignore it */
1569 if (q && q->handle == 0)
1572 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) {
1573 if (tcm->tcm_handle) {
1574 if (q && !(n->nlmsg_flags & NLM_F_REPLACE)) {
1575 NL_SET_ERR_MSG(extack, "NLM_F_REPLACE needed to override");
1578 if (TC_H_MIN(tcm->tcm_handle)) {
1579 NL_SET_ERR_MSG(extack, "Invalid minor handle");
1582 q = qdisc_lookup(dev, tcm->tcm_handle);
1584 goto create_n_graft;
1585 if (n->nlmsg_flags & NLM_F_EXCL) {
1586 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot override");
1589 if (tca[TCA_KIND] &&
1590 nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1591 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1594 if (q->flags & TCQ_F_INGRESS) {
1595 NL_SET_ERR_MSG(extack,
1596 "Cannot regraft ingress or clsact Qdiscs");
1600 (p && check_loop(q, p, 0))) {
1601 NL_SET_ERR_MSG(extack, "Qdisc parent/child loop detected");
1604 if (clid == TC_H_INGRESS) {
1605 NL_SET_ERR_MSG(extack, "Ingress cannot graft directly");
1608 qdisc_refcount_inc(q);
1612 goto create_n_graft;
1614 /* This magic test requires explanation.
1616 * We know, that some child q is already
1617 * attached to this parent and have choice:
1618 * either to change it or to create/graft new one.
1620 * 1. We are allowed to create/graft only
1621 * if CREATE and REPLACE flags are set.
1623 * 2. If EXCL is set, requestor wanted to say,
1624 * that qdisc tcm_handle is not expected
1625 * to exist, so that we choose create/graft too.
1627 * 3. The last case is when no flags are set.
1628 * Alas, it is sort of hole in API, we
1629 * cannot decide what to do unambiguously.
1630 * For now we select create/graft, if
1631 * user gave KIND, which does not match existing.
1633 if ((n->nlmsg_flags & NLM_F_CREATE) &&
1634 (n->nlmsg_flags & NLM_F_REPLACE) &&
1635 ((n->nlmsg_flags & NLM_F_EXCL) ||
1637 nla_strcmp(tca[TCA_KIND], q->ops->id))))
1638 goto create_n_graft;
1642 if (!tcm->tcm_handle) {
1643 NL_SET_ERR_MSG(extack, "Handle cannot be zero");
1646 q = qdisc_lookup(dev, tcm->tcm_handle);
1649 /* Change qdisc parameters */
1651 NL_SET_ERR_MSG(extack, "Specified qdisc not found");
1654 if (n->nlmsg_flags & NLM_F_EXCL) {
1655 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot modify");
1658 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1659 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1662 err = qdisc_change(q, tca, extack);
1664 qdisc_notify(net, skb, n, clid, NULL, q);
1668 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
1669 NL_SET_ERR_MSG(extack, "Qdisc not found. To create specify NLM_F_CREATE flag");
1672 if (clid == TC_H_INGRESS) {
1673 if (dev_ingress_queue(dev)) {
1674 q = qdisc_create(dev, dev_ingress_queue(dev),
1675 tcm->tcm_parent, tcm->tcm_parent,
1678 NL_SET_ERR_MSG(extack, "Cannot find ingress queue for specified device");
1682 struct netdev_queue *dev_queue;
1684 if (p && p->ops->cl_ops && p->ops->cl_ops->select_queue)
1685 dev_queue = p->ops->cl_ops->select_queue(p, tcm);
1687 dev_queue = p->dev_queue;
1689 dev_queue = netdev_get_tx_queue(dev, 0);
1691 q = qdisc_create(dev, dev_queue,
1692 tcm->tcm_parent, tcm->tcm_handle,
1702 err = qdisc_graft(dev, p, skb, n, clid, q, NULL, extack);
1712 static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
1713 struct netlink_callback *cb,
1714 int *q_idx_p, int s_q_idx, bool recur,
1715 bool dump_invisible)
1717 int ret = 0, q_idx = *q_idx_p;
1725 if (q_idx < s_q_idx) {
1728 if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
1729 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
1730 cb->nlh->nlmsg_seq, NLM_F_MULTI,
1736 /* If dumping singletons, there is no qdisc_dev(root) and the singleton
1737 * itself has already been dumped.
1739 * If we've already dumped the top-level (ingress) qdisc above and the global
1740 * qdisc hashtable, we don't want to hit it again
1742 if (!qdisc_dev(root) || !recur)
1745 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
1746 if (q_idx < s_q_idx) {
1750 if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
1751 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
1752 cb->nlh->nlmsg_seq, NLM_F_MULTI,
1766 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
1768 struct net *net = sock_net(skb->sk);
1771 struct net_device *dev;
1772 const struct nlmsghdr *nlh = cb->nlh;
1773 struct nlattr *tca[TCA_MAX + 1];
1776 s_idx = cb->args[0];
1777 s_q_idx = q_idx = cb->args[1];
1782 err = nlmsg_parse_deprecated(nlh, sizeof(struct tcmsg), tca, TCA_MAX,
1783 rtm_tca_policy, cb->extack);
1787 for_each_netdev(net, dev) {
1788 struct netdev_queue *dev_queue;
1796 if (tc_dump_qdisc_root(rtnl_dereference(dev->qdisc),
1797 skb, cb, &q_idx, s_q_idx,
1798 true, tca[TCA_DUMP_INVISIBLE]) < 0)
1801 dev_queue = dev_ingress_queue(dev);
1803 tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb,
1804 &q_idx, s_q_idx, false,
1805 tca[TCA_DUMP_INVISIBLE]) < 0)
1814 cb->args[1] = q_idx;
1821 /************************************************
1822 * Traffic classes manipulation. *
1823 ************************************************/
1825 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
1827 u32 portid, u32 seq, u16 flags, int event)
1830 struct nlmsghdr *nlh;
1831 unsigned char *b = skb_tail_pointer(skb);
1833 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops;
1836 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1838 goto out_nlmsg_trim;
1839 tcm = nlmsg_data(nlh);
1840 tcm->tcm_family = AF_UNSPEC;
1843 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1844 tcm->tcm_parent = q->handle;
1845 tcm->tcm_handle = q->handle;
1847 if (nla_put_string(skb, TCA_KIND, q->ops->id))
1848 goto nla_put_failure;
1849 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0)
1850 goto nla_put_failure;
1852 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
1853 NULL, &d, TCA_PAD) < 0)
1854 goto nla_put_failure;
1856 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)
1857 goto nla_put_failure;
1859 if (gnet_stats_finish_copy(&d) < 0)
1860 goto nla_put_failure;
1862 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1871 static int tclass_notify(struct net *net, struct sk_buff *oskb,
1872 struct nlmsghdr *n, struct Qdisc *q,
1873 unsigned long cl, int event)
1875 struct sk_buff *skb;
1876 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1878 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1882 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, event) < 0) {
1887 return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1888 n->nlmsg_flags & NLM_F_ECHO);
1891 static int tclass_del_notify(struct net *net,
1892 const struct Qdisc_class_ops *cops,
1893 struct sk_buff *oskb, struct nlmsghdr *n,
1894 struct Qdisc *q, unsigned long cl,
1895 struct netlink_ext_ack *extack)
1897 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1898 struct sk_buff *skb;
1904 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1908 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0,
1909 RTM_DELTCLASS) < 0) {
1914 err = cops->delete(q, cl, extack);
1920 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1921 n->nlmsg_flags & NLM_F_ECHO);
1925 #ifdef CONFIG_NET_CLS
1927 struct tcf_bind_args {
1928 struct tcf_walker w;
1934 static int tcf_node_bind(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
1936 struct tcf_bind_args *a = (void *)arg;
1938 if (n && tp->ops->bind_class) {
1939 struct Qdisc *q = tcf_block_q(tp->chain->block);
1942 tp->ops->bind_class(n, a->classid, a->cl, q, a->base);
1948 struct tc_bind_class_args {
1949 struct qdisc_walker w;
1950 unsigned long new_cl;
1955 static int tc_bind_class_walker(struct Qdisc *q, unsigned long cl,
1956 struct qdisc_walker *w)
1958 struct tc_bind_class_args *a = (struct tc_bind_class_args *)w;
1959 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1960 struct tcf_block *block;
1961 struct tcf_chain *chain;
1963 block = cops->tcf_block(q, cl, NULL);
1966 for (chain = tcf_get_next_chain(block, NULL);
1968 chain = tcf_get_next_chain(block, chain)) {
1969 struct tcf_proto *tp;
1971 for (tp = tcf_get_next_proto(chain, NULL);
1972 tp; tp = tcf_get_next_proto(chain, tp)) {
1973 struct tcf_bind_args arg = {};
1975 arg.w.fn = tcf_node_bind;
1976 arg.classid = a->clid;
1979 tp->ops->walk(tp, &arg.w, true);
1986 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid,
1987 unsigned long new_cl)
1989 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1990 struct tc_bind_class_args args = {};
1992 if (!cops->tcf_block)
1994 args.portid = portid;
1996 args.new_cl = new_cl;
1997 args.w.fn = tc_bind_class_walker;
1998 q->ops->cl_ops->walk(q, &args.w);
2003 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid,
2004 unsigned long new_cl)
2010 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n,
2011 struct netlink_ext_ack *extack)
2013 struct net *net = sock_net(skb->sk);
2014 struct tcmsg *tcm = nlmsg_data(n);
2015 struct nlattr *tca[TCA_MAX + 1];
2016 struct net_device *dev;
2017 struct Qdisc *q = NULL;
2018 const struct Qdisc_class_ops *cops;
2019 unsigned long cl = 0;
2020 unsigned long new_cl;
2026 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
2027 rtm_tca_policy, extack);
2031 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2036 parent == TC_H_UNSPEC - unspecified parent.
2037 parent == TC_H_ROOT - class is root, which has no parent.
2038 parent == X:0 - parent is root class.
2039 parent == X:Y - parent is a node in hierarchy.
2040 parent == 0:Y - parent is X:Y, where X:0 is qdisc.
2042 handle == 0:0 - generate handle from kernel pool.
2043 handle == 0:Y - class is X:Y, where X:0 is qdisc.
2044 handle == X:Y - clear.
2045 handle == X:0 - root class.
2048 /* Step 1. Determine qdisc handle X:0 */
2050 portid = tcm->tcm_parent;
2051 clid = tcm->tcm_handle;
2052 qid = TC_H_MAJ(clid);
2054 if (portid != TC_H_ROOT) {
2055 u32 qid1 = TC_H_MAJ(portid);
2058 /* If both majors are known, they must be identical. */
2063 } else if (qid == 0)
2064 qid = rtnl_dereference(dev->qdisc)->handle;
2066 /* Now qid is genuine qdisc handle consistent
2067 * both with parent and child.
2069 * TC_H_MAJ(portid) still may be unspecified, complete it now.
2072 portid = TC_H_MAKE(qid, portid);
2075 qid = rtnl_dereference(dev->qdisc)->handle;
2078 /* OK. Locate qdisc */
2079 q = qdisc_lookup(dev, qid);
2083 /* An check that it supports classes */
2084 cops = q->ops->cl_ops;
2088 /* Now try to get class */
2090 if (portid == TC_H_ROOT)
2093 clid = TC_H_MAKE(qid, clid);
2096 cl = cops->find(q, clid);
2100 if (n->nlmsg_type != RTM_NEWTCLASS ||
2101 !(n->nlmsg_flags & NLM_F_CREATE))
2104 switch (n->nlmsg_type) {
2107 if (n->nlmsg_flags & NLM_F_EXCL)
2111 err = tclass_del_notify(net, cops, skb, n, q, cl, extack);
2112 /* Unbind the class with flilters with 0 */
2113 tc_bind_tclass(q, portid, clid, 0);
2116 err = tclass_notify(net, skb, n, q, cl, RTM_NEWTCLASS);
2124 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) {
2125 NL_SET_ERR_MSG(extack, "Shared blocks are not supported for classes");
2132 err = cops->change(q, clid, portid, tca, &new_cl, extack);
2134 tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS);
2135 /* We just create a new class, need to do reverse binding. */
2137 tc_bind_tclass(q, portid, clid, new_cl);
2143 struct qdisc_dump_args {
2144 struct qdisc_walker w;
2145 struct sk_buff *skb;
2146 struct netlink_callback *cb;
2149 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl,
2150 struct qdisc_walker *arg)
2152 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg;
2154 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).portid,
2155 a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
2159 static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb,
2160 struct tcmsg *tcm, struct netlink_callback *cb,
2163 struct qdisc_dump_args arg;
2165 if (tc_qdisc_dump_ignore(q, false) ||
2166 *t_p < s_t || !q->ops->cl_ops ||
2168 TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
2173 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
2174 arg.w.fn = qdisc_class_dump;
2178 arg.w.skip = cb->args[1];
2180 q->ops->cl_ops->walk(q, &arg.w);
2181 cb->args[1] = arg.w.count;
2188 static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb,
2189 struct tcmsg *tcm, struct netlink_callback *cb,
2190 int *t_p, int s_t, bool recur)
2198 if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0)
2201 if (!qdisc_dev(root) || !recur)
2204 if (tcm->tcm_parent) {
2205 q = qdisc_match_from_root(root, TC_H_MAJ(tcm->tcm_parent));
2206 if (q && q != root &&
2207 tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
2211 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
2212 if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
2219 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb)
2221 struct tcmsg *tcm = nlmsg_data(cb->nlh);
2222 struct net *net = sock_net(skb->sk);
2223 struct netdev_queue *dev_queue;
2224 struct net_device *dev;
2227 if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2229 dev = dev_get_by_index(net, tcm->tcm_ifindex);
2236 if (tc_dump_tclass_root(rtnl_dereference(dev->qdisc),
2237 skb, tcm, cb, &t, s_t, true) < 0)
2240 dev_queue = dev_ingress_queue(dev);
2242 tc_dump_tclass_root(dev_queue->qdisc_sleeping, skb, tcm, cb,
2243 &t, s_t, false) < 0)
2253 #ifdef CONFIG_PROC_FS
2254 static int psched_show(struct seq_file *seq, void *v)
2256 seq_printf(seq, "%08x %08x %08x %08x\n",
2257 (u32)NSEC_PER_USEC, (u32)PSCHED_TICKS2NS(1),
2259 (u32)NSEC_PER_SEC / hrtimer_resolution);
2264 static int __net_init psched_net_init(struct net *net)
2266 struct proc_dir_entry *e;
2268 e = proc_create_single("psched", 0, net->proc_net, psched_show);
2275 static void __net_exit psched_net_exit(struct net *net)
2277 remove_proc_entry("psched", net->proc_net);
2280 static int __net_init psched_net_init(struct net *net)
2285 static void __net_exit psched_net_exit(struct net *net)
2290 static struct pernet_operations psched_net_ops = {
2291 .init = psched_net_init,
2292 .exit = psched_net_exit,
2295 static int __init pktsched_init(void)
2299 err = register_pernet_subsys(&psched_net_ops);
2301 pr_err("pktsched_init: "
2302 "cannot initialize per netns operations\n");
2306 register_qdisc(&pfifo_fast_ops);
2307 register_qdisc(&pfifo_qdisc_ops);
2308 register_qdisc(&bfifo_qdisc_ops);
2309 register_qdisc(&pfifo_head_drop_qdisc_ops);
2310 register_qdisc(&mq_qdisc_ops);
2311 register_qdisc(&noqueue_qdisc_ops);
2313 rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL, 0);
2314 rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL, 0);
2315 rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc,
2317 rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL, 0);
2318 rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL, 0);
2319 rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass,
2325 subsys_initcall(pktsched_init);