net: sched: remove qdisc->empty for lockless qdisc

[platform/kernel/linux-rpi.git] / include / net / sch_generic.h

diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index f7a6e14..c99ffe9 100644 (file)
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -36,8 +36,16 @@ struct qdisc_rate_table {
 enum qdisc_state_t {
        __QDISC_STATE_SCHED,
        __QDISC_STATE_DEACTIVATED,
+       __QDISC_STATE_MISSED,
+       __QDISC_STATE_DRAINING,
 };
 
+#define QDISC_STATE_MISSED     BIT(__QDISC_STATE_MISSED)
+#define QDISC_STATE_DRAINING   BIT(__QDISC_STATE_DRAINING)
+
+#define QDISC_STATE_NON_EMPTY  (QDISC_STATE_MISSED | \
+                                       QDISC_STATE_DRAINING)
+
 struct qdisc_size_table {
        struct rcu_head         rcu;
        struct list_head        list;
@@ -109,8 +117,6 @@ struct Qdisc {
        spinlock_t              busylock ____cacheline_aligned_in_smp;
        spinlock_t              seqlock;
 
-       /* for NOLOCK qdisc, true if there are no enqueued skbs */
-       bool                    empty;
        struct rcu_head         rcu;
 
        /* private data */
@@ -144,6 +150,11 @@ static inline bool qdisc_is_running(struct Qdisc *qdisc)
        return (raw_read_seqcount(&qdisc->running) & 1) ? true : false;
 }
 
+static inline bool nolock_qdisc_is_empty(const struct Qdisc *qdisc)
+{
+       return !(READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY);
+}
+
 static inline bool qdisc_is_percpu_stats(const struct Qdisc *q)
 {
        return q->flags & TCQ_F_CPUSTATS;
@@ -152,16 +163,37 @@ static inline bool qdisc_is_percpu_stats(const struct Qdisc *q)
 static inline bool qdisc_is_empty(const struct Qdisc *qdisc)
 {
        if (qdisc_is_percpu_stats(qdisc))
-               return READ_ONCE(qdisc->empty);
+               return nolock_qdisc_is_empty(qdisc);
        return !READ_ONCE(qdisc->q.qlen);
 }
 
 static inline bool qdisc_run_begin(struct Qdisc *qdisc)
 {
        if (qdisc->flags & TCQ_F_NOLOCK) {
-               if (!spin_trylock(&qdisc->seqlock))
+               if (spin_trylock(&qdisc->seqlock))
+                       return true;
+
+               /* If the MISSED flag is set, it means other thread has
+                * set the MISSED flag before second spin_trylock(), so
+                * we can return false here to avoid multi cpus doing
+                * the set_bit() and second spin_trylock() concurrently.
+                */
+               if (test_bit(__QDISC_STATE_MISSED, &qdisc->state))
                        return false;
-               WRITE_ONCE(qdisc->empty, false);
+
+               /* Set the MISSED flag before the second spin_trylock(),
+                * if the second spin_trylock() return false, it means
+                * other cpu holding the lock will do dequeuing for us
+                * or it will see the MISSED flag set after releasing
+                * lock and reschedule the net_tx_action() to do the
+                * dequeuing.
+                */
+               set_bit(__QDISC_STATE_MISSED, &qdisc->state);
+
+               /* Retry again in case other CPU may not see the new flag
+                * after it releases the lock at the end of qdisc_run_end().
+                */
+               return spin_trylock(&qdisc->seqlock);
        } else if (qdisc_is_running(qdisc)) {
                return false;
        }
@@ -175,9 +207,15 @@ static inline bool qdisc_run_begin(struct Qdisc *qdisc)
 
 static inline void qdisc_run_end(struct Qdisc *qdisc)
 {
-       write_seqcount_end(&qdisc->running);
-       if (qdisc->flags & TCQ_F_NOLOCK)
+       if (qdisc->flags & TCQ_F_NOLOCK) {
                spin_unlock(&qdisc->seqlock);
+
+               if (unlikely(test_bit(__QDISC_STATE_MISSED,
+                                     &qdisc->state)))
+                       __netif_schedule(qdisc);
+       } else {
+               write_seqcount_end(&qdisc->running);
+       }
 }
 
 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc)