Merge git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net

[platform/kernel/linux-starfive.git] / include / net / sch_generic.h

diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index 5771030..9ed33e6 100644 (file)
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -37,8 +37,15 @@ enum qdisc_state_t {
        __QDISC_STATE_SCHED,
        __QDISC_STATE_DEACTIVATED,
        __QDISC_STATE_MISSED,
+       __QDISC_STATE_DRAINING,
 };
 
+#define QDISC_STATE_MISSED     BIT(__QDISC_STATE_MISSED)
+#define QDISC_STATE_DRAINING   BIT(__QDISC_STATE_DRAINING)
+
+#define QDISC_STATE_NON_EMPTY  (QDISC_STATE_MISSED | \
+                                       QDISC_STATE_DRAINING)
+
 struct qdisc_size_table {
        struct rcu_head         rcu;
        struct list_head        list;
@@ -110,8 +117,6 @@ struct Qdisc {
        spinlock_t              busylock ____cacheline_aligned_in_smp;
        spinlock_t              seqlock;
 
-       /* for NOLOCK qdisc, true if there are no enqueued skbs */
-       bool                    empty;
        struct rcu_head         rcu;
 
        /* private data */
@@ -145,6 +150,11 @@ static inline bool qdisc_is_running(struct Qdisc *qdisc)
        return (raw_read_seqcount(&qdisc->running) & 1) ? true : false;
 }
 
+static inline bool nolock_qdisc_is_empty(const struct Qdisc *qdisc)
+{
+       return !(READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY);
+}
+
 static inline bool qdisc_is_percpu_stats(const struct Qdisc *q)
 {
        return q->flags & TCQ_F_CPUSTATS;
@@ -153,7 +163,7 @@ static inline bool qdisc_is_percpu_stats(const struct Qdisc *q)
 static inline bool qdisc_is_empty(const struct Qdisc *qdisc)
 {
        if (qdisc_is_percpu_stats(qdisc))
-               return READ_ONCE(qdisc->empty);
+               return nolock_qdisc_is_empty(qdisc);
        return !READ_ONCE(qdisc->q.qlen);
 }
 
@@ -161,7 +171,7 @@ static inline bool qdisc_run_begin(struct Qdisc *qdisc)
 {
        if (qdisc->flags & TCQ_F_NOLOCK) {
                if (spin_trylock(&qdisc->seqlock))
-                       goto nolock_empty;
+                       return true;
 
                /* Paired with smp_mb__after_atomic() to make sure
                 * STATE_MISSED checking is synchronized with clearing
@@ -195,11 +205,7 @@ static inline bool qdisc_run_begin(struct Qdisc *qdisc)
                /* Retry again in case other CPU may not see the new flag
                 * after it releases the lock at the end of qdisc_run_end().
                 */
-               if (!spin_trylock(&qdisc->seqlock))
-                       return false;
-
-nolock_empty:
-               WRITE_ONCE(qdisc->empty, false);
+               return spin_trylock(&qdisc->seqlock);
        } else if (qdisc_is_running(qdisc)) {
                return false;
        }
@@ -213,15 +219,14 @@ nolock_empty:
 
 static inline void qdisc_run_end(struct Qdisc *qdisc)
 {
-       write_seqcount_end(&qdisc->running);
        if (qdisc->flags & TCQ_F_NOLOCK) {
                spin_unlock(&qdisc->seqlock);
 
                if (unlikely(test_bit(__QDISC_STATE_MISSED,
-                                     &qdisc->state))) {
-                       clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
+                                     &qdisc->state)))
                        __netif_schedule(qdisc);
-               }
+       } else {
+               write_seqcount_end(&qdisc->running);
        }
 }