io_uring/net: make page accounting more consistent

Make network page accounting more consistent with how buffer
registration is working, i.e. account all memory to ctx->user.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/4aacfe64bbb81b27f9ecf5d5c219c69a07e5aa56.1658742118.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/net.c b/io_uring/net.c
index 8fb8469..c13d971 100644 (file)
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -985,7 +985,7 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)
                                          &msg.msg_iter);
                if (unlikely(ret))
                        return ret;
-               ret = mm_account_pinned_pages(&notif->uarg.mmp, zc->len);
+               ret = io_notif_account_mem(notif, zc->len);
                if (unlikely(ret))
                        return ret;
        }

diff --git a/io_uring/notif.c b/io_uring/notif.c
index a938874..e986a0e 100644 (file)
--- a/io_uring/notif.c
+++ b/io_uring/notif.c
@@ -14,12 +14,10 @@ static void __io_notif_complete_tw(struct callback_head *cb)
        struct io_notif *notif = container_of(cb, struct io_notif, task_work);
        struct io_rsrc_node *rsrc_node = notif->rsrc_node;
        struct io_ring_ctx *ctx = notif->ctx;
-       struct mmpin *mmp = &notif->uarg.mmp;
 
-       if (mmp->user) {
-               atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
-               free_uid(mmp->user);
-               mmp->user = NULL;
+       if (notif->account_pages && ctx->user) {
+               __io_unaccount_mem(ctx->user, notif->account_pages);
+               notif->account_pages = 0;
        }
        if (likely(notif->task)) {
                io_put_task(notif->task, 1);
@@ -121,6 +119,7 @@ struct io_notif *io_alloc_notif(struct io_ring_ctx *ctx,
                notif->ctx = ctx;
                notif->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
                notif->uarg.callback = io_uring_tx_zerocopy_callback;
+               notif->account_pages = 0;
        }
 
        notif->seq = slot->seq++;

diff --git a/io_uring/notif.h b/io_uring/notif.h
index 3e05d2c..d6f366b 100644 (file)
--- a/io_uring/notif.h
+++ b/io_uring/notif.h
@@ -5,6 +5,8 @@
 #include <net/sock.h>
 #include <linux/nospec.h>
 
+#include "rsrc.h"
+
 #define IO_NOTIF_SPLICE_BATCH  32
 #define IORING_MAX_NOTIF_SLOTS (1U << 10)
 
@@ -23,6 +25,8 @@ struct io_notif {
        /* hook into ctx->notif_list and ctx->notif_list_locked */
        struct list_head        cache_node;
 
+       unsigned long           account_pages;
+
        union {
                struct callback_head    task_work;
                struct work_struct      commit_work;
@@ -85,3 +89,18 @@ static inline void io_notif_slot_flush_submit(struct io_notif_slot *slot,
        }
        io_notif_slot_flush(slot);
 }
+
+static inline int io_notif_account_mem(struct io_notif *notif, unsigned len)
+{
+       struct io_ring_ctx *ctx = notif->ctx;
+       unsigned nr_pages = (len >> PAGE_SHIFT) + 2;
+       int ret;
+
+       if (ctx->user) {
+               ret = __io_account_mem(ctx->user, nr_pages);
+               if (ret)
+                       return ret;
+               notif->account_pages += nr_pages;
+       }
+       return 0;
+}

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 9165fdf..59704b9 100644 (file)
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -44,17 +44,13 @@ void io_rsrc_refs_drop(struct io_ring_ctx *ctx)
        }
 }
 
-static inline void __io_unaccount_mem(struct user_struct *user,
-                                     unsigned long nr_pages)
-{
-       atomic_long_sub(nr_pages, &user->locked_vm);
-}
-
-static inline int __io_account_mem(struct user_struct *user,
-                                  unsigned long nr_pages)
+int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
 {
        unsigned long page_limit, cur_pages, new_pages;
 
+       if (!nr_pages)
+               return 0;
+
        /* Don't allow more pages than we can safely lock */
        page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 

diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 21813a2..f3a9a17 100644 (file)
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -169,4 +169,13 @@ static inline u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx)
 
 int io_rsrc_update(struct io_kiocb *req, unsigned int issue_flags);
 int io_rsrc_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
+
+int __io_account_mem(struct user_struct *user, unsigned long nr_pages);
+
+static inline void __io_unaccount_mem(struct user_struct *user,
+                                     unsigned long nr_pages)
+{
+       atomic_long_sub(nr_pages, &user->locked_vm);
+}
+
 #endif