From b69de288e913030082bed3a324ddc58be6c1e983 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 17 Mar 2021 08:37:41 -0600 Subject: [PATCH] io_uring: allow events and user_data update of running poll requests This adds two new POLL_ADD flags, IORING_POLL_UPDATE_EVENTS and IORING_POLL_UPDATE_USER_DATA. As with the other POLL_ADD flag, these are masked into sqe->len. If set, the POLL_ADD will have the following behavior: - sqe->addr must contain the the user_data of the poll request that needs to be modified. This field is otherwise invalid for a POLL_ADD command. - If IORING_POLL_UPDATE_EVENTS is set, sqe->poll_events must contain the new mask for the existing poll request. There are no checks for whether these are identical or not, if a matching poll request is found, then it is re-armed with the new mask. - If IORING_POLL_UPDATE_USER_DATA is set, sqe->off must contain the new user_data for the existing poll request. A POLL_ADD with any of these flags set may complete with any of the following results: 1) 0, which means that we successfully found the existing poll request specified, and performed the re-arm procedure. Any error from that re-arm will be exposed as a completion event for that original poll request, not for the update request. 2) -ENOENT, if no existing poll request was found with the given user_data. 3) -EALREADY, if the existing poll request was already in the process of being removed/canceled/completing. 4) -EACCES, if an attempt was made to modify an internal poll request (eg not one originally issued ass IORING_OP_POLL_ADD). The usual -EINVAL cases apply as well, if any invalid fields are set in the sqe for this command type. Signed-off-by: Jens Axboe --- fs/io_uring.c | 95 +++++++++++++++++++++++++++++++++++++++---- include/uapi/linux/io_uring.h | 5 +++ 2 files changed, 92 insertions(+), 8 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 3c54e8c..eeb1652 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -486,7 +486,15 @@ struct io_poll_iocb { __poll_t events; bool done; bool canceled; - struct wait_queue_entry wait; + bool update_events; + bool update_user_data; + union { + struct wait_queue_entry wait; + struct { + u64 old_user_data; + u64 new_user_data; + }; + }; }; struct io_poll_remove { @@ -4911,8 +4919,9 @@ static bool io_poll_complete(struct io_kiocb *req, __poll_t mask, int error) } if (!error) error = mangle_poll(mask); - if (!__io_cqring_fill_event(req, error, flags) || - (req->poll.events & EPOLLONESHOT)) { + if (req->poll.events & EPOLLONESHOT) + flags = 0; + if (!__io_cqring_fill_event(req, error, flags)) { io_poll_remove_waitqs(req); req->poll.done = true; flags = 0; @@ -4992,6 +5001,7 @@ static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events, poll->head = NULL; poll->done = false; poll->canceled = false; + poll->update_events = poll->update_user_data = false; #define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP) /* mask in events that we always want/need */ poll->events = events | IO_POLL_UNMASK; @@ -5370,24 +5380,36 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; - if (sqe->addr || sqe->ioprio || sqe->off || sqe->buf_index) + if (sqe->ioprio || sqe->buf_index) return -EINVAL; flags = READ_ONCE(sqe->len); - if (flags & ~IORING_POLL_ADD_MULTI) + if (flags & ~(IORING_POLL_ADD_MULTI | IORING_POLL_UPDATE_EVENTS | + IORING_POLL_UPDATE_USER_DATA)) return -EINVAL; - events = READ_ONCE(sqe->poll32_events); #ifdef __BIG_ENDIAN events = swahw32(events); #endif - if (!flags) + if (!(flags & IORING_POLL_ADD_MULTI)) events |= EPOLLONESHOT; + poll->update_events = poll->update_user_data = false; + if (flags & IORING_POLL_UPDATE_EVENTS) { + poll->update_events = true; + poll->old_user_data = READ_ONCE(sqe->addr); + } + if (flags & IORING_POLL_UPDATE_USER_DATA) { + poll->update_user_data = true; + poll->new_user_data = READ_ONCE(sqe->off); + } + if (!(poll->update_events || poll->update_user_data) && + (sqe->off || sqe->addr)) + return -EINVAL; poll->events = demangle_poll(events) | (events & (EPOLLEXCLUSIVE|EPOLLONESHOT)); return 0; } -static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) +static int __io_poll_add(struct io_kiocb *req) { struct io_poll_iocb *poll = &req->poll; struct io_ring_ctx *ctx = req->ctx; @@ -5413,6 +5435,63 @@ static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) return ipt.error; } +static int io_poll_update(struct io_kiocb *req) +{ + struct io_ring_ctx *ctx = req->ctx; + struct io_kiocb *preq; + int ret; + + spin_lock_irq(&ctx->completion_lock); + preq = io_poll_find(ctx, req->poll.old_user_data); + if (!preq) { + ret = -ENOENT; + goto err; + } else if (preq->opcode != IORING_OP_POLL_ADD) { + /* don't allow internal poll updates */ + ret = -EACCES; + goto err; + } + if (!__io_poll_remove_one(preq, &preq->poll)) { + /* in process of completing/removal */ + ret = -EALREADY; + goto err; + } + /* we now have a detached poll request. reissue. */ + ret = 0; +err: + spin_unlock_irq(&ctx->completion_lock); + if (ret < 0) { + req_set_fail_links(req); + io_req_complete(req, ret); + return 0; + } + /* only mask one event flags, keep behavior flags */ + if (req->poll.update_events) { + preq->poll.events &= ~0xffff; + preq->poll.events |= req->poll.events & 0xffff; + preq->poll.events |= IO_POLL_UNMASK; + } + if (req->poll.update_user_data) + preq->user_data = req->poll.new_user_data; + + /* complete update request, we're done with it */ + io_req_complete(req, ret); + + ret = __io_poll_add(preq); + if (ret < 0) { + req_set_fail_links(preq); + io_req_complete(preq, ret); + } + return 0; +} + +static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) +{ + if (!req->poll.update_events && !req->poll.update_user_data) + return __io_poll_add(req); + return io_poll_update(req); +} + static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) { struct io_timeout_data *data = container_of(timer, diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 76c9676..5beaa6b 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -166,8 +166,13 @@ enum { * IORING_POLL_ADD_MULTI Multishot poll. Sets IORING_CQE_F_MORE if * the poll handler will continue to report * CQEs on behalf of the same SQE. + * + * IORING_POLL_UPDATE Update existing poll request, matching + * sqe->addr as the old user_data field. */ #define IORING_POLL_ADD_MULTI (1U << 0) +#define IORING_POLL_UPDATE_EVENTS (1U << 1) +#define IORING_POLL_UPDATE_USER_DATA (1U << 2) /* * IO completion data structure (Completion Queue Entry) -- 2.7.4