};
static void io_wq_submit_work(struct io_wq_work **workptr);
-static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data,
- long res);
+static void io_cqring_fill_event(struct io_kiocb *req, long res);
static void __io_free_req(struct io_kiocb *req);
static void io_put_req(struct io_kiocb *req, struct io_kiocb **nxtptr);
+static void io_double_put_req(struct io_kiocb *req);
static struct kmem_cache *req_cachep;
if (ret != -1) {
atomic_inc(&req->ctx->cq_timeouts);
list_del_init(&req->list);
- io_cqring_fill_event(req->ctx, req->user_data, 0);
- __io_free_req(req);
+ io_cqring_fill_event(req, 0);
+ io_put_req(req, NULL);
}
}
return &rings->cqes[tail & ctx->cq_mask];
}
-static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data,
- long res)
+static void io_cqring_fill_event(struct io_kiocb *req, long res)
{
+ struct io_ring_ctx *ctx = req->ctx;
struct io_uring_cqe *cqe;
- trace_io_uring_complete(ctx, ki_user_data, res);
+ trace_io_uring_complete(ctx, req->user_data, res);
/*
* If we can't get a cq entry, userspace overflowed the
*/
cqe = io_get_cqring(ctx);
if (cqe) {
- WRITE_ONCE(cqe->user_data, ki_user_data);
+ WRITE_ONCE(cqe->user_data, req->user_data);
WRITE_ONCE(cqe->res, res);
WRITE_ONCE(cqe->flags, 0);
} else {
eventfd_signal(ctx->cq_ev_fd, 1);
}
-static void io_cqring_add_event(struct io_ring_ctx *ctx, u64 user_data,
- long res)
+static void io_cqring_add_event(struct io_kiocb *req, long res)
{
+ struct io_ring_ctx *ctx = req->ctx;
unsigned long flags;
spin_lock_irqsave(&ctx->completion_lock, flags);
- io_cqring_fill_event(ctx, user_data, res);
+ io_cqring_fill_event(req, res);
io_commit_cqring(ctx);
spin_unlock_irqrestore(&ctx->completion_lock, flags);
ret = hrtimer_try_to_cancel(&req->timeout.timer);
if (ret != -1) {
- io_cqring_fill_event(ctx, req->user_data, -ECANCELED);
+ io_cqring_fill_event(req, -ECANCELED);
io_commit_cqring(ctx);
req->flags &= ~REQ_F_LINK;
- __io_free_req(req);
+ io_put_req(req, NULL);
return true;
}
link->submit.sqe->opcode == IORING_OP_LINK_TIMEOUT) {
io_link_cancel_timeout(ctx, link);
} else {
- io_cqring_fill_event(ctx, link->user_data, -ECANCELED);
- __io_free_req(link);
+ io_cqring_fill_event(link, -ECANCELED);
+ io_double_put_req(link);
}
}
}
}
+static void io_double_put_req(struct io_kiocb *req)
+{
+ /* drop both submit and complete references */
+ if (refcount_sub_and_test(2, &req->refs))
+ __io_free_req(req);
+}
+
static unsigned io_cqring_events(struct io_ring_ctx *ctx)
{
struct io_rings *rings = ctx->rings;
req = list_first_entry(done, struct io_kiocb, list);
list_del(&req->list);
- io_cqring_fill_event(ctx, req->user_data, req->result);
+ io_cqring_fill_event(req, req->result);
(*nr_events)++;
if (refcount_dec_and_test(&req->refs)) {
if ((req->flags & REQ_F_LINK) && res != req->result)
req->flags |= REQ_F_FAIL_LINK;
- io_cqring_add_event(req->ctx, req->user_data, res);
+ io_cqring_add_event(req, res);
}
static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
/*
* IORING_OP_NOP just posts a completion event, nothing else.
*/
-static int io_nop(struct io_kiocb *req, u64 user_data)
+static int io_nop(struct io_kiocb *req)
{
struct io_ring_ctx *ctx = req->ctx;
- long err = 0;
if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
return -EINVAL;
- io_cqring_add_event(ctx, user_data, err);
+ io_cqring_add_event(req, 0);
io_put_req(req, NULL);
return 0;
}
if (ret < 0 && (req->flags & REQ_F_LINK))
req->flags |= REQ_F_FAIL_LINK;
- io_cqring_add_event(req->ctx, sqe->user_data, ret);
+ io_cqring_add_event(req, ret);
io_put_req(req, nxt);
return 0;
}
if (ret < 0 && (req->flags & REQ_F_LINK))
req->flags |= REQ_F_FAIL_LINK;
- io_cqring_add_event(req->ctx, sqe->user_data, ret);
+ io_cqring_add_event(req, ret);
io_put_req(req, nxt);
return 0;
}
return ret;
}
- io_cqring_add_event(req->ctx, sqe->user_data, ret);
+ io_cqring_add_event(req, ret);
if (ret < 0 && (req->flags & REQ_F_LINK))
req->flags |= REQ_F_FAIL_LINK;
io_put_req(req, nxt);
}
if (ret < 0 && (req->flags & REQ_F_LINK))
req->flags |= REQ_F_FAIL_LINK;
- io_cqring_add_event(req->ctx, sqe->user_data, ret);
+ io_cqring_add_event(req, ret);
io_put_req(req, nxt);
return 0;
#else
}
spin_unlock_irq(&ctx->completion_lock);
- io_cqring_add_event(req->ctx, sqe->user_data, ret);
+ io_cqring_add_event(req, ret);
if (ret < 0 && (req->flags & REQ_F_LINK))
req->flags |= REQ_F_FAIL_LINK;
io_put_req(req, NULL);
__poll_t mask)
{
req->poll.done = true;
- io_cqring_fill_event(ctx, req->user_data, mangle_poll(mask));
+ io_cqring_fill_event(req, mangle_poll(mask));
io_commit_cqring(ctx);
}
list_del_init(&req->list);
}
- io_cqring_fill_event(ctx, req->user_data, -ETIME);
+ io_cqring_fill_event(req, -ETIME);
io_commit_cqring(ctx);
spin_unlock_irqrestore(&ctx->completion_lock, flags);
/* didn't find timeout */
if (ret) {
fill_ev:
- io_cqring_fill_event(ctx, req->user_data, ret);
+ io_cqring_fill_event(req, ret);
io_commit_cqring(ctx);
spin_unlock_irq(&ctx->completion_lock);
io_cqring_ev_posted(ctx);
goto fill_ev;
}
- io_cqring_fill_event(ctx, req->user_data, 0);
- io_cqring_fill_event(ctx, treq->user_data, -ECANCELED);
+ io_cqring_fill_event(req, 0);
+ io_cqring_fill_event(treq, -ECANCELED);
io_commit_cqring(ctx);
spin_unlock_irq(&ctx->completion_lock);
io_cqring_ev_posted(ctx);
if (ret < 0 && (req->flags & REQ_F_LINK))
req->flags |= REQ_F_FAIL_LINK;
- io_cqring_add_event(req->ctx, sqe->user_data, ret);
+ io_cqring_add_event(req, ret);
io_put_req(req, nxt);
return 0;
}
int ret, opcode;
struct sqe_submit *s = &req->submit;
- req->user_data = READ_ONCE(s->sqe->user_data);
-
opcode = READ_ONCE(s->sqe->opcode);
switch (opcode) {
case IORING_OP_NOP:
- ret = io_nop(req, req->user_data);
+ ret = io_nop(req);
break;
case IORING_OP_READV:
if (unlikely(s->sqe->buf_index))
if (ret) {
if (req->flags & REQ_F_LINK)
req->flags |= REQ_F_FAIL_LINK;
- io_cqring_add_event(ctx, sqe->user_data, ret);
+ io_cqring_add_event(req, ret);
io_put_req(req, NULL);
}
ret = io_async_cancel_one(ctx, user_data);
}
- io_cqring_add_event(ctx, req->user_data, ret);
+ io_cqring_add_event(req, ret);
io_put_req(req, NULL);
return HRTIMER_NORESTART;
}
* failed by the regular submission path.
*/
list_del(&nxt->list);
- io_cqring_fill_event(ctx, nxt->user_data, ret);
+ io_cqring_fill_event(nxt, ret);
trace_io_uring_fail_link(req, nxt);
io_commit_cqring(ctx);
io_put_req(nxt, NULL);
/* and drop final reference, if we failed */
if (ret) {
- io_cqring_add_event(ctx, req->user_data, ret);
+ io_cqring_add_event(req, ret);
if (req->flags & REQ_F_LINK)
req->flags |= REQ_F_FAIL_LINK;
io_put_req(req, NULL);
ret = io_req_defer(ctx, req);
if (ret) {
if (ret != -EIOCBQUEUED) {
- io_cqring_add_event(ctx, req->submit.sqe->user_data, ret);
- io_free_req(req, NULL);
+ io_cqring_add_event(req, ret);
+ io_double_put_req(req);
}
return 0;
}
ret = io_req_defer(ctx, req);
if (ret) {
if (ret != -EIOCBQUEUED) {
- io_cqring_add_event(ctx, req->submit.sqe->user_data, ret);
- io_free_req(req, NULL);
+ io_cqring_add_event(req, ret);
+ io_double_put_req(req);
__io_free_req(shadow);
return 0;
}
struct sqe_submit *s = &req->submit;
int ret;
+ req->user_data = s->sqe->user_data;
+
/* enforce forwards compatibility on users */
if (unlikely(s->sqe->flags & ~SQE_VALID_FLAGS)) {
ret = -EINVAL;
ret = io_req_set_file(ctx, state, req);
if (unlikely(ret)) {
err_req:
- io_cqring_add_event(ctx, s->sqe->user_data, ret);
- io_free_req(req, NULL);
+ io_cqring_add_event(req, ret);
+ io_double_put_req(req);
return;
}
- req->user_data = s->sqe->user_data;
-
/*
* If we already have a head request, queue this one for async
* submittal once the head completes. If we don't have a head but