struct io_uring_cqe *cqe = &ctx->completion_cqes[i];
if (!io_fill_cqe_aux(ctx, cqe->user_data, cqe->res, cqe->flags)) {
- if (ctx->task_complete) {
+ if (ctx->lockless_cq) {
spin_lock(&ctx->completion_lock);
io_cqring_event_overflow(ctx, cqe->user_data,
cqe->res, cqe->flags, 0, 0);
if (!(req->flags & REQ_F_CQE_SKIP) &&
unlikely(!io_fill_cqe_req(ctx, req))) {
- if (ctx->task_complete) {
+ if (ctx->lockless_cq) {
spin_lock(&ctx->completion_lock);
io_req_cqe_overflow(req);
spin_unlock(&ctx->completion_lock);
{
struct page **page_array;
unsigned int nr_pages;
- int ret;
+ int ret, i;
*npages = 0;
*/
if (page_array[0] != page_array[ret - 1])
goto err;
+
+ /*
+ * Can't support mapping user allocated ring memory on 32-bit archs
+ * where it could potentially reside in highmem. Just fail those with
+ * -EINVAL, just like we did on kernels that didn't support this
+ * feature.
+ */
+ for (i = 0; i < nr_pages; i++) {
+ if (PageHighMem(page_array[i])) {
+ ret = -EINVAL;
+ goto err;
+ }
+ }
+
*pages = page_array;
*npages = nr_pages;
return page_to_virt(page_array[0]);
return percpu_counter_sum(&tctx->inflight);
}
-static void io_uring_cancel_wq(struct io_uring_task *tctx)
-{
- int ret;
-
- if (!tctx->io_wq)
- return;
-
- /*
- * FIXED_FILE request isn't tracked in do_exit(), and these
- * requests may be submitted to our io_wq as iopoll, so have to
- * cancel them before destroying io_wq for avoiding IO hang
- */
- do {
- struct io_tctx_node *node;
- unsigned long index;
-
- ret = 0;
- xa_for_each(&tctx->xa, index, node) {
- struct io_ring_ctx *ctx = node->ctx;
- struct io_task_cancel cancel = { .task = current, .all = true, };
- enum io_wq_cancel cret;
-
- io_iopoll_try_reap_events(ctx);
- cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
- &cancel, true);
- ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
- cond_resched();
- }
- } while (ret);
-}
-
/*
* Find any io_uring ctx that this task has registered or done IO on, and cancel
* requests. @sqd should be not-null IFF it's an SQPOLL thread cancellation.
finish_wait(&tctx->wait, &wait);
} while (1);
- io_uring_cancel_wq(tctx);
io_uring_clean_tctx(tctx);
if (cancel_all) {
/*