struct llist_head rsrc_put_llist;
struct list_head rsrc_ref_list;
spinlock_t rsrc_ref_lock;
+ struct fixed_rsrc_ref_node *rsrc_backup_node;
struct io_restriction restrictions;
percpu_ref_kill(&ref_node->refs);
}
+static int io_rsrc_refnode_prealloc(struct io_ring_ctx *ctx)
+{
+ if (ctx->rsrc_backup_node)
+ return 0;
+ ctx->rsrc_backup_node = alloc_fixed_rsrc_ref_node(ctx);
+ return ctx->rsrc_backup_node ? 0 : -ENOMEM;
+}
+
+static struct fixed_rsrc_ref_node *
+io_rsrc_refnode_get(struct io_ring_ctx *ctx,
+ struct fixed_rsrc_data *rsrc_data,
+ void (*rsrc_put)(struct io_ring_ctx *ctx,
+ struct io_rsrc_put *prsrc))
+{
+ struct fixed_rsrc_ref_node *node = ctx->rsrc_backup_node;
+
+ WARN_ON_ONCE(!node);
+
+ ctx->rsrc_backup_node = NULL;
+ node->rsrc_data = rsrc_data;
+ node->rsrc_put = rsrc_put;
+ return node;
+}
+
static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data,
struct io_ring_ctx *ctx,
void (*rsrc_put)(struct io_ring_ctx *ctx,
struct io_rsrc_put *prsrc))
{
- struct fixed_rsrc_ref_node *backup_node;
+ struct fixed_rsrc_ref_node *node;
int ret;
if (data->quiesce)
data->quiesce = true;
do {
- ret = -ENOMEM;
- backup_node = alloc_fixed_rsrc_ref_node(ctx);
- if (!backup_node)
+ ret = io_rsrc_refnode_prealloc(ctx);
+ if (ret)
break;
- backup_node->rsrc_data = data;
- backup_node->rsrc_put = rsrc_put;
-
io_sqe_rsrc_kill_node(ctx, data);
percpu_ref_kill(&data->refs);
flush_delayed_work(&ctx->rsrc_put_work);
break;
percpu_ref_resurrect(&data->refs);
- io_sqe_rsrc_set_node(ctx, data, backup_node);
- backup_node = NULL;
+ node = io_rsrc_refnode_get(ctx, data, rsrc_put);
+ io_sqe_rsrc_set_node(ctx, data, node);
reinit_completion(&data->done);
+
mutex_unlock(&ctx->uring_lock);
ret = io_run_task_work_sig();
mutex_lock(&ctx->uring_lock);
} while (ret >= 0);
data->quiesce = false;
- if (backup_node)
- destroy_fixed_rsrc_ref_node(backup_node);
return ret;
}
return -EOVERFLOW;
if (done > ctx->nr_user_files)
return -EINVAL;
-
- ref_node = alloc_fixed_rsrc_ref_node(ctx);
- if (!ref_node)
- return -ENOMEM;
- init_fixed_file_ref_node(ctx, ref_node);
+ err = io_rsrc_refnode_prealloc(ctx);
+ if (err)
+ return err;
fds = u64_to_user_ptr(up->data);
for (done = 0; done < nr_args; done++) {
if (needs_switch) {
percpu_ref_kill(&data->node->refs);
+ ref_node = io_rsrc_refnode_get(ctx, data, io_ring_file_put);
io_sqe_rsrc_set_node(ctx, data, ref_node);
- } else
- destroy_fixed_rsrc_ref_node(ref_node);
-
+ }
return done ? done : err;
}
io_eventfd_unregister(ctx);
io_destroy_buffers(ctx);
+ if (ctx->rsrc_backup_node)
+ destroy_fixed_rsrc_ref_node(ctx->rsrc_backup_node);
+
#if defined(CONFIG_UNIX)
if (ctx->ring_sock) {
ctx->ring_sock->file = NULL; /* so that iput() is called */