From: Pavel Begunkov Date: Mon, 9 Jan 2023 14:46:03 +0000 (+0000) Subject: io_uring: move submitter_task out of cold cacheline X-Git-Tag: v6.6.7~3537^2~38 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=dde40322ae20f2d6b0bcb781a9eedcfc7ca3aa73;p=platform%2Fkernel%2Flinux-starfive.git io_uring: move submitter_task out of cold cacheline ->submitter_task is used somewhat more frequent now than before, i.e. for local tw enqueue and run, let's move it from the end of ctx, which is full of cold data, to the first cacheline with mostly constants. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/415ca91dc5ad1dec612b892e489cda98e1069542.1673274244.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 128a67a..8dfb6c4 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -195,11 +195,7 @@ struct io_alloc_cache { struct io_ring_ctx { /* const or read-mostly hot data */ struct { - struct percpu_ref refs; - - struct io_rings *rings; unsigned int flags; - enum task_work_notify_mode notify_method; unsigned int compat: 1; unsigned int drain_next: 1; unsigned int restricted: 1; @@ -210,6 +206,11 @@ struct io_ring_ctx { unsigned int syscall_iopoll: 1; /* all CQEs should be posted only by the submitter task */ unsigned int task_complete: 1; + + enum task_work_notify_mode notify_method; + struct io_rings *rings; + struct task_struct *submitter_task; + struct percpu_ref refs; } ____cacheline_aligned_in_smp; /* submission data */ @@ -320,7 +321,6 @@ struct io_ring_ctx { /* Keep this last, we don't need it for the fast path */ struct io_restriction restrictions; - struct task_struct *submitter_task; /* slow path rsrc auxilary data, used by update/register */ struct io_rsrc_node *rsrc_backup_node;