io_uring: move io_import_fixed()

Move io_import_fixed() into rsrc.c where it belongs.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/4d5becb21f332b4fef6a7cedd6a50e65e2371630.1655684496.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 68629eb..1106089 100644 (file)
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -1310,3 +1310,63 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
                io_rsrc_node_switch(ctx, NULL);
        return ret;
 }
+
+int io_import_fixed(int ddir, struct iov_iter *iter,
+                          struct io_mapped_ubuf *imu,
+                          u64 buf_addr, size_t len)
+{
+       u64 buf_end;
+       size_t offset;
+
+       if (WARN_ON_ONCE(!imu))
+               return -EFAULT;
+       if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))
+               return -EFAULT;
+       /* not inside the mapped region */
+       if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end))
+               return -EFAULT;
+
+       /*
+        * May not be a start of buffer, set size appropriately
+        * and advance us to the beginning.
+        */
+       offset = buf_addr - imu->ubuf;
+       iov_iter_bvec(iter, ddir, imu->bvec, imu->nr_bvecs, offset + len);
+
+       if (offset) {
+               /*
+                * Don't use iov_iter_advance() here, as it's really slow for
+                * using the latter parts of a big fixed buffer - it iterates
+                * over each segment manually. We can cheat a bit here, because
+                * we know that:
+                *
+                * 1) it's a BVEC iter, we set it up
+                * 2) all bvecs are PAGE_SIZE in size, except potentially the
+                *    first and last bvec
+                *
+                * So just find our index, and adjust the iterator afterwards.
+                * If the offset is within the first bvec (or the whole first
+                * bvec, just use iov_iter_advance(). This makes it easier
+                * since we can just skip the first segment, which may not
+                * be PAGE_SIZE aligned.
+                */
+               const struct bio_vec *bvec = imu->bvec;
+
+               if (offset <= bvec->bv_len) {
+                       iov_iter_advance(iter, offset);
+               } else {
+                       unsigned long seg_skip;
+
+                       /* skip first vec */
+                       offset -= bvec->bv_len;
+                       seg_skip = 1 + (offset >> PAGE_SHIFT);
+
+                       iter->bvec = bvec + seg_skip;
+                       iter->nr_segs -= seg_skip;
+                       iter->count -= bvec->bv_len + offset;
+                       iter->iov_offset = offset & ~PAGE_MASK;
+               }
+       }
+
+       return 0;
+}

diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 03f2651..87f5831 100644 (file)
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -64,6 +64,9 @@ int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
 void io_rsrc_node_switch(struct io_ring_ctx *ctx,
                         struct io_rsrc_data *data_to_kill);
 
+int io_import_fixed(int ddir, struct iov_iter *iter,
+                          struct io_mapped_ubuf *imu,
+                          u64 buf_addr, size_t len);
 
 void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx);
 int io_sqe_buffers_unregister(struct io_ring_ctx *ctx);

diff --git a/io_uring/rw.c b/io_uring/rw.c
index ded8ef0..e07f267 100644 (file)
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -273,66 +273,6 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret,
        return IOU_ISSUE_SKIP_COMPLETE;
 }
 
-static int io_import_fixed(int ddir, struct iov_iter *iter,
-                          struct io_mapped_ubuf *imu,
-                          u64 buf_addr, size_t len)
-{
-       u64 buf_end;
-       size_t offset;
-
-       if (WARN_ON_ONCE(!imu))
-               return -EFAULT;
-       if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))
-               return -EFAULT;
-       /* not inside the mapped region */
-       if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end))
-               return -EFAULT;
-
-       /*
-        * May not be a start of buffer, set size appropriately
-        * and advance us to the beginning.
-        */
-       offset = buf_addr - imu->ubuf;
-       iov_iter_bvec(iter, ddir, imu->bvec, imu->nr_bvecs, offset + len);
-
-       if (offset) {
-               /*
-                * Don't use iov_iter_advance() here, as it's really slow for
-                * using the latter parts of a big fixed buffer - it iterates
-                * over each segment manually. We can cheat a bit here, because
-                * we know that:
-                *
-                * 1) it's a BVEC iter, we set it up
-                * 2) all bvecs are PAGE_SIZE in size, except potentially the
-                *    first and last bvec
-                *
-                * So just find our index, and adjust the iterator afterwards.
-                * If the offset is within the first bvec (or the whole first
-                * bvec, just use iov_iter_advance(). This makes it easier
-                * since we can just skip the first segment, which may not
-                * be PAGE_SIZE aligned.
-                */
-               const struct bio_vec *bvec = imu->bvec;
-
-               if (offset <= bvec->bv_len) {
-                       iov_iter_advance(iter, offset);
-               } else {
-                       unsigned long seg_skip;
-
-                       /* skip first vec */
-                       offset -= bvec->bv_len;
-                       seg_skip = 1 + (offset >> PAGE_SHIFT);
-
-                       iter->bvec = bvec + seg_skip;
-                       iter->nr_segs -= seg_skip;
-                       iter->count -= bvec->bv_len + offset;
-                       iter->iov_offset = offset & ~PAGE_MASK;
-               }
-       }
-
-       return 0;
-}
-
 #ifdef CONFIG_COMPAT
 static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
                                unsigned int issue_flags)