1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* kiocb-using read/write
4 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
8 #include <linux/mount.h>
9 #include <linux/slab.h>
10 #include <linux/file.h>
11 #include <linux/uio.h>
12 #include <linux/falloc.h>
13 #include <linux/sched/mm.h>
14 #include <trace/events/fscache.h>
17 struct cachefiles_kiocb {
25 struct cachefiles_object *object;
26 netfs_io_terminated_t term_func;
29 unsigned int inval_counter; /* Copy of cookie->inval_counter */
33 static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki)
35 if (refcount_dec_and_test(&ki->ki_refcnt)) {
36 cachefiles_put_object(ki->object, cachefiles_obj_put_ioreq);
37 fput(ki->iocb.ki_filp);
43 * Handle completion of a read from the cache.
45 static void cachefiles_read_complete(struct kiocb *iocb, long ret)
47 struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
48 struct inode *inode = file_inode(ki->iocb.ki_filp);
53 trace_cachefiles_io_error(ki->object, inode, ret,
54 cachefiles_trace_read_error);
58 if (ki->object->cookie->inval_counter == ki->inval_counter)
64 ki->term_func(ki->term_func_priv, ret, ki->was_async);
67 cachefiles_put_kiocb(ki);
71 * Initiate a read from the cache.
73 static int cachefiles_read(struct netfs_cache_resources *cres,
75 struct iov_iter *iter,
76 enum netfs_read_from_hole read_hole,
77 netfs_io_terminated_t term_func,
80 struct cachefiles_object *object;
81 struct cachefiles_kiocb *ki;
83 unsigned int old_nofs;
84 ssize_t ret = -ENOBUFS;
85 size_t len = iov_iter_count(iter), skipped = 0;
87 if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
88 goto presubmission_error;
91 object = cachefiles_cres_object(cres);
92 file = cachefiles_cres_file(cres);
94 _enter("%pD,%li,%llx,%zx/%llx",
95 file, file_inode(file)->i_ino, start_pos, len,
96 i_size_read(file_inode(file)));
98 /* If the caller asked us to seek for data before doing the read, then
99 * we should do that now. If we find a gap, we fill it with zeros.
101 if (read_hole != NETFS_READ_HOLE_IGNORE) {
102 loff_t off = start_pos, off2;
104 off2 = cachefiles_inject_read_error();
106 off2 = vfs_llseek(file, off, SEEK_DATA);
107 if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO && off2 != -ENXIO) {
110 goto presubmission_error;
113 if (off2 == -ENXIO || off2 >= start_pos + len) {
114 /* The region is beyond the EOF or there's no more data
115 * in the region, so clear the rest of the buffer and
119 if (read_hole == NETFS_READ_HOLE_FAIL)
120 goto presubmission_error;
122 iov_iter_zero(len, iter);
125 goto presubmission_error;
128 skipped = off2 - off;
129 iov_iter_zero(skipped, iter);
133 ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL);
135 goto presubmission_error;
137 refcount_set(&ki->ki_refcnt, 2);
138 ki->iocb.ki_filp = file;
139 ki->iocb.ki_pos = start_pos + skipped;
140 ki->iocb.ki_flags = IOCB_DIRECT;
141 ki->iocb.ki_ioprio = get_current_ioprio();
142 ki->skipped = skipped;
144 ki->inval_counter = cres->inval_counter;
145 ki->term_func = term_func;
146 ki->term_func_priv = term_func_priv;
147 ki->was_async = true;
150 ki->iocb.ki_complete = cachefiles_read_complete;
152 get_file(ki->iocb.ki_filp);
153 cachefiles_grab_object(object, cachefiles_obj_get_ioreq);
155 trace_cachefiles_read(object, file_inode(file), ki->iocb.ki_pos, len - skipped);
156 old_nofs = memalloc_nofs_save();
157 ret = cachefiles_inject_read_error();
159 ret = vfs_iocb_iter_read(file, &ki->iocb, iter);
160 memalloc_nofs_restore(old_nofs);
166 case -ERESTARTNOINTR:
167 case -ERESTARTNOHAND:
168 case -ERESTART_RESTARTBLOCK:
169 /* There's no easy way to restart the syscall since other AIO's
170 * may be already running. Just fail this IO with EINTR.
175 ki->was_async = false;
176 cachefiles_read_complete(&ki->iocb, ret);
183 cachefiles_put_kiocb(ki);
184 _leave(" = %zd", ret);
189 term_func(term_func_priv, ret < 0 ? ret : skipped, false);
194 * Query the occupancy of the cache in a region, returning where the next chunk
195 * of data starts and how long it is.
197 static int cachefiles_query_occupancy(struct netfs_cache_resources *cres,
198 loff_t start, size_t len, size_t granularity,
199 loff_t *_data_start, size_t *_data_len)
201 struct cachefiles_object *object;
208 if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
211 object = cachefiles_cres_object(cres);
212 file = cachefiles_cres_file(cres);
213 granularity = max_t(size_t, object->volume->cache->bsize, granularity);
215 _enter("%pD,%li,%llx,%zx/%llx",
216 file, file_inode(file)->i_ino, start, len,
217 i_size_read(file_inode(file)));
219 off = cachefiles_inject_read_error();
221 off = vfs_llseek(file, start, SEEK_DATA);
223 return -ENODATA; /* Beyond EOF */
224 if (off < 0 && off >= (loff_t)-MAX_ERRNO)
225 return -ENOBUFS; /* Error. */
226 if (round_up(off, granularity) >= start + len)
227 return -ENODATA; /* No data in range */
229 off2 = cachefiles_inject_read_error();
231 off2 = vfs_llseek(file, off, SEEK_HOLE);
233 return -ENODATA; /* Beyond EOF */
234 if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO)
235 return -ENOBUFS; /* Error. */
237 /* Round away partial blocks */
238 off = round_up(off, granularity);
239 off2 = round_down(off2, granularity);
244 if (off2 > start + len)
247 *_data_len = off2 - off;
252 * Handle completion of a write to the cache.
254 static void cachefiles_write_complete(struct kiocb *iocb, long ret)
256 struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
257 struct cachefiles_object *object = ki->object;
258 struct inode *inode = file_inode(ki->iocb.ki_filp);
262 kiocb_end_write(iocb);
265 trace_cachefiles_io_error(object, inode, ret,
266 cachefiles_trace_write_error);
268 atomic_long_sub(ki->b_writing, &object->volume->cache->b_writing);
269 set_bit(FSCACHE_COOKIE_HAVE_DATA, &object->cookie->flags);
271 ki->term_func(ki->term_func_priv, ret, ki->was_async);
272 cachefiles_put_kiocb(ki);
276 * Initiate a write to the cache.
278 int __cachefiles_write(struct cachefiles_object *object,
281 struct iov_iter *iter,
282 netfs_io_terminated_t term_func,
283 void *term_func_priv)
285 struct cachefiles_cache *cache;
286 struct cachefiles_kiocb *ki;
287 unsigned int old_nofs;
289 size_t len = iov_iter_count(iter);
291 fscache_count_write();
292 cache = object->volume->cache;
294 _enter("%pD,%li,%llx,%zx/%llx",
295 file, file_inode(file)->i_ino, start_pos, len,
296 i_size_read(file_inode(file)));
298 ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL);
301 term_func(term_func_priv, -ENOMEM, false);
305 refcount_set(&ki->ki_refcnt, 2);
306 ki->iocb.ki_filp = file;
307 ki->iocb.ki_pos = start_pos;
308 ki->iocb.ki_flags = IOCB_DIRECT | IOCB_WRITE;
309 ki->iocb.ki_ioprio = get_current_ioprio();
311 ki->start = start_pos;
313 ki->term_func = term_func;
314 ki->term_func_priv = term_func_priv;
315 ki->was_async = true;
316 ki->b_writing = (len + (1 << cache->bshift) - 1) >> cache->bshift;
319 ki->iocb.ki_complete = cachefiles_write_complete;
320 atomic_long_add(ki->b_writing, &cache->b_writing);
322 kiocb_start_write(&ki->iocb);
324 get_file(ki->iocb.ki_filp);
325 cachefiles_grab_object(object, cachefiles_obj_get_ioreq);
327 trace_cachefiles_write(object, file_inode(file), ki->iocb.ki_pos, len);
328 old_nofs = memalloc_nofs_save();
329 ret = cachefiles_inject_write_error();
331 ret = vfs_iocb_iter_write(file, &ki->iocb, iter);
332 memalloc_nofs_restore(old_nofs);
338 case -ERESTARTNOINTR:
339 case -ERESTARTNOHAND:
340 case -ERESTART_RESTARTBLOCK:
341 /* There's no easy way to restart the syscall since other AIO's
342 * may be already running. Just fail this IO with EINTR.
347 ki->was_async = false;
348 cachefiles_write_complete(&ki->iocb, ret);
355 cachefiles_put_kiocb(ki);
356 _leave(" = %zd", ret);
360 static int cachefiles_write(struct netfs_cache_resources *cres,
362 struct iov_iter *iter,
363 netfs_io_terminated_t term_func,
364 void *term_func_priv)
366 if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) {
368 term_func(term_func_priv, -ENOBUFS, false);
372 return __cachefiles_write(cachefiles_cres_object(cres),
373 cachefiles_cres_file(cres),
375 term_func, term_func_priv);
378 static inline enum netfs_io_source
379 cachefiles_do_prepare_read(struct netfs_cache_resources *cres,
380 loff_t start, size_t *_len, loff_t i_size,
381 unsigned long *_flags, ino_t netfs_ino)
383 enum cachefiles_prepare_read_trace why;
384 struct cachefiles_object *object = NULL;
385 struct cachefiles_cache *cache;
386 struct fscache_cookie *cookie = fscache_cres_cookie(cres);
387 const struct cred *saved_cred;
388 struct file *file = cachefiles_cres_file(cres);
389 enum netfs_io_source ret = NETFS_DOWNLOAD_FROM_SERVER;
392 ino_t ino = file ? file_inode(file)->i_ino : 0;
395 _enter("%zx @%llx/%llx", len, start, i_size);
397 if (start >= i_size) {
398 ret = NETFS_FILL_WITH_ZEROES;
399 why = cachefiles_trace_read_after_eof;
403 if (test_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags)) {
404 __set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags);
405 why = cachefiles_trace_read_no_data;
406 if (!test_bit(NETFS_SREQ_ONDEMAND, _flags))
410 /* The object and the file may be being created in the background. */
412 why = cachefiles_trace_read_no_file;
413 if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
415 file = cachefiles_cres_file(cres);
418 ino = file_inode(file)->i_ino;
421 object = cachefiles_cres_object(cres);
422 cache = object->volume->cache;
423 cachefiles_begin_secure(cache, &saved_cred);
425 off = cachefiles_inject_read_error();
427 off = vfs_llseek(file, start, SEEK_DATA);
428 if (off < 0 && off >= (loff_t)-MAX_ERRNO) {
429 if (off == (loff_t)-ENXIO) {
430 why = cachefiles_trace_read_seek_nxio;
431 goto download_and_store;
433 trace_cachefiles_io_error(object, file_inode(file), off,
434 cachefiles_trace_seek_error);
435 why = cachefiles_trace_read_seek_error;
439 if (off >= start + len) {
440 why = cachefiles_trace_read_found_hole;
441 goto download_and_store;
445 off = round_up(off, cache->bsize);
448 why = cachefiles_trace_read_found_part;
449 goto download_and_store;
452 to = cachefiles_inject_read_error();
454 to = vfs_llseek(file, start, SEEK_HOLE);
455 if (to < 0 && to >= (loff_t)-MAX_ERRNO) {
456 trace_cachefiles_io_error(object, file_inode(file), to,
457 cachefiles_trace_seek_error);
458 why = cachefiles_trace_read_seek_error;
462 if (to < start + len) {
463 if (start + len >= i_size)
464 to = round_up(to, cache->bsize);
466 to = round_down(to, cache->bsize);
471 why = cachefiles_trace_read_have_data;
472 ret = NETFS_READ_FROM_CACHE;
476 __set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags);
477 if (test_bit(NETFS_SREQ_ONDEMAND, _flags)) {
478 rc = cachefiles_ondemand_read(object, start, len);
480 __clear_bit(NETFS_SREQ_ONDEMAND, _flags);
483 ret = NETFS_INVALID_READ;
486 cachefiles_end_secure(cache, saved_cred);
488 trace_cachefiles_prep_read(object, start, len, *_flags, ret, why, ino, netfs_ino);
493 * Prepare a read operation, shortening it to a cached/uncached
494 * boundary as appropriate.
496 static enum netfs_io_source cachefiles_prepare_read(struct netfs_io_subrequest *subreq,
499 return cachefiles_do_prepare_read(&subreq->rreq->cache_resources,
500 subreq->start, &subreq->len, i_size,
501 &subreq->flags, subreq->rreq->inode->i_ino);
505 * Prepare an on-demand read operation, shortening it to a cached/uncached
506 * boundary as appropriate.
508 static enum netfs_io_source
509 cachefiles_prepare_ondemand_read(struct netfs_cache_resources *cres,
510 loff_t start, size_t *_len, loff_t i_size,
511 unsigned long *_flags, ino_t ino)
513 return cachefiles_do_prepare_read(cres, start, _len, i_size, _flags, ino);
517 * Prepare for a write to occur.
519 int __cachefiles_prepare_write(struct cachefiles_object *object,
521 loff_t *_start, size_t *_len,
522 bool no_space_allocated_yet)
524 struct cachefiles_cache *cache = object->volume->cache;
525 loff_t start = *_start, pos;
526 size_t len = *_len, down;
529 /* Round to DIO size */
530 down = start - round_down(start, PAGE_SIZE);
531 *_start = start - down;
532 *_len = round_up(down + len, PAGE_SIZE);
534 /* We need to work out whether there's sufficient disk space to perform
535 * the write - but we can skip that check if we have space already
538 if (no_space_allocated_yet)
541 pos = cachefiles_inject_read_error();
543 pos = vfs_llseek(file, *_start, SEEK_DATA);
544 if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) {
546 goto check_space; /* Unallocated tail */
547 trace_cachefiles_io_error(object, file_inode(file), pos,
548 cachefiles_trace_seek_error);
551 if ((u64)pos >= (u64)*_start + *_len)
552 goto check_space; /* Unallocated region */
554 /* We have a block that's at least partially filled - if we're low on
555 * space, we need to see if it's fully allocated. If it's not, we may
558 if (cachefiles_has_space(cache, 0, *_len / PAGE_SIZE,
559 cachefiles_has_space_check) == 0)
560 return 0; /* Enough space to simply overwrite the whole block */
562 pos = cachefiles_inject_read_error();
564 pos = vfs_llseek(file, *_start, SEEK_HOLE);
565 if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) {
566 trace_cachefiles_io_error(object, file_inode(file), pos,
567 cachefiles_trace_seek_error);
570 if ((u64)pos >= (u64)*_start + *_len)
571 return 0; /* Fully allocated */
573 /* Partially allocated, but insufficient space: cull. */
574 fscache_count_no_write_space();
575 ret = cachefiles_inject_remove_error();
577 ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
580 trace_cachefiles_io_error(object, file_inode(file), ret,
581 cachefiles_trace_fallocate_error);
582 cachefiles_io_error_obj(object,
583 "CacheFiles: fallocate failed (%d)\n", ret);
590 return cachefiles_has_space(cache, 0, *_len / PAGE_SIZE,
591 cachefiles_has_space_for_write);
594 static int cachefiles_prepare_write(struct netfs_cache_resources *cres,
595 loff_t *_start, size_t *_len, loff_t i_size,
596 bool no_space_allocated_yet)
598 struct cachefiles_object *object = cachefiles_cres_object(cres);
599 struct cachefiles_cache *cache = object->volume->cache;
600 const struct cred *saved_cred;
603 if (!cachefiles_cres_file(cres)) {
604 if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE))
606 if (!cachefiles_cres_file(cres))
610 cachefiles_begin_secure(cache, &saved_cred);
611 ret = __cachefiles_prepare_write(object, cachefiles_cres_file(cres),
613 no_space_allocated_yet);
614 cachefiles_end_secure(cache, saved_cred);
619 * Clean up an operation.
621 static void cachefiles_end_operation(struct netfs_cache_resources *cres)
623 struct file *file = cachefiles_cres_file(cres);
627 fscache_end_cookie_access(fscache_cres_cookie(cres), fscache_access_io_end);
630 static const struct netfs_cache_ops cachefiles_netfs_cache_ops = {
631 .end_operation = cachefiles_end_operation,
632 .read = cachefiles_read,
633 .write = cachefiles_write,
634 .prepare_read = cachefiles_prepare_read,
635 .prepare_write = cachefiles_prepare_write,
636 .prepare_ondemand_read = cachefiles_prepare_ondemand_read,
637 .query_occupancy = cachefiles_query_occupancy,
641 * Open the cache file when beginning a cache operation.
643 bool cachefiles_begin_operation(struct netfs_cache_resources *cres,
644 enum fscache_want_state want_state)
646 struct cachefiles_object *object = cachefiles_cres_object(cres);
648 if (!cachefiles_cres_file(cres)) {
649 cres->ops = &cachefiles_netfs_cache_ops;
651 spin_lock(&object->lock);
652 if (!cres->cache_priv2 && object->file)
653 cres->cache_priv2 = get_file(object->file);
654 spin_unlock(&object->lock);
658 if (!cachefiles_cres_file(cres) && want_state != FSCACHE_WANT_PARAMS) {
659 pr_err("failed to get cres->file\n");