1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Network filesystem high-level buffered read support.
4 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
8 #include <linux/export.h>
9 #include <linux/task_io_accounting_ops.h>
13 * Unlock the folios in a read operation. We need to set PG_fscache on any
14 * folios we're going to write back before we unlock them.
16 void netfs_rreq_unlock_folios(struct netfs_io_request *rreq)
18 struct netfs_io_subrequest *subreq;
20 unsigned int iopos, account = 0;
21 pgoff_t start_page = rreq->start / PAGE_SIZE;
22 pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1;
23 bool subreq_failed = false;
25 XA_STATE(xas, &rreq->mapping->i_pages, start_page);
27 if (test_bit(NETFS_RREQ_FAILED, &rreq->flags)) {
28 __clear_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags);
29 list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
30 __clear_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags);
34 /* Walk through the pagecache and the I/O request lists simultaneously.
35 * We may have a mixture of cached and uncached sections and we only
36 * really want to write out the uncached sections. This is slightly
37 * complicated by the possibility that we might have huge pages with a
40 subreq = list_first_entry(&rreq->subrequests,
41 struct netfs_io_subrequest, rreq_link);
43 subreq_failed = (subreq->error < 0);
45 trace_netfs_rreq(rreq, netfs_rreq_trace_unlock);
48 xas_for_each(&xas, folio, last_page) {
49 unsigned int pgpos, pgend;
50 bool pg_failed = false;
52 if (xas_retry(&xas, folio))
55 pgpos = (folio_index(folio) - start_page) * PAGE_SIZE;
56 pgend = pgpos + folio_size(folio);
63 if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags))
64 folio_start_fscache(folio);
65 pg_failed |= subreq_failed;
66 if (pgend < iopos + subreq->len)
69 account += subreq->transferred;
71 if (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) {
72 subreq = list_next_entry(subreq, rreq_link);
73 subreq_failed = (subreq->error < 0);
76 subreq_failed = false;
83 flush_dcache_folio(folio);
84 folio_mark_uptodate(folio);
87 if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) {
88 if (folio_index(folio) == rreq->no_unlock_folio &&
89 test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags))
97 task_io_account_read(account);
98 if (rreq->netfs_ops->done)
99 rreq->netfs_ops->done(rreq);
102 static void netfs_cache_expand_readahead(struct netfs_io_request *rreq,
103 loff_t *_start, size_t *_len, loff_t i_size)
105 struct netfs_cache_resources *cres = &rreq->cache_resources;
107 if (cres->ops && cres->ops->expand_readahead)
108 cres->ops->expand_readahead(cres, _start, _len, i_size);
111 static void netfs_rreq_expand(struct netfs_io_request *rreq,
112 struct readahead_control *ractl)
114 /* Give the cache a chance to change the request parameters. The
115 * resultant request must contain the original region.
117 netfs_cache_expand_readahead(rreq, &rreq->start, &rreq->len, rreq->i_size);
119 /* Give the netfs a chance to change the request parameters. The
120 * resultant request must contain the original region.
122 if (rreq->netfs_ops->expand_readahead)
123 rreq->netfs_ops->expand_readahead(rreq);
125 /* Expand the request if the cache wants it to start earlier. Note
126 * that the expansion may get further extended if the VM wishes to
127 * insert THPs and the preferred start and/or end wind up in the middle
130 * If this is the case, however, the THP size should be an integer
131 * multiple of the cache granule size, so we get a whole number of
132 * granules to deal with.
134 if (rreq->start != readahead_pos(ractl) ||
135 rreq->len != readahead_length(ractl)) {
136 readahead_expand(ractl, rreq->start, rreq->len);
137 rreq->start = readahead_pos(ractl);
138 rreq->len = readahead_length(ractl);
140 trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl),
141 netfs_read_trace_expanded);
146 * netfs_readahead - Helper to manage a read request
147 * @ractl: The description of the readahead request
149 * Fulfil a readahead request by drawing data from the cache if possible, or
150 * the netfs if not. Space beyond the EOF is zero-filled. Multiple I/O
151 * requests from different sources will get munged together. If necessary, the
152 * readahead window can be expanded in either direction to a more convenient
153 * alighment for RPC efficiency or to make storage in the cache feasible.
155 * The calling netfs must initialise a netfs context contiguous to the vfs
156 * inode before calling this.
158 * This is usable whether or not caching is enabled.
160 void netfs_readahead(struct readahead_control *ractl)
162 struct netfs_io_request *rreq;
163 struct netfs_inode *ctx = netfs_inode(ractl->mapping->host);
166 _enter("%lx,%x", readahead_index(ractl), readahead_count(ractl));
168 if (readahead_count(ractl) == 0)
171 rreq = netfs_alloc_request(ractl->mapping, ractl->file,
172 readahead_pos(ractl),
173 readahead_length(ractl),
178 if (ctx->ops->begin_cache_operation) {
179 ret = ctx->ops->begin_cache_operation(rreq);
180 if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
184 netfs_stat(&netfs_n_rh_readahead);
185 trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl),
186 netfs_read_trace_readahead);
188 netfs_rreq_expand(rreq, ractl);
190 /* Drop the refs on the folios here rather than in the cache or
191 * filesystem. The locks will be dropped in netfs_rreq_unlock().
193 while (readahead_folio(ractl))
196 netfs_begin_read(rreq, false);
200 netfs_put_request(rreq, false, netfs_rreq_trace_put_failed);
203 EXPORT_SYMBOL(netfs_readahead);
206 * netfs_read_folio - Helper to manage a read_folio request
207 * @file: The file to read from
208 * @folio: The folio to read
210 * Fulfil a read_folio request by drawing data from the cache if
211 * possible, or the netfs if not. Space beyond the EOF is zero-filled.
212 * Multiple I/O requests from different sources will get munged together.
214 * The calling netfs must initialise a netfs context contiguous to the vfs
215 * inode before calling this.
217 * This is usable whether or not caching is enabled.
219 int netfs_read_folio(struct file *file, struct folio *folio)
221 struct address_space *mapping = folio_file_mapping(folio);
222 struct netfs_io_request *rreq;
223 struct netfs_inode *ctx = netfs_inode(mapping->host);
226 _enter("%lx", folio_index(folio));
228 rreq = netfs_alloc_request(mapping, file,
229 folio_file_pos(folio), folio_size(folio),
236 if (ctx->ops->begin_cache_operation) {
237 ret = ctx->ops->begin_cache_operation(rreq);
238 if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
242 netfs_stat(&netfs_n_rh_readpage);
243 trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_readpage);
244 return netfs_begin_read(rreq, true);
247 netfs_put_request(rreq, false, netfs_rreq_trace_put_discard);
252 EXPORT_SYMBOL(netfs_read_folio);
255 * Prepare a folio for writing without reading first
256 * @folio: The folio being prepared
257 * @pos: starting position for the write
258 * @len: length of write
259 * @always_fill: T if the folio should always be completely filled/cleared
261 * In some cases, write_begin doesn't need to read at all:
263 * - write that lies in a folio that is completely beyond EOF
264 * - write that covers the folio from start to EOF or beyond it
266 * If any of these criteria are met, then zero out the unwritten parts
267 * of the folio and return true. Otherwise, return false.
269 static bool netfs_skip_folio_read(struct folio *folio, loff_t pos, size_t len,
272 struct inode *inode = folio_inode(folio);
273 loff_t i_size = i_size_read(inode);
274 size_t offset = offset_in_folio(folio, pos);
275 size_t plen = folio_size(folio);
277 if (unlikely(always_fill)) {
278 if (pos - offset + len <= i_size)
279 return false; /* Page entirely before EOF */
280 zero_user_segment(&folio->page, 0, plen);
281 folio_mark_uptodate(folio);
285 /* Full folio write */
286 if (offset == 0 && len >= plen)
289 /* Page entirely beyond the end of the file */
290 if (pos - offset >= i_size)
293 /* Write that covers from the start of the folio to EOF or beyond */
294 if (offset == 0 && (pos + len) >= i_size)
299 zero_user_segments(&folio->page, 0, offset, offset + len, plen);
304 * netfs_write_begin - Helper to prepare for writing
305 * @ctx: The netfs context
306 * @file: The file to read from
307 * @mapping: The mapping to read from
308 * @pos: File position at which the write will begin
309 * @len: The length of the write (may extend beyond the end of the folio chosen)
310 * @_folio: Where to put the resultant folio
311 * @_fsdata: Place for the netfs to store a cookie
313 * Pre-read data for a write-begin request by drawing data from the cache if
314 * possible, or the netfs if not. Space beyond the EOF is zero-filled.
315 * Multiple I/O requests from different sources will get munged together. If
316 * necessary, the readahead window can be expanded in either direction to a
317 * more convenient alighment for RPC efficiency or to make storage in the cache
320 * The calling netfs must provide a table of operations, only one of which,
321 * issue_op, is mandatory.
323 * The check_write_begin() operation can be provided to check for and flush
324 * conflicting writes once the folio is grabbed and locked. It is passed a
325 * pointer to the fsdata cookie that gets returned to the VM to be passed to
326 * write_end. It is permitted to sleep. It should return 0 if the request
327 * should go ahead or it may return an error. It may also unlock and put the
328 * folio, provided it sets ``*foliop`` to NULL, in which case a return of 0
329 * will cause the folio to be re-got and the process to be retried.
331 * The calling netfs must initialise a netfs context contiguous to the vfs
332 * inode before calling this.
334 * This is usable whether or not caching is enabled.
336 int netfs_write_begin(struct netfs_inode *ctx,
337 struct file *file, struct address_space *mapping,
338 loff_t pos, unsigned int len, struct folio **_folio,
341 struct netfs_io_request *rreq;
343 unsigned int fgp_flags = FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_STABLE;
344 pgoff_t index = pos >> PAGE_SHIFT;
347 DEFINE_READAHEAD(ractl, file, NULL, mapping, index);
350 folio = __filemap_get_folio(mapping, index, fgp_flags,
351 mapping_gfp_mask(mapping));
355 if (ctx->ops->check_write_begin) {
356 /* Allow the netfs (eg. ceph) to flush conflicts. */
357 ret = ctx->ops->check_write_begin(file, pos, len, &folio, _fsdata);
359 trace_netfs_failure(NULL, NULL, ret, netfs_fail_check_write_begin);
366 if (folio_test_uptodate(folio))
369 /* If the page is beyond the EOF, we want to clear it - unless it's
370 * within the cache granule containing the EOF, in which case we need
371 * to preload the granule.
373 if (!netfs_is_cache_enabled(ctx) &&
374 netfs_skip_folio_read(folio, pos, len, false)) {
375 netfs_stat(&netfs_n_rh_write_zskip);
376 goto have_folio_no_wait;
379 rreq = netfs_alloc_request(mapping, file,
380 folio_file_pos(folio), folio_size(folio),
381 NETFS_READ_FOR_WRITE);
386 rreq->no_unlock_folio = folio_index(folio);
387 __set_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags);
389 if (ctx->ops->begin_cache_operation) {
390 ret = ctx->ops->begin_cache_operation(rreq);
391 if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
395 netfs_stat(&netfs_n_rh_write_begin);
396 trace_netfs_read(rreq, pos, len, netfs_read_trace_write_begin);
398 /* Expand the request to meet caching requirements and download
401 ractl._nr_pages = folio_nr_pages(folio);
402 netfs_rreq_expand(rreq, &ractl);
404 /* We hold the folio locks, so we can drop the references */
406 while (readahead_folio(&ractl))
409 ret = netfs_begin_read(rreq, true);
414 ret = folio_wait_fscache_killable(folio);
423 netfs_put_request(rreq, false, netfs_rreq_trace_put_failed);
429 _leave(" = %d", ret);
432 EXPORT_SYMBOL(netfs_write_begin);