1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (C) 2022, Alibaba Cloud
4 * Copyright (C) 2022, Bytedance Inc. All rights reserved.
6 #include <linux/fscache.h>
9 static DEFINE_MUTEX(erofs_domain_list_lock);
10 static DEFINE_MUTEX(erofs_domain_cookies_lock);
11 static LIST_HEAD(erofs_domain_list);
12 static struct vfsmount *erofs_pseudo_mnt;
14 static struct netfs_io_request *erofs_fscache_alloc_request(struct address_space *mapping,
15 loff_t start, size_t len)
17 struct netfs_io_request *rreq;
19 rreq = kzalloc(sizeof(struct netfs_io_request), GFP_KERNEL);
21 return ERR_PTR(-ENOMEM);
25 rreq->mapping = mapping;
26 rreq->inode = mapping->host;
27 INIT_LIST_HEAD(&rreq->subrequests);
28 refcount_set(&rreq->ref, 1);
32 static void erofs_fscache_put_request(struct netfs_io_request *rreq)
34 if (!refcount_dec_and_test(&rreq->ref))
36 if (rreq->cache_resources.ops)
37 rreq->cache_resources.ops->end_operation(&rreq->cache_resources);
41 static void erofs_fscache_put_subrequest(struct netfs_io_subrequest *subreq)
43 if (!refcount_dec_and_test(&subreq->ref))
45 erofs_fscache_put_request(subreq->rreq);
49 static void erofs_fscache_clear_subrequests(struct netfs_io_request *rreq)
51 struct netfs_io_subrequest *subreq;
53 while (!list_empty(&rreq->subrequests)) {
54 subreq = list_first_entry(&rreq->subrequests,
55 struct netfs_io_subrequest, rreq_link);
56 list_del(&subreq->rreq_link);
57 erofs_fscache_put_subrequest(subreq);
61 static void erofs_fscache_rreq_unlock_folios(struct netfs_io_request *rreq)
63 struct netfs_io_subrequest *subreq;
65 unsigned int iopos = 0;
66 pgoff_t start_page = rreq->start / PAGE_SIZE;
67 pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1;
68 bool subreq_failed = false;
70 XA_STATE(xas, &rreq->mapping->i_pages, start_page);
72 subreq = list_first_entry(&rreq->subrequests,
73 struct netfs_io_subrequest, rreq_link);
74 subreq_failed = (subreq->error < 0);
77 xas_for_each(&xas, folio, last_page) {
79 (folio_index(folio) - start_page) * PAGE_SIZE;
80 unsigned int pgend = pgpos + folio_size(folio);
81 bool pg_failed = false;
89 pg_failed |= subreq_failed;
90 if (pgend < iopos + subreq->len)
94 if (!list_is_last(&subreq->rreq_link,
95 &rreq->subrequests)) {
96 subreq = list_next_entry(subreq, rreq_link);
97 subreq_failed = (subreq->error < 0);
100 subreq_failed = false;
107 folio_mark_uptodate(folio);
114 static void erofs_fscache_rreq_complete(struct netfs_io_request *rreq)
116 erofs_fscache_rreq_unlock_folios(rreq);
117 erofs_fscache_clear_subrequests(rreq);
118 erofs_fscache_put_request(rreq);
121 static void erofc_fscache_subreq_complete(void *priv,
122 ssize_t transferred_or_error, bool was_async)
124 struct netfs_io_subrequest *subreq = priv;
125 struct netfs_io_request *rreq = subreq->rreq;
127 if (IS_ERR_VALUE(transferred_or_error))
128 subreq->error = transferred_or_error;
130 if (atomic_dec_and_test(&rreq->nr_outstanding))
131 erofs_fscache_rreq_complete(rreq);
133 erofs_fscache_put_subrequest(subreq);
137 * Read data from fscache and fill the read data into page cache described by
138 * @rreq, which shall be both aligned with PAGE_SIZE. @pstart describes
139 * the start physical address in the cache file.
141 static int erofs_fscache_read_folios_async(struct fscache_cookie *cookie,
142 struct netfs_io_request *rreq, loff_t pstart)
144 enum netfs_io_source source;
145 struct super_block *sb = rreq->mapping->host->i_sb;
146 struct netfs_io_subrequest *subreq;
147 struct netfs_cache_resources *cres = &rreq->cache_resources;
148 struct iov_iter iter;
149 loff_t start = rreq->start;
150 size_t len = rreq->len;
154 atomic_set(&rreq->nr_outstanding, 1);
156 ret = fscache_begin_read_operation(cres, cookie);
161 subreq = kzalloc(sizeof(struct netfs_io_subrequest),
164 INIT_LIST_HEAD(&subreq->rreq_link);
165 refcount_set(&subreq->ref, 2);
167 refcount_inc(&rreq->ref);
173 subreq->start = pstart + done;
174 subreq->len = len - done;
175 subreq->flags = 1 << NETFS_SREQ_ONDEMAND;
177 list_add_tail(&subreq->rreq_link, &rreq->subrequests);
179 source = cres->ops->prepare_read(subreq, LLONG_MAX);
180 if (WARN_ON(subreq->len == 0))
181 source = NETFS_INVALID_READ;
182 if (source != NETFS_READ_FROM_CACHE) {
183 erofs_err(sb, "failed to fscache prepare_read (source %d)",
187 erofs_fscache_put_subrequest(subreq);
191 atomic_inc(&rreq->nr_outstanding);
193 iov_iter_xarray(&iter, READ, &rreq->mapping->i_pages,
194 start + done, subreq->len);
196 ret = fscache_read(cres, subreq->start, &iter,
197 NETFS_READ_HOLE_FAIL,
198 erofc_fscache_subreq_complete, subreq);
199 if (ret == -EIOCBQUEUED)
202 erofs_err(sb, "failed to fscache_read (ret %d)", ret);
209 if (atomic_dec_and_test(&rreq->nr_outstanding))
210 erofs_fscache_rreq_complete(rreq);
215 static int erofs_fscache_meta_read_folio(struct file *data, struct folio *folio)
218 struct super_block *sb = folio_mapping(folio)->host->i_sb;
219 struct netfs_io_request *rreq;
220 struct erofs_map_dev mdev = {
222 .m_pa = folio_pos(folio),
225 ret = erofs_map_dev(sb, &mdev);
229 rreq = erofs_fscache_alloc_request(folio_mapping(folio),
230 folio_pos(folio), folio_size(folio));
236 return erofs_fscache_read_folios_async(mdev.m_fscache->cookie,
244 * Read into page cache in the range described by (@pos, @len).
246 * On return, the caller is responsible for page unlocking if the output @unlock
247 * is true, or the callee will take this responsibility through netfs_io_request
250 * The return value is the number of bytes successfully handled, or negative
251 * error code on failure. The only exception is that, the length of the range
252 * instead of the error code is returned on failure after netfs_io_request is
253 * allocated, so that .readahead() could advance rac accordingly.
255 static int erofs_fscache_data_read(struct address_space *mapping,
256 loff_t pos, size_t len, bool *unlock)
258 struct inode *inode = mapping->host;
259 struct super_block *sb = inode->i_sb;
260 struct netfs_io_request *rreq;
261 struct erofs_map_blocks map;
262 struct erofs_map_dev mdev;
263 struct iov_iter iter;
270 ret = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);
274 if (map.m_flags & EROFS_MAP_META) {
275 struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
280 /* For tail packing layout, the offset may be non-zero. */
281 offset = erofs_blkoff(map.m_pa);
282 blknr = erofs_blknr(map.m_pa);
285 src = erofs_read_metabuf(&buf, sb, blknr, EROFS_KMAP);
289 iov_iter_xarray(&iter, READ, &mapping->i_pages, pos, PAGE_SIZE);
290 if (copy_to_iter(src + offset, size, &iter) != size) {
291 erofs_put_metabuf(&buf);
294 iov_iter_zero(PAGE_SIZE - size, &iter);
295 erofs_put_metabuf(&buf);
299 if (!(map.m_flags & EROFS_MAP_MAPPED)) {
301 iov_iter_xarray(&iter, READ, &mapping->i_pages, pos, count);
302 iov_iter_zero(count, &iter);
306 count = min_t(size_t, map.m_llen - (pos - map.m_la), len);
307 DBG_BUGON(!count || count % PAGE_SIZE);
309 mdev = (struct erofs_map_dev) {
310 .m_deviceid = map.m_deviceid,
313 ret = erofs_map_dev(sb, &mdev);
317 rreq = erofs_fscache_alloc_request(mapping, pos, count);
319 return PTR_ERR(rreq);
322 erofs_fscache_read_folios_async(mdev.m_fscache->cookie,
323 rreq, mdev.m_pa + (pos - map.m_la));
327 static int erofs_fscache_read_folio(struct file *file, struct folio *folio)
332 DBG_BUGON(folio_size(folio) != EROFS_BLKSIZ);
334 ret = erofs_fscache_data_read(folio_mapping(folio), folio_pos(folio),
335 folio_size(folio), &unlock);
338 folio_mark_uptodate(folio);
341 return ret < 0 ? ret : 0;
344 static void erofs_fscache_readahead(struct readahead_control *rac)
347 size_t len, done = 0;
352 if (!readahead_count(rac))
355 start = readahead_pos(rac);
356 len = readahead_length(rac);
360 ret = erofs_fscache_data_read(rac->mapping, pos,
361 len - done, &unlock);
367 folio = readahead_folio(rac);
368 size -= folio_size(folio);
370 folio_mark_uptodate(folio);
374 } while ((done += ret) < len);
377 static const struct address_space_operations erofs_fscache_meta_aops = {
378 .read_folio = erofs_fscache_meta_read_folio,
381 const struct address_space_operations erofs_fscache_access_aops = {
382 .read_folio = erofs_fscache_read_folio,
383 .readahead = erofs_fscache_readahead,
386 static void erofs_fscache_domain_put(struct erofs_domain *domain)
390 mutex_lock(&erofs_domain_list_lock);
391 if (refcount_dec_and_test(&domain->ref)) {
392 list_del(&domain->list);
393 if (list_empty(&erofs_domain_list)) {
394 kern_unmount(erofs_pseudo_mnt);
395 erofs_pseudo_mnt = NULL;
397 mutex_unlock(&erofs_domain_list_lock);
398 fscache_relinquish_volume(domain->volume, NULL, false);
399 kfree(domain->domain_id);
403 mutex_unlock(&erofs_domain_list_lock);
406 static int erofs_fscache_register_volume(struct super_block *sb)
408 struct erofs_sb_info *sbi = EROFS_SB(sb);
409 char *domain_id = sbi->domain_id;
410 struct fscache_volume *volume;
414 name = kasprintf(GFP_KERNEL, "erofs,%s",
415 domain_id ? domain_id : sbi->fsid);
419 volume = fscache_acquire_volume(name, NULL, NULL, 0);
420 if (IS_ERR_OR_NULL(volume)) {
421 erofs_err(sb, "failed to register volume for %s", name);
422 ret = volume ? PTR_ERR(volume) : -EOPNOTSUPP;
426 sbi->volume = volume;
431 static int erofs_fscache_init_domain(struct super_block *sb)
434 struct erofs_domain *domain;
435 struct erofs_sb_info *sbi = EROFS_SB(sb);
437 domain = kzalloc(sizeof(struct erofs_domain), GFP_KERNEL);
441 domain->domain_id = kstrdup(sbi->domain_id, GFP_KERNEL);
442 if (!domain->domain_id) {
447 err = erofs_fscache_register_volume(sb);
451 if (!erofs_pseudo_mnt) {
452 erofs_pseudo_mnt = kern_mount(&erofs_fs_type);
453 if (IS_ERR(erofs_pseudo_mnt)) {
454 err = PTR_ERR(erofs_pseudo_mnt);
459 domain->volume = sbi->volume;
460 refcount_set(&domain->ref, 1);
461 list_add(&domain->list, &erofs_domain_list);
462 sbi->domain = domain;
465 kfree(domain->domain_id);
470 static int erofs_fscache_register_domain(struct super_block *sb)
473 struct erofs_domain *domain;
474 struct erofs_sb_info *sbi = EROFS_SB(sb);
476 mutex_lock(&erofs_domain_list_lock);
477 list_for_each_entry(domain, &erofs_domain_list, list) {
478 if (!strcmp(domain->domain_id, sbi->domain_id)) {
479 sbi->domain = domain;
480 sbi->volume = domain->volume;
481 refcount_inc(&domain->ref);
482 mutex_unlock(&erofs_domain_list_lock);
486 err = erofs_fscache_init_domain(sb);
487 mutex_unlock(&erofs_domain_list_lock);
492 struct erofs_fscache *erofs_fscache_acquire_cookie(struct super_block *sb,
493 char *name, bool need_inode)
495 struct fscache_volume *volume = EROFS_SB(sb)->volume;
496 struct erofs_fscache *ctx;
497 struct fscache_cookie *cookie;
500 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
502 return ERR_PTR(-ENOMEM);
504 cookie = fscache_acquire_cookie(volume, FSCACHE_ADV_WANT_CACHE_SIZE,
505 name, strlen(name), NULL, 0, 0);
507 erofs_err(sb, "failed to get cookie for %s", name);
512 fscache_use_cookie(cookie, false);
513 ctx->cookie = cookie;
516 struct inode *const inode = new_inode(sb);
519 erofs_err(sb, "failed to get anon inode for %s", name);
525 inode->i_size = OFFSET_MAX;
526 inode->i_mapping->a_ops = &erofs_fscache_meta_aops;
527 mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS);
535 fscache_unuse_cookie(ctx->cookie, NULL, NULL);
536 fscache_relinquish_cookie(ctx->cookie, false);
542 static void erofs_fscache_relinquish_cookie(struct erofs_fscache *ctx)
544 fscache_unuse_cookie(ctx->cookie, NULL, NULL);
545 fscache_relinquish_cookie(ctx->cookie, false);
552 struct erofs_fscache *erofs_fscache_domain_init_cookie(struct super_block *sb,
553 char *name, bool need_inode)
557 struct erofs_fscache *ctx;
558 struct erofs_domain *domain = EROFS_SB(sb)->domain;
560 ctx = erofs_fscache_acquire_cookie(sb, name, need_inode);
564 ctx->name = kstrdup(name, GFP_KERNEL);
570 inode = new_inode(erofs_pseudo_mnt->mnt_sb);
576 ctx->domain = domain;
577 ctx->anon_inode = inode;
578 inode->i_private = ctx;
579 refcount_inc(&domain->ref);
582 erofs_fscache_relinquish_cookie(ctx);
587 struct erofs_fscache *erofs_domain_register_cookie(struct super_block *sb,
588 char *name, bool need_inode)
591 struct erofs_fscache *ctx;
592 struct erofs_domain *domain = EROFS_SB(sb)->domain;
593 struct super_block *psb = erofs_pseudo_mnt->mnt_sb;
595 mutex_lock(&erofs_domain_cookies_lock);
596 spin_lock(&psb->s_inode_list_lock);
597 list_for_each_entry(inode, &psb->s_inodes, i_sb_list) {
598 ctx = inode->i_private;
599 if (!ctx || ctx->domain != domain || strcmp(ctx->name, name))
602 spin_unlock(&psb->s_inode_list_lock);
603 mutex_unlock(&erofs_domain_cookies_lock);
606 spin_unlock(&psb->s_inode_list_lock);
607 ctx = erofs_fscache_domain_init_cookie(sb, name, need_inode);
608 mutex_unlock(&erofs_domain_cookies_lock);
612 struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb,
613 char *name, bool need_inode)
615 if (EROFS_SB(sb)->domain_id)
616 return erofs_domain_register_cookie(sb, name, need_inode);
617 return erofs_fscache_acquire_cookie(sb, name, need_inode);
620 void erofs_fscache_unregister_cookie(struct erofs_fscache *ctx)
623 struct erofs_domain *domain;
627 domain = ctx->domain;
629 mutex_lock(&erofs_domain_cookies_lock);
630 drop = atomic_read(&ctx->anon_inode->i_count) == 1;
631 iput(ctx->anon_inode);
632 mutex_unlock(&erofs_domain_cookies_lock);
637 erofs_fscache_relinquish_cookie(ctx);
638 erofs_fscache_domain_put(domain);
641 int erofs_fscache_register_fs(struct super_block *sb)
644 struct erofs_sb_info *sbi = EROFS_SB(sb);
645 struct erofs_fscache *fscache;
648 ret = erofs_fscache_register_domain(sb);
650 ret = erofs_fscache_register_volume(sb);
654 /* acquired domain/volume will be relinquished in kill_sb() on error */
655 fscache = erofs_fscache_register_cookie(sb, sbi->fsid, true);
657 return PTR_ERR(fscache);
659 sbi->s_fscache = fscache;
663 void erofs_fscache_unregister_fs(struct super_block *sb)
665 struct erofs_sb_info *sbi = EROFS_SB(sb);
667 erofs_fscache_unregister_cookie(sbi->s_fscache);
670 erofs_fscache_domain_put(sbi->domain);
672 fscache_relinquish_volume(sbi->volume, NULL, false);
674 sbi->s_fscache = NULL;