ext4: add fs-verity read support

Make ext4_mpage_readpages() verify data as it is read from fs-verity
files, using the helper functions from fs/verity/.

To support both encryption and verity simultaneously, this required
refactoring the decryption workflow into a generic "post-read
processing" workflow which can do decryption, verification, or both.

The case where the ext4 block size is not equal to the PAGE_SIZE is not
supported yet, since in that case ext4_mpage_readpages() sometimes falls
back to block_read_full_page(), which does not support fs-verity yet.

Co-developed-by: Theodore Ts'o <tytso@mit.edu>
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
Signed-off-by: Eric Biggers <ebiggers@google.com>

diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
index 736972f..9c7f403 100644 (file)
--- a/fs/ext4/ext4.h
+++ b/fs/ext4/ext4.h
@@ -3191,6 +3191,8 @@ static inline void ext4_set_de_type(struct super_block *sb,
 extern int ext4_mpage_readpages(struct address_space *mapping,
                                struct list_head *pages, struct page *page,
                                unsigned nr_pages, bool is_readahead);
+extern int __init ext4_init_post_read_processing(void);
+extern void ext4_exit_post_read_processing(void);
 
 /* symlink.c */
 extern const struct inode_operations ext4_encrypted_symlink_inode_operations;

diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index 6de3d4b..cf0fce1 100644 (file)
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -3912,6 +3912,8 @@ static ssize_t ext4_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
        if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode))
                return 0;
 #endif
+       if (fsverity_active(inode))
+               return 0;
 
        /*
         * If we are doing data journalling we don't support O_DIRECT

diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c
index c916017..a30b203 100644 (file)
--- a/fs/ext4/readpage.c
+++ b/fs/ext4/readpage.c
@@ -47,13 +47,103 @@
 
 #include "ext4.h"
 
-static inline bool ext4_bio_encrypted(struct bio *bio)
+#define NUM_PREALLOC_POST_READ_CTXS    128
+
+static struct kmem_cache *bio_post_read_ctx_cache;
+static mempool_t *bio_post_read_ctx_pool;
+
+/* postprocessing steps for read bios */
+enum bio_post_read_step {
+       STEP_INITIAL = 0,
+       STEP_DECRYPT,
+       STEP_VERITY,
+};
+
+struct bio_post_read_ctx {
+       struct bio *bio;
+       struct work_struct work;
+       unsigned int cur_step;
+       unsigned int enabled_steps;
+};
+
+static void __read_end_io(struct bio *bio)
 {
-#ifdef CONFIG_FS_ENCRYPTION
-       return unlikely(bio->bi_private != NULL);
-#else
-       return false;
-#endif
+       struct page *page;
+       struct bio_vec *bv;
+       struct bvec_iter_all iter_all;
+
+       bio_for_each_segment_all(bv, bio, iter_all) {
+               page = bv->bv_page;
+
+               /* PG_error was set if any post_read step failed */
+               if (bio->bi_status || PageError(page)) {
+                       ClearPageUptodate(page);
+                       /* will re-read again later */
+                       ClearPageError(page);
+               } else {
+                       SetPageUptodate(page);
+               }
+               unlock_page(page);
+       }
+       if (bio->bi_private)
+               mempool_free(bio->bi_private, bio_post_read_ctx_pool);
+       bio_put(bio);
+}
+
+static void bio_post_read_processing(struct bio_post_read_ctx *ctx);
+
+static void decrypt_work(struct work_struct *work)
+{
+       struct bio_post_read_ctx *ctx =
+               container_of(work, struct bio_post_read_ctx, work);
+
+       fscrypt_decrypt_bio(ctx->bio);
+
+       bio_post_read_processing(ctx);
+}
+
+static void verity_work(struct work_struct *work)
+{
+       struct bio_post_read_ctx *ctx =
+               container_of(work, struct bio_post_read_ctx, work);
+
+       fsverity_verify_bio(ctx->bio);
+
+       bio_post_read_processing(ctx);
+}
+
+static void bio_post_read_processing(struct bio_post_read_ctx *ctx)
+{
+       /*
+        * We use different work queues for decryption and for verity because
+        * verity may require reading metadata pages that need decryption, and
+        * we shouldn't recurse to the same workqueue.
+        */
+       switch (++ctx->cur_step) {
+       case STEP_DECRYPT:
+               if (ctx->enabled_steps & (1 << STEP_DECRYPT)) {
+                       INIT_WORK(&ctx->work, decrypt_work);
+                       fscrypt_enqueue_decrypt_work(&ctx->work);
+                       return;
+               }
+               ctx->cur_step++;
+               /* fall-through */
+       case STEP_VERITY:
+               if (ctx->enabled_steps & (1 << STEP_VERITY)) {
+                       INIT_WORK(&ctx->work, verity_work);
+                       fsverity_enqueue_verify_work(&ctx->work);
+                       return;
+               }
+               ctx->cur_step++;
+               /* fall-through */
+       default:
+               __read_end_io(ctx->bio);
+       }
+}
+
+static bool bio_post_read_required(struct bio *bio)
+{
+       return bio->bi_private && !bio->bi_status;
 }
 
 /*
@@ -70,30 +160,53 @@ static inline bool ext4_bio_encrypted(struct bio *bio)
  */
 static void mpage_end_io(struct bio *bio)
 {
-       struct bio_vec *bv;
-       struct bvec_iter_all iter_all;
+       if (bio_post_read_required(bio)) {
+               struct bio_post_read_ctx *ctx = bio->bi_private;
 
-       if (ext4_bio_encrypted(bio)) {
-               if (bio->bi_status) {
-                       fscrypt_release_ctx(bio->bi_private);
-               } else {
-                       fscrypt_enqueue_decrypt_bio(bio->bi_private, bio);
-                       return;
-               }
+               ctx->cur_step = STEP_INITIAL;
+               bio_post_read_processing(ctx);
+               return;
        }
-       bio_for_each_segment_all(bv, bio, iter_all) {
-               struct page *page = bv->bv_page;
+       __read_end_io(bio);
+}
 
-               if (!bio->bi_status) {
-                       SetPageUptodate(page);
-               } else {
-                       ClearPageUptodate(page);
-                       SetPageError(page);
-               }
-               unlock_page(page);
+static inline bool ext4_need_verity(const struct inode *inode, pgoff_t idx)
+{
+       return fsverity_active(inode) &&
+              idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
+}
+
+static struct bio_post_read_ctx *get_bio_post_read_ctx(struct inode *inode,
+                                                      struct bio *bio,
+                                                      pgoff_t first_idx)
+{
+       unsigned int post_read_steps = 0;
+       struct bio_post_read_ctx *ctx = NULL;
+
+       if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode))
+               post_read_steps |= 1 << STEP_DECRYPT;
+
+       if (ext4_need_verity(inode, first_idx))
+               post_read_steps |= 1 << STEP_VERITY;
+
+       if (post_read_steps) {
+               ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
+               if (!ctx)
+                       return ERR_PTR(-ENOMEM);
+               ctx->bio = bio;
+               ctx->enabled_steps = post_read_steps;
+               bio->bi_private = ctx;
        }
+       return ctx;
+}
 
-       bio_put(bio);
+static inline loff_t ext4_readpage_limit(struct inode *inode)
+{
+       if (IS_ENABLED(CONFIG_FS_VERITY) &&
+           (IS_VERITY(inode) || ext4_verity_in_progress(inode)))
+               return inode->i_sb->s_maxbytes;
+
+       return i_size_read(inode);
 }
 
 int ext4_mpage_readpages(struct address_space *mapping,
@@ -141,7 +254,8 @@ int ext4_mpage_readpages(struct address_space *mapping,
 
                block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
                last_block = block_in_file + nr_pages * blocks_per_page;
-               last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;
+               last_block_in_file = (ext4_readpage_limit(inode) +
+                                     blocksize - 1) >> blkbits;
                if (last_block > last_block_in_file)
                        last_block = last_block_in_file;
                page_block = 0;
@@ -218,6 +332,9 @@ int ext4_mpage_readpages(struct address_space *mapping,
                        zero_user_segment(page, first_hole << blkbits,
                                          PAGE_SIZE);
                        if (first_hole == 0) {
+                               if (ext4_need_verity(inode, page->index) &&
+                                   !fsverity_verify_page(page))
+                                       goto set_error_page;
                                SetPageUptodate(page);
                                unlock_page(page);
                                goto next_page;
@@ -241,18 +358,16 @@ int ext4_mpage_readpages(struct address_space *mapping,
                        bio = NULL;
                }
                if (bio == NULL) {
-                       struct fscrypt_ctx *ctx = NULL;
+                       struct bio_post_read_ctx *ctx;
 
-                       if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode)) {
-                               ctx = fscrypt_get_ctx(GFP_NOFS);
-                               if (IS_ERR(ctx))
-                                       goto set_error_page;
-                       }
                        bio = bio_alloc(GFP_KERNEL,
                                min_t(int, nr_pages, BIO_MAX_PAGES));
-                       if (!bio) {
-                               if (ctx)
-                                       fscrypt_release_ctx(ctx);
+                       if (!bio)
+                               goto set_error_page;
+                       ctx = get_bio_post_read_ctx(inode, bio, page->index);
+                       if (IS_ERR(ctx)) {
+                               bio_put(bio);
+                               bio = NULL;
                                goto set_error_page;
                        }
                        bio_set_dev(bio, bdev);
@@ -293,3 +408,29 @@ int ext4_mpage_readpages(struct address_space *mapping,
                submit_bio(bio);
        return 0;
 }
+
+int __init ext4_init_post_read_processing(void)
+{
+       bio_post_read_ctx_cache =
+               kmem_cache_create("ext4_bio_post_read_ctx",
+                                 sizeof(struct bio_post_read_ctx), 0, 0, NULL);
+       if (!bio_post_read_ctx_cache)
+               goto fail;
+       bio_post_read_ctx_pool =
+               mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS,
+                                        bio_post_read_ctx_cache);
+       if (!bio_post_read_ctx_pool)
+               goto fail_free_cache;
+       return 0;
+
+fail_free_cache:
+       kmem_cache_destroy(bio_post_read_ctx_cache);
+fail:
+       return -ENOMEM;
+}
+
+void ext4_exit_post_read_processing(void)
+{
+       mempool_destroy(bio_post_read_ctx_pool);
+       kmem_cache_destroy(bio_post_read_ctx_cache);
+}

diff --git a/fs/ext4/super.c b/fs/ext4/super.c
index 05a9874..23e7acd 100644 (file)
--- a/fs/ext4/super.c
+++ b/fs/ext4/super.c
@@ -6104,6 +6104,10 @@ static int __init ext4_init_fs(void)
 
        err = ext4_init_pending();
        if (err)
+               goto out7;
+
+       err = ext4_init_post_read_processing();
+       if (err)
                goto out6;
 
        err = ext4_init_pageio();
@@ -6144,8 +6148,10 @@ out3:
 out4:
        ext4_exit_pageio();
 out5:
-       ext4_exit_pending();
+       ext4_exit_post_read_processing();
 out6:
+       ext4_exit_pending();
+out7:
        ext4_exit_es();
 
        return err;
@@ -6162,6 +6168,7 @@ static void __exit ext4_exit_fs(void)
        ext4_exit_sysfs();
        ext4_exit_system_zone();
        ext4_exit_pageio();
+       ext4_exit_post_read_processing();
        ext4_exit_es();
        ext4_exit_pending();
 }