drivers/md/dm-verity.c

   1 /*
   2  * Copyright (C) 2012 Red Hat, Inc.
   3  *
   4  * Author: Mikulas Patocka <mpatocka@redhat.com>
   5  *
   6  * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
   7  *
   8  * This file is released under the GPLv2.
   9  *
  10  * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
  11  * default prefetch value. Data are read in "prefetch_cluster" chunks from the
  12  * hash device. Setting this greatly improves performance when data and hash
  13  * are on the same disk on different partitions on devices with poor random
  14  * access behavior.
  15  */
  16
  17 #include "dm-bufio.h"
  18
  19 #include <linux/module.h>
  20 #include <linux/device-mapper.h>
  21 #include <crypto/hash.h>
  22
  23 #define DM_MSG_PREFIX                   "verity"
  24
  25 #define DM_VERITY_IO_VEC_INLINE         16
  26 #define DM_VERITY_MEMPOOL_SIZE          4
  27 #define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
  28
  29 #define DM_VERITY_MAX_LEVELS            63
  30
  31 static unsigned dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;
  32
  33 module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, S_IRUGO | S_IWUSR);
  34
  35 struct dm_verity {
  36         struct dm_dev *data_dev;
  37         struct dm_dev *hash_dev;
  38         struct dm_target *ti;
  39         struct dm_bufio_client *bufio;
  40         char *alg_name;
  41         struct crypto_shash *tfm;
  42         u8 *root_digest;        /* digest of the root block */
  43         u8 *salt;               /* salt: its size is salt_size */
  44         unsigned salt_size;
  45         sector_t data_start;    /* data offset in 512-byte sectors */
  46         sector_t hash_start;    /* hash start in blocks */
  47         sector_t data_blocks;   /* the number of data blocks */
  48         sector_t hash_blocks;   /* the number of hash blocks */
  49         unsigned char data_dev_block_bits;      /* log2(data blocksize) */
  50         unsigned char hash_dev_block_bits;      /* log2(hash blocksize) */
  51         unsigned char hash_per_block_bits;      /* log2(hashes in hash block) */
  52         unsigned char levels;   /* the number of tree levels */
  53         unsigned char version;
  54         unsigned digest_size;   /* digest size for the current hash algorithm */
  55         unsigned shash_descsize;/* the size of temporary space for crypto */
  56         int hash_failed;        /* set to 1 if hash of any block failed */
  57
  58         mempool_t *vec_mempool; /* mempool of bio vector */
  59
  60         struct workqueue_struct *verify_wq;
  61
  62         /* starting blocks for each tree level. 0 is the lowest level. */
  63         sector_t hash_level_block[DM_VERITY_MAX_LEVELS];
  64 };
  65
  66 struct dm_verity_io {
  67         struct dm_verity *v;
  68
  69         /* original values of bio->bi_end_io and bio->bi_private */
  70         bio_end_io_t *orig_bi_end_io;
  71         void *orig_bi_private;
  72
  73         sector_t block;
  74         unsigned n_blocks;
  75
  76         /* saved bio vector */
  77         struct bio_vec *io_vec;
  78         unsigned io_vec_size;
  79
  80         struct work_struct work;
  81
  82         /* A space for short vectors; longer vectors are allocated separately. */
  83         struct bio_vec io_vec_inline[DM_VERITY_IO_VEC_INLINE];
  84
  85         /*
  86          * Three variably-size fields follow this struct:
  87          *
  88          * u8 hash_desc[v->shash_descsize];
  89          * u8 real_digest[v->digest_size];
  90          * u8 want_digest[v->digest_size];
  91          *
  92          * To access them use: io_hash_desc(), io_real_digest() and io_want_digest().
  93          */
  94 };
  95
  96 struct dm_verity_prefetch_work {
  97         struct work_struct work;
  98         struct dm_verity *v;
  99         sector_t block;
 100         unsigned n_blocks;
 101 };
 102
 103 static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io)
 104 {
 105         return (struct shash_desc *)(io + 1);
 106 }
 107
 108 static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io)
 109 {
 110         return (u8 *)(io + 1) + v->shash_descsize;
 111 }
 112
 113 static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io)
 114 {
 115         return (u8 *)(io + 1) + v->shash_descsize + v->digest_size;
 116 }
 117
 118 /*
 119  * Auxiliary structure appended to each dm-bufio buffer. If the value
 120  * hash_verified is nonzero, hash of the block has been verified.
 121  *
 122  * The variable hash_verified is set to 0 when allocating the buffer, then
 123  * it can be changed to 1 and it is never reset to 0 again.
 124  *
 125  * There is no lock around this value, a race condition can at worst cause
 126  * that multiple processes verify the hash of the same buffer simultaneously
 127  * and write 1 to hash_verified simultaneously.
 128  * This condition is harmless, so we don't need locking.
 129  */
 130 struct buffer_aux {
 131         int hash_verified;
 132 };
 133
 134 /*
 135  * Initialize struct buffer_aux for a freshly created buffer.
 136  */
 137 static void dm_bufio_alloc_callback(struct dm_buffer *buf)
 138 {
 139         struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
 140
 141         aux->hash_verified = 0;
 142 }
 143
 144 /*
 145  * Translate input sector number to the sector number on the target device.
 146  */
 147 static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)
 148 {
 149         return v->data_start + dm_target_offset(v->ti, bi_sector);
 150 }
 151
 152 /*
 153  * Return hash position of a specified block at a specified tree level
 154  * (0 is the lowest level).
 155  * The lowest "hash_per_block_bits"-bits of the result denote hash position
 156  * inside a hash block. The remaining bits denote location of the hash block.
 157  */
 158 static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
 159                                          int level)
 160 {
 161         return block >> (level * v->hash_per_block_bits);
 162 }
 163
 164 static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
 165                                  sector_t *hash_block, unsigned *offset)
 166 {
 167         sector_t position = verity_position_at_level(v, block, level);
 168         unsigned idx;
 169
 170         *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);
 171
 172         if (!offset)
 173                 return;
 174
 175         idx = position & ((1 << v->hash_per_block_bits) - 1);
 176         if (!v->version)
 177                 *offset = idx * v->digest_size;
 178         else
 179                 *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits);
 180 }
 181
 182 /*
 183  * Verify hash of a metadata block pertaining to the specified data block
 184  * ("block" argument) at a specified level ("level" argument).
 185  *
 186  * On successful return, io_want_digest(v, io) contains the hash value for
 187  * a lower tree level or for the data block (if we're at the lowest leve).
 188  *
 189  * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned.
 190  * If "skip_unverified" is false, unverified buffer is hashed and verified
 191  * against current value of io_want_digest(v, io).
 192  */
 193 static int verity_verify_level(struct dm_verity_io *io, sector_t block,
 194                                int level, bool skip_unverified)
 195 {
 196         struct dm_verity *v = io->v;
 197         struct dm_buffer *buf;
 198         struct buffer_aux *aux;
 199         u8 *data;
 200         int r;
 201         sector_t hash_block;
 202         unsigned offset;
 203
 204         verity_hash_at_level(v, block, level, &hash_block, &offset);
 205
 206         data = dm_bufio_read(v->bufio, hash_block, &buf);
 207         if (unlikely(IS_ERR(data)))
 208                 return PTR_ERR(data);
 209
 210         aux = dm_bufio_get_aux_data(buf);
 211
 212         if (!aux->hash_verified) {
 213                 struct shash_desc *desc;
 214                 u8 *result;
 215
 216                 if (skip_unverified) {
 217                         r = 1;
 218                         goto release_ret_r;
 219                 }
 220
 221                 desc = io_hash_desc(v, io);
 222                 desc->tfm = v->tfm;
 223                 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
 224                 r = crypto_shash_init(desc);
 225                 if (r < 0) {
 226                         DMERR("crypto_shash_init failed: %d", r);
 227                         goto release_ret_r;
 228                 }
 229
 230                 if (likely(v->version >= 1)) {
 231                         r = crypto_shash_update(desc, v->salt, v->salt_size);
 232                         if (r < 0) {
 233                                 DMERR("crypto_shash_update failed: %d", r);
 234                                 goto release_ret_r;
 235                         }
 236                 }
 237
 238                 r = crypto_shash_update(desc, data, 1 << v->hash_dev_block_bits);
 239                 if (r < 0) {
 240                         DMERR("crypto_shash_update failed: %d", r);
 241                         goto release_ret_r;
 242                 }
 243
 244                 if (!v->version) {
 245                         r = crypto_shash_update(desc, v->salt, v->salt_size);
 246                         if (r < 0) {
 247                                 DMERR("crypto_shash_update failed: %d", r);
 248                                 goto release_ret_r;
 249                         }
 250                 }
 251
 252                 result = io_real_digest(v, io);
 253                 r = crypto_shash_final(desc, result);
 254                 if (r < 0) {
 255                         DMERR("crypto_shash_final failed: %d", r);
 256                         goto release_ret_r;
 257                 }
 258                 if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
 259                         DMERR_LIMIT("metadata block %llu is corrupted",
 260                                 (unsigned long long)hash_block);
 261                         v->hash_failed = 1;
 262                         r = -EIO;
 263                         goto release_ret_r;
 264                 } else
 265                         aux->hash_verified = 1;
 266         }
 267
 268         data += offset;
 269
 270         memcpy(io_want_digest(v, io), data, v->digest_size);
 271
 272         dm_bufio_release(buf);
 273         return 0;
 274
 275 release_ret_r:
 276         dm_bufio_release(buf);
 277
 278         return r;
 279 }
 280
 281 /*
 282  * Verify one "dm_verity_io" structure.
 283  */
 284 static int verity_verify_io(struct dm_verity_io *io)
 285 {
 286         struct dm_verity *v = io->v;
 287         unsigned b;
 288         int i;
 289         unsigned vector = 0, offset = 0;
 290
 291         for (b = 0; b < io->n_blocks; b++) {
 292                 struct shash_desc *desc;
 293                 u8 *result;
 294                 int r;
 295                 unsigned todo;
 296
 297                 if (likely(v->levels)) {
 298                         /*
 299                          * First, we try to get the requested hash for
 300                          * the current block. If the hash block itself is
 301                          * verified, zero is returned. If it isn't, this
 302                          * function returns 0 and we fall back to whole
 303                          * chain verification.
 304                          */
 305                         int r = verity_verify_level(io, io->block + b, 0, true);
 306                         if (likely(!r))
 307                                 goto test_block_hash;
 308                         if (r < 0)
 309                                 return r;
 310                 }
 311
 312                 memcpy(io_want_digest(v, io), v->root_digest, v->digest_size);
 313
 314                 for (i = v->levels - 1; i >= 0; i--) {
 315                         int r = verity_verify_level(io, io->block + b, i, false);
 316                         if (unlikely(r))
 317                                 return r;
 318                 }
 319
 320 test_block_hash:
 321                 desc = io_hash_desc(v, io);
 322                 desc->tfm = v->tfm;
 323                 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
 324                 r = crypto_shash_init(desc);
 325                 if (r < 0) {
 326                         DMERR("crypto_shash_init failed: %d", r);
 327                         return r;
 328                 }
 329
 330                 if (likely(v->version >= 1)) {
 331                         r = crypto_shash_update(desc, v->salt, v->salt_size);
 332                         if (r < 0) {
 333                                 DMERR("crypto_shash_update failed: %d", r);
 334                                 return r;
 335                         }
 336                 }
 337
 338                 todo = 1 << v->data_dev_block_bits;
 339                 do {
 340                         struct bio_vec *bv;
 341                         u8 *page;
 342                         unsigned len;
 343
 344                         BUG_ON(vector >= io->io_vec_size);
 345                         bv = &io->io_vec[vector];
 346                         page = kmap_atomic(bv->bv_page);
 347                         len = bv->bv_len - offset;
 348                         if (likely(len >= todo))
 349                                 len = todo;
 350                         r = crypto_shash_update(desc,
 351                                         page + bv->bv_offset + offset, len);
 352                         kunmap_atomic(page);
 353                         if (r < 0) {
 354                                 DMERR("crypto_shash_update failed: %d", r);
 355                                 return r;
 356                         }
 357                         offset += len;
 358                         if (likely(offset == bv->bv_len)) {
 359                                 offset = 0;
 360                                 vector++;
 361                         }
 362                         todo -= len;
 363                 } while (todo);
 364
 365                 if (!v->version) {
 366                         r = crypto_shash_update(desc, v->salt, v->salt_size);
 367                         if (r < 0) {
 368                                 DMERR("crypto_shash_update failed: %d", r);
 369                                 return r;
 370                         }
 371                 }
 372
 373                 result = io_real_digest(v, io);
 374                 r = crypto_shash_final(desc, result);
 375                 if (r < 0) {
 376                         DMERR("crypto_shash_final failed: %d", r);
 377                         return r;
 378                 }
 379                 if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) {
 380                         DMERR_LIMIT("data block %llu is corrupted",
 381                                 (unsigned long long)(io->block + b));
 382                         v->hash_failed = 1;
 383                         return -EIO;
 384                 }
 385         }
 386         BUG_ON(vector != io->io_vec_size);
 387         BUG_ON(offset);
 388
 389         return 0;
 390 }
 391
 392 /*
 393  * End one "io" structure with a given error.
 394  */
 395 static void verity_finish_io(struct dm_verity_io *io, int error)
 396 {
 397         struct dm_verity *v = io->v;
 398         struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_bio_data_size);
 399
 400         bio->bi_end_io = io->orig_bi_end_io;
 401         bio->bi_private = io->orig_bi_private;
 402
 403         if (io->io_vec != io->io_vec_inline)
 404                 mempool_free(io->io_vec, v->vec_mempool);
 405
 406         bio_endio(bio, error);
 407 }
 408
 409 static void verity_work(struct work_struct *w)
 410 {
 411         struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
 412
 413         verity_finish_io(io, verity_verify_io(io));
 414 }
 415
 416 static void verity_end_io(struct bio *bio, int error)
 417 {
 418         struct dm_verity_io *io = bio->bi_private;
 419
 420         if (error) {
 421                 verity_finish_io(io, error);
 422                 return;
 423         }
 424
 425         INIT_WORK(&io->work, verity_work);
 426         queue_work(io->v->verify_wq, &io->work);
 427 }
 428
 429 /*
 430  * Prefetch buffers for the specified io.
 431  * The root buffer is not prefetched, it is assumed that it will be cached
 432  * all the time.
 433  */
 434 static void verity_prefetch_io(struct work_struct *work)
 435 {
 436         struct dm_verity_prefetch_work *pw =
 437                 container_of(work, struct dm_verity_prefetch_work, work);
 438         struct dm_verity *v = pw->v;
 439         int i;
 440
 441         for (i = v->levels - 2; i >= 0; i--) {
 442                 sector_t hash_block_start;
 443                 sector_t hash_block_end;
 444                 verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL);
 445                 verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL);
 446                 if (!i) {
 447                         unsigned cluster = ACCESS_ONCE(dm_verity_prefetch_cluster);
 448
 449                         cluster >>= v->data_dev_block_bits;
 450                         if (unlikely(!cluster))
 451                                 goto no_prefetch_cluster;
 452
 453                         if (unlikely(cluster & (cluster - 1)))
 454                                 cluster = 1 << (fls(cluster) - 1);
 455
 456                         hash_block_start &= ~(sector_t)(cluster - 1);
 457                         hash_block_end |= cluster - 1;
 458                         if (unlikely(hash_block_end >= v->hash_blocks))
 459                                 hash_block_end = v->hash_blocks - 1;
 460                 }
 461 no_prefetch_cluster:
 462                 dm_bufio_prefetch(v->bufio, hash_block_start,
 463                                   hash_block_end - hash_block_start + 1);
 464         }
 465
 466         kfree(pw);
 467 }
 468
 469 static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io)
 470 {
 471         struct dm_verity_prefetch_work *pw;
 472
 473         pw = kmalloc(sizeof(struct dm_verity_prefetch_work),
 474                 GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
 475
 476         if (!pw)
 477                 return;
 478
 479         INIT_WORK(&pw->work, verity_prefetch_io);
 480         pw->v = v;
 481         pw->block = io->block;
 482         pw->n_blocks = io->n_blocks;
 483         queue_work(v->verify_wq, &pw->work);
 484 }
 485
 486 /*
 487  * Bio map function. It allocates dm_verity_io structure and bio vector and
 488  * fills them. Then it issues prefetches and the I/O.
 489  */
 490 static int verity_map(struct dm_target *ti, struct bio *bio)
 491 {
 492         struct dm_verity *v = ti->private;
 493         struct dm_verity_io *io;
 494
 495         bio->bi_bdev = v->data_dev->bdev;
 496         bio->bi_sector = verity_map_sector(v, bio->bi_sector);
 497
 498         if (((unsigned)bio->bi_sector | bio_sectors(bio)) &
 499             ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
 500                 DMERR_LIMIT("unaligned io");
 501                 return -EIO;
 502         }
 503
 504         if ((bio->bi_sector + bio_sectors(bio)) >>
 505             (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {
 506                 DMERR_LIMIT("io out of range");
 507                 return -EIO;
 508         }
 509
 510         if (bio_data_dir(bio) == WRITE)
 511                 return -EIO;
 512
 513         io = dm_per_bio_data(bio, ti->per_bio_data_size);
 514         io->v = v;
 515         io->orig_bi_end_io = bio->bi_end_io;
 516         io->orig_bi_private = bio->bi_private;
 517         io->block = bio->bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
 518         io->n_blocks = bio->bi_size >> v->data_dev_block_bits;
 519
 520         bio->bi_end_io = verity_end_io;
 521         bio->bi_private = io;
 522         io->io_vec_size = bio->bi_vcnt - bio->bi_idx;
 523         if (io->io_vec_size < DM_VERITY_IO_VEC_INLINE)
 524                 io->io_vec = io->io_vec_inline;
 525         else
 526                 io->io_vec = mempool_alloc(v->vec_mempool, GFP_NOIO);
 527         memcpy(io->io_vec, bio_iovec(bio),
 528                io->io_vec_size * sizeof(struct bio_vec));
 529
 530         verity_submit_prefetch(v, io);
 531
 532         generic_make_request(bio);
 533
 534         return DM_MAPIO_SUBMITTED;
 535 }
 536
 537 /*
 538  * Status: V (valid) or C (corruption found)
 539  */
 540 static void verity_status(struct dm_target *ti, status_type_t type,
 541                           unsigned status_flags, char *result, unsigned maxlen)
 542 {
 543         struct dm_verity *v = ti->private;
 544         unsigned sz = 0;
 545         unsigned x;
 546
 547         switch (type) {
 548         case STATUSTYPE_INFO:
 549                 DMEMIT("%c", v->hash_failed ? 'C' : 'V');
 550                 break;
 551         case STATUSTYPE_TABLE:
 552                 DMEMIT("%u %s %s %u %u %llu %llu %s ",
 553                         v->version,
 554                         v->data_dev->name,
 555                         v->hash_dev->name,
 556                         1 << v->data_dev_block_bits,
 557                         1 << v->hash_dev_block_bits,
 558                         (unsigned long long)v->data_blocks,
 559                         (unsigned long long)v->hash_start,
 560                         v->alg_name
 561                         );
 562                 for (x = 0; x < v->digest_size; x++)
 563                         DMEMIT("%02x", v->root_digest[x]);
 564                 DMEMIT(" ");
 565                 if (!v->salt_size)
 566                         DMEMIT("-");
 567                 else
 568                         for (x = 0; x < v->salt_size; x++)
 569                                 DMEMIT("%02x", v->salt[x]);
 570                 break;
 571         }
 572 }
 573
 574 static int verity_ioctl(struct dm_target *ti, unsigned cmd,
 575                         unsigned long arg)
 576 {
 577         struct dm_verity *v = ti->private;
 578         int r = 0;
 579
 580         if (v->data_start ||
 581             ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT)
 582                 r = scsi_verify_blk_ioctl(NULL, cmd);
 583
 584         return r ? : __blkdev_driver_ioctl(v->data_dev->bdev, v->data_dev->mode,
 585                                      cmd, arg);
 586 }
 587
 588 static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
 589                         struct bio_vec *biovec, int max_size)
 590 {
 591         struct dm_verity *v = ti->private;
 592         struct request_queue *q = bdev_get_queue(v->data_dev->bdev);
 593
 594         if (!q->merge_bvec_fn)
 595                 return max_size;
 596
 597         bvm->bi_bdev = v->data_dev->bdev;
 598         bvm->bi_sector = verity_map_sector(v, bvm->bi_sector);
 599
 600         return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
 601 }
 602
 603 static int verity_iterate_devices(struct dm_target *ti,
 604                                   iterate_devices_callout_fn fn, void *data)
 605 {
 606         struct dm_verity *v = ti->private;
 607
 608         return fn(ti, v->data_dev, v->data_start, ti->len, data);
 609 }
 610
 611 static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)
 612 {
 613         struct dm_verity *v = ti->private;
 614
 615         if (limits->logical_block_size < 1 << v->data_dev_block_bits)
 616                 limits->logical_block_size = 1 << v->data_dev_block_bits;
 617
 618         if (limits->physical_block_size < 1 << v->data_dev_block_bits)
 619                 limits->physical_block_size = 1 << v->data_dev_block_bits;
 620
 621         blk_limits_io_min(limits, limits->logical_block_size);
 622 }
 623
 624 static void verity_dtr(struct dm_target *ti)
 625 {
 626         struct dm_verity *v = ti->private;
 627
 628         if (v->verify_wq)
 629                 destroy_workqueue(v->verify_wq);
 630
 631         if (v->vec_mempool)
 632                 mempool_destroy(v->vec_mempool);
 633
 634         if (v->bufio)
 635                 dm_bufio_client_destroy(v->bufio);
 636
 637         kfree(v->salt);
 638         kfree(v->root_digest);
 639
 640         if (v->tfm)
 641                 crypto_free_shash(v->tfm);
 642
 643         kfree(v->alg_name);
 644
 645         if (v->hash_dev)
 646                 dm_put_device(ti, v->hash_dev);
 647
 648         if (v->data_dev)
 649                 dm_put_device(ti, v->data_dev);
 650
 651         kfree(v);
 652 }
 653
 654 /*
 655  * Target parameters:
 656  *      <version>       The current format is version 1.
 657  *                      Vsn 0 is compatible with original Chromium OS releases.
 658  *      <data device>
 659  *      <hash device>
 660  *      <data block size>
 661  *      <hash block size>
 662  *      <the number of data blocks>
 663  *      <hash start block>
 664  *      <algorithm>
 665  *      <digest>
 666  *      <salt>          Hex string or "-" if no salt.
 667  */
 668 static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
 669 {
 670         struct dm_verity *v;
 671         unsigned num;
 672         unsigned long long num_ll;
 673         int r;
 674         int i;
 675         sector_t hash_position;
 676         char dummy;
 677
 678         v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
 679         if (!v) {
 680                 ti->error = "Cannot allocate verity structure";
 681                 return -ENOMEM;
 682         }
 683         ti->private = v;
 684         v->ti = ti;
 685
 686         if ((dm_table_get_mode(ti->table) & ~FMODE_READ)) {
 687                 ti->error = "Device must be readonly";
 688                 r = -EINVAL;
 689                 goto bad;
 690         }
 691
 692         if (argc != 10) {
 693                 ti->error = "Invalid argument count: exactly 10 arguments required";
 694                 r = -EINVAL;
 695                 goto bad;
 696         }
 697
 698         if (sscanf(argv[0], "%d%c", &num, &dummy) != 1 ||
 699             num < 0 || num > 1) {
 700                 ti->error = "Invalid version";
 701                 r = -EINVAL;
 702                 goto bad;
 703         }
 704         v->version = num;
 705
 706         r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev);
 707         if (r) {
 708                 ti->error = "Data device lookup failed";
 709                 goto bad;
 710         }
 711
 712         r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev);
 713         if (r) {
 714                 ti->error = "Data device lookup failed";
 715                 goto bad;
 716         }
 717
 718         if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 ||
 719             !num || (num & (num - 1)) ||
 720             num < bdev_logical_block_size(v->data_dev->bdev) ||
 721             num > PAGE_SIZE) {
 722                 ti->error = "Invalid data device block size";
 723                 r = -EINVAL;
 724                 goto bad;
 725         }
 726         v->data_dev_block_bits = ffs(num) - 1;
 727
 728         if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||
 729             !num || (num & (num - 1)) ||
 730             num < bdev_logical_block_size(v->hash_dev->bdev) ||
 731             num > INT_MAX) {
 732                 ti->error = "Invalid hash device block size";
 733                 r = -EINVAL;
 734                 goto bad;
 735         }
 736         v->hash_dev_block_bits = ffs(num) - 1;
 737
 738         if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 ||
 739             (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT))
 740             >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) {
 741                 ti->error = "Invalid data blocks";
 742                 r = -EINVAL;
 743                 goto bad;
 744         }
 745         v->data_blocks = num_ll;
 746
 747         if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) {
 748                 ti->error = "Data device is too small";
 749                 r = -EINVAL;
 750                 goto bad;
 751         }
 752
 753         if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 ||
 754             (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT))
 755             >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) {
 756                 ti->error = "Invalid hash start";
 757                 r = -EINVAL;
 758                 goto bad;
 759         }
 760         v->hash_start = num_ll;
 761
 762         v->alg_name = kstrdup(argv[7], GFP_KERNEL);
 763         if (!v->alg_name) {
 764                 ti->error = "Cannot allocate algorithm name";
 765                 r = -ENOMEM;
 766                 goto bad;
 767         }
 768
 769         v->tfm = crypto_alloc_shash(v->alg_name, 0, 0);
 770         if (IS_ERR(v->tfm)) {
 771                 ti->error = "Cannot initialize hash function";
 772                 r = PTR_ERR(v->tfm);
 773                 v->tfm = NULL;
 774                 goto bad;
 775         }
 776         v->digest_size = crypto_shash_digestsize(v->tfm);
 777         if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
 778                 ti->error = "Digest size too big";
 779                 r = -EINVAL;
 780                 goto bad;
 781         }
 782         v->shash_descsize =
 783                 sizeof(struct shash_desc) + crypto_shash_descsize(v->tfm);
 784
 785         v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
 786         if (!v->root_digest) {
 787                 ti->error = "Cannot allocate root digest";
 788                 r = -ENOMEM;
 789                 goto bad;
 790         }
 791         if (strlen(argv[8]) != v->digest_size * 2 ||
 792             hex2bin(v->root_digest, argv[8], v->digest_size)) {
 793                 ti->error = "Invalid root digest";
 794                 r = -EINVAL;
 795                 goto bad;
 796         }
 797
 798         if (strcmp(argv[9], "-")) {
 799                 v->salt_size = strlen(argv[9]) / 2;
 800                 v->salt = kmalloc(v->salt_size, GFP_KERNEL);
 801                 if (!v->salt) {
 802                         ti->error = "Cannot allocate salt";
 803                         r = -ENOMEM;
 804                         goto bad;
 805                 }
 806                 if (strlen(argv[9]) != v->salt_size * 2 ||
 807                     hex2bin(v->salt, argv[9], v->salt_size)) {
 808                         ti->error = "Invalid salt";
 809                         r = -EINVAL;
 810                         goto bad;
 811                 }
 812         }
 813
 814         v->hash_per_block_bits =
 815                 fls((1 << v->hash_dev_block_bits) / v->digest_size) - 1;
 816
 817         v->levels = 0;
 818         if (v->data_blocks)
 819                 while (v->hash_per_block_bits * v->levels < 64 &&
 820                        (unsigned long long)(v->data_blocks - 1) >>
 821                        (v->hash_per_block_bits * v->levels))
 822                         v->levels++;
 823
 824         if (v->levels > DM_VERITY_MAX_LEVELS) {
 825                 ti->error = "Too many tree levels";
 826                 r = -E2BIG;
 827                 goto bad;
 828         }
 829
 830         hash_position = v->hash_start;
 831         for (i = v->levels - 1; i >= 0; i--) {
 832                 sector_t s;
 833                 v->hash_level_block[i] = hash_position;
 834                 s = verity_position_at_level(v, v->data_blocks, i);
 835                 s = (s >> v->hash_per_block_bits) +
 836                     !!(s & ((1 << v->hash_per_block_bits) - 1));
 837                 if (hash_position + s < hash_position) {
 838                         ti->error = "Hash device offset overflow";
 839                         r = -E2BIG;
 840                         goto bad;
 841                 }
 842                 hash_position += s;
 843         }
 844         v->hash_blocks = hash_position;
 845
 846         v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
 847                 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
 848                 dm_bufio_alloc_callback, NULL);
 849         if (IS_ERR(v->bufio)) {
 850                 ti->error = "Cannot initialize dm-bufio";
 851                 r = PTR_ERR(v->bufio);
 852                 v->bufio = NULL;
 853                 goto bad;
 854         }
 855
 856         if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {
 857                 ti->error = "Hash device is too small";
 858                 r = -E2BIG;
 859                 goto bad;
 860         }
 861
 862         ti->per_bio_data_size = roundup(sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2, __alignof__(struct dm_verity_io));
 863
 864         v->vec_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE,
 865                                         BIO_MAX_PAGES * sizeof(struct bio_vec));
 866         if (!v->vec_mempool) {
 867                 ti->error = "Cannot allocate vector mempool";
 868                 r = -ENOMEM;
 869                 goto bad;
 870         }
 871
 872         /* WQ_UNBOUND greatly improves performance when running on ramdisk */
 873         v->verify_wq = alloc_workqueue("kverityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus());
 874         if (!v->verify_wq) {
 875                 ti->error = "Cannot allocate workqueue";
 876                 r = -ENOMEM;
 877                 goto bad;
 878         }
 879
 880         return 0;
 881
 882 bad:
 883         verity_dtr(ti);
 884
 885         return r;
 886 }
 887
 888 static struct target_type verity_target = {
 889         .name           = "verity",
 890         .version        = {1, 2, 0},
 891         .module         = THIS_MODULE,
 892         .ctr            = verity_ctr,
 893         .dtr            = verity_dtr,
 894         .map            = verity_map,
 895         .status         = verity_status,
 896         .ioctl          = verity_ioctl,
 897         .merge          = verity_merge,
 898         .iterate_devices = verity_iterate_devices,
 899         .io_hints       = verity_io_hints,
 900 };
 901
 902 static int __init dm_verity_init(void)
 903 {
 904         int r;
 905
 906         r = dm_register_target(&verity_target);
 907         if (r < 0)
 908                 DMERR("register failed %d", r);
 909
 910         return r;
 911 }
 912
 913 static void __exit dm_verity_exit(void)
 914 {
 915         dm_unregister_target(&verity_target);
 916 }
 917
 918 module_init(dm_verity_init);
 919 module_exit(dm_verity_exit);
 920
 921 MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
 922 MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");
 923 MODULE_AUTHOR("Will Drewry <wad@chromium.org>");
 924 MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
 925 MODULE_LICENSE("GPL");