drivers/target/target_core_iblock.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*******************************************************************************
   3  * Filename:  target_core_iblock.c
   4  *
   5  * This file contains the Storage Engine  <-> Linux BlockIO transport
   6  * specific functions.
   7  *
   8  * (c) Copyright 2003-2013 Datera, Inc.
   9  *
  10  * Nicholas A. Bellinger <nab@kernel.org>
  11  *
  12  ******************************************************************************/
  13
  14 #include <linux/string.h>
  15 #include <linux/parser.h>
  16 #include <linux/timer.h>
  17 #include <linux/fs.h>
  18 #include <linux/blkdev.h>
  19 #include <linux/blk-integrity.h>
  20 #include <linux/slab.h>
  21 #include <linux/spinlock.h>
  22 #include <linux/bio.h>
  23 #include <linux/file.h>
  24 #include <linux/module.h>
  25 #include <linux/scatterlist.h>
  26 #include <linux/pr.h>
  27 #include <scsi/scsi_proto.h>
  28 #include <scsi/scsi_common.h>
  29 #include <asm/unaligned.h>
  30
  31 #include <target/target_core_base.h>
  32 #include <target/target_core_backend.h>
  33
  34 #include "target_core_iblock.h"
  35 #include "target_core_pr.h"
  36
  37 #define IBLOCK_MAX_BIO_PER_TASK  32     /* max # of bios to submit at a time */
  38 #define IBLOCK_BIO_POOL_SIZE    128
  39
  40 static inline struct iblock_dev *IBLOCK_DEV(struct se_device *dev)
  41 {
  42         return container_of(dev, struct iblock_dev, dev);
  43 }
  44
  45
  46 static int iblock_attach_hba(struct se_hba *hba, u32 host_id)
  47 {
  48         pr_debug("CORE_HBA[%d] - TCM iBlock HBA Driver %s on"
  49                 " Generic Target Core Stack %s\n", hba->hba_id,
  50                 IBLOCK_VERSION, TARGET_CORE_VERSION);
  51         return 0;
  52 }
  53
  54 static void iblock_detach_hba(struct se_hba *hba)
  55 {
  56 }
  57
  58 static struct se_device *iblock_alloc_device(struct se_hba *hba, const char *name)
  59 {
  60         struct iblock_dev *ib_dev = NULL;
  61
  62         ib_dev = kzalloc(sizeof(struct iblock_dev), GFP_KERNEL);
  63         if (!ib_dev) {
  64                 pr_err("Unable to allocate struct iblock_dev\n");
  65                 return NULL;
  66         }
  67
  68         ib_dev->ibd_plug = kcalloc(nr_cpu_ids, sizeof(*ib_dev->ibd_plug),
  69                                    GFP_KERNEL);
  70         if (!ib_dev->ibd_plug)
  71                 goto free_dev;
  72
  73         pr_debug( "IBLOCK: Allocated ib_dev for %s\n", name);
  74
  75         return &ib_dev->dev;
  76
  77 free_dev:
  78         kfree(ib_dev);
  79         return NULL;
  80 }
  81
  82 static bool iblock_configure_unmap(struct se_device *dev)
  83 {
  84         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
  85
  86         return target_configure_unmap_from_queue(&dev->dev_attrib,
  87                                                  ib_dev->ibd_bd);
  88 }
  89
  90 static int iblock_configure_device(struct se_device *dev)
  91 {
  92         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
  93         struct request_queue *q;
  94         struct block_device *bd = NULL;
  95         struct blk_integrity *bi;
  96         blk_mode_t mode = BLK_OPEN_READ;
  97         unsigned int max_write_zeroes_sectors;
  98         int ret;
  99
 100         if (!(ib_dev->ibd_flags & IBDF_HAS_UDEV_PATH)) {
 101                 pr_err("Missing udev_path= parameters for IBLOCK\n");
 102                 return -EINVAL;
 103         }
 104
 105         ret = bioset_init(&ib_dev->ibd_bio_set, IBLOCK_BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
 106         if (ret) {
 107                 pr_err("IBLOCK: Unable to create bioset\n");
 108                 goto out;
 109         }
 110
 111         pr_debug( "IBLOCK: Claiming struct block_device: %s\n",
 112                         ib_dev->ibd_udev_path);
 113
 114         if (!ib_dev->ibd_readonly)
 115                 mode |= BLK_OPEN_WRITE;
 116         else
 117                 dev->dev_flags |= DF_READ_ONLY;
 118
 119         bd = blkdev_get_by_path(ib_dev->ibd_udev_path, mode, ib_dev, NULL);
 120         if (IS_ERR(bd)) {
 121                 ret = PTR_ERR(bd);
 122                 goto out_free_bioset;
 123         }
 124         ib_dev->ibd_bd = bd;
 125
 126         q = bdev_get_queue(bd);
 127
 128         dev->dev_attrib.hw_block_size = bdev_logical_block_size(bd);
 129         dev->dev_attrib.hw_max_sectors = mult_frac(queue_max_hw_sectors(q),
 130                         SECTOR_SIZE,
 131                         dev->dev_attrib.hw_block_size);
 132         dev->dev_attrib.hw_queue_depth = q->nr_requests;
 133
 134         /*
 135          * Enable write same emulation for IBLOCK and use 0xFFFF as
 136          * the smaller WRITE_SAME(10) only has a two-byte block count.
 137          */
 138         max_write_zeroes_sectors = bdev_write_zeroes_sectors(bd);
 139         if (max_write_zeroes_sectors)
 140                 dev->dev_attrib.max_write_same_len = max_write_zeroes_sectors;
 141         else
 142                 dev->dev_attrib.max_write_same_len = 0xFFFF;
 143
 144         if (bdev_nonrot(bd))
 145                 dev->dev_attrib.is_nonrot = 1;
 146
 147         bi = bdev_get_integrity(bd);
 148         if (bi) {
 149                 struct bio_set *bs = &ib_dev->ibd_bio_set;
 150
 151                 if (!strcmp(bi->profile->name, "T10-DIF-TYPE3-IP") ||
 152                     !strcmp(bi->profile->name, "T10-DIF-TYPE1-IP")) {
 153                         pr_err("IBLOCK export of blk_integrity: %s not"
 154                                " supported\n", bi->profile->name);
 155                         ret = -ENOSYS;
 156                         goto out_blkdev_put;
 157                 }
 158
 159                 if (!strcmp(bi->profile->name, "T10-DIF-TYPE3-CRC")) {
 160                         dev->dev_attrib.pi_prot_type = TARGET_DIF_TYPE3_PROT;
 161                 } else if (!strcmp(bi->profile->name, "T10-DIF-TYPE1-CRC")) {
 162                         dev->dev_attrib.pi_prot_type = TARGET_DIF_TYPE1_PROT;
 163                 }
 164
 165                 if (dev->dev_attrib.pi_prot_type) {
 166                         if (bioset_integrity_create(bs, IBLOCK_BIO_POOL_SIZE) < 0) {
 167                                 pr_err("Unable to allocate bioset for PI\n");
 168                                 ret = -ENOMEM;
 169                                 goto out_blkdev_put;
 170                         }
 171                         pr_debug("IBLOCK setup BIP bs->bio_integrity_pool: %p\n",
 172                                  &bs->bio_integrity_pool);
 173                 }
 174                 dev->dev_attrib.hw_pi_prot_type = dev->dev_attrib.pi_prot_type;
 175         }
 176
 177         return 0;
 178
 179 out_blkdev_put:
 180         blkdev_put(ib_dev->ibd_bd, ib_dev);
 181 out_free_bioset:
 182         bioset_exit(&ib_dev->ibd_bio_set);
 183 out:
 184         return ret;
 185 }
 186
 187 static void iblock_dev_call_rcu(struct rcu_head *p)
 188 {
 189         struct se_device *dev = container_of(p, struct se_device, rcu_head);
 190         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 191
 192         kfree(ib_dev->ibd_plug);
 193         kfree(ib_dev);
 194 }
 195
 196 static void iblock_free_device(struct se_device *dev)
 197 {
 198         call_rcu(&dev->rcu_head, iblock_dev_call_rcu);
 199 }
 200
 201 static void iblock_destroy_device(struct se_device *dev)
 202 {
 203         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 204
 205         if (ib_dev->ibd_bd != NULL)
 206                 blkdev_put(ib_dev->ibd_bd, ib_dev);
 207         bioset_exit(&ib_dev->ibd_bio_set);
 208 }
 209
 210 static struct se_dev_plug *iblock_plug_device(struct se_device *se_dev)
 211 {
 212         struct iblock_dev *ib_dev = IBLOCK_DEV(se_dev);
 213         struct iblock_dev_plug *ib_dev_plug;
 214
 215         /*
 216          * Each se_device has a per cpu work this can be run from. We
 217          * shouldn't have multiple threads on the same cpu calling this
 218          * at the same time.
 219          */
 220         ib_dev_plug = &ib_dev->ibd_plug[raw_smp_processor_id()];
 221         if (test_and_set_bit(IBD_PLUGF_PLUGGED, &ib_dev_plug->flags))
 222                 return NULL;
 223
 224         blk_start_plug(&ib_dev_plug->blk_plug);
 225         return &ib_dev_plug->se_plug;
 226 }
 227
 228 static void iblock_unplug_device(struct se_dev_plug *se_plug)
 229 {
 230         struct iblock_dev_plug *ib_dev_plug = container_of(se_plug,
 231                                         struct iblock_dev_plug, se_plug);
 232
 233         blk_finish_plug(&ib_dev_plug->blk_plug);
 234         clear_bit(IBD_PLUGF_PLUGGED, &ib_dev_plug->flags);
 235 }
 236
 237 static sector_t iblock_get_blocks(struct se_device *dev)
 238 {
 239         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 240         u32 block_size = bdev_logical_block_size(ib_dev->ibd_bd);
 241         unsigned long long blocks_long =
 242                 div_u64(bdev_nr_bytes(ib_dev->ibd_bd), block_size) - 1;
 243
 244         if (block_size == dev->dev_attrib.block_size)
 245                 return blocks_long;
 246
 247         switch (block_size) {
 248         case 4096:
 249                 switch (dev->dev_attrib.block_size) {
 250                 case 2048:
 251                         blocks_long <<= 1;
 252                         break;
 253                 case 1024:
 254                         blocks_long <<= 2;
 255                         break;
 256                 case 512:
 257                         blocks_long <<= 3;
 258                         break;
 259                 default:
 260                         break;
 261                 }
 262                 break;
 263         case 2048:
 264                 switch (dev->dev_attrib.block_size) {
 265                 case 4096:
 266                         blocks_long >>= 1;
 267                         break;
 268                 case 1024:
 269                         blocks_long <<= 1;
 270                         break;
 271                 case 512:
 272                         blocks_long <<= 2;
 273                         break;
 274                 default:
 275                         break;
 276                 }
 277                 break;
 278         case 1024:
 279                 switch (dev->dev_attrib.block_size) {
 280                 case 4096:
 281                         blocks_long >>= 2;
 282                         break;
 283                 case 2048:
 284                         blocks_long >>= 1;
 285                         break;
 286                 case 512:
 287                         blocks_long <<= 1;
 288                         break;
 289                 default:
 290                         break;
 291                 }
 292                 break;
 293         case 512:
 294                 switch (dev->dev_attrib.block_size) {
 295                 case 4096:
 296                         blocks_long >>= 3;
 297                         break;
 298                 case 2048:
 299                         blocks_long >>= 2;
 300                         break;
 301                 case 1024:
 302                         blocks_long >>= 1;
 303                         break;
 304                 default:
 305                         break;
 306                 }
 307                 break;
 308         default:
 309                 break;
 310         }
 311
 312         return blocks_long;
 313 }
 314
 315 static void iblock_complete_cmd(struct se_cmd *cmd, blk_status_t blk_status)
 316 {
 317         struct iblock_req *ibr = cmd->priv;
 318         u8 status;
 319
 320         if (!refcount_dec_and_test(&ibr->pending))
 321                 return;
 322
 323         if (blk_status == BLK_STS_RESV_CONFLICT)
 324                 status = SAM_STAT_RESERVATION_CONFLICT;
 325         else if (atomic_read(&ibr->ib_bio_err_cnt))
 326                 status = SAM_STAT_CHECK_CONDITION;
 327         else
 328                 status = SAM_STAT_GOOD;
 329
 330         target_complete_cmd(cmd, status);
 331         kfree(ibr);
 332 }
 333
 334 static void iblock_bio_done(struct bio *bio)
 335 {
 336         struct se_cmd *cmd = bio->bi_private;
 337         struct iblock_req *ibr = cmd->priv;
 338         blk_status_t blk_status = bio->bi_status;
 339
 340         if (bio->bi_status) {
 341                 pr_err("bio error: %p,  err: %d\n", bio, bio->bi_status);
 342                 /*
 343                  * Bump the ib_bio_err_cnt and release bio.
 344                  */
 345                 atomic_inc(&ibr->ib_bio_err_cnt);
 346                 smp_mb__after_atomic();
 347         }
 348
 349         bio_put(bio);
 350
 351         iblock_complete_cmd(cmd, blk_status);
 352 }
 353
 354 static struct bio *iblock_get_bio(struct se_cmd *cmd, sector_t lba, u32 sg_num,
 355                                   blk_opf_t opf)
 356 {
 357         struct iblock_dev *ib_dev = IBLOCK_DEV(cmd->se_dev);
 358         struct bio *bio;
 359
 360         /*
 361          * Only allocate as many vector entries as the bio code allows us to,
 362          * we'll loop later on until we have handled the whole request.
 363          */
 364         bio = bio_alloc_bioset(ib_dev->ibd_bd, bio_max_segs(sg_num), opf,
 365                                GFP_NOIO, &ib_dev->ibd_bio_set);
 366         if (!bio) {
 367                 pr_err("Unable to allocate memory for bio\n");
 368                 return NULL;
 369         }
 370
 371         bio->bi_private = cmd;
 372         bio->bi_end_io = &iblock_bio_done;
 373         bio->bi_iter.bi_sector = lba;
 374
 375         return bio;
 376 }
 377
 378 static void iblock_submit_bios(struct bio_list *list)
 379 {
 380         struct blk_plug plug;
 381         struct bio *bio;
 382         /*
 383          * The block layer handles nested plugs, so just plug/unplug to handle
 384          * fabric drivers that didn't support batching and multi bio cmds.
 385          */
 386         blk_start_plug(&plug);
 387         while ((bio = bio_list_pop(list)))
 388                 submit_bio(bio);
 389         blk_finish_plug(&plug);
 390 }
 391
 392 static void iblock_end_io_flush(struct bio *bio)
 393 {
 394         struct se_cmd *cmd = bio->bi_private;
 395
 396         if (bio->bi_status)
 397                 pr_err("IBLOCK: cache flush failed: %d\n", bio->bi_status);
 398
 399         if (cmd) {
 400                 if (bio->bi_status)
 401                         target_complete_cmd(cmd, SAM_STAT_CHECK_CONDITION);
 402                 else
 403                         target_complete_cmd(cmd, SAM_STAT_GOOD);
 404         }
 405
 406         bio_put(bio);
 407 }
 408
 409 /*
 410  * Implement SYCHRONIZE CACHE.  Note that we can't handle lba ranges and must
 411  * always flush the whole cache.
 412  */
 413 static sense_reason_t
 414 iblock_execute_sync_cache(struct se_cmd *cmd)
 415 {
 416         struct iblock_dev *ib_dev = IBLOCK_DEV(cmd->se_dev);
 417         int immed = (cmd->t_task_cdb[1] & 0x2);
 418         struct bio *bio;
 419
 420         /*
 421          * If the Immediate bit is set, queue up the GOOD response
 422          * for this SYNCHRONIZE_CACHE op.
 423          */
 424         if (immed)
 425                 target_complete_cmd(cmd, SAM_STAT_GOOD);
 426
 427         bio = bio_alloc(ib_dev->ibd_bd, 0, REQ_OP_WRITE | REQ_PREFLUSH,
 428                         GFP_KERNEL);
 429         bio->bi_end_io = iblock_end_io_flush;
 430         if (!immed)
 431                 bio->bi_private = cmd;
 432         submit_bio(bio);
 433         return 0;
 434 }
 435
 436 static sense_reason_t
 437 iblock_execute_unmap(struct se_cmd *cmd, sector_t lba, sector_t nolb)
 438 {
 439         struct block_device *bdev = IBLOCK_DEV(cmd->se_dev)->ibd_bd;
 440         struct se_device *dev = cmd->se_dev;
 441         int ret;
 442
 443         ret = blkdev_issue_discard(bdev,
 444                                    target_to_linux_sector(dev, lba),
 445                                    target_to_linux_sector(dev,  nolb),
 446                                    GFP_KERNEL);
 447         if (ret < 0) {
 448                 pr_err("blkdev_issue_discard() failed: %d\n", ret);
 449                 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 450         }
 451
 452         return 0;
 453 }
 454
 455 static sense_reason_t
 456 iblock_execute_zero_out(struct block_device *bdev, struct se_cmd *cmd)
 457 {
 458         struct se_device *dev = cmd->se_dev;
 459         struct scatterlist *sg = &cmd->t_data_sg[0];
 460         unsigned char *buf, *not_zero;
 461         int ret;
 462
 463         buf = kmap(sg_page(sg)) + sg->offset;
 464         if (!buf)
 465                 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 466         /*
 467          * Fall back to block_execute_write_same() slow-path if
 468          * incoming WRITE_SAME payload does not contain zeros.
 469          */
 470         not_zero = memchr_inv(buf, 0x00, cmd->data_length);
 471         kunmap(sg_page(sg));
 472
 473         if (not_zero)
 474                 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 475
 476         ret = blkdev_issue_zeroout(bdev,
 477                                 target_to_linux_sector(dev, cmd->t_task_lba),
 478                                 target_to_linux_sector(dev,
 479                                         sbc_get_write_same_sectors(cmd)),
 480                                 GFP_KERNEL, BLKDEV_ZERO_NOUNMAP);
 481         if (ret)
 482                 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 483
 484         target_complete_cmd(cmd, SAM_STAT_GOOD);
 485         return 0;
 486 }
 487
 488 static sense_reason_t
 489 iblock_execute_write_same(struct se_cmd *cmd)
 490 {
 491         struct block_device *bdev = IBLOCK_DEV(cmd->se_dev)->ibd_bd;
 492         struct iblock_req *ibr;
 493         struct scatterlist *sg;
 494         struct bio *bio;
 495         struct bio_list list;
 496         struct se_device *dev = cmd->se_dev;
 497         sector_t block_lba = target_to_linux_sector(dev, cmd->t_task_lba);
 498         sector_t sectors = target_to_linux_sector(dev,
 499                                         sbc_get_write_same_sectors(cmd));
 500
 501         if (cmd->prot_op) {
 502                 pr_err("WRITE_SAME: Protection information with IBLOCK"
 503                        " backends not supported\n");
 504                 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 505         }
 506
 507         if (!cmd->t_data_nents)
 508                 return TCM_INVALID_CDB_FIELD;
 509
 510         sg = &cmd->t_data_sg[0];
 511
 512         if (cmd->t_data_nents > 1 ||
 513             sg->length != cmd->se_dev->dev_attrib.block_size) {
 514                 pr_err("WRITE_SAME: Illegal SGL t_data_nents: %u length: %u"
 515                         " block_size: %u\n", cmd->t_data_nents, sg->length,
 516                         cmd->se_dev->dev_attrib.block_size);
 517                 return TCM_INVALID_CDB_FIELD;
 518         }
 519
 520         if (bdev_write_zeroes_sectors(bdev)) {
 521                 if (!iblock_execute_zero_out(bdev, cmd))
 522                         return 0;
 523         }
 524
 525         ibr = kzalloc(sizeof(struct iblock_req), GFP_KERNEL);
 526         if (!ibr)
 527                 goto fail;
 528         cmd->priv = ibr;
 529
 530         bio = iblock_get_bio(cmd, block_lba, 1, REQ_OP_WRITE);
 531         if (!bio)
 532                 goto fail_free_ibr;
 533
 534         bio_list_init(&list);
 535         bio_list_add(&list, bio);
 536
 537         refcount_set(&ibr->pending, 1);
 538
 539         while (sectors) {
 540                 while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
 541                                 != sg->length) {
 542
 543                         bio = iblock_get_bio(cmd, block_lba, 1, REQ_OP_WRITE);
 544                         if (!bio)
 545                                 goto fail_put_bios;
 546
 547                         refcount_inc(&ibr->pending);
 548                         bio_list_add(&list, bio);
 549                 }
 550
 551                 /* Always in 512 byte units for Linux/Block */
 552                 block_lba += sg->length >> SECTOR_SHIFT;
 553                 sectors -= sg->length >> SECTOR_SHIFT;
 554         }
 555
 556         iblock_submit_bios(&list);
 557         return 0;
 558
 559 fail_put_bios:
 560         while ((bio = bio_list_pop(&list)))
 561                 bio_put(bio);
 562 fail_free_ibr:
 563         kfree(ibr);
 564 fail:
 565         return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 566 }
 567
 568 enum {
 569         Opt_udev_path, Opt_readonly, Opt_force, Opt_err
 570 };
 571
 572 static match_table_t tokens = {
 573         {Opt_udev_path, "udev_path=%s"},
 574         {Opt_readonly, "readonly=%d"},
 575         {Opt_force, "force=%d"},
 576         {Opt_err, NULL}
 577 };
 578
 579 static ssize_t iblock_set_configfs_dev_params(struct se_device *dev,
 580                 const char *page, ssize_t count)
 581 {
 582         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 583         char *orig, *ptr, *arg_p, *opts;
 584         substring_t args[MAX_OPT_ARGS];
 585         int ret = 0, token;
 586         unsigned long tmp_readonly;
 587
 588         opts = kstrdup(page, GFP_KERNEL);
 589         if (!opts)
 590                 return -ENOMEM;
 591
 592         orig = opts;
 593
 594         while ((ptr = strsep(&opts, ",\n")) != NULL) {
 595                 if (!*ptr)
 596                         continue;
 597
 598                 token = match_token(ptr, tokens, args);
 599                 switch (token) {
 600                 case Opt_udev_path:
 601                         if (ib_dev->ibd_bd) {
 602                                 pr_err("Unable to set udev_path= while"
 603                                         " ib_dev->ibd_bd exists\n");
 604                                 ret = -EEXIST;
 605                                 goto out;
 606                         }
 607                         if (match_strlcpy(ib_dev->ibd_udev_path, &args[0],
 608                                 SE_UDEV_PATH_LEN) == 0) {
 609                                 ret = -EINVAL;
 610                                 break;
 611                         }
 612                         pr_debug("IBLOCK: Referencing UDEV path: %s\n",
 613                                         ib_dev->ibd_udev_path);
 614                         ib_dev->ibd_flags |= IBDF_HAS_UDEV_PATH;
 615                         break;
 616                 case Opt_readonly:
 617                         arg_p = match_strdup(&args[0]);
 618                         if (!arg_p) {
 619                                 ret = -ENOMEM;
 620                                 break;
 621                         }
 622                         ret = kstrtoul(arg_p, 0, &tmp_readonly);
 623                         kfree(arg_p);
 624                         if (ret < 0) {
 625                                 pr_err("kstrtoul() failed for"
 626                                                 " readonly=\n");
 627                                 goto out;
 628                         }
 629                         ib_dev->ibd_readonly = tmp_readonly;
 630                         pr_debug("IBLOCK: readonly: %d\n", ib_dev->ibd_readonly);
 631                         break;
 632                 case Opt_force:
 633                         break;
 634                 default:
 635                         break;
 636                 }
 637         }
 638
 639 out:
 640         kfree(orig);
 641         return (!ret) ? count : ret;
 642 }
 643
 644 static ssize_t iblock_show_configfs_dev_params(struct se_device *dev, char *b)
 645 {
 646         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 647         struct block_device *bd = ib_dev->ibd_bd;
 648         ssize_t bl = 0;
 649
 650         if (bd)
 651                 bl += sprintf(b + bl, "iBlock device: %pg", bd);
 652         if (ib_dev->ibd_flags & IBDF_HAS_UDEV_PATH)
 653                 bl += sprintf(b + bl, "  UDEV PATH: %s",
 654                                 ib_dev->ibd_udev_path);
 655         bl += sprintf(b + bl, "  readonly: %d\n", ib_dev->ibd_readonly);
 656
 657         bl += sprintf(b + bl, "        ");
 658         if (bd) {
 659                 bl += sprintf(b + bl, "Major: %d Minor: %d  %s\n",
 660                         MAJOR(bd->bd_dev), MINOR(bd->bd_dev),
 661                         "CLAIMED: IBLOCK");
 662         } else {
 663                 bl += sprintf(b + bl, "Major: 0 Minor: 0\n");
 664         }
 665
 666         return bl;
 667 }
 668
 669 static int
 670 iblock_alloc_bip(struct se_cmd *cmd, struct bio *bio,
 671                  struct sg_mapping_iter *miter)
 672 {
 673         struct se_device *dev = cmd->se_dev;
 674         struct blk_integrity *bi;
 675         struct bio_integrity_payload *bip;
 676         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 677         int rc;
 678         size_t resid, len;
 679
 680         bi = bdev_get_integrity(ib_dev->ibd_bd);
 681         if (!bi) {
 682                 pr_err("Unable to locate bio_integrity\n");
 683                 return -ENODEV;
 684         }
 685
 686         bip = bio_integrity_alloc(bio, GFP_NOIO, bio_max_segs(cmd->t_prot_nents));
 687         if (IS_ERR(bip)) {
 688                 pr_err("Unable to allocate bio_integrity_payload\n");
 689                 return PTR_ERR(bip);
 690         }
 691
 692         bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
 693         /* virtual start sector must be in integrity interval units */
 694         bip_set_seed(bip, bio->bi_iter.bi_sector >>
 695                                   (bi->interval_exp - SECTOR_SHIFT));
 696
 697         pr_debug("IBLOCK BIP Size: %u Sector: %llu\n", bip->bip_iter.bi_size,
 698                  (unsigned long long)bip->bip_iter.bi_sector);
 699
 700         resid = bip->bip_iter.bi_size;
 701         while (resid > 0 && sg_miter_next(miter)) {
 702
 703                 len = min_t(size_t, miter->length, resid);
 704                 rc = bio_integrity_add_page(bio, miter->page, len,
 705                                             offset_in_page(miter->addr));
 706                 if (rc != len) {
 707                         pr_err("bio_integrity_add_page() failed; %d\n", rc);
 708                         sg_miter_stop(miter);
 709                         return -ENOMEM;
 710                 }
 711
 712                 pr_debug("Added bio integrity page: %p length: %zu offset: %lu\n",
 713                           miter->page, len, offset_in_page(miter->addr));
 714
 715                 resid -= len;
 716                 if (len < miter->length)
 717                         miter->consumed -= miter->length - len;
 718         }
 719         sg_miter_stop(miter);
 720
 721         return 0;
 722 }
 723
 724 static sense_reason_t
 725 iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
 726                   enum dma_data_direction data_direction)
 727 {
 728         struct se_device *dev = cmd->se_dev;
 729         sector_t block_lba = target_to_linux_sector(dev, cmd->t_task_lba);
 730         struct iblock_req *ibr;
 731         struct bio *bio;
 732         struct bio_list list;
 733         struct scatterlist *sg;
 734         u32 sg_num = sgl_nents;
 735         blk_opf_t opf;
 736         unsigned bio_cnt;
 737         int i, rc;
 738         struct sg_mapping_iter prot_miter;
 739         unsigned int miter_dir;
 740
 741         if (data_direction == DMA_TO_DEVICE) {
 742                 struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 743                 /*
 744                  * Force writethrough using REQ_FUA if a volatile write cache
 745                  * is not enabled, or if initiator set the Force Unit Access bit.
 746                  */
 747                 opf = REQ_OP_WRITE;
 748                 miter_dir = SG_MITER_TO_SG;
 749                 if (bdev_fua(ib_dev->ibd_bd)) {
 750                         if (cmd->se_cmd_flags & SCF_FUA)
 751                                 opf |= REQ_FUA;
 752                         else if (!bdev_write_cache(ib_dev->ibd_bd))
 753                                 opf |= REQ_FUA;
 754                 }
 755         } else {
 756                 opf = REQ_OP_READ;
 757                 miter_dir = SG_MITER_FROM_SG;
 758         }
 759
 760         ibr = kzalloc(sizeof(struct iblock_req), GFP_KERNEL);
 761         if (!ibr)
 762                 goto fail;
 763         cmd->priv = ibr;
 764
 765         if (!sgl_nents) {
 766                 refcount_set(&ibr->pending, 1);
 767                 iblock_complete_cmd(cmd, BLK_STS_OK);
 768                 return 0;
 769         }
 770
 771         bio = iblock_get_bio(cmd, block_lba, sgl_nents, opf);
 772         if (!bio)
 773                 goto fail_free_ibr;
 774
 775         bio_list_init(&list);
 776         bio_list_add(&list, bio);
 777
 778         refcount_set(&ibr->pending, 2);
 779         bio_cnt = 1;
 780
 781         if (cmd->prot_type && dev->dev_attrib.pi_prot_type)
 782                 sg_miter_start(&prot_miter, cmd->t_prot_sg, cmd->t_prot_nents,
 783                                miter_dir);
 784
 785         for_each_sg(sgl, sg, sgl_nents, i) {
 786                 /*
 787                  * XXX: if the length the device accepts is shorter than the
 788                  *      length of the S/G list entry this will cause and
 789                  *      endless loop.  Better hope no driver uses huge pages.
 790                  */
 791                 while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
 792                                 != sg->length) {
 793                         if (cmd->prot_type && dev->dev_attrib.pi_prot_type) {
 794                                 rc = iblock_alloc_bip(cmd, bio, &prot_miter);
 795                                 if (rc)
 796                                         goto fail_put_bios;
 797                         }
 798
 799                         if (bio_cnt >= IBLOCK_MAX_BIO_PER_TASK) {
 800                                 iblock_submit_bios(&list);
 801                                 bio_cnt = 0;
 802                         }
 803
 804                         bio = iblock_get_bio(cmd, block_lba, sg_num, opf);
 805                         if (!bio)
 806                                 goto fail_put_bios;
 807
 808                         refcount_inc(&ibr->pending);
 809                         bio_list_add(&list, bio);
 810                         bio_cnt++;
 811                 }
 812
 813                 /* Always in 512 byte units for Linux/Block */
 814                 block_lba += sg->length >> SECTOR_SHIFT;
 815                 sg_num--;
 816         }
 817
 818         if (cmd->prot_type && dev->dev_attrib.pi_prot_type) {
 819                 rc = iblock_alloc_bip(cmd, bio, &prot_miter);
 820                 if (rc)
 821                         goto fail_put_bios;
 822         }
 823
 824         iblock_submit_bios(&list);
 825         iblock_complete_cmd(cmd, BLK_STS_OK);
 826         return 0;
 827
 828 fail_put_bios:
 829         while ((bio = bio_list_pop(&list)))
 830                 bio_put(bio);
 831 fail_free_ibr:
 832         kfree(ibr);
 833 fail:
 834         return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 835 }
 836
 837 static sense_reason_t iblock_execute_pr_out(struct se_cmd *cmd, u8 sa, u64 key,
 838                                             u64 sa_key, u8 type, bool aptpl)
 839 {
 840         struct se_device *dev = cmd->se_dev;
 841         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 842         struct block_device *bdev = ib_dev->ibd_bd;
 843         const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops;
 844         int ret;
 845
 846         if (!ops) {
 847                 pr_err("Block device does not support pr_ops but iblock device has been configured for PR passthrough.\n");
 848                 return TCM_UNSUPPORTED_SCSI_OPCODE;
 849         }
 850
 851         switch (sa) {
 852         case PRO_REGISTER:
 853         case PRO_REGISTER_AND_IGNORE_EXISTING_KEY:
 854                 if (!ops->pr_register) {
 855                         pr_err("block device does not support pr_register.\n");
 856                         return TCM_UNSUPPORTED_SCSI_OPCODE;
 857                 }
 858
 859                 /* The block layer pr ops always enables aptpl */
 860                 if (!aptpl)
 861                         pr_info("APTPL not set by initiator, but will be used.\n");
 862
 863                 ret = ops->pr_register(bdev, key, sa_key,
 864                                 sa == PRO_REGISTER ? 0 : PR_FL_IGNORE_KEY);
 865                 break;
 866         case PRO_RESERVE:
 867                 if (!ops->pr_reserve) {
 868                         pr_err("block_device does not support pr_reserve.\n");
 869                         return TCM_UNSUPPORTED_SCSI_OPCODE;
 870                 }
 871
 872                 ret = ops->pr_reserve(bdev, key, scsi_pr_type_to_block(type), 0);
 873                 break;
 874         case PRO_CLEAR:
 875                 if (!ops->pr_clear) {
 876                         pr_err("block_device does not support pr_clear.\n");
 877                         return TCM_UNSUPPORTED_SCSI_OPCODE;
 878                 }
 879
 880                 ret = ops->pr_clear(bdev, key);
 881                 break;
 882         case PRO_PREEMPT:
 883         case PRO_PREEMPT_AND_ABORT:
 884                 if (!ops->pr_clear) {
 885                         pr_err("block_device does not support pr_preempt.\n");
 886                         return TCM_UNSUPPORTED_SCSI_OPCODE;
 887                 }
 888
 889                 ret = ops->pr_preempt(bdev, key, sa_key,
 890                                       scsi_pr_type_to_block(type),
 891                                       sa == PRO_PREEMPT_AND_ABORT);
 892                 break;
 893         case PRO_RELEASE:
 894                 if (!ops->pr_clear) {
 895                         pr_err("block_device does not support pr_pclear.\n");
 896                         return TCM_UNSUPPORTED_SCSI_OPCODE;
 897                 }
 898
 899                 ret = ops->pr_release(bdev, key, scsi_pr_type_to_block(type));
 900                 break;
 901         default:
 902                 pr_err("Unknown PERSISTENT_RESERVE_OUT SA: 0x%02x\n", sa);
 903                 return TCM_UNSUPPORTED_SCSI_OPCODE;
 904         }
 905
 906         if (!ret)
 907                 return TCM_NO_SENSE;
 908         else if (ret == PR_STS_RESERVATION_CONFLICT)
 909                 return TCM_RESERVATION_CONFLICT;
 910         else
 911                 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 912 }
 913
 914 static void iblock_pr_report_caps(unsigned char *param_data)
 915 {
 916         u16 len = 8;
 917
 918         put_unaligned_be16(len, &param_data[0]);
 919         /*
 920          * When using the pr_ops passthrough method we only support exporting
 921          * the device through one target port because from the backend module
 922          * level we can't see the target port config. As a result we only
 923          * support registration directly from the I_T nexus the cmd is sent
 924          * through and do not set ATP_C here.
 925          *
 926          * The block layer pr_ops do not support passing in initiators so
 927          * we don't set SIP_C here.
 928          */
 929         /* PTPL_C: Persistence across Target Power Loss bit */
 930         param_data[2] |= 0x01;
 931         /*
 932          * We are filling in the PERSISTENT RESERVATION TYPE MASK below, so
 933          * set the TMV: Task Mask Valid bit.
 934          */
 935         param_data[3] |= 0x80;
 936         /*
 937          * Change ALLOW COMMANDs to 0x20 or 0x40 later from Table 166
 938          */
 939         param_data[3] |= 0x10; /* ALLOW COMMANDs field 001b */
 940         /*
 941          * PTPL_A: Persistence across Target Power Loss Active bit. The block
 942          * layer pr ops always enables this so report it active.
 943          */
 944         param_data[3] |= 0x01;
 945         /*
 946          * Setup the PERSISTENT RESERVATION TYPE MASK from Table 212 spc4r37.
 947          */
 948         param_data[4] |= 0x80; /* PR_TYPE_EXCLUSIVE_ACCESS_ALLREG */
 949         param_data[4] |= 0x40; /* PR_TYPE_EXCLUSIVE_ACCESS_REGONLY */
 950         param_data[4] |= 0x20; /* PR_TYPE_WRITE_EXCLUSIVE_REGONLY */
 951         param_data[4] |= 0x08; /* PR_TYPE_EXCLUSIVE_ACCESS */
 952         param_data[4] |= 0x02; /* PR_TYPE_WRITE_EXCLUSIVE */
 953         param_data[5] |= 0x01; /* PR_TYPE_EXCLUSIVE_ACCESS_ALLREG */
 954 }
 955
 956 static sense_reason_t iblock_pr_read_keys(struct se_cmd *cmd,
 957                                           unsigned char *param_data)
 958 {
 959         struct se_device *dev = cmd->se_dev;
 960         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 961         struct block_device *bdev = ib_dev->ibd_bd;
 962         const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops;
 963         int i, len, paths, data_offset;
 964         struct pr_keys *keys;
 965         sense_reason_t ret;
 966
 967         if (!ops) {
 968                 pr_err("Block device does not support pr_ops but iblock device has been configured for PR passthrough.\n");
 969                 return TCM_UNSUPPORTED_SCSI_OPCODE;
 970         }
 971
 972         if (!ops->pr_read_keys) {
 973                 pr_err("Block device does not support read_keys.\n");
 974                 return TCM_UNSUPPORTED_SCSI_OPCODE;
 975         }
 976
 977         /*
 978          * We don't know what's under us, but dm-multipath will register every
 979          * path with the same key, so start off with enough space for 16 paths.
 980          * which is not a lot of memory and should normally be enough.
 981          */
 982         paths = 16;
 983 retry:
 984         len = 8 * paths;
 985         keys = kzalloc(sizeof(*keys) + len, GFP_KERNEL);
 986         if (!keys)
 987                 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 988
 989         keys->num_keys = paths;
 990         if (!ops->pr_read_keys(bdev, keys)) {
 991                 if (keys->num_keys > paths) {
 992                         kfree(keys);
 993                         paths *= 2;
 994                         goto retry;
 995                 }
 996         } else {
 997                 ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 998                 goto free_keys;
 999         }
1000
1001         ret = TCM_NO_SENSE;
1002
1003         put_unaligned_be32(keys->generation, &param_data[0]);
1004         if (!keys->num_keys) {
1005                 put_unaligned_be32(0, &param_data[4]);
1006                 goto free_keys;
1007         }
1008
1009         put_unaligned_be32(8 * keys->num_keys, &param_data[4]);
1010
1011         data_offset = 8;
1012         for (i = 0; i < keys->num_keys; i++) {
1013                 if (data_offset + 8 > cmd->data_length)
1014                         break;
1015
1016                 put_unaligned_be64(keys->keys[i], &param_data[data_offset]);
1017                 data_offset += 8;
1018         }
1019
1020 free_keys:
1021         kfree(keys);
1022         return ret;
1023 }
1024
1025 static sense_reason_t iblock_pr_read_reservation(struct se_cmd *cmd,
1026                                                  unsigned char *param_data)
1027 {
1028         struct se_device *dev = cmd->se_dev;
1029         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
1030         struct block_device *bdev = ib_dev->ibd_bd;
1031         const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops;
1032         struct pr_held_reservation rsv = { };
1033
1034         if (!ops) {
1035                 pr_err("Block device does not support pr_ops but iblock device has been configured for PR passthrough.\n");
1036                 return TCM_UNSUPPORTED_SCSI_OPCODE;
1037         }
1038
1039         if (!ops->pr_read_reservation) {
1040                 pr_err("Block device does not support read_keys.\n");
1041                 return TCM_UNSUPPORTED_SCSI_OPCODE;
1042         }
1043
1044         if (ops->pr_read_reservation(bdev, &rsv))
1045                 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
1046
1047         put_unaligned_be32(rsv.generation, &param_data[0]);
1048         if (!block_pr_type_to_scsi(rsv.type)) {
1049                 put_unaligned_be32(0, &param_data[4]);
1050                 return TCM_NO_SENSE;
1051         }
1052
1053         put_unaligned_be32(16, &param_data[4]);
1054
1055         if (cmd->data_length < 16)
1056                 return TCM_NO_SENSE;
1057         put_unaligned_be64(rsv.key, &param_data[8]);
1058
1059         if (cmd->data_length < 22)
1060                 return TCM_NO_SENSE;
1061         param_data[21] = block_pr_type_to_scsi(rsv.type);
1062
1063         return TCM_NO_SENSE;
1064 }
1065
1066 static sense_reason_t iblock_execute_pr_in(struct se_cmd *cmd, u8 sa,
1067                                            unsigned char *param_data)
1068 {
1069         sense_reason_t ret = TCM_NO_SENSE;
1070
1071         switch (sa) {
1072         case PRI_REPORT_CAPABILITIES:
1073                 iblock_pr_report_caps(param_data);
1074                 break;
1075         case PRI_READ_KEYS:
1076                 ret = iblock_pr_read_keys(cmd, param_data);
1077                 break;
1078         case PRI_READ_RESERVATION:
1079                 ret = iblock_pr_read_reservation(cmd, param_data);
1080                 break;
1081         default:
1082                 pr_err("Unknown PERSISTENT_RESERVE_IN SA: 0x%02x\n", sa);
1083                 return TCM_UNSUPPORTED_SCSI_OPCODE;
1084         }
1085
1086         return ret;
1087 }
1088
1089 static sector_t iblock_get_alignment_offset_lbas(struct se_device *dev)
1090 {
1091         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
1092         struct block_device *bd = ib_dev->ibd_bd;
1093         int ret;
1094
1095         ret = bdev_alignment_offset(bd);
1096         if (ret == -1)
1097                 return 0;
1098
1099         /* convert offset-bytes to offset-lbas */
1100         return ret / bdev_logical_block_size(bd);
1101 }
1102
1103 static unsigned int iblock_get_lbppbe(struct se_device *dev)
1104 {
1105         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
1106         struct block_device *bd = ib_dev->ibd_bd;
1107         unsigned int logs_per_phys =
1108                 bdev_physical_block_size(bd) / bdev_logical_block_size(bd);
1109
1110         return ilog2(logs_per_phys);
1111 }
1112
1113 static unsigned int iblock_get_io_min(struct se_device *dev)
1114 {
1115         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
1116         struct block_device *bd = ib_dev->ibd_bd;
1117
1118         return bdev_io_min(bd);
1119 }
1120
1121 static unsigned int iblock_get_io_opt(struct se_device *dev)
1122 {
1123         struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
1124         struct block_device *bd = ib_dev->ibd_bd;
1125
1126         return bdev_io_opt(bd);
1127 }
1128
1129 static struct exec_cmd_ops iblock_exec_cmd_ops = {
1130         .execute_rw             = iblock_execute_rw,
1131         .execute_sync_cache     = iblock_execute_sync_cache,
1132         .execute_write_same     = iblock_execute_write_same,
1133         .execute_unmap          = iblock_execute_unmap,
1134         .execute_pr_out         = iblock_execute_pr_out,
1135         .execute_pr_in          = iblock_execute_pr_in,
1136 };
1137
1138 static sense_reason_t
1139 iblock_parse_cdb(struct se_cmd *cmd)
1140 {
1141         return sbc_parse_cdb(cmd, &iblock_exec_cmd_ops);
1142 }
1143
1144 static bool iblock_get_write_cache(struct se_device *dev)
1145 {
1146         return bdev_write_cache(IBLOCK_DEV(dev)->ibd_bd);
1147 }
1148
1149 static const struct target_backend_ops iblock_ops = {
1150         .name                   = "iblock",
1151         .inquiry_prod           = "IBLOCK",
1152         .transport_flags_changeable = TRANSPORT_FLAG_PASSTHROUGH_PGR,
1153         .inquiry_rev            = IBLOCK_VERSION,
1154         .owner                  = THIS_MODULE,
1155         .attach_hba             = iblock_attach_hba,
1156         .detach_hba             = iblock_detach_hba,
1157         .alloc_device           = iblock_alloc_device,
1158         .configure_device       = iblock_configure_device,
1159         .destroy_device         = iblock_destroy_device,
1160         .free_device            = iblock_free_device,
1161         .configure_unmap        = iblock_configure_unmap,
1162         .plug_device            = iblock_plug_device,
1163         .unplug_device          = iblock_unplug_device,
1164         .parse_cdb              = iblock_parse_cdb,
1165         .set_configfs_dev_params = iblock_set_configfs_dev_params,
1166         .show_configfs_dev_params = iblock_show_configfs_dev_params,
1167         .get_device_type        = sbc_get_device_type,
1168         .get_blocks             = iblock_get_blocks,
1169         .get_alignment_offset_lbas = iblock_get_alignment_offset_lbas,
1170         .get_lbppbe             = iblock_get_lbppbe,
1171         .get_io_min             = iblock_get_io_min,
1172         .get_io_opt             = iblock_get_io_opt,
1173         .get_write_cache        = iblock_get_write_cache,
1174         .tb_dev_attrib_attrs    = sbc_attrib_attrs,
1175 };
1176
1177 static int __init iblock_module_init(void)
1178 {
1179         return transport_backend_register(&iblock_ops);
1180 }
1181
1182 static void __exit iblock_module_exit(void)
1183 {
1184         target_backend_unregister(&iblock_ops);
1185 }
1186
1187 MODULE_DESCRIPTION("TCM IBLOCK subsystem plugin");
1188 MODULE_AUTHOR("nab@Linux-iSCSI.org");
1189 MODULE_LICENSE("GPL");
1190
1191 module_init(iblock_module_init);
1192 module_exit(iblock_module_exit);