drivers/block/ublk_drv.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Userspace block device - block device which IO is handled from userspace
   4  *
   5  * Take full use of io_uring passthrough command for communicating with
   6  * ublk userspace daemon(ublksrvd) for handling basic IO request.
   7  *
   8  * Copyright 2022 Ming Lei <ming.lei@redhat.com>
   9  *
  10  * (part of code stolen from loop.c)
  11  */
  12 #include <linux/module.h>
  13 #include <linux/moduleparam.h>
  14 #include <linux/sched.h>
  15 #include <linux/fs.h>
  16 #include <linux/pagemap.h>
  17 #include <linux/file.h>
  18 #include <linux/stat.h>
  19 #include <linux/errno.h>
  20 #include <linux/major.h>
  21 #include <linux/wait.h>
  22 #include <linux/blkdev.h>
  23 #include <linux/init.h>
  24 #include <linux/swap.h>
  25 #include <linux/slab.h>
  26 #include <linux/compat.h>
  27 #include <linux/mutex.h>
  28 #include <linux/writeback.h>
  29 #include <linux/completion.h>
  30 #include <linux/highmem.h>
  31 #include <linux/sysfs.h>
  32 #include <linux/miscdevice.h>
  33 #include <linux/falloc.h>
  34 #include <linux/uio.h>
  35 #include <linux/ioprio.h>
  36 #include <linux/sched/mm.h>
  37 #include <linux/uaccess.h>
  38 #include <linux/cdev.h>
  39 #include <linux/io_uring.h>
  40 #include <linux/blk-mq.h>
  41 #include <linux/delay.h>
  42 #include <linux/mm.h>
  43 #include <asm/page.h>
  44 #include <linux/task_work.h>
  45 #include <uapi/linux/ublk_cmd.h>
  46
  47 #define UBLK_MINORS             (1U << MINORBITS)
  48
  49 /* All UBLK_F_* have to be included into UBLK_F_ALL */
  50 #define UBLK_F_ALL (UBLK_F_SUPPORT_ZERO_COPY \
  51                 | UBLK_F_URING_CMD_COMP_IN_TASK \
  52                 | UBLK_F_NEED_GET_DATA \
  53                 | UBLK_F_USER_RECOVERY \
  54                 | UBLK_F_USER_RECOVERY_REISSUE)
  55
  56 /* All UBLK_PARAM_TYPE_* should be included here */
  57 #define UBLK_PARAM_TYPE_ALL (UBLK_PARAM_TYPE_BASIC | UBLK_PARAM_TYPE_DISCARD)
  58
  59 struct ublk_rq_data {
  60         struct callback_head work;
  61 };
  62
  63 struct ublk_uring_cmd_pdu {
  64         struct request *req;
  65 };
  66
  67 /*
  68  * io command is active: sqe cmd is received, and its cqe isn't done
  69  *
  70  * If the flag is set, the io command is owned by ublk driver, and waited
  71  * for incoming blk-mq request from the ublk block device.
  72  *
  73  * If the flag is cleared, the io command will be completed, and owned by
  74  * ublk server.
  75  */
  76 #define UBLK_IO_FLAG_ACTIVE     0x01
  77
  78 /*
  79  * IO command is completed via cqe, and it is being handled by ublksrv, and
  80  * not committed yet
  81  *
  82  * Basically exclusively with UBLK_IO_FLAG_ACTIVE, so can be served for
  83  * cross verification
  84  */
  85 #define UBLK_IO_FLAG_OWNED_BY_SRV 0x02
  86
  87 /*
  88  * IO command is aborted, so this flag is set in case of
  89  * !UBLK_IO_FLAG_ACTIVE.
  90  *
  91  * After this flag is observed, any pending or new incoming request
  92  * associated with this io command will be failed immediately
  93  */
  94 #define UBLK_IO_FLAG_ABORTED 0x04
  95
  96 /*
  97  * UBLK_IO_FLAG_NEED_GET_DATA is set because IO command requires
  98  * get data buffer address from ublksrv.
  99  *
 100  * Then, bio data could be copied into this data buffer for a WRITE request
 101  * after the IO command is issued again and UBLK_IO_FLAG_NEED_GET_DATA is unset.
 102  */
 103 #define UBLK_IO_FLAG_NEED_GET_DATA 0x08
 104
 105 struct ublk_io {
 106         /* userspace buffer address from io cmd */
 107         __u64   addr;
 108         unsigned int flags;
 109         int res;
 110
 111         struct io_uring_cmd *cmd;
 112 };
 113
 114 struct ublk_queue {
 115         int q_id;
 116         int q_depth;
 117
 118         unsigned long flags;
 119         struct task_struct      *ubq_daemon;
 120         char *io_cmd_buf;
 121
 122         unsigned long io_addr;  /* mapped vm address */
 123         unsigned int max_io_sz;
 124         bool force_abort;
 125         unsigned short nr_io_ready;     /* how many ios setup */
 126         struct ublk_device *dev;
 127         struct ublk_io ios[0];
 128 };
 129
 130 #define UBLK_DAEMON_MONITOR_PERIOD      (5 * HZ)
 131
 132 struct ublk_device {
 133         struct gendisk          *ub_disk;
 134
 135         char    *__queues;
 136
 137         unsigned short  queue_size;
 138         struct ublksrv_ctrl_dev_info    dev_info;
 139
 140         struct blk_mq_tag_set   tag_set;
 141
 142         struct cdev             cdev;
 143         struct device           cdev_dev;
 144
 145 #define UB_STATE_OPEN           0
 146 #define UB_STATE_USED           1
 147         unsigned long           state;
 148         int                     ub_number;
 149
 150         struct mutex            mutex;
 151
 152         spinlock_t              mm_lock;
 153         struct mm_struct        *mm;
 154
 155         struct ublk_params      params;
 156
 157         struct completion       completion;
 158         unsigned int            nr_queues_ready;
 159         atomic_t                nr_aborted_queues;
 160
 161         /*
 162          * Our ubq->daemon may be killed without any notification, so
 163          * monitor each queue's daemon periodically
 164          */
 165         struct delayed_work     monitor_work;
 166         struct work_struct      quiesce_work;
 167         struct work_struct      stop_work;
 168 };
 169
 170 /* header of ublk_params */
 171 struct ublk_params_header {
 172         __u32   len;
 173         __u32   types;
 174 };
 175
 176 static dev_t ublk_chr_devt;
 177 static struct class *ublk_chr_class;
 178
 179 static DEFINE_IDR(ublk_index_idr);
 180 static DEFINE_SPINLOCK(ublk_idr_lock);
 181 static wait_queue_head_t ublk_idr_wq;   /* wait until one idr is freed */
 182
 183 static DEFINE_MUTEX(ublk_ctl_mutex);
 184
 185 static struct miscdevice ublk_misc;
 186
 187 static void ublk_dev_param_basic_apply(struct ublk_device *ub)
 188 {
 189         struct request_queue *q = ub->ub_disk->queue;
 190         const struct ublk_param_basic *p = &ub->params.basic;
 191
 192         blk_queue_logical_block_size(q, 1 << p->logical_bs_shift);
 193         blk_queue_physical_block_size(q, 1 << p->physical_bs_shift);
 194         blk_queue_io_min(q, 1 << p->io_min_shift);
 195         blk_queue_io_opt(q, 1 << p->io_opt_shift);
 196
 197         blk_queue_write_cache(q, p->attrs & UBLK_ATTR_VOLATILE_CACHE,
 198                         p->attrs & UBLK_ATTR_FUA);
 199         if (p->attrs & UBLK_ATTR_ROTATIONAL)
 200                 blk_queue_flag_clear(QUEUE_FLAG_NONROT, q);
 201         else
 202                 blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
 203
 204         blk_queue_max_hw_sectors(q, p->max_sectors);
 205         blk_queue_chunk_sectors(q, p->chunk_sectors);
 206         blk_queue_virt_boundary(q, p->virt_boundary_mask);
 207
 208         if (p->attrs & UBLK_ATTR_READ_ONLY)
 209                 set_disk_ro(ub->ub_disk, true);
 210
 211         set_capacity(ub->ub_disk, p->dev_sectors);
 212 }
 213
 214 static void ublk_dev_param_discard_apply(struct ublk_device *ub)
 215 {
 216         struct request_queue *q = ub->ub_disk->queue;
 217         const struct ublk_param_discard *p = &ub->params.discard;
 218
 219         q->limits.discard_alignment = p->discard_alignment;
 220         q->limits.discard_granularity = p->discard_granularity;
 221         blk_queue_max_discard_sectors(q, p->max_discard_sectors);
 222         blk_queue_max_write_zeroes_sectors(q,
 223                         p->max_write_zeroes_sectors);
 224         blk_queue_max_discard_segments(q, p->max_discard_segments);
 225 }
 226
 227 static int ublk_validate_params(const struct ublk_device *ub)
 228 {
 229         /* basic param is the only one which must be set */
 230         if (ub->params.types & UBLK_PARAM_TYPE_BASIC) {
 231                 const struct ublk_param_basic *p = &ub->params.basic;
 232
 233                 if (p->logical_bs_shift > PAGE_SHIFT)
 234                         return -EINVAL;
 235
 236                 if (p->logical_bs_shift > p->physical_bs_shift)
 237                         return -EINVAL;
 238
 239                 if (p->max_sectors > (ub->dev_info.max_io_buf_bytes >> 9))
 240                         return -EINVAL;
 241         } else
 242                 return -EINVAL;
 243
 244         if (ub->params.types & UBLK_PARAM_TYPE_DISCARD) {
 245                 const struct ublk_param_discard *p = &ub->params.discard;
 246
 247                 /* So far, only support single segment discard */
 248                 if (p->max_discard_sectors && p->max_discard_segments != 1)
 249                         return -EINVAL;
 250
 251                 if (!p->discard_granularity)
 252                         return -EINVAL;
 253         }
 254
 255         return 0;
 256 }
 257
 258 static int ublk_apply_params(struct ublk_device *ub)
 259 {
 260         if (!(ub->params.types & UBLK_PARAM_TYPE_BASIC))
 261                 return -EINVAL;
 262
 263         ublk_dev_param_basic_apply(ub);
 264
 265         if (ub->params.types & UBLK_PARAM_TYPE_DISCARD)
 266                 ublk_dev_param_discard_apply(ub);
 267
 268         return 0;
 269 }
 270
 271 static inline bool ublk_can_use_task_work(const struct ublk_queue *ubq)
 272 {
 273         if (IS_BUILTIN(CONFIG_BLK_DEV_UBLK) &&
 274                         !(ubq->flags & UBLK_F_URING_CMD_COMP_IN_TASK))
 275                 return true;
 276         return false;
 277 }
 278
 279 static inline bool ublk_need_get_data(const struct ublk_queue *ubq)
 280 {
 281         if (ubq->flags & UBLK_F_NEED_GET_DATA)
 282                 return true;
 283         return false;
 284 }
 285
 286 static struct ublk_device *ublk_get_device(struct ublk_device *ub)
 287 {
 288         if (kobject_get_unless_zero(&ub->cdev_dev.kobj))
 289                 return ub;
 290         return NULL;
 291 }
 292
 293 static void ublk_put_device(struct ublk_device *ub)
 294 {
 295         put_device(&ub->cdev_dev);
 296 }
 297
 298 static inline struct ublk_queue *ublk_get_queue(struct ublk_device *dev,
 299                 int qid)
 300 {
 301        return (struct ublk_queue *)&(dev->__queues[qid * dev->queue_size]);
 302 }
 303
 304 static inline bool ublk_rq_has_data(const struct request *rq)
 305 {
 306         return rq->bio && bio_has_data(rq->bio);
 307 }
 308
 309 static inline struct ublksrv_io_desc *ublk_get_iod(struct ublk_queue *ubq,
 310                 int tag)
 311 {
 312         return (struct ublksrv_io_desc *)
 313                 &(ubq->io_cmd_buf[tag * sizeof(struct ublksrv_io_desc)]);
 314 }
 315
 316 static inline char *ublk_queue_cmd_buf(struct ublk_device *ub, int q_id)
 317 {
 318         return ublk_get_queue(ub, q_id)->io_cmd_buf;
 319 }
 320
 321 static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub, int q_id)
 322 {
 323         struct ublk_queue *ubq = ublk_get_queue(ub, q_id);
 324
 325         return round_up(ubq->q_depth * sizeof(struct ublksrv_io_desc),
 326                         PAGE_SIZE);
 327 }
 328
 329 static inline bool ublk_queue_can_use_recovery_reissue(
 330                 struct ublk_queue *ubq)
 331 {
 332         if ((ubq->flags & UBLK_F_USER_RECOVERY) &&
 333                         (ubq->flags & UBLK_F_USER_RECOVERY_REISSUE))
 334                 return true;
 335         return false;
 336 }
 337
 338 static inline bool ublk_queue_can_use_recovery(
 339                 struct ublk_queue *ubq)
 340 {
 341         if (ubq->flags & UBLK_F_USER_RECOVERY)
 342                 return true;
 343         return false;
 344 }
 345
 346 static inline bool ublk_can_use_recovery(struct ublk_device *ub)
 347 {
 348         if (ub->dev_info.flags & UBLK_F_USER_RECOVERY)
 349                 return true;
 350         return false;
 351 }
 352
 353 static void ublk_free_disk(struct gendisk *disk)
 354 {
 355         struct ublk_device *ub = disk->private_data;
 356
 357         clear_bit(UB_STATE_USED, &ub->state);
 358         put_device(&ub->cdev_dev);
 359 }
 360
 361 static const struct block_device_operations ub_fops = {
 362         .owner =        THIS_MODULE,
 363         .free_disk =    ublk_free_disk,
 364 };
 365
 366 #define UBLK_MAX_PIN_PAGES      32
 367
 368 struct ublk_map_data {
 369         const struct ublk_queue *ubq;
 370         const struct request *rq;
 371         const struct ublk_io *io;
 372         unsigned max_bytes;
 373 };
 374
 375 struct ublk_io_iter {
 376         struct page *pages[UBLK_MAX_PIN_PAGES];
 377         unsigned pg_off;        /* offset in the 1st page in pages */
 378         int nr_pages;           /* how many page pointers in pages */
 379         struct bio *bio;
 380         struct bvec_iter iter;
 381 };
 382
 383 static inline unsigned ublk_copy_io_pages(struct ublk_io_iter *data,
 384                 unsigned max_bytes, bool to_vm)
 385 {
 386         const unsigned total = min_t(unsigned, max_bytes,
 387                         PAGE_SIZE - data->pg_off +
 388                         ((data->nr_pages - 1) << PAGE_SHIFT));
 389         unsigned done = 0;
 390         unsigned pg_idx = 0;
 391
 392         while (done < total) {
 393                 struct bio_vec bv = bio_iter_iovec(data->bio, data->iter);
 394                 const unsigned int bytes = min3(bv.bv_len, total - done,
 395                                 (unsigned)(PAGE_SIZE - data->pg_off));
 396                 void *bv_buf = bvec_kmap_local(&bv);
 397                 void *pg_buf = kmap_local_page(data->pages[pg_idx]);
 398
 399                 if (to_vm)
 400                         memcpy(pg_buf + data->pg_off, bv_buf, bytes);
 401                 else
 402                         memcpy(bv_buf, pg_buf + data->pg_off, bytes);
 403
 404                 kunmap_local(pg_buf);
 405                 kunmap_local(bv_buf);
 406
 407                 /* advance page array */
 408                 data->pg_off += bytes;
 409                 if (data->pg_off == PAGE_SIZE) {
 410                         pg_idx += 1;
 411                         data->pg_off = 0;
 412                 }
 413
 414                 done += bytes;
 415
 416                 /* advance bio */
 417                 bio_advance_iter_single(data->bio, &data->iter, bytes);
 418                 if (!data->iter.bi_size) {
 419                         data->bio = data->bio->bi_next;
 420                         if (data->bio == NULL)
 421                                 break;
 422                         data->iter = data->bio->bi_iter;
 423                 }
 424         }
 425
 426         return done;
 427 }
 428
 429 static inline int ublk_copy_user_pages(struct ublk_map_data *data,
 430                 bool to_vm)
 431 {
 432         const unsigned int gup_flags = to_vm ? FOLL_WRITE : 0;
 433         const unsigned long start_vm = data->io->addr;
 434         unsigned int done = 0;
 435         struct ublk_io_iter iter = {
 436                 .pg_off = start_vm & (PAGE_SIZE - 1),
 437                 .bio    = data->rq->bio,
 438                 .iter   = data->rq->bio->bi_iter,
 439         };
 440         const unsigned int nr_pages = round_up(data->max_bytes +
 441                         (start_vm & (PAGE_SIZE - 1)), PAGE_SIZE) >> PAGE_SHIFT;
 442
 443         while (done < nr_pages) {
 444                 const unsigned to_pin = min_t(unsigned, UBLK_MAX_PIN_PAGES,
 445                                 nr_pages - done);
 446                 unsigned i, len;
 447
 448                 iter.nr_pages = get_user_pages_fast(start_vm +
 449                                 (done << PAGE_SHIFT), to_pin, gup_flags,
 450                                 iter.pages);
 451                 if (iter.nr_pages <= 0)
 452                         return done == 0 ? iter.nr_pages : done;
 453                 len = ublk_copy_io_pages(&iter, data->max_bytes, to_vm);
 454                 for (i = 0; i < iter.nr_pages; i++) {
 455                         if (to_vm)
 456                                 set_page_dirty(iter.pages[i]);
 457                         put_page(iter.pages[i]);
 458                 }
 459                 data->max_bytes -= len;
 460                 done += iter.nr_pages;
 461         }
 462
 463         return done;
 464 }
 465
 466 static int ublk_map_io(const struct ublk_queue *ubq, const struct request *req,
 467                 struct ublk_io *io)
 468 {
 469         const unsigned int rq_bytes = blk_rq_bytes(req);
 470         /*
 471          * no zero copy, we delay copy WRITE request data into ublksrv
 472          * context and the big benefit is that pinning pages in current
 473          * context is pretty fast, see ublk_pin_user_pages
 474          */
 475         if (req_op(req) != REQ_OP_WRITE && req_op(req) != REQ_OP_FLUSH)
 476                 return rq_bytes;
 477
 478         if (ublk_rq_has_data(req)) {
 479                 struct ublk_map_data data = {
 480                         .ubq    =       ubq,
 481                         .rq     =       req,
 482                         .io     =       io,
 483                         .max_bytes =    rq_bytes,
 484                 };
 485
 486                 ublk_copy_user_pages(&data, true);
 487
 488                 return rq_bytes - data.max_bytes;
 489         }
 490         return rq_bytes;
 491 }
 492
 493 static int ublk_unmap_io(const struct ublk_queue *ubq,
 494                 const struct request *req,
 495                 struct ublk_io *io)
 496 {
 497         const unsigned int rq_bytes = blk_rq_bytes(req);
 498
 499         if (req_op(req) == REQ_OP_READ && ublk_rq_has_data(req)) {
 500                 struct ublk_map_data data = {
 501                         .ubq    =       ubq,
 502                         .rq     =       req,
 503                         .io     =       io,
 504                         .max_bytes =    io->res,
 505                 };
 506
 507                 WARN_ON_ONCE(io->res > rq_bytes);
 508
 509                 ublk_copy_user_pages(&data, false);
 510
 511                 return io->res - data.max_bytes;
 512         }
 513         return rq_bytes;
 514 }
 515
 516 static inline unsigned int ublk_req_build_flags(struct request *req)
 517 {
 518         unsigned flags = 0;
 519
 520         if (req->cmd_flags & REQ_FAILFAST_DEV)
 521                 flags |= UBLK_IO_F_FAILFAST_DEV;
 522
 523         if (req->cmd_flags & REQ_FAILFAST_TRANSPORT)
 524                 flags |= UBLK_IO_F_FAILFAST_TRANSPORT;
 525
 526         if (req->cmd_flags & REQ_FAILFAST_DRIVER)
 527                 flags |= UBLK_IO_F_FAILFAST_DRIVER;
 528
 529         if (req->cmd_flags & REQ_META)
 530                 flags |= UBLK_IO_F_META;
 531
 532         if (req->cmd_flags & REQ_FUA)
 533                 flags |= UBLK_IO_F_FUA;
 534
 535         if (req->cmd_flags & REQ_NOUNMAP)
 536                 flags |= UBLK_IO_F_NOUNMAP;
 537
 538         if (req->cmd_flags & REQ_SWAP)
 539                 flags |= UBLK_IO_F_SWAP;
 540
 541         return flags;
 542 }
 543
 544 static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req)
 545 {
 546         struct ublksrv_io_desc *iod = ublk_get_iod(ubq, req->tag);
 547         struct ublk_io *io = &ubq->ios[req->tag];
 548         u32 ublk_op;
 549
 550         switch (req_op(req)) {
 551         case REQ_OP_READ:
 552                 ublk_op = UBLK_IO_OP_READ;
 553                 break;
 554         case REQ_OP_WRITE:
 555                 ublk_op = UBLK_IO_OP_WRITE;
 556                 break;
 557         case REQ_OP_FLUSH:
 558                 ublk_op = UBLK_IO_OP_FLUSH;
 559                 break;
 560         case REQ_OP_DISCARD:
 561                 ublk_op = UBLK_IO_OP_DISCARD;
 562                 break;
 563         case REQ_OP_WRITE_ZEROES:
 564                 ublk_op = UBLK_IO_OP_WRITE_ZEROES;
 565                 break;
 566         default:
 567                 return BLK_STS_IOERR;
 568         }
 569
 570         /* need to translate since kernel may change */
 571         iod->op_flags = ublk_op | ublk_req_build_flags(req);
 572         iod->nr_sectors = blk_rq_sectors(req);
 573         iod->start_sector = blk_rq_pos(req);
 574         iod->addr = io->addr;
 575
 576         return BLK_STS_OK;
 577 }
 578
 579 static inline struct ublk_uring_cmd_pdu *ublk_get_uring_cmd_pdu(
 580                 struct io_uring_cmd *ioucmd)
 581 {
 582         return (struct ublk_uring_cmd_pdu *)&ioucmd->pdu;
 583 }
 584
 585 static inline bool ubq_daemon_is_dying(struct ublk_queue *ubq)
 586 {
 587         return ubq->ubq_daemon->flags & PF_EXITING;
 588 }
 589
 590 /* todo: handle partial completion */
 591 static void ublk_complete_rq(struct request *req)
 592 {
 593         struct ublk_queue *ubq = req->mq_hctx->driver_data;
 594         struct ublk_io *io = &ubq->ios[req->tag];
 595         unsigned int unmapped_bytes;
 596
 597         /* failed read IO if nothing is read */
 598         if (!io->res && req_op(req) == REQ_OP_READ)
 599                 io->res = -EIO;
 600
 601         if (io->res < 0) {
 602                 blk_mq_end_request(req, errno_to_blk_status(io->res));
 603                 return;
 604         }
 605
 606         /*
 607          * FLUSH or DISCARD usually won't return bytes returned, so end them
 608          * directly.
 609          *
 610          * Both the two needn't unmap.
 611          */
 612         if (req_op(req) != REQ_OP_READ && req_op(req) != REQ_OP_WRITE) {
 613                 blk_mq_end_request(req, BLK_STS_OK);
 614                 return;
 615         }
 616
 617         /* for READ request, writing data in iod->addr to rq buffers */
 618         unmapped_bytes = ublk_unmap_io(ubq, req, io);
 619
 620         /*
 621          * Extremely impossible since we got data filled in just before
 622          *
 623          * Re-read simply for this unlikely case.
 624          */
 625         if (unlikely(unmapped_bytes < io->res))
 626                 io->res = unmapped_bytes;
 627
 628         if (blk_update_request(req, BLK_STS_OK, io->res))
 629                 blk_mq_requeue_request(req, true);
 630         else
 631                 __blk_mq_end_request(req, BLK_STS_OK);
 632 }
 633
 634 /*
 635  * Since __ublk_rq_task_work always fails requests immediately during
 636  * exiting, __ublk_fail_req() is only called from abort context during
 637  * exiting. So lock is unnecessary.
 638  *
 639  * Also aborting may not be started yet, keep in mind that one failed
 640  * request may be issued by block layer again.
 641  */
 642 static void __ublk_fail_req(struct ublk_queue *ubq, struct ublk_io *io,
 643                 struct request *req)
 644 {
 645         WARN_ON_ONCE(io->flags & UBLK_IO_FLAG_ACTIVE);
 646
 647         if (!(io->flags & UBLK_IO_FLAG_ABORTED)) {
 648                 io->flags |= UBLK_IO_FLAG_ABORTED;
 649                 if (ublk_queue_can_use_recovery_reissue(ubq))
 650                         blk_mq_requeue_request(req, false);
 651                 else
 652                         blk_mq_end_request(req, BLK_STS_IOERR);
 653         }
 654 }
 655
 656 static void ubq_complete_io_cmd(struct ublk_io *io, int res)
 657 {
 658         /* mark this cmd owned by ublksrv */
 659         io->flags |= UBLK_IO_FLAG_OWNED_BY_SRV;
 660
 661         /*
 662          * clear ACTIVE since we are done with this sqe/cmd slot
 663          * We can only accept io cmd in case of being not active.
 664          */
 665         io->flags &= ~UBLK_IO_FLAG_ACTIVE;
 666
 667         /* tell ublksrv one io request is coming */
 668         io_uring_cmd_done(io->cmd, res, 0);
 669 }
 670
 671 #define UBLK_REQUEUE_DELAY_MS   3
 672
 673 static inline void __ublk_abort_rq(struct ublk_queue *ubq,
 674                 struct request *rq)
 675 {
 676         /* We cannot process this rq so just requeue it. */
 677         if (ublk_queue_can_use_recovery(ubq))
 678                 blk_mq_requeue_request(rq, false);
 679         else
 680                 blk_mq_end_request(rq, BLK_STS_IOERR);
 681
 682         mod_delayed_work(system_wq, &ubq->dev->monitor_work, 0);
 683 }
 684
 685 static inline void __ublk_rq_task_work(struct request *req)
 686 {
 687         struct ublk_queue *ubq = req->mq_hctx->driver_data;
 688         int tag = req->tag;
 689         struct ublk_io *io = &ubq->ios[tag];
 690         unsigned int mapped_bytes;
 691
 692         pr_devel("%s: complete: op %d, qid %d tag %d io_flags %x addr %llx\n",
 693                         __func__, io->cmd->cmd_op, ubq->q_id, req->tag, io->flags,
 694                         ublk_get_iod(ubq, req->tag)->addr);
 695
 696         /*
 697          * Task is exiting if either:
 698          *
 699          * (1) current != ubq_daemon.
 700          * io_uring_cmd_complete_in_task() tries to run task_work
 701          * in a workqueue if ubq_daemon(cmd's task) is PF_EXITING.
 702          *
 703          * (2) current->flags & PF_EXITING.
 704          */
 705         if (unlikely(current != ubq->ubq_daemon || current->flags & PF_EXITING)) {
 706                 __ublk_abort_rq(ubq, req);
 707                 return;
 708         }
 709
 710         if (ublk_need_get_data(ubq) &&
 711                         (req_op(req) == REQ_OP_WRITE ||
 712                         req_op(req) == REQ_OP_FLUSH)) {
 713                 /*
 714                  * We have not handled UBLK_IO_NEED_GET_DATA command yet,
 715                  * so immepdately pass UBLK_IO_RES_NEED_GET_DATA to ublksrv
 716                  * and notify it.
 717                  */
 718                 if (!(io->flags & UBLK_IO_FLAG_NEED_GET_DATA)) {
 719                         io->flags |= UBLK_IO_FLAG_NEED_GET_DATA;
 720                         pr_devel("%s: need get data. op %d, qid %d tag %d io_flags %x\n",
 721                                         __func__, io->cmd->cmd_op, ubq->q_id,
 722                                         req->tag, io->flags);
 723                         ubq_complete_io_cmd(io, UBLK_IO_RES_NEED_GET_DATA);
 724                         return;
 725                 }
 726                 /*
 727                  * We have handled UBLK_IO_NEED_GET_DATA command,
 728                  * so clear UBLK_IO_FLAG_NEED_GET_DATA now and just
 729                  * do the copy work.
 730                  */
 731                 io->flags &= ~UBLK_IO_FLAG_NEED_GET_DATA;
 732                 /* update iod->addr because ublksrv may have passed a new io buffer */
 733                 ublk_get_iod(ubq, req->tag)->addr = io->addr;
 734                 pr_devel("%s: update iod->addr: op %d, qid %d tag %d io_flags %x addr %llx\n",
 735                                 __func__, io->cmd->cmd_op, ubq->q_id, req->tag, io->flags,
 736                                 ublk_get_iod(ubq, req->tag)->addr);
 737         }
 738
 739         mapped_bytes = ublk_map_io(ubq, req, io);
 740
 741         /* partially mapped, update io descriptor */
 742         if (unlikely(mapped_bytes != blk_rq_bytes(req))) {
 743                 /*
 744                  * Nothing mapped, retry until we succeed.
 745                  *
 746                  * We may never succeed in mapping any bytes here because
 747                  * of OOM. TODO: reserve one buffer with single page pinned
 748                  * for providing forward progress guarantee.
 749                  */
 750                 if (unlikely(!mapped_bytes)) {
 751                         blk_mq_requeue_request(req, false);
 752                         blk_mq_delay_kick_requeue_list(req->q,
 753                                         UBLK_REQUEUE_DELAY_MS);
 754                         return;
 755                 }
 756
 757                 ublk_get_iod(ubq, req->tag)->nr_sectors =
 758                         mapped_bytes >> 9;
 759         }
 760
 761         ubq_complete_io_cmd(io, UBLK_IO_RES_OK);
 762 }
 763
 764 static void ublk_rq_task_work_cb(struct io_uring_cmd *cmd)
 765 {
 766         struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
 767
 768         __ublk_rq_task_work(pdu->req);
 769 }
 770
 771 static void ublk_rq_task_work_fn(struct callback_head *work)
 772 {
 773         struct ublk_rq_data *data = container_of(work,
 774                         struct ublk_rq_data, work);
 775         struct request *req = blk_mq_rq_from_pdu(data);
 776
 777         __ublk_rq_task_work(req);
 778 }
 779
 780 static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx,
 781                 const struct blk_mq_queue_data *bd)
 782 {
 783         struct ublk_queue *ubq = hctx->driver_data;
 784         struct request *rq = bd->rq;
 785         blk_status_t res;
 786
 787         /* fill iod to slot in io cmd buffer */
 788         res = ublk_setup_iod(ubq, rq);
 789         if (unlikely(res != BLK_STS_OK))
 790                 return BLK_STS_IOERR;
 791         /* With recovery feature enabled, force_abort is set in
 792          * ublk_stop_dev() before calling del_gendisk(). We have to
 793          * abort all requeued and new rqs here to let del_gendisk()
 794          * move on. Besides, we cannot not call io_uring_cmd_complete_in_task()
 795          * to avoid UAF on io_uring ctx.
 796          *
 797          * Note: force_abort is guaranteed to be seen because it is set
 798          * before request queue is unqiuesced.
 799          */
 800         if (ublk_queue_can_use_recovery(ubq) && unlikely(ubq->force_abort))
 801                 return BLK_STS_IOERR;
 802
 803         blk_mq_start_request(bd->rq);
 804
 805         if (unlikely(ubq_daemon_is_dying(ubq))) {
 806  fail:
 807                 __ublk_abort_rq(ubq, rq);
 808                 return BLK_STS_OK;
 809         }
 810
 811         if (ublk_can_use_task_work(ubq)) {
 812                 struct ublk_rq_data *data = blk_mq_rq_to_pdu(rq);
 813                 enum task_work_notify_mode notify_mode = bd->last ?
 814                         TWA_SIGNAL_NO_IPI : TWA_NONE;
 815
 816                 if (task_work_add(ubq->ubq_daemon, &data->work, notify_mode))
 817                         goto fail;
 818         } else {
 819                 struct ublk_io *io = &ubq->ios[rq->tag];
 820                 struct io_uring_cmd *cmd = io->cmd;
 821                 struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
 822
 823                 /*
 824                  * If the check pass, we know that this is a re-issued request aborted
 825                  * previously in monitor_work because the ubq_daemon(cmd's task) is
 826                  * PF_EXITING. We cannot call io_uring_cmd_complete_in_task() anymore
 827                  * because this ioucmd's io_uring context may be freed now if no inflight
 828                  * ioucmd exists. Otherwise we may cause null-deref in ctx->fallback_work.
 829                  *
 830                  * Note: monitor_work sets UBLK_IO_FLAG_ABORTED and ends this request(releasing
 831                  * the tag). Then the request is re-started(allocating the tag) and we are here.
 832                  * Since releasing/allocating a tag implies smp_mb(), finding UBLK_IO_FLAG_ABORTED
 833                  * guarantees that here is a re-issued request aborted previously.
 834                  */
 835                 if ((io->flags & UBLK_IO_FLAG_ABORTED))
 836                         goto fail;
 837
 838                 pdu->req = rq;
 839                 io_uring_cmd_complete_in_task(cmd, ublk_rq_task_work_cb);
 840         }
 841
 842         return BLK_STS_OK;
 843 }
 844
 845 static void ublk_commit_rqs(struct blk_mq_hw_ctx *hctx)
 846 {
 847         struct ublk_queue *ubq = hctx->driver_data;
 848
 849         if (ublk_can_use_task_work(ubq))
 850                 __set_notify_signal(ubq->ubq_daemon);
 851 }
 852
 853 static int ublk_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
 854                 unsigned int hctx_idx)
 855 {
 856         struct ublk_device *ub = driver_data;
 857         struct ublk_queue *ubq = ublk_get_queue(ub, hctx->queue_num);
 858
 859         hctx->driver_data = ubq;
 860         return 0;
 861 }
 862
 863 static int ublk_init_rq(struct blk_mq_tag_set *set, struct request *req,
 864                 unsigned int hctx_idx, unsigned int numa_node)
 865 {
 866         struct ublk_rq_data *data = blk_mq_rq_to_pdu(req);
 867
 868         init_task_work(&data->work, ublk_rq_task_work_fn);
 869         return 0;
 870 }
 871
 872 static const struct blk_mq_ops ublk_mq_ops = {
 873         .queue_rq       = ublk_queue_rq,
 874         .commit_rqs     = ublk_commit_rqs,
 875         .init_hctx      = ublk_init_hctx,
 876         .init_request   = ublk_init_rq,
 877 };
 878
 879 static int ublk_ch_open(struct inode *inode, struct file *filp)
 880 {
 881         struct ublk_device *ub = container_of(inode->i_cdev,
 882                         struct ublk_device, cdev);
 883
 884         if (test_and_set_bit(UB_STATE_OPEN, &ub->state))
 885                 return -EBUSY;
 886         filp->private_data = ub;
 887         return 0;
 888 }
 889
 890 static int ublk_ch_release(struct inode *inode, struct file *filp)
 891 {
 892         struct ublk_device *ub = filp->private_data;
 893
 894         clear_bit(UB_STATE_OPEN, &ub->state);
 895         return 0;
 896 }
 897
 898 /* map pre-allocated per-queue cmd buffer to ublksrv daemon */
 899 static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma)
 900 {
 901         struct ublk_device *ub = filp->private_data;
 902         size_t sz = vma->vm_end - vma->vm_start;
 903         unsigned max_sz = UBLK_MAX_QUEUE_DEPTH * sizeof(struct ublksrv_io_desc);
 904         unsigned long pfn, end, phys_off = vma->vm_pgoff << PAGE_SHIFT;
 905         int q_id, ret = 0;
 906
 907         spin_lock(&ub->mm_lock);
 908         if (!ub->mm)
 909                 ub->mm = current->mm;
 910         if (current->mm != ub->mm)
 911                 ret = -EINVAL;
 912         spin_unlock(&ub->mm_lock);
 913
 914         if (ret)
 915                 return ret;
 916
 917         if (vma->vm_flags & VM_WRITE)
 918                 return -EPERM;
 919
 920         end = UBLKSRV_CMD_BUF_OFFSET + ub->dev_info.nr_hw_queues * max_sz;
 921         if (phys_off < UBLKSRV_CMD_BUF_OFFSET || phys_off >= end)
 922                 return -EINVAL;
 923
 924         q_id = (phys_off - UBLKSRV_CMD_BUF_OFFSET) / max_sz;
 925         pr_devel("%s: qid %d, pid %d, addr %lx pg_off %lx sz %lu\n",
 926                         __func__, q_id, current->pid, vma->vm_start,
 927                         phys_off, (unsigned long)sz);
 928
 929         if (sz != ublk_queue_cmd_buf_size(ub, q_id))
 930                 return -EINVAL;
 931
 932         pfn = virt_to_phys(ublk_queue_cmd_buf(ub, q_id)) >> PAGE_SHIFT;
 933         return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
 934 }
 935
 936 static void ublk_commit_completion(struct ublk_device *ub,
 937                 struct ublksrv_io_cmd *ub_cmd)
 938 {
 939         u32 qid = ub_cmd->q_id, tag = ub_cmd->tag;
 940         struct ublk_queue *ubq = ublk_get_queue(ub, qid);
 941         struct ublk_io *io = &ubq->ios[tag];
 942         struct request *req;
 943
 944         /* now this cmd slot is owned by nbd driver */
 945         io->flags &= ~UBLK_IO_FLAG_OWNED_BY_SRV;
 946         io->res = ub_cmd->result;
 947
 948         /* find the io request and complete */
 949         req = blk_mq_tag_to_rq(ub->tag_set.tags[qid], tag);
 950
 951         if (req && likely(!blk_should_fake_timeout(req->q)))
 952                 ublk_complete_rq(req);
 953 }
 954
 955 /*
 956  * When ->ubq_daemon is exiting, either new request is ended immediately,
 957  * or any queued io command is drained, so it is safe to abort queue
 958  * lockless
 959  */
 960 static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq)
 961 {
 962         int i;
 963
 964         if (!ublk_get_device(ub))
 965                 return;
 966
 967         for (i = 0; i < ubq->q_depth; i++) {
 968                 struct ublk_io *io = &ubq->ios[i];
 969
 970                 if (!(io->flags & UBLK_IO_FLAG_ACTIVE)) {
 971                         struct request *rq;
 972
 973                         /*
 974                          * Either we fail the request or ublk_rq_task_work_fn
 975                          * will do it
 976                          */
 977                         rq = blk_mq_tag_to_rq(ub->tag_set.tags[ubq->q_id], i);
 978                         if (rq)
 979                                 __ublk_fail_req(ubq, io, rq);
 980                 }
 981         }
 982         ublk_put_device(ub);
 983 }
 984
 985 static void ublk_daemon_monitor_work(struct work_struct *work)
 986 {
 987         struct ublk_device *ub =
 988                 container_of(work, struct ublk_device, monitor_work.work);
 989         int i;
 990
 991         for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
 992                 struct ublk_queue *ubq = ublk_get_queue(ub, i);
 993
 994                 if (ubq_daemon_is_dying(ubq)) {
 995                         if (ublk_queue_can_use_recovery(ubq))
 996                                 schedule_work(&ub->quiesce_work);
 997                         else
 998                                 schedule_work(&ub->stop_work);
 999
1000                         /* abort queue is for making forward progress */
1001                         ublk_abort_queue(ub, ubq);
1002                 }
1003         }
1004
1005         /*
1006          * We can't schedule monitor work after ub's state is not UBLK_S_DEV_LIVE.
1007          * after ublk_remove() or __ublk_quiesce_dev() is started.
1008          *
1009          * No need ub->mutex, monitor work are canceled after state is marked
1010          * as not LIVE, so new state is observed reliably.
1011          */
1012         if (ub->dev_info.state == UBLK_S_DEV_LIVE)
1013                 schedule_delayed_work(&ub->monitor_work,
1014                                 UBLK_DAEMON_MONITOR_PERIOD);
1015 }
1016
1017 static inline bool ublk_queue_ready(struct ublk_queue *ubq)
1018 {
1019         return ubq->nr_io_ready == ubq->q_depth;
1020 }
1021
1022 static void ublk_cancel_queue(struct ublk_queue *ubq)
1023 {
1024         int i;
1025
1026         if (!ublk_queue_ready(ubq))
1027                 return;
1028
1029         for (i = 0; i < ubq->q_depth; i++) {
1030                 struct ublk_io *io = &ubq->ios[i];
1031
1032                 if (io->flags & UBLK_IO_FLAG_ACTIVE)
1033                         io_uring_cmd_done(io->cmd, UBLK_IO_RES_ABORT, 0);
1034         }
1035
1036         /* all io commands are canceled */
1037         ubq->nr_io_ready = 0;
1038 }
1039
1040 /* Cancel all pending commands, must be called after del_gendisk() returns */
1041 static void ublk_cancel_dev(struct ublk_device *ub)
1042 {
1043         int i;
1044
1045         for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1046                 ublk_cancel_queue(ublk_get_queue(ub, i));
1047 }
1048
1049 static bool ublk_check_inflight_rq(struct request *rq, void *data)
1050 {
1051         bool *idle = data;
1052
1053         if (blk_mq_request_started(rq)) {
1054                 *idle = false;
1055                 return false;
1056         }
1057         return true;
1058 }
1059
1060 static void ublk_wait_tagset_rqs_idle(struct ublk_device *ub)
1061 {
1062         bool idle;
1063
1064         WARN_ON_ONCE(!blk_queue_quiesced(ub->ub_disk->queue));
1065         while (true) {
1066                 idle = true;
1067                 blk_mq_tagset_busy_iter(&ub->tag_set,
1068                                 ublk_check_inflight_rq, &idle);
1069                 if (idle)
1070                         break;
1071                 msleep(UBLK_REQUEUE_DELAY_MS);
1072         }
1073 }
1074
1075 static void __ublk_quiesce_dev(struct ublk_device *ub)
1076 {
1077         pr_devel("%s: quiesce ub: dev_id %d state %s\n",
1078                         __func__, ub->dev_info.dev_id,
1079                         ub->dev_info.state == UBLK_S_DEV_LIVE ?
1080                         "LIVE" : "QUIESCED");
1081         blk_mq_quiesce_queue(ub->ub_disk->queue);
1082         ublk_wait_tagset_rqs_idle(ub);
1083         ub->dev_info.state = UBLK_S_DEV_QUIESCED;
1084         ublk_cancel_dev(ub);
1085         /* we are going to release task_struct of ubq_daemon and resets
1086          * ->ubq_daemon to NULL. So in monitor_work, check on ubq_daemon causes UAF.
1087          * Besides, monitor_work is not necessary in QUIESCED state since we have
1088          * already scheduled quiesce_work and quiesced all ubqs.
1089          *
1090          * Do not let monitor_work schedule itself if state it QUIESCED. And we cancel
1091          * it here and re-schedule it in END_USER_RECOVERY to avoid UAF.
1092          */
1093         cancel_delayed_work_sync(&ub->monitor_work);
1094 }
1095
1096 static void ublk_quiesce_work_fn(struct work_struct *work)
1097 {
1098         struct ublk_device *ub =
1099                 container_of(work, struct ublk_device, quiesce_work);
1100
1101         mutex_lock(&ub->mutex);
1102         if (ub->dev_info.state != UBLK_S_DEV_LIVE)
1103                 goto unlock;
1104         __ublk_quiesce_dev(ub);
1105  unlock:
1106         mutex_unlock(&ub->mutex);
1107 }
1108
1109 static void ublk_unquiesce_dev(struct ublk_device *ub)
1110 {
1111         int i;
1112
1113         pr_devel("%s: unquiesce ub: dev_id %d state %s\n",
1114                         __func__, ub->dev_info.dev_id,
1115                         ub->dev_info.state == UBLK_S_DEV_LIVE ?
1116                         "LIVE" : "QUIESCED");
1117         /* quiesce_work has run. We let requeued rqs be aborted
1118          * before running fallback_wq. "force_abort" must be seen
1119          * after request queue is unqiuesced. Then del_gendisk()
1120          * can move on.
1121          */
1122         for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1123                 ublk_get_queue(ub, i)->force_abort = true;
1124
1125         blk_mq_unquiesce_queue(ub->ub_disk->queue);
1126         /* We may have requeued some rqs in ublk_quiesce_queue() */
1127         blk_mq_kick_requeue_list(ub->ub_disk->queue);
1128 }
1129
1130 static void ublk_stop_dev(struct ublk_device *ub)
1131 {
1132         mutex_lock(&ub->mutex);
1133         if (ub->dev_info.state == UBLK_S_DEV_DEAD)
1134                 goto unlock;
1135         if (ublk_can_use_recovery(ub)) {
1136                 if (ub->dev_info.state == UBLK_S_DEV_LIVE)
1137                         __ublk_quiesce_dev(ub);
1138                 ublk_unquiesce_dev(ub);
1139         }
1140         del_gendisk(ub->ub_disk);
1141         ub->dev_info.state = UBLK_S_DEV_DEAD;
1142         ub->dev_info.ublksrv_pid = -1;
1143         put_disk(ub->ub_disk);
1144         ub->ub_disk = NULL;
1145  unlock:
1146         ublk_cancel_dev(ub);
1147         mutex_unlock(&ub->mutex);
1148         cancel_delayed_work_sync(&ub->monitor_work);
1149 }
1150
1151 /* device can only be started after all IOs are ready */
1152 static void ublk_mark_io_ready(struct ublk_device *ub, struct ublk_queue *ubq)
1153 {
1154         mutex_lock(&ub->mutex);
1155         ubq->nr_io_ready++;
1156         if (ublk_queue_ready(ubq)) {
1157                 ubq->ubq_daemon = current;
1158                 get_task_struct(ubq->ubq_daemon);
1159                 ub->nr_queues_ready++;
1160         }
1161         if (ub->nr_queues_ready == ub->dev_info.nr_hw_queues)
1162                 complete_all(&ub->completion);
1163         mutex_unlock(&ub->mutex);
1164 }
1165
1166 static void ublk_handle_need_get_data(struct ublk_device *ub, int q_id,
1167                 int tag, struct io_uring_cmd *cmd)
1168 {
1169         struct ublk_queue *ubq = ublk_get_queue(ub, q_id);
1170         struct request *req = blk_mq_tag_to_rq(ub->tag_set.tags[q_id], tag);
1171
1172         if (ublk_can_use_task_work(ubq)) {
1173                 struct ublk_rq_data *data = blk_mq_rq_to_pdu(req);
1174
1175                 /* should not fail since we call it just in ubq->ubq_daemon */
1176                 task_work_add(ubq->ubq_daemon, &data->work, TWA_SIGNAL_NO_IPI);
1177         } else {
1178                 struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
1179
1180                 pdu->req = req;
1181                 io_uring_cmd_complete_in_task(cmd, ublk_rq_task_work_cb);
1182         }
1183 }
1184
1185 static int ublk_ch_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
1186 {
1187         struct ublksrv_io_cmd *ub_cmd = (struct ublksrv_io_cmd *)cmd->cmd;
1188         struct ublk_device *ub = cmd->file->private_data;
1189         struct ublk_queue *ubq;
1190         struct ublk_io *io;
1191         u32 cmd_op = cmd->cmd_op;
1192         unsigned tag = ub_cmd->tag;
1193         int ret = -EINVAL;
1194
1195         pr_devel("%s: received: cmd op %d queue %d tag %d result %d\n",
1196                         __func__, cmd->cmd_op, ub_cmd->q_id, tag,
1197                         ub_cmd->result);
1198
1199         if (!(issue_flags & IO_URING_F_SQE128))
1200                 goto out;
1201
1202         if (ub_cmd->q_id >= ub->dev_info.nr_hw_queues)
1203                 goto out;
1204
1205         ubq = ublk_get_queue(ub, ub_cmd->q_id);
1206         if (!ubq || ub_cmd->q_id != ubq->q_id)
1207                 goto out;
1208
1209         if (ubq->ubq_daemon && ubq->ubq_daemon != current)
1210                 goto out;
1211
1212         if (tag >= ubq->q_depth)
1213                 goto out;
1214
1215         io = &ubq->ios[tag];
1216
1217         /* there is pending io cmd, something must be wrong */
1218         if (io->flags & UBLK_IO_FLAG_ACTIVE) {
1219                 ret = -EBUSY;
1220                 goto out;
1221         }
1222
1223         /*
1224          * ensure that the user issues UBLK_IO_NEED_GET_DATA
1225          * iff the driver have set the UBLK_IO_FLAG_NEED_GET_DATA.
1226          */
1227         if ((!!(io->flags & UBLK_IO_FLAG_NEED_GET_DATA))
1228                         ^ (cmd_op == UBLK_IO_NEED_GET_DATA))
1229                 goto out;
1230
1231         switch (cmd_op) {
1232         case UBLK_IO_FETCH_REQ:
1233                 /* UBLK_IO_FETCH_REQ is only allowed before queue is setup */
1234                 if (ublk_queue_ready(ubq)) {
1235                         ret = -EBUSY;
1236                         goto out;
1237                 }
1238                 /*
1239                  * The io is being handled by server, so COMMIT_RQ is expected
1240                  * instead of FETCH_REQ
1241                  */
1242                 if (io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)
1243                         goto out;
1244                 /* FETCH_RQ has to provide IO buffer */
1245                 if (!ub_cmd->addr)
1246                         goto out;
1247                 io->cmd = cmd;
1248                 io->flags |= UBLK_IO_FLAG_ACTIVE;
1249                 io->addr = ub_cmd->addr;
1250
1251                 ublk_mark_io_ready(ub, ubq);
1252                 break;
1253         case UBLK_IO_COMMIT_AND_FETCH_REQ:
1254                 /* FETCH_RQ has to provide IO buffer */
1255                 if (!ub_cmd->addr)
1256                         goto out;
1257                 if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV))
1258                         goto out;
1259                 io->addr = ub_cmd->addr;
1260                 io->flags |= UBLK_IO_FLAG_ACTIVE;
1261                 io->cmd = cmd;
1262                 ublk_commit_completion(ub, ub_cmd);
1263                 break;
1264         case UBLK_IO_NEED_GET_DATA:
1265                 if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV))
1266                         goto out;
1267                 io->addr = ub_cmd->addr;
1268                 io->cmd = cmd;
1269                 io->flags |= UBLK_IO_FLAG_ACTIVE;
1270                 ublk_handle_need_get_data(ub, ub_cmd->q_id, ub_cmd->tag, cmd);
1271                 break;
1272         default:
1273                 goto out;
1274         }
1275         return -EIOCBQUEUED;
1276
1277  out:
1278         io_uring_cmd_done(cmd, ret, 0);
1279         pr_devel("%s: complete: cmd op %d, tag %d ret %x io_flags %x\n",
1280                         __func__, cmd_op, tag, ret, io->flags);
1281         return -EIOCBQUEUED;
1282 }
1283
1284 static const struct file_operations ublk_ch_fops = {
1285         .owner = THIS_MODULE,
1286         .open = ublk_ch_open,
1287         .release = ublk_ch_release,
1288         .llseek = no_llseek,
1289         .uring_cmd = ublk_ch_uring_cmd,
1290         .mmap = ublk_ch_mmap,
1291 };
1292
1293 static void ublk_deinit_queue(struct ublk_device *ub, int q_id)
1294 {
1295         int size = ublk_queue_cmd_buf_size(ub, q_id);
1296         struct ublk_queue *ubq = ublk_get_queue(ub, q_id);
1297
1298         if (ubq->ubq_daemon)
1299                 put_task_struct(ubq->ubq_daemon);
1300         if (ubq->io_cmd_buf)
1301                 free_pages((unsigned long)ubq->io_cmd_buf, get_order(size));
1302 }
1303
1304 static int ublk_init_queue(struct ublk_device *ub, int q_id)
1305 {
1306         struct ublk_queue *ubq = ublk_get_queue(ub, q_id);
1307         gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO;
1308         void *ptr;
1309         int size;
1310
1311         ubq->flags = ub->dev_info.flags;
1312         ubq->q_id = q_id;
1313         ubq->q_depth = ub->dev_info.queue_depth;
1314         size = ublk_queue_cmd_buf_size(ub, q_id);
1315
1316         ptr = (void *) __get_free_pages(gfp_flags, get_order(size));
1317         if (!ptr)
1318                 return -ENOMEM;
1319
1320         ubq->io_cmd_buf = ptr;
1321         ubq->dev = ub;
1322         return 0;
1323 }
1324
1325 static void ublk_deinit_queues(struct ublk_device *ub)
1326 {
1327         int nr_queues = ub->dev_info.nr_hw_queues;
1328         int i;
1329
1330         if (!ub->__queues)
1331                 return;
1332
1333         for (i = 0; i < nr_queues; i++)
1334                 ublk_deinit_queue(ub, i);
1335         kfree(ub->__queues);
1336 }
1337
1338 static int ublk_init_queues(struct ublk_device *ub)
1339 {
1340         int nr_queues = ub->dev_info.nr_hw_queues;
1341         int depth = ub->dev_info.queue_depth;
1342         int ubq_size = sizeof(struct ublk_queue) + depth * sizeof(struct ublk_io);
1343         int i, ret = -ENOMEM;
1344
1345         ub->queue_size = ubq_size;
1346         ub->__queues = kcalloc(nr_queues, ubq_size, GFP_KERNEL);
1347         if (!ub->__queues)
1348                 return ret;
1349
1350         for (i = 0; i < nr_queues; i++) {
1351                 if (ublk_init_queue(ub, i))
1352                         goto fail;
1353         }
1354
1355         init_completion(&ub->completion);
1356         return 0;
1357
1358  fail:
1359         ublk_deinit_queues(ub);
1360         return ret;
1361 }
1362
1363 static int ublk_alloc_dev_number(struct ublk_device *ub, int idx)
1364 {
1365         int i = idx;
1366         int err;
1367
1368         spin_lock(&ublk_idr_lock);
1369         /* allocate id, if @id >= 0, we're requesting that specific id */
1370         if (i >= 0) {
1371                 err = idr_alloc(&ublk_index_idr, ub, i, i + 1, GFP_NOWAIT);
1372                 if (err == -ENOSPC)
1373                         err = -EEXIST;
1374         } else {
1375                 err = idr_alloc(&ublk_index_idr, ub, 0, 0, GFP_NOWAIT);
1376         }
1377         spin_unlock(&ublk_idr_lock);
1378
1379         if (err >= 0)
1380                 ub->ub_number = err;
1381
1382         return err;
1383 }
1384
1385 static void ublk_free_dev_number(struct ublk_device *ub)
1386 {
1387         spin_lock(&ublk_idr_lock);
1388         idr_remove(&ublk_index_idr, ub->ub_number);
1389         wake_up_all(&ublk_idr_wq);
1390         spin_unlock(&ublk_idr_lock);
1391 }
1392
1393 static void ublk_cdev_rel(struct device *dev)
1394 {
1395         struct ublk_device *ub = container_of(dev, struct ublk_device, cdev_dev);
1396
1397         blk_mq_free_tag_set(&ub->tag_set);
1398         ublk_deinit_queues(ub);
1399         ublk_free_dev_number(ub);
1400         mutex_destroy(&ub->mutex);
1401         kfree(ub);
1402 }
1403
1404 static int ublk_add_chdev(struct ublk_device *ub)
1405 {
1406         struct device *dev = &ub->cdev_dev;
1407         int minor = ub->ub_number;
1408         int ret;
1409
1410         dev->parent = ublk_misc.this_device;
1411         dev->devt = MKDEV(MAJOR(ublk_chr_devt), minor);
1412         dev->class = ublk_chr_class;
1413         dev->release = ublk_cdev_rel;
1414         device_initialize(dev);
1415
1416         ret = dev_set_name(dev, "ublkc%d", minor);
1417         if (ret)
1418                 goto fail;
1419
1420         cdev_init(&ub->cdev, &ublk_ch_fops);
1421         ret = cdev_device_add(&ub->cdev, dev);
1422         if (ret)
1423                 goto fail;
1424         return 0;
1425  fail:
1426         put_device(dev);
1427         return ret;
1428 }
1429
1430 static void ublk_stop_work_fn(struct work_struct *work)
1431 {
1432         struct ublk_device *ub =
1433                 container_of(work, struct ublk_device, stop_work);
1434
1435         ublk_stop_dev(ub);
1436 }
1437
1438 /* align max io buffer size with PAGE_SIZE */
1439 static void ublk_align_max_io_size(struct ublk_device *ub)
1440 {
1441         unsigned int max_io_bytes = ub->dev_info.max_io_buf_bytes;
1442
1443         ub->dev_info.max_io_buf_bytes =
1444                 round_down(max_io_bytes, PAGE_SIZE);
1445 }
1446
1447 static int ublk_add_tag_set(struct ublk_device *ub)
1448 {
1449         ub->tag_set.ops = &ublk_mq_ops;
1450         ub->tag_set.nr_hw_queues = ub->dev_info.nr_hw_queues;
1451         ub->tag_set.queue_depth = ub->dev_info.queue_depth;
1452         ub->tag_set.numa_node = NUMA_NO_NODE;
1453         ub->tag_set.cmd_size = sizeof(struct ublk_rq_data);
1454         ub->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
1455         ub->tag_set.driver_data = ub;
1456         return blk_mq_alloc_tag_set(&ub->tag_set);
1457 }
1458
1459 static void ublk_remove(struct ublk_device *ub)
1460 {
1461         ublk_stop_dev(ub);
1462         cancel_work_sync(&ub->stop_work);
1463         cancel_work_sync(&ub->quiesce_work);
1464         cdev_device_del(&ub->cdev, &ub->cdev_dev);
1465         put_device(&ub->cdev_dev);
1466 }
1467
1468 static struct ublk_device *ublk_get_device_from_id(int idx)
1469 {
1470         struct ublk_device *ub = NULL;
1471
1472         if (idx < 0)
1473                 return NULL;
1474
1475         spin_lock(&ublk_idr_lock);
1476         ub = idr_find(&ublk_index_idr, idx);
1477         if (ub)
1478                 ub = ublk_get_device(ub);
1479         spin_unlock(&ublk_idr_lock);
1480
1481         return ub;
1482 }
1483
1484 static int ublk_ctrl_start_dev(struct io_uring_cmd *cmd)
1485 {
1486         struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1487         int ublksrv_pid = (int)header->data[0];
1488         struct ublk_device *ub;
1489         struct gendisk *disk;
1490         int ret = -EINVAL;
1491
1492         if (ublksrv_pid <= 0)
1493                 return -EINVAL;
1494
1495         ub = ublk_get_device_from_id(header->dev_id);
1496         if (!ub)
1497                 return -EINVAL;
1498
1499         wait_for_completion_interruptible(&ub->completion);
1500
1501         schedule_delayed_work(&ub->monitor_work, UBLK_DAEMON_MONITOR_PERIOD);
1502
1503         mutex_lock(&ub->mutex);
1504         if (ub->dev_info.state == UBLK_S_DEV_LIVE ||
1505             test_bit(UB_STATE_USED, &ub->state)) {
1506                 ret = -EEXIST;
1507                 goto out_unlock;
1508         }
1509
1510         disk = blk_mq_alloc_disk(&ub->tag_set, ub);
1511         if (IS_ERR(disk)) {
1512                 ret = PTR_ERR(disk);
1513                 goto out_unlock;
1514         }
1515         sprintf(disk->disk_name, "ublkb%d", ub->ub_number);
1516         disk->fops = &ub_fops;
1517         disk->private_data = ub;
1518
1519         ub->dev_info.ublksrv_pid = ublksrv_pid;
1520         ub->ub_disk = disk;
1521
1522         ret = ublk_apply_params(ub);
1523         if (ret)
1524                 goto out_put_disk;
1525
1526         get_device(&ub->cdev_dev);
1527         ret = add_disk(disk);
1528         if (ret) {
1529                 /*
1530                  * Has to drop the reference since ->free_disk won't be
1531                  * called in case of add_disk failure.
1532                  */
1533                 ublk_put_device(ub);
1534                 goto out_put_disk;
1535         }
1536         set_bit(UB_STATE_USED, &ub->state);
1537         ub->dev_info.state = UBLK_S_DEV_LIVE;
1538 out_put_disk:
1539         if (ret)
1540                 put_disk(disk);
1541 out_unlock:
1542         mutex_unlock(&ub->mutex);
1543         ublk_put_device(ub);
1544         return ret;
1545 }
1546
1547 static int ublk_ctrl_get_queue_affinity(struct io_uring_cmd *cmd)
1548 {
1549         struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1550         void __user *argp = (void __user *)(unsigned long)header->addr;
1551         struct ublk_device *ub;
1552         cpumask_var_t cpumask;
1553         unsigned long queue;
1554         unsigned int retlen;
1555         unsigned int i;
1556         int ret = -EINVAL;
1557
1558         if (header->len * BITS_PER_BYTE < nr_cpu_ids)
1559                 return -EINVAL;
1560         if (header->len & (sizeof(unsigned long)-1))
1561                 return -EINVAL;
1562         if (!header->addr)
1563                 return -EINVAL;
1564
1565         ub = ublk_get_device_from_id(header->dev_id);
1566         if (!ub)
1567                 return -EINVAL;
1568
1569         queue = header->data[0];
1570         if (queue >= ub->dev_info.nr_hw_queues)
1571                 goto out_put_device;
1572
1573         ret = -ENOMEM;
1574         if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
1575                 goto out_put_device;
1576
1577         for_each_possible_cpu(i) {
1578                 if (ub->tag_set.map[HCTX_TYPE_DEFAULT].mq_map[i] == queue)
1579                         cpumask_set_cpu(i, cpumask);
1580         }
1581
1582         ret = -EFAULT;
1583         retlen = min_t(unsigned short, header->len, cpumask_size());
1584         if (copy_to_user(argp, cpumask, retlen))
1585                 goto out_free_cpumask;
1586         if (retlen != header->len &&
1587             clear_user(argp + retlen, header->len - retlen))
1588                 goto out_free_cpumask;
1589
1590         ret = 0;
1591 out_free_cpumask:
1592         free_cpumask_var(cpumask);
1593 out_put_device:
1594         ublk_put_device(ub);
1595         return ret;
1596 }
1597
1598 static inline void ublk_dump_dev_info(struct ublksrv_ctrl_dev_info *info)
1599 {
1600         pr_devel("%s: dev id %d flags %llx\n", __func__,
1601                         info->dev_id, info->flags);
1602         pr_devel("\t nr_hw_queues %d queue_depth %d\n",
1603                         info->nr_hw_queues, info->queue_depth);
1604 }
1605
1606 static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd)
1607 {
1608         struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1609         void __user *argp = (void __user *)(unsigned long)header->addr;
1610         struct ublksrv_ctrl_dev_info info;
1611         struct ublk_device *ub;
1612         int ret = -EINVAL;
1613
1614         if (header->len < sizeof(info) || !header->addr)
1615                 return -EINVAL;
1616         if (header->queue_id != (u16)-1) {
1617                 pr_warn("%s: queue_id is wrong %x\n",
1618                         __func__, header->queue_id);
1619                 return -EINVAL;
1620         }
1621         if (copy_from_user(&info, argp, sizeof(info)))
1622                 return -EFAULT;
1623         ublk_dump_dev_info(&info);
1624         if (header->dev_id != info.dev_id) {
1625                 pr_warn("%s: dev id not match %u %u\n",
1626                         __func__, header->dev_id, info.dev_id);
1627                 return -EINVAL;
1628         }
1629
1630         ret = mutex_lock_killable(&ublk_ctl_mutex);
1631         if (ret)
1632                 return ret;
1633
1634         ret = -ENOMEM;
1635         ub = kzalloc(sizeof(*ub), GFP_KERNEL);
1636         if (!ub)
1637                 goto out_unlock;
1638         mutex_init(&ub->mutex);
1639         spin_lock_init(&ub->mm_lock);
1640         INIT_WORK(&ub->quiesce_work, ublk_quiesce_work_fn);
1641         INIT_WORK(&ub->stop_work, ublk_stop_work_fn);
1642         INIT_DELAYED_WORK(&ub->monitor_work, ublk_daemon_monitor_work);
1643
1644         ret = ublk_alloc_dev_number(ub, header->dev_id);
1645         if (ret < 0)
1646                 goto out_free_ub;
1647
1648         memcpy(&ub->dev_info, &info, sizeof(info));
1649
1650         /* update device id */
1651         ub->dev_info.dev_id = ub->ub_number;
1652
1653         /*
1654          * 64bit flags will be copied back to userspace as feature
1655          * negotiation result, so have to clear flags which driver
1656          * doesn't support yet, then userspace can get correct flags
1657          * (features) to handle.
1658          */
1659         ub->dev_info.flags &= UBLK_F_ALL;
1660
1661         /* We are not ready to support zero copy */
1662         ub->dev_info.flags &= ~UBLK_F_SUPPORT_ZERO_COPY;
1663
1664         ub->dev_info.nr_hw_queues = min_t(unsigned int,
1665                         ub->dev_info.nr_hw_queues, nr_cpu_ids);
1666         ublk_align_max_io_size(ub);
1667
1668         ret = ublk_init_queues(ub);
1669         if (ret)
1670                 goto out_free_dev_number;
1671
1672         ret = ublk_add_tag_set(ub);
1673         if (ret)
1674                 goto out_deinit_queues;
1675
1676         ret = -EFAULT;
1677         if (copy_to_user(argp, &ub->dev_info, sizeof(info)))
1678                 goto out_free_tag_set;
1679
1680         /*
1681          * Add the char dev so that ublksrv daemon can be setup.
1682          * ublk_add_chdev() will cleanup everything if it fails.
1683          */
1684         ret = ublk_add_chdev(ub);
1685         goto out_unlock;
1686
1687 out_free_tag_set:
1688         blk_mq_free_tag_set(&ub->tag_set);
1689 out_deinit_queues:
1690         ublk_deinit_queues(ub);
1691 out_free_dev_number:
1692         ublk_free_dev_number(ub);
1693 out_free_ub:
1694         mutex_destroy(&ub->mutex);
1695         kfree(ub);
1696 out_unlock:
1697         mutex_unlock(&ublk_ctl_mutex);
1698         return ret;
1699 }
1700
1701 static inline bool ublk_idr_freed(int id)
1702 {
1703         void *ptr;
1704
1705         spin_lock(&ublk_idr_lock);
1706         ptr = idr_find(&ublk_index_idr, id);
1707         spin_unlock(&ublk_idr_lock);
1708
1709         return ptr == NULL;
1710 }
1711
1712 static int ublk_ctrl_del_dev(int idx)
1713 {
1714         struct ublk_device *ub;
1715         int ret;
1716
1717         ret = mutex_lock_killable(&ublk_ctl_mutex);
1718         if (ret)
1719                 return ret;
1720
1721         ub = ublk_get_device_from_id(idx);
1722         if (ub) {
1723                 ublk_remove(ub);
1724                 ublk_put_device(ub);
1725                 ret = 0;
1726         } else {
1727                 ret = -ENODEV;
1728         }
1729
1730         /*
1731          * Wait until the idr is removed, then it can be reused after
1732          * DEL_DEV command is returned.
1733          */
1734         if (!ret)
1735                 wait_event(ublk_idr_wq, ublk_idr_freed(idx));
1736         mutex_unlock(&ublk_ctl_mutex);
1737
1738         return ret;
1739 }
1740
1741 static inline void ublk_ctrl_cmd_dump(struct io_uring_cmd *cmd)
1742 {
1743         struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1744
1745         pr_devel("%s: cmd_op %x, dev id %d qid %d data %llx buf %llx len %u\n",
1746                         __func__, cmd->cmd_op, header->dev_id, header->queue_id,
1747                         header->data[0], header->addr, header->len);
1748 }
1749
1750 static int ublk_ctrl_stop_dev(struct io_uring_cmd *cmd)
1751 {
1752         struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1753         struct ublk_device *ub;
1754
1755         ub = ublk_get_device_from_id(header->dev_id);
1756         if (!ub)
1757                 return -EINVAL;
1758
1759         ublk_stop_dev(ub);
1760         cancel_work_sync(&ub->stop_work);
1761         cancel_work_sync(&ub->quiesce_work);
1762
1763         ublk_put_device(ub);
1764         return 0;
1765 }
1766
1767 static int ublk_ctrl_get_dev_info(struct io_uring_cmd *cmd)
1768 {
1769         struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1770         void __user *argp = (void __user *)(unsigned long)header->addr;
1771         struct ublk_device *ub;
1772         int ret = 0;
1773
1774         if (header->len < sizeof(struct ublksrv_ctrl_dev_info) || !header->addr)
1775                 return -EINVAL;
1776
1777         ub = ublk_get_device_from_id(header->dev_id);
1778         if (!ub)
1779                 return -EINVAL;
1780
1781         if (copy_to_user(argp, &ub->dev_info, sizeof(ub->dev_info)))
1782                 ret = -EFAULT;
1783         ublk_put_device(ub);
1784
1785         return ret;
1786 }
1787
1788 static int ublk_ctrl_get_params(struct io_uring_cmd *cmd)
1789 {
1790         struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1791         void __user *argp = (void __user *)(unsigned long)header->addr;
1792         struct ublk_params_header ph;
1793         struct ublk_device *ub;
1794         int ret;
1795
1796         if (header->len <= sizeof(ph) || !header->addr)
1797                 return -EINVAL;
1798
1799         if (copy_from_user(&ph, argp, sizeof(ph)))
1800                 return -EFAULT;
1801
1802         if (ph.len > header->len || !ph.len)
1803                 return -EINVAL;
1804
1805         if (ph.len > sizeof(struct ublk_params))
1806                 ph.len = sizeof(struct ublk_params);
1807
1808         ub = ublk_get_device_from_id(header->dev_id);
1809         if (!ub)
1810                 return -EINVAL;
1811
1812         mutex_lock(&ub->mutex);
1813         if (copy_to_user(argp, &ub->params, ph.len))
1814                 ret = -EFAULT;
1815         else
1816                 ret = 0;
1817         mutex_unlock(&ub->mutex);
1818
1819         ublk_put_device(ub);
1820         return ret;
1821 }
1822
1823 static int ublk_ctrl_set_params(struct io_uring_cmd *cmd)
1824 {
1825         struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1826         void __user *argp = (void __user *)(unsigned long)header->addr;
1827         struct ublk_params_header ph;
1828         struct ublk_device *ub;
1829         int ret = -EFAULT;
1830
1831         if (header->len <= sizeof(ph) || !header->addr)
1832                 return -EINVAL;
1833
1834         if (copy_from_user(&ph, argp, sizeof(ph)))
1835                 return -EFAULT;
1836
1837         if (ph.len > header->len || !ph.len || !ph.types)
1838                 return -EINVAL;
1839
1840         if (ph.len > sizeof(struct ublk_params))
1841                 ph.len = sizeof(struct ublk_params);
1842
1843         ub = ublk_get_device_from_id(header->dev_id);
1844         if (!ub)
1845                 return -EINVAL;
1846
1847         /* parameters can only be changed when device isn't live */
1848         mutex_lock(&ub->mutex);
1849         if (ub->dev_info.state == UBLK_S_DEV_LIVE) {
1850                 ret = -EACCES;
1851         } else if (copy_from_user(&ub->params, argp, ph.len)) {
1852                 ret = -EFAULT;
1853         } else {
1854                 /* clear all we don't support yet */
1855                 ub->params.types &= UBLK_PARAM_TYPE_ALL;
1856                 ret = ublk_validate_params(ub);
1857         }
1858         mutex_unlock(&ub->mutex);
1859         ublk_put_device(ub);
1860
1861         return ret;
1862 }
1863
1864 static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq)
1865 {
1866         int i;
1867
1868         WARN_ON_ONCE(!(ubq->ubq_daemon && ubq_daemon_is_dying(ubq)));
1869         /* All old ioucmds have to be completed */
1870         WARN_ON_ONCE(ubq->nr_io_ready);
1871         /* old daemon is PF_EXITING, put it now */
1872         put_task_struct(ubq->ubq_daemon);
1873         /* We have to reset it to NULL, otherwise ub won't accept new FETCH_REQ */
1874         ubq->ubq_daemon = NULL;
1875
1876         for (i = 0; i < ubq->q_depth; i++) {
1877                 struct ublk_io *io = &ubq->ios[i];
1878
1879                 /* forget everything now and be ready for new FETCH_REQ */
1880                 io->flags = 0;
1881                 io->cmd = NULL;
1882                 io->addr = 0;
1883         }
1884 }
1885
1886 static int ublk_ctrl_start_recovery(struct io_uring_cmd *cmd)
1887 {
1888         struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1889         struct ublk_device *ub;
1890         int ret = -EINVAL;
1891         int i;
1892
1893         ub = ublk_get_device_from_id(header->dev_id);
1894         if (!ub)
1895                 return ret;
1896
1897         mutex_lock(&ub->mutex);
1898         if (!ublk_can_use_recovery(ub))
1899                 goto out_unlock;
1900         /*
1901          * START_RECOVERY is only allowd after:
1902          *
1903          * (1) UB_STATE_OPEN is not set, which means the dying process is exited
1904          *     and related io_uring ctx is freed so file struct of /dev/ublkcX is
1905          *     released.
1906          *
1907          * (2) UBLK_S_DEV_QUIESCED is set, which means the quiesce_work:
1908          *     (a)has quiesced request queue
1909          *     (b)has requeued every inflight rqs whose io_flags is ACTIVE
1910          *     (c)has requeued/aborted every inflight rqs whose io_flags is NOT ACTIVE
1911          *     (d)has completed/camceled all ioucmds owned by ther dying process
1912          */
1913         if (test_bit(UB_STATE_OPEN, &ub->state) ||
1914                         ub->dev_info.state != UBLK_S_DEV_QUIESCED) {
1915                 ret = -EBUSY;
1916                 goto out_unlock;
1917         }
1918         pr_devel("%s: start recovery for dev id %d.\n", __func__, header->dev_id);
1919         for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1920                 ublk_queue_reinit(ub, ublk_get_queue(ub, i));
1921         /* set to NULL, otherwise new ubq_daemon cannot mmap the io_cmd_buf */
1922         ub->mm = NULL;
1923         ub->nr_queues_ready = 0;
1924         init_completion(&ub->completion);
1925         ret = 0;
1926  out_unlock:
1927         mutex_unlock(&ub->mutex);
1928         ublk_put_device(ub);
1929         return ret;
1930 }
1931
1932 static int ublk_ctrl_end_recovery(struct io_uring_cmd *cmd)
1933 {
1934         struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1935         int ublksrv_pid = (int)header->data[0];
1936         struct ublk_device *ub;
1937         int ret = -EINVAL;
1938
1939         ub = ublk_get_device_from_id(header->dev_id);
1940         if (!ub)
1941                 return ret;
1942
1943         pr_devel("%s: Waiting for new ubq_daemons(nr: %d) are ready, dev id %d...\n",
1944                         __func__, ub->dev_info.nr_hw_queues, header->dev_id);
1945         /* wait until new ubq_daemon sending all FETCH_REQ */
1946         wait_for_completion_interruptible(&ub->completion);
1947         pr_devel("%s: All new ubq_daemons(nr: %d) are ready, dev id %d\n",
1948                         __func__, ub->dev_info.nr_hw_queues, header->dev_id);
1949
1950         mutex_lock(&ub->mutex);
1951         if (!ublk_can_use_recovery(ub))
1952                 goto out_unlock;
1953
1954         if (ub->dev_info.state != UBLK_S_DEV_QUIESCED) {
1955                 ret = -EBUSY;
1956                 goto out_unlock;
1957         }
1958         ub->dev_info.ublksrv_pid = ublksrv_pid;
1959         pr_devel("%s: new ublksrv_pid %d, dev id %d\n",
1960                         __func__, ublksrv_pid, header->dev_id);
1961         blk_mq_unquiesce_queue(ub->ub_disk->queue);
1962         pr_devel("%s: queue unquiesced, dev id %d.\n",
1963                         __func__, header->dev_id);
1964         blk_mq_kick_requeue_list(ub->ub_disk->queue);
1965         ub->dev_info.state = UBLK_S_DEV_LIVE;
1966         schedule_delayed_work(&ub->monitor_work, UBLK_DAEMON_MONITOR_PERIOD);
1967         ret = 0;
1968  out_unlock:
1969         mutex_unlock(&ub->mutex);
1970         ublk_put_device(ub);
1971         return ret;
1972 }
1973
1974 static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd,
1975                 unsigned int issue_flags)
1976 {
1977         struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1978         int ret = -EINVAL;
1979
1980         ublk_ctrl_cmd_dump(cmd);
1981
1982         if (!(issue_flags & IO_URING_F_SQE128))
1983                 goto out;
1984
1985         ret = -EPERM;
1986         if (!capable(CAP_SYS_ADMIN))
1987                 goto out;
1988
1989         ret = -ENODEV;
1990         switch (cmd->cmd_op) {
1991         case UBLK_CMD_START_DEV:
1992                 ret = ublk_ctrl_start_dev(cmd);
1993                 break;
1994         case UBLK_CMD_STOP_DEV:
1995                 ret = ublk_ctrl_stop_dev(cmd);
1996                 break;
1997         case UBLK_CMD_GET_DEV_INFO:
1998                 ret = ublk_ctrl_get_dev_info(cmd);
1999                 break;
2000         case UBLK_CMD_ADD_DEV:
2001                 ret = ublk_ctrl_add_dev(cmd);
2002                 break;
2003         case UBLK_CMD_DEL_DEV:
2004                 ret = ublk_ctrl_del_dev(header->dev_id);
2005                 break;
2006         case UBLK_CMD_GET_QUEUE_AFFINITY:
2007                 ret = ublk_ctrl_get_queue_affinity(cmd);
2008                 break;
2009         case UBLK_CMD_GET_PARAMS:
2010                 ret = ublk_ctrl_get_params(cmd);
2011                 break;
2012         case UBLK_CMD_SET_PARAMS:
2013                 ret = ublk_ctrl_set_params(cmd);
2014                 break;
2015         case UBLK_CMD_START_USER_RECOVERY:
2016                 ret = ublk_ctrl_start_recovery(cmd);
2017                 break;
2018         case UBLK_CMD_END_USER_RECOVERY:
2019                 ret = ublk_ctrl_end_recovery(cmd);
2020                 break;
2021         default:
2022                 break;
2023         }
2024  out:
2025         io_uring_cmd_done(cmd, ret, 0);
2026         pr_devel("%s: cmd done ret %d cmd_op %x, dev id %d qid %d\n",
2027                         __func__, ret, cmd->cmd_op, header->dev_id, header->queue_id);
2028         return -EIOCBQUEUED;
2029 }
2030
2031 static const struct file_operations ublk_ctl_fops = {
2032         .open           = nonseekable_open,
2033         .uring_cmd      = ublk_ctrl_uring_cmd,
2034         .owner          = THIS_MODULE,
2035         .llseek         = noop_llseek,
2036 };
2037
2038 static struct miscdevice ublk_misc = {
2039         .minor          = MISC_DYNAMIC_MINOR,
2040         .name           = "ublk-control",
2041         .fops           = &ublk_ctl_fops,
2042 };
2043
2044 static int __init ublk_init(void)
2045 {
2046         int ret;
2047
2048         init_waitqueue_head(&ublk_idr_wq);
2049
2050         ret = misc_register(&ublk_misc);
2051         if (ret)
2052                 return ret;
2053
2054         ret = alloc_chrdev_region(&ublk_chr_devt, 0, UBLK_MINORS, "ublk-char");
2055         if (ret)
2056                 goto unregister_mis;
2057
2058         ublk_chr_class = class_create(THIS_MODULE, "ublk-char");
2059         if (IS_ERR(ublk_chr_class)) {
2060                 ret = PTR_ERR(ublk_chr_class);
2061                 goto free_chrdev_region;
2062         }
2063         return 0;
2064
2065 free_chrdev_region:
2066         unregister_chrdev_region(ublk_chr_devt, UBLK_MINORS);
2067 unregister_mis:
2068         misc_deregister(&ublk_misc);
2069         return ret;
2070 }
2071
2072 static void __exit ublk_exit(void)
2073 {
2074         struct ublk_device *ub;
2075         int id;
2076
2077         class_destroy(ublk_chr_class);
2078
2079         misc_deregister(&ublk_misc);
2080
2081         idr_for_each_entry(&ublk_index_idr, ub, id)
2082                 ublk_remove(ub);
2083
2084         idr_destroy(&ublk_index_idr);
2085         unregister_chrdev_region(ublk_chr_devt, UBLK_MINORS);
2086 }
2087
2088 module_init(ublk_init);
2089 module_exit(ublk_exit);
2090
2091 MODULE_AUTHOR("Ming Lei <ming.lei@redhat.com>");
2092 MODULE_LICENSE("GPL");