drivers/vhost/scsi.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*******************************************************************************
   3  * Vhost kernel TCM fabric driver for virtio SCSI initiators
   4  *
   5  * (C) Copyright 2010-2013 Datera, Inc.
   6  * (C) Copyright 2010-2012 IBM Corp.
   7  *
   8  * Authors: Nicholas A. Bellinger <nab@daterainc.com>
   9  *          Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
  10  ****************************************************************************/
  11
  12 #include <linux/module.h>
  13 #include <linux/moduleparam.h>
  14 #include <generated/utsrelease.h>
  15 #include <linux/utsname.h>
  16 #include <linux/init.h>
  17 #include <linux/slab.h>
  18 #include <linux/kthread.h>
  19 #include <linux/types.h>
  20 #include <linux/string.h>
  21 #include <linux/configfs.h>
  22 #include <linux/ctype.h>
  23 #include <linux/compat.h>
  24 #include <linux/eventfd.h>
  25 #include <linux/fs.h>
  26 #include <linux/vmalloc.h>
  27 #include <linux/miscdevice.h>
  28 #include <asm/unaligned.h>
  29 #include <scsi/scsi_common.h>
  30 #include <scsi/scsi_proto.h>
  31 #include <target/target_core_base.h>
  32 #include <target/target_core_fabric.h>
  33 #include <linux/vhost.h>
  34 #include <linux/virtio_scsi.h>
  35 #include <linux/llist.h>
  36 #include <linux/bitmap.h>
  37
  38 #include "vhost.h"
  39
  40 #define VHOST_SCSI_VERSION  "v0.1"
  41 #define VHOST_SCSI_NAMELEN 256
  42 #define VHOST_SCSI_MAX_CDB_SIZE 32
  43 #define VHOST_SCSI_PREALLOC_SGLS 2048
  44 #define VHOST_SCSI_PREALLOC_UPAGES 2048
  45 #define VHOST_SCSI_PREALLOC_PROT_SGLS 2048
  46
  47 /* Max number of requests before requeueing the job.
  48  * Using this limit prevents one virtqueue from starving others with
  49  * request.
  50  */
  51 #define VHOST_SCSI_WEIGHT 256
  52
  53 struct vhost_scsi_inflight {
  54         /* Wait for the flush operation to finish */
  55         struct completion comp;
  56         /* Refcount for the inflight reqs */
  57         struct kref kref;
  58 };
  59
  60 struct vhost_scsi_cmd {
  61         /* Descriptor from vhost_get_vq_desc() for virt_queue segment */
  62         int tvc_vq_desc;
  63         /* virtio-scsi initiator task attribute */
  64         int tvc_task_attr;
  65         /* virtio-scsi response incoming iovecs */
  66         int tvc_in_iovs;
  67         /* virtio-scsi initiator data direction */
  68         enum dma_data_direction tvc_data_direction;
  69         /* Expected data transfer length from virtio-scsi header */
  70         u32 tvc_exp_data_len;
  71         /* The Tag from include/linux/virtio_scsi.h:struct virtio_scsi_cmd_req */
  72         u64 tvc_tag;
  73         /* The number of scatterlists associated with this cmd */
  74         u32 tvc_sgl_count;
  75         u32 tvc_prot_sgl_count;
  76         /* Saved unpacked SCSI LUN for vhost_scsi_target_queue_cmd() */
  77         u32 tvc_lun;
  78         /* Pointer to the SGL formatted memory from virtio-scsi */
  79         struct scatterlist *tvc_sgl;
  80         struct scatterlist *tvc_prot_sgl;
  81         struct page **tvc_upages;
  82         /* Pointer to response header iovec */
  83         struct iovec *tvc_resp_iov;
  84         /* Pointer to vhost_scsi for our device */
  85         struct vhost_scsi *tvc_vhost;
  86         /* Pointer to vhost_virtqueue for the cmd */
  87         struct vhost_virtqueue *tvc_vq;
  88         /* Pointer to vhost nexus memory */
  89         struct vhost_scsi_nexus *tvc_nexus;
  90         /* The TCM I/O descriptor that is accessed via container_of() */
  91         struct se_cmd tvc_se_cmd;
  92         /* Copy of the incoming SCSI command descriptor block (CDB) */
  93         unsigned char tvc_cdb[VHOST_SCSI_MAX_CDB_SIZE];
  94         /* Sense buffer that will be mapped into outgoing status */
  95         unsigned char tvc_sense_buf[TRANSPORT_SENSE_BUFFER];
  96         /* Completed commands list, serviced from vhost worker thread */
  97         struct llist_node tvc_completion_list;
  98         /* Used to track inflight cmd */
  99         struct vhost_scsi_inflight *inflight;
 100 };
 101
 102 struct vhost_scsi_nexus {
 103         /* Pointer to TCM session for I_T Nexus */
 104         struct se_session *tvn_se_sess;
 105 };
 106
 107 struct vhost_scsi_tpg {
 108         /* Vhost port target portal group tag for TCM */
 109         u16 tport_tpgt;
 110         /* Used to track number of TPG Port/Lun Links wrt to explict I_T Nexus shutdown */
 111         int tv_tpg_port_count;
 112         /* Used for vhost_scsi device reference to tpg_nexus, protected by tv_tpg_mutex */
 113         int tv_tpg_vhost_count;
 114         /* Used for enabling T10-PI with legacy devices */
 115         int tv_fabric_prot_type;
 116         /* list for vhost_scsi_list */
 117         struct list_head tv_tpg_list;
 118         /* Used to protect access for tpg_nexus */
 119         struct mutex tv_tpg_mutex;
 120         /* Pointer to the TCM VHost I_T Nexus for this TPG endpoint */
 121         struct vhost_scsi_nexus *tpg_nexus;
 122         /* Pointer back to vhost_scsi_tport */
 123         struct vhost_scsi_tport *tport;
 124         /* Returned by vhost_scsi_make_tpg() */
 125         struct se_portal_group se_tpg;
 126         /* Pointer back to vhost_scsi, protected by tv_tpg_mutex */
 127         struct vhost_scsi *vhost_scsi;
 128 };
 129
 130 struct vhost_scsi_tport {
 131         /* SCSI protocol the tport is providing */
 132         u8 tport_proto_id;
 133         /* Binary World Wide unique Port Name for Vhost Target port */
 134         u64 tport_wwpn;
 135         /* ASCII formatted WWPN for Vhost Target port */
 136         char tport_name[VHOST_SCSI_NAMELEN];
 137         /* Returned by vhost_scsi_make_tport() */
 138         struct se_wwn tport_wwn;
 139 };
 140
 141 struct vhost_scsi_evt {
 142         /* event to be sent to guest */
 143         struct virtio_scsi_event event;
 144         /* event list, serviced from vhost worker thread */
 145         struct llist_node list;
 146 };
 147
 148 enum {
 149         VHOST_SCSI_VQ_CTL = 0,
 150         VHOST_SCSI_VQ_EVT = 1,
 151         VHOST_SCSI_VQ_IO = 2,
 152 };
 153
 154 /* Note: can't set VIRTIO_F_VERSION_1 yet, since that implies ANY_LAYOUT. */
 155 enum {
 156         VHOST_SCSI_FEATURES = VHOST_FEATURES | (1ULL << VIRTIO_SCSI_F_HOTPLUG) |
 157                                                (1ULL << VIRTIO_SCSI_F_T10_PI)
 158 };
 159
 160 #define VHOST_SCSI_MAX_TARGET   256
 161 #define VHOST_SCSI_MAX_IO_VQ    1024
 162 #define VHOST_SCSI_MAX_EVENT    128
 163
 164 static unsigned vhost_scsi_max_io_vqs = 128;
 165 module_param_named(max_io_vqs, vhost_scsi_max_io_vqs, uint, 0644);
 166 MODULE_PARM_DESC(max_io_vqs, "Set the max number of IO virtqueues a vhost scsi device can support. The default is 128. The max is 1024.");
 167
 168 struct vhost_scsi_virtqueue {
 169         struct vhost_virtqueue vq;
 170         struct vhost_scsi *vs;
 171         /*
 172          * Reference counting for inflight reqs, used for flush operation. At
 173          * each time, one reference tracks new commands submitted, while we
 174          * wait for another one to reach 0.
 175          */
 176         struct vhost_scsi_inflight inflights[2];
 177         /*
 178          * Indicate current inflight in use, protected by vq->mutex.
 179          * Writers must also take dev mutex and flush under it.
 180          */
 181         int inflight_idx;
 182         struct vhost_scsi_cmd *scsi_cmds;
 183         struct sbitmap scsi_tags;
 184         int max_cmds;
 185
 186         struct vhost_work completion_work;
 187         struct llist_head completion_list;
 188 };
 189
 190 struct vhost_scsi {
 191         /* Protected by vhost_scsi->dev.mutex */
 192         struct vhost_scsi_tpg **vs_tpg;
 193         char vs_vhost_wwpn[TRANSPORT_IQN_LEN];
 194
 195         struct vhost_dev dev;
 196         struct vhost_scsi_virtqueue *vqs;
 197         struct vhost_scsi_inflight **old_inflight;
 198
 199         struct vhost_work vs_event_work; /* evt injection work item */
 200         struct llist_head vs_event_list; /* evt injection queue */
 201
 202         bool vs_events_missed; /* any missed events, protected by vq->mutex */
 203         int vs_events_nr; /* num of pending events, protected by vq->mutex */
 204 };
 205
 206 struct vhost_scsi_tmf {
 207         struct vhost_work vwork;
 208         struct vhost_scsi *vhost;
 209         struct vhost_scsi_virtqueue *svq;
 210
 211         struct se_cmd se_cmd;
 212         u8 scsi_resp;
 213         struct vhost_scsi_inflight *inflight;
 214         struct iovec resp_iov;
 215         int in_iovs;
 216         int vq_desc;
 217 };
 218
 219 /*
 220  * Context for processing request and control queue operations.
 221  */
 222 struct vhost_scsi_ctx {
 223         int head;
 224         unsigned int out, in;
 225         size_t req_size, rsp_size;
 226         size_t out_size, in_size;
 227         u8 *target, *lunp;
 228         void *req;
 229         struct iov_iter out_iter;
 230 };
 231
 232 /*
 233  * Global mutex to protect vhost_scsi TPG list for vhost IOCTLs and LIO
 234  * configfs management operations.
 235  */
 236 static DEFINE_MUTEX(vhost_scsi_mutex);
 237 static LIST_HEAD(vhost_scsi_list);
 238
 239 static void vhost_scsi_done_inflight(struct kref *kref)
 240 {
 241         struct vhost_scsi_inflight *inflight;
 242
 243         inflight = container_of(kref, struct vhost_scsi_inflight, kref);
 244         complete(&inflight->comp);
 245 }
 246
 247 static void vhost_scsi_init_inflight(struct vhost_scsi *vs,
 248                                     struct vhost_scsi_inflight *old_inflight[])
 249 {
 250         struct vhost_scsi_inflight *new_inflight;
 251         struct vhost_virtqueue *vq;
 252         int idx, i;
 253
 254         for (i = 0; i < vs->dev.nvqs;  i++) {
 255                 vq = &vs->vqs[i].vq;
 256
 257                 mutex_lock(&vq->mutex);
 258
 259                 /* store old infight */
 260                 idx = vs->vqs[i].inflight_idx;
 261                 if (old_inflight)
 262                         old_inflight[i] = &vs->vqs[i].inflights[idx];
 263
 264                 /* setup new infight */
 265                 vs->vqs[i].inflight_idx = idx ^ 1;
 266                 new_inflight = &vs->vqs[i].inflights[idx ^ 1];
 267                 kref_init(&new_inflight->kref);
 268                 init_completion(&new_inflight->comp);
 269
 270                 mutex_unlock(&vq->mutex);
 271         }
 272 }
 273
 274 static struct vhost_scsi_inflight *
 275 vhost_scsi_get_inflight(struct vhost_virtqueue *vq)
 276 {
 277         struct vhost_scsi_inflight *inflight;
 278         struct vhost_scsi_virtqueue *svq;
 279
 280         svq = container_of(vq, struct vhost_scsi_virtqueue, vq);
 281         inflight = &svq->inflights[svq->inflight_idx];
 282         kref_get(&inflight->kref);
 283
 284         return inflight;
 285 }
 286
 287 static void vhost_scsi_put_inflight(struct vhost_scsi_inflight *inflight)
 288 {
 289         kref_put(&inflight->kref, vhost_scsi_done_inflight);
 290 }
 291
 292 static int vhost_scsi_check_true(struct se_portal_group *se_tpg)
 293 {
 294         return 1;
 295 }
 296
 297 static char *vhost_scsi_get_fabric_wwn(struct se_portal_group *se_tpg)
 298 {
 299         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
 300                                 struct vhost_scsi_tpg, se_tpg);
 301         struct vhost_scsi_tport *tport = tpg->tport;
 302
 303         return &tport->tport_name[0];
 304 }
 305
 306 static u16 vhost_scsi_get_tpgt(struct se_portal_group *se_tpg)
 307 {
 308         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
 309                                 struct vhost_scsi_tpg, se_tpg);
 310         return tpg->tport_tpgt;
 311 }
 312
 313 static int vhost_scsi_check_prot_fabric_only(struct se_portal_group *se_tpg)
 314 {
 315         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
 316                                 struct vhost_scsi_tpg, se_tpg);
 317
 318         return tpg->tv_fabric_prot_type;
 319 }
 320
 321 static void vhost_scsi_release_cmd_res(struct se_cmd *se_cmd)
 322 {
 323         struct vhost_scsi_cmd *tv_cmd = container_of(se_cmd,
 324                                 struct vhost_scsi_cmd, tvc_se_cmd);
 325         struct vhost_scsi_virtqueue *svq = container_of(tv_cmd->tvc_vq,
 326                                 struct vhost_scsi_virtqueue, vq);
 327         struct vhost_scsi_inflight *inflight = tv_cmd->inflight;
 328         int i;
 329
 330         if (tv_cmd->tvc_sgl_count) {
 331                 for (i = 0; i < tv_cmd->tvc_sgl_count; i++)
 332                         put_page(sg_page(&tv_cmd->tvc_sgl[i]));
 333         }
 334         if (tv_cmd->tvc_prot_sgl_count) {
 335                 for (i = 0; i < tv_cmd->tvc_prot_sgl_count; i++)
 336                         put_page(sg_page(&tv_cmd->tvc_prot_sgl[i]));
 337         }
 338
 339         sbitmap_clear_bit(&svq->scsi_tags, se_cmd->map_tag);
 340         vhost_scsi_put_inflight(inflight);
 341 }
 342
 343 static void vhost_scsi_release_tmf_res(struct vhost_scsi_tmf *tmf)
 344 {
 345         struct vhost_scsi_inflight *inflight = tmf->inflight;
 346
 347         kfree(tmf);
 348         vhost_scsi_put_inflight(inflight);
 349 }
 350
 351 static void vhost_scsi_release_cmd(struct se_cmd *se_cmd)
 352 {
 353         if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) {
 354                 struct vhost_scsi_tmf *tmf = container_of(se_cmd,
 355                                         struct vhost_scsi_tmf, se_cmd);
 356                 struct vhost_virtqueue *vq = &tmf->svq->vq;
 357
 358                 vhost_vq_work_queue(vq, &tmf->vwork);
 359         } else {
 360                 struct vhost_scsi_cmd *cmd = container_of(se_cmd,
 361                                         struct vhost_scsi_cmd, tvc_se_cmd);
 362                 struct vhost_scsi_virtqueue *svq =  container_of(cmd->tvc_vq,
 363                                         struct vhost_scsi_virtqueue, vq);
 364
 365                 llist_add(&cmd->tvc_completion_list, &svq->completion_list);
 366                 vhost_vq_work_queue(&svq->vq, &svq->completion_work);
 367         }
 368 }
 369
 370 static int vhost_scsi_write_pending(struct se_cmd *se_cmd)
 371 {
 372         /* Go ahead and process the write immediately */
 373         target_execute_cmd(se_cmd);
 374         return 0;
 375 }
 376
 377 static int vhost_scsi_queue_data_in(struct se_cmd *se_cmd)
 378 {
 379         transport_generic_free_cmd(se_cmd, 0);
 380         return 0;
 381 }
 382
 383 static int vhost_scsi_queue_status(struct se_cmd *se_cmd)
 384 {
 385         transport_generic_free_cmd(se_cmd, 0);
 386         return 0;
 387 }
 388
 389 static void vhost_scsi_queue_tm_rsp(struct se_cmd *se_cmd)
 390 {
 391         struct vhost_scsi_tmf *tmf = container_of(se_cmd, struct vhost_scsi_tmf,
 392                                                   se_cmd);
 393
 394         tmf->scsi_resp = se_cmd->se_tmr_req->response;
 395         transport_generic_free_cmd(&tmf->se_cmd, 0);
 396 }
 397
 398 static void vhost_scsi_aborted_task(struct se_cmd *se_cmd)
 399 {
 400         return;
 401 }
 402
 403 static void vhost_scsi_free_evt(struct vhost_scsi *vs, struct vhost_scsi_evt *evt)
 404 {
 405         vs->vs_events_nr--;
 406         kfree(evt);
 407 }
 408
 409 static struct vhost_scsi_evt *
 410 vhost_scsi_allocate_evt(struct vhost_scsi *vs,
 411                        u32 event, u32 reason)
 412 {
 413         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 414         struct vhost_scsi_evt *evt;
 415
 416         if (vs->vs_events_nr > VHOST_SCSI_MAX_EVENT) {
 417                 vs->vs_events_missed = true;
 418                 return NULL;
 419         }
 420
 421         evt = kzalloc(sizeof(*evt), GFP_KERNEL);
 422         if (!evt) {
 423                 vq_err(vq, "Failed to allocate vhost_scsi_evt\n");
 424                 vs->vs_events_missed = true;
 425                 return NULL;
 426         }
 427
 428         evt->event.event = cpu_to_vhost32(vq, event);
 429         evt->event.reason = cpu_to_vhost32(vq, reason);
 430         vs->vs_events_nr++;
 431
 432         return evt;
 433 }
 434
 435 static int vhost_scsi_check_stop_free(struct se_cmd *se_cmd)
 436 {
 437         return target_put_sess_cmd(se_cmd);
 438 }
 439
 440 static void
 441 vhost_scsi_do_evt_work(struct vhost_scsi *vs, struct vhost_scsi_evt *evt)
 442 {
 443         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 444         struct virtio_scsi_event *event = &evt->event;
 445         struct virtio_scsi_event __user *eventp;
 446         unsigned out, in;
 447         int head, ret;
 448
 449         if (!vhost_vq_get_backend(vq)) {
 450                 vs->vs_events_missed = true;
 451                 return;
 452         }
 453
 454 again:
 455         vhost_disable_notify(&vs->dev, vq);
 456         head = vhost_get_vq_desc(vq, vq->iov,
 457                         ARRAY_SIZE(vq->iov), &out, &in,
 458                         NULL, NULL);
 459         if (head < 0) {
 460                 vs->vs_events_missed = true;
 461                 return;
 462         }
 463         if (head == vq->num) {
 464                 if (vhost_enable_notify(&vs->dev, vq))
 465                         goto again;
 466                 vs->vs_events_missed = true;
 467                 return;
 468         }
 469
 470         if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) {
 471                 vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n",
 472                                 vq->iov[out].iov_len);
 473                 vs->vs_events_missed = true;
 474                 return;
 475         }
 476
 477         if (vs->vs_events_missed) {
 478                 event->event |= cpu_to_vhost32(vq, VIRTIO_SCSI_T_EVENTS_MISSED);
 479                 vs->vs_events_missed = false;
 480         }
 481
 482         eventp = vq->iov[out].iov_base;
 483         ret = __copy_to_user(eventp, event, sizeof(*event));
 484         if (!ret)
 485                 vhost_add_used_and_signal(&vs->dev, vq, head, 0);
 486         else
 487                 vq_err(vq, "Faulted on vhost_scsi_send_event\n");
 488 }
 489
 490 static void vhost_scsi_evt_work(struct vhost_work *work)
 491 {
 492         struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
 493                                         vs_event_work);
 494         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 495         struct vhost_scsi_evt *evt, *t;
 496         struct llist_node *llnode;
 497
 498         mutex_lock(&vq->mutex);
 499         llnode = llist_del_all(&vs->vs_event_list);
 500         llist_for_each_entry_safe(evt, t, llnode, list) {
 501                 vhost_scsi_do_evt_work(vs, evt);
 502                 vhost_scsi_free_evt(vs, evt);
 503         }
 504         mutex_unlock(&vq->mutex);
 505 }
 506
 507 /* Fill in status and signal that we are done processing this command
 508  *
 509  * This is scheduled in the vhost work queue so we are called with the owner
 510  * process mm and can access the vring.
 511  */
 512 static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
 513 {
 514         struct vhost_scsi_virtqueue *svq = container_of(work,
 515                                 struct vhost_scsi_virtqueue, completion_work);
 516         struct virtio_scsi_cmd_resp v_rsp;
 517         struct vhost_scsi_cmd *cmd, *t;
 518         struct llist_node *llnode;
 519         struct se_cmd *se_cmd;
 520         struct iov_iter iov_iter;
 521         bool signal = false;
 522         int ret;
 523
 524         llnode = llist_del_all(&svq->completion_list);
 525         llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) {
 526                 se_cmd = &cmd->tvc_se_cmd;
 527
 528                 pr_debug("%s tv_cmd %p resid %u status %#02x\n", __func__,
 529                         cmd, se_cmd->residual_count, se_cmd->scsi_status);
 530
 531                 memset(&v_rsp, 0, sizeof(v_rsp));
 532                 v_rsp.resid = cpu_to_vhost32(cmd->tvc_vq, se_cmd->residual_count);
 533                 /* TODO is status_qualifier field needed? */
 534                 v_rsp.status = se_cmd->scsi_status;
 535                 v_rsp.sense_len = cpu_to_vhost32(cmd->tvc_vq,
 536                                                  se_cmd->scsi_sense_length);
 537                 memcpy(v_rsp.sense, cmd->tvc_sense_buf,
 538                        se_cmd->scsi_sense_length);
 539
 540                 iov_iter_init(&iov_iter, ITER_DEST, cmd->tvc_resp_iov,
 541                               cmd->tvc_in_iovs, sizeof(v_rsp));
 542                 ret = copy_to_iter(&v_rsp, sizeof(v_rsp), &iov_iter);
 543                 if (likely(ret == sizeof(v_rsp))) {
 544                         signal = true;
 545
 546                         vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0);
 547                 } else
 548                         pr_err("Faulted on virtio_scsi_cmd_resp\n");
 549
 550                 vhost_scsi_release_cmd_res(se_cmd);
 551         }
 552
 553         if (signal)
 554                 vhost_signal(&svq->vs->dev, &svq->vq);
 555 }
 556
 557 static struct vhost_scsi_cmd *
 558 vhost_scsi_get_cmd(struct vhost_virtqueue *vq, struct vhost_scsi_tpg *tpg,
 559                    unsigned char *cdb, u64 scsi_tag, u16 lun, u8 task_attr,
 560                    u32 exp_data_len, int data_direction)
 561 {
 562         struct vhost_scsi_virtqueue *svq = container_of(vq,
 563                                         struct vhost_scsi_virtqueue, vq);
 564         struct vhost_scsi_cmd *cmd;
 565         struct vhost_scsi_nexus *tv_nexus;
 566         struct scatterlist *sg, *prot_sg;
 567         struct iovec *tvc_resp_iov;
 568         struct page **pages;
 569         int tag;
 570
 571         tv_nexus = tpg->tpg_nexus;
 572         if (!tv_nexus) {
 573                 pr_err("Unable to locate active struct vhost_scsi_nexus\n");
 574                 return ERR_PTR(-EIO);
 575         }
 576
 577         tag = sbitmap_get(&svq->scsi_tags);
 578         if (tag < 0) {
 579                 pr_err("Unable to obtain tag for vhost_scsi_cmd\n");
 580                 return ERR_PTR(-ENOMEM);
 581         }
 582
 583         cmd = &svq->scsi_cmds[tag];
 584         sg = cmd->tvc_sgl;
 585         prot_sg = cmd->tvc_prot_sgl;
 586         pages = cmd->tvc_upages;
 587         tvc_resp_iov = cmd->tvc_resp_iov;
 588         memset(cmd, 0, sizeof(*cmd));
 589         cmd->tvc_sgl = sg;
 590         cmd->tvc_prot_sgl = prot_sg;
 591         cmd->tvc_upages = pages;
 592         cmd->tvc_se_cmd.map_tag = tag;
 593         cmd->tvc_tag = scsi_tag;
 594         cmd->tvc_lun = lun;
 595         cmd->tvc_task_attr = task_attr;
 596         cmd->tvc_exp_data_len = exp_data_len;
 597         cmd->tvc_data_direction = data_direction;
 598         cmd->tvc_nexus = tv_nexus;
 599         cmd->inflight = vhost_scsi_get_inflight(vq);
 600         cmd->tvc_resp_iov = tvc_resp_iov;
 601
 602         memcpy(cmd->tvc_cdb, cdb, VHOST_SCSI_MAX_CDB_SIZE);
 603
 604         return cmd;
 605 }
 606
 607 /*
 608  * Map a user memory range into a scatterlist
 609  *
 610  * Returns the number of scatterlist entries used or -errno on error.
 611  */
 612 static int
 613 vhost_scsi_map_to_sgl(struct vhost_scsi_cmd *cmd,
 614                       struct iov_iter *iter,
 615                       struct scatterlist *sgl,
 616                       bool write)
 617 {
 618         struct page **pages = cmd->tvc_upages;
 619         struct scatterlist *sg = sgl;
 620         ssize_t bytes;
 621         size_t offset;
 622         unsigned int npages = 0;
 623
 624         bytes = iov_iter_get_pages2(iter, pages, LONG_MAX,
 625                                 VHOST_SCSI_PREALLOC_UPAGES, &offset);
 626         /* No pages were pinned */
 627         if (bytes <= 0)
 628                 return bytes < 0 ? bytes : -EFAULT;
 629
 630         while (bytes) {
 631                 unsigned n = min_t(unsigned, PAGE_SIZE - offset, bytes);
 632                 sg_set_page(sg++, pages[npages++], n, offset);
 633                 bytes -= n;
 634                 offset = 0;
 635         }
 636         return npages;
 637 }
 638
 639 static int
 640 vhost_scsi_calc_sgls(struct iov_iter *iter, size_t bytes, int max_sgls)
 641 {
 642         int sgl_count = 0;
 643
 644         if (!iter || !iter_iov(iter)) {
 645                 pr_err("%s: iter->iov is NULL, but expected bytes: %zu"
 646                        " present\n", __func__, bytes);
 647                 return -EINVAL;
 648         }
 649
 650         sgl_count = iov_iter_npages(iter, 0xffff);
 651         if (sgl_count > max_sgls) {
 652                 pr_err("%s: requested sgl_count: %d exceeds pre-allocated"
 653                        " max_sgls: %d\n", __func__, sgl_count, max_sgls);
 654                 return -EINVAL;
 655         }
 656         return sgl_count;
 657 }
 658
 659 static int
 660 vhost_scsi_iov_to_sgl(struct vhost_scsi_cmd *cmd, bool write,
 661                       struct iov_iter *iter,
 662                       struct scatterlist *sg, int sg_count)
 663 {
 664         struct scatterlist *p = sg;
 665         int ret;
 666
 667         while (iov_iter_count(iter)) {
 668                 ret = vhost_scsi_map_to_sgl(cmd, iter, sg, write);
 669                 if (ret < 0) {
 670                         while (p < sg) {
 671                                 struct page *page = sg_page(p++);
 672                                 if (page)
 673                                         put_page(page);
 674                         }
 675                         return ret;
 676                 }
 677                 sg += ret;
 678         }
 679         return 0;
 680 }
 681
 682 static int
 683 vhost_scsi_mapal(struct vhost_scsi_cmd *cmd,
 684                  size_t prot_bytes, struct iov_iter *prot_iter,
 685                  size_t data_bytes, struct iov_iter *data_iter)
 686 {
 687         int sgl_count, ret;
 688         bool write = (cmd->tvc_data_direction == DMA_FROM_DEVICE);
 689
 690         if (prot_bytes) {
 691                 sgl_count = vhost_scsi_calc_sgls(prot_iter, prot_bytes,
 692                                                  VHOST_SCSI_PREALLOC_PROT_SGLS);
 693                 if (sgl_count < 0)
 694                         return sgl_count;
 695
 696                 sg_init_table(cmd->tvc_prot_sgl, sgl_count);
 697                 cmd->tvc_prot_sgl_count = sgl_count;
 698                 pr_debug("%s prot_sg %p prot_sgl_count %u\n", __func__,
 699                          cmd->tvc_prot_sgl, cmd->tvc_prot_sgl_count);
 700
 701                 ret = vhost_scsi_iov_to_sgl(cmd, write, prot_iter,
 702                                             cmd->tvc_prot_sgl,
 703                                             cmd->tvc_prot_sgl_count);
 704                 if (ret < 0) {
 705                         cmd->tvc_prot_sgl_count = 0;
 706                         return ret;
 707                 }
 708         }
 709         sgl_count = vhost_scsi_calc_sgls(data_iter, data_bytes,
 710                                          VHOST_SCSI_PREALLOC_SGLS);
 711         if (sgl_count < 0)
 712                 return sgl_count;
 713
 714         sg_init_table(cmd->tvc_sgl, sgl_count);
 715         cmd->tvc_sgl_count = sgl_count;
 716         pr_debug("%s data_sg %p data_sgl_count %u\n", __func__,
 717                   cmd->tvc_sgl, cmd->tvc_sgl_count);
 718
 719         ret = vhost_scsi_iov_to_sgl(cmd, write, data_iter,
 720                                     cmd->tvc_sgl, cmd->tvc_sgl_count);
 721         if (ret < 0) {
 722                 cmd->tvc_sgl_count = 0;
 723                 return ret;
 724         }
 725         return 0;
 726 }
 727
 728 static int vhost_scsi_to_tcm_attr(int attr)
 729 {
 730         switch (attr) {
 731         case VIRTIO_SCSI_S_SIMPLE:
 732                 return TCM_SIMPLE_TAG;
 733         case VIRTIO_SCSI_S_ORDERED:
 734                 return TCM_ORDERED_TAG;
 735         case VIRTIO_SCSI_S_HEAD:
 736                 return TCM_HEAD_TAG;
 737         case VIRTIO_SCSI_S_ACA:
 738                 return TCM_ACA_TAG;
 739         default:
 740                 break;
 741         }
 742         return TCM_SIMPLE_TAG;
 743 }
 744
 745 static void vhost_scsi_target_queue_cmd(struct vhost_scsi_cmd *cmd)
 746 {
 747         struct se_cmd *se_cmd = &cmd->tvc_se_cmd;
 748         struct vhost_scsi_nexus *tv_nexus;
 749         struct scatterlist *sg_ptr, *sg_prot_ptr = NULL;
 750
 751         /* FIXME: BIDI operation */
 752         if (cmd->tvc_sgl_count) {
 753                 sg_ptr = cmd->tvc_sgl;
 754
 755                 if (cmd->tvc_prot_sgl_count)
 756                         sg_prot_ptr = cmd->tvc_prot_sgl;
 757                 else
 758                         se_cmd->prot_pto = true;
 759         } else {
 760                 sg_ptr = NULL;
 761         }
 762         tv_nexus = cmd->tvc_nexus;
 763
 764         se_cmd->tag = 0;
 765         target_init_cmd(se_cmd, tv_nexus->tvn_se_sess, &cmd->tvc_sense_buf[0],
 766                         cmd->tvc_lun, cmd->tvc_exp_data_len,
 767                         vhost_scsi_to_tcm_attr(cmd->tvc_task_attr),
 768                         cmd->tvc_data_direction, TARGET_SCF_ACK_KREF);
 769
 770         if (target_submit_prep(se_cmd, cmd->tvc_cdb, sg_ptr,
 771                                cmd->tvc_sgl_count, NULL, 0, sg_prot_ptr,
 772                                cmd->tvc_prot_sgl_count, GFP_KERNEL))
 773                 return;
 774
 775         target_queue_submission(se_cmd);
 776 }
 777
 778 static void
 779 vhost_scsi_send_bad_target(struct vhost_scsi *vs,
 780                            struct vhost_virtqueue *vq,
 781                            int head, unsigned out)
 782 {
 783         struct virtio_scsi_cmd_resp __user *resp;
 784         struct virtio_scsi_cmd_resp rsp;
 785         int ret;
 786
 787         memset(&rsp, 0, sizeof(rsp));
 788         rsp.response = VIRTIO_SCSI_S_BAD_TARGET;
 789         resp = vq->iov[out].iov_base;
 790         ret = __copy_to_user(resp, &rsp, sizeof(rsp));
 791         if (!ret)
 792                 vhost_add_used_and_signal(&vs->dev, vq, head, 0);
 793         else
 794                 pr_err("Faulted on virtio_scsi_cmd_resp\n");
 795 }
 796
 797 static int
 798 vhost_scsi_get_desc(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
 799                     struct vhost_scsi_ctx *vc)
 800 {
 801         int ret = -ENXIO;
 802
 803         vc->head = vhost_get_vq_desc(vq, vq->iov,
 804                                      ARRAY_SIZE(vq->iov), &vc->out, &vc->in,
 805                                      NULL, NULL);
 806
 807         pr_debug("vhost_get_vq_desc: head: %d, out: %u in: %u\n",
 808                  vc->head, vc->out, vc->in);
 809
 810         /* On error, stop handling until the next kick. */
 811         if (unlikely(vc->head < 0))
 812                 goto done;
 813
 814         /* Nothing new?  Wait for eventfd to tell us they refilled. */
 815         if (vc->head == vq->num) {
 816                 if (unlikely(vhost_enable_notify(&vs->dev, vq))) {
 817                         vhost_disable_notify(&vs->dev, vq);
 818                         ret = -EAGAIN;
 819                 }
 820                 goto done;
 821         }
 822
 823         /*
 824          * Get the size of request and response buffers.
 825          * FIXME: Not correct for BIDI operation
 826          */
 827         vc->out_size = iov_length(vq->iov, vc->out);
 828         vc->in_size = iov_length(&vq->iov[vc->out], vc->in);
 829
 830         /*
 831          * Copy over the virtio-scsi request header, which for a
 832          * ANY_LAYOUT enabled guest may span multiple iovecs, or a
 833          * single iovec may contain both the header + outgoing
 834          * WRITE payloads.
 835          *
 836          * copy_from_iter() will advance out_iter, so that it will
 837          * point at the start of the outgoing WRITE payload, if
 838          * DMA_TO_DEVICE is set.
 839          */
 840         iov_iter_init(&vc->out_iter, ITER_SOURCE, vq->iov, vc->out, vc->out_size);
 841         ret = 0;
 842
 843 done:
 844         return ret;
 845 }
 846
 847 static int
 848 vhost_scsi_chk_size(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc)
 849 {
 850         if (unlikely(vc->in_size < vc->rsp_size)) {
 851                 vq_err(vq,
 852                        "Response buf too small, need min %zu bytes got %zu",
 853                        vc->rsp_size, vc->in_size);
 854                 return -EINVAL;
 855         } else if (unlikely(vc->out_size < vc->req_size)) {
 856                 vq_err(vq,
 857                        "Request buf too small, need min %zu bytes got %zu",
 858                        vc->req_size, vc->out_size);
 859                 return -EIO;
 860         }
 861
 862         return 0;
 863 }
 864
 865 static int
 866 vhost_scsi_get_req(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc,
 867                    struct vhost_scsi_tpg **tpgp)
 868 {
 869         int ret = -EIO;
 870
 871         if (unlikely(!copy_from_iter_full(vc->req, vc->req_size,
 872                                           &vc->out_iter))) {
 873                 vq_err(vq, "Faulted on copy_from_iter_full\n");
 874         } else if (unlikely(*vc->lunp != 1)) {
 875                 /* virtio-scsi spec requires byte 0 of the lun to be 1 */
 876                 vq_err(vq, "Illegal virtio-scsi lun: %u\n", *vc->lunp);
 877         } else {
 878                 struct vhost_scsi_tpg **vs_tpg, *tpg;
 879
 880                 vs_tpg = vhost_vq_get_backend(vq);      /* validated at handler entry */
 881
 882                 tpg = READ_ONCE(vs_tpg[*vc->target]);
 883                 if (unlikely(!tpg)) {
 884                         vq_err(vq, "Target 0x%x does not exist\n", *vc->target);
 885                 } else {
 886                         if (tpgp)
 887                                 *tpgp = tpg;
 888                         ret = 0;
 889                 }
 890         }
 891
 892         return ret;
 893 }
 894
 895 static u16 vhost_buf_to_lun(u8 *lun_buf)
 896 {
 897         return ((lun_buf[2] << 8) | lun_buf[3]) & 0x3FFF;
 898 }
 899
 900 static void
 901 vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
 902 {
 903         struct vhost_scsi_tpg **vs_tpg, *tpg;
 904         struct virtio_scsi_cmd_req v_req;
 905         struct virtio_scsi_cmd_req_pi v_req_pi;
 906         struct vhost_scsi_ctx vc;
 907         struct vhost_scsi_cmd *cmd;
 908         struct iov_iter in_iter, prot_iter, data_iter;
 909         u64 tag;
 910         u32 exp_data_len, data_direction;
 911         int ret, prot_bytes, i, c = 0;
 912         u16 lun;
 913         u8 task_attr;
 914         bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI);
 915         void *cdb;
 916
 917         mutex_lock(&vq->mutex);
 918         /*
 919          * We can handle the vq only after the endpoint is setup by calling the
 920          * VHOST_SCSI_SET_ENDPOINT ioctl.
 921          */
 922         vs_tpg = vhost_vq_get_backend(vq);
 923         if (!vs_tpg)
 924                 goto out;
 925
 926         memset(&vc, 0, sizeof(vc));
 927         vc.rsp_size = sizeof(struct virtio_scsi_cmd_resp);
 928
 929         vhost_disable_notify(&vs->dev, vq);
 930
 931         do {
 932                 ret = vhost_scsi_get_desc(vs, vq, &vc);
 933                 if (ret)
 934                         goto err;
 935
 936                 /*
 937                  * Setup pointers and values based upon different virtio-scsi
 938                  * request header if T10_PI is enabled in KVM guest.
 939                  */
 940                 if (t10_pi) {
 941                         vc.req = &v_req_pi;
 942                         vc.req_size = sizeof(v_req_pi);
 943                         vc.lunp = &v_req_pi.lun[0];
 944                         vc.target = &v_req_pi.lun[1];
 945                 } else {
 946                         vc.req = &v_req;
 947                         vc.req_size = sizeof(v_req);
 948                         vc.lunp = &v_req.lun[0];
 949                         vc.target = &v_req.lun[1];
 950                 }
 951
 952                 /*
 953                  * Validate the size of request and response buffers.
 954                  * Check for a sane response buffer so we can report
 955                  * early errors back to the guest.
 956                  */
 957                 ret = vhost_scsi_chk_size(vq, &vc);
 958                 if (ret)
 959                         goto err;
 960
 961                 ret = vhost_scsi_get_req(vq, &vc, &tpg);
 962                 if (ret)
 963                         goto err;
 964
 965                 ret = -EIO;     /* bad target on any error from here on */
 966
 967                 /*
 968                  * Determine data_direction by calculating the total outgoing
 969                  * iovec sizes + incoming iovec sizes vs. virtio-scsi request +
 970                  * response headers respectively.
 971                  *
 972                  * For DMA_TO_DEVICE this is out_iter, which is already pointing
 973                  * to the right place.
 974                  *
 975                  * For DMA_FROM_DEVICE, the iovec will be just past the end
 976                  * of the virtio-scsi response header in either the same
 977                  * or immediately following iovec.
 978                  *
 979                  * Any associated T10_PI bytes for the outgoing / incoming
 980                  * payloads are included in calculation of exp_data_len here.
 981                  */
 982                 prot_bytes = 0;
 983
 984                 if (vc.out_size > vc.req_size) {
 985                         data_direction = DMA_TO_DEVICE;
 986                         exp_data_len = vc.out_size - vc.req_size;
 987                         data_iter = vc.out_iter;
 988                 } else if (vc.in_size > vc.rsp_size) {
 989                         data_direction = DMA_FROM_DEVICE;
 990                         exp_data_len = vc.in_size - vc.rsp_size;
 991
 992                         iov_iter_init(&in_iter, ITER_DEST, &vq->iov[vc.out], vc.in,
 993                                       vc.rsp_size + exp_data_len);
 994                         iov_iter_advance(&in_iter, vc.rsp_size);
 995                         data_iter = in_iter;
 996                 } else {
 997                         data_direction = DMA_NONE;
 998                         exp_data_len = 0;
 999                 }
1000                 /*
1001                  * If T10_PI header + payload is present, setup prot_iter values
1002                  * and recalculate data_iter for vhost_scsi_mapal() mapping to
1003                  * host scatterlists via get_user_pages_fast().
1004                  */
1005                 if (t10_pi) {
1006                         if (v_req_pi.pi_bytesout) {
1007                                 if (data_direction != DMA_TO_DEVICE) {
1008                                         vq_err(vq, "Received non zero pi_bytesout,"
1009                                                 " but wrong data_direction\n");
1010                                         goto err;
1011                                 }
1012                                 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout);
1013                         } else if (v_req_pi.pi_bytesin) {
1014                                 if (data_direction != DMA_FROM_DEVICE) {
1015                                         vq_err(vq, "Received non zero pi_bytesin,"
1016                                                 " but wrong data_direction\n");
1017                                         goto err;
1018                                 }
1019                                 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin);
1020                         }
1021                         /*
1022                          * Set prot_iter to data_iter and truncate it to
1023                          * prot_bytes, and advance data_iter past any
1024                          * preceeding prot_bytes that may be present.
1025                          *
1026                          * Also fix up the exp_data_len to reflect only the
1027                          * actual data payload length.
1028                          */
1029                         if (prot_bytes) {
1030                                 exp_data_len -= prot_bytes;
1031                                 prot_iter = data_iter;
1032                                 iov_iter_truncate(&prot_iter, prot_bytes);
1033                                 iov_iter_advance(&data_iter, prot_bytes);
1034                         }
1035                         tag = vhost64_to_cpu(vq, v_req_pi.tag);
1036                         task_attr = v_req_pi.task_attr;
1037                         cdb = &v_req_pi.cdb[0];
1038                         lun = vhost_buf_to_lun(v_req_pi.lun);
1039                 } else {
1040                         tag = vhost64_to_cpu(vq, v_req.tag);
1041                         task_attr = v_req.task_attr;
1042                         cdb = &v_req.cdb[0];
1043                         lun = vhost_buf_to_lun(v_req.lun);
1044                 }
1045                 /*
1046                  * Check that the received CDB size does not exceeded our
1047                  * hardcoded max for vhost-scsi, then get a pre-allocated
1048                  * cmd descriptor for the new virtio-scsi tag.
1049                  *
1050                  * TODO what if cdb was too small for varlen cdb header?
1051                  */
1052                 if (unlikely(scsi_command_size(cdb) > VHOST_SCSI_MAX_CDB_SIZE)) {
1053                         vq_err(vq, "Received SCSI CDB with command_size: %d that"
1054                                 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1055                                 scsi_command_size(cdb), VHOST_SCSI_MAX_CDB_SIZE);
1056                                 goto err;
1057                 }
1058                 cmd = vhost_scsi_get_cmd(vq, tpg, cdb, tag, lun, task_attr,
1059                                          exp_data_len + prot_bytes,
1060                                          data_direction);
1061                 if (IS_ERR(cmd)) {
1062                         vq_err(vq, "vhost_scsi_get_cmd failed %ld\n",
1063                                PTR_ERR(cmd));
1064                         goto err;
1065                 }
1066                 cmd->tvc_vhost = vs;
1067                 cmd->tvc_vq = vq;
1068                 for (i = 0; i < vc.in ; i++)
1069                         cmd->tvc_resp_iov[i] = vq->iov[vc.out + i];
1070                 cmd->tvc_in_iovs = vc.in;
1071
1072                 pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n",
1073                          cmd->tvc_cdb[0], cmd->tvc_lun);
1074                 pr_debug("cmd: %p exp_data_len: %d, prot_bytes: %d data_direction:"
1075                          " %d\n", cmd, exp_data_len, prot_bytes, data_direction);
1076
1077                 if (data_direction != DMA_NONE) {
1078                         if (unlikely(vhost_scsi_mapal(cmd, prot_bytes,
1079                                                       &prot_iter, exp_data_len,
1080                                                       &data_iter))) {
1081                                 vq_err(vq, "Failed to map iov to sgl\n");
1082                                 vhost_scsi_release_cmd_res(&cmd->tvc_se_cmd);
1083                                 goto err;
1084                         }
1085                 }
1086                 /*
1087                  * Save the descriptor from vhost_get_vq_desc() to be used to
1088                  * complete the virtio-scsi request in TCM callback context via
1089                  * vhost_scsi_queue_data_in() and vhost_scsi_queue_status()
1090                  */
1091                 cmd->tvc_vq_desc = vc.head;
1092                 vhost_scsi_target_queue_cmd(cmd);
1093                 ret = 0;
1094 err:
1095                 /*
1096                  * ENXIO:  No more requests, or read error, wait for next kick
1097                  * EINVAL: Invalid response buffer, drop the request
1098                  * EIO:    Respond with bad target
1099                  * EAGAIN: Pending request
1100                  */
1101                 if (ret == -ENXIO)
1102                         break;
1103                 else if (ret == -EIO)
1104                         vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out);
1105         } while (likely(!vhost_exceeds_weight(vq, ++c, 0)));
1106 out:
1107         mutex_unlock(&vq->mutex);
1108 }
1109
1110 static void
1111 vhost_scsi_send_tmf_resp(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
1112                          int in_iovs, int vq_desc, struct iovec *resp_iov,
1113                          int tmf_resp_code)
1114 {
1115         struct virtio_scsi_ctrl_tmf_resp rsp;
1116         struct iov_iter iov_iter;
1117         int ret;
1118
1119         pr_debug("%s\n", __func__);
1120         memset(&rsp, 0, sizeof(rsp));
1121         rsp.response = tmf_resp_code;
1122
1123         iov_iter_init(&iov_iter, ITER_DEST, resp_iov, in_iovs, sizeof(rsp));
1124
1125         ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);
1126         if (likely(ret == sizeof(rsp)))
1127                 vhost_add_used_and_signal(&vs->dev, vq, vq_desc, 0);
1128         else
1129                 pr_err("Faulted on virtio_scsi_ctrl_tmf_resp\n");
1130 }
1131
1132 static void vhost_scsi_tmf_resp_work(struct vhost_work *work)
1133 {
1134         struct vhost_scsi_tmf *tmf = container_of(work, struct vhost_scsi_tmf,
1135                                                   vwork);
1136         struct vhost_virtqueue *ctl_vq, *vq;
1137         int resp_code, i;
1138
1139         if (tmf->scsi_resp == TMR_FUNCTION_COMPLETE) {
1140                 /*
1141                  * Flush IO vqs that don't share a worker with the ctl to make
1142                  * sure they have sent their responses before us.
1143                  */
1144                 ctl_vq = &tmf->vhost->vqs[VHOST_SCSI_VQ_CTL].vq;
1145                 for (i = VHOST_SCSI_VQ_IO; i < tmf->vhost->dev.nvqs; i++) {
1146                         vq = &tmf->vhost->vqs[i].vq;
1147
1148                         if (vhost_vq_is_setup(vq) &&
1149                             vq->worker != ctl_vq->worker)
1150                                 vhost_vq_flush(vq);
1151                 }
1152
1153                 resp_code = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED;
1154         } else {
1155                 resp_code = VIRTIO_SCSI_S_FUNCTION_REJECTED;
1156         }
1157
1158         vhost_scsi_send_tmf_resp(tmf->vhost, &tmf->svq->vq, tmf->in_iovs,
1159                                  tmf->vq_desc, &tmf->resp_iov, resp_code);
1160         vhost_scsi_release_tmf_res(tmf);
1161 }
1162
1163 static void
1164 vhost_scsi_handle_tmf(struct vhost_scsi *vs, struct vhost_scsi_tpg *tpg,
1165                       struct vhost_virtqueue *vq,
1166                       struct virtio_scsi_ctrl_tmf_req *vtmf,
1167                       struct vhost_scsi_ctx *vc)
1168 {
1169         struct vhost_scsi_virtqueue *svq = container_of(vq,
1170                                         struct vhost_scsi_virtqueue, vq);
1171         struct vhost_scsi_tmf *tmf;
1172
1173         if (vhost32_to_cpu(vq, vtmf->subtype) !=
1174             VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET)
1175                 goto send_reject;
1176
1177         if (!tpg->tpg_nexus || !tpg->tpg_nexus->tvn_se_sess) {
1178                 pr_err("Unable to locate active struct vhost_scsi_nexus for LUN RESET.\n");
1179                 goto send_reject;
1180         }
1181
1182         tmf = kzalloc(sizeof(*tmf), GFP_KERNEL);
1183         if (!tmf)
1184                 goto send_reject;
1185
1186         vhost_work_init(&tmf->vwork, vhost_scsi_tmf_resp_work);
1187         tmf->vhost = vs;
1188         tmf->svq = svq;
1189         tmf->resp_iov = vq->iov[vc->out];
1190         tmf->vq_desc = vc->head;
1191         tmf->in_iovs = vc->in;
1192         tmf->inflight = vhost_scsi_get_inflight(vq);
1193
1194         if (target_submit_tmr(&tmf->se_cmd, tpg->tpg_nexus->tvn_se_sess, NULL,
1195                               vhost_buf_to_lun(vtmf->lun), NULL,
1196                               TMR_LUN_RESET, GFP_KERNEL, 0,
1197                               TARGET_SCF_ACK_KREF) < 0) {
1198                 vhost_scsi_release_tmf_res(tmf);
1199                 goto send_reject;
1200         }
1201
1202         return;
1203
1204 send_reject:
1205         vhost_scsi_send_tmf_resp(vs, vq, vc->in, vc->head, &vq->iov[vc->out],
1206                                  VIRTIO_SCSI_S_FUNCTION_REJECTED);
1207 }
1208
1209 static void
1210 vhost_scsi_send_an_resp(struct vhost_scsi *vs,
1211                         struct vhost_virtqueue *vq,
1212                         struct vhost_scsi_ctx *vc)
1213 {
1214         struct virtio_scsi_ctrl_an_resp rsp;
1215         struct iov_iter iov_iter;
1216         int ret;
1217
1218         pr_debug("%s\n", __func__);
1219         memset(&rsp, 0, sizeof(rsp));   /* event_actual = 0 */
1220         rsp.response = VIRTIO_SCSI_S_OK;
1221
1222         iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[vc->out], vc->in, sizeof(rsp));
1223
1224         ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);
1225         if (likely(ret == sizeof(rsp)))
1226                 vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);
1227         else
1228                 pr_err("Faulted on virtio_scsi_ctrl_an_resp\n");
1229 }
1230
1231 static void
1232 vhost_scsi_ctl_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
1233 {
1234         struct vhost_scsi_tpg *tpg;
1235         union {
1236                 __virtio32 type;
1237                 struct virtio_scsi_ctrl_an_req an;
1238                 struct virtio_scsi_ctrl_tmf_req tmf;
1239         } v_req;
1240         struct vhost_scsi_ctx vc;
1241         size_t typ_size;
1242         int ret, c = 0;
1243
1244         mutex_lock(&vq->mutex);
1245         /*
1246          * We can handle the vq only after the endpoint is setup by calling the
1247          * VHOST_SCSI_SET_ENDPOINT ioctl.
1248          */
1249         if (!vhost_vq_get_backend(vq))
1250                 goto out;
1251
1252         memset(&vc, 0, sizeof(vc));
1253
1254         vhost_disable_notify(&vs->dev, vq);
1255
1256         do {
1257                 ret = vhost_scsi_get_desc(vs, vq, &vc);
1258                 if (ret)
1259                         goto err;
1260
1261                 /*
1262                  * Get the request type first in order to setup
1263                  * other parameters dependent on the type.
1264                  */
1265                 vc.req = &v_req.type;
1266                 typ_size = sizeof(v_req.type);
1267
1268                 if (unlikely(!copy_from_iter_full(vc.req, typ_size,
1269                                                   &vc.out_iter))) {
1270                         vq_err(vq, "Faulted on copy_from_iter tmf type\n");
1271                         /*
1272                          * The size of the response buffer depends on the
1273                          * request type and must be validated against it.
1274                          * Since the request type is not known, don't send
1275                          * a response.
1276                          */
1277                         continue;
1278                 }
1279
1280                 switch (vhost32_to_cpu(vq, v_req.type)) {
1281                 case VIRTIO_SCSI_T_TMF:
1282                         vc.req = &v_req.tmf;
1283                         vc.req_size = sizeof(struct virtio_scsi_ctrl_tmf_req);
1284                         vc.rsp_size = sizeof(struct virtio_scsi_ctrl_tmf_resp);
1285                         vc.lunp = &v_req.tmf.lun[0];
1286                         vc.target = &v_req.tmf.lun[1];
1287                         break;
1288                 case VIRTIO_SCSI_T_AN_QUERY:
1289                 case VIRTIO_SCSI_T_AN_SUBSCRIBE:
1290                         vc.req = &v_req.an;
1291                         vc.req_size = sizeof(struct virtio_scsi_ctrl_an_req);
1292                         vc.rsp_size = sizeof(struct virtio_scsi_ctrl_an_resp);
1293                         vc.lunp = &v_req.an.lun[0];
1294                         vc.target = NULL;
1295                         break;
1296                 default:
1297                         vq_err(vq, "Unknown control request %d", v_req.type);
1298                         continue;
1299                 }
1300
1301                 /*
1302                  * Validate the size of request and response buffers.
1303                  * Check for a sane response buffer so we can report
1304                  * early errors back to the guest.
1305                  */
1306                 ret = vhost_scsi_chk_size(vq, &vc);
1307                 if (ret)
1308                         goto err;
1309
1310                 /*
1311                  * Get the rest of the request now that its size is known.
1312                  */
1313                 vc.req += typ_size;
1314                 vc.req_size -= typ_size;
1315
1316                 ret = vhost_scsi_get_req(vq, &vc, &tpg);
1317                 if (ret)
1318                         goto err;
1319
1320                 if (v_req.type == VIRTIO_SCSI_T_TMF)
1321                         vhost_scsi_handle_tmf(vs, tpg, vq, &v_req.tmf, &vc);
1322                 else
1323                         vhost_scsi_send_an_resp(vs, vq, &vc);
1324 err:
1325                 /*
1326                  * ENXIO:  No more requests, or read error, wait for next kick
1327                  * EINVAL: Invalid response buffer, drop the request
1328                  * EIO:    Respond with bad target
1329                  * EAGAIN: Pending request
1330                  */
1331                 if (ret == -ENXIO)
1332                         break;
1333                 else if (ret == -EIO)
1334                         vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out);
1335         } while (likely(!vhost_exceeds_weight(vq, ++c, 0)));
1336 out:
1337         mutex_unlock(&vq->mutex);
1338 }
1339
1340 static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)
1341 {
1342         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1343                                                 poll.work);
1344         struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1345
1346         pr_debug("%s: The handling func for control queue.\n", __func__);
1347         vhost_scsi_ctl_handle_vq(vs, vq);
1348 }
1349
1350 static void
1351 vhost_scsi_send_evt(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
1352                     struct vhost_scsi_tpg *tpg, struct se_lun *lun,
1353                     u32 event, u32 reason)
1354 {
1355         struct vhost_scsi_evt *evt;
1356
1357         evt = vhost_scsi_allocate_evt(vs, event, reason);
1358         if (!evt)
1359                 return;
1360
1361         if (tpg && lun) {
1362                 /* TODO: share lun setup code with virtio-scsi.ko */
1363                 /*
1364                  * Note: evt->event is zeroed when we allocate it and
1365                  * lun[4-7] need to be zero according to virtio-scsi spec.
1366                  */
1367                 evt->event.lun[0] = 0x01;
1368                 evt->event.lun[1] = tpg->tport_tpgt;
1369                 if (lun->unpacked_lun >= 256)
1370                         evt->event.lun[2] = lun->unpacked_lun >> 8 | 0x40 ;
1371                 evt->event.lun[3] = lun->unpacked_lun & 0xFF;
1372         }
1373
1374         llist_add(&evt->list, &vs->vs_event_list);
1375         vhost_vq_work_queue(vq, &vs->vs_event_work);
1376 }
1377
1378 static void vhost_scsi_evt_handle_kick(struct vhost_work *work)
1379 {
1380         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1381                                                 poll.work);
1382         struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1383
1384         mutex_lock(&vq->mutex);
1385         if (!vhost_vq_get_backend(vq))
1386                 goto out;
1387
1388         if (vs->vs_events_missed)
1389                 vhost_scsi_send_evt(vs, vq, NULL, NULL, VIRTIO_SCSI_T_NO_EVENT,
1390                                     0);
1391 out:
1392         mutex_unlock(&vq->mutex);
1393 }
1394
1395 static void vhost_scsi_handle_kick(struct vhost_work *work)
1396 {
1397         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1398                                                 poll.work);
1399         struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1400
1401         vhost_scsi_handle_vq(vs, vq);
1402 }
1403
1404 /* Callers must hold dev mutex */
1405 static void vhost_scsi_flush(struct vhost_scsi *vs)
1406 {
1407         int i;
1408
1409         /* Init new inflight and remember the old inflight */
1410         vhost_scsi_init_inflight(vs, vs->old_inflight);
1411
1412         /*
1413          * The inflight->kref was initialized to 1. We decrement it here to
1414          * indicate the start of the flush operation so that it will reach 0
1415          * when all the reqs are finished.
1416          */
1417         for (i = 0; i < vs->dev.nvqs; i++)
1418                 kref_put(&vs->old_inflight[i]->kref, vhost_scsi_done_inflight);
1419
1420         /* Flush both the vhost poll and vhost work */
1421         vhost_dev_flush(&vs->dev);
1422
1423         /* Wait for all reqs issued before the flush to be finished */
1424         for (i = 0; i < vs->dev.nvqs; i++)
1425                 wait_for_completion(&vs->old_inflight[i]->comp);
1426 }
1427
1428 static void vhost_scsi_destroy_vq_cmds(struct vhost_virtqueue *vq)
1429 {
1430         struct vhost_scsi_virtqueue *svq = container_of(vq,
1431                                         struct vhost_scsi_virtqueue, vq);
1432         struct vhost_scsi_cmd *tv_cmd;
1433         unsigned int i;
1434
1435         if (!svq->scsi_cmds)
1436                 return;
1437
1438         for (i = 0; i < svq->max_cmds; i++) {
1439                 tv_cmd = &svq->scsi_cmds[i];
1440
1441                 kfree(tv_cmd->tvc_sgl);
1442                 kfree(tv_cmd->tvc_prot_sgl);
1443                 kfree(tv_cmd->tvc_upages);
1444                 kfree(tv_cmd->tvc_resp_iov);
1445         }
1446
1447         sbitmap_free(&svq->scsi_tags);
1448         kfree(svq->scsi_cmds);
1449         svq->scsi_cmds = NULL;
1450 }
1451
1452 static int vhost_scsi_setup_vq_cmds(struct vhost_virtqueue *vq, int max_cmds)
1453 {
1454         struct vhost_scsi_virtqueue *svq = container_of(vq,
1455                                         struct vhost_scsi_virtqueue, vq);
1456         struct vhost_scsi_cmd *tv_cmd;
1457         unsigned int i;
1458
1459         if (svq->scsi_cmds)
1460                 return 0;
1461
1462         if (sbitmap_init_node(&svq->scsi_tags, max_cmds, -1, GFP_KERNEL,
1463                               NUMA_NO_NODE, false, true))
1464                 return -ENOMEM;
1465         svq->max_cmds = max_cmds;
1466
1467         svq->scsi_cmds = kcalloc(max_cmds, sizeof(*tv_cmd), GFP_KERNEL);
1468         if (!svq->scsi_cmds) {
1469                 sbitmap_free(&svq->scsi_tags);
1470                 return -ENOMEM;
1471         }
1472
1473         for (i = 0; i < max_cmds; i++) {
1474                 tv_cmd = &svq->scsi_cmds[i];
1475
1476                 tv_cmd->tvc_sgl = kcalloc(VHOST_SCSI_PREALLOC_SGLS,
1477                                           sizeof(struct scatterlist),
1478                                           GFP_KERNEL);
1479                 if (!tv_cmd->tvc_sgl) {
1480                         pr_err("Unable to allocate tv_cmd->tvc_sgl\n");
1481                         goto out;
1482                 }
1483
1484                 tv_cmd->tvc_upages = kcalloc(VHOST_SCSI_PREALLOC_UPAGES,
1485                                              sizeof(struct page *),
1486                                              GFP_KERNEL);
1487                 if (!tv_cmd->tvc_upages) {
1488                         pr_err("Unable to allocate tv_cmd->tvc_upages\n");
1489                         goto out;
1490                 }
1491
1492                 tv_cmd->tvc_resp_iov = kcalloc(UIO_MAXIOV,
1493                                                sizeof(struct iovec),
1494                                                GFP_KERNEL);
1495                 if (!tv_cmd->tvc_resp_iov) {
1496                         pr_err("Unable to allocate tv_cmd->tvc_resp_iov\n");
1497                         goto out;
1498                 }
1499
1500                 tv_cmd->tvc_prot_sgl = kcalloc(VHOST_SCSI_PREALLOC_PROT_SGLS,
1501                                                sizeof(struct scatterlist),
1502                                                GFP_KERNEL);
1503                 if (!tv_cmd->tvc_prot_sgl) {
1504                         pr_err("Unable to allocate tv_cmd->tvc_prot_sgl\n");
1505                         goto out;
1506                 }
1507         }
1508         return 0;
1509 out:
1510         vhost_scsi_destroy_vq_cmds(vq);
1511         return -ENOMEM;
1512 }
1513
1514 /*
1515  * Called from vhost_scsi_ioctl() context to walk the list of available
1516  * vhost_scsi_tpg with an active struct vhost_scsi_nexus
1517  *
1518  *  The lock nesting rule is:
1519  *    vs->dev.mutex -> vhost_scsi_mutex -> tpg->tv_tpg_mutex -> vq->mutex
1520  */
1521 static int
1522 vhost_scsi_set_endpoint(struct vhost_scsi *vs,
1523                         struct vhost_scsi_target *t)
1524 {
1525         struct se_portal_group *se_tpg;
1526         struct vhost_scsi_tport *tv_tport;
1527         struct vhost_scsi_tpg *tpg;
1528         struct vhost_scsi_tpg **vs_tpg;
1529         struct vhost_virtqueue *vq;
1530         int index, ret, i, len;
1531         bool match = false;
1532
1533         mutex_lock(&vs->dev.mutex);
1534
1535         /* Verify that ring has been setup correctly. */
1536         for (index = 0; index < vs->dev.nvqs; ++index) {
1537                 /* Verify that ring has been setup correctly. */
1538                 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {
1539                         ret = -EFAULT;
1540                         goto out;
1541                 }
1542         }
1543
1544         len = sizeof(vs_tpg[0]) * VHOST_SCSI_MAX_TARGET;
1545         vs_tpg = kzalloc(len, GFP_KERNEL);
1546         if (!vs_tpg) {
1547                 ret = -ENOMEM;
1548                 goto out;
1549         }
1550         if (vs->vs_tpg)
1551                 memcpy(vs_tpg, vs->vs_tpg, len);
1552
1553         mutex_lock(&vhost_scsi_mutex);
1554         list_for_each_entry(tpg, &vhost_scsi_list, tv_tpg_list) {
1555                 mutex_lock(&tpg->tv_tpg_mutex);
1556                 if (!tpg->tpg_nexus) {
1557                         mutex_unlock(&tpg->tv_tpg_mutex);
1558                         continue;
1559                 }
1560                 if (tpg->tv_tpg_vhost_count != 0) {
1561                         mutex_unlock(&tpg->tv_tpg_mutex);
1562                         continue;
1563                 }
1564                 tv_tport = tpg->tport;
1565
1566                 if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1567                         if (vs->vs_tpg && vs->vs_tpg[tpg->tport_tpgt]) {
1568                                 mutex_unlock(&tpg->tv_tpg_mutex);
1569                                 mutex_unlock(&vhost_scsi_mutex);
1570                                 ret = -EEXIST;
1571                                 goto undepend;
1572                         }
1573                         /*
1574                          * In order to ensure individual vhost-scsi configfs
1575                          * groups cannot be removed while in use by vhost ioctl,
1576                          * go ahead and take an explicit se_tpg->tpg_group.cg_item
1577                          * dependency now.
1578                          */
1579                         se_tpg = &tpg->se_tpg;
1580                         ret = target_depend_item(&se_tpg->tpg_group.cg_item);
1581                         if (ret) {
1582                                 pr_warn("target_depend_item() failed: %d\n", ret);
1583                                 mutex_unlock(&tpg->tv_tpg_mutex);
1584                                 mutex_unlock(&vhost_scsi_mutex);
1585                                 goto undepend;
1586                         }
1587                         tpg->tv_tpg_vhost_count++;
1588                         tpg->vhost_scsi = vs;
1589                         vs_tpg[tpg->tport_tpgt] = tpg;
1590                         match = true;
1591                 }
1592                 mutex_unlock(&tpg->tv_tpg_mutex);
1593         }
1594         mutex_unlock(&vhost_scsi_mutex);
1595
1596         if (match) {
1597                 memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
1598                        sizeof(vs->vs_vhost_wwpn));
1599
1600                 for (i = VHOST_SCSI_VQ_IO; i < vs->dev.nvqs; i++) {
1601                         vq = &vs->vqs[i].vq;
1602                         if (!vhost_vq_is_setup(vq))
1603                                 continue;
1604
1605                         ret = vhost_scsi_setup_vq_cmds(vq, vq->num);
1606                         if (ret)
1607                                 goto destroy_vq_cmds;
1608                 }
1609
1610                 for (i = 0; i < vs->dev.nvqs; i++) {
1611                         vq = &vs->vqs[i].vq;
1612                         mutex_lock(&vq->mutex);
1613                         vhost_vq_set_backend(vq, vs_tpg);
1614                         vhost_vq_init_access(vq);
1615                         mutex_unlock(&vq->mutex);
1616                 }
1617                 ret = 0;
1618         } else {
1619                 ret = -EEXIST;
1620         }
1621
1622         /*
1623          * Act as synchronize_rcu to make sure access to
1624          * old vs->vs_tpg is finished.
1625          */
1626         vhost_scsi_flush(vs);
1627         kfree(vs->vs_tpg);
1628         vs->vs_tpg = vs_tpg;
1629         goto out;
1630
1631 destroy_vq_cmds:
1632         for (i--; i >= VHOST_SCSI_VQ_IO; i--) {
1633                 if (!vhost_vq_get_backend(&vs->vqs[i].vq))
1634                         vhost_scsi_destroy_vq_cmds(&vs->vqs[i].vq);
1635         }
1636 undepend:
1637         for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
1638                 tpg = vs_tpg[i];
1639                 if (tpg) {
1640                         mutex_lock(&tpg->tv_tpg_mutex);
1641                         tpg->vhost_scsi = NULL;
1642                         tpg->tv_tpg_vhost_count--;
1643                         mutex_unlock(&tpg->tv_tpg_mutex);
1644                         target_undepend_item(&tpg->se_tpg.tpg_group.cg_item);
1645                 }
1646         }
1647         kfree(vs_tpg);
1648 out:
1649         mutex_unlock(&vs->dev.mutex);
1650         return ret;
1651 }
1652
1653 static int
1654 vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
1655                           struct vhost_scsi_target *t)
1656 {
1657         struct se_portal_group *se_tpg;
1658         struct vhost_scsi_tport *tv_tport;
1659         struct vhost_scsi_tpg *tpg;
1660         struct vhost_virtqueue *vq;
1661         bool match = false;
1662         int index, ret, i;
1663         u8 target;
1664
1665         mutex_lock(&vs->dev.mutex);
1666         /* Verify that ring has been setup correctly. */
1667         for (index = 0; index < vs->dev.nvqs; ++index) {
1668                 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {
1669                         ret = -EFAULT;
1670                         goto err_dev;
1671                 }
1672         }
1673
1674         if (!vs->vs_tpg) {
1675                 ret = 0;
1676                 goto err_dev;
1677         }
1678
1679         for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
1680                 target = i;
1681                 tpg = vs->vs_tpg[target];
1682                 if (!tpg)
1683                         continue;
1684
1685                 tv_tport = tpg->tport;
1686                 if (!tv_tport) {
1687                         ret = -ENODEV;
1688                         goto err_dev;
1689                 }
1690
1691                 if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1692                         pr_warn("tv_tport->tport_name: %s, tpg->tport_tpgt: %hu"
1693                                 " does not match t->vhost_wwpn: %s, t->vhost_tpgt: %hu\n",
1694                                 tv_tport->tport_name, tpg->tport_tpgt,
1695                                 t->vhost_wwpn, t->vhost_tpgt);
1696                         ret = -EINVAL;
1697                         goto err_dev;
1698                 }
1699                 match = true;
1700         }
1701         if (!match)
1702                 goto free_vs_tpg;
1703
1704         /* Prevent new cmds from starting and accessing the tpgs/sessions */
1705         for (i = 0; i < vs->dev.nvqs; i++) {
1706                 vq = &vs->vqs[i].vq;
1707                 mutex_lock(&vq->mutex);
1708                 vhost_vq_set_backend(vq, NULL);
1709                 mutex_unlock(&vq->mutex);
1710         }
1711         /* Make sure cmds are not running before tearing them down. */
1712         vhost_scsi_flush(vs);
1713
1714         for (i = 0; i < vs->dev.nvqs; i++) {
1715                 vq = &vs->vqs[i].vq;
1716                 vhost_scsi_destroy_vq_cmds(vq);
1717         }
1718
1719         /*
1720          * We can now release our hold on the tpg and sessions and userspace
1721          * can free them after this point.
1722          */
1723         for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
1724                 target = i;
1725                 tpg = vs->vs_tpg[target];
1726                 if (!tpg)
1727                         continue;
1728
1729                 mutex_lock(&tpg->tv_tpg_mutex);
1730
1731                 tpg->tv_tpg_vhost_count--;
1732                 tpg->vhost_scsi = NULL;
1733                 vs->vs_tpg[target] = NULL;
1734
1735                 mutex_unlock(&tpg->tv_tpg_mutex);
1736
1737                 se_tpg = &tpg->se_tpg;
1738                 target_undepend_item(&se_tpg->tpg_group.cg_item);
1739         }
1740
1741 free_vs_tpg:
1742         /*
1743          * Act as synchronize_rcu to make sure access to
1744          * old vs->vs_tpg is finished.
1745          */
1746         vhost_scsi_flush(vs);
1747         kfree(vs->vs_tpg);
1748         vs->vs_tpg = NULL;
1749         WARN_ON(vs->vs_events_nr);
1750         mutex_unlock(&vs->dev.mutex);
1751         return 0;
1752
1753 err_dev:
1754         mutex_unlock(&vs->dev.mutex);
1755         return ret;
1756 }
1757
1758 static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features)
1759 {
1760         struct vhost_virtqueue *vq;
1761         int i;
1762
1763         if (features & ~VHOST_SCSI_FEATURES)
1764                 return -EOPNOTSUPP;
1765
1766         mutex_lock(&vs->dev.mutex);
1767         if ((features & (1 << VHOST_F_LOG_ALL)) &&
1768             !vhost_log_access_ok(&vs->dev)) {
1769                 mutex_unlock(&vs->dev.mutex);
1770                 return -EFAULT;
1771         }
1772
1773         for (i = 0; i < vs->dev.nvqs; i++) {
1774                 vq = &vs->vqs[i].vq;
1775                 mutex_lock(&vq->mutex);
1776                 vq->acked_features = features;
1777                 mutex_unlock(&vq->mutex);
1778         }
1779         mutex_unlock(&vs->dev.mutex);
1780         return 0;
1781 }
1782
1783 static int vhost_scsi_open(struct inode *inode, struct file *f)
1784 {
1785         struct vhost_scsi_virtqueue *svq;
1786         struct vhost_scsi *vs;
1787         struct vhost_virtqueue **vqs;
1788         int r = -ENOMEM, i, nvqs = vhost_scsi_max_io_vqs;
1789
1790         vs = kvzalloc(sizeof(*vs), GFP_KERNEL);
1791         if (!vs)
1792                 goto err_vs;
1793
1794         if (nvqs > VHOST_SCSI_MAX_IO_VQ) {
1795                 pr_err("Invalid max_io_vqs of %d. Using %d.\n", nvqs,
1796                        VHOST_SCSI_MAX_IO_VQ);
1797                 nvqs = VHOST_SCSI_MAX_IO_VQ;
1798         } else if (nvqs == 0) {
1799                 pr_err("Invalid max_io_vqs of %d. Using 1.\n", nvqs);
1800                 nvqs = 1;
1801         }
1802         nvqs += VHOST_SCSI_VQ_IO;
1803
1804         vs->old_inflight = kmalloc_array(nvqs, sizeof(*vs->old_inflight),
1805                                          GFP_KERNEL | __GFP_ZERO);
1806         if (!vs->old_inflight)
1807                 goto err_inflight;
1808
1809         vs->vqs = kmalloc_array(nvqs, sizeof(*vs->vqs),
1810                                 GFP_KERNEL | __GFP_ZERO);
1811         if (!vs->vqs)
1812                 goto err_vqs;
1813
1814         vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL);
1815         if (!vqs)
1816                 goto err_local_vqs;
1817
1818         vhost_work_init(&vs->vs_event_work, vhost_scsi_evt_work);
1819
1820         vs->vs_events_nr = 0;
1821         vs->vs_events_missed = false;
1822
1823         vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq;
1824         vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1825         vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;
1826         vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;
1827         for (i = VHOST_SCSI_VQ_IO; i < nvqs; i++) {
1828                 svq = &vs->vqs[i];
1829
1830                 vqs[i] = &svq->vq;
1831                 svq->vs = vs;
1832                 init_llist_head(&svq->completion_list);
1833                 vhost_work_init(&svq->completion_work,
1834                                 vhost_scsi_complete_cmd_work);
1835                 svq->vq.handle_kick = vhost_scsi_handle_kick;
1836         }
1837         vhost_dev_init(&vs->dev, vqs, nvqs, UIO_MAXIOV,
1838                        VHOST_SCSI_WEIGHT, 0, true, NULL);
1839
1840         vhost_scsi_init_inflight(vs, NULL);
1841
1842         f->private_data = vs;
1843         return 0;
1844
1845 err_local_vqs:
1846         kfree(vs->vqs);
1847 err_vqs:
1848         kfree(vs->old_inflight);
1849 err_inflight:
1850         kvfree(vs);
1851 err_vs:
1852         return r;
1853 }
1854
1855 static int vhost_scsi_release(struct inode *inode, struct file *f)
1856 {
1857         struct vhost_scsi *vs = f->private_data;
1858         struct vhost_scsi_target t;
1859
1860         mutex_lock(&vs->dev.mutex);
1861         memcpy(t.vhost_wwpn, vs->vs_vhost_wwpn, sizeof(t.vhost_wwpn));
1862         mutex_unlock(&vs->dev.mutex);
1863         vhost_scsi_clear_endpoint(vs, &t);
1864         vhost_dev_stop(&vs->dev);
1865         vhost_dev_cleanup(&vs->dev);
1866         kfree(vs->dev.vqs);
1867         kfree(vs->vqs);
1868         kfree(vs->old_inflight);
1869         kvfree(vs);
1870         return 0;
1871 }
1872
1873 static long
1874 vhost_scsi_ioctl(struct file *f,
1875                  unsigned int ioctl,
1876                  unsigned long arg)
1877 {
1878         struct vhost_scsi *vs = f->private_data;
1879         struct vhost_scsi_target backend;
1880         void __user *argp = (void __user *)arg;
1881         u64 __user *featurep = argp;
1882         u32 __user *eventsp = argp;
1883         u32 events_missed;
1884         u64 features;
1885         int r, abi_version = VHOST_SCSI_ABI_VERSION;
1886         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1887
1888         switch (ioctl) {
1889         case VHOST_SCSI_SET_ENDPOINT:
1890                 if (copy_from_user(&backend, argp, sizeof backend))
1891                         return -EFAULT;
1892                 if (backend.reserved != 0)
1893                         return -EOPNOTSUPP;
1894
1895                 return vhost_scsi_set_endpoint(vs, &backend);
1896         case VHOST_SCSI_CLEAR_ENDPOINT:
1897                 if (copy_from_user(&backend, argp, sizeof backend))
1898                         return -EFAULT;
1899                 if (backend.reserved != 0)
1900                         return -EOPNOTSUPP;
1901
1902                 return vhost_scsi_clear_endpoint(vs, &backend);
1903         case VHOST_SCSI_GET_ABI_VERSION:
1904                 if (copy_to_user(argp, &abi_version, sizeof abi_version))
1905                         return -EFAULT;
1906                 return 0;
1907         case VHOST_SCSI_SET_EVENTS_MISSED:
1908                 if (get_user(events_missed, eventsp))
1909                         return -EFAULT;
1910                 mutex_lock(&vq->mutex);
1911                 vs->vs_events_missed = events_missed;
1912                 mutex_unlock(&vq->mutex);
1913                 return 0;
1914         case VHOST_SCSI_GET_EVENTS_MISSED:
1915                 mutex_lock(&vq->mutex);
1916                 events_missed = vs->vs_events_missed;
1917                 mutex_unlock(&vq->mutex);
1918                 if (put_user(events_missed, eventsp))
1919                         return -EFAULT;
1920                 return 0;
1921         case VHOST_GET_FEATURES:
1922                 features = VHOST_SCSI_FEATURES;
1923                 if (copy_to_user(featurep, &features, sizeof features))
1924                         return -EFAULT;
1925                 return 0;
1926         case VHOST_SET_FEATURES:
1927                 if (copy_from_user(&features, featurep, sizeof features))
1928                         return -EFAULT;
1929                 return vhost_scsi_set_features(vs, features);
1930         case VHOST_NEW_WORKER:
1931         case VHOST_FREE_WORKER:
1932         case VHOST_ATTACH_VRING_WORKER:
1933         case VHOST_GET_VRING_WORKER:
1934                 mutex_lock(&vs->dev.mutex);
1935                 r = vhost_worker_ioctl(&vs->dev, ioctl, argp);
1936                 mutex_unlock(&vs->dev.mutex);
1937                 return r;
1938         default:
1939                 mutex_lock(&vs->dev.mutex);
1940                 r = vhost_dev_ioctl(&vs->dev, ioctl, argp);
1941                 /* TODO: flush backend after dev ioctl. */
1942                 if (r == -ENOIOCTLCMD)
1943                         r = vhost_vring_ioctl(&vs->dev, ioctl, argp);
1944                 mutex_unlock(&vs->dev.mutex);
1945                 return r;
1946         }
1947 }
1948
1949 static const struct file_operations vhost_scsi_fops = {
1950         .owner          = THIS_MODULE,
1951         .release        = vhost_scsi_release,
1952         .unlocked_ioctl = vhost_scsi_ioctl,
1953         .compat_ioctl   = compat_ptr_ioctl,
1954         .open           = vhost_scsi_open,
1955         .llseek         = noop_llseek,
1956 };
1957
1958 static struct miscdevice vhost_scsi_misc = {
1959         MISC_DYNAMIC_MINOR,
1960         "vhost-scsi",
1961         &vhost_scsi_fops,
1962 };
1963
1964 static int __init vhost_scsi_register(void)
1965 {
1966         return misc_register(&vhost_scsi_misc);
1967 }
1968
1969 static void vhost_scsi_deregister(void)
1970 {
1971         misc_deregister(&vhost_scsi_misc);
1972 }
1973
1974 static char *vhost_scsi_dump_proto_id(struct vhost_scsi_tport *tport)
1975 {
1976         switch (tport->tport_proto_id) {
1977         case SCSI_PROTOCOL_SAS:
1978                 return "SAS";
1979         case SCSI_PROTOCOL_FCP:
1980                 return "FCP";
1981         case SCSI_PROTOCOL_ISCSI:
1982                 return "iSCSI";
1983         default:
1984                 break;
1985         }
1986
1987         return "Unknown";
1988 }
1989
1990 static void
1991 vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg,
1992                   struct se_lun *lun, bool plug)
1993 {
1994
1995         struct vhost_scsi *vs = tpg->vhost_scsi;
1996         struct vhost_virtqueue *vq;
1997         u32 reason;
1998
1999         if (!vs)
2000                 return;
2001
2002         if (plug)
2003                 reason = VIRTIO_SCSI_EVT_RESET_RESCAN;
2004         else
2005                 reason = VIRTIO_SCSI_EVT_RESET_REMOVED;
2006
2007         vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
2008         mutex_lock(&vq->mutex);
2009         /*
2010          * We can't queue events if the backend has been cleared, because
2011          * we could end up queueing an event after the flush.
2012          */
2013         if (!vhost_vq_get_backend(vq))
2014                 goto unlock;
2015
2016         if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG))
2017                 vhost_scsi_send_evt(vs, vq, tpg, lun,
2018                                    VIRTIO_SCSI_T_TRANSPORT_RESET, reason);
2019 unlock:
2020         mutex_unlock(&vq->mutex);
2021 }
2022
2023 static void vhost_scsi_hotplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
2024 {
2025         vhost_scsi_do_plug(tpg, lun, true);
2026 }
2027
2028 static void vhost_scsi_hotunplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
2029 {
2030         vhost_scsi_do_plug(tpg, lun, false);
2031 }
2032
2033 static int vhost_scsi_port_link(struct se_portal_group *se_tpg,
2034                                struct se_lun *lun)
2035 {
2036         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2037                                 struct vhost_scsi_tpg, se_tpg);
2038
2039         mutex_lock(&tpg->tv_tpg_mutex);
2040         tpg->tv_tpg_port_count++;
2041         vhost_scsi_hotplug(tpg, lun);
2042         mutex_unlock(&tpg->tv_tpg_mutex);
2043
2044         return 0;
2045 }
2046
2047 static void vhost_scsi_port_unlink(struct se_portal_group *se_tpg,
2048                                   struct se_lun *lun)
2049 {
2050         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2051                                 struct vhost_scsi_tpg, se_tpg);
2052
2053         mutex_lock(&tpg->tv_tpg_mutex);
2054         tpg->tv_tpg_port_count--;
2055         vhost_scsi_hotunplug(tpg, lun);
2056         mutex_unlock(&tpg->tv_tpg_mutex);
2057 }
2058
2059 static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_store(
2060                 struct config_item *item, const char *page, size_t count)
2061 {
2062         struct se_portal_group *se_tpg = attrib_to_tpg(item);
2063         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2064                                 struct vhost_scsi_tpg, se_tpg);
2065         unsigned long val;
2066         int ret = kstrtoul(page, 0, &val);
2067
2068         if (ret) {
2069                 pr_err("kstrtoul() returned %d for fabric_prot_type\n", ret);
2070                 return ret;
2071         }
2072         if (val != 0 && val != 1 && val != 3) {
2073                 pr_err("Invalid vhost_scsi fabric_prot_type: %lu\n", val);
2074                 return -EINVAL;
2075         }
2076         tpg->tv_fabric_prot_type = val;
2077
2078         return count;
2079 }
2080
2081 static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_show(
2082                 struct config_item *item, char *page)
2083 {
2084         struct se_portal_group *se_tpg = attrib_to_tpg(item);
2085         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2086                                 struct vhost_scsi_tpg, se_tpg);
2087
2088         return sysfs_emit(page, "%d\n", tpg->tv_fabric_prot_type);
2089 }
2090
2091 CONFIGFS_ATTR(vhost_scsi_tpg_attrib_, fabric_prot_type);
2092
2093 static struct configfs_attribute *vhost_scsi_tpg_attrib_attrs[] = {
2094         &vhost_scsi_tpg_attrib_attr_fabric_prot_type,
2095         NULL,
2096 };
2097
2098 static int vhost_scsi_make_nexus(struct vhost_scsi_tpg *tpg,
2099                                 const char *name)
2100 {
2101         struct vhost_scsi_nexus *tv_nexus;
2102
2103         mutex_lock(&tpg->tv_tpg_mutex);
2104         if (tpg->tpg_nexus) {
2105                 mutex_unlock(&tpg->tv_tpg_mutex);
2106                 pr_debug("tpg->tpg_nexus already exists\n");
2107                 return -EEXIST;
2108         }
2109
2110         tv_nexus = kzalloc(sizeof(*tv_nexus), GFP_KERNEL);
2111         if (!tv_nexus) {
2112                 mutex_unlock(&tpg->tv_tpg_mutex);
2113                 pr_err("Unable to allocate struct vhost_scsi_nexus\n");
2114                 return -ENOMEM;
2115         }
2116         /*
2117          * Since we are running in 'demo mode' this call with generate a
2118          * struct se_node_acl for the vhost_scsi struct se_portal_group with
2119          * the SCSI Initiator port name of the passed configfs group 'name'.
2120          */
2121         tv_nexus->tvn_se_sess = target_setup_session(&tpg->se_tpg, 0, 0,
2122                                         TARGET_PROT_DIN_PASS | TARGET_PROT_DOUT_PASS,
2123                                         (unsigned char *)name, tv_nexus, NULL);
2124         if (IS_ERR(tv_nexus->tvn_se_sess)) {
2125                 mutex_unlock(&tpg->tv_tpg_mutex);
2126                 kfree(tv_nexus);
2127                 return -ENOMEM;
2128         }
2129         tpg->tpg_nexus = tv_nexus;
2130
2131         mutex_unlock(&tpg->tv_tpg_mutex);
2132         return 0;
2133 }
2134
2135 static int vhost_scsi_drop_nexus(struct vhost_scsi_tpg *tpg)
2136 {
2137         struct se_session *se_sess;
2138         struct vhost_scsi_nexus *tv_nexus;
2139
2140         mutex_lock(&tpg->tv_tpg_mutex);
2141         tv_nexus = tpg->tpg_nexus;
2142         if (!tv_nexus) {
2143                 mutex_unlock(&tpg->tv_tpg_mutex);
2144                 return -ENODEV;
2145         }
2146
2147         se_sess = tv_nexus->tvn_se_sess;
2148         if (!se_sess) {
2149                 mutex_unlock(&tpg->tv_tpg_mutex);
2150                 return -ENODEV;
2151         }
2152
2153         if (tpg->tv_tpg_port_count != 0) {
2154                 mutex_unlock(&tpg->tv_tpg_mutex);
2155                 pr_err("Unable to remove TCM_vhost I_T Nexus with"
2156                         " active TPG port count: %d\n",
2157                         tpg->tv_tpg_port_count);
2158                 return -EBUSY;
2159         }
2160
2161         if (tpg->tv_tpg_vhost_count != 0) {
2162                 mutex_unlock(&tpg->tv_tpg_mutex);
2163                 pr_err("Unable to remove TCM_vhost I_T Nexus with"
2164                         " active TPG vhost count: %d\n",
2165                         tpg->tv_tpg_vhost_count);
2166                 return -EBUSY;
2167         }
2168
2169         pr_debug("TCM_vhost_ConfigFS: Removing I_T Nexus to emulated"
2170                 " %s Initiator Port: %s\n", vhost_scsi_dump_proto_id(tpg->tport),
2171                 tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
2172
2173         /*
2174          * Release the SCSI I_T Nexus to the emulated vhost Target Port
2175          */
2176         target_remove_session(se_sess);
2177         tpg->tpg_nexus = NULL;
2178         mutex_unlock(&tpg->tv_tpg_mutex);
2179
2180         kfree(tv_nexus);
2181         return 0;
2182 }
2183
2184 static ssize_t vhost_scsi_tpg_nexus_show(struct config_item *item, char *page)
2185 {
2186         struct se_portal_group *se_tpg = to_tpg(item);
2187         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2188                                 struct vhost_scsi_tpg, se_tpg);
2189         struct vhost_scsi_nexus *tv_nexus;
2190         ssize_t ret;
2191
2192         mutex_lock(&tpg->tv_tpg_mutex);
2193         tv_nexus = tpg->tpg_nexus;
2194         if (!tv_nexus) {
2195                 mutex_unlock(&tpg->tv_tpg_mutex);
2196                 return -ENODEV;
2197         }
2198         ret = sysfs_emit(page, "%s\n",
2199                         tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
2200         mutex_unlock(&tpg->tv_tpg_mutex);
2201
2202         return ret;
2203 }
2204
2205 static ssize_t vhost_scsi_tpg_nexus_store(struct config_item *item,
2206                 const char *page, size_t count)
2207 {
2208         struct se_portal_group *se_tpg = to_tpg(item);
2209         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2210                                 struct vhost_scsi_tpg, se_tpg);
2211         struct vhost_scsi_tport *tport_wwn = tpg->tport;
2212         unsigned char i_port[VHOST_SCSI_NAMELEN], *ptr, *port_ptr;
2213         int ret;
2214         /*
2215          * Shutdown the active I_T nexus if 'NULL' is passed..
2216          */
2217         if (!strncmp(page, "NULL", 4)) {
2218                 ret = vhost_scsi_drop_nexus(tpg);
2219                 return (!ret) ? count : ret;
2220         }
2221         /*
2222          * Otherwise make sure the passed virtual Initiator port WWN matches
2223          * the fabric protocol_id set in vhost_scsi_make_tport(), and call
2224          * vhost_scsi_make_nexus().
2225          */
2226         if (strlen(page) >= VHOST_SCSI_NAMELEN) {
2227                 pr_err("Emulated NAA Sas Address: %s, exceeds"
2228                                 " max: %d\n", page, VHOST_SCSI_NAMELEN);
2229                 return -EINVAL;
2230         }
2231         snprintf(&i_port[0], VHOST_SCSI_NAMELEN, "%s", page);
2232
2233         ptr = strstr(i_port, "naa.");
2234         if (ptr) {
2235                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_SAS) {
2236                         pr_err("Passed SAS Initiator Port %s does not"
2237                                 " match target port protoid: %s\n", i_port,
2238                                 vhost_scsi_dump_proto_id(tport_wwn));
2239                         return -EINVAL;
2240                 }
2241                 port_ptr = &i_port[0];
2242                 goto check_newline;
2243         }
2244         ptr = strstr(i_port, "fc.");
2245         if (ptr) {
2246                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_FCP) {
2247                         pr_err("Passed FCP Initiator Port %s does not"
2248                                 " match target port protoid: %s\n", i_port,
2249                                 vhost_scsi_dump_proto_id(tport_wwn));
2250                         return -EINVAL;
2251                 }
2252                 port_ptr = &i_port[3]; /* Skip over "fc." */
2253                 goto check_newline;
2254         }
2255         ptr = strstr(i_port, "iqn.");
2256         if (ptr) {
2257                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_ISCSI) {
2258                         pr_err("Passed iSCSI Initiator Port %s does not"
2259                                 " match target port protoid: %s\n", i_port,
2260                                 vhost_scsi_dump_proto_id(tport_wwn));
2261                         return -EINVAL;
2262                 }
2263                 port_ptr = &i_port[0];
2264                 goto check_newline;
2265         }
2266         pr_err("Unable to locate prefix for emulated Initiator Port:"
2267                         " %s\n", i_port);
2268         return -EINVAL;
2269         /*
2270          * Clear any trailing newline for the NAA WWN
2271          */
2272 check_newline:
2273         if (i_port[strlen(i_port)-1] == '\n')
2274                 i_port[strlen(i_port)-1] = '\0';
2275
2276         ret = vhost_scsi_make_nexus(tpg, port_ptr);
2277         if (ret < 0)
2278                 return ret;
2279
2280         return count;
2281 }
2282
2283 CONFIGFS_ATTR(vhost_scsi_tpg_, nexus);
2284
2285 static struct configfs_attribute *vhost_scsi_tpg_attrs[] = {
2286         &vhost_scsi_tpg_attr_nexus,
2287         NULL,
2288 };
2289
2290 static struct se_portal_group *
2291 vhost_scsi_make_tpg(struct se_wwn *wwn, const char *name)
2292 {
2293         struct vhost_scsi_tport *tport = container_of(wwn,
2294                         struct vhost_scsi_tport, tport_wwn);
2295
2296         struct vhost_scsi_tpg *tpg;
2297         u16 tpgt;
2298         int ret;
2299
2300         if (strstr(name, "tpgt_") != name)
2301                 return ERR_PTR(-EINVAL);
2302         if (kstrtou16(name + 5, 10, &tpgt) || tpgt >= VHOST_SCSI_MAX_TARGET)
2303                 return ERR_PTR(-EINVAL);
2304
2305         tpg = kzalloc(sizeof(*tpg), GFP_KERNEL);
2306         if (!tpg) {
2307                 pr_err("Unable to allocate struct vhost_scsi_tpg");
2308                 return ERR_PTR(-ENOMEM);
2309         }
2310         mutex_init(&tpg->tv_tpg_mutex);
2311         INIT_LIST_HEAD(&tpg->tv_tpg_list);
2312         tpg->tport = tport;
2313         tpg->tport_tpgt = tpgt;
2314
2315         ret = core_tpg_register(wwn, &tpg->se_tpg, tport->tport_proto_id);
2316         if (ret < 0) {
2317                 kfree(tpg);
2318                 return NULL;
2319         }
2320         mutex_lock(&vhost_scsi_mutex);
2321         list_add_tail(&tpg->tv_tpg_list, &vhost_scsi_list);
2322         mutex_unlock(&vhost_scsi_mutex);
2323
2324         return &tpg->se_tpg;
2325 }
2326
2327 static void vhost_scsi_drop_tpg(struct se_portal_group *se_tpg)
2328 {
2329         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2330                                 struct vhost_scsi_tpg, se_tpg);
2331
2332         mutex_lock(&vhost_scsi_mutex);
2333         list_del(&tpg->tv_tpg_list);
2334         mutex_unlock(&vhost_scsi_mutex);
2335         /*
2336          * Release the virtual I_T Nexus for this vhost TPG
2337          */
2338         vhost_scsi_drop_nexus(tpg);
2339         /*
2340          * Deregister the se_tpg from TCM..
2341          */
2342         core_tpg_deregister(se_tpg);
2343         kfree(tpg);
2344 }
2345
2346 static struct se_wwn *
2347 vhost_scsi_make_tport(struct target_fabric_configfs *tf,
2348                      struct config_group *group,
2349                      const char *name)
2350 {
2351         struct vhost_scsi_tport *tport;
2352         char *ptr;
2353         u64 wwpn = 0;
2354         int off = 0;
2355
2356         /* if (vhost_scsi_parse_wwn(name, &wwpn, 1) < 0)
2357                 return ERR_PTR(-EINVAL); */
2358
2359         tport = kzalloc(sizeof(*tport), GFP_KERNEL);
2360         if (!tport) {
2361                 pr_err("Unable to allocate struct vhost_scsi_tport");
2362                 return ERR_PTR(-ENOMEM);
2363         }
2364         tport->tport_wwpn = wwpn;
2365         /*
2366          * Determine the emulated Protocol Identifier and Target Port Name
2367          * based on the incoming configfs directory name.
2368          */
2369         ptr = strstr(name, "naa.");
2370         if (ptr) {
2371                 tport->tport_proto_id = SCSI_PROTOCOL_SAS;
2372                 goto check_len;
2373         }
2374         ptr = strstr(name, "fc.");
2375         if (ptr) {
2376                 tport->tport_proto_id = SCSI_PROTOCOL_FCP;
2377                 off = 3; /* Skip over "fc." */
2378                 goto check_len;
2379         }
2380         ptr = strstr(name, "iqn.");
2381         if (ptr) {
2382                 tport->tport_proto_id = SCSI_PROTOCOL_ISCSI;
2383                 goto check_len;
2384         }
2385
2386         pr_err("Unable to locate prefix for emulated Target Port:"
2387                         " %s\n", name);
2388         kfree(tport);
2389         return ERR_PTR(-EINVAL);
2390
2391 check_len:
2392         if (strlen(name) >= VHOST_SCSI_NAMELEN) {
2393                 pr_err("Emulated %s Address: %s, exceeds"
2394                         " max: %d\n", name, vhost_scsi_dump_proto_id(tport),
2395                         VHOST_SCSI_NAMELEN);
2396                 kfree(tport);
2397                 return ERR_PTR(-EINVAL);
2398         }
2399         snprintf(&tport->tport_name[0], VHOST_SCSI_NAMELEN, "%s", &name[off]);
2400
2401         pr_debug("TCM_VHost_ConfigFS: Allocated emulated Target"
2402                 " %s Address: %s\n", vhost_scsi_dump_proto_id(tport), name);
2403
2404         return &tport->tport_wwn;
2405 }
2406
2407 static void vhost_scsi_drop_tport(struct se_wwn *wwn)
2408 {
2409         struct vhost_scsi_tport *tport = container_of(wwn,
2410                                 struct vhost_scsi_tport, tport_wwn);
2411
2412         pr_debug("TCM_VHost_ConfigFS: Deallocating emulated Target"
2413                 " %s Address: %s\n", vhost_scsi_dump_proto_id(tport),
2414                 tport->tport_name);
2415
2416         kfree(tport);
2417 }
2418
2419 static ssize_t
2420 vhost_scsi_wwn_version_show(struct config_item *item, char *page)
2421 {
2422         return sysfs_emit(page, "TCM_VHOST fabric module %s on %s/%s"
2423                 "on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,
2424                 utsname()->machine);
2425 }
2426
2427 CONFIGFS_ATTR_RO(vhost_scsi_wwn_, version);
2428
2429 static struct configfs_attribute *vhost_scsi_wwn_attrs[] = {
2430         &vhost_scsi_wwn_attr_version,
2431         NULL,
2432 };
2433
2434 static const struct target_core_fabric_ops vhost_scsi_ops = {
2435         .module                         = THIS_MODULE,
2436         .fabric_name                    = "vhost",
2437         .max_data_sg_nents              = VHOST_SCSI_PREALLOC_SGLS,
2438         .tpg_get_wwn                    = vhost_scsi_get_fabric_wwn,
2439         .tpg_get_tag                    = vhost_scsi_get_tpgt,
2440         .tpg_check_demo_mode            = vhost_scsi_check_true,
2441         .tpg_check_demo_mode_cache      = vhost_scsi_check_true,
2442         .tpg_check_prot_fabric_only     = vhost_scsi_check_prot_fabric_only,
2443         .release_cmd                    = vhost_scsi_release_cmd,
2444         .check_stop_free                = vhost_scsi_check_stop_free,
2445         .sess_get_initiator_sid         = NULL,
2446         .write_pending                  = vhost_scsi_write_pending,
2447         .queue_data_in                  = vhost_scsi_queue_data_in,
2448         .queue_status                   = vhost_scsi_queue_status,
2449         .queue_tm_rsp                   = vhost_scsi_queue_tm_rsp,
2450         .aborted_task                   = vhost_scsi_aborted_task,
2451         /*
2452          * Setup callers for generic logic in target_core_fabric_configfs.c
2453          */
2454         .fabric_make_wwn                = vhost_scsi_make_tport,
2455         .fabric_drop_wwn                = vhost_scsi_drop_tport,
2456         .fabric_make_tpg                = vhost_scsi_make_tpg,
2457         .fabric_drop_tpg                = vhost_scsi_drop_tpg,
2458         .fabric_post_link               = vhost_scsi_port_link,
2459         .fabric_pre_unlink              = vhost_scsi_port_unlink,
2460
2461         .tfc_wwn_attrs                  = vhost_scsi_wwn_attrs,
2462         .tfc_tpg_base_attrs             = vhost_scsi_tpg_attrs,
2463         .tfc_tpg_attrib_attrs           = vhost_scsi_tpg_attrib_attrs,
2464 };
2465
2466 static int __init vhost_scsi_init(void)
2467 {
2468         int ret = -ENOMEM;
2469
2470         pr_debug("TCM_VHOST fabric module %s on %s/%s"
2471                 " on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,
2472                 utsname()->machine);
2473
2474         ret = vhost_scsi_register();
2475         if (ret < 0)
2476                 goto out;
2477
2478         ret = target_register_template(&vhost_scsi_ops);
2479         if (ret < 0)
2480                 goto out_vhost_scsi_deregister;
2481
2482         return 0;
2483
2484 out_vhost_scsi_deregister:
2485         vhost_scsi_deregister();
2486 out:
2487         return ret;
2488 };
2489
2490 static void vhost_scsi_exit(void)
2491 {
2492         target_unregister_template(&vhost_scsi_ops);
2493         vhost_scsi_deregister();
2494 };
2495
2496 MODULE_DESCRIPTION("VHOST_SCSI series fabric driver");
2497 MODULE_ALIAS("tcm_vhost");
2498 MODULE_LICENSE("GPL");
2499 module_init(vhost_scsi_init);
2500 module_exit(vhost_scsi_exit);