drivers/nvme/host/fabrics.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVMe over Fabrics common host code.
   4  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
   5  */
   6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7 #include <linux/init.h>
   8 #include <linux/miscdevice.h>
   9 #include <linux/module.h>
  10 #include <linux/mutex.h>
  11 #include <linux/parser.h>
  12 #include <linux/seq_file.h>
  13 #include "nvme.h"
  14 #include "fabrics.h"
  15
  16 static LIST_HEAD(nvmf_transports);
  17 static DECLARE_RWSEM(nvmf_transports_rwsem);
  18
  19 static LIST_HEAD(nvmf_hosts);
  20 static DEFINE_MUTEX(nvmf_hosts_mutex);
  21
  22 static struct nvmf_host *nvmf_default_host;
  23
  24 static struct nvmf_host *nvmf_host_alloc(const char *hostnqn, uuid_t *id)
  25 {
  26         struct nvmf_host *host;
  27
  28         host = kmalloc(sizeof(*host), GFP_KERNEL);
  29         if (!host)
  30                 return NULL;
  31
  32         kref_init(&host->ref);
  33         uuid_copy(&host->id, id);
  34         strscpy(host->nqn, hostnqn, NVMF_NQN_SIZE);
  35
  36         return host;
  37 }
  38
  39 static struct nvmf_host *nvmf_host_add(const char *hostnqn, uuid_t *id)
  40 {
  41         struct nvmf_host *host;
  42
  43         mutex_lock(&nvmf_hosts_mutex);
  44
  45         /*
  46          * We have defined a host as how it is perceived by the target.
  47          * Therefore, we don't allow different Host NQNs with the same Host ID.
  48          * Similarly, we do not allow the usage of the same Host NQN with
  49          * different Host IDs. This'll maintain unambiguous host identification.
  50          */
  51         list_for_each_entry(host, &nvmf_hosts, list) {
  52                 bool same_hostnqn = !strcmp(host->nqn, hostnqn);
  53                 bool same_hostid = uuid_equal(&host->id, id);
  54
  55                 if (same_hostnqn && same_hostid) {
  56                         kref_get(&host->ref);
  57                         goto out_unlock;
  58                 }
  59                 if (same_hostnqn) {
  60                         pr_err("found same hostnqn %s but different hostid %pUb\n",
  61                                hostnqn, id);
  62                         host = ERR_PTR(-EINVAL);
  63                         goto out_unlock;
  64                 }
  65                 if (same_hostid) {
  66                         pr_err("found same hostid %pUb but different hostnqn %s\n",
  67                                id, hostnqn);
  68                         host = ERR_PTR(-EINVAL);
  69                         goto out_unlock;
  70                 }
  71         }
  72
  73         host = nvmf_host_alloc(hostnqn, id);
  74         if (!host) {
  75                 host = ERR_PTR(-ENOMEM);
  76                 goto out_unlock;
  77         }
  78
  79         list_add_tail(&host->list, &nvmf_hosts);
  80 out_unlock:
  81         mutex_unlock(&nvmf_hosts_mutex);
  82         return host;
  83 }
  84
  85 static struct nvmf_host *nvmf_host_default(void)
  86 {
  87         struct nvmf_host *host;
  88         char nqn[NVMF_NQN_SIZE];
  89         uuid_t id;
  90
  91         uuid_gen(&id);
  92         snprintf(nqn, NVMF_NQN_SIZE,
  93                 "nqn.2014-08.org.nvmexpress:uuid:%pUb", &id);
  94
  95         host = nvmf_host_alloc(nqn, &id);
  96         if (!host)
  97                 return NULL;
  98
  99         mutex_lock(&nvmf_hosts_mutex);
 100         list_add_tail(&host->list, &nvmf_hosts);
 101         mutex_unlock(&nvmf_hosts_mutex);
 102
 103         return host;
 104 }
 105
 106 static void nvmf_host_destroy(struct kref *ref)
 107 {
 108         struct nvmf_host *host = container_of(ref, struct nvmf_host, ref);
 109
 110         mutex_lock(&nvmf_hosts_mutex);
 111         list_del(&host->list);
 112         mutex_unlock(&nvmf_hosts_mutex);
 113
 114         kfree(host);
 115 }
 116
 117 static void nvmf_host_put(struct nvmf_host *host)
 118 {
 119         if (host)
 120                 kref_put(&host->ref, nvmf_host_destroy);
 121 }
 122
 123 /**
 124  * nvmf_get_address() -  Get address/port
 125  * @ctrl:       Host NVMe controller instance which we got the address
 126  * @buf:        OUTPUT parameter that will contain the address/port
 127  * @size:       buffer size
 128  */
 129 int nvmf_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
 130 {
 131         int len = 0;
 132
 133         if (ctrl->opts->mask & NVMF_OPT_TRADDR)
 134                 len += scnprintf(buf, size, "traddr=%s", ctrl->opts->traddr);
 135         if (ctrl->opts->mask & NVMF_OPT_TRSVCID)
 136                 len += scnprintf(buf + len, size - len, "%strsvcid=%s",
 137                                 (len) ? "," : "", ctrl->opts->trsvcid);
 138         if (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)
 139                 len += scnprintf(buf + len, size - len, "%shost_traddr=%s",
 140                                 (len) ? "," : "", ctrl->opts->host_traddr);
 141         if (ctrl->opts->mask & NVMF_OPT_HOST_IFACE)
 142                 len += scnprintf(buf + len, size - len, "%shost_iface=%s",
 143                                 (len) ? "," : "", ctrl->opts->host_iface);
 144         len += scnprintf(buf + len, size - len, "\n");
 145
 146         return len;
 147 }
 148 EXPORT_SYMBOL_GPL(nvmf_get_address);
 149
 150 /**
 151  * nvmf_reg_read32() -  NVMe Fabrics "Property Get" API function.
 152  * @ctrl:       Host NVMe controller instance maintaining the admin
 153  *              queue used to submit the property read command to
 154  *              the allocated NVMe controller resource on the target system.
 155  * @off:        Starting offset value of the targeted property
 156  *              register (see the fabrics section of the NVMe standard).
 157  * @val:        OUTPUT parameter that will contain the value of
 158  *              the property after a successful read.
 159  *
 160  * Used by the host system to retrieve a 32-bit capsule property value
 161  * from an NVMe controller on the target system.
 162  *
 163  * ("Capsule property" is an "PCIe register concept" applied to the
 164  * NVMe fabrics space.)
 165  *
 166  * Return:
 167  *      0: successful read
 168  *      > 0: NVMe error status code
 169  *      < 0: Linux errno error code
 170  */
 171 int nvmf_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
 172 {
 173         struct nvme_command cmd = { };
 174         union nvme_result res;
 175         int ret;
 176
 177         cmd.prop_get.opcode = nvme_fabrics_command;
 178         cmd.prop_get.fctype = nvme_fabrics_type_property_get;
 179         cmd.prop_get.offset = cpu_to_le32(off);
 180
 181         ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, &res, NULL, 0,
 182                         NVME_QID_ANY, 0, 0);
 183
 184         if (ret >= 0)
 185                 *val = le64_to_cpu(res.u64);
 186         if (unlikely(ret != 0))
 187                 dev_err(ctrl->device,
 188                         "Property Get error: %d, offset %#x\n",
 189                         ret > 0 ? ret & ~NVME_SC_DNR : ret, off);
 190
 191         return ret;
 192 }
 193 EXPORT_SYMBOL_GPL(nvmf_reg_read32);
 194
 195 /**
 196  * nvmf_reg_read64() -  NVMe Fabrics "Property Get" API function.
 197  * @ctrl:       Host NVMe controller instance maintaining the admin
 198  *              queue used to submit the property read command to
 199  *              the allocated controller resource on the target system.
 200  * @off:        Starting offset value of the targeted property
 201  *              register (see the fabrics section of the NVMe standard).
 202  * @val:        OUTPUT parameter that will contain the value of
 203  *              the property after a successful read.
 204  *
 205  * Used by the host system to retrieve a 64-bit capsule property value
 206  * from an NVMe controller on the target system.
 207  *
 208  * ("Capsule property" is an "PCIe register concept" applied to the
 209  * NVMe fabrics space.)
 210  *
 211  * Return:
 212  *      0: successful read
 213  *      > 0: NVMe error status code
 214  *      < 0: Linux errno error code
 215  */
 216 int nvmf_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
 217 {
 218         struct nvme_command cmd = { };
 219         union nvme_result res;
 220         int ret;
 221
 222         cmd.prop_get.opcode = nvme_fabrics_command;
 223         cmd.prop_get.fctype = nvme_fabrics_type_property_get;
 224         cmd.prop_get.attrib = 1;
 225         cmd.prop_get.offset = cpu_to_le32(off);
 226
 227         ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, &res, NULL, 0,
 228                         NVME_QID_ANY, 0, 0);
 229
 230         if (ret >= 0)
 231                 *val = le64_to_cpu(res.u64);
 232         if (unlikely(ret != 0))
 233                 dev_err(ctrl->device,
 234                         "Property Get error: %d, offset %#x\n",
 235                         ret > 0 ? ret & ~NVME_SC_DNR : ret, off);
 236         return ret;
 237 }
 238 EXPORT_SYMBOL_GPL(nvmf_reg_read64);
 239
 240 /**
 241  * nvmf_reg_write32() -  NVMe Fabrics "Property Write" API function.
 242  * @ctrl:       Host NVMe controller instance maintaining the admin
 243  *              queue used to submit the property read command to
 244  *              the allocated NVMe controller resource on the target system.
 245  * @off:        Starting offset value of the targeted property
 246  *              register (see the fabrics section of the NVMe standard).
 247  * @val:        Input parameter that contains the value to be
 248  *              written to the property.
 249  *
 250  * Used by the NVMe host system to write a 32-bit capsule property value
 251  * to an NVMe controller on the target system.
 252  *
 253  * ("Capsule property" is an "PCIe register concept" applied to the
 254  * NVMe fabrics space.)
 255  *
 256  * Return:
 257  *      0: successful write
 258  *      > 0: NVMe error status code
 259  *      < 0: Linux errno error code
 260  */
 261 int nvmf_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
 262 {
 263         struct nvme_command cmd = { };
 264         int ret;
 265
 266         cmd.prop_set.opcode = nvme_fabrics_command;
 267         cmd.prop_set.fctype = nvme_fabrics_type_property_set;
 268         cmd.prop_set.attrib = 0;
 269         cmd.prop_set.offset = cpu_to_le32(off);
 270         cmd.prop_set.value = cpu_to_le64(val);
 271
 272         ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, NULL, NULL, 0,
 273                         NVME_QID_ANY, 0, 0);
 274         if (unlikely(ret))
 275                 dev_err(ctrl->device,
 276                         "Property Set error: %d, offset %#x\n",
 277                         ret > 0 ? ret & ~NVME_SC_DNR : ret, off);
 278         return ret;
 279 }
 280 EXPORT_SYMBOL_GPL(nvmf_reg_write32);
 281
 282 /**
 283  * nvmf_log_connect_error() - Error-parsing-diagnostic print out function for
 284  *                              connect() errors.
 285  * @ctrl:       The specific /dev/nvmeX device that had the error.
 286  * @errval:     Error code to be decoded in a more human-friendly
 287  *              printout.
 288  * @offset:     For use with the NVMe error code
 289  *              NVME_SC_CONNECT_INVALID_PARAM.
 290  * @cmd:        This is the SQE portion of a submission capsule.
 291  * @data:       This is the "Data" portion of a submission capsule.
 292  */
 293 static void nvmf_log_connect_error(struct nvme_ctrl *ctrl,
 294                 int errval, int offset, struct nvme_command *cmd,
 295                 struct nvmf_connect_data *data)
 296 {
 297         int err_sctype = errval & ~NVME_SC_DNR;
 298
 299         if (errval < 0) {
 300                 dev_err(ctrl->device,
 301                         "Connect command failed, errno: %d\n", errval);
 302                 return;
 303         }
 304
 305         switch (err_sctype) {
 306         case NVME_SC_CONNECT_INVALID_PARAM:
 307                 if (offset >> 16) {
 308                         char *inv_data = "Connect Invalid Data Parameter";
 309
 310                         switch (offset & 0xffff) {
 311                         case (offsetof(struct nvmf_connect_data, cntlid)):
 312                                 dev_err(ctrl->device,
 313                                         "%s, cntlid: %d\n",
 314                                         inv_data, data->cntlid);
 315                                 break;
 316                         case (offsetof(struct nvmf_connect_data, hostnqn)):
 317                                 dev_err(ctrl->device,
 318                                         "%s, hostnqn \"%s\"\n",
 319                                         inv_data, data->hostnqn);
 320                                 break;
 321                         case (offsetof(struct nvmf_connect_data, subsysnqn)):
 322                                 dev_err(ctrl->device,
 323                                         "%s, subsysnqn \"%s\"\n",
 324                                         inv_data, data->subsysnqn);
 325                                 break;
 326                         default:
 327                                 dev_err(ctrl->device,
 328                                         "%s, starting byte offset: %d\n",
 329                                        inv_data, offset & 0xffff);
 330                                 break;
 331                         }
 332                 } else {
 333                         char *inv_sqe = "Connect Invalid SQE Parameter";
 334
 335                         switch (offset) {
 336                         case (offsetof(struct nvmf_connect_command, qid)):
 337                                 dev_err(ctrl->device,
 338                                        "%s, qid %d\n",
 339                                         inv_sqe, cmd->connect.qid);
 340                                 break;
 341                         default:
 342                                 dev_err(ctrl->device,
 343                                         "%s, starting byte offset: %d\n",
 344                                         inv_sqe, offset);
 345                         }
 346                 }
 347                 break;
 348         case NVME_SC_CONNECT_INVALID_HOST:
 349                 dev_err(ctrl->device,
 350                         "Connect for subsystem %s is not allowed, hostnqn: %s\n",
 351                         data->subsysnqn, data->hostnqn);
 352                 break;
 353         case NVME_SC_CONNECT_CTRL_BUSY:
 354                 dev_err(ctrl->device,
 355                         "Connect command failed: controller is busy or not available\n");
 356                 break;
 357         case NVME_SC_CONNECT_FORMAT:
 358                 dev_err(ctrl->device,
 359                         "Connect incompatible format: %d",
 360                         cmd->connect.recfmt);
 361                 break;
 362         case NVME_SC_HOST_PATH_ERROR:
 363                 dev_err(ctrl->device,
 364                         "Connect command failed: host path error\n");
 365                 break;
 366         case NVME_SC_AUTH_REQUIRED:
 367                 dev_err(ctrl->device,
 368                         "Connect command failed: authentication required\n");
 369                 break;
 370         default:
 371                 dev_err(ctrl->device,
 372                         "Connect command failed, error wo/DNR bit: %d\n",
 373                         err_sctype);
 374                 break;
 375         }
 376 }
 377
 378 static struct nvmf_connect_data *nvmf_connect_data_prep(struct nvme_ctrl *ctrl,
 379                 u16 cntlid)
 380 {
 381         struct nvmf_connect_data *data;
 382
 383         data = kzalloc(sizeof(*data), GFP_KERNEL);
 384         if (!data)
 385                 return NULL;
 386
 387         uuid_copy(&data->hostid, &ctrl->opts->host->id);
 388         data->cntlid = cpu_to_le16(cntlid);
 389         strncpy(data->subsysnqn, ctrl->opts->subsysnqn, NVMF_NQN_SIZE);
 390         strncpy(data->hostnqn, ctrl->opts->host->nqn, NVMF_NQN_SIZE);
 391
 392         return data;
 393 }
 394
 395 static void nvmf_connect_cmd_prep(struct nvme_ctrl *ctrl, u16 qid,
 396                 struct nvme_command *cmd)
 397 {
 398         cmd->connect.opcode = nvme_fabrics_command;
 399         cmd->connect.fctype = nvme_fabrics_type_connect;
 400         cmd->connect.qid = cpu_to_le16(qid);
 401
 402         if (qid) {
 403                 cmd->connect.sqsize = cpu_to_le16(ctrl->sqsize);
 404         } else {
 405                 cmd->connect.sqsize = cpu_to_le16(NVME_AQ_DEPTH - 1);
 406
 407                 /*
 408                  * set keep-alive timeout in seconds granularity (ms * 1000)
 409                  */
 410                 cmd->connect.kato = cpu_to_le32(ctrl->kato * 1000);
 411         }
 412
 413         if (ctrl->opts->disable_sqflow)
 414                 cmd->connect.cattr |= NVME_CONNECT_DISABLE_SQFLOW;
 415 }
 416
 417 /**
 418  * nvmf_connect_admin_queue() - NVMe Fabrics Admin Queue "Connect"
 419  *                              API function.
 420  * @ctrl:       Host nvme controller instance used to request
 421  *              a new NVMe controller allocation on the target
 422  *              system and  establish an NVMe Admin connection to
 423  *              that controller.
 424  *
 425  * This function enables an NVMe host device to request a new allocation of
 426  * an NVMe controller resource on a target system as well establish a
 427  * fabrics-protocol connection of the NVMe Admin queue between the
 428  * host system device and the allocated NVMe controller on the
 429  * target system via a NVMe Fabrics "Connect" command.
 430  *
 431  * Return:
 432  *      0: success
 433  *      > 0: NVMe error status code
 434  *      < 0: Linux errno error code
 435  *
 436  */
 437 int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl)
 438 {
 439         struct nvme_command cmd = { };
 440         union nvme_result res;
 441         struct nvmf_connect_data *data;
 442         int ret;
 443         u32 result;
 444
 445         nvmf_connect_cmd_prep(ctrl, 0, &cmd);
 446
 447         data = nvmf_connect_data_prep(ctrl, 0xffff);
 448         if (!data)
 449                 return -ENOMEM;
 450
 451         ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, &res,
 452                         data, sizeof(*data), NVME_QID_ANY, 1,
 453                         BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
 454         if (ret) {
 455                 nvmf_log_connect_error(ctrl, ret, le32_to_cpu(res.u32),
 456                                        &cmd, data);
 457                 goto out_free_data;
 458         }
 459
 460         result = le32_to_cpu(res.u32);
 461         ctrl->cntlid = result & 0xFFFF;
 462         if (result & (NVME_CONNECT_AUTHREQ_ATR | NVME_CONNECT_AUTHREQ_ASCR)) {
 463                 /* Secure concatenation is not implemented */
 464                 if (result & NVME_CONNECT_AUTHREQ_ASCR) {
 465                         dev_warn(ctrl->device,
 466                                  "qid 0: secure concatenation is not supported\n");
 467                         ret = NVME_SC_AUTH_REQUIRED;
 468                         goto out_free_data;
 469                 }
 470                 /* Authentication required */
 471                 ret = nvme_auth_negotiate(ctrl, 0);
 472                 if (ret) {
 473                         dev_warn(ctrl->device,
 474                                  "qid 0: authentication setup failed\n");
 475                         ret = NVME_SC_AUTH_REQUIRED;
 476                         goto out_free_data;
 477                 }
 478                 ret = nvme_auth_wait(ctrl, 0);
 479                 if (ret)
 480                         dev_warn(ctrl->device,
 481                                  "qid 0: authentication failed\n");
 482                 else
 483                         dev_info(ctrl->device,
 484                                  "qid 0: authenticated\n");
 485         }
 486 out_free_data:
 487         kfree(data);
 488         return ret;
 489 }
 490 EXPORT_SYMBOL_GPL(nvmf_connect_admin_queue);
 491
 492 /**
 493  * nvmf_connect_io_queue() - NVMe Fabrics I/O Queue "Connect"
 494  *                           API function.
 495  * @ctrl:       Host nvme controller instance used to establish an
 496  *              NVMe I/O queue connection to the already allocated NVMe
 497  *              controller on the target system.
 498  * @qid:        NVMe I/O queue number for the new I/O connection between
 499  *              host and target (note qid == 0 is illegal as this is
 500  *              the Admin queue, per NVMe standard).
 501  *
 502  * This function issues a fabrics-protocol connection
 503  * of a NVMe I/O queue (via NVMe Fabrics "Connect" command)
 504  * between the host system device and the allocated NVMe controller
 505  * on the target system.
 506  *
 507  * Return:
 508  *      0: success
 509  *      > 0: NVMe error status code
 510  *      < 0: Linux errno error code
 511  */
 512 int nvmf_connect_io_queue(struct nvme_ctrl *ctrl, u16 qid)
 513 {
 514         struct nvme_command cmd = { };
 515         struct nvmf_connect_data *data;
 516         union nvme_result res;
 517         int ret;
 518         u32 result;
 519
 520         nvmf_connect_cmd_prep(ctrl, qid, &cmd);
 521
 522         data = nvmf_connect_data_prep(ctrl, ctrl->cntlid);
 523         if (!data)
 524                 return -ENOMEM;
 525
 526         ret = __nvme_submit_sync_cmd(ctrl->connect_q, &cmd, &res,
 527                         data, sizeof(*data), qid, 1,
 528                         BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
 529         if (ret) {
 530                 nvmf_log_connect_error(ctrl, ret, le32_to_cpu(res.u32),
 531                                        &cmd, data);
 532         }
 533         result = le32_to_cpu(res.u32);
 534         if (result & (NVME_CONNECT_AUTHREQ_ATR | NVME_CONNECT_AUTHREQ_ASCR)) {
 535                 /* Secure concatenation is not implemented */
 536                 if (result & NVME_CONNECT_AUTHREQ_ASCR) {
 537                         dev_warn(ctrl->device,
 538                                  "qid 0: secure concatenation is not supported\n");
 539                         ret = NVME_SC_AUTH_REQUIRED;
 540                         goto out_free_data;
 541                 }
 542                 /* Authentication required */
 543                 ret = nvme_auth_negotiate(ctrl, qid);
 544                 if (ret) {
 545                         dev_warn(ctrl->device,
 546                                  "qid %d: authentication setup failed\n", qid);
 547                         ret = NVME_SC_AUTH_REQUIRED;
 548                 } else {
 549                         ret = nvme_auth_wait(ctrl, qid);
 550                         if (ret)
 551                                 dev_warn(ctrl->device,
 552                                          "qid %u: authentication failed\n", qid);
 553                 }
 554         }
 555 out_free_data:
 556         kfree(data);
 557         return ret;
 558 }
 559 EXPORT_SYMBOL_GPL(nvmf_connect_io_queue);
 560
 561 bool nvmf_should_reconnect(struct nvme_ctrl *ctrl)
 562 {
 563         if (ctrl->opts->max_reconnects == -1 ||
 564             ctrl->nr_reconnects < ctrl->opts->max_reconnects)
 565                 return true;
 566
 567         return false;
 568 }
 569 EXPORT_SYMBOL_GPL(nvmf_should_reconnect);
 570
 571 /**
 572  * nvmf_register_transport() - NVMe Fabrics Library registration function.
 573  * @ops:        Transport ops instance to be registered to the
 574  *              common fabrics library.
 575  *
 576  * API function that registers the type of specific transport fabric
 577  * being implemented to the common NVMe fabrics library. Part of
 578  * the overall init sequence of starting up a fabrics driver.
 579  */
 580 int nvmf_register_transport(struct nvmf_transport_ops *ops)
 581 {
 582         if (!ops->create_ctrl)
 583                 return -EINVAL;
 584
 585         down_write(&nvmf_transports_rwsem);
 586         list_add_tail(&ops->entry, &nvmf_transports);
 587         up_write(&nvmf_transports_rwsem);
 588
 589         return 0;
 590 }
 591 EXPORT_SYMBOL_GPL(nvmf_register_transport);
 592
 593 /**
 594  * nvmf_unregister_transport() - NVMe Fabrics Library unregistration function.
 595  * @ops:        Transport ops instance to be unregistered from the
 596  *              common fabrics library.
 597  *
 598  * Fabrics API function that unregisters the type of specific transport
 599  * fabric being implemented from the common NVMe fabrics library.
 600  * Part of the overall exit sequence of unloading the implemented driver.
 601  */
 602 void nvmf_unregister_transport(struct nvmf_transport_ops *ops)
 603 {
 604         down_write(&nvmf_transports_rwsem);
 605         list_del(&ops->entry);
 606         up_write(&nvmf_transports_rwsem);
 607 }
 608 EXPORT_SYMBOL_GPL(nvmf_unregister_transport);
 609
 610 static struct nvmf_transport_ops *nvmf_lookup_transport(
 611                 struct nvmf_ctrl_options *opts)
 612 {
 613         struct nvmf_transport_ops *ops;
 614
 615         lockdep_assert_held(&nvmf_transports_rwsem);
 616
 617         list_for_each_entry(ops, &nvmf_transports, entry) {
 618                 if (strcmp(ops->name, opts->transport) == 0)
 619                         return ops;
 620         }
 621
 622         return NULL;
 623 }
 624
 625 static const match_table_t opt_tokens = {
 626         { NVMF_OPT_TRANSPORT,           "transport=%s"          },
 627         { NVMF_OPT_TRADDR,              "traddr=%s"             },
 628         { NVMF_OPT_TRSVCID,             "trsvcid=%s"            },
 629         { NVMF_OPT_NQN,                 "nqn=%s"                },
 630         { NVMF_OPT_QUEUE_SIZE,          "queue_size=%d"         },
 631         { NVMF_OPT_NR_IO_QUEUES,        "nr_io_queues=%d"       },
 632         { NVMF_OPT_RECONNECT_DELAY,     "reconnect_delay=%d"    },
 633         { NVMF_OPT_CTRL_LOSS_TMO,       "ctrl_loss_tmo=%d"      },
 634         { NVMF_OPT_KATO,                "keep_alive_tmo=%d"     },
 635         { NVMF_OPT_HOSTNQN,             "hostnqn=%s"            },
 636         { NVMF_OPT_HOST_TRADDR,         "host_traddr=%s"        },
 637         { NVMF_OPT_HOST_IFACE,          "host_iface=%s"         },
 638         { NVMF_OPT_HOST_ID,             "hostid=%s"             },
 639         { NVMF_OPT_DUP_CONNECT,         "duplicate_connect"     },
 640         { NVMF_OPT_DISABLE_SQFLOW,      "disable_sqflow"        },
 641         { NVMF_OPT_HDR_DIGEST,          "hdr_digest"            },
 642         { NVMF_OPT_DATA_DIGEST,         "data_digest"           },
 643         { NVMF_OPT_NR_WRITE_QUEUES,     "nr_write_queues=%d"    },
 644         { NVMF_OPT_NR_POLL_QUEUES,      "nr_poll_queues=%d"     },
 645         { NVMF_OPT_TOS,                 "tos=%d"                },
 646         { NVMF_OPT_FAIL_FAST_TMO,       "fast_io_fail_tmo=%d"   },
 647         { NVMF_OPT_DISCOVERY,           "discovery"             },
 648 #ifdef CONFIG_NVME_HOST_AUTH
 649         { NVMF_OPT_DHCHAP_SECRET,       "dhchap_secret=%s"      },
 650         { NVMF_OPT_DHCHAP_CTRL_SECRET,  "dhchap_ctrl_secret=%s" },
 651 #endif
 652         { NVMF_OPT_ERR,                 NULL                    }
 653 };
 654
 655 static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
 656                 const char *buf)
 657 {
 658         substring_t args[MAX_OPT_ARGS];
 659         char *options, *o, *p;
 660         int token, ret = 0;
 661         size_t nqnlen  = 0;
 662         int ctrl_loss_tmo = NVMF_DEF_CTRL_LOSS_TMO;
 663         uuid_t hostid;
 664         char hostnqn[NVMF_NQN_SIZE];
 665
 666         /* Set defaults */
 667         opts->queue_size = NVMF_DEF_QUEUE_SIZE;
 668         opts->nr_io_queues = num_online_cpus();
 669         opts->reconnect_delay = NVMF_DEF_RECONNECT_DELAY;
 670         opts->kato = 0;
 671         opts->duplicate_connect = false;
 672         opts->fast_io_fail_tmo = NVMF_DEF_FAIL_FAST_TMO;
 673         opts->hdr_digest = false;
 674         opts->data_digest = false;
 675         opts->tos = -1; /* < 0 == use transport default */
 676
 677         options = o = kstrdup(buf, GFP_KERNEL);
 678         if (!options)
 679                 return -ENOMEM;
 680
 681         /* use default host if not given by user space */
 682         uuid_copy(&hostid, &nvmf_default_host->id);
 683         strscpy(hostnqn, nvmf_default_host->nqn, NVMF_NQN_SIZE);
 684
 685         while ((p = strsep(&o, ",\n")) != NULL) {
 686                 if (!*p)
 687                         continue;
 688
 689                 token = match_token(p, opt_tokens, args);
 690                 opts->mask |= token;
 691                 switch (token) {
 692                 case NVMF_OPT_TRANSPORT:
 693                         p = match_strdup(args);
 694                         if (!p) {
 695                                 ret = -ENOMEM;
 696                                 goto out;
 697                         }
 698                         kfree(opts->transport);
 699                         opts->transport = p;
 700                         break;
 701                 case NVMF_OPT_NQN:
 702                         p = match_strdup(args);
 703                         if (!p) {
 704                                 ret = -ENOMEM;
 705                                 goto out;
 706                         }
 707                         kfree(opts->subsysnqn);
 708                         opts->subsysnqn = p;
 709                         nqnlen = strlen(opts->subsysnqn);
 710                         if (nqnlen >= NVMF_NQN_SIZE) {
 711                                 pr_err("%s needs to be < %d bytes\n",
 712                                         opts->subsysnqn, NVMF_NQN_SIZE);
 713                                 ret = -EINVAL;
 714                                 goto out;
 715                         }
 716                         opts->discovery_nqn =
 717                                 !(strcmp(opts->subsysnqn,
 718                                          NVME_DISC_SUBSYS_NAME));
 719                         break;
 720                 case NVMF_OPT_TRADDR:
 721                         p = match_strdup(args);
 722                         if (!p) {
 723                                 ret = -ENOMEM;
 724                                 goto out;
 725                         }
 726                         kfree(opts->traddr);
 727                         opts->traddr = p;
 728                         break;
 729                 case NVMF_OPT_TRSVCID:
 730                         p = match_strdup(args);
 731                         if (!p) {
 732                                 ret = -ENOMEM;
 733                                 goto out;
 734                         }
 735                         kfree(opts->trsvcid);
 736                         opts->trsvcid = p;
 737                         break;
 738                 case NVMF_OPT_QUEUE_SIZE:
 739                         if (match_int(args, &token)) {
 740                                 ret = -EINVAL;
 741                                 goto out;
 742                         }
 743                         if (token < NVMF_MIN_QUEUE_SIZE ||
 744                             token > NVMF_MAX_QUEUE_SIZE) {
 745                                 pr_err("Invalid queue_size %d\n", token);
 746                                 ret = -EINVAL;
 747                                 goto out;
 748                         }
 749                         opts->queue_size = token;
 750                         break;
 751                 case NVMF_OPT_NR_IO_QUEUES:
 752                         if (match_int(args, &token)) {
 753                                 ret = -EINVAL;
 754                                 goto out;
 755                         }
 756                         if (token <= 0) {
 757                                 pr_err("Invalid number of IOQs %d\n", token);
 758                                 ret = -EINVAL;
 759                                 goto out;
 760                         }
 761                         if (opts->discovery_nqn) {
 762                                 pr_debug("Ignoring nr_io_queues value for discovery controller\n");
 763                                 break;
 764                         }
 765
 766                         opts->nr_io_queues = min_t(unsigned int,
 767                                         num_online_cpus(), token);
 768                         break;
 769                 case NVMF_OPT_KATO:
 770                         if (match_int(args, &token)) {
 771                                 ret = -EINVAL;
 772                                 goto out;
 773                         }
 774
 775                         if (token < 0) {
 776                                 pr_err("Invalid keep_alive_tmo %d\n", token);
 777                                 ret = -EINVAL;
 778                                 goto out;
 779                         } else if (token == 0 && !opts->discovery_nqn) {
 780                                 /* Allowed for debug */
 781                                 pr_warn("keep_alive_tmo 0 won't execute keep alives!!!\n");
 782                         }
 783                         opts->kato = token;
 784                         break;
 785                 case NVMF_OPT_CTRL_LOSS_TMO:
 786                         if (match_int(args, &token)) {
 787                                 ret = -EINVAL;
 788                                 goto out;
 789                         }
 790
 791                         if (token < 0)
 792                                 pr_warn("ctrl_loss_tmo < 0 will reconnect forever\n");
 793                         ctrl_loss_tmo = token;
 794                         break;
 795                 case NVMF_OPT_FAIL_FAST_TMO:
 796                         if (match_int(args, &token)) {
 797                                 ret = -EINVAL;
 798                                 goto out;
 799                         }
 800
 801                         if (token >= 0)
 802                                 pr_warn("I/O fail on reconnect controller after %d sec\n",
 803                                         token);
 804                         else
 805                                 token = -1;
 806
 807                         opts->fast_io_fail_tmo = token;
 808                         break;
 809                 case NVMF_OPT_HOSTNQN:
 810                         if (opts->host) {
 811                                 pr_err("hostnqn already user-assigned: %s\n",
 812                                        opts->host->nqn);
 813                                 ret = -EADDRINUSE;
 814                                 goto out;
 815                         }
 816                         p = match_strdup(args);
 817                         if (!p) {
 818                                 ret = -ENOMEM;
 819                                 goto out;
 820                         }
 821                         nqnlen = strlen(p);
 822                         if (nqnlen >= NVMF_NQN_SIZE) {
 823                                 pr_err("%s needs to be < %d bytes\n",
 824                                         p, NVMF_NQN_SIZE);
 825                                 kfree(p);
 826                                 ret = -EINVAL;
 827                                 goto out;
 828                         }
 829                         strscpy(hostnqn, p, NVMF_NQN_SIZE);
 830                         kfree(p);
 831                         break;
 832                 case NVMF_OPT_RECONNECT_DELAY:
 833                         if (match_int(args, &token)) {
 834                                 ret = -EINVAL;
 835                                 goto out;
 836                         }
 837                         if (token <= 0) {
 838                                 pr_err("Invalid reconnect_delay %d\n", token);
 839                                 ret = -EINVAL;
 840                                 goto out;
 841                         }
 842                         opts->reconnect_delay = token;
 843                         break;
 844                 case NVMF_OPT_HOST_TRADDR:
 845                         p = match_strdup(args);
 846                         if (!p) {
 847                                 ret = -ENOMEM;
 848                                 goto out;
 849                         }
 850                         kfree(opts->host_traddr);
 851                         opts->host_traddr = p;
 852                         break;
 853                 case NVMF_OPT_HOST_IFACE:
 854                         p = match_strdup(args);
 855                         if (!p) {
 856                                 ret = -ENOMEM;
 857                                 goto out;
 858                         }
 859                         kfree(opts->host_iface);
 860                         opts->host_iface = p;
 861                         break;
 862                 case NVMF_OPT_HOST_ID:
 863                         p = match_strdup(args);
 864                         if (!p) {
 865                                 ret = -ENOMEM;
 866                                 goto out;
 867                         }
 868                         ret = uuid_parse(p, &hostid);
 869                         if (ret) {
 870                                 pr_err("Invalid hostid %s\n", p);
 871                                 ret = -EINVAL;
 872                                 kfree(p);
 873                                 goto out;
 874                         }
 875                         kfree(p);
 876                         break;
 877                 case NVMF_OPT_DUP_CONNECT:
 878                         opts->duplicate_connect = true;
 879                         break;
 880                 case NVMF_OPT_DISABLE_SQFLOW:
 881                         opts->disable_sqflow = true;
 882                         break;
 883                 case NVMF_OPT_HDR_DIGEST:
 884                         opts->hdr_digest = true;
 885                         break;
 886                 case NVMF_OPT_DATA_DIGEST:
 887                         opts->data_digest = true;
 888                         break;
 889                 case NVMF_OPT_NR_WRITE_QUEUES:
 890                         if (match_int(args, &token)) {
 891                                 ret = -EINVAL;
 892                                 goto out;
 893                         }
 894                         if (token <= 0) {
 895                                 pr_err("Invalid nr_write_queues %d\n", token);
 896                                 ret = -EINVAL;
 897                                 goto out;
 898                         }
 899                         opts->nr_write_queues = token;
 900                         break;
 901                 case NVMF_OPT_NR_POLL_QUEUES:
 902                         if (match_int(args, &token)) {
 903                                 ret = -EINVAL;
 904                                 goto out;
 905                         }
 906                         if (token <= 0) {
 907                                 pr_err("Invalid nr_poll_queues %d\n", token);
 908                                 ret = -EINVAL;
 909                                 goto out;
 910                         }
 911                         opts->nr_poll_queues = token;
 912                         break;
 913                 case NVMF_OPT_TOS:
 914                         if (match_int(args, &token)) {
 915                                 ret = -EINVAL;
 916                                 goto out;
 917                         }
 918                         if (token < 0) {
 919                                 pr_err("Invalid type of service %d\n", token);
 920                                 ret = -EINVAL;
 921                                 goto out;
 922                         }
 923                         if (token > 255) {
 924                                 pr_warn("Clamping type of service to 255\n");
 925                                 token = 255;
 926                         }
 927                         opts->tos = token;
 928                         break;
 929                 case NVMF_OPT_DISCOVERY:
 930                         opts->discovery_nqn = true;
 931                         break;
 932                 case NVMF_OPT_DHCHAP_SECRET:
 933                         p = match_strdup(args);
 934                         if (!p) {
 935                                 ret = -ENOMEM;
 936                                 goto out;
 937                         }
 938                         if (strlen(p) < 11 || strncmp(p, "DHHC-1:", 7)) {
 939                                 pr_err("Invalid DH-CHAP secret %s\n", p);
 940                                 ret = -EINVAL;
 941                                 goto out;
 942                         }
 943                         kfree(opts->dhchap_secret);
 944                         opts->dhchap_secret = p;
 945                         break;
 946                 case NVMF_OPT_DHCHAP_CTRL_SECRET:
 947                         p = match_strdup(args);
 948                         if (!p) {
 949                                 ret = -ENOMEM;
 950                                 goto out;
 951                         }
 952                         if (strlen(p) < 11 || strncmp(p, "DHHC-1:", 7)) {
 953                                 pr_err("Invalid DH-CHAP secret %s\n", p);
 954                                 ret = -EINVAL;
 955                                 goto out;
 956                         }
 957                         kfree(opts->dhchap_ctrl_secret);
 958                         opts->dhchap_ctrl_secret = p;
 959                         break;
 960                 default:
 961                         pr_warn("unknown parameter or missing value '%s' in ctrl creation request\n",
 962                                 p);
 963                         ret = -EINVAL;
 964                         goto out;
 965                 }
 966         }
 967
 968         if (opts->discovery_nqn) {
 969                 opts->nr_io_queues = 0;
 970                 opts->nr_write_queues = 0;
 971                 opts->nr_poll_queues = 0;
 972                 opts->duplicate_connect = true;
 973         } else {
 974                 if (!opts->kato)
 975                         opts->kato = NVME_DEFAULT_KATO;
 976         }
 977         if (ctrl_loss_tmo < 0) {
 978                 opts->max_reconnects = -1;
 979         } else {
 980                 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
 981                                                 opts->reconnect_delay);
 982                 if (ctrl_loss_tmo < opts->fast_io_fail_tmo)
 983                         pr_warn("failfast tmo (%d) larger than controller loss tmo (%d)\n",
 984                                 opts->fast_io_fail_tmo, ctrl_loss_tmo);
 985         }
 986
 987         opts->host = nvmf_host_add(hostnqn, &hostid);
 988         if (IS_ERR(opts->host)) {
 989                 ret = PTR_ERR(opts->host);
 990                 opts->host = NULL;
 991                 goto out;
 992         }
 993
 994 out:
 995         kfree(options);
 996         return ret;
 997 }
 998
 999 void nvmf_set_io_queues(struct nvmf_ctrl_options *opts, u32 nr_io_queues,
1000                         u32 io_queues[HCTX_MAX_TYPES])
1001 {
1002         if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) {
1003                 /*
1004                  * separate read/write queues
1005                  * hand out dedicated default queues only after we have
1006                  * sufficient read queues.
1007                  */
1008                 io_queues[HCTX_TYPE_READ] = opts->nr_io_queues;
1009                 nr_io_queues -= io_queues[HCTX_TYPE_READ];
1010                 io_queues[HCTX_TYPE_DEFAULT] =
1011                         min(opts->nr_write_queues, nr_io_queues);
1012                 nr_io_queues -= io_queues[HCTX_TYPE_DEFAULT];
1013         } else {
1014                 /*
1015                  * shared read/write queues
1016                  * either no write queues were requested, or we don't have
1017                  * sufficient queue count to have dedicated default queues.
1018                  */
1019                 io_queues[HCTX_TYPE_DEFAULT] =
1020                         min(opts->nr_io_queues, nr_io_queues);
1021                 nr_io_queues -= io_queues[HCTX_TYPE_DEFAULT];
1022         }
1023
1024         if (opts->nr_poll_queues && nr_io_queues) {
1025                 /* map dedicated poll queues only if we have queues left */
1026                 io_queues[HCTX_TYPE_POLL] =
1027                         min(opts->nr_poll_queues, nr_io_queues);
1028         }
1029 }
1030 EXPORT_SYMBOL_GPL(nvmf_set_io_queues);
1031
1032 void nvmf_map_queues(struct blk_mq_tag_set *set, struct nvme_ctrl *ctrl,
1033                      u32 io_queues[HCTX_MAX_TYPES])
1034 {
1035         struct nvmf_ctrl_options *opts = ctrl->opts;
1036
1037         if (opts->nr_write_queues && io_queues[HCTX_TYPE_READ]) {
1038                 /* separate read/write queues */
1039                 set->map[HCTX_TYPE_DEFAULT].nr_queues =
1040                         io_queues[HCTX_TYPE_DEFAULT];
1041                 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
1042                 set->map[HCTX_TYPE_READ].nr_queues =
1043                         io_queues[HCTX_TYPE_READ];
1044                 set->map[HCTX_TYPE_READ].queue_offset =
1045                         io_queues[HCTX_TYPE_DEFAULT];
1046         } else {
1047                 /* shared read/write queues */
1048                 set->map[HCTX_TYPE_DEFAULT].nr_queues =
1049                         io_queues[HCTX_TYPE_DEFAULT];
1050                 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
1051                 set->map[HCTX_TYPE_READ].nr_queues =
1052                         io_queues[HCTX_TYPE_DEFAULT];
1053                 set->map[HCTX_TYPE_READ].queue_offset = 0;
1054         }
1055
1056         blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
1057         blk_mq_map_queues(&set->map[HCTX_TYPE_READ]);
1058         if (opts->nr_poll_queues && io_queues[HCTX_TYPE_POLL]) {
1059                 /* map dedicated poll queues only if we have queues left */
1060                 set->map[HCTX_TYPE_POLL].nr_queues = io_queues[HCTX_TYPE_POLL];
1061                 set->map[HCTX_TYPE_POLL].queue_offset =
1062                         io_queues[HCTX_TYPE_DEFAULT] +
1063                         io_queues[HCTX_TYPE_READ];
1064                 blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
1065         }
1066
1067         dev_info(ctrl->device,
1068                 "mapped %d/%d/%d default/read/poll queues.\n",
1069                 io_queues[HCTX_TYPE_DEFAULT],
1070                 io_queues[HCTX_TYPE_READ],
1071                 io_queues[HCTX_TYPE_POLL]);
1072 }
1073 EXPORT_SYMBOL_GPL(nvmf_map_queues);
1074
1075 static int nvmf_check_required_opts(struct nvmf_ctrl_options *opts,
1076                 unsigned int required_opts)
1077 {
1078         if ((opts->mask & required_opts) != required_opts) {
1079                 unsigned int i;
1080
1081                 for (i = 0; i < ARRAY_SIZE(opt_tokens); i++) {
1082                         if ((opt_tokens[i].token & required_opts) &&
1083                             !(opt_tokens[i].token & opts->mask)) {
1084                                 pr_warn("missing parameter '%s'\n",
1085                                         opt_tokens[i].pattern);
1086                         }
1087                 }
1088
1089                 return -EINVAL;
1090         }
1091
1092         return 0;
1093 }
1094
1095 bool nvmf_ip_options_match(struct nvme_ctrl *ctrl,
1096                 struct nvmf_ctrl_options *opts)
1097 {
1098         if (!nvmf_ctlr_matches_baseopts(ctrl, opts) ||
1099             strcmp(opts->traddr, ctrl->opts->traddr) ||
1100             strcmp(opts->trsvcid, ctrl->opts->trsvcid))
1101                 return false;
1102
1103         /*
1104          * Checking the local address or host interfaces is rough.
1105          *
1106          * In most cases, none is specified and the host port or
1107          * host interface is selected by the stack.
1108          *
1109          * Assume no match if:
1110          * -  local address or host interface is specified and address
1111          *    or host interface is not the same
1112          * -  local address or host interface is not specified but
1113          *    remote is, or vice versa (admin using specific
1114          *    host_traddr/host_iface when it matters).
1115          */
1116         if ((opts->mask & NVMF_OPT_HOST_TRADDR) &&
1117             (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)) {
1118                 if (strcmp(opts->host_traddr, ctrl->opts->host_traddr))
1119                         return false;
1120         } else if ((opts->mask & NVMF_OPT_HOST_TRADDR) ||
1121                    (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)) {
1122                 return false;
1123         }
1124
1125         if ((opts->mask & NVMF_OPT_HOST_IFACE) &&
1126             (ctrl->opts->mask & NVMF_OPT_HOST_IFACE)) {
1127                 if (strcmp(opts->host_iface, ctrl->opts->host_iface))
1128                         return false;
1129         } else if ((opts->mask & NVMF_OPT_HOST_IFACE) ||
1130                    (ctrl->opts->mask & NVMF_OPT_HOST_IFACE)) {
1131                 return false;
1132         }
1133
1134         return true;
1135 }
1136 EXPORT_SYMBOL_GPL(nvmf_ip_options_match);
1137
1138 static int nvmf_check_allowed_opts(struct nvmf_ctrl_options *opts,
1139                 unsigned int allowed_opts)
1140 {
1141         if (opts->mask & ~allowed_opts) {
1142                 unsigned int i;
1143
1144                 for (i = 0; i < ARRAY_SIZE(opt_tokens); i++) {
1145                         if ((opt_tokens[i].token & opts->mask) &&
1146                             (opt_tokens[i].token & ~allowed_opts)) {
1147                                 pr_warn("invalid parameter '%s'\n",
1148                                         opt_tokens[i].pattern);
1149                         }
1150                 }
1151
1152                 return -EINVAL;
1153         }
1154
1155         return 0;
1156 }
1157
1158 void nvmf_free_options(struct nvmf_ctrl_options *opts)
1159 {
1160         nvmf_host_put(opts->host);
1161         kfree(opts->transport);
1162         kfree(opts->traddr);
1163         kfree(opts->trsvcid);
1164         kfree(opts->subsysnqn);
1165         kfree(opts->host_traddr);
1166         kfree(opts->host_iface);
1167         kfree(opts->dhchap_secret);
1168         kfree(opts->dhchap_ctrl_secret);
1169         kfree(opts);
1170 }
1171 EXPORT_SYMBOL_GPL(nvmf_free_options);
1172
1173 #define NVMF_REQUIRED_OPTS      (NVMF_OPT_TRANSPORT | NVMF_OPT_NQN)
1174 #define NVMF_ALLOWED_OPTS       (NVMF_OPT_QUEUE_SIZE | NVMF_OPT_NR_IO_QUEUES | \
1175                                  NVMF_OPT_KATO | NVMF_OPT_HOSTNQN | \
1176                                  NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT |\
1177                                  NVMF_OPT_DISABLE_SQFLOW | NVMF_OPT_DISCOVERY |\
1178                                  NVMF_OPT_FAIL_FAST_TMO | NVMF_OPT_DHCHAP_SECRET |\
1179                                  NVMF_OPT_DHCHAP_CTRL_SECRET)
1180
1181 static struct nvme_ctrl *
1182 nvmf_create_ctrl(struct device *dev, const char *buf)
1183 {
1184         struct nvmf_ctrl_options *opts;
1185         struct nvmf_transport_ops *ops;
1186         struct nvme_ctrl *ctrl;
1187         int ret;
1188
1189         opts = kzalloc(sizeof(*opts), GFP_KERNEL);
1190         if (!opts)
1191                 return ERR_PTR(-ENOMEM);
1192
1193         ret = nvmf_parse_options(opts, buf);
1194         if (ret)
1195                 goto out_free_opts;
1196
1197
1198         request_module("nvme-%s", opts->transport);
1199
1200         /*
1201          * Check the generic options first as we need a valid transport for
1202          * the lookup below.  Then clear the generic flags so that transport
1203          * drivers don't have to care about them.
1204          */
1205         ret = nvmf_check_required_opts(opts, NVMF_REQUIRED_OPTS);
1206         if (ret)
1207                 goto out_free_opts;
1208         opts->mask &= ~NVMF_REQUIRED_OPTS;
1209
1210         down_read(&nvmf_transports_rwsem);
1211         ops = nvmf_lookup_transport(opts);
1212         if (!ops) {
1213                 pr_info("no handler found for transport %s.\n",
1214                         opts->transport);
1215                 ret = -EINVAL;
1216                 goto out_unlock;
1217         }
1218
1219         if (!try_module_get(ops->module)) {
1220                 ret = -EBUSY;
1221                 goto out_unlock;
1222         }
1223         up_read(&nvmf_transports_rwsem);
1224
1225         ret = nvmf_check_required_opts(opts, ops->required_opts);
1226         if (ret)
1227                 goto out_module_put;
1228         ret = nvmf_check_allowed_opts(opts, NVMF_ALLOWED_OPTS |
1229                                 ops->allowed_opts | ops->required_opts);
1230         if (ret)
1231                 goto out_module_put;
1232
1233         ctrl = ops->create_ctrl(dev, opts);
1234         if (IS_ERR(ctrl)) {
1235                 ret = PTR_ERR(ctrl);
1236                 goto out_module_put;
1237         }
1238
1239         module_put(ops->module);
1240         return ctrl;
1241
1242 out_module_put:
1243         module_put(ops->module);
1244         goto out_free_opts;
1245 out_unlock:
1246         up_read(&nvmf_transports_rwsem);
1247 out_free_opts:
1248         nvmf_free_options(opts);
1249         return ERR_PTR(ret);
1250 }
1251
1252 static struct class *nvmf_class;
1253 static struct device *nvmf_device;
1254 static DEFINE_MUTEX(nvmf_dev_mutex);
1255
1256 static ssize_t nvmf_dev_write(struct file *file, const char __user *ubuf,
1257                 size_t count, loff_t *pos)
1258 {
1259         struct seq_file *seq_file = file->private_data;
1260         struct nvme_ctrl *ctrl;
1261         const char *buf;
1262         int ret = 0;
1263
1264         if (count > PAGE_SIZE)
1265                 return -ENOMEM;
1266
1267         buf = memdup_user_nul(ubuf, count);
1268         if (IS_ERR(buf))
1269                 return PTR_ERR(buf);
1270
1271         mutex_lock(&nvmf_dev_mutex);
1272         if (seq_file->private) {
1273                 ret = -EINVAL;
1274                 goto out_unlock;
1275         }
1276
1277         ctrl = nvmf_create_ctrl(nvmf_device, buf);
1278         if (IS_ERR(ctrl)) {
1279                 ret = PTR_ERR(ctrl);
1280                 goto out_unlock;
1281         }
1282
1283         seq_file->private = ctrl;
1284
1285 out_unlock:
1286         mutex_unlock(&nvmf_dev_mutex);
1287         kfree(buf);
1288         return ret ? ret : count;
1289 }
1290
1291 static void __nvmf_concat_opt_tokens(struct seq_file *seq_file)
1292 {
1293         const struct match_token *tok;
1294         int idx;
1295
1296         /*
1297          * Add dummy entries for instance and cntlid to
1298          * signal an invalid/non-existing controller
1299          */
1300         seq_puts(seq_file, "instance=-1,cntlid=-1");
1301         for (idx = 0; idx < ARRAY_SIZE(opt_tokens); idx++) {
1302                 tok = &opt_tokens[idx];
1303                 if (tok->token == NVMF_OPT_ERR)
1304                         continue;
1305                 seq_puts(seq_file, ",");
1306                 seq_puts(seq_file, tok->pattern);
1307         }
1308         seq_puts(seq_file, "\n");
1309 }
1310
1311 static int nvmf_dev_show(struct seq_file *seq_file, void *private)
1312 {
1313         struct nvme_ctrl *ctrl;
1314
1315         mutex_lock(&nvmf_dev_mutex);
1316         ctrl = seq_file->private;
1317         if (!ctrl) {
1318                 __nvmf_concat_opt_tokens(seq_file);
1319                 goto out_unlock;
1320         }
1321
1322         seq_printf(seq_file, "instance=%d,cntlid=%d\n",
1323                         ctrl->instance, ctrl->cntlid);
1324
1325 out_unlock:
1326         mutex_unlock(&nvmf_dev_mutex);
1327         return 0;
1328 }
1329
1330 static int nvmf_dev_open(struct inode *inode, struct file *file)
1331 {
1332         /*
1333          * The miscdevice code initializes file->private_data, but doesn't
1334          * make use of it later.
1335          */
1336         file->private_data = NULL;
1337         return single_open(file, nvmf_dev_show, NULL);
1338 }
1339
1340 static int nvmf_dev_release(struct inode *inode, struct file *file)
1341 {
1342         struct seq_file *seq_file = file->private_data;
1343         struct nvme_ctrl *ctrl = seq_file->private;
1344
1345         if (ctrl)
1346                 nvme_put_ctrl(ctrl);
1347         return single_release(inode, file);
1348 }
1349
1350 static const struct file_operations nvmf_dev_fops = {
1351         .owner          = THIS_MODULE,
1352         .write          = nvmf_dev_write,
1353         .read           = seq_read,
1354         .open           = nvmf_dev_open,
1355         .release        = nvmf_dev_release,
1356 };
1357
1358 static struct miscdevice nvmf_misc = {
1359         .minor          = MISC_DYNAMIC_MINOR,
1360         .name           = "nvme-fabrics",
1361         .fops           = &nvmf_dev_fops,
1362 };
1363
1364 static int __init nvmf_init(void)
1365 {
1366         int ret;
1367
1368         nvmf_default_host = nvmf_host_default();
1369         if (!nvmf_default_host)
1370                 return -ENOMEM;
1371
1372         nvmf_class = class_create("nvme-fabrics");
1373         if (IS_ERR(nvmf_class)) {
1374                 pr_err("couldn't register class nvme-fabrics\n");
1375                 ret = PTR_ERR(nvmf_class);
1376                 goto out_free_host;
1377         }
1378
1379         nvmf_device =
1380                 device_create(nvmf_class, NULL, MKDEV(0, 0), NULL, "ctl");
1381         if (IS_ERR(nvmf_device)) {
1382                 pr_err("couldn't create nvme-fabrics device!\n");
1383                 ret = PTR_ERR(nvmf_device);
1384                 goto out_destroy_class;
1385         }
1386
1387         ret = misc_register(&nvmf_misc);
1388         if (ret) {
1389                 pr_err("couldn't register misc device: %d\n", ret);
1390                 goto out_destroy_device;
1391         }
1392
1393         return 0;
1394
1395 out_destroy_device:
1396         device_destroy(nvmf_class, MKDEV(0, 0));
1397 out_destroy_class:
1398         class_destroy(nvmf_class);
1399 out_free_host:
1400         nvmf_host_put(nvmf_default_host);
1401         return ret;
1402 }
1403
1404 static void __exit nvmf_exit(void)
1405 {
1406         misc_deregister(&nvmf_misc);
1407         device_destroy(nvmf_class, MKDEV(0, 0));
1408         class_destroy(nvmf_class);
1409         nvmf_host_put(nvmf_default_host);
1410
1411         BUILD_BUG_ON(sizeof(struct nvmf_common_command) != 64);
1412         BUILD_BUG_ON(sizeof(struct nvmf_connect_command) != 64);
1413         BUILD_BUG_ON(sizeof(struct nvmf_property_get_command) != 64);
1414         BUILD_BUG_ON(sizeof(struct nvmf_property_set_command) != 64);
1415         BUILD_BUG_ON(sizeof(struct nvmf_auth_send_command) != 64);
1416         BUILD_BUG_ON(sizeof(struct nvmf_auth_receive_command) != 64);
1417         BUILD_BUG_ON(sizeof(struct nvmf_connect_data) != 1024);
1418         BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_negotiate_data) != 8);
1419         BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_challenge_data) != 16);
1420         BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_reply_data) != 16);
1421         BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_success1_data) != 16);
1422         BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_success2_data) != 16);
1423 }
1424
1425 MODULE_LICENSE("GPL v2");
1426
1427 module_init(nvmf_init);
1428 module_exit(nvmf_exit);