drivers/cxl/core/memdev.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2020 Intel Corporation. */
   3
   4 #include <linux/io-64-nonatomic-lo-hi.h>
   5 #include <linux/firmware.h>
   6 #include <linux/device.h>
   7 #include <linux/slab.h>
   8 #include <linux/idr.h>
   9 #include <linux/pci.h>
  10 #include <cxlmem.h>
  11 #include "trace.h"
  12 #include "core.h"
  13
  14 static DECLARE_RWSEM(cxl_memdev_rwsem);
  15
  16 /*
  17  * An entire PCI topology full of devices should be enough for any
  18  * config
  19  */
  20 #define CXL_MEM_MAX_DEVS 65536
  21
  22 static int cxl_mem_major;
  23 static DEFINE_IDA(cxl_memdev_ida);
  24
  25 static void cxl_memdev_release(struct device *dev)
  26 {
  27         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  28
  29         ida_free(&cxl_memdev_ida, cxlmd->id);
  30         kfree(cxlmd);
  31 }
  32
  33 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid,
  34                                 kgid_t *gid)
  35 {
  36         return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev));
  37 }
  38
  39 static ssize_t firmware_version_show(struct device *dev,
  40                                      struct device_attribute *attr, char *buf)
  41 {
  42         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  43         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  44         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
  45
  46         if (!mds)
  47                 return sysfs_emit(buf, "\n");
  48         return sysfs_emit(buf, "%.16s\n", mds->firmware_version);
  49 }
  50 static DEVICE_ATTR_RO(firmware_version);
  51
  52 static ssize_t payload_max_show(struct device *dev,
  53                                 struct device_attribute *attr, char *buf)
  54 {
  55         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  56         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  57         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
  58
  59         if (!mds)
  60                 return sysfs_emit(buf, "\n");
  61         return sysfs_emit(buf, "%zu\n", mds->payload_size);
  62 }
  63 static DEVICE_ATTR_RO(payload_max);
  64
  65 static ssize_t label_storage_size_show(struct device *dev,
  66                                        struct device_attribute *attr, char *buf)
  67 {
  68         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  69         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  70         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
  71
  72         if (!mds)
  73                 return sysfs_emit(buf, "\n");
  74         return sysfs_emit(buf, "%zu\n", mds->lsa_size);
  75 }
  76 static DEVICE_ATTR_RO(label_storage_size);
  77
  78 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr,
  79                              char *buf)
  80 {
  81         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  82         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  83         unsigned long long len = resource_size(&cxlds->ram_res);
  84
  85         return sysfs_emit(buf, "%#llx\n", len);
  86 }
  87
  88 static struct device_attribute dev_attr_ram_size =
  89         __ATTR(size, 0444, ram_size_show, NULL);
  90
  91 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr,
  92                               char *buf)
  93 {
  94         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
  95         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  96         unsigned long long len = resource_size(&cxlds->pmem_res);
  97
  98         return sysfs_emit(buf, "%#llx\n", len);
  99 }
 100
 101 static struct device_attribute dev_attr_pmem_size =
 102         __ATTR(size, 0444, pmem_size_show, NULL);
 103
 104 static ssize_t serial_show(struct device *dev, struct device_attribute *attr,
 105                            char *buf)
 106 {
 107         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 108         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 109
 110         return sysfs_emit(buf, "%#llx\n", cxlds->serial);
 111 }
 112 static DEVICE_ATTR_RO(serial);
 113
 114 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr,
 115                               char *buf)
 116 {
 117         return sprintf(buf, "%d\n", dev_to_node(dev));
 118 }
 119 static DEVICE_ATTR_RO(numa_node);
 120
 121 static ssize_t security_state_show(struct device *dev,
 122                                    struct device_attribute *attr,
 123                                    char *buf)
 124 {
 125         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 126         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 127         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
 128         u64 reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET);
 129         u32 pct = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK, reg);
 130         u16 cmd = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK, reg);
 131         unsigned long state = mds->security.state;
 132
 133         if (cmd == CXL_MBOX_OP_SANITIZE && pct != 100)
 134                 return sysfs_emit(buf, "sanitize\n");
 135
 136         if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET))
 137                 return sysfs_emit(buf, "disabled\n");
 138         if (state & CXL_PMEM_SEC_STATE_FROZEN ||
 139             state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT ||
 140             state & CXL_PMEM_SEC_STATE_USER_PLIMIT)
 141                 return sysfs_emit(buf, "frozen\n");
 142         if (state & CXL_PMEM_SEC_STATE_LOCKED)
 143                 return sysfs_emit(buf, "locked\n");
 144         else
 145                 return sysfs_emit(buf, "unlocked\n");
 146 }
 147 static struct device_attribute dev_attr_security_state =
 148         __ATTR(state, 0444, security_state_show, NULL);
 149
 150 static ssize_t security_sanitize_store(struct device *dev,
 151                                        struct device_attribute *attr,
 152                                        const char *buf, size_t len)
 153 {
 154         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 155         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 156         struct cxl_port *port = cxlmd->endpoint;
 157         bool sanitize;
 158         ssize_t rc;
 159
 160         if (kstrtobool(buf, &sanitize) || !sanitize)
 161                 return -EINVAL;
 162
 163         if (!port || !is_cxl_endpoint(port))
 164                 return -EINVAL;
 165
 166         /* ensure no regions are mapped to this memdev */
 167         if (port->commit_end != -1)
 168                 return -EBUSY;
 169
 170         rc = cxl_mem_sanitize(mds, CXL_MBOX_OP_SANITIZE);
 171
 172         return rc ? rc : len;
 173 }
 174 static struct device_attribute dev_attr_security_sanitize =
 175         __ATTR(sanitize, 0200, NULL, security_sanitize_store);
 176
 177 static ssize_t security_erase_store(struct device *dev,
 178                                     struct device_attribute *attr,
 179                                     const char *buf, size_t len)
 180 {
 181         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 182         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 183         struct cxl_port *port = cxlmd->endpoint;
 184         ssize_t rc;
 185         bool erase;
 186
 187         if (kstrtobool(buf, &erase) || !erase)
 188                 return -EINVAL;
 189
 190         if (!port || !is_cxl_endpoint(port))
 191                 return -EINVAL;
 192
 193         /* ensure no regions are mapped to this memdev */
 194         if (port->commit_end != -1)
 195                 return -EBUSY;
 196
 197         rc = cxl_mem_sanitize(mds, CXL_MBOX_OP_SECURE_ERASE);
 198
 199         return rc ? rc : len;
 200 }
 201 static struct device_attribute dev_attr_security_erase =
 202         __ATTR(erase, 0200, NULL, security_erase_store);
 203
 204 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd)
 205 {
 206         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 207         u64 offset, length;
 208         int rc = 0;
 209
 210         /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */
 211         if (resource_size(&cxlds->pmem_res)) {
 212                 offset = cxlds->pmem_res.start;
 213                 length = resource_size(&cxlds->pmem_res);
 214                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
 215                 if (rc)
 216                         return rc;
 217         }
 218         if (resource_size(&cxlds->ram_res)) {
 219                 offset = cxlds->ram_res.start;
 220                 length = resource_size(&cxlds->ram_res);
 221                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
 222                 /*
 223                  * Invalid Physical Address is not an error for
 224                  * volatile addresses. Device support is optional.
 225                  */
 226                 if (rc == -EFAULT)
 227                         rc = 0;
 228         }
 229         return rc;
 230 }
 231
 232 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)
 233 {
 234         struct cxl_port *port;
 235         int rc;
 236
 237         port = cxlmd->endpoint;
 238         if (!port || !is_cxl_endpoint(port))
 239                 return -EINVAL;
 240
 241         rc = down_read_interruptible(&cxl_dpa_rwsem);
 242         if (rc)
 243                 return rc;
 244
 245         if (port->commit_end == -1) {
 246                 /* No regions mapped to this memdev */
 247                 rc = cxl_get_poison_by_memdev(cxlmd);
 248         } else {
 249                 /* Regions mapped, collect poison by endpoint */
 250                 rc =  cxl_get_poison_by_endpoint(port);
 251         }
 252         up_read(&cxl_dpa_rwsem);
 253
 254         return rc;
 255 }
 256 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, CXL);
 257
 258 struct cxl_dpa_to_region_context {
 259         struct cxl_region *cxlr;
 260         u64 dpa;
 261 };
 262
 263 static int __cxl_dpa_to_region(struct device *dev, void *arg)
 264 {
 265         struct cxl_dpa_to_region_context *ctx = arg;
 266         struct cxl_endpoint_decoder *cxled;
 267         u64 dpa = ctx->dpa;
 268
 269         if (!is_endpoint_decoder(dev))
 270                 return 0;
 271
 272         cxled = to_cxl_endpoint_decoder(dev);
 273         if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
 274                 return 0;
 275
 276         if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start)
 277                 return 0;
 278
 279         dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa,
 280                 dev_name(&cxled->cxld.region->dev));
 281
 282         ctx->cxlr = cxled->cxld.region;
 283
 284         return 1;
 285 }
 286
 287 static struct cxl_region *cxl_dpa_to_region(struct cxl_memdev *cxlmd, u64 dpa)
 288 {
 289         struct cxl_dpa_to_region_context ctx;
 290         struct cxl_port *port;
 291
 292         ctx = (struct cxl_dpa_to_region_context) {
 293                 .dpa = dpa,
 294         };
 295         port = cxlmd->endpoint;
 296         if (port && is_cxl_endpoint(port) && port->commit_end != -1)
 297                 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region);
 298
 299         return ctx.cxlr;
 300 }
 301
 302 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa)
 303 {
 304         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 305
 306         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 307                 return 0;
 308
 309         if (!resource_size(&cxlds->dpa_res)) {
 310                 dev_dbg(cxlds->dev, "device has no dpa resource\n");
 311                 return -EINVAL;
 312         }
 313         if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) {
 314                 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n",
 315                         dpa, &cxlds->dpa_res);
 316                 return -EINVAL;
 317         }
 318         if (!IS_ALIGNED(dpa, 64)) {
 319                 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa);
 320                 return -EINVAL;
 321         }
 322
 323         return 0;
 324 }
 325
 326 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
 327 {
 328         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 329         struct cxl_mbox_inject_poison inject;
 330         struct cxl_poison_record record;
 331         struct cxl_mbox_cmd mbox_cmd;
 332         struct cxl_region *cxlr;
 333         int rc;
 334
 335         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 336                 return 0;
 337
 338         rc = down_read_interruptible(&cxl_dpa_rwsem);
 339         if (rc)
 340                 return rc;
 341
 342         rc = cxl_validate_poison_dpa(cxlmd, dpa);
 343         if (rc)
 344                 goto out;
 345
 346         inject.address = cpu_to_le64(dpa);
 347         mbox_cmd = (struct cxl_mbox_cmd) {
 348                 .opcode = CXL_MBOX_OP_INJECT_POISON,
 349                 .size_in = sizeof(inject),
 350                 .payload_in = &inject,
 351         };
 352         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 353         if (rc)
 354                 goto out;
 355
 356         cxlr = cxl_dpa_to_region(cxlmd, dpa);
 357         if (cxlr)
 358                 dev_warn_once(mds->cxlds.dev,
 359                               "poison inject dpa:%#llx region: %s\n", dpa,
 360                               dev_name(&cxlr->dev));
 361
 362         record = (struct cxl_poison_record) {
 363                 .address = cpu_to_le64(dpa),
 364                 .length = cpu_to_le32(1),
 365         };
 366         trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT);
 367 out:
 368         up_read(&cxl_dpa_rwsem);
 369
 370         return rc;
 371 }
 372 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL);
 373
 374 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
 375 {
 376         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 377         struct cxl_mbox_clear_poison clear;
 378         struct cxl_poison_record record;
 379         struct cxl_mbox_cmd mbox_cmd;
 380         struct cxl_region *cxlr;
 381         int rc;
 382
 383         if (!IS_ENABLED(CONFIG_DEBUG_FS))
 384                 return 0;
 385
 386         rc = down_read_interruptible(&cxl_dpa_rwsem);
 387         if (rc)
 388                 return rc;
 389
 390         rc = cxl_validate_poison_dpa(cxlmd, dpa);
 391         if (rc)
 392                 goto out;
 393
 394         /*
 395          * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command
 396          * is defined to accept 64 bytes of write-data, along with the
 397          * address to clear. This driver uses zeroes as write-data.
 398          */
 399         clear = (struct cxl_mbox_clear_poison) {
 400                 .address = cpu_to_le64(dpa)
 401         };
 402
 403         mbox_cmd = (struct cxl_mbox_cmd) {
 404                 .opcode = CXL_MBOX_OP_CLEAR_POISON,
 405                 .size_in = sizeof(clear),
 406                 .payload_in = &clear,
 407         };
 408
 409         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 410         if (rc)
 411                 goto out;
 412
 413         cxlr = cxl_dpa_to_region(cxlmd, dpa);
 414         if (cxlr)
 415                 dev_warn_once(mds->cxlds.dev,
 416                               "poison clear dpa:%#llx region: %s\n", dpa,
 417                               dev_name(&cxlr->dev));
 418
 419         record = (struct cxl_poison_record) {
 420                 .address = cpu_to_le64(dpa),
 421                 .length = cpu_to_le32(1),
 422         };
 423         trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR);
 424 out:
 425         up_read(&cxl_dpa_rwsem);
 426
 427         return rc;
 428 }
 429 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, CXL);
 430
 431 static struct attribute *cxl_memdev_attributes[] = {
 432         &dev_attr_serial.attr,
 433         &dev_attr_firmware_version.attr,
 434         &dev_attr_payload_max.attr,
 435         &dev_attr_label_storage_size.attr,
 436         &dev_attr_numa_node.attr,
 437         NULL,
 438 };
 439
 440 static struct attribute *cxl_memdev_pmem_attributes[] = {
 441         &dev_attr_pmem_size.attr,
 442         NULL,
 443 };
 444
 445 static struct attribute *cxl_memdev_ram_attributes[] = {
 446         &dev_attr_ram_size.attr,
 447         NULL,
 448 };
 449
 450 static struct attribute *cxl_memdev_security_attributes[] = {
 451         &dev_attr_security_state.attr,
 452         &dev_attr_security_sanitize.attr,
 453         &dev_attr_security_erase.attr,
 454         NULL,
 455 };
 456
 457 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a,
 458                                   int n)
 459 {
 460         if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr)
 461                 return 0;
 462         return a->mode;
 463 }
 464
 465 static struct attribute_group cxl_memdev_attribute_group = {
 466         .attrs = cxl_memdev_attributes,
 467         .is_visible = cxl_memdev_visible,
 468 };
 469
 470 static struct attribute_group cxl_memdev_ram_attribute_group = {
 471         .name = "ram",
 472         .attrs = cxl_memdev_ram_attributes,
 473 };
 474
 475 static struct attribute_group cxl_memdev_pmem_attribute_group = {
 476         .name = "pmem",
 477         .attrs = cxl_memdev_pmem_attributes,
 478 };
 479
 480 static umode_t cxl_memdev_security_visible(struct kobject *kobj,
 481                                            struct attribute *a, int n)
 482 {
 483         struct device *dev = kobj_to_dev(kobj);
 484         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 485         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 486
 487         if (a == &dev_attr_security_sanitize.attr &&
 488             !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
 489                 return 0;
 490
 491         if (a == &dev_attr_security_erase.attr &&
 492             !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds))
 493                 return 0;
 494
 495         return a->mode;
 496 }
 497
 498 static struct attribute_group cxl_memdev_security_attribute_group = {
 499         .name = "security",
 500         .attrs = cxl_memdev_security_attributes,
 501         .is_visible = cxl_memdev_security_visible,
 502 };
 503
 504 static const struct attribute_group *cxl_memdev_attribute_groups[] = {
 505         &cxl_memdev_attribute_group,
 506         &cxl_memdev_ram_attribute_group,
 507         &cxl_memdev_pmem_attribute_group,
 508         &cxl_memdev_security_attribute_group,
 509         NULL,
 510 };
 511
 512 static const struct device_type cxl_memdev_type = {
 513         .name = "cxl_memdev",
 514         .release = cxl_memdev_release,
 515         .devnode = cxl_memdev_devnode,
 516         .groups = cxl_memdev_attribute_groups,
 517 };
 518
 519 bool is_cxl_memdev(const struct device *dev)
 520 {
 521         return dev->type == &cxl_memdev_type;
 522 }
 523 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL);
 524
 525 /**
 526  * set_exclusive_cxl_commands() - atomically disable user cxl commands
 527  * @mds: The device state to operate on
 528  * @cmds: bitmap of commands to mark exclusive
 529  *
 530  * Grab the cxl_memdev_rwsem in write mode to flush in-flight
 531  * invocations of the ioctl path and then disable future execution of
 532  * commands with the command ids set in @cmds.
 533  */
 534 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
 535                                 unsigned long *cmds)
 536 {
 537         down_write(&cxl_memdev_rwsem);
 538         bitmap_or(mds->exclusive_cmds, mds->exclusive_cmds, cmds,
 539                   CXL_MEM_COMMAND_ID_MAX);
 540         up_write(&cxl_memdev_rwsem);
 541 }
 542 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL);
 543
 544 /**
 545  * clear_exclusive_cxl_commands() - atomically enable user cxl commands
 546  * @mds: The device state to modify
 547  * @cmds: bitmap of commands to mark available for userspace
 548  */
 549 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,
 550                                   unsigned long *cmds)
 551 {
 552         down_write(&cxl_memdev_rwsem);
 553         bitmap_andnot(mds->exclusive_cmds, mds->exclusive_cmds, cmds,
 554                       CXL_MEM_COMMAND_ID_MAX);
 555         up_write(&cxl_memdev_rwsem);
 556 }
 557 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL);
 558
 559 static void cxl_memdev_security_shutdown(struct device *dev)
 560 {
 561         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 562         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 563
 564         cancel_delayed_work_sync(&mds->security.poll_dwork);
 565 }
 566
 567 static void cxl_memdev_shutdown(struct device *dev)
 568 {
 569         struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
 570
 571         down_write(&cxl_memdev_rwsem);
 572         cxl_memdev_security_shutdown(dev);
 573         cxlmd->cxlds = NULL;
 574         up_write(&cxl_memdev_rwsem);
 575 }
 576
 577 static void cxl_memdev_unregister(void *_cxlmd)
 578 {
 579         struct cxl_memdev *cxlmd = _cxlmd;
 580         struct device *dev = &cxlmd->dev;
 581
 582         cxl_memdev_shutdown(dev);
 583         cdev_device_del(&cxlmd->cdev, dev);
 584         put_device(dev);
 585 }
 586
 587 static void detach_memdev(struct work_struct *work)
 588 {
 589         struct cxl_memdev *cxlmd;
 590
 591         cxlmd = container_of(work, typeof(*cxlmd), detach_work);
 592         device_release_driver(&cxlmd->dev);
 593         put_device(&cxlmd->dev);
 594 }
 595
 596 static struct lock_class_key cxl_memdev_key;
 597
 598 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
 599                                            const struct file_operations *fops)
 600 {
 601         struct cxl_memdev *cxlmd;
 602         struct device *dev;
 603         struct cdev *cdev;
 604         int rc;
 605
 606         cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL);
 607         if (!cxlmd)
 608                 return ERR_PTR(-ENOMEM);
 609
 610         rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL);
 611         if (rc < 0)
 612                 goto err;
 613         cxlmd->id = rc;
 614         cxlmd->depth = -1;
 615
 616         dev = &cxlmd->dev;
 617         device_initialize(dev);
 618         lockdep_set_class(&dev->mutex, &cxl_memdev_key);
 619         dev->parent = cxlds->dev;
 620         dev->bus = &cxl_bus_type;
 621         dev->devt = MKDEV(cxl_mem_major, cxlmd->id);
 622         dev->type = &cxl_memdev_type;
 623         device_set_pm_not_required(dev);
 624         INIT_WORK(&cxlmd->detach_work, detach_memdev);
 625
 626         cdev = &cxlmd->cdev;
 627         cdev_init(cdev, fops);
 628         return cxlmd;
 629
 630 err:
 631         kfree(cxlmd);
 632         return ERR_PTR(rc);
 633 }
 634
 635 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
 636                                unsigned long arg)
 637 {
 638         switch (cmd) {
 639         case CXL_MEM_QUERY_COMMANDS:
 640                 return cxl_query_cmd(cxlmd, (void __user *)arg);
 641         case CXL_MEM_SEND_COMMAND:
 642                 return cxl_send_cmd(cxlmd, (void __user *)arg);
 643         default:
 644                 return -ENOTTY;
 645         }
 646 }
 647
 648 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,
 649                              unsigned long arg)
 650 {
 651         struct cxl_memdev *cxlmd = file->private_data;
 652         struct cxl_dev_state *cxlds;
 653         int rc = -ENXIO;
 654
 655         down_read(&cxl_memdev_rwsem);
 656         cxlds = cxlmd->cxlds;
 657         if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM)
 658                 rc = __cxl_memdev_ioctl(cxlmd, cmd, arg);
 659         up_read(&cxl_memdev_rwsem);
 660
 661         return rc;
 662 }
 663
 664 static int cxl_memdev_open(struct inode *inode, struct file *file)
 665 {
 666         struct cxl_memdev *cxlmd =
 667                 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
 668
 669         get_device(&cxlmd->dev);
 670         file->private_data = cxlmd;
 671
 672         return 0;
 673 }
 674
 675 static int cxl_memdev_release_file(struct inode *inode, struct file *file)
 676 {
 677         struct cxl_memdev *cxlmd =
 678                 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
 679
 680         put_device(&cxlmd->dev);
 681
 682         return 0;
 683 }
 684
 685 /**
 686  * cxl_mem_get_fw_info - Get Firmware info
 687  * @mds: The device data for the operation
 688  *
 689  * Retrieve firmware info for the device specified.
 690  *
 691  * Return: 0 if no error: or the result of the mailbox command.
 692  *
 693  * See CXL-3.0 8.2.9.3.1 Get FW Info
 694  */
 695 static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)
 696 {
 697         struct cxl_mbox_get_fw_info info;
 698         struct cxl_mbox_cmd mbox_cmd;
 699         int rc;
 700
 701         mbox_cmd = (struct cxl_mbox_cmd) {
 702                 .opcode = CXL_MBOX_OP_GET_FW_INFO,
 703                 .size_out = sizeof(info),
 704                 .payload_out = &info,
 705         };
 706
 707         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 708         if (rc < 0)
 709                 return rc;
 710
 711         mds->fw.num_slots = info.num_slots;
 712         mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK,
 713                                        info.slot_info);
 714
 715         return 0;
 716 }
 717
 718 /**
 719  * cxl_mem_activate_fw - Activate Firmware
 720  * @mds: The device data for the operation
 721  * @slot: slot number to activate
 722  *
 723  * Activate firmware in a given slot for the device specified.
 724  *
 725  * Return: 0 if no error: or the result of the mailbox command.
 726  *
 727  * See CXL-3.0 8.2.9.3.3 Activate FW
 728  */
 729 static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)
 730 {
 731         struct cxl_mbox_activate_fw activate;
 732         struct cxl_mbox_cmd mbox_cmd;
 733
 734         if (slot == 0 || slot > mds->fw.num_slots)
 735                 return -EINVAL;
 736
 737         mbox_cmd = (struct cxl_mbox_cmd) {
 738                 .opcode = CXL_MBOX_OP_ACTIVATE_FW,
 739                 .size_in = sizeof(activate),
 740                 .payload_in = &activate,
 741         };
 742
 743         /* Only offline activation supported for now */
 744         activate.action = CXL_FW_ACTIVATE_OFFLINE;
 745         activate.slot = slot;
 746
 747         return cxl_internal_send_cmd(mds, &mbox_cmd);
 748 }
 749
 750 /**
 751  * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer
 752  * @mds: The device data for the operation
 753  *
 754  * Abort an in-progress firmware transfer for the device specified.
 755  *
 756  * Return: 0 if no error: or the result of the mailbox command.
 757  *
 758  * See CXL-3.0 8.2.9.3.2 Transfer FW
 759  */
 760 static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds)
 761 {
 762         struct cxl_mbox_transfer_fw *transfer;
 763         struct cxl_mbox_cmd mbox_cmd;
 764         int rc;
 765
 766         transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL);
 767         if (!transfer)
 768                 return -ENOMEM;
 769
 770         /* Set a 1s poll interval and a total wait time of 30s */
 771         mbox_cmd = (struct cxl_mbox_cmd) {
 772                 .opcode = CXL_MBOX_OP_TRANSFER_FW,
 773                 .size_in = sizeof(*transfer),
 774                 .payload_in = transfer,
 775                 .poll_interval_ms = 1000,
 776                 .poll_count = 30,
 777         };
 778
 779         transfer->action = CXL_FW_TRANSFER_ACTION_ABORT;
 780
 781         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 782         kfree(transfer);
 783         return rc;
 784 }
 785
 786 static void cxl_fw_cleanup(struct fw_upload *fwl)
 787 {
 788         struct cxl_memdev_state *mds = fwl->dd_handle;
 789
 790         mds->fw.next_slot = 0;
 791 }
 792
 793 static int cxl_fw_do_cancel(struct fw_upload *fwl)
 794 {
 795         struct cxl_memdev_state *mds = fwl->dd_handle;
 796         struct cxl_dev_state *cxlds = &mds->cxlds;
 797         struct cxl_memdev *cxlmd = cxlds->cxlmd;
 798         int rc;
 799
 800         rc = cxl_mem_abort_fw_xfer(mds);
 801         if (rc < 0)
 802                 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc);
 803
 804         return FW_UPLOAD_ERR_CANCELED;
 805 }
 806
 807 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,
 808                                          u32 size)
 809 {
 810         struct cxl_memdev_state *mds = fwl->dd_handle;
 811         struct cxl_mbox_transfer_fw *transfer;
 812
 813         if (!size)
 814                 return FW_UPLOAD_ERR_INVALID_SIZE;
 815
 816         mds->fw.oneshot = struct_size(transfer, data, size) <
 817                             mds->payload_size;
 818
 819         if (cxl_mem_get_fw_info(mds))
 820                 return FW_UPLOAD_ERR_HW_ERROR;
 821
 822         /*
 823          * So far no state has been changed, hence no other cleanup is
 824          * necessary. Simply return the cancelled status.
 825          */
 826         if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
 827                 return FW_UPLOAD_ERR_CANCELED;
 828
 829         return FW_UPLOAD_ERR_NONE;
 830 }
 831
 832 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
 833                                        u32 offset, u32 size, u32 *written)
 834 {
 835         struct cxl_memdev_state *mds = fwl->dd_handle;
 836         struct cxl_dev_state *cxlds = &mds->cxlds;
 837         struct cxl_memdev *cxlmd = cxlds->cxlmd;
 838         struct cxl_mbox_transfer_fw *transfer;
 839         struct cxl_mbox_cmd mbox_cmd;
 840         u32 cur_size, remaining;
 841         size_t size_in;
 842         int rc;
 843
 844         *written = 0;
 845
 846         /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */
 847         if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) {
 848                 dev_err(&cxlmd->dev,
 849                         "misaligned offset for FW transfer slice (%u)\n",
 850                         offset);
 851                 return FW_UPLOAD_ERR_RW_ERROR;
 852         }
 853
 854         /*
 855          * Pick transfer size based on mds->payload_size @size must bw 128-byte
 856          * aligned, ->payload_size is a power of 2 starting at 256 bytes, and
 857          * sizeof(*transfer) is 128.  These constraints imply that @cur_size
 858          * will always be 128b aligned.
 859          */
 860         cur_size = min_t(size_t, size, mds->payload_size - sizeof(*transfer));
 861
 862         remaining = size - cur_size;
 863         size_in = struct_size(transfer, data, cur_size);
 864
 865         if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
 866                 return cxl_fw_do_cancel(fwl);
 867
 868         /*
 869          * Slot numbers are 1-indexed
 870          * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1')
 871          * Check for rollover using modulo, and 1-index it by adding 1
 872          */
 873         mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1;
 874
 875         /* Do the transfer via mailbox cmd */
 876         transfer = kzalloc(size_in, GFP_KERNEL);
 877         if (!transfer)
 878                 return FW_UPLOAD_ERR_RW_ERROR;
 879
 880         transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT);
 881         memcpy(transfer->data, data + offset, cur_size);
 882         if (mds->fw.oneshot) {
 883                 transfer->action = CXL_FW_TRANSFER_ACTION_FULL;
 884                 transfer->slot = mds->fw.next_slot;
 885         } else {
 886                 if (offset == 0) {
 887                         transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE;
 888                 } else if (remaining == 0) {
 889                         transfer->action = CXL_FW_TRANSFER_ACTION_END;
 890                         transfer->slot = mds->fw.next_slot;
 891                 } else {
 892                         transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE;
 893                 }
 894         }
 895
 896         mbox_cmd = (struct cxl_mbox_cmd) {
 897                 .opcode = CXL_MBOX_OP_TRANSFER_FW,
 898                 .size_in = size_in,
 899                 .payload_in = transfer,
 900                 .poll_interval_ms = 1000,
 901                 .poll_count = 30,
 902         };
 903
 904         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 905         if (rc < 0) {
 906                 rc = FW_UPLOAD_ERR_RW_ERROR;
 907                 goto out_free;
 908         }
 909
 910         *written = cur_size;
 911
 912         /* Activate FW if oneshot or if the last slice was written */
 913         if (mds->fw.oneshot || remaining == 0) {
 914                 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n",
 915                         mds->fw.next_slot);
 916                 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot);
 917                 if (rc < 0) {
 918                         dev_err(&cxlmd->dev, "Error activating firmware: %d\n",
 919                                 rc);
 920                         rc = FW_UPLOAD_ERR_HW_ERROR;
 921                         goto out_free;
 922                 }
 923         }
 924
 925         rc = FW_UPLOAD_ERR_NONE;
 926
 927 out_free:
 928         kfree(transfer);
 929         return rc;
 930 }
 931
 932 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl)
 933 {
 934         struct cxl_memdev_state *mds = fwl->dd_handle;
 935
 936         /*
 937          * cxl_internal_send_cmd() handles background operations synchronously.
 938          * No need to wait for completions here - any errors would've been
 939          * reported and handled during the ->write() call(s).
 940          * Just check if a cancel request was received, and return success.
 941          */
 942         if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
 943                 return cxl_fw_do_cancel(fwl);
 944
 945         return FW_UPLOAD_ERR_NONE;
 946 }
 947
 948 static void cxl_fw_cancel(struct fw_upload *fwl)
 949 {
 950         struct cxl_memdev_state *mds = fwl->dd_handle;
 951
 952         set_bit(CXL_FW_CANCEL, mds->fw.state);
 953 }
 954
 955 static const struct fw_upload_ops cxl_memdev_fw_ops = {
 956         .prepare = cxl_fw_prepare,
 957         .write = cxl_fw_write,
 958         .poll_complete = cxl_fw_poll_complete,
 959         .cancel = cxl_fw_cancel,
 960         .cleanup = cxl_fw_cleanup,
 961 };
 962
 963 static void devm_cxl_remove_fw_upload(void *fwl)
 964 {
 965         firmware_upload_unregister(fwl);
 966 }
 967
 968 int cxl_memdev_setup_fw_upload(struct cxl_memdev_state *mds)
 969 {
 970         struct cxl_dev_state *cxlds = &mds->cxlds;
 971         struct device *dev = &cxlds->cxlmd->dev;
 972         struct fw_upload *fwl;
 973         int rc;
 974
 975         if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, mds->enabled_cmds))
 976                 return 0;
 977
 978         fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev),
 979                                        &cxl_memdev_fw_ops, mds);
 980         if (IS_ERR(fwl))
 981                 return dev_err_probe(dev, PTR_ERR(fwl),
 982                                      "Failed to register firmware loader\n");
 983
 984         rc = devm_add_action_or_reset(cxlds->dev, devm_cxl_remove_fw_upload,
 985                                       fwl);
 986         if (rc)
 987                 dev_err(dev,
 988                         "Failed to add firmware loader remove action: %d\n",
 989                         rc);
 990
 991         return rc;
 992 }
 993 EXPORT_SYMBOL_NS_GPL(cxl_memdev_setup_fw_upload, CXL);
 994
 995 static const struct file_operations cxl_memdev_fops = {
 996         .owner = THIS_MODULE,
 997         .unlocked_ioctl = cxl_memdev_ioctl,
 998         .open = cxl_memdev_open,
 999         .release = cxl_memdev_release_file,
1000         .compat_ioctl = compat_ptr_ioctl,
1001         .llseek = noop_llseek,
1002 };
1003
1004 static void put_sanitize(void *data)
1005 {
1006         struct cxl_memdev_state *mds = data;
1007
1008         sysfs_put(mds->security.sanitize_node);
1009 }
1010
1011 static int cxl_memdev_security_init(struct cxl_memdev *cxlmd)
1012 {
1013         struct cxl_dev_state *cxlds = cxlmd->cxlds;
1014         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
1015         struct device *dev = &cxlmd->dev;
1016         struct kernfs_node *sec;
1017
1018         sec = sysfs_get_dirent(dev->kobj.sd, "security");
1019         if (!sec) {
1020                 dev_err(dev, "sysfs_get_dirent 'security' failed\n");
1021                 return -ENODEV;
1022         }
1023         mds->security.sanitize_node = sysfs_get_dirent(sec, "state");
1024         sysfs_put(sec);
1025         if (!mds->security.sanitize_node) {
1026                 dev_err(dev, "sysfs_get_dirent 'state' failed\n");
1027                 return -ENODEV;
1028         }
1029
1030         return devm_add_action_or_reset(cxlds->dev, put_sanitize, mds);
1031  }
1032
1033 struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds)
1034 {
1035         struct cxl_memdev *cxlmd;
1036         struct device *dev;
1037         struct cdev *cdev;
1038         int rc;
1039
1040         cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops);
1041         if (IS_ERR(cxlmd))
1042                 return cxlmd;
1043
1044         dev = &cxlmd->dev;
1045         rc = dev_set_name(dev, "mem%d", cxlmd->id);
1046         if (rc)
1047                 goto err;
1048
1049         /*
1050          * Activate ioctl operations, no cxl_memdev_rwsem manipulation
1051          * needed as this is ordered with cdev_add() publishing the device.
1052          */
1053         cxlmd->cxlds = cxlds;
1054         cxlds->cxlmd = cxlmd;
1055
1056         cdev = &cxlmd->cdev;
1057         rc = cdev_device_add(cdev, dev);
1058         if (rc)
1059                 goto err;
1060
1061         rc = cxl_memdev_security_init(cxlmd);
1062         if (rc)
1063                 goto err;
1064
1065         rc = devm_add_action_or_reset(cxlds->dev, cxl_memdev_unregister, cxlmd);
1066         if (rc)
1067                 return ERR_PTR(rc);
1068         return cxlmd;
1069
1070 err:
1071         /*
1072          * The cdev was briefly live, shutdown any ioctl operations that
1073          * saw that state.
1074          */
1075         cxl_memdev_shutdown(dev);
1076         put_device(dev);
1077         return ERR_PTR(rc);
1078 }
1079 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
1080
1081 __init int cxl_memdev_init(void)
1082 {
1083         dev_t devt;
1084         int rc;
1085
1086         rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl");
1087         if (rc)
1088                 return rc;
1089
1090         cxl_mem_major = MAJOR(devt);
1091
1092         return 0;
1093 }
1094
1095 void cxl_memdev_exit(void)
1096 {
1097         unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS);
1098 }