drivers/cxl/pmem.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2021 Intel Corporation. All rights reserved. */
   3 #include <linux/libnvdimm.h>
   4 #include <asm/unaligned.h>
   5 #include <linux/device.h>
   6 #include <linux/module.h>
   7 #include <linux/ndctl.h>
   8 #include <linux/async.h>
   9 #include <linux/slab.h>
  10 #include <linux/nd.h>
  11 #include "cxlmem.h"
  12 #include "cxl.h"
  13
  14 /*
  15  * Ordered workqueue for cxl nvdimm device arrival and departure
  16  * to coordinate bus rescans when a bridge arrives and trigger remove
  17  * operations when the bridge is removed.
  18  */
  19 static struct workqueue_struct *cxl_pmem_wq;
  20
  21 static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
  22
  23 static void clear_exclusive(void *cxlds)
  24 {
  25         clear_exclusive_cxl_commands(cxlds, exclusive_cmds);
  26 }
  27
  28 static void unregister_nvdimm(void *nvdimm)
  29 {
  30         struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
  31         struct cxl_nvdimm_bridge *cxl_nvb = cxl_nvd->bridge;
  32         struct cxl_pmem_region *cxlr_pmem;
  33         unsigned long index;
  34
  35         device_lock(&cxl_nvb->dev);
  36         dev_set_drvdata(&cxl_nvd->dev, NULL);
  37         xa_for_each(&cxl_nvd->pmem_regions, index, cxlr_pmem) {
  38                 get_device(&cxlr_pmem->dev);
  39                 device_unlock(&cxl_nvb->dev);
  40
  41                 device_release_driver(&cxlr_pmem->dev);
  42                 put_device(&cxlr_pmem->dev);
  43
  44                 device_lock(&cxl_nvb->dev);
  45         }
  46         device_unlock(&cxl_nvb->dev);
  47
  48         nvdimm_delete(nvdimm);
  49         cxl_nvd->bridge = NULL;
  50 }
  51
  52 static int cxl_nvdimm_probe(struct device *dev)
  53 {
  54         struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
  55         struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
  56         unsigned long flags = 0, cmd_mask = 0;
  57         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  58         struct cxl_nvdimm_bridge *cxl_nvb;
  59         struct nvdimm *nvdimm;
  60         int rc;
  61
  62         cxl_nvb = cxl_find_nvdimm_bridge(dev);
  63         if (!cxl_nvb)
  64                 return -ENXIO;
  65
  66         device_lock(&cxl_nvb->dev);
  67         if (!cxl_nvb->nvdimm_bus) {
  68                 rc = -ENXIO;
  69                 goto out;
  70         }
  71
  72         set_exclusive_cxl_commands(cxlds, exclusive_cmds);
  73         rc = devm_add_action_or_reset(dev, clear_exclusive, cxlds);
  74         if (rc)
  75                 goto out;
  76
  77         set_bit(NDD_LABELING, &flags);
  78         set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
  79         set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
  80         set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
  81         nvdimm = nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd, NULL, flags,
  82                                cmd_mask, 0, NULL);
  83         if (!nvdimm) {
  84                 rc = -ENOMEM;
  85                 goto out;
  86         }
  87
  88         dev_set_drvdata(dev, nvdimm);
  89         cxl_nvd->bridge = cxl_nvb;
  90         rc = devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
  91 out:
  92         device_unlock(&cxl_nvb->dev);
  93         put_device(&cxl_nvb->dev);
  94
  95         return rc;
  96 }
  97
  98 static struct cxl_driver cxl_nvdimm_driver = {
  99         .name = "cxl_nvdimm",
 100         .probe = cxl_nvdimm_probe,
 101         .id = CXL_DEVICE_NVDIMM,
 102 };
 103
 104 static int cxl_pmem_get_config_size(struct cxl_dev_state *cxlds,
 105                                     struct nd_cmd_get_config_size *cmd,
 106                                     unsigned int buf_len)
 107 {
 108         if (sizeof(*cmd) > buf_len)
 109                 return -EINVAL;
 110
 111         *cmd = (struct nd_cmd_get_config_size) {
 112                  .config_size = cxlds->lsa_size,
 113                  .max_xfer = cxlds->payload_size - sizeof(struct cxl_mbox_set_lsa),
 114         };
 115
 116         return 0;
 117 }
 118
 119 static int cxl_pmem_get_config_data(struct cxl_dev_state *cxlds,
 120                                     struct nd_cmd_get_config_data_hdr *cmd,
 121                                     unsigned int buf_len)
 122 {
 123         struct cxl_mbox_get_lsa get_lsa;
 124         int rc;
 125
 126         if (sizeof(*cmd) > buf_len)
 127                 return -EINVAL;
 128         if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
 129                 return -EINVAL;
 130
 131         get_lsa = (struct cxl_mbox_get_lsa) {
 132                 .offset = cpu_to_le32(cmd->in_offset),
 133                 .length = cpu_to_le32(cmd->in_length),
 134         };
 135
 136         rc = cxl_mbox_send_cmd(cxlds, CXL_MBOX_OP_GET_LSA, &get_lsa,
 137                                sizeof(get_lsa), cmd->out_buf, cmd->in_length);
 138         cmd->status = 0;
 139
 140         return rc;
 141 }
 142
 143 static int cxl_pmem_set_config_data(struct cxl_dev_state *cxlds,
 144                                     struct nd_cmd_set_config_hdr *cmd,
 145                                     unsigned int buf_len)
 146 {
 147         struct cxl_mbox_set_lsa *set_lsa;
 148         int rc;
 149
 150         if (sizeof(*cmd) > buf_len)
 151                 return -EINVAL;
 152
 153         /* 4-byte status follows the input data in the payload */
 154         if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len)
 155                 return -EINVAL;
 156
 157         set_lsa =
 158                 kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);
 159         if (!set_lsa)
 160                 return -ENOMEM;
 161
 162         *set_lsa = (struct cxl_mbox_set_lsa) {
 163                 .offset = cpu_to_le32(cmd->in_offset),
 164         };
 165         memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
 166
 167         rc = cxl_mbox_send_cmd(cxlds, CXL_MBOX_OP_SET_LSA, set_lsa,
 168                                struct_size(set_lsa, data, cmd->in_length),
 169                                NULL, 0);
 170
 171         /*
 172          * Set "firmware" status (4-packed bytes at the end of the input
 173          * payload.
 174          */
 175         put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
 176         kvfree(set_lsa);
 177
 178         return rc;
 179 }
 180
 181 static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
 182                                void *buf, unsigned int buf_len)
 183 {
 184         struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
 185         unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
 186         struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
 187         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 188
 189         if (!test_bit(cmd, &cmd_mask))
 190                 return -ENOTTY;
 191
 192         switch (cmd) {
 193         case ND_CMD_GET_CONFIG_SIZE:
 194                 return cxl_pmem_get_config_size(cxlds, buf, buf_len);
 195         case ND_CMD_GET_CONFIG_DATA:
 196                 return cxl_pmem_get_config_data(cxlds, buf, buf_len);
 197         case ND_CMD_SET_CONFIG_DATA:
 198                 return cxl_pmem_set_config_data(cxlds, buf, buf_len);
 199         default:
 200                 return -ENOTTY;
 201         }
 202 }
 203
 204 static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
 205                         struct nvdimm *nvdimm, unsigned int cmd, void *buf,
 206                         unsigned int buf_len, int *cmd_rc)
 207 {
 208         /*
 209          * No firmware response to translate, let the transport error
 210          * code take precedence.
 211          */
 212         *cmd_rc = 0;
 213
 214         if (!nvdimm)
 215                 return -ENOTTY;
 216         return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
 217 }
 218
 219 static bool online_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb)
 220 {
 221         if (cxl_nvb->nvdimm_bus)
 222                 return true;
 223         cxl_nvb->nvdimm_bus =
 224                 nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);
 225         return cxl_nvb->nvdimm_bus != NULL;
 226 }
 227
 228 static int cxl_nvdimm_release_driver(struct device *dev, void *cxl_nvb)
 229 {
 230         struct cxl_nvdimm *cxl_nvd;
 231
 232         if (!is_cxl_nvdimm(dev))
 233                 return 0;
 234
 235         cxl_nvd = to_cxl_nvdimm(dev);
 236         if (cxl_nvd->bridge != cxl_nvb)
 237                 return 0;
 238
 239         device_release_driver(dev);
 240         return 0;
 241 }
 242
 243 static int cxl_pmem_region_release_driver(struct device *dev, void *cxl_nvb)
 244 {
 245         struct cxl_pmem_region *cxlr_pmem;
 246
 247         if (!is_cxl_pmem_region(dev))
 248                 return 0;
 249
 250         cxlr_pmem = to_cxl_pmem_region(dev);
 251         if (cxlr_pmem->bridge != cxl_nvb)
 252                 return 0;
 253
 254         device_release_driver(dev);
 255         return 0;
 256 }
 257
 258 static void offline_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb,
 259                                struct nvdimm_bus *nvdimm_bus)
 260 {
 261         if (!nvdimm_bus)
 262                 return;
 263
 264         /*
 265          * Set the state of cxl_nvdimm devices to unbound / idle before
 266          * nvdimm_bus_unregister() rips the nvdimm objects out from
 267          * underneath them.
 268          */
 269         bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb,
 270                          cxl_pmem_region_release_driver);
 271         bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb,
 272                          cxl_nvdimm_release_driver);
 273         nvdimm_bus_unregister(nvdimm_bus);
 274 }
 275
 276 static void cxl_nvb_update_state(struct work_struct *work)
 277 {
 278         struct cxl_nvdimm_bridge *cxl_nvb =
 279                 container_of(work, typeof(*cxl_nvb), state_work);
 280         struct nvdimm_bus *victim_bus = NULL;
 281         bool release = false, rescan = false;
 282
 283         device_lock(&cxl_nvb->dev);
 284         switch (cxl_nvb->state) {
 285         case CXL_NVB_ONLINE:
 286                 if (!online_nvdimm_bus(cxl_nvb)) {
 287                         dev_err(&cxl_nvb->dev,
 288                                 "failed to establish nvdimm bus\n");
 289                         release = true;
 290                 } else
 291                         rescan = true;
 292                 break;
 293         case CXL_NVB_OFFLINE:
 294         case CXL_NVB_DEAD:
 295                 victim_bus = cxl_nvb->nvdimm_bus;
 296                 cxl_nvb->nvdimm_bus = NULL;
 297                 break;
 298         default:
 299                 break;
 300         }
 301         device_unlock(&cxl_nvb->dev);
 302
 303         if (release)
 304                 device_release_driver(&cxl_nvb->dev);
 305         if (rescan) {
 306                 int rc = bus_rescan_devices(&cxl_bus_type);
 307
 308                 dev_dbg(&cxl_nvb->dev, "rescan: %d\n", rc);
 309         }
 310         offline_nvdimm_bus(cxl_nvb, victim_bus);
 311
 312         put_device(&cxl_nvb->dev);
 313 }
 314
 315 static void cxl_nvdimm_bridge_state_work(struct cxl_nvdimm_bridge *cxl_nvb)
 316 {
 317         /*
 318          * Take a reference that the workqueue will drop if new work
 319          * gets queued.
 320          */
 321         get_device(&cxl_nvb->dev);
 322         if (!queue_work(cxl_pmem_wq, &cxl_nvb->state_work))
 323                 put_device(&cxl_nvb->dev);
 324 }
 325
 326 static void cxl_nvdimm_bridge_remove(struct device *dev)
 327 {
 328         struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
 329
 330         if (cxl_nvb->state == CXL_NVB_ONLINE)
 331                 cxl_nvb->state = CXL_NVB_OFFLINE;
 332         cxl_nvdimm_bridge_state_work(cxl_nvb);
 333 }
 334
 335 static int cxl_nvdimm_bridge_probe(struct device *dev)
 336 {
 337         struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
 338
 339         if (cxl_nvb->state == CXL_NVB_DEAD)
 340                 return -ENXIO;
 341
 342         if (cxl_nvb->state == CXL_NVB_NEW) {
 343                 cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {
 344                         .provider_name = "CXL",
 345                         .module = THIS_MODULE,
 346                         .ndctl = cxl_pmem_ctl,
 347                 };
 348
 349                 INIT_WORK(&cxl_nvb->state_work, cxl_nvb_update_state);
 350         }
 351
 352         cxl_nvb->state = CXL_NVB_ONLINE;
 353         cxl_nvdimm_bridge_state_work(cxl_nvb);
 354
 355         return 0;
 356 }
 357
 358 static struct cxl_driver cxl_nvdimm_bridge_driver = {
 359         .name = "cxl_nvdimm_bridge",
 360         .probe = cxl_nvdimm_bridge_probe,
 361         .remove = cxl_nvdimm_bridge_remove,
 362         .id = CXL_DEVICE_NVDIMM_BRIDGE,
 363 };
 364
 365 static int match_cxl_nvdimm(struct device *dev, void *data)
 366 {
 367         return is_cxl_nvdimm(dev);
 368 }
 369
 370 static void unregister_nvdimm_region(void *nd_region)
 371 {
 372         nvdimm_region_delete(nd_region);
 373 }
 374
 375 static int cxl_nvdimm_add_region(struct cxl_nvdimm *cxl_nvd,
 376                                  struct cxl_pmem_region *cxlr_pmem)
 377 {
 378         int rc;
 379
 380         rc = xa_insert(&cxl_nvd->pmem_regions, (unsigned long)cxlr_pmem,
 381                        cxlr_pmem, GFP_KERNEL);
 382         if (rc)
 383                 return rc;
 384
 385         get_device(&cxlr_pmem->dev);
 386         return 0;
 387 }
 388
 389 static void cxl_nvdimm_del_region(struct cxl_nvdimm *cxl_nvd,
 390                                   struct cxl_pmem_region *cxlr_pmem)
 391 {
 392         /*
 393          * It is possible this is called without a corresponding
 394          * cxl_nvdimm_add_region for @cxlr_pmem
 395          */
 396         cxlr_pmem = xa_erase(&cxl_nvd->pmem_regions, (unsigned long)cxlr_pmem);
 397         if (cxlr_pmem)
 398                 put_device(&cxlr_pmem->dev);
 399 }
 400
 401 static void release_mappings(void *data)
 402 {
 403         int i;
 404         struct cxl_pmem_region *cxlr_pmem = data;
 405         struct cxl_nvdimm_bridge *cxl_nvb = cxlr_pmem->bridge;
 406
 407         device_lock(&cxl_nvb->dev);
 408         for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
 409                 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
 410                 struct cxl_nvdimm *cxl_nvd = m->cxl_nvd;
 411
 412                 cxl_nvdimm_del_region(cxl_nvd, cxlr_pmem);
 413         }
 414         device_unlock(&cxl_nvb->dev);
 415 }
 416
 417 static void cxlr_pmem_remove_resource(void *res)
 418 {
 419         remove_resource(res);
 420 }
 421
 422 struct cxl_pmem_region_info {
 423         u64 offset;
 424         u64 serial;
 425 };
 426
 427 static int cxl_pmem_region_probe(struct device *dev)
 428 {
 429         struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE];
 430         struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
 431         struct cxl_region *cxlr = cxlr_pmem->cxlr;
 432         struct cxl_pmem_region_info *info = NULL;
 433         struct cxl_nvdimm_bridge *cxl_nvb;
 434         struct nd_interleave_set *nd_set;
 435         struct nd_region_desc ndr_desc;
 436         struct cxl_nvdimm *cxl_nvd;
 437         struct nvdimm *nvdimm;
 438         struct resource *res;
 439         int rc, i = 0;
 440
 441         cxl_nvb = cxl_find_nvdimm_bridge(&cxlr_pmem->mapping[0].cxlmd->dev);
 442         if (!cxl_nvb) {
 443                 dev_dbg(dev, "bridge not found\n");
 444                 return -ENXIO;
 445         }
 446         cxlr_pmem->bridge = cxl_nvb;
 447
 448         device_lock(&cxl_nvb->dev);
 449         if (!cxl_nvb->nvdimm_bus) {
 450                 dev_dbg(dev, "nvdimm bus not found\n");
 451                 rc = -ENXIO;
 452                 goto out_nvb;
 453         }
 454
 455         memset(&mappings, 0, sizeof(mappings));
 456         memset(&ndr_desc, 0, sizeof(ndr_desc));
 457
 458         res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
 459         if (!res) {
 460                 rc = -ENOMEM;
 461                 goto out_nvb;
 462         }
 463
 464         res->name = "Persistent Memory";
 465         res->start = cxlr_pmem->hpa_range.start;
 466         res->end = cxlr_pmem->hpa_range.end;
 467         res->flags = IORESOURCE_MEM;
 468         res->desc = IORES_DESC_PERSISTENT_MEMORY;
 469
 470         rc = insert_resource(&iomem_resource, res);
 471         if (rc)
 472                 goto out_nvb;
 473
 474         rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res);
 475         if (rc)
 476                 goto out_nvb;
 477
 478         ndr_desc.res = res;
 479         ndr_desc.provider_data = cxlr_pmem;
 480
 481         ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start);
 482         ndr_desc.target_node = phys_to_target_node(res->start);
 483         if (ndr_desc.target_node == NUMA_NO_NODE) {
 484                 ndr_desc.target_node = ndr_desc.numa_node;
 485                 dev_dbg(&cxlr->dev, "changing target node from %d to %d",
 486                         NUMA_NO_NODE, ndr_desc.target_node);
 487         }
 488
 489         nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
 490         if (!nd_set) {
 491                 rc = -ENOMEM;
 492                 goto out_nvb;
 493         }
 494
 495         ndr_desc.memregion = cxlr->id;
 496         set_bit(ND_REGION_CXL, &ndr_desc.flags);
 497         set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags);
 498
 499         info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL);
 500         if (!info) {
 501                 rc = -ENOMEM;
 502                 goto out_nvb;
 503         }
 504
 505         rc = devm_add_action_or_reset(dev, release_mappings, cxlr_pmem);
 506         if (rc)
 507                 goto out_nvd;
 508
 509         for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
 510                 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
 511                 struct cxl_memdev *cxlmd = m->cxlmd;
 512                 struct cxl_dev_state *cxlds = cxlmd->cxlds;
 513                 struct device *d;
 514
 515                 d = device_find_child(&cxlmd->dev, NULL, match_cxl_nvdimm);
 516                 if (!d) {
 517                         dev_dbg(dev, "[%d]: %s: no cxl_nvdimm found\n", i,
 518                                 dev_name(&cxlmd->dev));
 519                         rc = -ENODEV;
 520                         goto out_nvd;
 521                 }
 522
 523                 /* safe to drop ref now with bridge lock held */
 524                 put_device(d);
 525
 526                 cxl_nvd = to_cxl_nvdimm(d);
 527                 nvdimm = dev_get_drvdata(&cxl_nvd->dev);
 528                 if (!nvdimm) {
 529                         dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i,
 530                                 dev_name(&cxlmd->dev));
 531                         rc = -ENODEV;
 532                         goto out_nvd;
 533                 }
 534
 535                 /*
 536                  * Pin the region per nvdimm device as those may be released
 537                  * out-of-order with respect to the region, and a single nvdimm
 538                  * maybe associated with multiple regions
 539                  */
 540                 rc = cxl_nvdimm_add_region(cxl_nvd, cxlr_pmem);
 541                 if (rc)
 542                         goto out_nvd;
 543                 m->cxl_nvd = cxl_nvd;
 544                 mappings[i] = (struct nd_mapping_desc) {
 545                         .nvdimm = nvdimm,
 546                         .start = m->start,
 547                         .size = m->size,
 548                         .position = i,
 549                 };
 550                 info[i].offset = m->start;
 551                 info[i].serial = cxlds->serial;
 552         }
 553         ndr_desc.num_mappings = cxlr_pmem->nr_mappings;
 554         ndr_desc.mapping = mappings;
 555
 556         /*
 557          * TODO enable CXL labels which skip the need for 'interleave-set cookie'
 558          */
 559         nd_set->cookie1 =
 560                 nd_fletcher64(info, sizeof(*info) * cxlr_pmem->nr_mappings, 0);
 561         nd_set->cookie2 = nd_set->cookie1;
 562         ndr_desc.nd_set = nd_set;
 563
 564         cxlr_pmem->nd_region =
 565                 nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc);
 566         if (!cxlr_pmem->nd_region) {
 567                 rc = -ENOMEM;
 568                 goto out_nvd;
 569         }
 570
 571         rc = devm_add_action_or_reset(dev, unregister_nvdimm_region,
 572                                       cxlr_pmem->nd_region);
 573 out_nvd:
 574         kfree(info);
 575 out_nvb:
 576         device_unlock(&cxl_nvb->dev);
 577         put_device(&cxl_nvb->dev);
 578
 579         return rc;
 580 }
 581
 582 static struct cxl_driver cxl_pmem_region_driver = {
 583         .name = "cxl_pmem_region",
 584         .probe = cxl_pmem_region_probe,
 585         .id = CXL_DEVICE_PMEM_REGION,
 586 };
 587
 588 /*
 589  * Return all bridges to the CXL_NVB_NEW state to invalidate any
 590  * ->state_work referring to the now destroyed cxl_pmem_wq.
 591  */
 592 static int cxl_nvdimm_bridge_reset(struct device *dev, void *data)
 593 {
 594         struct cxl_nvdimm_bridge *cxl_nvb;
 595
 596         if (!is_cxl_nvdimm_bridge(dev))
 597                 return 0;
 598
 599         cxl_nvb = to_cxl_nvdimm_bridge(dev);
 600         device_lock(dev);
 601         cxl_nvb->state = CXL_NVB_NEW;
 602         device_unlock(dev);
 603
 604         return 0;
 605 }
 606
 607 static void destroy_cxl_pmem_wq(void)
 608 {
 609         destroy_workqueue(cxl_pmem_wq);
 610         bus_for_each_dev(&cxl_bus_type, NULL, NULL, cxl_nvdimm_bridge_reset);
 611 }
 612
 613 static __init int cxl_pmem_init(void)
 614 {
 615         int rc;
 616
 617         set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
 618         set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);
 619
 620         cxl_pmem_wq = alloc_ordered_workqueue("cxl_pmem", 0);
 621         if (!cxl_pmem_wq)
 622                 return -ENXIO;
 623
 624         rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);
 625         if (rc)
 626                 goto err_bridge;
 627
 628         rc = cxl_driver_register(&cxl_nvdimm_driver);
 629         if (rc)
 630                 goto err_nvdimm;
 631
 632         rc = cxl_driver_register(&cxl_pmem_region_driver);
 633         if (rc)
 634                 goto err_region;
 635
 636         return 0;
 637
 638 err_region:
 639         cxl_driver_unregister(&cxl_nvdimm_driver);
 640 err_nvdimm:
 641         cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
 642 err_bridge:
 643         destroy_cxl_pmem_wq();
 644         return rc;
 645 }
 646
 647 static __exit void cxl_pmem_exit(void)
 648 {
 649         cxl_driver_unregister(&cxl_pmem_region_driver);
 650         cxl_driver_unregister(&cxl_nvdimm_driver);
 651         cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
 652         destroy_cxl_pmem_wq();
 653 }
 654
 655 MODULE_LICENSE("GPL v2");
 656 module_init(cxl_pmem_init);
 657 module_exit(cxl_pmem_exit);
 658 MODULE_IMPORT_NS(CXL);
 659 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
 660 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
 661 MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION);