drivers/cxl/core/port.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
   3 #include <linux/platform_device.h>
   4 #include <linux/memregion.h>
   5 #include <linux/workqueue.h>
   6 #include <linux/debugfs.h>
   7 #include <linux/device.h>
   8 #include <linux/module.h>
   9 #include <linux/pci.h>
  10 #include <linux/slab.h>
  11 #include <linux/idr.h>
  12 #include <cxlmem.h>
  13 #include <cxlpci.h>
  14 #include <cxl.h>
  15 #include "core.h"
  16
  17 /**
  18  * DOC: cxl core
  19  *
  20  * The CXL core provides a set of interfaces that can be consumed by CXL aware
  21  * drivers. The interfaces allow for creation, modification, and destruction of
  22  * regions, memory devices, ports, and decoders. CXL aware drivers must register
  23  * with the CXL core via these interfaces in order to be able to participate in
  24  * cross-device interleave coordination. The CXL core also establishes and
  25  * maintains the bridge to the nvdimm subsystem.
  26  *
  27  * CXL core introduces sysfs hierarchy to control the devices that are
  28  * instantiated by the core.
  29  */
  30
  31 /*
  32  * All changes to the interleave configuration occur with this lock held
  33  * for write.
  34  */
  35 DECLARE_RWSEM(cxl_region_rwsem);
  36
  37 static DEFINE_IDA(cxl_port_ida);
  38 static DEFINE_XARRAY(cxl_root_buses);
  39
  40 int cxl_num_decoders_committed(struct cxl_port *port)
  41 {
  42         lockdep_assert_held(&cxl_region_rwsem);
  43
  44         return port->commit_end + 1;
  45 }
  46
  47 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
  48                             char *buf)
  49 {
  50         return sysfs_emit(buf, "%s\n", dev->type->name);
  51 }
  52 static DEVICE_ATTR_RO(devtype);
  53
  54 static int cxl_device_id(const struct device *dev)
  55 {
  56         if (dev->type == &cxl_nvdimm_bridge_type)
  57                 return CXL_DEVICE_NVDIMM_BRIDGE;
  58         if (dev->type == &cxl_nvdimm_type)
  59                 return CXL_DEVICE_NVDIMM;
  60         if (dev->type == CXL_PMEM_REGION_TYPE())
  61                 return CXL_DEVICE_PMEM_REGION;
  62         if (dev->type == CXL_DAX_REGION_TYPE())
  63                 return CXL_DEVICE_DAX_REGION;
  64         if (is_cxl_port(dev)) {
  65                 if (is_cxl_root(to_cxl_port(dev)))
  66                         return CXL_DEVICE_ROOT;
  67                 return CXL_DEVICE_PORT;
  68         }
  69         if (is_cxl_memdev(dev))
  70                 return CXL_DEVICE_MEMORY_EXPANDER;
  71         if (dev->type == CXL_REGION_TYPE())
  72                 return CXL_DEVICE_REGION;
  73         if (dev->type == &cxl_pmu_type)
  74                 return CXL_DEVICE_PMU;
  75         return 0;
  76 }
  77
  78 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
  79                              char *buf)
  80 {
  81         return sysfs_emit(buf, CXL_MODALIAS_FMT "\n", cxl_device_id(dev));
  82 }
  83 static DEVICE_ATTR_RO(modalias);
  84
  85 static struct attribute *cxl_base_attributes[] = {
  86         &dev_attr_devtype.attr,
  87         &dev_attr_modalias.attr,
  88         NULL,
  89 };
  90
  91 struct attribute_group cxl_base_attribute_group = {
  92         .attrs = cxl_base_attributes,
  93 };
  94
  95 static ssize_t start_show(struct device *dev, struct device_attribute *attr,
  96                           char *buf)
  97 {
  98         struct cxl_decoder *cxld = to_cxl_decoder(dev);
  99
 100         return sysfs_emit(buf, "%#llx\n", cxld->hpa_range.start);
 101 }
 102 static DEVICE_ATTR_ADMIN_RO(start);
 103
 104 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
 105                         char *buf)
 106 {
 107         struct cxl_decoder *cxld = to_cxl_decoder(dev);
 108
 109         return sysfs_emit(buf, "%#llx\n", range_len(&cxld->hpa_range));
 110 }
 111 static DEVICE_ATTR_RO(size);
 112
 113 #define CXL_DECODER_FLAG_ATTR(name, flag)                            \
 114 static ssize_t name##_show(struct device *dev,                       \
 115                            struct device_attribute *attr, char *buf) \
 116 {                                                                    \
 117         struct cxl_decoder *cxld = to_cxl_decoder(dev);              \
 118                                                                      \
 119         return sysfs_emit(buf, "%s\n",                               \
 120                           (cxld->flags & (flag)) ? "1" : "0");       \
 121 }                                                                    \
 122 static DEVICE_ATTR_RO(name)
 123
 124 CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
 125 CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
 126 CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
 127 CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
 128 CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
 129
 130 static ssize_t target_type_show(struct device *dev,
 131                                 struct device_attribute *attr, char *buf)
 132 {
 133         struct cxl_decoder *cxld = to_cxl_decoder(dev);
 134
 135         switch (cxld->target_type) {
 136         case CXL_DECODER_DEVMEM:
 137                 return sysfs_emit(buf, "accelerator\n");
 138         case CXL_DECODER_HOSTONLYMEM:
 139                 return sysfs_emit(buf, "expander\n");
 140         }
 141         return -ENXIO;
 142 }
 143 static DEVICE_ATTR_RO(target_type);
 144
 145 static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf)
 146 {
 147         struct cxl_decoder *cxld = &cxlsd->cxld;
 148         ssize_t offset = 0;
 149         int i, rc = 0;
 150
 151         for (i = 0; i < cxld->interleave_ways; i++) {
 152                 struct cxl_dport *dport = cxlsd->target[i];
 153                 struct cxl_dport *next = NULL;
 154
 155                 if (!dport)
 156                         break;
 157
 158                 if (i + 1 < cxld->interleave_ways)
 159                         next = cxlsd->target[i + 1];
 160                 rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
 161                                    next ? "," : "");
 162                 if (rc < 0)
 163                         return rc;
 164                 offset += rc;
 165         }
 166
 167         return offset;
 168 }
 169
 170 static ssize_t target_list_show(struct device *dev,
 171                                 struct device_attribute *attr, char *buf)
 172 {
 173         struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
 174         ssize_t offset;
 175         int rc;
 176
 177         guard(rwsem_read)(&cxl_region_rwsem);
 178         rc = emit_target_list(cxlsd, buf);
 179         if (rc < 0)
 180                 return rc;
 181         offset = rc;
 182
 183         rc = sysfs_emit_at(buf, offset, "\n");
 184         if (rc < 0)
 185                 return rc;
 186
 187         return offset + rc;
 188 }
 189 static DEVICE_ATTR_RO(target_list);
 190
 191 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
 192                          char *buf)
 193 {
 194         struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 195
 196         return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxled->mode));
 197 }
 198
 199 static ssize_t mode_store(struct device *dev, struct device_attribute *attr,
 200                           const char *buf, size_t len)
 201 {
 202         struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 203         enum cxl_decoder_mode mode;
 204         ssize_t rc;
 205
 206         if (sysfs_streq(buf, "pmem"))
 207                 mode = CXL_DECODER_PMEM;
 208         else if (sysfs_streq(buf, "ram"))
 209                 mode = CXL_DECODER_RAM;
 210         else
 211                 return -EINVAL;
 212
 213         rc = cxl_dpa_set_mode(cxled, mode);
 214         if (rc)
 215                 return rc;
 216
 217         return len;
 218 }
 219 static DEVICE_ATTR_RW(mode);
 220
 221 static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *attr,
 222                             char *buf)
 223 {
 224         struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 225
 226         guard(rwsem_read)(&cxl_dpa_rwsem);
 227         return sysfs_emit(buf, "%#llx\n", (u64)cxl_dpa_resource_start(cxled));
 228 }
 229 static DEVICE_ATTR_RO(dpa_resource);
 230
 231 static ssize_t dpa_size_show(struct device *dev, struct device_attribute *attr,
 232                              char *buf)
 233 {
 234         struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 235         resource_size_t size = cxl_dpa_size(cxled);
 236
 237         return sysfs_emit(buf, "%pa\n", &size);
 238 }
 239
 240 static ssize_t dpa_size_store(struct device *dev, struct device_attribute *attr,
 241                               const char *buf, size_t len)
 242 {
 243         struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 244         unsigned long long size;
 245         ssize_t rc;
 246
 247         rc = kstrtoull(buf, 0, &size);
 248         if (rc)
 249                 return rc;
 250
 251         if (!IS_ALIGNED(size, SZ_256M))
 252                 return -EINVAL;
 253
 254         rc = cxl_dpa_free(cxled);
 255         if (rc)
 256                 return rc;
 257
 258         if (size == 0)
 259                 return len;
 260
 261         rc = cxl_dpa_alloc(cxled, size);
 262         if (rc)
 263                 return rc;
 264
 265         return len;
 266 }
 267 static DEVICE_ATTR_RW(dpa_size);
 268
 269 static ssize_t interleave_granularity_show(struct device *dev,
 270                                            struct device_attribute *attr,
 271                                            char *buf)
 272 {
 273         struct cxl_decoder *cxld = to_cxl_decoder(dev);
 274
 275         return sysfs_emit(buf, "%d\n", cxld->interleave_granularity);
 276 }
 277
 278 static DEVICE_ATTR_RO(interleave_granularity);
 279
 280 static ssize_t interleave_ways_show(struct device *dev,
 281                                     struct device_attribute *attr, char *buf)
 282 {
 283         struct cxl_decoder *cxld = to_cxl_decoder(dev);
 284
 285         return sysfs_emit(buf, "%d\n", cxld->interleave_ways);
 286 }
 287
 288 static DEVICE_ATTR_RO(interleave_ways);
 289
 290 static struct attribute *cxl_decoder_base_attrs[] = {
 291         &dev_attr_start.attr,
 292         &dev_attr_size.attr,
 293         &dev_attr_locked.attr,
 294         &dev_attr_interleave_granularity.attr,
 295         &dev_attr_interleave_ways.attr,
 296         NULL,
 297 };
 298
 299 static struct attribute_group cxl_decoder_base_attribute_group = {
 300         .attrs = cxl_decoder_base_attrs,
 301 };
 302
 303 static struct attribute *cxl_decoder_root_attrs[] = {
 304         &dev_attr_cap_pmem.attr,
 305         &dev_attr_cap_ram.attr,
 306         &dev_attr_cap_type2.attr,
 307         &dev_attr_cap_type3.attr,
 308         &dev_attr_target_list.attr,
 309         SET_CXL_REGION_ATTR(create_pmem_region)
 310         SET_CXL_REGION_ATTR(create_ram_region)
 311         SET_CXL_REGION_ATTR(delete_region)
 312         NULL,
 313 };
 314
 315 static bool can_create_pmem(struct cxl_root_decoder *cxlrd)
 316 {
 317         unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_PMEM;
 318
 319         return (cxlrd->cxlsd.cxld.flags & flags) == flags;
 320 }
 321
 322 static bool can_create_ram(struct cxl_root_decoder *cxlrd)
 323 {
 324         unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_RAM;
 325
 326         return (cxlrd->cxlsd.cxld.flags & flags) == flags;
 327 }
 328
 329 static umode_t cxl_root_decoder_visible(struct kobject *kobj, struct attribute *a, int n)
 330 {
 331         struct device *dev = kobj_to_dev(kobj);
 332         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
 333
 334         if (a == CXL_REGION_ATTR(create_pmem_region) && !can_create_pmem(cxlrd))
 335                 return 0;
 336
 337         if (a == CXL_REGION_ATTR(create_ram_region) && !can_create_ram(cxlrd))
 338                 return 0;
 339
 340         if (a == CXL_REGION_ATTR(delete_region) &&
 341             !(can_create_pmem(cxlrd) || can_create_ram(cxlrd)))
 342                 return 0;
 343
 344         return a->mode;
 345 }
 346
 347 static struct attribute_group cxl_decoder_root_attribute_group = {
 348         .attrs = cxl_decoder_root_attrs,
 349         .is_visible = cxl_root_decoder_visible,
 350 };
 351
 352 static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
 353         &cxl_decoder_root_attribute_group,
 354         &cxl_decoder_base_attribute_group,
 355         &cxl_base_attribute_group,
 356         NULL,
 357 };
 358
 359 static struct attribute *cxl_decoder_switch_attrs[] = {
 360         &dev_attr_target_type.attr,
 361         &dev_attr_target_list.attr,
 362         SET_CXL_REGION_ATTR(region)
 363         NULL,
 364 };
 365
 366 static struct attribute_group cxl_decoder_switch_attribute_group = {
 367         .attrs = cxl_decoder_switch_attrs,
 368 };
 369
 370 static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
 371         &cxl_decoder_switch_attribute_group,
 372         &cxl_decoder_base_attribute_group,
 373         &cxl_base_attribute_group,
 374         NULL,
 375 };
 376
 377 static struct attribute *cxl_decoder_endpoint_attrs[] = {
 378         &dev_attr_target_type.attr,
 379         &dev_attr_mode.attr,
 380         &dev_attr_dpa_size.attr,
 381         &dev_attr_dpa_resource.attr,
 382         SET_CXL_REGION_ATTR(region)
 383         NULL,
 384 };
 385
 386 static struct attribute_group cxl_decoder_endpoint_attribute_group = {
 387         .attrs = cxl_decoder_endpoint_attrs,
 388 };
 389
 390 static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = {
 391         &cxl_decoder_base_attribute_group,
 392         &cxl_decoder_endpoint_attribute_group,
 393         &cxl_base_attribute_group,
 394         NULL,
 395 };
 396
 397 static void __cxl_decoder_release(struct cxl_decoder *cxld)
 398 {
 399         struct cxl_port *port = to_cxl_port(cxld->dev.parent);
 400
 401         ida_free(&port->decoder_ida, cxld->id);
 402         put_device(&port->dev);
 403 }
 404
 405 static void cxl_endpoint_decoder_release(struct device *dev)
 406 {
 407         struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 408
 409         __cxl_decoder_release(&cxled->cxld);
 410         kfree(cxled);
 411 }
 412
 413 static void cxl_switch_decoder_release(struct device *dev)
 414 {
 415         struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
 416
 417         __cxl_decoder_release(&cxlsd->cxld);
 418         kfree(cxlsd);
 419 }
 420
 421 struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev)
 422 {
 423         if (dev_WARN_ONCE(dev, !is_root_decoder(dev),
 424                           "not a cxl_root_decoder device\n"))
 425                 return NULL;
 426         return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev);
 427 }
 428 EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, CXL);
 429
 430 static void cxl_root_decoder_release(struct device *dev)
 431 {
 432         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
 433
 434         if (atomic_read(&cxlrd->region_id) >= 0)
 435                 memregion_free(atomic_read(&cxlrd->region_id));
 436         __cxl_decoder_release(&cxlrd->cxlsd.cxld);
 437         kfree(cxlrd);
 438 }
 439
 440 static const struct device_type cxl_decoder_endpoint_type = {
 441         .name = "cxl_decoder_endpoint",
 442         .release = cxl_endpoint_decoder_release,
 443         .groups = cxl_decoder_endpoint_attribute_groups,
 444 };
 445
 446 static const struct device_type cxl_decoder_switch_type = {
 447         .name = "cxl_decoder_switch",
 448         .release = cxl_switch_decoder_release,
 449         .groups = cxl_decoder_switch_attribute_groups,
 450 };
 451
 452 static const struct device_type cxl_decoder_root_type = {
 453         .name = "cxl_decoder_root",
 454         .release = cxl_root_decoder_release,
 455         .groups = cxl_decoder_root_attribute_groups,
 456 };
 457
 458 bool is_endpoint_decoder(struct device *dev)
 459 {
 460         return dev->type == &cxl_decoder_endpoint_type;
 461 }
 462 EXPORT_SYMBOL_NS_GPL(is_endpoint_decoder, CXL);
 463
 464 bool is_root_decoder(struct device *dev)
 465 {
 466         return dev->type == &cxl_decoder_root_type;
 467 }
 468 EXPORT_SYMBOL_NS_GPL(is_root_decoder, CXL);
 469
 470 bool is_switch_decoder(struct device *dev)
 471 {
 472         return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type;
 473 }
 474 EXPORT_SYMBOL_NS_GPL(is_switch_decoder, CXL);
 475
 476 struct cxl_decoder *to_cxl_decoder(struct device *dev)
 477 {
 478         if (dev_WARN_ONCE(dev,
 479                           !is_switch_decoder(dev) && !is_endpoint_decoder(dev),
 480                           "not a cxl_decoder device\n"))
 481                 return NULL;
 482         return container_of(dev, struct cxl_decoder, dev);
 483 }
 484 EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, CXL);
 485
 486 struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev)
 487 {
 488         if (dev_WARN_ONCE(dev, !is_endpoint_decoder(dev),
 489                           "not a cxl_endpoint_decoder device\n"))
 490                 return NULL;
 491         return container_of(dev, struct cxl_endpoint_decoder, cxld.dev);
 492 }
 493 EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, CXL);
 494
 495 struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev)
 496 {
 497         if (dev_WARN_ONCE(dev, !is_switch_decoder(dev),
 498                           "not a cxl_switch_decoder device\n"))
 499                 return NULL;
 500         return container_of(dev, struct cxl_switch_decoder, cxld.dev);
 501 }
 502 EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, CXL);
 503
 504 static void cxl_ep_release(struct cxl_ep *ep)
 505 {
 506         put_device(ep->ep);
 507         kfree(ep);
 508 }
 509
 510 static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep)
 511 {
 512         if (!ep)
 513                 return;
 514         xa_erase(&port->endpoints, (unsigned long) ep->ep);
 515         cxl_ep_release(ep);
 516 }
 517
 518 static void cxl_port_release(struct device *dev)
 519 {
 520         struct cxl_port *port = to_cxl_port(dev);
 521         unsigned long index;
 522         struct cxl_ep *ep;
 523
 524         xa_for_each(&port->endpoints, index, ep)
 525                 cxl_ep_remove(port, ep);
 526         xa_destroy(&port->endpoints);
 527         xa_destroy(&port->dports);
 528         xa_destroy(&port->regions);
 529         ida_free(&cxl_port_ida, port->id);
 530         kfree(port);
 531 }
 532
 533 static const struct attribute_group *cxl_port_attribute_groups[] = {
 534         &cxl_base_attribute_group,
 535         NULL,
 536 };
 537
 538 static const struct device_type cxl_port_type = {
 539         .name = "cxl_port",
 540         .release = cxl_port_release,
 541         .groups = cxl_port_attribute_groups,
 542 };
 543
 544 bool is_cxl_port(const struct device *dev)
 545 {
 546         return dev->type == &cxl_port_type;
 547 }
 548 EXPORT_SYMBOL_NS_GPL(is_cxl_port, CXL);
 549
 550 struct cxl_port *to_cxl_port(const struct device *dev)
 551 {
 552         if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
 553                           "not a cxl_port device\n"))
 554                 return NULL;
 555         return container_of(dev, struct cxl_port, dev);
 556 }
 557 EXPORT_SYMBOL_NS_GPL(to_cxl_port, CXL);
 558
 559 static void unregister_port(void *_port)
 560 {
 561         struct cxl_port *port = _port;
 562         struct cxl_port *parent;
 563         struct device *lock_dev;
 564
 565         if (is_cxl_root(port))
 566                 parent = NULL;
 567         else
 568                 parent = to_cxl_port(port->dev.parent);
 569
 570         /*
 571          * CXL root port's and the first level of ports are unregistered
 572          * under the platform firmware device lock, all other ports are
 573          * unregistered while holding their parent port lock.
 574          */
 575         if (!parent)
 576                 lock_dev = port->uport_dev;
 577         else if (is_cxl_root(parent))
 578                 lock_dev = parent->uport_dev;
 579         else
 580                 lock_dev = &parent->dev;
 581
 582         device_lock_assert(lock_dev);
 583         port->dead = true;
 584         device_unregister(&port->dev);
 585 }
 586
 587 static void cxl_unlink_uport(void *_port)
 588 {
 589         struct cxl_port *port = _port;
 590
 591         sysfs_remove_link(&port->dev.kobj, "uport");
 592 }
 593
 594 static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
 595 {
 596         int rc;
 597
 598         rc = sysfs_create_link(&port->dev.kobj, &port->uport_dev->kobj,
 599                                "uport");
 600         if (rc)
 601                 return rc;
 602         return devm_add_action_or_reset(host, cxl_unlink_uport, port);
 603 }
 604
 605 static void cxl_unlink_parent_dport(void *_port)
 606 {
 607         struct cxl_port *port = _port;
 608
 609         sysfs_remove_link(&port->dev.kobj, "parent_dport");
 610 }
 611
 612 static int devm_cxl_link_parent_dport(struct device *host,
 613                                       struct cxl_port *port,
 614                                       struct cxl_dport *parent_dport)
 615 {
 616         int rc;
 617
 618         if (!parent_dport)
 619                 return 0;
 620
 621         rc = sysfs_create_link(&port->dev.kobj, &parent_dport->dport_dev->kobj,
 622                                "parent_dport");
 623         if (rc)
 624                 return rc;
 625         return devm_add_action_or_reset(host, cxl_unlink_parent_dport, port);
 626 }
 627
 628 static struct lock_class_key cxl_port_key;
 629
 630 static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
 631                                        resource_size_t component_reg_phys,
 632                                        struct cxl_dport *parent_dport)
 633 {
 634         struct cxl_port *port;
 635         struct device *dev;
 636         int rc;
 637
 638         port = kzalloc(sizeof(*port), GFP_KERNEL);
 639         if (!port)
 640                 return ERR_PTR(-ENOMEM);
 641
 642         rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
 643         if (rc < 0)
 644                 goto err;
 645         port->id = rc;
 646         port->uport_dev = uport_dev;
 647
 648         /*
 649          * The top-level cxl_port "cxl_root" does not have a cxl_port as
 650          * its parent and it does not have any corresponding component
 651          * registers as its decode is described by a fixed platform
 652          * description.
 653          */
 654         dev = &port->dev;
 655         if (parent_dport) {
 656                 struct cxl_port *parent_port = parent_dport->port;
 657                 struct cxl_port *iter;
 658
 659                 dev->parent = &parent_port->dev;
 660                 port->depth = parent_port->depth + 1;
 661                 port->parent_dport = parent_dport;
 662
 663                 /*
 664                  * walk to the host bridge, or the first ancestor that knows
 665                  * the host bridge
 666                  */
 667                 iter = port;
 668                 while (!iter->host_bridge &&
 669                        !is_cxl_root(to_cxl_port(iter->dev.parent)))
 670                         iter = to_cxl_port(iter->dev.parent);
 671                 if (iter->host_bridge)
 672                         port->host_bridge = iter->host_bridge;
 673                 else if (parent_dport->rch)
 674                         port->host_bridge = parent_dport->dport_dev;
 675                 else
 676                         port->host_bridge = iter->uport_dev;
 677                 dev_dbg(uport_dev, "host-bridge: %s\n",
 678                         dev_name(port->host_bridge));
 679         } else
 680                 dev->parent = uport_dev;
 681
 682         port->component_reg_phys = component_reg_phys;
 683         ida_init(&port->decoder_ida);
 684         port->hdm_end = -1;
 685         port->commit_end = -1;
 686         xa_init(&port->dports);
 687         xa_init(&port->endpoints);
 688         xa_init(&port->regions);
 689
 690         device_initialize(dev);
 691         lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
 692         device_set_pm_not_required(dev);
 693         dev->bus = &cxl_bus_type;
 694         dev->type = &cxl_port_type;
 695
 696         return port;
 697
 698 err:
 699         kfree(port);
 700         return ERR_PTR(rc);
 701 }
 702
 703 static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map,
 704                                resource_size_t component_reg_phys)
 705 {
 706         if (component_reg_phys == CXL_RESOURCE_NONE)
 707                 return 0;
 708
 709         *map = (struct cxl_register_map) {
 710                 .host = host,
 711                 .reg_type = CXL_REGLOC_RBI_COMPONENT,
 712                 .resource = component_reg_phys,
 713                 .max_size = CXL_COMPONENT_REG_BLOCK_SIZE,
 714         };
 715
 716         return cxl_setup_regs(map);
 717 }
 718
 719 static int cxl_port_setup_regs(struct cxl_port *port,
 720                         resource_size_t component_reg_phys)
 721 {
 722         if (dev_is_platform(port->uport_dev))
 723                 return 0;
 724         return cxl_setup_comp_regs(&port->dev, &port->comp_map,
 725                                    component_reg_phys);
 726 }
 727
 728 static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
 729                                 resource_size_t component_reg_phys)
 730 {
 731         int rc;
 732
 733         if (dev_is_platform(dport->dport_dev))
 734                 return 0;
 735
 736         /*
 737          * use @dport->dport_dev for the context for error messages during
 738          * register probing, and fixup @host after the fact, since @host may be
 739          * NULL.
 740          */
 741         rc = cxl_setup_comp_regs(dport->dport_dev, &dport->comp_map,
 742                                  component_reg_phys);
 743         dport->comp_map.host = host;
 744         return rc;
 745 }
 746
 747 static struct cxl_port *__devm_cxl_add_port(struct device *host,
 748                                             struct device *uport_dev,
 749                                             resource_size_t component_reg_phys,
 750                                             struct cxl_dport *parent_dport)
 751 {
 752         struct cxl_port *port;
 753         struct device *dev;
 754         int rc;
 755
 756         port = cxl_port_alloc(uport_dev, component_reg_phys, parent_dport);
 757         if (IS_ERR(port))
 758                 return port;
 759
 760         dev = &port->dev;
 761         if (is_cxl_memdev(uport_dev))
 762                 rc = dev_set_name(dev, "endpoint%d", port->id);
 763         else if (parent_dport)
 764                 rc = dev_set_name(dev, "port%d", port->id);
 765         else
 766                 rc = dev_set_name(dev, "root%d", port->id);
 767         if (rc)
 768                 goto err;
 769
 770         rc = cxl_port_setup_regs(port, component_reg_phys);
 771         if (rc)
 772                 goto err;
 773
 774         rc = device_add(dev);
 775         if (rc)
 776                 goto err;
 777
 778         rc = devm_add_action_or_reset(host, unregister_port, port);
 779         if (rc)
 780                 return ERR_PTR(rc);
 781
 782         rc = devm_cxl_link_uport(host, port);
 783         if (rc)
 784                 return ERR_PTR(rc);
 785
 786         rc = devm_cxl_link_parent_dport(host, port, parent_dport);
 787         if (rc)
 788                 return ERR_PTR(rc);
 789
 790         return port;
 791
 792 err:
 793         put_device(dev);
 794         return ERR_PTR(rc);
 795 }
 796
 797 /**
 798  * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
 799  * @host: host device for devm operations
 800  * @uport_dev: "physical" device implementing this upstream port
 801  * @component_reg_phys: (optional) for configurable cxl_port instances
 802  * @parent_dport: next hop up in the CXL memory decode hierarchy
 803  */
 804 struct cxl_port *devm_cxl_add_port(struct device *host,
 805                                    struct device *uport_dev,
 806                                    resource_size_t component_reg_phys,
 807                                    struct cxl_dport *parent_dport)
 808 {
 809         struct cxl_port *port, *parent_port;
 810
 811         port = __devm_cxl_add_port(host, uport_dev, component_reg_phys,
 812                                    parent_dport);
 813
 814         parent_port = parent_dport ? parent_dport->port : NULL;
 815         if (IS_ERR(port)) {
 816                 dev_dbg(uport_dev, "Failed to add%s%s%s: %ld\n",
 817                         parent_port ? " port to " : "",
 818                         parent_port ? dev_name(&parent_port->dev) : "",
 819                         parent_port ? "" : " root port",
 820                         PTR_ERR(port));
 821         } else {
 822                 dev_dbg(uport_dev, "%s added%s%s%s\n",
 823                         dev_name(&port->dev),
 824                         parent_port ? " to " : "",
 825                         parent_port ? dev_name(&parent_port->dev) : "",
 826                         parent_port ? "" : " (root port)");
 827         }
 828
 829         return port;
 830 }
 831 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, CXL);
 832
 833 struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port)
 834 {
 835         /* There is no pci_bus associated with a CXL platform-root port */
 836         if (is_cxl_root(port))
 837                 return NULL;
 838
 839         if (dev_is_pci(port->uport_dev)) {
 840                 struct pci_dev *pdev = to_pci_dev(port->uport_dev);
 841
 842                 return pdev->subordinate;
 843         }
 844
 845         return xa_load(&cxl_root_buses, (unsigned long)port->uport_dev);
 846 }
 847 EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, CXL);
 848
 849 static void unregister_pci_bus(void *uport_dev)
 850 {
 851         xa_erase(&cxl_root_buses, (unsigned long)uport_dev);
 852 }
 853
 854 int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,
 855                               struct pci_bus *bus)
 856 {
 857         int rc;
 858
 859         if (dev_is_pci(uport_dev))
 860                 return -EINVAL;
 861
 862         rc = xa_insert(&cxl_root_buses, (unsigned long)uport_dev, bus,
 863                        GFP_KERNEL);
 864         if (rc)
 865                 return rc;
 866         return devm_add_action_or_reset(host, unregister_pci_bus, uport_dev);
 867 }
 868 EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, CXL);
 869
 870 static bool dev_is_cxl_root_child(struct device *dev)
 871 {
 872         struct cxl_port *port, *parent;
 873
 874         if (!is_cxl_port(dev))
 875                 return false;
 876
 877         port = to_cxl_port(dev);
 878         if (is_cxl_root(port))
 879                 return false;
 880
 881         parent = to_cxl_port(port->dev.parent);
 882         if (is_cxl_root(parent))
 883                 return true;
 884
 885         return false;
 886 }
 887
 888 struct cxl_port *find_cxl_root(struct cxl_port *port)
 889 {
 890         struct cxl_port *iter = port;
 891
 892         while (iter && !is_cxl_root(iter))
 893                 iter = to_cxl_port(iter->dev.parent);
 894
 895         if (!iter)
 896                 return NULL;
 897         get_device(&iter->dev);
 898         return iter;
 899 }
 900 EXPORT_SYMBOL_NS_GPL(find_cxl_root, CXL);
 901
 902 static struct cxl_dport *find_dport(struct cxl_port *port, int id)
 903 {
 904         struct cxl_dport *dport;
 905         unsigned long index;
 906
 907         device_lock_assert(&port->dev);
 908         xa_for_each(&port->dports, index, dport)
 909                 if (dport->port_id == id)
 910                         return dport;
 911         return NULL;
 912 }
 913
 914 static int add_dport(struct cxl_port *port, struct cxl_dport *dport)
 915 {
 916         struct cxl_dport *dup;
 917         int rc;
 918
 919         device_lock_assert(&port->dev);
 920         dup = find_dport(port, dport->port_id);
 921         if (dup) {
 922                 dev_err(&port->dev,
 923                         "unable to add dport%d-%s non-unique port id (%s)\n",
 924                         dport->port_id, dev_name(dport->dport_dev),
 925                         dev_name(dup->dport_dev));
 926                 return -EBUSY;
 927         }
 928
 929         rc = xa_insert(&port->dports, (unsigned long)dport->dport_dev, dport,
 930                        GFP_KERNEL);
 931         if (rc)
 932                 return rc;
 933
 934         port->nr_dports++;
 935         return 0;
 936 }
 937
 938 /*
 939  * Since root-level CXL dports cannot be enumerated by PCI they are not
 940  * enumerated by the common port driver that acquires the port lock over
 941  * dport add/remove. Instead, root dports are manually added by a
 942  * platform driver and cond_cxl_root_lock() is used to take the missing
 943  * port lock in that case.
 944  */
 945 static void cond_cxl_root_lock(struct cxl_port *port)
 946 {
 947         if (is_cxl_root(port))
 948                 device_lock(&port->dev);
 949 }
 950
 951 static void cond_cxl_root_unlock(struct cxl_port *port)
 952 {
 953         if (is_cxl_root(port))
 954                 device_unlock(&port->dev);
 955 }
 956
 957 static void cxl_dport_remove(void *data)
 958 {
 959         struct cxl_dport *dport = data;
 960         struct cxl_port *port = dport->port;
 961
 962         xa_erase(&port->dports, (unsigned long) dport->dport_dev);
 963         put_device(dport->dport_dev);
 964 }
 965
 966 static void cxl_dport_unlink(void *data)
 967 {
 968         struct cxl_dport *dport = data;
 969         struct cxl_port *port = dport->port;
 970         char link_name[CXL_TARGET_STRLEN];
 971
 972         sprintf(link_name, "dport%d", dport->port_id);
 973         sysfs_remove_link(&port->dev.kobj, link_name);
 974 }
 975
 976 static struct cxl_dport *
 977 __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
 978                      int port_id, resource_size_t component_reg_phys,
 979                      resource_size_t rcrb)
 980 {
 981         char link_name[CXL_TARGET_STRLEN];
 982         struct cxl_dport *dport;
 983         struct device *host;
 984         int rc;
 985
 986         if (is_cxl_root(port))
 987                 host = port->uport_dev;
 988         else
 989                 host = &port->dev;
 990
 991         if (!host->driver) {
 992                 dev_WARN_ONCE(&port->dev, 1, "dport:%s bad devm context\n",
 993                               dev_name(dport_dev));
 994                 return ERR_PTR(-ENXIO);
 995         }
 996
 997         if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
 998             CXL_TARGET_STRLEN)
 999                 return ERR_PTR(-EINVAL);
1000
1001         dport = devm_kzalloc(host, sizeof(*dport), GFP_KERNEL);
1002         if (!dport)
1003                 return ERR_PTR(-ENOMEM);
1004
1005         dport->dport_dev = dport_dev;
1006         dport->port_id = port_id;
1007         dport->port = port;
1008
1009         if (rcrb == CXL_RESOURCE_NONE) {
1010                 rc = cxl_dport_setup_regs(&port->dev, dport,
1011                                           component_reg_phys);
1012                 if (rc)
1013                         return ERR_PTR(rc);
1014         } else {
1015                 dport->rcrb.base = rcrb;
1016                 component_reg_phys = __rcrb_to_component(dport_dev, &dport->rcrb,
1017                                                          CXL_RCRB_DOWNSTREAM);
1018                 if (component_reg_phys == CXL_RESOURCE_NONE) {
1019                         dev_warn(dport_dev, "Invalid Component Registers in RCRB");
1020                         return ERR_PTR(-ENXIO);
1021                 }
1022
1023                 /*
1024                  * RCH @dport is not ready to map until associated with its
1025                  * memdev
1026                  */
1027                 rc = cxl_dport_setup_regs(NULL, dport, component_reg_phys);
1028                 if (rc)
1029                         return ERR_PTR(rc);
1030
1031                 dport->rch = true;
1032         }
1033
1034         if (component_reg_phys != CXL_RESOURCE_NONE)
1035                 dev_dbg(dport_dev, "Component Registers found for dport: %pa\n",
1036                         &component_reg_phys);
1037
1038         cond_cxl_root_lock(port);
1039         rc = add_dport(port, dport);
1040         cond_cxl_root_unlock(port);
1041         if (rc)
1042                 return ERR_PTR(rc);
1043
1044         get_device(dport_dev);
1045         rc = devm_add_action_or_reset(host, cxl_dport_remove, dport);
1046         if (rc)
1047                 return ERR_PTR(rc);
1048
1049         rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
1050         if (rc)
1051                 return ERR_PTR(rc);
1052
1053         rc = devm_add_action_or_reset(host, cxl_dport_unlink, dport);
1054         if (rc)
1055                 return ERR_PTR(rc);
1056
1057         return dport;
1058 }
1059
1060 /**
1061  * devm_cxl_add_dport - append VH downstream port data to a cxl_port
1062  * @port: the cxl_port that references this dport
1063  * @dport_dev: firmware or PCI device representing the dport
1064  * @port_id: identifier for this dport in a decoder's target list
1065  * @component_reg_phys: optional location of CXL component registers
1066  *
1067  * Note that dports are appended to the devm release action's of the
1068  * either the port's host (for root ports), or the port itself (for
1069  * switch ports)
1070  */
1071 struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
1072                                      struct device *dport_dev, int port_id,
1073                                      resource_size_t component_reg_phys)
1074 {
1075         struct cxl_dport *dport;
1076
1077         dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1078                                      component_reg_phys, CXL_RESOURCE_NONE);
1079         if (IS_ERR(dport)) {
1080                 dev_dbg(dport_dev, "failed to add dport to %s: %ld\n",
1081                         dev_name(&port->dev), PTR_ERR(dport));
1082         } else {
1083                 dev_dbg(dport_dev, "dport added to %s\n",
1084                         dev_name(&port->dev));
1085         }
1086
1087         return dport;
1088 }
1089 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL);
1090
1091 /**
1092  * devm_cxl_add_rch_dport - append RCH downstream port data to a cxl_port
1093  * @port: the cxl_port that references this dport
1094  * @dport_dev: firmware or PCI device representing the dport
1095  * @port_id: identifier for this dport in a decoder's target list
1096  * @rcrb: mandatory location of a Root Complex Register Block
1097  *
1098  * See CXL 3.0 9.11.8 CXL Devices Attached to an RCH
1099  */
1100 struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
1101                                          struct device *dport_dev, int port_id,
1102                                          resource_size_t rcrb)
1103 {
1104         struct cxl_dport *dport;
1105
1106         if (rcrb == CXL_RESOURCE_NONE) {
1107                 dev_dbg(&port->dev, "failed to add RCH dport, missing RCRB\n");
1108                 return ERR_PTR(-EINVAL);
1109         }
1110
1111         dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1112                                      CXL_RESOURCE_NONE, rcrb);
1113         if (IS_ERR(dport)) {
1114                 dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n",
1115                         dev_name(&port->dev), PTR_ERR(dport));
1116         } else {
1117                 dev_dbg(dport_dev, "RCH dport added to %s\n",
1118                         dev_name(&port->dev));
1119         }
1120
1121         return dport;
1122 }
1123 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_rch_dport, CXL);
1124
1125 static int add_ep(struct cxl_ep *new)
1126 {
1127         struct cxl_port *port = new->dport->port;
1128         int rc;
1129
1130         device_lock(&port->dev);
1131         if (port->dead) {
1132                 device_unlock(&port->dev);
1133                 return -ENXIO;
1134         }
1135         rc = xa_insert(&port->endpoints, (unsigned long)new->ep, new,
1136                        GFP_KERNEL);
1137         device_unlock(&port->dev);
1138
1139         return rc;
1140 }
1141
1142 /**
1143  * cxl_add_ep - register an endpoint's interest in a port
1144  * @dport: the dport that routes to @ep_dev
1145  * @ep_dev: device representing the endpoint
1146  *
1147  * Intermediate CXL ports are scanned based on the arrival of endpoints.
1148  * When those endpoints depart the port can be destroyed once all
1149  * endpoints that care about that port have been removed.
1150  */
1151 static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev)
1152 {
1153         struct cxl_ep *ep;
1154         int rc;
1155
1156         ep = kzalloc(sizeof(*ep), GFP_KERNEL);
1157         if (!ep)
1158                 return -ENOMEM;
1159
1160         ep->ep = get_device(ep_dev);
1161         ep->dport = dport;
1162
1163         rc = add_ep(ep);
1164         if (rc)
1165                 cxl_ep_release(ep);
1166         return rc;
1167 }
1168
1169 struct cxl_find_port_ctx {
1170         const struct device *dport_dev;
1171         const struct cxl_port *parent_port;
1172         struct cxl_dport **dport;
1173 };
1174
1175 static int match_port_by_dport(struct device *dev, const void *data)
1176 {
1177         const struct cxl_find_port_ctx *ctx = data;
1178         struct cxl_dport *dport;
1179         struct cxl_port *port;
1180
1181         if (!is_cxl_port(dev))
1182                 return 0;
1183         if (ctx->parent_port && dev->parent != &ctx->parent_port->dev)
1184                 return 0;
1185
1186         port = to_cxl_port(dev);
1187         dport = cxl_find_dport_by_dev(port, ctx->dport_dev);
1188         if (ctx->dport)
1189                 *ctx->dport = dport;
1190         return dport != NULL;
1191 }
1192
1193 static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx)
1194 {
1195         struct device *dev;
1196
1197         if (!ctx->dport_dev)
1198                 return NULL;
1199
1200         dev = bus_find_device(&cxl_bus_type, NULL, ctx, match_port_by_dport);
1201         if (dev)
1202                 return to_cxl_port(dev);
1203         return NULL;
1204 }
1205
1206 static struct cxl_port *find_cxl_port(struct device *dport_dev,
1207                                       struct cxl_dport **dport)
1208 {
1209         struct cxl_find_port_ctx ctx = {
1210                 .dport_dev = dport_dev,
1211                 .dport = dport,
1212         };
1213         struct cxl_port *port;
1214
1215         port = __find_cxl_port(&ctx);
1216         return port;
1217 }
1218
1219 static struct cxl_port *find_cxl_port_at(struct cxl_port *parent_port,
1220                                          struct device *dport_dev,
1221                                          struct cxl_dport **dport)
1222 {
1223         struct cxl_find_port_ctx ctx = {
1224                 .dport_dev = dport_dev,
1225                 .parent_port = parent_port,
1226                 .dport = dport,
1227         };
1228         struct cxl_port *port;
1229
1230         port = __find_cxl_port(&ctx);
1231         return port;
1232 }
1233
1234 /*
1235  * All users of grandparent() are using it to walk PCIe-like switch port
1236  * hierarchy. A PCIe switch is comprised of a bridge device representing the
1237  * upstream switch port and N bridges representing downstream switch ports. When
1238  * bridges stack the grand-parent of a downstream switch port is another
1239  * downstream switch port in the immediate ancestor switch.
1240  */
1241 static struct device *grandparent(struct device *dev)
1242 {
1243         if (dev && dev->parent)
1244                 return dev->parent->parent;
1245         return NULL;
1246 }
1247
1248 static struct device *endpoint_host(struct cxl_port *endpoint)
1249 {
1250         struct cxl_port *port = to_cxl_port(endpoint->dev.parent);
1251
1252         if (is_cxl_root(port))
1253                 return port->uport_dev;
1254         return &port->dev;
1255 }
1256
1257 static void delete_endpoint(void *data)
1258 {
1259         struct cxl_memdev *cxlmd = data;
1260         struct cxl_port *endpoint = cxlmd->endpoint;
1261         struct device *host = endpoint_host(endpoint);
1262
1263         device_lock(host);
1264         if (host->driver && !endpoint->dead) {
1265                 devm_release_action(host, cxl_unlink_parent_dport, endpoint);
1266                 devm_release_action(host, cxl_unlink_uport, endpoint);
1267                 devm_release_action(host, unregister_port, endpoint);
1268         }
1269         cxlmd->endpoint = NULL;
1270         device_unlock(host);
1271         put_device(&endpoint->dev);
1272         put_device(host);
1273 }
1274
1275 int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
1276 {
1277         struct device *host = endpoint_host(endpoint);
1278         struct device *dev = &cxlmd->dev;
1279
1280         get_device(host);
1281         get_device(&endpoint->dev);
1282         cxlmd->endpoint = endpoint;
1283         cxlmd->depth = endpoint->depth;
1284         return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
1285 }
1286 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, CXL);
1287
1288 /*
1289  * The natural end of life of a non-root 'cxl_port' is when its parent port goes
1290  * through a ->remove() event ("top-down" unregistration). The unnatural trigger
1291  * for a port to be unregistered is when all memdevs beneath that port have gone
1292  * through ->remove(). This "bottom-up" removal selectively removes individual
1293  * child ports manually. This depends on devm_cxl_add_port() to not change is
1294  * devm action registration order, and for dports to have already been
1295  * destroyed by reap_dports().
1296  */
1297 static void delete_switch_port(struct cxl_port *port)
1298 {
1299         devm_release_action(port->dev.parent, cxl_unlink_parent_dport, port);
1300         devm_release_action(port->dev.parent, cxl_unlink_uport, port);
1301         devm_release_action(port->dev.parent, unregister_port, port);
1302 }
1303
1304 static void reap_dports(struct cxl_port *port)
1305 {
1306         struct cxl_dport *dport;
1307         unsigned long index;
1308
1309         device_lock_assert(&port->dev);
1310
1311         xa_for_each(&port->dports, index, dport) {
1312                 devm_release_action(&port->dev, cxl_dport_unlink, dport);
1313                 devm_release_action(&port->dev, cxl_dport_remove, dport);
1314                 devm_kfree(&port->dev, dport);
1315         }
1316 }
1317
1318 struct detach_ctx {
1319         struct cxl_memdev *cxlmd;
1320         int depth;
1321 };
1322
1323 static int port_has_memdev(struct device *dev, const void *data)
1324 {
1325         const struct detach_ctx *ctx = data;
1326         struct cxl_port *port;
1327
1328         if (!is_cxl_port(dev))
1329                 return 0;
1330
1331         port = to_cxl_port(dev);
1332         if (port->depth != ctx->depth)
1333                 return 0;
1334
1335         return !!cxl_ep_load(port, ctx->cxlmd);
1336 }
1337
1338 static void cxl_detach_ep(void *data)
1339 {
1340         struct cxl_memdev *cxlmd = data;
1341
1342         for (int i = cxlmd->depth - 1; i >= 1; i--) {
1343                 struct cxl_port *port, *parent_port;
1344                 struct detach_ctx ctx = {
1345                         .cxlmd = cxlmd,
1346                         .depth = i,
1347                 };
1348                 struct device *dev;
1349                 struct cxl_ep *ep;
1350                 bool died = false;
1351
1352                 dev = bus_find_device(&cxl_bus_type, NULL, &ctx,
1353                                       port_has_memdev);
1354                 if (!dev)
1355                         continue;
1356                 port = to_cxl_port(dev);
1357
1358                 parent_port = to_cxl_port(port->dev.parent);
1359                 device_lock(&parent_port->dev);
1360                 device_lock(&port->dev);
1361                 ep = cxl_ep_load(port, cxlmd);
1362                 dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
1363                         ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
1364                 cxl_ep_remove(port, ep);
1365                 if (ep && !port->dead && xa_empty(&port->endpoints) &&
1366                     !is_cxl_root(parent_port) && parent_port->dev.driver) {
1367                         /*
1368                          * This was the last ep attached to a dynamically
1369                          * enumerated port. Block new cxl_add_ep() and garbage
1370                          * collect the port.
1371                          */
1372                         died = true;
1373                         port->dead = true;
1374                         reap_dports(port);
1375                 }
1376                 device_unlock(&port->dev);
1377
1378                 if (died) {
1379                         dev_dbg(&cxlmd->dev, "delete %s\n",
1380                                 dev_name(&port->dev));
1381                         delete_switch_port(port);
1382                 }
1383                 put_device(&port->dev);
1384                 device_unlock(&parent_port->dev);
1385         }
1386 }
1387
1388 static resource_size_t find_component_registers(struct device *dev)
1389 {
1390         struct cxl_register_map map;
1391         struct pci_dev *pdev;
1392
1393         /*
1394          * Theoretically, CXL component registers can be hosted on a
1395          * non-PCI device, in practice, only cxl_test hits this case.
1396          */
1397         if (!dev_is_pci(dev))
1398                 return CXL_RESOURCE_NONE;
1399
1400         pdev = to_pci_dev(dev);
1401
1402         cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
1403         return map.resource;
1404 }
1405
1406 static int add_port_attach_ep(struct cxl_memdev *cxlmd,
1407                               struct device *uport_dev,
1408                               struct device *dport_dev)
1409 {
1410         struct device *dparent = grandparent(dport_dev);
1411         struct cxl_port *port, *parent_port = NULL;
1412         struct cxl_dport *dport, *parent_dport;
1413         resource_size_t component_reg_phys;
1414         int rc;
1415
1416         if (!dparent) {
1417                 /*
1418                  * The iteration reached the topology root without finding the
1419                  * CXL-root 'cxl_port' on a previous iteration, fail for now to
1420                  * be re-probed after platform driver attaches.
1421                  */
1422                 dev_dbg(&cxlmd->dev, "%s is a root dport\n",
1423                         dev_name(dport_dev));
1424                 return -ENXIO;
1425         }
1426
1427         parent_port = find_cxl_port(dparent, &parent_dport);
1428         if (!parent_port) {
1429                 /* iterate to create this parent_port */
1430                 return -EAGAIN;
1431         }
1432
1433         device_lock(&parent_port->dev);
1434         if (!parent_port->dev.driver) {
1435                 dev_warn(&cxlmd->dev,
1436                          "port %s:%s disabled, failed to enumerate CXL.mem\n",
1437                          dev_name(&parent_port->dev), dev_name(uport_dev));
1438                 port = ERR_PTR(-ENXIO);
1439                 goto out;
1440         }
1441
1442         port = find_cxl_port_at(parent_port, dport_dev, &dport);
1443         if (!port) {
1444                 component_reg_phys = find_component_registers(uport_dev);
1445                 port = devm_cxl_add_port(&parent_port->dev, uport_dev,
1446                                          component_reg_phys, parent_dport);
1447                 /* retry find to pick up the new dport information */
1448                 if (!IS_ERR(port))
1449                         port = find_cxl_port_at(parent_port, dport_dev, &dport);
1450         }
1451 out:
1452         device_unlock(&parent_port->dev);
1453
1454         if (IS_ERR(port))
1455                 rc = PTR_ERR(port);
1456         else {
1457                 dev_dbg(&cxlmd->dev, "add to new port %s:%s\n",
1458                         dev_name(&port->dev), dev_name(port->uport_dev));
1459                 rc = cxl_add_ep(dport, &cxlmd->dev);
1460                 if (rc == -EBUSY) {
1461                         /*
1462                          * "can't" happen, but this error code means
1463                          * something to the caller, so translate it.
1464                          */
1465                         rc = -ENXIO;
1466                 }
1467                 put_device(&port->dev);
1468         }
1469
1470         put_device(&parent_port->dev);
1471         return rc;
1472 }
1473
1474 int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
1475 {
1476         struct device *dev = &cxlmd->dev;
1477         struct device *iter;
1478         int rc;
1479
1480         /*
1481          * Skip intermediate port enumeration in the RCH case, there
1482          * are no ports in between a host bridge and an endpoint.
1483          */
1484         if (cxlmd->cxlds->rcd)
1485                 return 0;
1486
1487         rc = devm_add_action_or_reset(&cxlmd->dev, cxl_detach_ep, cxlmd);
1488         if (rc)
1489                 return rc;
1490
1491         /*
1492          * Scan for and add all cxl_ports in this device's ancestry.
1493          * Repeat until no more ports are added. Abort if a port add
1494          * attempt fails.
1495          */
1496 retry:
1497         for (iter = dev; iter; iter = grandparent(iter)) {
1498                 struct device *dport_dev = grandparent(iter);
1499                 struct device *uport_dev;
1500                 struct cxl_dport *dport;
1501                 struct cxl_port *port;
1502
1503                 if (!dport_dev)
1504                         return 0;
1505
1506                 uport_dev = dport_dev->parent;
1507                 if (!uport_dev) {
1508                         dev_warn(dev, "at %s no parent for dport: %s\n",
1509                                  dev_name(iter), dev_name(dport_dev));
1510                         return -ENXIO;
1511                 }
1512
1513                 dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
1514                         dev_name(iter), dev_name(dport_dev),
1515                         dev_name(uport_dev));
1516                 port = find_cxl_port(dport_dev, &dport);
1517                 if (port) {
1518                         dev_dbg(&cxlmd->dev,
1519                                 "found already registered port %s:%s\n",
1520                                 dev_name(&port->dev),
1521                                 dev_name(port->uport_dev));
1522                         rc = cxl_add_ep(dport, &cxlmd->dev);
1523
1524                         /*
1525                          * If the endpoint already exists in the port's list,
1526                          * that's ok, it was added on a previous pass.
1527                          * Otherwise, retry in add_port_attach_ep() after taking
1528                          * the parent_port lock as the current port may be being
1529                          * reaped.
1530                          */
1531                         if (rc && rc != -EBUSY) {
1532                                 put_device(&port->dev);
1533                                 return rc;
1534                         }
1535
1536                         /* Any more ports to add between this one and the root? */
1537                         if (!dev_is_cxl_root_child(&port->dev)) {
1538                                 put_device(&port->dev);
1539                                 continue;
1540                         }
1541
1542                         put_device(&port->dev);
1543                         return 0;
1544                 }
1545
1546                 rc = add_port_attach_ep(cxlmd, uport_dev, dport_dev);
1547                 /* port missing, try to add parent */
1548                 if (rc == -EAGAIN)
1549                         continue;
1550                 /* failed to add ep or port */
1551                 if (rc)
1552                         return rc;
1553                 /* port added, new descendants possible, start over */
1554                 goto retry;
1555         }
1556
1557         return 0;
1558 }
1559 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL);
1560
1561 struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
1562                                    struct cxl_dport **dport)
1563 {
1564         return find_cxl_port(pdev->dev.parent, dport);
1565 }
1566 EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, CXL);
1567
1568 struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
1569                                    struct cxl_dport **dport)
1570 {
1571         return find_cxl_port(grandparent(&cxlmd->dev), dport);
1572 }
1573 EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, CXL);
1574
1575 static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
1576                                     struct cxl_port *port, int *target_map)
1577 {
1578         int i;
1579
1580         if (!target_map)
1581                 return 0;
1582
1583         device_lock_assert(&port->dev);
1584
1585         if (xa_empty(&port->dports))
1586                 return -EINVAL;
1587
1588         guard(rwsem_write)(&cxl_region_rwsem);
1589         for (i = 0; i < cxlsd->cxld.interleave_ways; i++) {
1590                 struct cxl_dport *dport = find_dport(port, target_map[i]);
1591
1592                 if (!dport)
1593                         return -ENXIO;
1594                 cxlsd->target[i] = dport;
1595         }
1596
1597         return 0;
1598 }
1599
1600 struct cxl_dport *cxl_hb_modulo(struct cxl_root_decoder *cxlrd, int pos)
1601 {
1602         struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
1603         struct cxl_decoder *cxld = &cxlsd->cxld;
1604         int iw;
1605
1606         iw = cxld->interleave_ways;
1607         if (dev_WARN_ONCE(&cxld->dev, iw != cxlsd->nr_targets,
1608                           "misconfigured root decoder\n"))
1609                 return NULL;
1610
1611         return cxlrd->cxlsd.target[pos % iw];
1612 }
1613 EXPORT_SYMBOL_NS_GPL(cxl_hb_modulo, CXL);
1614
1615 static struct lock_class_key cxl_decoder_key;
1616
1617 /**
1618  * cxl_decoder_init - Common decoder setup / initialization
1619  * @port: owning port of this decoder
1620  * @cxld: common decoder properties to initialize
1621  *
1622  * A port may contain one or more decoders. Each of those decoders
1623  * enable some address space for CXL.mem utilization. A decoder is
1624  * expected to be configured by the caller before registering via
1625  * cxl_decoder_add()
1626  */
1627 static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld)
1628 {
1629         struct device *dev;
1630         int rc;
1631
1632         rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
1633         if (rc < 0)
1634                 return rc;
1635
1636         /* need parent to stick around to release the id */
1637         get_device(&port->dev);
1638         cxld->id = rc;
1639
1640         dev = &cxld->dev;
1641         device_initialize(dev);
1642         lockdep_set_class(&dev->mutex, &cxl_decoder_key);
1643         device_set_pm_not_required(dev);
1644         dev->parent = &port->dev;
1645         dev->bus = &cxl_bus_type;
1646
1647         /* Pre initialize an "empty" decoder */
1648         cxld->interleave_ways = 1;
1649         cxld->interleave_granularity = PAGE_SIZE;
1650         cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1651         cxld->hpa_range = (struct range) {
1652                 .start = 0,
1653                 .end = -1,
1654         };
1655
1656         return 0;
1657 }
1658
1659 static int cxl_switch_decoder_init(struct cxl_port *port,
1660                                    struct cxl_switch_decoder *cxlsd,
1661                                    int nr_targets)
1662 {
1663         if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1664                 return -EINVAL;
1665
1666         cxlsd->nr_targets = nr_targets;
1667         return cxl_decoder_init(port, &cxlsd->cxld);
1668 }
1669
1670 /**
1671  * cxl_root_decoder_alloc - Allocate a root level decoder
1672  * @port: owning CXL root of this decoder
1673  * @nr_targets: static number of downstream targets
1674  * @calc_hb: which host bridge covers the n'th position by granularity
1675  *
1676  * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1677  * 'CXL root' decoder is one that decodes from a top-level / static platform
1678  * firmware description of CXL resources into a CXL standard decode
1679  * topology.
1680  */
1681 struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
1682                                                 unsigned int nr_targets,
1683                                                 cxl_calc_hb_fn calc_hb)
1684 {
1685         struct cxl_root_decoder *cxlrd;
1686         struct cxl_switch_decoder *cxlsd;
1687         struct cxl_decoder *cxld;
1688         int rc;
1689
1690         if (!is_cxl_root(port))
1691                 return ERR_PTR(-EINVAL);
1692
1693         cxlrd = kzalloc(struct_size(cxlrd, cxlsd.target, nr_targets),
1694                         GFP_KERNEL);
1695         if (!cxlrd)
1696                 return ERR_PTR(-ENOMEM);
1697
1698         cxlsd = &cxlrd->cxlsd;
1699         rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1700         if (rc) {
1701                 kfree(cxlrd);
1702                 return ERR_PTR(rc);
1703         }
1704
1705         cxlrd->calc_hb = calc_hb;
1706         mutex_init(&cxlrd->range_lock);
1707
1708         cxld = &cxlsd->cxld;
1709         cxld->dev.type = &cxl_decoder_root_type;
1710         /*
1711          * cxl_root_decoder_release() special cases negative ids to
1712          * detect memregion_alloc() failures.
1713          */
1714         atomic_set(&cxlrd->region_id, -1);
1715         rc = memregion_alloc(GFP_KERNEL);
1716         if (rc < 0) {
1717                 put_device(&cxld->dev);
1718                 return ERR_PTR(rc);
1719         }
1720
1721         atomic_set(&cxlrd->region_id, rc);
1722         return cxlrd;
1723 }
1724 EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL);
1725
1726 /**
1727  * cxl_switch_decoder_alloc - Allocate a switch level decoder
1728  * @port: owning CXL switch port of this decoder
1729  * @nr_targets: max number of dynamically addressable downstream targets
1730  *
1731  * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1732  * 'switch' decoder is any decoder that can be enumerated by PCIe
1733  * topology and the HDM Decoder Capability. This includes the decoders
1734  * that sit between Switch Upstream Ports / Switch Downstream Ports and
1735  * Host Bridges / Root Ports.
1736  */
1737 struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
1738                                                     unsigned int nr_targets)
1739 {
1740         struct cxl_switch_decoder *cxlsd;
1741         struct cxl_decoder *cxld;
1742         int rc;
1743
1744         if (is_cxl_root(port) || is_cxl_endpoint(port))
1745                 return ERR_PTR(-EINVAL);
1746
1747         cxlsd = kzalloc(struct_size(cxlsd, target, nr_targets), GFP_KERNEL);
1748         if (!cxlsd)
1749                 return ERR_PTR(-ENOMEM);
1750
1751         rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1752         if (rc) {
1753                 kfree(cxlsd);
1754                 return ERR_PTR(rc);
1755         }
1756
1757         cxld = &cxlsd->cxld;
1758         cxld->dev.type = &cxl_decoder_switch_type;
1759         return cxlsd;
1760 }
1761 EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, CXL);
1762
1763 /**
1764  * cxl_endpoint_decoder_alloc - Allocate an endpoint decoder
1765  * @port: owning port of this decoder
1766  *
1767  * Return: A new cxl decoder to be registered by cxl_decoder_add()
1768  */
1769 struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port)
1770 {
1771         struct cxl_endpoint_decoder *cxled;
1772         struct cxl_decoder *cxld;
1773         int rc;
1774
1775         if (!is_cxl_endpoint(port))
1776                 return ERR_PTR(-EINVAL);
1777
1778         cxled = kzalloc(sizeof(*cxled), GFP_KERNEL);
1779         if (!cxled)
1780                 return ERR_PTR(-ENOMEM);
1781
1782         cxled->pos = -1;
1783         cxld = &cxled->cxld;
1784         rc = cxl_decoder_init(port, cxld);
1785         if (rc)  {
1786                 kfree(cxled);
1787                 return ERR_PTR(rc);
1788         }
1789
1790         cxld->dev.type = &cxl_decoder_endpoint_type;
1791         return cxled;
1792 }
1793 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, CXL);
1794
1795 /**
1796  * cxl_decoder_add_locked - Add a decoder with targets
1797  * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1798  * @target_map: A list of downstream ports that this decoder can direct memory
1799  *              traffic to. These numbers should correspond with the port number
1800  *              in the PCIe Link Capabilities structure.
1801  *
1802  * Certain types of decoders may not have any targets. The main example of this
1803  * is an endpoint device. A more awkward example is a hostbridge whose root
1804  * ports get hot added (technically possible, though unlikely).
1805  *
1806  * This is the locked variant of cxl_decoder_add().
1807  *
1808  * Context: Process context. Expects the device lock of the port that owns the
1809  *          @cxld to be held.
1810  *
1811  * Return: Negative error code if the decoder wasn't properly configured; else
1812  *         returns 0.
1813  */
1814 int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map)
1815 {
1816         struct cxl_port *port;
1817         struct device *dev;
1818         int rc;
1819
1820         if (WARN_ON_ONCE(!cxld))
1821                 return -EINVAL;
1822
1823         if (WARN_ON_ONCE(IS_ERR(cxld)))
1824                 return PTR_ERR(cxld);
1825
1826         if (cxld->interleave_ways < 1)
1827                 return -EINVAL;
1828
1829         dev = &cxld->dev;
1830
1831         port = to_cxl_port(cxld->dev.parent);
1832         if (!is_endpoint_decoder(dev)) {
1833                 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
1834
1835                 rc = decoder_populate_targets(cxlsd, port, target_map);
1836                 if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {
1837                         dev_err(&port->dev,
1838                                 "Failed to populate active decoder targets\n");
1839                         return rc;
1840                 }
1841         }
1842
1843         rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
1844         if (rc)
1845                 return rc;
1846
1847         return device_add(dev);
1848 }
1849 EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, CXL);
1850
1851 /**
1852  * cxl_decoder_add - Add a decoder with targets
1853  * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1854  * @target_map: A list of downstream ports that this decoder can direct memory
1855  *              traffic to. These numbers should correspond with the port number
1856  *              in the PCIe Link Capabilities structure.
1857  *
1858  * This is the unlocked variant of cxl_decoder_add_locked().
1859  * See cxl_decoder_add_locked().
1860  *
1861  * Context: Process context. Takes and releases the device lock of the port that
1862  *          owns the @cxld.
1863  */
1864 int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map)
1865 {
1866         struct cxl_port *port;
1867         int rc;
1868
1869         if (WARN_ON_ONCE(!cxld))
1870                 return -EINVAL;
1871
1872         if (WARN_ON_ONCE(IS_ERR(cxld)))
1873                 return PTR_ERR(cxld);
1874
1875         port = to_cxl_port(cxld->dev.parent);
1876
1877         device_lock(&port->dev);
1878         rc = cxl_decoder_add_locked(cxld, target_map);
1879         device_unlock(&port->dev);
1880
1881         return rc;
1882 }
1883 EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, CXL);
1884
1885 static void cxld_unregister(void *dev)
1886 {
1887         struct cxl_endpoint_decoder *cxled;
1888
1889         if (is_endpoint_decoder(dev)) {
1890                 cxled = to_cxl_endpoint_decoder(dev);
1891                 cxl_decoder_kill_region(cxled);
1892         }
1893
1894         device_unregister(dev);
1895 }
1896
1897 int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
1898 {
1899         return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
1900 }
1901 EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, CXL);
1902
1903 /**
1904  * __cxl_driver_register - register a driver for the cxl bus
1905  * @cxl_drv: cxl driver structure to attach
1906  * @owner: owning module/driver
1907  * @modname: KBUILD_MODNAME for parent driver
1908  */
1909 int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
1910                           const char *modname)
1911 {
1912         if (!cxl_drv->probe) {
1913                 pr_debug("%s ->probe() must be specified\n", modname);
1914                 return -EINVAL;
1915         }
1916
1917         if (!cxl_drv->name) {
1918                 pr_debug("%s ->name must be specified\n", modname);
1919                 return -EINVAL;
1920         }
1921
1922         if (!cxl_drv->id) {
1923                 pr_debug("%s ->id must be specified\n", modname);
1924                 return -EINVAL;
1925         }
1926
1927         cxl_drv->drv.bus = &cxl_bus_type;
1928         cxl_drv->drv.owner = owner;
1929         cxl_drv->drv.mod_name = modname;
1930         cxl_drv->drv.name = cxl_drv->name;
1931
1932         return driver_register(&cxl_drv->drv);
1933 }
1934 EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, CXL);
1935
1936 void cxl_driver_unregister(struct cxl_driver *cxl_drv)
1937 {
1938         driver_unregister(&cxl_drv->drv);
1939 }
1940 EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, CXL);
1941
1942 static int cxl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
1943 {
1944         return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,
1945                               cxl_device_id(dev));
1946 }
1947
1948 static int cxl_bus_match(struct device *dev, struct device_driver *drv)
1949 {
1950         return cxl_device_id(dev) == to_cxl_drv(drv)->id;
1951 }
1952
1953 static int cxl_bus_probe(struct device *dev)
1954 {
1955         int rc;
1956
1957         rc = to_cxl_drv(dev->driver)->probe(dev);
1958         dev_dbg(dev, "probe: %d\n", rc);
1959         return rc;
1960 }
1961
1962 static void cxl_bus_remove(struct device *dev)
1963 {
1964         struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
1965
1966         if (cxl_drv->remove)
1967                 cxl_drv->remove(dev);
1968 }
1969
1970 static struct workqueue_struct *cxl_bus_wq;
1971
1972 static void cxl_bus_rescan_queue(struct work_struct *w)
1973 {
1974         int rc = bus_rescan_devices(&cxl_bus_type);
1975
1976         pr_debug("CXL bus rescan result: %d\n", rc);
1977 }
1978
1979 void cxl_bus_rescan(void)
1980 {
1981         static DECLARE_WORK(rescan_work, cxl_bus_rescan_queue);
1982
1983         queue_work(cxl_bus_wq, &rescan_work);
1984 }
1985 EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, CXL);
1986
1987 void cxl_bus_drain(void)
1988 {
1989         drain_workqueue(cxl_bus_wq);
1990 }
1991 EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, CXL);
1992
1993 bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
1994 {
1995         return queue_work(cxl_bus_wq, &cxlmd->detach_work);
1996 }
1997 EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL);
1998
1999 /* for user tooling to ensure port disable work has completed */
2000 static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count)
2001 {
2002         if (sysfs_streq(buf, "1")) {
2003                 flush_workqueue(cxl_bus_wq);
2004                 return count;
2005         }
2006
2007         return -EINVAL;
2008 }
2009
2010 static BUS_ATTR_WO(flush);
2011
2012 static struct attribute *cxl_bus_attributes[] = {
2013         &bus_attr_flush.attr,
2014         NULL,
2015 };
2016
2017 static struct attribute_group cxl_bus_attribute_group = {
2018         .attrs = cxl_bus_attributes,
2019 };
2020
2021 static const struct attribute_group *cxl_bus_attribute_groups[] = {
2022         &cxl_bus_attribute_group,
2023         NULL,
2024 };
2025
2026 struct bus_type cxl_bus_type = {
2027         .name = "cxl",
2028         .uevent = cxl_bus_uevent,
2029         .match = cxl_bus_match,
2030         .probe = cxl_bus_probe,
2031         .remove = cxl_bus_remove,
2032         .bus_groups = cxl_bus_attribute_groups,
2033 };
2034 EXPORT_SYMBOL_NS_GPL(cxl_bus_type, CXL);
2035
2036 static struct dentry *cxl_debugfs;
2037
2038 struct dentry *cxl_debugfs_create_dir(const char *dir)
2039 {
2040         return debugfs_create_dir(dir, cxl_debugfs);
2041 }
2042 EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, CXL);
2043
2044 static __init int cxl_core_init(void)
2045 {
2046         int rc;
2047
2048         cxl_debugfs = debugfs_create_dir("cxl", NULL);
2049
2050         cxl_mbox_init();
2051
2052         rc = cxl_memdev_init();
2053         if (rc)
2054                 return rc;
2055
2056         cxl_bus_wq = alloc_ordered_workqueue("cxl_port", 0);
2057         if (!cxl_bus_wq) {
2058                 rc = -ENOMEM;
2059                 goto err_wq;
2060         }
2061
2062         rc = bus_register(&cxl_bus_type);
2063         if (rc)
2064                 goto err_bus;
2065
2066         rc = cxl_region_init();
2067         if (rc)
2068                 goto err_region;
2069
2070         return 0;
2071
2072 err_region:
2073         bus_unregister(&cxl_bus_type);
2074 err_bus:
2075         destroy_workqueue(cxl_bus_wq);
2076 err_wq:
2077         cxl_memdev_exit();
2078         return rc;
2079 }
2080
2081 static void cxl_core_exit(void)
2082 {
2083         cxl_region_exit();
2084         bus_unregister(&cxl_bus_type);
2085         destroy_workqueue(cxl_bus_wq);
2086         cxl_memdev_exit();
2087         debugfs_remove_recursive(cxl_debugfs);
2088 }
2089
2090 subsys_initcall(cxl_core_init);
2091 module_exit(cxl_core_exit);
2092 MODULE_LICENSE("GPL v2");