drivers/cxl/core/port.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
   3 #include <linux/memregion.h>
   4 #include <linux/workqueue.h>
   5 #include <linux/debugfs.h>
   6 #include <linux/device.h>
   7 #include <linux/module.h>
   8 #include <linux/pci.h>
   9 #include <linux/slab.h>
  10 #include <linux/idr.h>
  11 #include <cxlmem.h>
  12 #include <cxlpci.h>
  13 #include <cxl.h>
  14 #include "core.h"
  15
  16 /**
  17  * DOC: cxl core
  18  *
  19  * The CXL core provides a set of interfaces that can be consumed by CXL aware
  20  * drivers. The interfaces allow for creation, modification, and destruction of
  21  * regions, memory devices, ports, and decoders. CXL aware drivers must register
  22  * with the CXL core via these interfaces in order to be able to participate in
  23  * cross-device interleave coordination. The CXL core also establishes and
  24  * maintains the bridge to the nvdimm subsystem.
  25  *
  26  * CXL core introduces sysfs hierarchy to control the devices that are
  27  * instantiated by the core.
  28  */
  29
  30 static DEFINE_IDA(cxl_port_ida);
  31 static DEFINE_XARRAY(cxl_root_buses);
  32
  33 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
  34                             char *buf)
  35 {
  36         return sysfs_emit(buf, "%s\n", dev->type->name);
  37 }
  38 static DEVICE_ATTR_RO(devtype);
  39
  40 static int cxl_device_id(const struct device *dev)
  41 {
  42         if (dev->type == &cxl_nvdimm_bridge_type)
  43                 return CXL_DEVICE_NVDIMM_BRIDGE;
  44         if (dev->type == &cxl_nvdimm_type)
  45                 return CXL_DEVICE_NVDIMM;
  46         if (dev->type == CXL_PMEM_REGION_TYPE())
  47                 return CXL_DEVICE_PMEM_REGION;
  48         if (dev->type == CXL_DAX_REGION_TYPE())
  49                 return CXL_DEVICE_DAX_REGION;
  50         if (is_cxl_port(dev)) {
  51                 if (is_cxl_root(to_cxl_port(dev)))
  52                         return CXL_DEVICE_ROOT;
  53                 return CXL_DEVICE_PORT;
  54         }
  55         if (is_cxl_memdev(dev))
  56                 return CXL_DEVICE_MEMORY_EXPANDER;
  57         if (dev->type == CXL_REGION_TYPE())
  58                 return CXL_DEVICE_REGION;
  59         if (dev->type == &cxl_pmu_type)
  60                 return CXL_DEVICE_PMU;
  61         return 0;
  62 }
  63
  64 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
  65                              char *buf)
  66 {
  67         return sysfs_emit(buf, CXL_MODALIAS_FMT "\n", cxl_device_id(dev));
  68 }
  69 static DEVICE_ATTR_RO(modalias);
  70
  71 static struct attribute *cxl_base_attributes[] = {
  72         &dev_attr_devtype.attr,
  73         &dev_attr_modalias.attr,
  74         NULL,
  75 };
  76
  77 struct attribute_group cxl_base_attribute_group = {
  78         .attrs = cxl_base_attributes,
  79 };
  80
  81 static ssize_t start_show(struct device *dev, struct device_attribute *attr,
  82                           char *buf)
  83 {
  84         struct cxl_decoder *cxld = to_cxl_decoder(dev);
  85
  86         return sysfs_emit(buf, "%#llx\n", cxld->hpa_range.start);
  87 }
  88 static DEVICE_ATTR_ADMIN_RO(start);
  89
  90 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
  91                         char *buf)
  92 {
  93         struct cxl_decoder *cxld = to_cxl_decoder(dev);
  94
  95         return sysfs_emit(buf, "%#llx\n", range_len(&cxld->hpa_range));
  96 }
  97 static DEVICE_ATTR_RO(size);
  98
  99 #define CXL_DECODER_FLAG_ATTR(name, flag)                            \
 100 static ssize_t name##_show(struct device *dev,                       \
 101                            struct device_attribute *attr, char *buf) \
 102 {                                                                    \
 103         struct cxl_decoder *cxld = to_cxl_decoder(dev);              \
 104                                                                      \
 105         return sysfs_emit(buf, "%s\n",                               \
 106                           (cxld->flags & (flag)) ? "1" : "0");       \
 107 }                                                                    \
 108 static DEVICE_ATTR_RO(name)
 109
 110 CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
 111 CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
 112 CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
 113 CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
 114 CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
 115
 116 static ssize_t target_type_show(struct device *dev,
 117                                 struct device_attribute *attr, char *buf)
 118 {
 119         struct cxl_decoder *cxld = to_cxl_decoder(dev);
 120
 121         switch (cxld->target_type) {
 122         case CXL_DECODER_DEVMEM:
 123                 return sysfs_emit(buf, "accelerator\n");
 124         case CXL_DECODER_HOSTONLYMEM:
 125                 return sysfs_emit(buf, "expander\n");
 126         }
 127         return -ENXIO;
 128 }
 129 static DEVICE_ATTR_RO(target_type);
 130
 131 static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf)
 132 {
 133         struct cxl_decoder *cxld = &cxlsd->cxld;
 134         ssize_t offset = 0;
 135         int i, rc = 0;
 136
 137         for (i = 0; i < cxld->interleave_ways; i++) {
 138                 struct cxl_dport *dport = cxlsd->target[i];
 139                 struct cxl_dport *next = NULL;
 140
 141                 if (!dport)
 142                         break;
 143
 144                 if (i + 1 < cxld->interleave_ways)
 145                         next = cxlsd->target[i + 1];
 146                 rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
 147                                    next ? "," : "");
 148                 if (rc < 0)
 149                         return rc;
 150                 offset += rc;
 151         }
 152
 153         return offset;
 154 }
 155
 156 static ssize_t target_list_show(struct device *dev,
 157                                 struct device_attribute *attr, char *buf)
 158 {
 159         struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
 160         ssize_t offset;
 161         unsigned int seq;
 162         int rc;
 163
 164         do {
 165                 seq = read_seqbegin(&cxlsd->target_lock);
 166                 rc = emit_target_list(cxlsd, buf);
 167         } while (read_seqretry(&cxlsd->target_lock, seq));
 168
 169         if (rc < 0)
 170                 return rc;
 171         offset = rc;
 172
 173         rc = sysfs_emit_at(buf, offset, "\n");
 174         if (rc < 0)
 175                 return rc;
 176
 177         return offset + rc;
 178 }
 179 static DEVICE_ATTR_RO(target_list);
 180
 181 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
 182                          char *buf)
 183 {
 184         struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 185
 186         return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxled->mode));
 187 }
 188
 189 static ssize_t mode_store(struct device *dev, struct device_attribute *attr,
 190                           const char *buf, size_t len)
 191 {
 192         struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 193         enum cxl_decoder_mode mode;
 194         ssize_t rc;
 195
 196         if (sysfs_streq(buf, "pmem"))
 197                 mode = CXL_DECODER_PMEM;
 198         else if (sysfs_streq(buf, "ram"))
 199                 mode = CXL_DECODER_RAM;
 200         else
 201                 return -EINVAL;
 202
 203         rc = cxl_dpa_set_mode(cxled, mode);
 204         if (rc)
 205                 return rc;
 206
 207         return len;
 208 }
 209 static DEVICE_ATTR_RW(mode);
 210
 211 static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *attr,
 212                             char *buf)
 213 {
 214         struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 215         u64 base = cxl_dpa_resource_start(cxled);
 216
 217         return sysfs_emit(buf, "%#llx\n", base);
 218 }
 219 static DEVICE_ATTR_RO(dpa_resource);
 220
 221 static ssize_t dpa_size_show(struct device *dev, struct device_attribute *attr,
 222                              char *buf)
 223 {
 224         struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 225         resource_size_t size = cxl_dpa_size(cxled);
 226
 227         return sysfs_emit(buf, "%pa\n", &size);
 228 }
 229
 230 static ssize_t dpa_size_store(struct device *dev, struct device_attribute *attr,
 231                               const char *buf, size_t len)
 232 {
 233         struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 234         unsigned long long size;
 235         ssize_t rc;
 236
 237         rc = kstrtoull(buf, 0, &size);
 238         if (rc)
 239                 return rc;
 240
 241         if (!IS_ALIGNED(size, SZ_256M))
 242                 return -EINVAL;
 243
 244         rc = cxl_dpa_free(cxled);
 245         if (rc)
 246                 return rc;
 247
 248         if (size == 0)
 249                 return len;
 250
 251         rc = cxl_dpa_alloc(cxled, size);
 252         if (rc)
 253                 return rc;
 254
 255         return len;
 256 }
 257 static DEVICE_ATTR_RW(dpa_size);
 258
 259 static ssize_t interleave_granularity_show(struct device *dev,
 260                                            struct device_attribute *attr,
 261                                            char *buf)
 262 {
 263         struct cxl_decoder *cxld = to_cxl_decoder(dev);
 264
 265         return sysfs_emit(buf, "%d\n", cxld->interleave_granularity);
 266 }
 267
 268 static DEVICE_ATTR_RO(interleave_granularity);
 269
 270 static ssize_t interleave_ways_show(struct device *dev,
 271                                     struct device_attribute *attr, char *buf)
 272 {
 273         struct cxl_decoder *cxld = to_cxl_decoder(dev);
 274
 275         return sysfs_emit(buf, "%d\n", cxld->interleave_ways);
 276 }
 277
 278 static DEVICE_ATTR_RO(interleave_ways);
 279
 280 static struct attribute *cxl_decoder_base_attrs[] = {
 281         &dev_attr_start.attr,
 282         &dev_attr_size.attr,
 283         &dev_attr_locked.attr,
 284         &dev_attr_interleave_granularity.attr,
 285         &dev_attr_interleave_ways.attr,
 286         NULL,
 287 };
 288
 289 static struct attribute_group cxl_decoder_base_attribute_group = {
 290         .attrs = cxl_decoder_base_attrs,
 291 };
 292
 293 static struct attribute *cxl_decoder_root_attrs[] = {
 294         &dev_attr_cap_pmem.attr,
 295         &dev_attr_cap_ram.attr,
 296         &dev_attr_cap_type2.attr,
 297         &dev_attr_cap_type3.attr,
 298         &dev_attr_target_list.attr,
 299         SET_CXL_REGION_ATTR(create_pmem_region)
 300         SET_CXL_REGION_ATTR(create_ram_region)
 301         SET_CXL_REGION_ATTR(delete_region)
 302         NULL,
 303 };
 304
 305 static bool can_create_pmem(struct cxl_root_decoder *cxlrd)
 306 {
 307         unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_PMEM;
 308
 309         return (cxlrd->cxlsd.cxld.flags & flags) == flags;
 310 }
 311
 312 static bool can_create_ram(struct cxl_root_decoder *cxlrd)
 313 {
 314         unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_RAM;
 315
 316         return (cxlrd->cxlsd.cxld.flags & flags) == flags;
 317 }
 318
 319 static umode_t cxl_root_decoder_visible(struct kobject *kobj, struct attribute *a, int n)
 320 {
 321         struct device *dev = kobj_to_dev(kobj);
 322         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
 323
 324         if (a == CXL_REGION_ATTR(create_pmem_region) && !can_create_pmem(cxlrd))
 325                 return 0;
 326
 327         if (a == CXL_REGION_ATTR(create_ram_region) && !can_create_ram(cxlrd))
 328                 return 0;
 329
 330         if (a == CXL_REGION_ATTR(delete_region) &&
 331             !(can_create_pmem(cxlrd) || can_create_ram(cxlrd)))
 332                 return 0;
 333
 334         return a->mode;
 335 }
 336
 337 static struct attribute_group cxl_decoder_root_attribute_group = {
 338         .attrs = cxl_decoder_root_attrs,
 339         .is_visible = cxl_root_decoder_visible,
 340 };
 341
 342 static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
 343         &cxl_decoder_root_attribute_group,
 344         &cxl_decoder_base_attribute_group,
 345         &cxl_base_attribute_group,
 346         NULL,
 347 };
 348
 349 static struct attribute *cxl_decoder_switch_attrs[] = {
 350         &dev_attr_target_type.attr,
 351         &dev_attr_target_list.attr,
 352         SET_CXL_REGION_ATTR(region)
 353         NULL,
 354 };
 355
 356 static struct attribute_group cxl_decoder_switch_attribute_group = {
 357         .attrs = cxl_decoder_switch_attrs,
 358 };
 359
 360 static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
 361         &cxl_decoder_switch_attribute_group,
 362         &cxl_decoder_base_attribute_group,
 363         &cxl_base_attribute_group,
 364         NULL,
 365 };
 366
 367 static struct attribute *cxl_decoder_endpoint_attrs[] = {
 368         &dev_attr_target_type.attr,
 369         &dev_attr_mode.attr,
 370         &dev_attr_dpa_size.attr,
 371         &dev_attr_dpa_resource.attr,
 372         SET_CXL_REGION_ATTR(region)
 373         NULL,
 374 };
 375
 376 static struct attribute_group cxl_decoder_endpoint_attribute_group = {
 377         .attrs = cxl_decoder_endpoint_attrs,
 378 };
 379
 380 static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = {
 381         &cxl_decoder_base_attribute_group,
 382         &cxl_decoder_endpoint_attribute_group,
 383         &cxl_base_attribute_group,
 384         NULL,
 385 };
 386
 387 static void __cxl_decoder_release(struct cxl_decoder *cxld)
 388 {
 389         struct cxl_port *port = to_cxl_port(cxld->dev.parent);
 390
 391         ida_free(&port->decoder_ida, cxld->id);
 392         put_device(&port->dev);
 393 }
 394
 395 static void cxl_endpoint_decoder_release(struct device *dev)
 396 {
 397         struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 398
 399         __cxl_decoder_release(&cxled->cxld);
 400         kfree(cxled);
 401 }
 402
 403 static void cxl_switch_decoder_release(struct device *dev)
 404 {
 405         struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
 406
 407         __cxl_decoder_release(&cxlsd->cxld);
 408         kfree(cxlsd);
 409 }
 410
 411 struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev)
 412 {
 413         if (dev_WARN_ONCE(dev, !is_root_decoder(dev),
 414                           "not a cxl_root_decoder device\n"))
 415                 return NULL;
 416         return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev);
 417 }
 418 EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, CXL);
 419
 420 static void cxl_root_decoder_release(struct device *dev)
 421 {
 422         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
 423
 424         if (atomic_read(&cxlrd->region_id) >= 0)
 425                 memregion_free(atomic_read(&cxlrd->region_id));
 426         __cxl_decoder_release(&cxlrd->cxlsd.cxld);
 427         kfree(cxlrd);
 428 }
 429
 430 static const struct device_type cxl_decoder_endpoint_type = {
 431         .name = "cxl_decoder_endpoint",
 432         .release = cxl_endpoint_decoder_release,
 433         .groups = cxl_decoder_endpoint_attribute_groups,
 434 };
 435
 436 static const struct device_type cxl_decoder_switch_type = {
 437         .name = "cxl_decoder_switch",
 438         .release = cxl_switch_decoder_release,
 439         .groups = cxl_decoder_switch_attribute_groups,
 440 };
 441
 442 static const struct device_type cxl_decoder_root_type = {
 443         .name = "cxl_decoder_root",
 444         .release = cxl_root_decoder_release,
 445         .groups = cxl_decoder_root_attribute_groups,
 446 };
 447
 448 bool is_endpoint_decoder(struct device *dev)
 449 {
 450         return dev->type == &cxl_decoder_endpoint_type;
 451 }
 452 EXPORT_SYMBOL_NS_GPL(is_endpoint_decoder, CXL);
 453
 454 bool is_root_decoder(struct device *dev)
 455 {
 456         return dev->type == &cxl_decoder_root_type;
 457 }
 458 EXPORT_SYMBOL_NS_GPL(is_root_decoder, CXL);
 459
 460 bool is_switch_decoder(struct device *dev)
 461 {
 462         return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type;
 463 }
 464 EXPORT_SYMBOL_NS_GPL(is_switch_decoder, CXL);
 465
 466 struct cxl_decoder *to_cxl_decoder(struct device *dev)
 467 {
 468         if (dev_WARN_ONCE(dev,
 469                           !is_switch_decoder(dev) && !is_endpoint_decoder(dev),
 470                           "not a cxl_decoder device\n"))
 471                 return NULL;
 472         return container_of(dev, struct cxl_decoder, dev);
 473 }
 474 EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, CXL);
 475
 476 struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev)
 477 {
 478         if (dev_WARN_ONCE(dev, !is_endpoint_decoder(dev),
 479                           "not a cxl_endpoint_decoder device\n"))
 480                 return NULL;
 481         return container_of(dev, struct cxl_endpoint_decoder, cxld.dev);
 482 }
 483 EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, CXL);
 484
 485 struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev)
 486 {
 487         if (dev_WARN_ONCE(dev, !is_switch_decoder(dev),
 488                           "not a cxl_switch_decoder device\n"))
 489                 return NULL;
 490         return container_of(dev, struct cxl_switch_decoder, cxld.dev);
 491 }
 492 EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, CXL);
 493
 494 static void cxl_ep_release(struct cxl_ep *ep)
 495 {
 496         put_device(ep->ep);
 497         kfree(ep);
 498 }
 499
 500 static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep)
 501 {
 502         if (!ep)
 503                 return;
 504         xa_erase(&port->endpoints, (unsigned long) ep->ep);
 505         cxl_ep_release(ep);
 506 }
 507
 508 static void cxl_port_release(struct device *dev)
 509 {
 510         struct cxl_port *port = to_cxl_port(dev);
 511         unsigned long index;
 512         struct cxl_ep *ep;
 513
 514         xa_for_each(&port->endpoints, index, ep)
 515                 cxl_ep_remove(port, ep);
 516         xa_destroy(&port->endpoints);
 517         xa_destroy(&port->dports);
 518         xa_destroy(&port->regions);
 519         ida_free(&cxl_port_ida, port->id);
 520         kfree(port);
 521 }
 522
 523 static const struct attribute_group *cxl_port_attribute_groups[] = {
 524         &cxl_base_attribute_group,
 525         NULL,
 526 };
 527
 528 static const struct device_type cxl_port_type = {
 529         .name = "cxl_port",
 530         .release = cxl_port_release,
 531         .groups = cxl_port_attribute_groups,
 532 };
 533
 534 bool is_cxl_port(const struct device *dev)
 535 {
 536         return dev->type == &cxl_port_type;
 537 }
 538 EXPORT_SYMBOL_NS_GPL(is_cxl_port, CXL);
 539
 540 struct cxl_port *to_cxl_port(const struct device *dev)
 541 {
 542         if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
 543                           "not a cxl_port device\n"))
 544                 return NULL;
 545         return container_of(dev, struct cxl_port, dev);
 546 }
 547 EXPORT_SYMBOL_NS_GPL(to_cxl_port, CXL);
 548
 549 static void unregister_port(void *_port)
 550 {
 551         struct cxl_port *port = _port;
 552         struct cxl_port *parent;
 553         struct device *lock_dev;
 554
 555         if (is_cxl_root(port))
 556                 parent = NULL;
 557         else
 558                 parent = to_cxl_port(port->dev.parent);
 559
 560         /*
 561          * CXL root port's and the first level of ports are unregistered
 562          * under the platform firmware device lock, all other ports are
 563          * unregistered while holding their parent port lock.
 564          */
 565         if (!parent)
 566                 lock_dev = port->uport_dev;
 567         else if (is_cxl_root(parent))
 568                 lock_dev = parent->uport_dev;
 569         else
 570                 lock_dev = &parent->dev;
 571
 572         device_lock_assert(lock_dev);
 573         port->dead = true;
 574         device_unregister(&port->dev);
 575 }
 576
 577 static void cxl_unlink_uport(void *_port)
 578 {
 579         struct cxl_port *port = _port;
 580
 581         sysfs_remove_link(&port->dev.kobj, "uport");
 582 }
 583
 584 static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
 585 {
 586         int rc;
 587
 588         rc = sysfs_create_link(&port->dev.kobj, &port->uport_dev->kobj,
 589                                "uport");
 590         if (rc)
 591                 return rc;
 592         return devm_add_action_or_reset(host, cxl_unlink_uport, port);
 593 }
 594
 595 static void cxl_unlink_parent_dport(void *_port)
 596 {
 597         struct cxl_port *port = _port;
 598
 599         sysfs_remove_link(&port->dev.kobj, "parent_dport");
 600 }
 601
 602 static int devm_cxl_link_parent_dport(struct device *host,
 603                                       struct cxl_port *port,
 604                                       struct cxl_dport *parent_dport)
 605 {
 606         int rc;
 607
 608         if (!parent_dport)
 609                 return 0;
 610
 611         rc = sysfs_create_link(&port->dev.kobj, &parent_dport->dport_dev->kobj,
 612                                "parent_dport");
 613         if (rc)
 614                 return rc;
 615         return devm_add_action_or_reset(host, cxl_unlink_parent_dport, port);
 616 }
 617
 618 static struct lock_class_key cxl_port_key;
 619
 620 static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
 621                                        resource_size_t component_reg_phys,
 622                                        struct cxl_dport *parent_dport)
 623 {
 624         struct cxl_port *port;
 625         struct device *dev;
 626         int rc;
 627
 628         port = kzalloc(sizeof(*port), GFP_KERNEL);
 629         if (!port)
 630                 return ERR_PTR(-ENOMEM);
 631
 632         rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
 633         if (rc < 0)
 634                 goto err;
 635         port->id = rc;
 636         port->uport_dev = uport_dev;
 637
 638         /*
 639          * The top-level cxl_port "cxl_root" does not have a cxl_port as
 640          * its parent and it does not have any corresponding component
 641          * registers as its decode is described by a fixed platform
 642          * description.
 643          */
 644         dev = &port->dev;
 645         if (parent_dport) {
 646                 struct cxl_port *parent_port = parent_dport->port;
 647                 struct cxl_port *iter;
 648
 649                 dev->parent = &parent_port->dev;
 650                 port->depth = parent_port->depth + 1;
 651                 port->parent_dport = parent_dport;
 652
 653                 /*
 654                  * walk to the host bridge, or the first ancestor that knows
 655                  * the host bridge
 656                  */
 657                 iter = port;
 658                 while (!iter->host_bridge &&
 659                        !is_cxl_root(to_cxl_port(iter->dev.parent)))
 660                         iter = to_cxl_port(iter->dev.parent);
 661                 if (iter->host_bridge)
 662                         port->host_bridge = iter->host_bridge;
 663                 else if (parent_dport->rch)
 664                         port->host_bridge = parent_dport->dport_dev;
 665                 else
 666                         port->host_bridge = iter->uport_dev;
 667                 dev_dbg(uport_dev, "host-bridge: %s\n",
 668                         dev_name(port->host_bridge));
 669         } else
 670                 dev->parent = uport_dev;
 671
 672         port->component_reg_phys = component_reg_phys;
 673         ida_init(&port->decoder_ida);
 674         port->hdm_end = -1;
 675         port->commit_end = -1;
 676         xa_init(&port->dports);
 677         xa_init(&port->endpoints);
 678         xa_init(&port->regions);
 679
 680         device_initialize(dev);
 681         lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
 682         device_set_pm_not_required(dev);
 683         dev->bus = &cxl_bus_type;
 684         dev->type = &cxl_port_type;
 685
 686         return port;
 687
 688 err:
 689         kfree(port);
 690         return ERR_PTR(rc);
 691 }
 692
 693 static int cxl_setup_comp_regs(struct device *dev, struct cxl_register_map *map,
 694                                resource_size_t component_reg_phys)
 695 {
 696         if (component_reg_phys == CXL_RESOURCE_NONE)
 697                 return 0;
 698
 699         *map = (struct cxl_register_map) {
 700                 .dev = dev,
 701                 .reg_type = CXL_REGLOC_RBI_COMPONENT,
 702                 .resource = component_reg_phys,
 703                 .max_size = CXL_COMPONENT_REG_BLOCK_SIZE,
 704         };
 705
 706         return cxl_setup_regs(map);
 707 }
 708
 709 static inline int cxl_port_setup_regs(struct cxl_port *port,
 710                                       resource_size_t component_reg_phys)
 711 {
 712         return cxl_setup_comp_regs(&port->dev, &port->comp_map,
 713                                    component_reg_phys);
 714 }
 715
 716 static inline int cxl_dport_setup_regs(struct cxl_dport *dport,
 717                                        resource_size_t component_reg_phys)
 718 {
 719         return cxl_setup_comp_regs(dport->dport_dev, &dport->comp_map,
 720                                    component_reg_phys);
 721 }
 722
 723 static struct cxl_port *__devm_cxl_add_port(struct device *host,
 724                                             struct device *uport_dev,
 725                                             resource_size_t component_reg_phys,
 726                                             struct cxl_dport *parent_dport)
 727 {
 728         struct cxl_port *port;
 729         struct device *dev;
 730         int rc;
 731
 732         port = cxl_port_alloc(uport_dev, component_reg_phys, parent_dport);
 733         if (IS_ERR(port))
 734                 return port;
 735
 736         dev = &port->dev;
 737         if (is_cxl_memdev(uport_dev))
 738                 rc = dev_set_name(dev, "endpoint%d", port->id);
 739         else if (parent_dport)
 740                 rc = dev_set_name(dev, "port%d", port->id);
 741         else
 742                 rc = dev_set_name(dev, "root%d", port->id);
 743         if (rc)
 744                 goto err;
 745
 746         rc = cxl_port_setup_regs(port, component_reg_phys);
 747         if (rc)
 748                 goto err;
 749
 750         rc = device_add(dev);
 751         if (rc)
 752                 goto err;
 753
 754         rc = devm_add_action_or_reset(host, unregister_port, port);
 755         if (rc)
 756                 return ERR_PTR(rc);
 757
 758         rc = devm_cxl_link_uport(host, port);
 759         if (rc)
 760                 return ERR_PTR(rc);
 761
 762         rc = devm_cxl_link_parent_dport(host, port, parent_dport);
 763         if (rc)
 764                 return ERR_PTR(rc);
 765
 766         return port;
 767
 768 err:
 769         put_device(dev);
 770         return ERR_PTR(rc);
 771 }
 772
 773 /**
 774  * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
 775  * @host: host device for devm operations
 776  * @uport_dev: "physical" device implementing this upstream port
 777  * @component_reg_phys: (optional) for configurable cxl_port instances
 778  * @parent_dport: next hop up in the CXL memory decode hierarchy
 779  */
 780 struct cxl_port *devm_cxl_add_port(struct device *host,
 781                                    struct device *uport_dev,
 782                                    resource_size_t component_reg_phys,
 783                                    struct cxl_dport *parent_dport)
 784 {
 785         struct cxl_port *port, *parent_port;
 786
 787         port = __devm_cxl_add_port(host, uport_dev, component_reg_phys,
 788                                    parent_dport);
 789
 790         parent_port = parent_dport ? parent_dport->port : NULL;
 791         if (IS_ERR(port)) {
 792                 dev_dbg(uport_dev, "Failed to add%s%s%s: %ld\n",
 793                         parent_port ? " port to " : "",
 794                         parent_port ? dev_name(&parent_port->dev) : "",
 795                         parent_port ? "" : " root port",
 796                         PTR_ERR(port));
 797         } else {
 798                 dev_dbg(uport_dev, "%s added%s%s%s\n",
 799                         dev_name(&port->dev),
 800                         parent_port ? " to " : "",
 801                         parent_port ? dev_name(&parent_port->dev) : "",
 802                         parent_port ? "" : " (root port)");
 803         }
 804
 805         return port;
 806 }
 807 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, CXL);
 808
 809 struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port)
 810 {
 811         /* There is no pci_bus associated with a CXL platform-root port */
 812         if (is_cxl_root(port))
 813                 return NULL;
 814
 815         if (dev_is_pci(port->uport_dev)) {
 816                 struct pci_dev *pdev = to_pci_dev(port->uport_dev);
 817
 818                 return pdev->subordinate;
 819         }
 820
 821         return xa_load(&cxl_root_buses, (unsigned long)port->uport_dev);
 822 }
 823 EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, CXL);
 824
 825 static void unregister_pci_bus(void *uport_dev)
 826 {
 827         xa_erase(&cxl_root_buses, (unsigned long)uport_dev);
 828 }
 829
 830 int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,
 831                               struct pci_bus *bus)
 832 {
 833         int rc;
 834
 835         if (dev_is_pci(uport_dev))
 836                 return -EINVAL;
 837
 838         rc = xa_insert(&cxl_root_buses, (unsigned long)uport_dev, bus,
 839                        GFP_KERNEL);
 840         if (rc)
 841                 return rc;
 842         return devm_add_action_or_reset(host, unregister_pci_bus, uport_dev);
 843 }
 844 EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, CXL);
 845
 846 static bool dev_is_cxl_root_child(struct device *dev)
 847 {
 848         struct cxl_port *port, *parent;
 849
 850         if (!is_cxl_port(dev))
 851                 return false;
 852
 853         port = to_cxl_port(dev);
 854         if (is_cxl_root(port))
 855                 return false;
 856
 857         parent = to_cxl_port(port->dev.parent);
 858         if (is_cxl_root(parent))
 859                 return true;
 860
 861         return false;
 862 }
 863
 864 struct cxl_port *find_cxl_root(struct cxl_port *port)
 865 {
 866         struct cxl_port *iter = port;
 867
 868         while (iter && !is_cxl_root(iter))
 869                 iter = to_cxl_port(iter->dev.parent);
 870
 871         if (!iter)
 872                 return NULL;
 873         get_device(&iter->dev);
 874         return iter;
 875 }
 876 EXPORT_SYMBOL_NS_GPL(find_cxl_root, CXL);
 877
 878 static struct cxl_dport *find_dport(struct cxl_port *port, int id)
 879 {
 880         struct cxl_dport *dport;
 881         unsigned long index;
 882
 883         device_lock_assert(&port->dev);
 884         xa_for_each(&port->dports, index, dport)
 885                 if (dport->port_id == id)
 886                         return dport;
 887         return NULL;
 888 }
 889
 890 static int add_dport(struct cxl_port *port, struct cxl_dport *dport)
 891 {
 892         struct cxl_dport *dup;
 893         int rc;
 894
 895         device_lock_assert(&port->dev);
 896         dup = find_dport(port, dport->port_id);
 897         if (dup) {
 898                 dev_err(&port->dev,
 899                         "unable to add dport%d-%s non-unique port id (%s)\n",
 900                         dport->port_id, dev_name(dport->dport_dev),
 901                         dev_name(dup->dport_dev));
 902                 return -EBUSY;
 903         }
 904
 905         rc = xa_insert(&port->dports, (unsigned long)dport->dport_dev, dport,
 906                        GFP_KERNEL);
 907         if (rc)
 908                 return rc;
 909
 910         port->nr_dports++;
 911         return 0;
 912 }
 913
 914 /*
 915  * Since root-level CXL dports cannot be enumerated by PCI they are not
 916  * enumerated by the common port driver that acquires the port lock over
 917  * dport add/remove. Instead, root dports are manually added by a
 918  * platform driver and cond_cxl_root_lock() is used to take the missing
 919  * port lock in that case.
 920  */
 921 static void cond_cxl_root_lock(struct cxl_port *port)
 922 {
 923         if (is_cxl_root(port))
 924                 device_lock(&port->dev);
 925 }
 926
 927 static void cond_cxl_root_unlock(struct cxl_port *port)
 928 {
 929         if (is_cxl_root(port))
 930                 device_unlock(&port->dev);
 931 }
 932
 933 static void cxl_dport_remove(void *data)
 934 {
 935         struct cxl_dport *dport = data;
 936         struct cxl_port *port = dport->port;
 937
 938         xa_erase(&port->dports, (unsigned long) dport->dport_dev);
 939         put_device(dport->dport_dev);
 940 }
 941
 942 static void cxl_dport_unlink(void *data)
 943 {
 944         struct cxl_dport *dport = data;
 945         struct cxl_port *port = dport->port;
 946         char link_name[CXL_TARGET_STRLEN];
 947
 948         sprintf(link_name, "dport%d", dport->port_id);
 949         sysfs_remove_link(&port->dev.kobj, link_name);
 950 }
 951
 952 static struct cxl_dport *
 953 __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
 954                      int port_id, resource_size_t component_reg_phys,
 955                      resource_size_t rcrb)
 956 {
 957         char link_name[CXL_TARGET_STRLEN];
 958         struct cxl_dport *dport;
 959         struct device *host;
 960         int rc;
 961
 962         if (is_cxl_root(port))
 963                 host = port->uport_dev;
 964         else
 965                 host = &port->dev;
 966
 967         if (!host->driver) {
 968                 dev_WARN_ONCE(&port->dev, 1, "dport:%s bad devm context\n",
 969                               dev_name(dport_dev));
 970                 return ERR_PTR(-ENXIO);
 971         }
 972
 973         if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
 974             CXL_TARGET_STRLEN)
 975                 return ERR_PTR(-EINVAL);
 976
 977         dport = devm_kzalloc(host, sizeof(*dport), GFP_KERNEL);
 978         if (!dport)
 979                 return ERR_PTR(-ENOMEM);
 980
 981         if (rcrb != CXL_RESOURCE_NONE) {
 982                 dport->rcrb.base = rcrb;
 983                 component_reg_phys = __rcrb_to_component(dport_dev, &dport->rcrb,
 984                                                          CXL_RCRB_DOWNSTREAM);
 985                 if (component_reg_phys == CXL_RESOURCE_NONE) {
 986                         dev_warn(dport_dev, "Invalid Component Registers in RCRB");
 987                         return ERR_PTR(-ENXIO);
 988                 }
 989
 990                 dport->rch = true;
 991         }
 992
 993         if (component_reg_phys != CXL_RESOURCE_NONE)
 994                 dev_dbg(dport_dev, "Component Registers found for dport: %pa\n",
 995                         &component_reg_phys);
 996
 997         dport->dport_dev = dport_dev;
 998         dport->port_id = port_id;
 999         dport->port = port;
1000
1001         rc = cxl_dport_setup_regs(dport, component_reg_phys);
1002         if (rc)
1003                 return ERR_PTR(rc);
1004
1005         cond_cxl_root_lock(port);
1006         rc = add_dport(port, dport);
1007         cond_cxl_root_unlock(port);
1008         if (rc)
1009                 return ERR_PTR(rc);
1010
1011         get_device(dport_dev);
1012         rc = devm_add_action_or_reset(host, cxl_dport_remove, dport);
1013         if (rc)
1014                 return ERR_PTR(rc);
1015
1016         rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
1017         if (rc)
1018                 return ERR_PTR(rc);
1019
1020         rc = devm_add_action_or_reset(host, cxl_dport_unlink, dport);
1021         if (rc)
1022                 return ERR_PTR(rc);
1023
1024         return dport;
1025 }
1026
1027 /**
1028  * devm_cxl_add_dport - append VH downstream port data to a cxl_port
1029  * @port: the cxl_port that references this dport
1030  * @dport_dev: firmware or PCI device representing the dport
1031  * @port_id: identifier for this dport in a decoder's target list
1032  * @component_reg_phys: optional location of CXL component registers
1033  *
1034  * Note that dports are appended to the devm release action's of the
1035  * either the port's host (for root ports), or the port itself (for
1036  * switch ports)
1037  */
1038 struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
1039                                      struct device *dport_dev, int port_id,
1040                                      resource_size_t component_reg_phys)
1041 {
1042         struct cxl_dport *dport;
1043
1044         dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1045                                      component_reg_phys, CXL_RESOURCE_NONE);
1046         if (IS_ERR(dport)) {
1047                 dev_dbg(dport_dev, "failed to add dport to %s: %ld\n",
1048                         dev_name(&port->dev), PTR_ERR(dport));
1049         } else {
1050                 dev_dbg(dport_dev, "dport added to %s\n",
1051                         dev_name(&port->dev));
1052         }
1053
1054         return dport;
1055 }
1056 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL);
1057
1058 /**
1059  * devm_cxl_add_rch_dport - append RCH downstream port data to a cxl_port
1060  * @port: the cxl_port that references this dport
1061  * @dport_dev: firmware or PCI device representing the dport
1062  * @port_id: identifier for this dport in a decoder's target list
1063  * @rcrb: mandatory location of a Root Complex Register Block
1064  *
1065  * See CXL 3.0 9.11.8 CXL Devices Attached to an RCH
1066  */
1067 struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
1068                                          struct device *dport_dev, int port_id,
1069                                          resource_size_t rcrb)
1070 {
1071         struct cxl_dport *dport;
1072
1073         if (rcrb == CXL_RESOURCE_NONE) {
1074                 dev_dbg(&port->dev, "failed to add RCH dport, missing RCRB\n");
1075                 return ERR_PTR(-EINVAL);
1076         }
1077
1078         dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1079                                      CXL_RESOURCE_NONE, rcrb);
1080         if (IS_ERR(dport)) {
1081                 dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n",
1082                         dev_name(&port->dev), PTR_ERR(dport));
1083         } else {
1084                 dev_dbg(dport_dev, "RCH dport added to %s\n",
1085                         dev_name(&port->dev));
1086         }
1087
1088         return dport;
1089 }
1090 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_rch_dport, CXL);
1091
1092 static int add_ep(struct cxl_ep *new)
1093 {
1094         struct cxl_port *port = new->dport->port;
1095         int rc;
1096
1097         device_lock(&port->dev);
1098         if (port->dead) {
1099                 device_unlock(&port->dev);
1100                 return -ENXIO;
1101         }
1102         rc = xa_insert(&port->endpoints, (unsigned long)new->ep, new,
1103                        GFP_KERNEL);
1104         device_unlock(&port->dev);
1105
1106         return rc;
1107 }
1108
1109 /**
1110  * cxl_add_ep - register an endpoint's interest in a port
1111  * @dport: the dport that routes to @ep_dev
1112  * @ep_dev: device representing the endpoint
1113  *
1114  * Intermediate CXL ports are scanned based on the arrival of endpoints.
1115  * When those endpoints depart the port can be destroyed once all
1116  * endpoints that care about that port have been removed.
1117  */
1118 static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev)
1119 {
1120         struct cxl_ep *ep;
1121         int rc;
1122
1123         ep = kzalloc(sizeof(*ep), GFP_KERNEL);
1124         if (!ep)
1125                 return -ENOMEM;
1126
1127         ep->ep = get_device(ep_dev);
1128         ep->dport = dport;
1129
1130         rc = add_ep(ep);
1131         if (rc)
1132                 cxl_ep_release(ep);
1133         return rc;
1134 }
1135
1136 struct cxl_find_port_ctx {
1137         const struct device *dport_dev;
1138         const struct cxl_port *parent_port;
1139         struct cxl_dport **dport;
1140 };
1141
1142 static int match_port_by_dport(struct device *dev, const void *data)
1143 {
1144         const struct cxl_find_port_ctx *ctx = data;
1145         struct cxl_dport *dport;
1146         struct cxl_port *port;
1147
1148         if (!is_cxl_port(dev))
1149                 return 0;
1150         if (ctx->parent_port && dev->parent != &ctx->parent_port->dev)
1151                 return 0;
1152
1153         port = to_cxl_port(dev);
1154         dport = cxl_find_dport_by_dev(port, ctx->dport_dev);
1155         if (ctx->dport)
1156                 *ctx->dport = dport;
1157         return dport != NULL;
1158 }
1159
1160 static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx)
1161 {
1162         struct device *dev;
1163
1164         if (!ctx->dport_dev)
1165                 return NULL;
1166
1167         dev = bus_find_device(&cxl_bus_type, NULL, ctx, match_port_by_dport);
1168         if (dev)
1169                 return to_cxl_port(dev);
1170         return NULL;
1171 }
1172
1173 static struct cxl_port *find_cxl_port(struct device *dport_dev,
1174                                       struct cxl_dport **dport)
1175 {
1176         struct cxl_find_port_ctx ctx = {
1177                 .dport_dev = dport_dev,
1178                 .dport = dport,
1179         };
1180         struct cxl_port *port;
1181
1182         port = __find_cxl_port(&ctx);
1183         return port;
1184 }
1185
1186 static struct cxl_port *find_cxl_port_at(struct cxl_port *parent_port,
1187                                          struct device *dport_dev,
1188                                          struct cxl_dport **dport)
1189 {
1190         struct cxl_find_port_ctx ctx = {
1191                 .dport_dev = dport_dev,
1192                 .parent_port = parent_port,
1193                 .dport = dport,
1194         };
1195         struct cxl_port *port;
1196
1197         port = __find_cxl_port(&ctx);
1198         return port;
1199 }
1200
1201 /*
1202  * All users of grandparent() are using it to walk PCIe-like switch port
1203  * hierarchy. A PCIe switch is comprised of a bridge device representing the
1204  * upstream switch port and N bridges representing downstream switch ports. When
1205  * bridges stack the grand-parent of a downstream switch port is another
1206  * downstream switch port in the immediate ancestor switch.
1207  */
1208 static struct device *grandparent(struct device *dev)
1209 {
1210         if (dev && dev->parent)
1211                 return dev->parent->parent;
1212         return NULL;
1213 }
1214
1215 static void delete_endpoint(void *data)
1216 {
1217         struct cxl_memdev *cxlmd = data;
1218         struct cxl_port *endpoint = cxlmd->endpoint;
1219         struct cxl_port *parent_port;
1220         struct device *parent;
1221
1222         parent_port = cxl_mem_find_port(cxlmd, NULL);
1223         if (!parent_port)
1224                 goto out;
1225         parent = &parent_port->dev;
1226
1227         device_lock(parent);
1228         if (parent->driver && !endpoint->dead) {
1229                 devm_release_action(parent, cxl_unlink_parent_dport, endpoint);
1230                 devm_release_action(parent, cxl_unlink_uport, endpoint);
1231                 devm_release_action(parent, unregister_port, endpoint);
1232         }
1233         cxlmd->endpoint = NULL;
1234         device_unlock(parent);
1235         put_device(parent);
1236 out:
1237         put_device(&endpoint->dev);
1238 }
1239
1240 int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
1241 {
1242         struct device *dev = &cxlmd->dev;
1243
1244         get_device(&endpoint->dev);
1245         cxlmd->endpoint = endpoint;
1246         cxlmd->depth = endpoint->depth;
1247         return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
1248 }
1249 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, CXL);
1250
1251 /*
1252  * The natural end of life of a non-root 'cxl_port' is when its parent port goes
1253  * through a ->remove() event ("top-down" unregistration). The unnatural trigger
1254  * for a port to be unregistered is when all memdevs beneath that port have gone
1255  * through ->remove(). This "bottom-up" removal selectively removes individual
1256  * child ports manually. This depends on devm_cxl_add_port() to not change is
1257  * devm action registration order, and for dports to have already been
1258  * destroyed by reap_dports().
1259  */
1260 static void delete_switch_port(struct cxl_port *port)
1261 {
1262         devm_release_action(port->dev.parent, cxl_unlink_parent_dport, port);
1263         devm_release_action(port->dev.parent, cxl_unlink_uport, port);
1264         devm_release_action(port->dev.parent, unregister_port, port);
1265 }
1266
1267 static void reap_dports(struct cxl_port *port)
1268 {
1269         struct cxl_dport *dport;
1270         unsigned long index;
1271
1272         device_lock_assert(&port->dev);
1273
1274         xa_for_each(&port->dports, index, dport) {
1275                 devm_release_action(&port->dev, cxl_dport_unlink, dport);
1276                 devm_release_action(&port->dev, cxl_dport_remove, dport);
1277                 devm_kfree(&port->dev, dport);
1278         }
1279 }
1280
1281 struct detach_ctx {
1282         struct cxl_memdev *cxlmd;
1283         int depth;
1284 };
1285
1286 static int port_has_memdev(struct device *dev, const void *data)
1287 {
1288         const struct detach_ctx *ctx = data;
1289         struct cxl_port *port;
1290
1291         if (!is_cxl_port(dev))
1292                 return 0;
1293
1294         port = to_cxl_port(dev);
1295         if (port->depth != ctx->depth)
1296                 return 0;
1297
1298         return !!cxl_ep_load(port, ctx->cxlmd);
1299 }
1300
1301 static void cxl_detach_ep(void *data)
1302 {
1303         struct cxl_memdev *cxlmd = data;
1304
1305         for (int i = cxlmd->depth - 1; i >= 1; i--) {
1306                 struct cxl_port *port, *parent_port;
1307                 struct detach_ctx ctx = {
1308                         .cxlmd = cxlmd,
1309                         .depth = i,
1310                 };
1311                 struct device *dev;
1312                 struct cxl_ep *ep;
1313                 bool died = false;
1314
1315                 dev = bus_find_device(&cxl_bus_type, NULL, &ctx,
1316                                       port_has_memdev);
1317                 if (!dev)
1318                         continue;
1319                 port = to_cxl_port(dev);
1320
1321                 parent_port = to_cxl_port(port->dev.parent);
1322                 device_lock(&parent_port->dev);
1323                 device_lock(&port->dev);
1324                 ep = cxl_ep_load(port, cxlmd);
1325                 dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
1326                         ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
1327                 cxl_ep_remove(port, ep);
1328                 if (ep && !port->dead && xa_empty(&port->endpoints) &&
1329                     !is_cxl_root(parent_port) && parent_port->dev.driver) {
1330                         /*
1331                          * This was the last ep attached to a dynamically
1332                          * enumerated port. Block new cxl_add_ep() and garbage
1333                          * collect the port.
1334                          */
1335                         died = true;
1336                         port->dead = true;
1337                         reap_dports(port);
1338                 }
1339                 device_unlock(&port->dev);
1340
1341                 if (died) {
1342                         dev_dbg(&cxlmd->dev, "delete %s\n",
1343                                 dev_name(&port->dev));
1344                         delete_switch_port(port);
1345                 }
1346                 put_device(&port->dev);
1347                 device_unlock(&parent_port->dev);
1348         }
1349 }
1350
1351 static resource_size_t find_component_registers(struct device *dev)
1352 {
1353         struct cxl_register_map map;
1354         struct pci_dev *pdev;
1355
1356         /*
1357          * Theoretically, CXL component registers can be hosted on a
1358          * non-PCI device, in practice, only cxl_test hits this case.
1359          */
1360         if (!dev_is_pci(dev))
1361                 return CXL_RESOURCE_NONE;
1362
1363         pdev = to_pci_dev(dev);
1364
1365         cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
1366         return map.resource;
1367 }
1368
1369 static int add_port_attach_ep(struct cxl_memdev *cxlmd,
1370                               struct device *uport_dev,
1371                               struct device *dport_dev)
1372 {
1373         struct device *dparent = grandparent(dport_dev);
1374         struct cxl_port *port, *parent_port = NULL;
1375         struct cxl_dport *dport, *parent_dport;
1376         resource_size_t component_reg_phys;
1377         int rc;
1378
1379         if (!dparent) {
1380                 /*
1381                  * The iteration reached the topology root without finding the
1382                  * CXL-root 'cxl_port' on a previous iteration, fail for now to
1383                  * be re-probed after platform driver attaches.
1384                  */
1385                 dev_dbg(&cxlmd->dev, "%s is a root dport\n",
1386                         dev_name(dport_dev));
1387                 return -ENXIO;
1388         }
1389
1390         parent_port = find_cxl_port(dparent, &parent_dport);
1391         if (!parent_port) {
1392                 /* iterate to create this parent_port */
1393                 return -EAGAIN;
1394         }
1395
1396         device_lock(&parent_port->dev);
1397         if (!parent_port->dev.driver) {
1398                 dev_warn(&cxlmd->dev,
1399                          "port %s:%s disabled, failed to enumerate CXL.mem\n",
1400                          dev_name(&parent_port->dev), dev_name(uport_dev));
1401                 port = ERR_PTR(-ENXIO);
1402                 goto out;
1403         }
1404
1405         port = find_cxl_port_at(parent_port, dport_dev, &dport);
1406         if (!port) {
1407                 component_reg_phys = find_component_registers(uport_dev);
1408                 port = devm_cxl_add_port(&parent_port->dev, uport_dev,
1409                                          component_reg_phys, parent_dport);
1410                 /* retry find to pick up the new dport information */
1411                 if (!IS_ERR(port))
1412                         port = find_cxl_port_at(parent_port, dport_dev, &dport);
1413         }
1414 out:
1415         device_unlock(&parent_port->dev);
1416
1417         if (IS_ERR(port))
1418                 rc = PTR_ERR(port);
1419         else {
1420                 dev_dbg(&cxlmd->dev, "add to new port %s:%s\n",
1421                         dev_name(&port->dev), dev_name(port->uport_dev));
1422                 rc = cxl_add_ep(dport, &cxlmd->dev);
1423                 if (rc == -EBUSY) {
1424                         /*
1425                          * "can't" happen, but this error code means
1426                          * something to the caller, so translate it.
1427                          */
1428                         rc = -ENXIO;
1429                 }
1430                 put_device(&port->dev);
1431         }
1432
1433         put_device(&parent_port->dev);
1434         return rc;
1435 }
1436
1437 int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
1438 {
1439         struct device *dev = &cxlmd->dev;
1440         struct device *iter;
1441         int rc;
1442
1443         /*
1444          * Skip intermediate port enumeration in the RCH case, there
1445          * are no ports in between a host bridge and an endpoint.
1446          */
1447         if (cxlmd->cxlds->rcd)
1448                 return 0;
1449
1450         rc = devm_add_action_or_reset(&cxlmd->dev, cxl_detach_ep, cxlmd);
1451         if (rc)
1452                 return rc;
1453
1454         /*
1455          * Scan for and add all cxl_ports in this device's ancestry.
1456          * Repeat until no more ports are added. Abort if a port add
1457          * attempt fails.
1458          */
1459 retry:
1460         for (iter = dev; iter; iter = grandparent(iter)) {
1461                 struct device *dport_dev = grandparent(iter);
1462                 struct device *uport_dev;
1463                 struct cxl_dport *dport;
1464                 struct cxl_port *port;
1465
1466                 if (!dport_dev)
1467                         return 0;
1468
1469                 uport_dev = dport_dev->parent;
1470                 if (!uport_dev) {
1471                         dev_warn(dev, "at %s no parent for dport: %s\n",
1472                                  dev_name(iter), dev_name(dport_dev));
1473                         return -ENXIO;
1474                 }
1475
1476                 dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
1477                         dev_name(iter), dev_name(dport_dev),
1478                         dev_name(uport_dev));
1479                 port = find_cxl_port(dport_dev, &dport);
1480                 if (port) {
1481                         dev_dbg(&cxlmd->dev,
1482                                 "found already registered port %s:%s\n",
1483                                 dev_name(&port->dev),
1484                                 dev_name(port->uport_dev));
1485                         rc = cxl_add_ep(dport, &cxlmd->dev);
1486
1487                         /*
1488                          * If the endpoint already exists in the port's list,
1489                          * that's ok, it was added on a previous pass.
1490                          * Otherwise, retry in add_port_attach_ep() after taking
1491                          * the parent_port lock as the current port may be being
1492                          * reaped.
1493                          */
1494                         if (rc && rc != -EBUSY) {
1495                                 put_device(&port->dev);
1496                                 return rc;
1497                         }
1498
1499                         /* Any more ports to add between this one and the root? */
1500                         if (!dev_is_cxl_root_child(&port->dev)) {
1501                                 put_device(&port->dev);
1502                                 continue;
1503                         }
1504
1505                         put_device(&port->dev);
1506                         return 0;
1507                 }
1508
1509                 rc = add_port_attach_ep(cxlmd, uport_dev, dport_dev);
1510                 /* port missing, try to add parent */
1511                 if (rc == -EAGAIN)
1512                         continue;
1513                 /* failed to add ep or port */
1514                 if (rc)
1515                         return rc;
1516                 /* port added, new descendants possible, start over */
1517                 goto retry;
1518         }
1519
1520         return 0;
1521 }
1522 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL);
1523
1524 struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
1525                                    struct cxl_dport **dport)
1526 {
1527         return find_cxl_port(pdev->dev.parent, dport);
1528 }
1529 EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, CXL);
1530
1531 struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
1532                                    struct cxl_dport **dport)
1533 {
1534         return find_cxl_port(grandparent(&cxlmd->dev), dport);
1535 }
1536 EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, CXL);
1537
1538 static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
1539                                     struct cxl_port *port, int *target_map)
1540 {
1541         int i, rc = 0;
1542
1543         if (!target_map)
1544                 return 0;
1545
1546         device_lock_assert(&port->dev);
1547
1548         if (xa_empty(&port->dports))
1549                 return -EINVAL;
1550
1551         write_seqlock(&cxlsd->target_lock);
1552         for (i = 0; i < cxlsd->nr_targets; i++) {
1553                 struct cxl_dport *dport = find_dport(port, target_map[i]);
1554
1555                 if (!dport) {
1556                         rc = -ENXIO;
1557                         break;
1558                 }
1559                 cxlsd->target[i] = dport;
1560         }
1561         write_sequnlock(&cxlsd->target_lock);
1562
1563         return rc;
1564 }
1565
1566 struct cxl_dport *cxl_hb_modulo(struct cxl_root_decoder *cxlrd, int pos)
1567 {
1568         struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
1569         struct cxl_decoder *cxld = &cxlsd->cxld;
1570         int iw;
1571
1572         iw = cxld->interleave_ways;
1573         if (dev_WARN_ONCE(&cxld->dev, iw != cxlsd->nr_targets,
1574                           "misconfigured root decoder\n"))
1575                 return NULL;
1576
1577         return cxlrd->cxlsd.target[pos % iw];
1578 }
1579 EXPORT_SYMBOL_NS_GPL(cxl_hb_modulo, CXL);
1580
1581 static struct lock_class_key cxl_decoder_key;
1582
1583 /**
1584  * cxl_decoder_init - Common decoder setup / initialization
1585  * @port: owning port of this decoder
1586  * @cxld: common decoder properties to initialize
1587  *
1588  * A port may contain one or more decoders. Each of those decoders
1589  * enable some address space for CXL.mem utilization. A decoder is
1590  * expected to be configured by the caller before registering via
1591  * cxl_decoder_add()
1592  */
1593 static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld)
1594 {
1595         struct device *dev;
1596         int rc;
1597
1598         rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
1599         if (rc < 0)
1600                 return rc;
1601
1602         /* need parent to stick around to release the id */
1603         get_device(&port->dev);
1604         cxld->id = rc;
1605
1606         dev = &cxld->dev;
1607         device_initialize(dev);
1608         lockdep_set_class(&dev->mutex, &cxl_decoder_key);
1609         device_set_pm_not_required(dev);
1610         dev->parent = &port->dev;
1611         dev->bus = &cxl_bus_type;
1612
1613         /* Pre initialize an "empty" decoder */
1614         cxld->interleave_ways = 1;
1615         cxld->interleave_granularity = PAGE_SIZE;
1616         cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1617         cxld->hpa_range = (struct range) {
1618                 .start = 0,
1619                 .end = -1,
1620         };
1621
1622         return 0;
1623 }
1624
1625 static int cxl_switch_decoder_init(struct cxl_port *port,
1626                                    struct cxl_switch_decoder *cxlsd,
1627                                    int nr_targets)
1628 {
1629         if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1630                 return -EINVAL;
1631
1632         cxlsd->nr_targets = nr_targets;
1633         seqlock_init(&cxlsd->target_lock);
1634         return cxl_decoder_init(port, &cxlsd->cxld);
1635 }
1636
1637 /**
1638  * cxl_root_decoder_alloc - Allocate a root level decoder
1639  * @port: owning CXL root of this decoder
1640  * @nr_targets: static number of downstream targets
1641  * @calc_hb: which host bridge covers the n'th position by granularity
1642  *
1643  * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1644  * 'CXL root' decoder is one that decodes from a top-level / static platform
1645  * firmware description of CXL resources into a CXL standard decode
1646  * topology.
1647  */
1648 struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
1649                                                 unsigned int nr_targets,
1650                                                 cxl_calc_hb_fn calc_hb)
1651 {
1652         struct cxl_root_decoder *cxlrd;
1653         struct cxl_switch_decoder *cxlsd;
1654         struct cxl_decoder *cxld;
1655         int rc;
1656
1657         if (!is_cxl_root(port))
1658                 return ERR_PTR(-EINVAL);
1659
1660         cxlrd = kzalloc(struct_size(cxlrd, cxlsd.target, nr_targets),
1661                         GFP_KERNEL);
1662         if (!cxlrd)
1663                 return ERR_PTR(-ENOMEM);
1664
1665         cxlsd = &cxlrd->cxlsd;
1666         rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1667         if (rc) {
1668                 kfree(cxlrd);
1669                 return ERR_PTR(rc);
1670         }
1671
1672         cxlrd->calc_hb = calc_hb;
1673         mutex_init(&cxlrd->range_lock);
1674
1675         cxld = &cxlsd->cxld;
1676         cxld->dev.type = &cxl_decoder_root_type;
1677         /*
1678          * cxl_root_decoder_release() special cases negative ids to
1679          * detect memregion_alloc() failures.
1680          */
1681         atomic_set(&cxlrd->region_id, -1);
1682         rc = memregion_alloc(GFP_KERNEL);
1683         if (rc < 0) {
1684                 put_device(&cxld->dev);
1685                 return ERR_PTR(rc);
1686         }
1687
1688         atomic_set(&cxlrd->region_id, rc);
1689         return cxlrd;
1690 }
1691 EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL);
1692
1693 /**
1694  * cxl_switch_decoder_alloc - Allocate a switch level decoder
1695  * @port: owning CXL switch port of this decoder
1696  * @nr_targets: max number of dynamically addressable downstream targets
1697  *
1698  * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1699  * 'switch' decoder is any decoder that can be enumerated by PCIe
1700  * topology and the HDM Decoder Capability. This includes the decoders
1701  * that sit between Switch Upstream Ports / Switch Downstream Ports and
1702  * Host Bridges / Root Ports.
1703  */
1704 struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
1705                                                     unsigned int nr_targets)
1706 {
1707         struct cxl_switch_decoder *cxlsd;
1708         struct cxl_decoder *cxld;
1709         int rc;
1710
1711         if (is_cxl_root(port) || is_cxl_endpoint(port))
1712                 return ERR_PTR(-EINVAL);
1713
1714         cxlsd = kzalloc(struct_size(cxlsd, target, nr_targets), GFP_KERNEL);
1715         if (!cxlsd)
1716                 return ERR_PTR(-ENOMEM);
1717
1718         rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1719         if (rc) {
1720                 kfree(cxlsd);
1721                 return ERR_PTR(rc);
1722         }
1723
1724         cxld = &cxlsd->cxld;
1725         cxld->dev.type = &cxl_decoder_switch_type;
1726         return cxlsd;
1727 }
1728 EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, CXL);
1729
1730 /**
1731  * cxl_endpoint_decoder_alloc - Allocate an endpoint decoder
1732  * @port: owning port of this decoder
1733  *
1734  * Return: A new cxl decoder to be registered by cxl_decoder_add()
1735  */
1736 struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port)
1737 {
1738         struct cxl_endpoint_decoder *cxled;
1739         struct cxl_decoder *cxld;
1740         int rc;
1741
1742         if (!is_cxl_endpoint(port))
1743                 return ERR_PTR(-EINVAL);
1744
1745         cxled = kzalloc(sizeof(*cxled), GFP_KERNEL);
1746         if (!cxled)
1747                 return ERR_PTR(-ENOMEM);
1748
1749         cxled->pos = -1;
1750         cxld = &cxled->cxld;
1751         rc = cxl_decoder_init(port, cxld);
1752         if (rc)  {
1753                 kfree(cxled);
1754                 return ERR_PTR(rc);
1755         }
1756
1757         cxld->dev.type = &cxl_decoder_endpoint_type;
1758         return cxled;
1759 }
1760 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, CXL);
1761
1762 /**
1763  * cxl_decoder_add_locked - Add a decoder with targets
1764  * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1765  * @target_map: A list of downstream ports that this decoder can direct memory
1766  *              traffic to. These numbers should correspond with the port number
1767  *              in the PCIe Link Capabilities structure.
1768  *
1769  * Certain types of decoders may not have any targets. The main example of this
1770  * is an endpoint device. A more awkward example is a hostbridge whose root
1771  * ports get hot added (technically possible, though unlikely).
1772  *
1773  * This is the locked variant of cxl_decoder_add().
1774  *
1775  * Context: Process context. Expects the device lock of the port that owns the
1776  *          @cxld to be held.
1777  *
1778  * Return: Negative error code if the decoder wasn't properly configured; else
1779  *         returns 0.
1780  */
1781 int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map)
1782 {
1783         struct cxl_port *port;
1784         struct device *dev;
1785         int rc;
1786
1787         if (WARN_ON_ONCE(!cxld))
1788                 return -EINVAL;
1789
1790         if (WARN_ON_ONCE(IS_ERR(cxld)))
1791                 return PTR_ERR(cxld);
1792
1793         if (cxld->interleave_ways < 1)
1794                 return -EINVAL;
1795
1796         dev = &cxld->dev;
1797
1798         port = to_cxl_port(cxld->dev.parent);
1799         if (!is_endpoint_decoder(dev)) {
1800                 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
1801
1802                 rc = decoder_populate_targets(cxlsd, port, target_map);
1803                 if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {
1804                         dev_err(&port->dev,
1805                                 "Failed to populate active decoder targets\n");
1806                         return rc;
1807                 }
1808         }
1809
1810         rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
1811         if (rc)
1812                 return rc;
1813
1814         return device_add(dev);
1815 }
1816 EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, CXL);
1817
1818 /**
1819  * cxl_decoder_add - Add a decoder with targets
1820  * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1821  * @target_map: A list of downstream ports that this decoder can direct memory
1822  *              traffic to. These numbers should correspond with the port number
1823  *              in the PCIe Link Capabilities structure.
1824  *
1825  * This is the unlocked variant of cxl_decoder_add_locked().
1826  * See cxl_decoder_add_locked().
1827  *
1828  * Context: Process context. Takes and releases the device lock of the port that
1829  *          owns the @cxld.
1830  */
1831 int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map)
1832 {
1833         struct cxl_port *port;
1834         int rc;
1835
1836         if (WARN_ON_ONCE(!cxld))
1837                 return -EINVAL;
1838
1839         if (WARN_ON_ONCE(IS_ERR(cxld)))
1840                 return PTR_ERR(cxld);
1841
1842         port = to_cxl_port(cxld->dev.parent);
1843
1844         device_lock(&port->dev);
1845         rc = cxl_decoder_add_locked(cxld, target_map);
1846         device_unlock(&port->dev);
1847
1848         return rc;
1849 }
1850 EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, CXL);
1851
1852 static void cxld_unregister(void *dev)
1853 {
1854         struct cxl_endpoint_decoder *cxled;
1855
1856         if (is_endpoint_decoder(dev)) {
1857                 cxled = to_cxl_endpoint_decoder(dev);
1858                 cxl_decoder_kill_region(cxled);
1859         }
1860
1861         device_unregister(dev);
1862 }
1863
1864 int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
1865 {
1866         return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
1867 }
1868 EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, CXL);
1869
1870 /**
1871  * __cxl_driver_register - register a driver for the cxl bus
1872  * @cxl_drv: cxl driver structure to attach
1873  * @owner: owning module/driver
1874  * @modname: KBUILD_MODNAME for parent driver
1875  */
1876 int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
1877                           const char *modname)
1878 {
1879         if (!cxl_drv->probe) {
1880                 pr_debug("%s ->probe() must be specified\n", modname);
1881                 return -EINVAL;
1882         }
1883
1884         if (!cxl_drv->name) {
1885                 pr_debug("%s ->name must be specified\n", modname);
1886                 return -EINVAL;
1887         }
1888
1889         if (!cxl_drv->id) {
1890                 pr_debug("%s ->id must be specified\n", modname);
1891                 return -EINVAL;
1892         }
1893
1894         cxl_drv->drv.bus = &cxl_bus_type;
1895         cxl_drv->drv.owner = owner;
1896         cxl_drv->drv.mod_name = modname;
1897         cxl_drv->drv.name = cxl_drv->name;
1898
1899         return driver_register(&cxl_drv->drv);
1900 }
1901 EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, CXL);
1902
1903 void cxl_driver_unregister(struct cxl_driver *cxl_drv)
1904 {
1905         driver_unregister(&cxl_drv->drv);
1906 }
1907 EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, CXL);
1908
1909 static int cxl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
1910 {
1911         return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,
1912                               cxl_device_id(dev));
1913 }
1914
1915 static int cxl_bus_match(struct device *dev, struct device_driver *drv)
1916 {
1917         return cxl_device_id(dev) == to_cxl_drv(drv)->id;
1918 }
1919
1920 static int cxl_bus_probe(struct device *dev)
1921 {
1922         int rc;
1923
1924         rc = to_cxl_drv(dev->driver)->probe(dev);
1925         dev_dbg(dev, "probe: %d\n", rc);
1926         return rc;
1927 }
1928
1929 static void cxl_bus_remove(struct device *dev)
1930 {
1931         struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
1932
1933         if (cxl_drv->remove)
1934                 cxl_drv->remove(dev);
1935 }
1936
1937 static struct workqueue_struct *cxl_bus_wq;
1938
1939 static void cxl_bus_rescan_queue(struct work_struct *w)
1940 {
1941         int rc = bus_rescan_devices(&cxl_bus_type);
1942
1943         pr_debug("CXL bus rescan result: %d\n", rc);
1944 }
1945
1946 void cxl_bus_rescan(void)
1947 {
1948         static DECLARE_WORK(rescan_work, cxl_bus_rescan_queue);
1949
1950         queue_work(cxl_bus_wq, &rescan_work);
1951 }
1952 EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, CXL);
1953
1954 void cxl_bus_drain(void)
1955 {
1956         drain_workqueue(cxl_bus_wq);
1957 }
1958 EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, CXL);
1959
1960 bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
1961 {
1962         return queue_work(cxl_bus_wq, &cxlmd->detach_work);
1963 }
1964 EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL);
1965
1966 /* for user tooling to ensure port disable work has completed */
1967 static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count)
1968 {
1969         if (sysfs_streq(buf, "1")) {
1970                 flush_workqueue(cxl_bus_wq);
1971                 return count;
1972         }
1973
1974         return -EINVAL;
1975 }
1976
1977 static BUS_ATTR_WO(flush);
1978
1979 static struct attribute *cxl_bus_attributes[] = {
1980         &bus_attr_flush.attr,
1981         NULL,
1982 };
1983
1984 static struct attribute_group cxl_bus_attribute_group = {
1985         .attrs = cxl_bus_attributes,
1986 };
1987
1988 static const struct attribute_group *cxl_bus_attribute_groups[] = {
1989         &cxl_bus_attribute_group,
1990         NULL,
1991 };
1992
1993 struct bus_type cxl_bus_type = {
1994         .name = "cxl",
1995         .uevent = cxl_bus_uevent,
1996         .match = cxl_bus_match,
1997         .probe = cxl_bus_probe,
1998         .remove = cxl_bus_remove,
1999         .bus_groups = cxl_bus_attribute_groups,
2000 };
2001 EXPORT_SYMBOL_NS_GPL(cxl_bus_type, CXL);
2002
2003 static struct dentry *cxl_debugfs;
2004
2005 struct dentry *cxl_debugfs_create_dir(const char *dir)
2006 {
2007         return debugfs_create_dir(dir, cxl_debugfs);
2008 }
2009 EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, CXL);
2010
2011 static __init int cxl_core_init(void)
2012 {
2013         int rc;
2014
2015         cxl_debugfs = debugfs_create_dir("cxl", NULL);
2016
2017         cxl_mbox_init();
2018
2019         rc = cxl_memdev_init();
2020         if (rc)
2021                 return rc;
2022
2023         cxl_bus_wq = alloc_ordered_workqueue("cxl_port", 0);
2024         if (!cxl_bus_wq) {
2025                 rc = -ENOMEM;
2026                 goto err_wq;
2027         }
2028
2029         rc = bus_register(&cxl_bus_type);
2030         if (rc)
2031                 goto err_bus;
2032
2033         rc = cxl_region_init();
2034         if (rc)
2035                 goto err_region;
2036
2037         return 0;
2038
2039 err_region:
2040         bus_unregister(&cxl_bus_type);
2041 err_bus:
2042         destroy_workqueue(cxl_bus_wq);
2043 err_wq:
2044         cxl_memdev_exit();
2045         return rc;
2046 }
2047
2048 static void cxl_core_exit(void)
2049 {
2050         cxl_region_exit();
2051         bus_unregister(&cxl_bus_type);
2052         destroy_workqueue(cxl_bus_wq);
2053         cxl_memdev_exit();
2054         debugfs_remove_recursive(cxl_debugfs);
2055 }
2056
2057 subsys_initcall(cxl_core_init);
2058 module_exit(cxl_core_exit);
2059 MODULE_LICENSE("GPL v2");