1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3 #include <linux/memregion.h>
4 #include <linux/workqueue.h>
5 #include <linux/debugfs.h>
6 #include <linux/device.h>
7 #include <linux/module.h>
9 #include <linux/slab.h>
10 #include <linux/idr.h>
19 * The CXL core provides a set of interfaces that can be consumed by CXL aware
20 * drivers. The interfaces allow for creation, modification, and destruction of
21 * regions, memory devices, ports, and decoders. CXL aware drivers must register
22 * with the CXL core via these interfaces in order to be able to participate in
23 * cross-device interleave coordination. The CXL core also establishes and
24 * maintains the bridge to the nvdimm subsystem.
26 * CXL core introduces sysfs hierarchy to control the devices that are
27 * instantiated by the core.
30 static DEFINE_IDA(cxl_port_ida);
31 static DEFINE_XARRAY(cxl_root_buses);
33 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
36 return sysfs_emit(buf, "%s\n", dev->type->name);
38 static DEVICE_ATTR_RO(devtype);
40 static int cxl_device_id(const struct device *dev)
42 if (dev->type == &cxl_nvdimm_bridge_type)
43 return CXL_DEVICE_NVDIMM_BRIDGE;
44 if (dev->type == &cxl_nvdimm_type)
45 return CXL_DEVICE_NVDIMM;
46 if (dev->type == CXL_PMEM_REGION_TYPE())
47 return CXL_DEVICE_PMEM_REGION;
48 if (dev->type == CXL_DAX_REGION_TYPE())
49 return CXL_DEVICE_DAX_REGION;
50 if (is_cxl_port(dev)) {
51 if (is_cxl_root(to_cxl_port(dev)))
52 return CXL_DEVICE_ROOT;
53 return CXL_DEVICE_PORT;
55 if (is_cxl_memdev(dev))
56 return CXL_DEVICE_MEMORY_EXPANDER;
57 if (dev->type == CXL_REGION_TYPE())
58 return CXL_DEVICE_REGION;
59 if (dev->type == &cxl_pmu_type)
60 return CXL_DEVICE_PMU;
64 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
67 return sysfs_emit(buf, CXL_MODALIAS_FMT "\n", cxl_device_id(dev));
69 static DEVICE_ATTR_RO(modalias);
71 static struct attribute *cxl_base_attributes[] = {
72 &dev_attr_devtype.attr,
73 &dev_attr_modalias.attr,
77 struct attribute_group cxl_base_attribute_group = {
78 .attrs = cxl_base_attributes,
81 static ssize_t start_show(struct device *dev, struct device_attribute *attr,
84 struct cxl_decoder *cxld = to_cxl_decoder(dev);
86 return sysfs_emit(buf, "%#llx\n", cxld->hpa_range.start);
88 static DEVICE_ATTR_ADMIN_RO(start);
90 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
93 struct cxl_decoder *cxld = to_cxl_decoder(dev);
95 return sysfs_emit(buf, "%#llx\n", range_len(&cxld->hpa_range));
97 static DEVICE_ATTR_RO(size);
99 #define CXL_DECODER_FLAG_ATTR(name, flag) \
100 static ssize_t name##_show(struct device *dev, \
101 struct device_attribute *attr, char *buf) \
103 struct cxl_decoder *cxld = to_cxl_decoder(dev); \
105 return sysfs_emit(buf, "%s\n", \
106 (cxld->flags & (flag)) ? "1" : "0"); \
108 static DEVICE_ATTR_RO(name)
110 CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
111 CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
112 CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
113 CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
114 CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
116 static ssize_t target_type_show(struct device *dev,
117 struct device_attribute *attr, char *buf)
119 struct cxl_decoder *cxld = to_cxl_decoder(dev);
121 switch (cxld->target_type) {
122 case CXL_DECODER_DEVMEM:
123 return sysfs_emit(buf, "accelerator\n");
124 case CXL_DECODER_HOSTONLYMEM:
125 return sysfs_emit(buf, "expander\n");
129 static DEVICE_ATTR_RO(target_type);
131 static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf)
133 struct cxl_decoder *cxld = &cxlsd->cxld;
137 for (i = 0; i < cxld->interleave_ways; i++) {
138 struct cxl_dport *dport = cxlsd->target[i];
139 struct cxl_dport *next = NULL;
144 if (i + 1 < cxld->interleave_ways)
145 next = cxlsd->target[i + 1];
146 rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
156 static ssize_t target_list_show(struct device *dev,
157 struct device_attribute *attr, char *buf)
159 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
165 seq = read_seqbegin(&cxlsd->target_lock);
166 rc = emit_target_list(cxlsd, buf);
167 } while (read_seqretry(&cxlsd->target_lock, seq));
173 rc = sysfs_emit_at(buf, offset, "\n");
179 static DEVICE_ATTR_RO(target_list);
181 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
184 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
186 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxled->mode));
189 static ssize_t mode_store(struct device *dev, struct device_attribute *attr,
190 const char *buf, size_t len)
192 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
193 enum cxl_decoder_mode mode;
196 if (sysfs_streq(buf, "pmem"))
197 mode = CXL_DECODER_PMEM;
198 else if (sysfs_streq(buf, "ram"))
199 mode = CXL_DECODER_RAM;
203 rc = cxl_dpa_set_mode(cxled, mode);
209 static DEVICE_ATTR_RW(mode);
211 static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *attr,
214 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
215 u64 base = cxl_dpa_resource_start(cxled);
217 return sysfs_emit(buf, "%#llx\n", base);
219 static DEVICE_ATTR_RO(dpa_resource);
221 static ssize_t dpa_size_show(struct device *dev, struct device_attribute *attr,
224 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
225 resource_size_t size = cxl_dpa_size(cxled);
227 return sysfs_emit(buf, "%pa\n", &size);
230 static ssize_t dpa_size_store(struct device *dev, struct device_attribute *attr,
231 const char *buf, size_t len)
233 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
234 unsigned long long size;
237 rc = kstrtoull(buf, 0, &size);
241 if (!IS_ALIGNED(size, SZ_256M))
244 rc = cxl_dpa_free(cxled);
251 rc = cxl_dpa_alloc(cxled, size);
257 static DEVICE_ATTR_RW(dpa_size);
259 static ssize_t interleave_granularity_show(struct device *dev,
260 struct device_attribute *attr,
263 struct cxl_decoder *cxld = to_cxl_decoder(dev);
265 return sysfs_emit(buf, "%d\n", cxld->interleave_granularity);
268 static DEVICE_ATTR_RO(interleave_granularity);
270 static ssize_t interleave_ways_show(struct device *dev,
271 struct device_attribute *attr, char *buf)
273 struct cxl_decoder *cxld = to_cxl_decoder(dev);
275 return sysfs_emit(buf, "%d\n", cxld->interleave_ways);
278 static DEVICE_ATTR_RO(interleave_ways);
280 static struct attribute *cxl_decoder_base_attrs[] = {
281 &dev_attr_start.attr,
283 &dev_attr_locked.attr,
284 &dev_attr_interleave_granularity.attr,
285 &dev_attr_interleave_ways.attr,
289 static struct attribute_group cxl_decoder_base_attribute_group = {
290 .attrs = cxl_decoder_base_attrs,
293 static struct attribute *cxl_decoder_root_attrs[] = {
294 &dev_attr_cap_pmem.attr,
295 &dev_attr_cap_ram.attr,
296 &dev_attr_cap_type2.attr,
297 &dev_attr_cap_type3.attr,
298 &dev_attr_target_list.attr,
299 SET_CXL_REGION_ATTR(create_pmem_region)
300 SET_CXL_REGION_ATTR(create_ram_region)
301 SET_CXL_REGION_ATTR(delete_region)
305 static bool can_create_pmem(struct cxl_root_decoder *cxlrd)
307 unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_PMEM;
309 return (cxlrd->cxlsd.cxld.flags & flags) == flags;
312 static bool can_create_ram(struct cxl_root_decoder *cxlrd)
314 unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_RAM;
316 return (cxlrd->cxlsd.cxld.flags & flags) == flags;
319 static umode_t cxl_root_decoder_visible(struct kobject *kobj, struct attribute *a, int n)
321 struct device *dev = kobj_to_dev(kobj);
322 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
324 if (a == CXL_REGION_ATTR(create_pmem_region) && !can_create_pmem(cxlrd))
327 if (a == CXL_REGION_ATTR(create_ram_region) && !can_create_ram(cxlrd))
330 if (a == CXL_REGION_ATTR(delete_region) &&
331 !(can_create_pmem(cxlrd) || can_create_ram(cxlrd)))
337 static struct attribute_group cxl_decoder_root_attribute_group = {
338 .attrs = cxl_decoder_root_attrs,
339 .is_visible = cxl_root_decoder_visible,
342 static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
343 &cxl_decoder_root_attribute_group,
344 &cxl_decoder_base_attribute_group,
345 &cxl_base_attribute_group,
349 static struct attribute *cxl_decoder_switch_attrs[] = {
350 &dev_attr_target_type.attr,
351 &dev_attr_target_list.attr,
352 SET_CXL_REGION_ATTR(region)
356 static struct attribute_group cxl_decoder_switch_attribute_group = {
357 .attrs = cxl_decoder_switch_attrs,
360 static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
361 &cxl_decoder_switch_attribute_group,
362 &cxl_decoder_base_attribute_group,
363 &cxl_base_attribute_group,
367 static struct attribute *cxl_decoder_endpoint_attrs[] = {
368 &dev_attr_target_type.attr,
370 &dev_attr_dpa_size.attr,
371 &dev_attr_dpa_resource.attr,
372 SET_CXL_REGION_ATTR(region)
376 static struct attribute_group cxl_decoder_endpoint_attribute_group = {
377 .attrs = cxl_decoder_endpoint_attrs,
380 static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = {
381 &cxl_decoder_base_attribute_group,
382 &cxl_decoder_endpoint_attribute_group,
383 &cxl_base_attribute_group,
387 static void __cxl_decoder_release(struct cxl_decoder *cxld)
389 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
391 ida_free(&port->decoder_ida, cxld->id);
392 put_device(&port->dev);
395 static void cxl_endpoint_decoder_release(struct device *dev)
397 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
399 __cxl_decoder_release(&cxled->cxld);
403 static void cxl_switch_decoder_release(struct device *dev)
405 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
407 __cxl_decoder_release(&cxlsd->cxld);
411 struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev)
413 if (dev_WARN_ONCE(dev, !is_root_decoder(dev),
414 "not a cxl_root_decoder device\n"))
416 return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev);
418 EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, CXL);
420 static void cxl_root_decoder_release(struct device *dev)
422 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
424 if (atomic_read(&cxlrd->region_id) >= 0)
425 memregion_free(atomic_read(&cxlrd->region_id));
426 __cxl_decoder_release(&cxlrd->cxlsd.cxld);
430 static const struct device_type cxl_decoder_endpoint_type = {
431 .name = "cxl_decoder_endpoint",
432 .release = cxl_endpoint_decoder_release,
433 .groups = cxl_decoder_endpoint_attribute_groups,
436 static const struct device_type cxl_decoder_switch_type = {
437 .name = "cxl_decoder_switch",
438 .release = cxl_switch_decoder_release,
439 .groups = cxl_decoder_switch_attribute_groups,
442 static const struct device_type cxl_decoder_root_type = {
443 .name = "cxl_decoder_root",
444 .release = cxl_root_decoder_release,
445 .groups = cxl_decoder_root_attribute_groups,
448 bool is_endpoint_decoder(struct device *dev)
450 return dev->type == &cxl_decoder_endpoint_type;
452 EXPORT_SYMBOL_NS_GPL(is_endpoint_decoder, CXL);
454 bool is_root_decoder(struct device *dev)
456 return dev->type == &cxl_decoder_root_type;
458 EXPORT_SYMBOL_NS_GPL(is_root_decoder, CXL);
460 bool is_switch_decoder(struct device *dev)
462 return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type;
464 EXPORT_SYMBOL_NS_GPL(is_switch_decoder, CXL);
466 struct cxl_decoder *to_cxl_decoder(struct device *dev)
468 if (dev_WARN_ONCE(dev,
469 !is_switch_decoder(dev) && !is_endpoint_decoder(dev),
470 "not a cxl_decoder device\n"))
472 return container_of(dev, struct cxl_decoder, dev);
474 EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, CXL);
476 struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev)
478 if (dev_WARN_ONCE(dev, !is_endpoint_decoder(dev),
479 "not a cxl_endpoint_decoder device\n"))
481 return container_of(dev, struct cxl_endpoint_decoder, cxld.dev);
483 EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, CXL);
485 struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev)
487 if (dev_WARN_ONCE(dev, !is_switch_decoder(dev),
488 "not a cxl_switch_decoder device\n"))
490 return container_of(dev, struct cxl_switch_decoder, cxld.dev);
492 EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, CXL);
494 static void cxl_ep_release(struct cxl_ep *ep)
500 static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep)
504 xa_erase(&port->endpoints, (unsigned long) ep->ep);
508 static void cxl_port_release(struct device *dev)
510 struct cxl_port *port = to_cxl_port(dev);
514 xa_for_each(&port->endpoints, index, ep)
515 cxl_ep_remove(port, ep);
516 xa_destroy(&port->endpoints);
517 xa_destroy(&port->dports);
518 xa_destroy(&port->regions);
519 ida_free(&cxl_port_ida, port->id);
523 static const struct attribute_group *cxl_port_attribute_groups[] = {
524 &cxl_base_attribute_group,
528 static const struct device_type cxl_port_type = {
530 .release = cxl_port_release,
531 .groups = cxl_port_attribute_groups,
534 bool is_cxl_port(const struct device *dev)
536 return dev->type == &cxl_port_type;
538 EXPORT_SYMBOL_NS_GPL(is_cxl_port, CXL);
540 struct cxl_port *to_cxl_port(const struct device *dev)
542 if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
543 "not a cxl_port device\n"))
545 return container_of(dev, struct cxl_port, dev);
547 EXPORT_SYMBOL_NS_GPL(to_cxl_port, CXL);
549 static void unregister_port(void *_port)
551 struct cxl_port *port = _port;
552 struct cxl_port *parent;
553 struct device *lock_dev;
555 if (is_cxl_root(port))
558 parent = to_cxl_port(port->dev.parent);
561 * CXL root port's and the first level of ports are unregistered
562 * under the platform firmware device lock, all other ports are
563 * unregistered while holding their parent port lock.
566 lock_dev = port->uport_dev;
567 else if (is_cxl_root(parent))
568 lock_dev = parent->uport_dev;
570 lock_dev = &parent->dev;
572 device_lock_assert(lock_dev);
574 device_unregister(&port->dev);
577 static void cxl_unlink_uport(void *_port)
579 struct cxl_port *port = _port;
581 sysfs_remove_link(&port->dev.kobj, "uport");
584 static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
588 rc = sysfs_create_link(&port->dev.kobj, &port->uport_dev->kobj,
592 return devm_add_action_or_reset(host, cxl_unlink_uport, port);
595 static void cxl_unlink_parent_dport(void *_port)
597 struct cxl_port *port = _port;
599 sysfs_remove_link(&port->dev.kobj, "parent_dport");
602 static int devm_cxl_link_parent_dport(struct device *host,
603 struct cxl_port *port,
604 struct cxl_dport *parent_dport)
611 rc = sysfs_create_link(&port->dev.kobj, &parent_dport->dport_dev->kobj,
615 return devm_add_action_or_reset(host, cxl_unlink_parent_dport, port);
618 static struct lock_class_key cxl_port_key;
620 static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
621 resource_size_t component_reg_phys,
622 struct cxl_dport *parent_dport)
624 struct cxl_port *port;
628 port = kzalloc(sizeof(*port), GFP_KERNEL);
630 return ERR_PTR(-ENOMEM);
632 rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
636 port->uport_dev = uport_dev;
639 * The top-level cxl_port "cxl_root" does not have a cxl_port as
640 * its parent and it does not have any corresponding component
641 * registers as its decode is described by a fixed platform
646 struct cxl_port *parent_port = parent_dport->port;
647 struct cxl_port *iter;
649 dev->parent = &parent_port->dev;
650 port->depth = parent_port->depth + 1;
651 port->parent_dport = parent_dport;
654 * walk to the host bridge, or the first ancestor that knows
658 while (!iter->host_bridge &&
659 !is_cxl_root(to_cxl_port(iter->dev.parent)))
660 iter = to_cxl_port(iter->dev.parent);
661 if (iter->host_bridge)
662 port->host_bridge = iter->host_bridge;
663 else if (parent_dport->rch)
664 port->host_bridge = parent_dport->dport_dev;
666 port->host_bridge = iter->uport_dev;
667 dev_dbg(uport_dev, "host-bridge: %s\n",
668 dev_name(port->host_bridge));
670 dev->parent = uport_dev;
672 port->component_reg_phys = component_reg_phys;
673 ida_init(&port->decoder_ida);
675 port->commit_end = -1;
676 xa_init(&port->dports);
677 xa_init(&port->endpoints);
678 xa_init(&port->regions);
680 device_initialize(dev);
681 lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
682 device_set_pm_not_required(dev);
683 dev->bus = &cxl_bus_type;
684 dev->type = &cxl_port_type;
693 static int cxl_setup_comp_regs(struct device *dev, struct cxl_register_map *map,
694 resource_size_t component_reg_phys)
696 if (component_reg_phys == CXL_RESOURCE_NONE)
699 *map = (struct cxl_register_map) {
701 .reg_type = CXL_REGLOC_RBI_COMPONENT,
702 .resource = component_reg_phys,
703 .max_size = CXL_COMPONENT_REG_BLOCK_SIZE,
706 return cxl_setup_regs(map);
709 static inline int cxl_port_setup_regs(struct cxl_port *port,
710 resource_size_t component_reg_phys)
712 return cxl_setup_comp_regs(&port->dev, &port->comp_map,
716 static inline int cxl_dport_setup_regs(struct cxl_dport *dport,
717 resource_size_t component_reg_phys)
719 return cxl_setup_comp_regs(dport->dport_dev, &dport->comp_map,
723 static struct cxl_port *__devm_cxl_add_port(struct device *host,
724 struct device *uport_dev,
725 resource_size_t component_reg_phys,
726 struct cxl_dport *parent_dport)
728 struct cxl_port *port;
732 port = cxl_port_alloc(uport_dev, component_reg_phys, parent_dport);
737 if (is_cxl_memdev(uport_dev))
738 rc = dev_set_name(dev, "endpoint%d", port->id);
739 else if (parent_dport)
740 rc = dev_set_name(dev, "port%d", port->id);
742 rc = dev_set_name(dev, "root%d", port->id);
746 rc = cxl_port_setup_regs(port, component_reg_phys);
750 rc = device_add(dev);
754 rc = devm_add_action_or_reset(host, unregister_port, port);
758 rc = devm_cxl_link_uport(host, port);
762 rc = devm_cxl_link_parent_dport(host, port, parent_dport);
774 * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
775 * @host: host device for devm operations
776 * @uport_dev: "physical" device implementing this upstream port
777 * @component_reg_phys: (optional) for configurable cxl_port instances
778 * @parent_dport: next hop up in the CXL memory decode hierarchy
780 struct cxl_port *devm_cxl_add_port(struct device *host,
781 struct device *uport_dev,
782 resource_size_t component_reg_phys,
783 struct cxl_dport *parent_dport)
785 struct cxl_port *port, *parent_port;
787 port = __devm_cxl_add_port(host, uport_dev, component_reg_phys,
790 parent_port = parent_dport ? parent_dport->port : NULL;
792 dev_dbg(uport_dev, "Failed to add%s%s%s: %ld\n",
793 parent_port ? " port to " : "",
794 parent_port ? dev_name(&parent_port->dev) : "",
795 parent_port ? "" : " root port",
798 dev_dbg(uport_dev, "%s added%s%s%s\n",
799 dev_name(&port->dev),
800 parent_port ? " to " : "",
801 parent_port ? dev_name(&parent_port->dev) : "",
802 parent_port ? "" : " (root port)");
807 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, CXL);
809 struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port)
811 /* There is no pci_bus associated with a CXL platform-root port */
812 if (is_cxl_root(port))
815 if (dev_is_pci(port->uport_dev)) {
816 struct pci_dev *pdev = to_pci_dev(port->uport_dev);
818 return pdev->subordinate;
821 return xa_load(&cxl_root_buses, (unsigned long)port->uport_dev);
823 EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, CXL);
825 static void unregister_pci_bus(void *uport_dev)
827 xa_erase(&cxl_root_buses, (unsigned long)uport_dev);
830 int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,
835 if (dev_is_pci(uport_dev))
838 rc = xa_insert(&cxl_root_buses, (unsigned long)uport_dev, bus,
842 return devm_add_action_or_reset(host, unregister_pci_bus, uport_dev);
844 EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, CXL);
846 static bool dev_is_cxl_root_child(struct device *dev)
848 struct cxl_port *port, *parent;
850 if (!is_cxl_port(dev))
853 port = to_cxl_port(dev);
854 if (is_cxl_root(port))
857 parent = to_cxl_port(port->dev.parent);
858 if (is_cxl_root(parent))
864 struct cxl_port *find_cxl_root(struct cxl_port *port)
866 struct cxl_port *iter = port;
868 while (iter && !is_cxl_root(iter))
869 iter = to_cxl_port(iter->dev.parent);
873 get_device(&iter->dev);
876 EXPORT_SYMBOL_NS_GPL(find_cxl_root, CXL);
878 static struct cxl_dport *find_dport(struct cxl_port *port, int id)
880 struct cxl_dport *dport;
883 device_lock_assert(&port->dev);
884 xa_for_each(&port->dports, index, dport)
885 if (dport->port_id == id)
890 static int add_dport(struct cxl_port *port, struct cxl_dport *dport)
892 struct cxl_dport *dup;
895 device_lock_assert(&port->dev);
896 dup = find_dport(port, dport->port_id);
899 "unable to add dport%d-%s non-unique port id (%s)\n",
900 dport->port_id, dev_name(dport->dport_dev),
901 dev_name(dup->dport_dev));
905 rc = xa_insert(&port->dports, (unsigned long)dport->dport_dev, dport,
915 * Since root-level CXL dports cannot be enumerated by PCI they are not
916 * enumerated by the common port driver that acquires the port lock over
917 * dport add/remove. Instead, root dports are manually added by a
918 * platform driver and cond_cxl_root_lock() is used to take the missing
919 * port lock in that case.
921 static void cond_cxl_root_lock(struct cxl_port *port)
923 if (is_cxl_root(port))
924 device_lock(&port->dev);
927 static void cond_cxl_root_unlock(struct cxl_port *port)
929 if (is_cxl_root(port))
930 device_unlock(&port->dev);
933 static void cxl_dport_remove(void *data)
935 struct cxl_dport *dport = data;
936 struct cxl_port *port = dport->port;
938 xa_erase(&port->dports, (unsigned long) dport->dport_dev);
939 put_device(dport->dport_dev);
942 static void cxl_dport_unlink(void *data)
944 struct cxl_dport *dport = data;
945 struct cxl_port *port = dport->port;
946 char link_name[CXL_TARGET_STRLEN];
948 sprintf(link_name, "dport%d", dport->port_id);
949 sysfs_remove_link(&port->dev.kobj, link_name);
952 static struct cxl_dport *
953 __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
954 int port_id, resource_size_t component_reg_phys,
955 resource_size_t rcrb)
957 char link_name[CXL_TARGET_STRLEN];
958 struct cxl_dport *dport;
962 if (is_cxl_root(port))
963 host = port->uport_dev;
968 dev_WARN_ONCE(&port->dev, 1, "dport:%s bad devm context\n",
969 dev_name(dport_dev));
970 return ERR_PTR(-ENXIO);
973 if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
975 return ERR_PTR(-EINVAL);
977 dport = devm_kzalloc(host, sizeof(*dport), GFP_KERNEL);
979 return ERR_PTR(-ENOMEM);
981 if (rcrb != CXL_RESOURCE_NONE) {
982 dport->rcrb.base = rcrb;
983 component_reg_phys = __rcrb_to_component(dport_dev, &dport->rcrb,
984 CXL_RCRB_DOWNSTREAM);
985 if (component_reg_phys == CXL_RESOURCE_NONE) {
986 dev_warn(dport_dev, "Invalid Component Registers in RCRB");
987 return ERR_PTR(-ENXIO);
993 if (component_reg_phys != CXL_RESOURCE_NONE)
994 dev_dbg(dport_dev, "Component Registers found for dport: %pa\n",
995 &component_reg_phys);
997 dport->dport_dev = dport_dev;
998 dport->port_id = port_id;
1001 rc = cxl_dport_setup_regs(dport, component_reg_phys);
1005 cond_cxl_root_lock(port);
1006 rc = add_dport(port, dport);
1007 cond_cxl_root_unlock(port);
1011 get_device(dport_dev);
1012 rc = devm_add_action_or_reset(host, cxl_dport_remove, dport);
1016 rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
1020 rc = devm_add_action_or_reset(host, cxl_dport_unlink, dport);
1028 * devm_cxl_add_dport - append VH downstream port data to a cxl_port
1029 * @port: the cxl_port that references this dport
1030 * @dport_dev: firmware or PCI device representing the dport
1031 * @port_id: identifier for this dport in a decoder's target list
1032 * @component_reg_phys: optional location of CXL component registers
1034 * Note that dports are appended to the devm release action's of the
1035 * either the port's host (for root ports), or the port itself (for
1038 struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
1039 struct device *dport_dev, int port_id,
1040 resource_size_t component_reg_phys)
1042 struct cxl_dport *dport;
1044 dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1045 component_reg_phys, CXL_RESOURCE_NONE);
1046 if (IS_ERR(dport)) {
1047 dev_dbg(dport_dev, "failed to add dport to %s: %ld\n",
1048 dev_name(&port->dev), PTR_ERR(dport));
1050 dev_dbg(dport_dev, "dport added to %s\n",
1051 dev_name(&port->dev));
1056 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL);
1059 * devm_cxl_add_rch_dport - append RCH downstream port data to a cxl_port
1060 * @port: the cxl_port that references this dport
1061 * @dport_dev: firmware or PCI device representing the dport
1062 * @port_id: identifier for this dport in a decoder's target list
1063 * @rcrb: mandatory location of a Root Complex Register Block
1065 * See CXL 3.0 9.11.8 CXL Devices Attached to an RCH
1067 struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
1068 struct device *dport_dev, int port_id,
1069 resource_size_t rcrb)
1071 struct cxl_dport *dport;
1073 if (rcrb == CXL_RESOURCE_NONE) {
1074 dev_dbg(&port->dev, "failed to add RCH dport, missing RCRB\n");
1075 return ERR_PTR(-EINVAL);
1078 dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1079 CXL_RESOURCE_NONE, rcrb);
1080 if (IS_ERR(dport)) {
1081 dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n",
1082 dev_name(&port->dev), PTR_ERR(dport));
1084 dev_dbg(dport_dev, "RCH dport added to %s\n",
1085 dev_name(&port->dev));
1090 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_rch_dport, CXL);
1092 static int add_ep(struct cxl_ep *new)
1094 struct cxl_port *port = new->dport->port;
1097 device_lock(&port->dev);
1099 device_unlock(&port->dev);
1102 rc = xa_insert(&port->endpoints, (unsigned long)new->ep, new,
1104 device_unlock(&port->dev);
1110 * cxl_add_ep - register an endpoint's interest in a port
1111 * @dport: the dport that routes to @ep_dev
1112 * @ep_dev: device representing the endpoint
1114 * Intermediate CXL ports are scanned based on the arrival of endpoints.
1115 * When those endpoints depart the port can be destroyed once all
1116 * endpoints that care about that port have been removed.
1118 static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev)
1123 ep = kzalloc(sizeof(*ep), GFP_KERNEL);
1127 ep->ep = get_device(ep_dev);
1136 struct cxl_find_port_ctx {
1137 const struct device *dport_dev;
1138 const struct cxl_port *parent_port;
1139 struct cxl_dport **dport;
1142 static int match_port_by_dport(struct device *dev, const void *data)
1144 const struct cxl_find_port_ctx *ctx = data;
1145 struct cxl_dport *dport;
1146 struct cxl_port *port;
1148 if (!is_cxl_port(dev))
1150 if (ctx->parent_port && dev->parent != &ctx->parent_port->dev)
1153 port = to_cxl_port(dev);
1154 dport = cxl_find_dport_by_dev(port, ctx->dport_dev);
1156 *ctx->dport = dport;
1157 return dport != NULL;
1160 static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx)
1164 if (!ctx->dport_dev)
1167 dev = bus_find_device(&cxl_bus_type, NULL, ctx, match_port_by_dport);
1169 return to_cxl_port(dev);
1173 static struct cxl_port *find_cxl_port(struct device *dport_dev,
1174 struct cxl_dport **dport)
1176 struct cxl_find_port_ctx ctx = {
1177 .dport_dev = dport_dev,
1180 struct cxl_port *port;
1182 port = __find_cxl_port(&ctx);
1186 static struct cxl_port *find_cxl_port_at(struct cxl_port *parent_port,
1187 struct device *dport_dev,
1188 struct cxl_dport **dport)
1190 struct cxl_find_port_ctx ctx = {
1191 .dport_dev = dport_dev,
1192 .parent_port = parent_port,
1195 struct cxl_port *port;
1197 port = __find_cxl_port(&ctx);
1202 * All users of grandparent() are using it to walk PCIe-like switch port
1203 * hierarchy. A PCIe switch is comprised of a bridge device representing the
1204 * upstream switch port and N bridges representing downstream switch ports. When
1205 * bridges stack the grand-parent of a downstream switch port is another
1206 * downstream switch port in the immediate ancestor switch.
1208 static struct device *grandparent(struct device *dev)
1210 if (dev && dev->parent)
1211 return dev->parent->parent;
1215 static void delete_endpoint(void *data)
1217 struct cxl_memdev *cxlmd = data;
1218 struct cxl_port *endpoint = cxlmd->endpoint;
1219 struct cxl_port *parent_port;
1220 struct device *parent;
1222 parent_port = cxl_mem_find_port(cxlmd, NULL);
1225 parent = &parent_port->dev;
1227 device_lock(parent);
1228 if (parent->driver && !endpoint->dead) {
1229 devm_release_action(parent, cxl_unlink_parent_dport, endpoint);
1230 devm_release_action(parent, cxl_unlink_uport, endpoint);
1231 devm_release_action(parent, unregister_port, endpoint);
1233 cxlmd->endpoint = NULL;
1234 device_unlock(parent);
1237 put_device(&endpoint->dev);
1240 int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
1242 struct device *dev = &cxlmd->dev;
1244 get_device(&endpoint->dev);
1245 cxlmd->endpoint = endpoint;
1246 cxlmd->depth = endpoint->depth;
1247 return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
1249 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, CXL);
1252 * The natural end of life of a non-root 'cxl_port' is when its parent port goes
1253 * through a ->remove() event ("top-down" unregistration). The unnatural trigger
1254 * for a port to be unregistered is when all memdevs beneath that port have gone
1255 * through ->remove(). This "bottom-up" removal selectively removes individual
1256 * child ports manually. This depends on devm_cxl_add_port() to not change is
1257 * devm action registration order, and for dports to have already been
1258 * destroyed by reap_dports().
1260 static void delete_switch_port(struct cxl_port *port)
1262 devm_release_action(port->dev.parent, cxl_unlink_parent_dport, port);
1263 devm_release_action(port->dev.parent, cxl_unlink_uport, port);
1264 devm_release_action(port->dev.parent, unregister_port, port);
1267 static void reap_dports(struct cxl_port *port)
1269 struct cxl_dport *dport;
1270 unsigned long index;
1272 device_lock_assert(&port->dev);
1274 xa_for_each(&port->dports, index, dport) {
1275 devm_release_action(&port->dev, cxl_dport_unlink, dport);
1276 devm_release_action(&port->dev, cxl_dport_remove, dport);
1277 devm_kfree(&port->dev, dport);
1282 struct cxl_memdev *cxlmd;
1286 static int port_has_memdev(struct device *dev, const void *data)
1288 const struct detach_ctx *ctx = data;
1289 struct cxl_port *port;
1291 if (!is_cxl_port(dev))
1294 port = to_cxl_port(dev);
1295 if (port->depth != ctx->depth)
1298 return !!cxl_ep_load(port, ctx->cxlmd);
1301 static void cxl_detach_ep(void *data)
1303 struct cxl_memdev *cxlmd = data;
1305 for (int i = cxlmd->depth - 1; i >= 1; i--) {
1306 struct cxl_port *port, *parent_port;
1307 struct detach_ctx ctx = {
1315 dev = bus_find_device(&cxl_bus_type, NULL, &ctx,
1319 port = to_cxl_port(dev);
1321 parent_port = to_cxl_port(port->dev.parent);
1322 device_lock(&parent_port->dev);
1323 device_lock(&port->dev);
1324 ep = cxl_ep_load(port, cxlmd);
1325 dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
1326 ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
1327 cxl_ep_remove(port, ep);
1328 if (ep && !port->dead && xa_empty(&port->endpoints) &&
1329 !is_cxl_root(parent_port) && parent_port->dev.driver) {
1331 * This was the last ep attached to a dynamically
1332 * enumerated port. Block new cxl_add_ep() and garbage
1339 device_unlock(&port->dev);
1342 dev_dbg(&cxlmd->dev, "delete %s\n",
1343 dev_name(&port->dev));
1344 delete_switch_port(port);
1346 put_device(&port->dev);
1347 device_unlock(&parent_port->dev);
1351 static resource_size_t find_component_registers(struct device *dev)
1353 struct cxl_register_map map;
1354 struct pci_dev *pdev;
1357 * Theoretically, CXL component registers can be hosted on a
1358 * non-PCI device, in practice, only cxl_test hits this case.
1360 if (!dev_is_pci(dev))
1361 return CXL_RESOURCE_NONE;
1363 pdev = to_pci_dev(dev);
1365 cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
1366 return map.resource;
1369 static int add_port_attach_ep(struct cxl_memdev *cxlmd,
1370 struct device *uport_dev,
1371 struct device *dport_dev)
1373 struct device *dparent = grandparent(dport_dev);
1374 struct cxl_port *port, *parent_port = NULL;
1375 struct cxl_dport *dport, *parent_dport;
1376 resource_size_t component_reg_phys;
1381 * The iteration reached the topology root without finding the
1382 * CXL-root 'cxl_port' on a previous iteration, fail for now to
1383 * be re-probed after platform driver attaches.
1385 dev_dbg(&cxlmd->dev, "%s is a root dport\n",
1386 dev_name(dport_dev));
1390 parent_port = find_cxl_port(dparent, &parent_dport);
1392 /* iterate to create this parent_port */
1396 device_lock(&parent_port->dev);
1397 if (!parent_port->dev.driver) {
1398 dev_warn(&cxlmd->dev,
1399 "port %s:%s disabled, failed to enumerate CXL.mem\n",
1400 dev_name(&parent_port->dev), dev_name(uport_dev));
1401 port = ERR_PTR(-ENXIO);
1405 port = find_cxl_port_at(parent_port, dport_dev, &dport);
1407 component_reg_phys = find_component_registers(uport_dev);
1408 port = devm_cxl_add_port(&parent_port->dev, uport_dev,
1409 component_reg_phys, parent_dport);
1410 /* retry find to pick up the new dport information */
1412 port = find_cxl_port_at(parent_port, dport_dev, &dport);
1415 device_unlock(&parent_port->dev);
1420 dev_dbg(&cxlmd->dev, "add to new port %s:%s\n",
1421 dev_name(&port->dev), dev_name(port->uport_dev));
1422 rc = cxl_add_ep(dport, &cxlmd->dev);
1425 * "can't" happen, but this error code means
1426 * something to the caller, so translate it.
1430 put_device(&port->dev);
1433 put_device(&parent_port->dev);
1437 int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
1439 struct device *dev = &cxlmd->dev;
1440 struct device *iter;
1444 * Skip intermediate port enumeration in the RCH case, there
1445 * are no ports in between a host bridge and an endpoint.
1447 if (cxlmd->cxlds->rcd)
1450 rc = devm_add_action_or_reset(&cxlmd->dev, cxl_detach_ep, cxlmd);
1455 * Scan for and add all cxl_ports in this device's ancestry.
1456 * Repeat until no more ports are added. Abort if a port add
1460 for (iter = dev; iter; iter = grandparent(iter)) {
1461 struct device *dport_dev = grandparent(iter);
1462 struct device *uport_dev;
1463 struct cxl_dport *dport;
1464 struct cxl_port *port;
1469 uport_dev = dport_dev->parent;
1471 dev_warn(dev, "at %s no parent for dport: %s\n",
1472 dev_name(iter), dev_name(dport_dev));
1476 dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
1477 dev_name(iter), dev_name(dport_dev),
1478 dev_name(uport_dev));
1479 port = find_cxl_port(dport_dev, &dport);
1481 dev_dbg(&cxlmd->dev,
1482 "found already registered port %s:%s\n",
1483 dev_name(&port->dev),
1484 dev_name(port->uport_dev));
1485 rc = cxl_add_ep(dport, &cxlmd->dev);
1488 * If the endpoint already exists in the port's list,
1489 * that's ok, it was added on a previous pass.
1490 * Otherwise, retry in add_port_attach_ep() after taking
1491 * the parent_port lock as the current port may be being
1494 if (rc && rc != -EBUSY) {
1495 put_device(&port->dev);
1499 /* Any more ports to add between this one and the root? */
1500 if (!dev_is_cxl_root_child(&port->dev)) {
1501 put_device(&port->dev);
1505 put_device(&port->dev);
1509 rc = add_port_attach_ep(cxlmd, uport_dev, dport_dev);
1510 /* port missing, try to add parent */
1513 /* failed to add ep or port */
1516 /* port added, new descendants possible, start over */
1522 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL);
1524 struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
1525 struct cxl_dport **dport)
1527 return find_cxl_port(pdev->dev.parent, dport);
1529 EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, CXL);
1531 struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
1532 struct cxl_dport **dport)
1534 return find_cxl_port(grandparent(&cxlmd->dev), dport);
1536 EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, CXL);
1538 static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
1539 struct cxl_port *port, int *target_map)
1546 device_lock_assert(&port->dev);
1548 if (xa_empty(&port->dports))
1551 write_seqlock(&cxlsd->target_lock);
1552 for (i = 0; i < cxlsd->nr_targets; i++) {
1553 struct cxl_dport *dport = find_dport(port, target_map[i]);
1559 cxlsd->target[i] = dport;
1561 write_sequnlock(&cxlsd->target_lock);
1566 struct cxl_dport *cxl_hb_modulo(struct cxl_root_decoder *cxlrd, int pos)
1568 struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
1569 struct cxl_decoder *cxld = &cxlsd->cxld;
1572 iw = cxld->interleave_ways;
1573 if (dev_WARN_ONCE(&cxld->dev, iw != cxlsd->nr_targets,
1574 "misconfigured root decoder\n"))
1577 return cxlrd->cxlsd.target[pos % iw];
1579 EXPORT_SYMBOL_NS_GPL(cxl_hb_modulo, CXL);
1581 static struct lock_class_key cxl_decoder_key;
1584 * cxl_decoder_init - Common decoder setup / initialization
1585 * @port: owning port of this decoder
1586 * @cxld: common decoder properties to initialize
1588 * A port may contain one or more decoders. Each of those decoders
1589 * enable some address space for CXL.mem utilization. A decoder is
1590 * expected to be configured by the caller before registering via
1593 static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld)
1598 rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
1602 /* need parent to stick around to release the id */
1603 get_device(&port->dev);
1607 device_initialize(dev);
1608 lockdep_set_class(&dev->mutex, &cxl_decoder_key);
1609 device_set_pm_not_required(dev);
1610 dev->parent = &port->dev;
1611 dev->bus = &cxl_bus_type;
1613 /* Pre initialize an "empty" decoder */
1614 cxld->interleave_ways = 1;
1615 cxld->interleave_granularity = PAGE_SIZE;
1616 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1617 cxld->hpa_range = (struct range) {
1625 static int cxl_switch_decoder_init(struct cxl_port *port,
1626 struct cxl_switch_decoder *cxlsd,
1629 if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1632 cxlsd->nr_targets = nr_targets;
1633 seqlock_init(&cxlsd->target_lock);
1634 return cxl_decoder_init(port, &cxlsd->cxld);
1638 * cxl_root_decoder_alloc - Allocate a root level decoder
1639 * @port: owning CXL root of this decoder
1640 * @nr_targets: static number of downstream targets
1641 * @calc_hb: which host bridge covers the n'th position by granularity
1643 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1644 * 'CXL root' decoder is one that decodes from a top-level / static platform
1645 * firmware description of CXL resources into a CXL standard decode
1648 struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
1649 unsigned int nr_targets,
1650 cxl_calc_hb_fn calc_hb)
1652 struct cxl_root_decoder *cxlrd;
1653 struct cxl_switch_decoder *cxlsd;
1654 struct cxl_decoder *cxld;
1657 if (!is_cxl_root(port))
1658 return ERR_PTR(-EINVAL);
1660 cxlrd = kzalloc(struct_size(cxlrd, cxlsd.target, nr_targets),
1663 return ERR_PTR(-ENOMEM);
1665 cxlsd = &cxlrd->cxlsd;
1666 rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1672 cxlrd->calc_hb = calc_hb;
1673 mutex_init(&cxlrd->range_lock);
1675 cxld = &cxlsd->cxld;
1676 cxld->dev.type = &cxl_decoder_root_type;
1678 * cxl_root_decoder_release() special cases negative ids to
1679 * detect memregion_alloc() failures.
1681 atomic_set(&cxlrd->region_id, -1);
1682 rc = memregion_alloc(GFP_KERNEL);
1684 put_device(&cxld->dev);
1688 atomic_set(&cxlrd->region_id, rc);
1691 EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL);
1694 * cxl_switch_decoder_alloc - Allocate a switch level decoder
1695 * @port: owning CXL switch port of this decoder
1696 * @nr_targets: max number of dynamically addressable downstream targets
1698 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1699 * 'switch' decoder is any decoder that can be enumerated by PCIe
1700 * topology and the HDM Decoder Capability. This includes the decoders
1701 * that sit between Switch Upstream Ports / Switch Downstream Ports and
1702 * Host Bridges / Root Ports.
1704 struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
1705 unsigned int nr_targets)
1707 struct cxl_switch_decoder *cxlsd;
1708 struct cxl_decoder *cxld;
1711 if (is_cxl_root(port) || is_cxl_endpoint(port))
1712 return ERR_PTR(-EINVAL);
1714 cxlsd = kzalloc(struct_size(cxlsd, target, nr_targets), GFP_KERNEL);
1716 return ERR_PTR(-ENOMEM);
1718 rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1724 cxld = &cxlsd->cxld;
1725 cxld->dev.type = &cxl_decoder_switch_type;
1728 EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, CXL);
1731 * cxl_endpoint_decoder_alloc - Allocate an endpoint decoder
1732 * @port: owning port of this decoder
1734 * Return: A new cxl decoder to be registered by cxl_decoder_add()
1736 struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port)
1738 struct cxl_endpoint_decoder *cxled;
1739 struct cxl_decoder *cxld;
1742 if (!is_cxl_endpoint(port))
1743 return ERR_PTR(-EINVAL);
1745 cxled = kzalloc(sizeof(*cxled), GFP_KERNEL);
1747 return ERR_PTR(-ENOMEM);
1750 cxld = &cxled->cxld;
1751 rc = cxl_decoder_init(port, cxld);
1757 cxld->dev.type = &cxl_decoder_endpoint_type;
1760 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, CXL);
1763 * cxl_decoder_add_locked - Add a decoder with targets
1764 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1765 * @target_map: A list of downstream ports that this decoder can direct memory
1766 * traffic to. These numbers should correspond with the port number
1767 * in the PCIe Link Capabilities structure.
1769 * Certain types of decoders may not have any targets. The main example of this
1770 * is an endpoint device. A more awkward example is a hostbridge whose root
1771 * ports get hot added (technically possible, though unlikely).
1773 * This is the locked variant of cxl_decoder_add().
1775 * Context: Process context. Expects the device lock of the port that owns the
1778 * Return: Negative error code if the decoder wasn't properly configured; else
1781 int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map)
1783 struct cxl_port *port;
1787 if (WARN_ON_ONCE(!cxld))
1790 if (WARN_ON_ONCE(IS_ERR(cxld)))
1791 return PTR_ERR(cxld);
1793 if (cxld->interleave_ways < 1)
1798 port = to_cxl_port(cxld->dev.parent);
1799 if (!is_endpoint_decoder(dev)) {
1800 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
1802 rc = decoder_populate_targets(cxlsd, port, target_map);
1803 if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {
1805 "Failed to populate active decoder targets\n");
1810 rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
1814 return device_add(dev);
1816 EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, CXL);
1819 * cxl_decoder_add - Add a decoder with targets
1820 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1821 * @target_map: A list of downstream ports that this decoder can direct memory
1822 * traffic to. These numbers should correspond with the port number
1823 * in the PCIe Link Capabilities structure.
1825 * This is the unlocked variant of cxl_decoder_add_locked().
1826 * See cxl_decoder_add_locked().
1828 * Context: Process context. Takes and releases the device lock of the port that
1831 int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map)
1833 struct cxl_port *port;
1836 if (WARN_ON_ONCE(!cxld))
1839 if (WARN_ON_ONCE(IS_ERR(cxld)))
1840 return PTR_ERR(cxld);
1842 port = to_cxl_port(cxld->dev.parent);
1844 device_lock(&port->dev);
1845 rc = cxl_decoder_add_locked(cxld, target_map);
1846 device_unlock(&port->dev);
1850 EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, CXL);
1852 static void cxld_unregister(void *dev)
1854 struct cxl_endpoint_decoder *cxled;
1856 if (is_endpoint_decoder(dev)) {
1857 cxled = to_cxl_endpoint_decoder(dev);
1858 cxl_decoder_kill_region(cxled);
1861 device_unregister(dev);
1864 int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
1866 return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
1868 EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, CXL);
1871 * __cxl_driver_register - register a driver for the cxl bus
1872 * @cxl_drv: cxl driver structure to attach
1873 * @owner: owning module/driver
1874 * @modname: KBUILD_MODNAME for parent driver
1876 int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
1877 const char *modname)
1879 if (!cxl_drv->probe) {
1880 pr_debug("%s ->probe() must be specified\n", modname);
1884 if (!cxl_drv->name) {
1885 pr_debug("%s ->name must be specified\n", modname);
1890 pr_debug("%s ->id must be specified\n", modname);
1894 cxl_drv->drv.bus = &cxl_bus_type;
1895 cxl_drv->drv.owner = owner;
1896 cxl_drv->drv.mod_name = modname;
1897 cxl_drv->drv.name = cxl_drv->name;
1899 return driver_register(&cxl_drv->drv);
1901 EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, CXL);
1903 void cxl_driver_unregister(struct cxl_driver *cxl_drv)
1905 driver_unregister(&cxl_drv->drv);
1907 EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, CXL);
1909 static int cxl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
1911 return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,
1912 cxl_device_id(dev));
1915 static int cxl_bus_match(struct device *dev, struct device_driver *drv)
1917 return cxl_device_id(dev) == to_cxl_drv(drv)->id;
1920 static int cxl_bus_probe(struct device *dev)
1924 rc = to_cxl_drv(dev->driver)->probe(dev);
1925 dev_dbg(dev, "probe: %d\n", rc);
1929 static void cxl_bus_remove(struct device *dev)
1931 struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
1933 if (cxl_drv->remove)
1934 cxl_drv->remove(dev);
1937 static struct workqueue_struct *cxl_bus_wq;
1939 static void cxl_bus_rescan_queue(struct work_struct *w)
1941 int rc = bus_rescan_devices(&cxl_bus_type);
1943 pr_debug("CXL bus rescan result: %d\n", rc);
1946 void cxl_bus_rescan(void)
1948 static DECLARE_WORK(rescan_work, cxl_bus_rescan_queue);
1950 queue_work(cxl_bus_wq, &rescan_work);
1952 EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, CXL);
1954 void cxl_bus_drain(void)
1956 drain_workqueue(cxl_bus_wq);
1958 EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, CXL);
1960 bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
1962 return queue_work(cxl_bus_wq, &cxlmd->detach_work);
1964 EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL);
1966 /* for user tooling to ensure port disable work has completed */
1967 static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count)
1969 if (sysfs_streq(buf, "1")) {
1970 flush_workqueue(cxl_bus_wq);
1977 static BUS_ATTR_WO(flush);
1979 static struct attribute *cxl_bus_attributes[] = {
1980 &bus_attr_flush.attr,
1984 static struct attribute_group cxl_bus_attribute_group = {
1985 .attrs = cxl_bus_attributes,
1988 static const struct attribute_group *cxl_bus_attribute_groups[] = {
1989 &cxl_bus_attribute_group,
1993 struct bus_type cxl_bus_type = {
1995 .uevent = cxl_bus_uevent,
1996 .match = cxl_bus_match,
1997 .probe = cxl_bus_probe,
1998 .remove = cxl_bus_remove,
1999 .bus_groups = cxl_bus_attribute_groups,
2001 EXPORT_SYMBOL_NS_GPL(cxl_bus_type, CXL);
2003 static struct dentry *cxl_debugfs;
2005 struct dentry *cxl_debugfs_create_dir(const char *dir)
2007 return debugfs_create_dir(dir, cxl_debugfs);
2009 EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, CXL);
2011 static __init int cxl_core_init(void)
2015 cxl_debugfs = debugfs_create_dir("cxl", NULL);
2019 rc = cxl_memdev_init();
2023 cxl_bus_wq = alloc_ordered_workqueue("cxl_port", 0);
2029 rc = bus_register(&cxl_bus_type);
2033 rc = cxl_region_init();
2040 bus_unregister(&cxl_bus_type);
2042 destroy_workqueue(cxl_bus_wq);
2048 static void cxl_core_exit(void)
2051 bus_unregister(&cxl_bus_type);
2052 destroy_workqueue(cxl_bus_wq);
2054 debugfs_remove_recursive(cxl_debugfs);
2057 subsys_initcall(cxl_core_init);
2058 module_exit(cxl_core_exit);
2059 MODULE_LICENSE("GPL v2");