1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3 #include <linux/memregion.h>
4 #include <linux/genalloc.h>
5 #include <linux/device.h>
6 #include <linux/module.h>
7 #include <linux/slab.h>
8 #include <linux/uuid.h>
9 #include <linux/sort.h>
10 #include <linux/idr.h>
16 * DOC: cxl core region
18 * CXL Regions represent mapped memory capacity in system physical address
19 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
20 * Memory ranges, Regions represent the active mapped capacity by the HDM
21 * Decoder Capability structures throughout the Host Bridges, Switches, and
22 * Endpoints in the topology.
24 * Region configuration has ordering constraints. UUID may be set at any time
25 * but is only visible for persistent regions.
26 * 1. Interleave granularity
31 static struct cxl_region *to_cxl_region(struct device *dev);
33 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
36 struct cxl_region *cxlr = to_cxl_region(dev);
37 struct cxl_region_params *p = &cxlr->params;
40 rc = down_read_interruptible(&cxl_region_rwsem);
43 if (cxlr->mode != CXL_DECODER_PMEM)
44 rc = sysfs_emit(buf, "\n");
46 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
47 up_read(&cxl_region_rwsem);
52 static int is_dup(struct device *match, void *data)
54 struct cxl_region_params *p;
55 struct cxl_region *cxlr;
58 if (!is_cxl_region(match))
61 lockdep_assert_held(&cxl_region_rwsem);
62 cxlr = to_cxl_region(match);
65 if (uuid_equal(&p->uuid, uuid)) {
66 dev_dbg(match, "already has uuid: %pUb\n", uuid);
73 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
74 const char *buf, size_t len)
76 struct cxl_region *cxlr = to_cxl_region(dev);
77 struct cxl_region_params *p = &cxlr->params;
81 if (len != UUID_STRING_LEN + 1)
84 rc = uuid_parse(buf, &temp);
88 if (uuid_is_null(&temp))
91 rc = down_write_killable(&cxl_region_rwsem);
95 if (uuid_equal(&p->uuid, &temp))
99 if (p->state >= CXL_CONFIG_ACTIVE)
102 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
106 uuid_copy(&p->uuid, &temp);
108 up_write(&cxl_region_rwsem);
114 static DEVICE_ATTR_RW(uuid);
116 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
117 struct cxl_region *cxlr)
119 return xa_load(&port->regions, (unsigned long)cxlr);
122 static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
124 if (!cpu_cache_has_invalidate_memregion()) {
125 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
128 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
132 "Failed to synchronize CPU cache state\n");
137 cpu_cache_invalidate_memregion(IORES_DESC_CXL);
141 static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
143 struct cxl_region_params *p = &cxlr->params;
147 * Before region teardown attempt to flush, and if the flush
148 * fails cancel the region teardown for data consistency
151 rc = cxl_region_invalidate_memregion(cxlr);
155 for (i = count - 1; i >= 0; i--) {
156 struct cxl_endpoint_decoder *cxled = p->targets[i];
157 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
158 struct cxl_port *iter = cxled_to_port(cxled);
159 struct cxl_dev_state *cxlds = cxlmd->cxlds;
165 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
166 iter = to_cxl_port(iter->dev.parent);
168 for (ep = cxl_ep_load(iter, cxlmd); iter;
169 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
170 struct cxl_region_ref *cxl_rr;
171 struct cxl_decoder *cxld;
173 cxl_rr = cxl_rr_load(iter, cxlr);
174 cxld = cxl_rr->decoder;
176 rc = cxld->reset(cxld);
179 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
183 rc = cxled->cxld.reset(&cxled->cxld);
186 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
189 /* all decoders associated with this region have been torn down */
190 clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
195 static int commit_decoder(struct cxl_decoder *cxld)
197 struct cxl_switch_decoder *cxlsd = NULL;
200 return cxld->commit(cxld);
202 if (is_switch_decoder(&cxld->dev))
203 cxlsd = to_cxl_switch_decoder(&cxld->dev);
205 if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
206 "->commit() is required\n"))
211 static int cxl_region_decode_commit(struct cxl_region *cxlr)
213 struct cxl_region_params *p = &cxlr->params;
216 for (i = 0; i < p->nr_targets; i++) {
217 struct cxl_endpoint_decoder *cxled = p->targets[i];
218 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
219 struct cxl_region_ref *cxl_rr;
220 struct cxl_decoder *cxld;
221 struct cxl_port *iter;
224 /* commit bottom up */
225 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
226 iter = to_cxl_port(iter->dev.parent)) {
227 cxl_rr = cxl_rr_load(iter, cxlr);
228 cxld = cxl_rr->decoder;
229 rc = commit_decoder(cxld);
235 /* programming @iter failed, teardown */
236 for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
237 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
238 cxl_rr = cxl_rr_load(iter, cxlr);
239 cxld = cxl_rr->decoder;
244 cxled->cxld.reset(&cxled->cxld);
252 /* undo the targets that were successfully committed */
253 cxl_region_decode_reset(cxlr, i);
257 static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
258 const char *buf, size_t len)
260 struct cxl_region *cxlr = to_cxl_region(dev);
261 struct cxl_region_params *p = &cxlr->params;
265 rc = kstrtobool(buf, &commit);
269 rc = down_write_killable(&cxl_region_rwsem);
273 /* Already in the requested state? */
274 if (commit && p->state >= CXL_CONFIG_COMMIT)
276 if (!commit && p->state < CXL_CONFIG_COMMIT)
279 /* Not ready to commit? */
280 if (commit && p->state < CXL_CONFIG_ACTIVE) {
286 * Invalidate caches before region setup to drop any speculative
287 * consumption of this address space
289 rc = cxl_region_invalidate_memregion(cxlr);
294 rc = cxl_region_decode_commit(cxlr);
296 p->state = CXL_CONFIG_COMMIT;
298 p->state = CXL_CONFIG_RESET_PENDING;
299 up_write(&cxl_region_rwsem);
300 device_release_driver(&cxlr->dev);
301 down_write(&cxl_region_rwsem);
304 * The lock was dropped, so need to revalidate that the reset is
307 if (p->state == CXL_CONFIG_RESET_PENDING) {
308 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
310 * Revert to committed since there may still be active
311 * decoders associated with this region, or move forward
312 * to active to mark the reset successful
315 p->state = CXL_CONFIG_COMMIT;
317 p->state = CXL_CONFIG_ACTIVE;
322 up_write(&cxl_region_rwsem);
329 static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
332 struct cxl_region *cxlr = to_cxl_region(dev);
333 struct cxl_region_params *p = &cxlr->params;
336 rc = down_read_interruptible(&cxl_region_rwsem);
339 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
340 up_read(&cxl_region_rwsem);
344 static DEVICE_ATTR_RW(commit);
346 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
349 struct device *dev = kobj_to_dev(kobj);
350 struct cxl_region *cxlr = to_cxl_region(dev);
353 * Support tooling that expects to find a 'uuid' attribute for all
354 * regions regardless of mode.
356 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
361 static ssize_t interleave_ways_show(struct device *dev,
362 struct device_attribute *attr, char *buf)
364 struct cxl_region *cxlr = to_cxl_region(dev);
365 struct cxl_region_params *p = &cxlr->params;
368 rc = down_read_interruptible(&cxl_region_rwsem);
371 rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
372 up_read(&cxl_region_rwsem);
377 static const struct attribute_group *get_cxl_region_target_group(void);
379 static ssize_t interleave_ways_store(struct device *dev,
380 struct device_attribute *attr,
381 const char *buf, size_t len)
383 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
384 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
385 struct cxl_region *cxlr = to_cxl_region(dev);
386 struct cxl_region_params *p = &cxlr->params;
387 unsigned int val, save;
391 rc = kstrtouint(buf, 0, &val);
395 rc = ways_to_eiw(val, &iw);
400 * Even for x3, x9, and x12 interleaves the region interleave must be a
401 * power of 2 multiple of the host bridge interleave.
403 if (!is_power_of_2(val / cxld->interleave_ways) ||
404 (val % cxld->interleave_ways)) {
405 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
409 rc = down_write_killable(&cxl_region_rwsem);
412 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
417 save = p->interleave_ways;
418 p->interleave_ways = val;
419 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
421 p->interleave_ways = save;
423 up_write(&cxl_region_rwsem);
428 static DEVICE_ATTR_RW(interleave_ways);
430 static ssize_t interleave_granularity_show(struct device *dev,
431 struct device_attribute *attr,
434 struct cxl_region *cxlr = to_cxl_region(dev);
435 struct cxl_region_params *p = &cxlr->params;
438 rc = down_read_interruptible(&cxl_region_rwsem);
441 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
442 up_read(&cxl_region_rwsem);
447 static ssize_t interleave_granularity_store(struct device *dev,
448 struct device_attribute *attr,
449 const char *buf, size_t len)
451 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
452 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
453 struct cxl_region *cxlr = to_cxl_region(dev);
454 struct cxl_region_params *p = &cxlr->params;
458 rc = kstrtoint(buf, 0, &val);
462 rc = granularity_to_eig(val, &ig);
467 * When the host-bridge is interleaved, disallow region granularity !=
468 * root granularity. Regions with a granularity less than the root
469 * interleave result in needing multiple endpoints to support a single
470 * slot in the interleave (possible to support in the future). Regions
471 * with a granularity greater than the root interleave result in invalid
472 * DPA translations (invalid to support).
474 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
477 rc = down_write_killable(&cxl_region_rwsem);
480 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
485 p->interleave_granularity = val;
487 up_write(&cxl_region_rwsem);
492 static DEVICE_ATTR_RW(interleave_granularity);
494 static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
497 struct cxl_region *cxlr = to_cxl_region(dev);
498 struct cxl_region_params *p = &cxlr->params;
499 u64 resource = -1ULL;
502 rc = down_read_interruptible(&cxl_region_rwsem);
506 resource = p->res->start;
507 rc = sysfs_emit(buf, "%#llx\n", resource);
508 up_read(&cxl_region_rwsem);
512 static DEVICE_ATTR_RO(resource);
514 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
517 struct cxl_region *cxlr = to_cxl_region(dev);
519 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
521 static DEVICE_ATTR_RO(mode);
523 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
525 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
526 struct cxl_region_params *p = &cxlr->params;
527 struct resource *res;
530 lockdep_assert_held_write(&cxl_region_rwsem);
532 /* Nothing to do... */
533 if (p->res && resource_size(p->res) == size)
536 /* To change size the old size must be freed first */
540 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
543 /* ways, granularity and uuid (if PMEM) need to be set before HPA */
544 if (!p->interleave_ways || !p->interleave_granularity ||
545 (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
548 div_u64_rem(size, SZ_256M * p->interleave_ways, &remainder);
552 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
553 dev_name(&cxlr->dev));
555 dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n",
561 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
566 static void cxl_region_iomem_release(struct cxl_region *cxlr)
568 struct cxl_region_params *p = &cxlr->params;
570 if (device_is_registered(&cxlr->dev))
571 lockdep_assert_held_write(&cxl_region_rwsem);
574 * Autodiscovered regions may not have been able to insert their
578 remove_resource(p->res);
584 static int free_hpa(struct cxl_region *cxlr)
586 struct cxl_region_params *p = &cxlr->params;
588 lockdep_assert_held_write(&cxl_region_rwsem);
593 if (p->state >= CXL_CONFIG_ACTIVE)
596 cxl_region_iomem_release(cxlr);
597 p->state = CXL_CONFIG_IDLE;
601 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
602 const char *buf, size_t len)
604 struct cxl_region *cxlr = to_cxl_region(dev);
608 rc = kstrtou64(buf, 0, &val);
612 rc = down_write_killable(&cxl_region_rwsem);
617 rc = alloc_hpa(cxlr, val);
620 up_write(&cxl_region_rwsem);
628 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
631 struct cxl_region *cxlr = to_cxl_region(dev);
632 struct cxl_region_params *p = &cxlr->params;
636 rc = down_read_interruptible(&cxl_region_rwsem);
640 size = resource_size(p->res);
641 rc = sysfs_emit(buf, "%#llx\n", size);
642 up_read(&cxl_region_rwsem);
646 static DEVICE_ATTR_RW(size);
648 static struct attribute *cxl_region_attrs[] = {
650 &dev_attr_commit.attr,
651 &dev_attr_interleave_ways.attr,
652 &dev_attr_interleave_granularity.attr,
653 &dev_attr_resource.attr,
659 static const struct attribute_group cxl_region_group = {
660 .attrs = cxl_region_attrs,
661 .is_visible = cxl_region_visible,
664 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
666 struct cxl_region_params *p = &cxlr->params;
667 struct cxl_endpoint_decoder *cxled;
670 rc = down_read_interruptible(&cxl_region_rwsem);
674 if (pos >= p->interleave_ways) {
675 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
681 cxled = p->targets[pos];
683 rc = sysfs_emit(buf, "\n");
685 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
687 up_read(&cxl_region_rwsem);
692 static int match_free_decoder(struct device *dev, void *data)
694 struct cxl_decoder *cxld;
697 if (!is_switch_decoder(dev))
700 cxld = to_cxl_decoder(dev);
702 /* enforce ordered allocation */
714 static int match_auto_decoder(struct device *dev, void *data)
716 struct cxl_region_params *p = data;
717 struct cxl_decoder *cxld;
720 if (!is_switch_decoder(dev))
723 cxld = to_cxl_decoder(dev);
724 r = &cxld->hpa_range;
726 if (p->res && p->res->start == r->start && p->res->end == r->end)
732 static struct cxl_decoder *cxl_region_find_decoder(struct cxl_port *port,
733 struct cxl_region *cxlr)
738 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
739 dev = device_find_child(&port->dev, &cxlr->params,
742 dev = device_find_child(&port->dev, &id, match_free_decoder);
746 * This decoder is pinned registered as long as the endpoint decoder is
747 * registered, and endpoint decoder unregistration holds the
748 * cxl_region_rwsem over unregister events, so no need to hold on to
749 * this extra reference.
752 return to_cxl_decoder(dev);
755 static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port,
756 struct cxl_region *cxlr)
758 struct cxl_region_params *p = &cxlr->params;
759 struct cxl_region_ref *cxl_rr, *iter;
763 xa_for_each(&port->regions, index, iter) {
764 struct cxl_region_params *ip = &iter->region->params;
769 if (ip->res->start > p->res->start) {
771 "%s: HPA order violation %s:%pr vs %pr\n",
772 dev_name(&port->dev),
773 dev_name(&iter->region->dev), ip->res, p->res);
774 return ERR_PTR(-EBUSY);
778 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
780 return ERR_PTR(-ENOMEM);
782 cxl_rr->region = cxlr;
783 cxl_rr->nr_targets = 1;
784 xa_init(&cxl_rr->endpoints);
786 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
789 "%s: failed to track region reference: %d\n",
790 dev_name(&port->dev), rc);
798 static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
800 struct cxl_region *cxlr = cxl_rr->region;
801 struct cxl_decoder *cxld = cxl_rr->decoder;
806 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
807 if (cxld->region == cxlr) {
809 put_device(&cxlr->dev);
813 static void free_region_ref(struct cxl_region_ref *cxl_rr)
815 struct cxl_port *port = cxl_rr->port;
816 struct cxl_region *cxlr = cxl_rr->region;
818 cxl_rr_free_decoder(cxl_rr);
819 xa_erase(&port->regions, (unsigned long)cxlr);
820 xa_destroy(&cxl_rr->endpoints);
824 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
825 struct cxl_endpoint_decoder *cxled)
828 struct cxl_port *port = cxl_rr->port;
829 struct cxl_region *cxlr = cxl_rr->region;
830 struct cxl_decoder *cxld = cxl_rr->decoder;
831 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
834 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
843 get_device(&cxlr->dev);
849 static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
850 struct cxl_endpoint_decoder *cxled,
851 struct cxl_region_ref *cxl_rr)
853 struct cxl_decoder *cxld;
855 if (port == cxled_to_port(cxled))
858 cxld = cxl_region_find_decoder(port, cxlr);
860 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
861 dev_name(&port->dev));
866 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
867 dev_name(&port->dev), dev_name(&cxld->dev),
868 dev_name(&cxld->region->dev));
873 * Endpoints should already match the region type, but backstop that
874 * assumption with an assertion. Switch-decoders change mapping-type
875 * based on what is mapped when they are assigned to a region.
877 dev_WARN_ONCE(&cxlr->dev,
878 port == cxled_to_port(cxled) &&
879 cxld->target_type != cxlr->type,
880 "%s:%s mismatch decoder type %d -> %d\n",
881 dev_name(&cxled_to_memdev(cxled)->dev),
882 dev_name(&cxld->dev), cxld->target_type, cxlr->type);
883 cxld->target_type = cxlr->type;
884 cxl_rr->decoder = cxld;
889 * cxl_port_attach_region() - track a region's interest in a port by endpoint
890 * @port: port to add a new region reference 'struct cxl_region_ref'
891 * @cxlr: region to attach to @port
892 * @cxled: endpoint decoder used to create or further pin a region reference
893 * @pos: interleave position of @cxled in @cxlr
895 * The attach event is an opportunity to validate CXL decode setup
896 * constraints and record metadata needed for programming HDM decoders,
897 * in particular decoder target lists.
901 * - validate that there are no other regions with a higher HPA already
902 * associated with @port
903 * - establish a region reference if one is not already present
905 * - additionally allocate a decoder instance that will host @cxlr on
908 * - pin the region reference by the endpoint
909 * - account for how many entries in @port's target list are needed to
910 * cover all of the added endpoints.
912 static int cxl_port_attach_region(struct cxl_port *port,
913 struct cxl_region *cxlr,
914 struct cxl_endpoint_decoder *cxled, int pos)
916 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
917 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
918 struct cxl_region_ref *cxl_rr;
919 bool nr_targets_inc = false;
920 struct cxl_decoder *cxld;
924 lockdep_assert_held_write(&cxl_region_rwsem);
926 cxl_rr = cxl_rr_load(port, cxlr);
928 struct cxl_ep *ep_iter;
932 * Walk the existing endpoints that have been attached to
933 * @cxlr at @port and see if they share the same 'next' port
934 * in the downstream direction. I.e. endpoints that share common
937 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
940 if (ep_iter->next == ep->next) {
947 * New target port, or @port is an endpoint port that always
948 * accounts its own local decode as a target.
950 if (!found || !ep->next) {
951 cxl_rr->nr_targets++;
952 nr_targets_inc = true;
955 cxl_rr = alloc_region_ref(port, cxlr);
956 if (IS_ERR(cxl_rr)) {
958 "%s: failed to allocate region reference\n",
959 dev_name(&port->dev));
960 return PTR_ERR(cxl_rr);
962 nr_targets_inc = true;
964 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
968 cxld = cxl_rr->decoder;
970 rc = cxl_rr_ep_add(cxl_rr, cxled);
973 "%s: failed to track endpoint %s:%s reference\n",
974 dev_name(&port->dev), dev_name(&cxlmd->dev),
975 dev_name(&cxld->dev));
980 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
981 dev_name(port->uport_dev), dev_name(&port->dev),
982 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
983 dev_name(&cxled->cxld.dev), pos,
984 ep ? ep->next ? dev_name(ep->next->uport_dev) :
985 dev_name(&cxlmd->dev) :
987 cxl_rr->nr_eps, cxl_rr->nr_targets);
992 cxl_rr->nr_targets--;
993 if (cxl_rr->nr_eps == 0)
994 free_region_ref(cxl_rr);
998 static void cxl_port_detach_region(struct cxl_port *port,
999 struct cxl_region *cxlr,
1000 struct cxl_endpoint_decoder *cxled)
1002 struct cxl_region_ref *cxl_rr;
1003 struct cxl_ep *ep = NULL;
1005 lockdep_assert_held_write(&cxl_region_rwsem);
1007 cxl_rr = cxl_rr_load(port, cxlr);
1012 * Endpoint ports do not carry cxl_ep references, and they
1013 * never target more than one endpoint by definition
1015 if (cxl_rr->decoder == &cxled->cxld)
1018 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1020 struct cxl_ep *ep_iter;
1021 unsigned long index;
1025 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1026 if (ep_iter->next == ep->next) {
1032 cxl_rr->nr_targets--;
1035 if (cxl_rr->nr_eps == 0)
1036 free_region_ref(cxl_rr);
1039 static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1040 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1043 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1044 struct cxl_region *cxlr = cxl_rr->region;
1045 struct cxl_region_params *p = &cxlr->params;
1046 struct cxl_endpoint_decoder *cxled_peer;
1047 struct cxl_port *port = cxl_rr->port;
1048 struct cxl_memdev *cxlmd_peer;
1049 struct cxl_ep *ep_peer;
1050 int pos = cxled->pos;
1053 * If this position wants to share a dport with the last endpoint mapped
1054 * then that endpoint, at index 'position - distance', must also be
1055 * mapped by this dport.
1057 if (pos < distance) {
1058 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1059 dev_name(port->uport_dev), dev_name(&port->dev),
1060 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1063 cxled_peer = p->targets[pos - distance];
1064 cxlmd_peer = cxled_to_memdev(cxled_peer);
1065 ep_peer = cxl_ep_load(port, cxlmd_peer);
1066 if (ep->dport != ep_peer->dport) {
1068 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1069 dev_name(port->uport_dev), dev_name(&port->dev),
1070 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1071 dev_name(&cxlmd_peer->dev),
1072 dev_name(&cxled_peer->cxld.dev));
1079 static int cxl_port_setup_targets(struct cxl_port *port,
1080 struct cxl_region *cxlr,
1081 struct cxl_endpoint_decoder *cxled)
1083 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1084 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1085 struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1086 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1087 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1088 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1089 struct cxl_region_params *p = &cxlr->params;
1090 struct cxl_decoder *cxld = cxl_rr->decoder;
1091 struct cxl_switch_decoder *cxlsd;
1096 * While root level decoders support x3, x6, x12, switch level
1097 * decoders only support powers of 2 up to x16.
1099 if (!is_power_of_2(cxl_rr->nr_targets)) {
1100 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1101 dev_name(port->uport_dev), dev_name(&port->dev),
1102 cxl_rr->nr_targets);
1106 cxlsd = to_cxl_switch_decoder(&cxld->dev);
1107 if (cxl_rr->nr_targets_set) {
1111 * Passthrough decoders impose no distance requirements between
1114 if (cxl_rr->nr_targets == 1)
1117 distance = p->nr_targets / cxl_rr->nr_targets;
1118 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1119 if (ep->dport == cxlsd->target[i]) {
1120 rc = check_last_peer(cxled, ep, cxl_rr,
1124 goto out_target_set;
1129 if (is_cxl_root(parent_port)) {
1131 * Root decoder IG is always set to value in CFMWS which
1132 * may be different than this region's IG. We can use the
1133 * region's IG here since interleave_granularity_store()
1134 * does not allow interleaved host-bridges with
1135 * root IG != region IG.
1137 parent_ig = p->interleave_granularity;
1138 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1140 * For purposes of address bit routing, use power-of-2 math for
1143 if (!is_power_of_2(parent_iw))
1146 struct cxl_region_ref *parent_rr;
1147 struct cxl_decoder *parent_cxld;
1149 parent_rr = cxl_rr_load(parent_port, cxlr);
1150 parent_cxld = parent_rr->decoder;
1151 parent_ig = parent_cxld->interleave_granularity;
1152 parent_iw = parent_cxld->interleave_ways;
1155 rc = granularity_to_eig(parent_ig, &peig);
1157 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1158 dev_name(parent_port->uport_dev),
1159 dev_name(&parent_port->dev), parent_ig);
1163 rc = ways_to_eiw(parent_iw, &peiw);
1165 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1166 dev_name(parent_port->uport_dev),
1167 dev_name(&parent_port->dev), parent_iw);
1171 iw = cxl_rr->nr_targets;
1172 rc = ways_to_eiw(iw, &eiw);
1174 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1175 dev_name(port->uport_dev), dev_name(&port->dev), iw);
1180 * Interleave granularity is a multiple of @parent_port granularity.
1181 * Multiplier is the parent port interleave ways.
1183 rc = granularity_to_eig(parent_ig * parent_iw, &eig);
1186 "%s: invalid granularity calculation (%d * %d)\n",
1187 dev_name(&parent_port->dev), parent_ig, parent_iw);
1191 rc = eig_to_granularity(eig, &ig);
1193 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1194 dev_name(port->uport_dev), dev_name(&port->dev),
1199 if (iw > 8 || iw > cxlsd->nr_targets) {
1201 "%s:%s:%s: ways: %d overflows targets: %d\n",
1202 dev_name(port->uport_dev), dev_name(&port->dev),
1203 dev_name(&cxld->dev), iw, cxlsd->nr_targets);
1207 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1208 if (cxld->interleave_ways != iw ||
1209 cxld->interleave_granularity != ig ||
1210 cxld->hpa_range.start != p->res->start ||
1211 cxld->hpa_range.end != p->res->end ||
1212 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1214 "%s:%s %s expected iw: %d ig: %d %pr\n",
1215 dev_name(port->uport_dev), dev_name(&port->dev),
1216 __func__, iw, ig, p->res);
1218 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1219 dev_name(port->uport_dev), dev_name(&port->dev),
1220 __func__, cxld->interleave_ways,
1221 cxld->interleave_granularity,
1222 (cxld->flags & CXL_DECODER_F_ENABLE) ?
1225 cxld->hpa_range.start, cxld->hpa_range.end);
1229 cxld->interleave_ways = iw;
1230 cxld->interleave_granularity = ig;
1231 cxld->hpa_range = (struct range) {
1232 .start = p->res->start,
1236 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1237 dev_name(&port->dev), iw, ig);
1239 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1241 "%s:%s: targets full trying to add %s:%s at %d\n",
1242 dev_name(port->uport_dev), dev_name(&port->dev),
1243 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1246 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1247 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1248 dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1249 dev_name(port->uport_dev), dev_name(&port->dev),
1250 dev_name(&cxlsd->cxld.dev),
1251 dev_name(ep->dport->dport_dev),
1252 cxl_rr->nr_targets_set);
1256 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1259 cxl_rr->nr_targets_set += inc;
1260 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1261 dev_name(port->uport_dev), dev_name(&port->dev),
1262 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1263 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1268 static void cxl_port_reset_targets(struct cxl_port *port,
1269 struct cxl_region *cxlr)
1271 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1272 struct cxl_decoder *cxld;
1275 * After the last endpoint has been detached the entire cxl_rr may now
1280 cxl_rr->nr_targets_set = 0;
1282 cxld = cxl_rr->decoder;
1283 cxld->hpa_range = (struct range) {
1289 static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1291 struct cxl_region_params *p = &cxlr->params;
1292 struct cxl_endpoint_decoder *cxled;
1293 struct cxl_dev_state *cxlds;
1294 struct cxl_memdev *cxlmd;
1295 struct cxl_port *iter;
1300 * In the auto-discovery case skip automatic teardown since the
1301 * address space is already active
1303 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1306 for (i = 0; i < p->nr_targets; i++) {
1307 cxled = p->targets[i];
1308 cxlmd = cxled_to_memdev(cxled);
1309 cxlds = cxlmd->cxlds;
1314 iter = cxled_to_port(cxled);
1315 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1316 iter = to_cxl_port(iter->dev.parent);
1318 for (ep = cxl_ep_load(iter, cxlmd); iter;
1319 iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1320 cxl_port_reset_targets(iter, cxlr);
1324 static int cxl_region_setup_targets(struct cxl_region *cxlr)
1326 struct cxl_region_params *p = &cxlr->params;
1327 struct cxl_endpoint_decoder *cxled;
1328 struct cxl_dev_state *cxlds;
1329 int i, rc, rch = 0, vh = 0;
1330 struct cxl_memdev *cxlmd;
1331 struct cxl_port *iter;
1334 for (i = 0; i < p->nr_targets; i++) {
1335 cxled = p->targets[i];
1336 cxlmd = cxled_to_memdev(cxled);
1337 cxlds = cxlmd->cxlds;
1339 /* validate that all targets agree on topology */
1347 iter = cxled_to_port(cxled);
1348 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1349 iter = to_cxl_port(iter->dev.parent);
1352 * Descend the topology tree programming / validating
1353 * targets while looking for conflicts.
1355 for (ep = cxl_ep_load(iter, cxlmd); iter;
1356 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1357 rc = cxl_port_setup_targets(iter, cxlr, cxled);
1359 cxl_region_teardown_targets(cxlr);
1366 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1367 cxl_region_teardown_targets(cxlr);
1374 static int cxl_region_validate_position(struct cxl_region *cxlr,
1375 struct cxl_endpoint_decoder *cxled,
1378 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1379 struct cxl_region_params *p = &cxlr->params;
1382 if (pos < 0 || pos >= p->interleave_ways) {
1383 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1384 p->interleave_ways);
1388 if (p->targets[pos] == cxled)
1391 if (p->targets[pos]) {
1392 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1393 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1395 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1396 pos, dev_name(&cxlmd_target->dev),
1397 dev_name(&cxled_target->cxld.dev));
1401 for (i = 0; i < p->interleave_ways; i++) {
1402 struct cxl_endpoint_decoder *cxled_target;
1403 struct cxl_memdev *cxlmd_target;
1405 cxled_target = p->targets[i];
1409 cxlmd_target = cxled_to_memdev(cxled_target);
1410 if (cxlmd_target == cxlmd) {
1412 "%s already specified at position %d via: %s\n",
1413 dev_name(&cxlmd->dev), pos,
1414 dev_name(&cxled_target->cxld.dev));
1422 static int cxl_region_attach_position(struct cxl_region *cxlr,
1423 struct cxl_root_decoder *cxlrd,
1424 struct cxl_endpoint_decoder *cxled,
1425 const struct cxl_dport *dport, int pos)
1427 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1428 struct cxl_port *iter;
1431 if (cxlrd->calc_hb(cxlrd, pos) != dport) {
1432 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1433 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1434 dev_name(&cxlrd->cxlsd.cxld.dev));
1438 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1439 iter = to_cxl_port(iter->dev.parent)) {
1440 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1448 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1449 iter = to_cxl_port(iter->dev.parent))
1450 cxl_port_detach_region(iter, cxlr, cxled);
1454 static int cxl_region_attach_auto(struct cxl_region *cxlr,
1455 struct cxl_endpoint_decoder *cxled, int pos)
1457 struct cxl_region_params *p = &cxlr->params;
1459 if (cxled->state != CXL_DECODER_STATE_AUTO) {
1461 "%s: unable to add decoder to autodetected region\n",
1462 dev_name(&cxled->cxld.dev));
1467 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1468 dev_name(&cxled->cxld.dev), pos);
1472 if (p->nr_targets >= p->interleave_ways) {
1473 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1474 dev_name(&cxled->cxld.dev));
1479 * Temporarily record the endpoint decoder into the target array. Yes,
1480 * this means that userspace can view devices in the wrong position
1481 * before the region activates, and must be careful to understand when
1482 * it might be racing region autodiscovery.
1484 pos = p->nr_targets;
1485 p->targets[pos] = cxled;
1492 static int cmp_interleave_pos(const void *a, const void *b)
1494 struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1495 struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1497 return cxled_a->pos - cxled_b->pos;
1500 static struct cxl_port *next_port(struct cxl_port *port)
1502 if (!port->parent_dport)
1504 return port->parent_dport->port;
1507 static int match_switch_decoder_by_range(struct device *dev, void *data)
1509 struct cxl_switch_decoder *cxlsd;
1510 struct range *r1, *r2 = data;
1512 if (!is_switch_decoder(dev))
1515 cxlsd = to_cxl_switch_decoder(dev);
1516 r1 = &cxlsd->cxld.hpa_range;
1518 if (is_root_decoder(dev))
1519 return range_contains(r1, r2);
1520 return (r1->start == r2->start && r1->end == r2->end);
1523 static int find_pos_and_ways(struct cxl_port *port, struct range *range,
1524 int *pos, int *ways)
1526 struct cxl_switch_decoder *cxlsd;
1527 struct cxl_port *parent;
1531 parent = next_port(port);
1535 dev = device_find_child(&parent->dev, range,
1536 match_switch_decoder_by_range);
1538 dev_err(port->uport_dev,
1539 "failed to find decoder mapping %#llx-%#llx\n",
1540 range->start, range->end);
1543 cxlsd = to_cxl_switch_decoder(dev);
1544 *ways = cxlsd->cxld.interleave_ways;
1546 for (int i = 0; i < *ways; i++) {
1547 if (cxlsd->target[i] == port->parent_dport) {
1559 * cxl_calc_interleave_pos() - calculate an endpoint position in a region
1560 * @cxled: endpoint decoder member of given region
1562 * The endpoint position is calculated by traversing the topology from
1563 * the endpoint to the root decoder and iteratively applying this
1566 * position = position * parent_ways + parent_pos;
1568 * ...where @position is inferred from switch and root decoder target lists.
1570 * Return: position >= 0 on success
1573 static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled)
1575 struct cxl_port *iter, *port = cxled_to_port(cxled);
1576 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1577 struct range *range = &cxled->cxld.hpa_range;
1578 int parent_ways = 0, parent_pos = 0, pos = 0;
1582 * Example: the expected interleave order of the 4-way region shown
1583 * below is: mem0, mem2, mem1, mem3
1587 * host_bridge_0 host_bridge_1
1589 * mem0 mem1 mem2 mem3
1591 * In the example the calculator will iterate twice. The first iteration
1592 * uses the mem position in the host-bridge and the ways of the host-
1593 * bridge to generate the first, or local, position. The second
1594 * iteration uses the host-bridge position in the root_port and the ways
1595 * of the root_port to refine the position.
1597 * A trace of the calculation per endpoint looks like this:
1598 * mem0: pos = 0 * 2 + 0 mem2: pos = 0 * 2 + 0
1599 * pos = 0 * 2 + 0 pos = 0 * 2 + 1
1602 * mem1: pos = 0 * 2 + 1 mem3: pos = 0 * 2 + 1
1603 * pos = 1 * 2 + 0 pos = 1 * 2 + 1
1606 * Note that while this example is simple, the method applies to more
1607 * complex topologies, including those with switches.
1610 /* Iterate from endpoint to root_port refining the position */
1611 for (iter = port; iter; iter = next_port(iter)) {
1612 if (is_cxl_root(iter))
1615 rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways);
1619 pos = pos * parent_ways + parent_pos;
1622 dev_dbg(&cxlmd->dev,
1623 "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n",
1624 dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent),
1625 dev_name(&port->dev), range->start, range->end, pos);
1630 static int cxl_region_sort_targets(struct cxl_region *cxlr)
1632 struct cxl_region_params *p = &cxlr->params;
1635 for (i = 0; i < p->nr_targets; i++) {
1636 struct cxl_endpoint_decoder *cxled = p->targets[i];
1638 cxled->pos = cxl_calc_interleave_pos(cxled);
1640 * Record that sorting failed, but still continue to calc
1641 * cxled->pos so that follow-on code paths can reliably
1642 * do p->targets[cxled->pos] to self-reference their entry.
1647 /* Keep the cxlr target list in interleave position order */
1648 sort(p->targets, p->nr_targets, sizeof(p->targets[0]),
1649 cmp_interleave_pos, NULL);
1651 dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1655 static int cxl_region_attach(struct cxl_region *cxlr,
1656 struct cxl_endpoint_decoder *cxled, int pos)
1658 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1659 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1660 struct cxl_region_params *p = &cxlr->params;
1661 struct cxl_port *ep_port, *root_port;
1662 struct cxl_dport *dport;
1665 if (cxled->mode != cxlr->mode) {
1666 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1667 dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1671 if (cxled->mode == CXL_DECODER_DEAD) {
1672 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1676 /* all full of members, or interleave config not established? */
1677 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1678 dev_dbg(&cxlr->dev, "region already active\n");
1680 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1681 dev_dbg(&cxlr->dev, "interleave config missing\n");
1685 if (p->nr_targets >= p->interleave_ways) {
1686 dev_dbg(&cxlr->dev, "region already has %d endpoints\n",
1691 ep_port = cxled_to_port(cxled);
1692 root_port = cxlrd_to_port(cxlrd);
1693 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1695 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1696 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1697 dev_name(cxlr->dev.parent));
1701 if (cxled->cxld.target_type != cxlr->type) {
1702 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1703 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1704 cxled->cxld.target_type, cxlr->type);
1708 if (!cxled->dpa_res) {
1709 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1710 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1714 if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1715 resource_size(p->res)) {
1717 "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1718 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1719 (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1720 (u64)resource_size(p->res));
1724 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1727 rc = cxl_region_attach_auto(cxlr, cxled, pos);
1731 /* await more targets to arrive... */
1732 if (p->nr_targets < p->interleave_ways)
1736 * All targets are here, which implies all PCI enumeration that
1737 * affects this region has been completed. Walk the topology to
1738 * sort the devices into their relative region decode position.
1740 rc = cxl_region_sort_targets(cxlr);
1744 for (i = 0; i < p->nr_targets; i++) {
1745 cxled = p->targets[i];
1746 ep_port = cxled_to_port(cxled);
1747 dport = cxl_find_dport_by_dev(root_port,
1748 ep_port->host_bridge);
1749 rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1755 rc = cxl_region_setup_targets(cxlr);
1760 * If target setup succeeds in the autodiscovery case
1761 * then the region is already committed.
1763 p->state = CXL_CONFIG_COMMIT;
1768 rc = cxl_region_validate_position(cxlr, cxled, pos);
1772 rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
1776 p->targets[pos] = cxled;
1780 if (p->nr_targets == p->interleave_ways) {
1781 rc = cxl_region_setup_targets(cxlr);
1784 p->state = CXL_CONFIG_ACTIVE;
1787 cxled->cxld.interleave_ways = p->interleave_ways;
1788 cxled->cxld.interleave_granularity = p->interleave_granularity;
1789 cxled->cxld.hpa_range = (struct range) {
1790 .start = p->res->start,
1794 if (p->nr_targets != p->interleave_ways)
1798 * Test the auto-discovery position calculator function
1799 * against this successfully created user-defined region.
1800 * A fail message here means that this interleave config
1801 * will fail when presented as CXL_REGION_F_AUTO.
1803 for (int i = 0; i < p->nr_targets; i++) {
1804 struct cxl_endpoint_decoder *cxled = p->targets[i];
1807 test_pos = cxl_calc_interleave_pos(cxled);
1808 dev_dbg(&cxled->cxld.dev,
1809 "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n",
1810 (test_pos == cxled->pos) ? "success" : "fail",
1811 test_pos, cxled->pos);
1817 static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
1819 struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
1820 struct cxl_region *cxlr = cxled->cxld.region;
1821 struct cxl_region_params *p;
1824 lockdep_assert_held_write(&cxl_region_rwsem);
1830 get_device(&cxlr->dev);
1832 if (p->state > CXL_CONFIG_ACTIVE) {
1834 * TODO: tear down all impacted regions if a device is
1835 * removed out of order
1837 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
1840 p->state = CXL_CONFIG_ACTIVE;
1843 for (iter = ep_port; !is_cxl_root(iter);
1844 iter = to_cxl_port(iter->dev.parent))
1845 cxl_port_detach_region(iter, cxlr, cxled);
1847 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
1848 p->targets[cxled->pos] != cxled) {
1849 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1851 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
1852 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1857 if (p->state == CXL_CONFIG_ACTIVE) {
1858 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
1859 cxl_region_teardown_targets(cxlr);
1861 p->targets[cxled->pos] = NULL;
1863 cxled->cxld.hpa_range = (struct range) {
1868 /* notify the region driver that one of its targets has departed */
1869 up_write(&cxl_region_rwsem);
1870 device_release_driver(&cxlr->dev);
1871 down_write(&cxl_region_rwsem);
1873 put_device(&cxlr->dev);
1877 void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
1879 down_write(&cxl_region_rwsem);
1880 cxled->mode = CXL_DECODER_DEAD;
1881 cxl_region_detach(cxled);
1882 up_write(&cxl_region_rwsem);
1885 static int attach_target(struct cxl_region *cxlr,
1886 struct cxl_endpoint_decoder *cxled, int pos,
1891 if (state == TASK_INTERRUPTIBLE)
1892 rc = down_write_killable(&cxl_region_rwsem);
1894 down_write(&cxl_region_rwsem);
1898 down_read(&cxl_dpa_rwsem);
1899 rc = cxl_region_attach(cxlr, cxled, pos);
1900 up_read(&cxl_dpa_rwsem);
1901 up_write(&cxl_region_rwsem);
1905 static int detach_target(struct cxl_region *cxlr, int pos)
1907 struct cxl_region_params *p = &cxlr->params;
1910 rc = down_write_killable(&cxl_region_rwsem);
1914 if (pos >= p->interleave_ways) {
1915 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1916 p->interleave_ways);
1921 if (!p->targets[pos]) {
1926 rc = cxl_region_detach(p->targets[pos]);
1928 up_write(&cxl_region_rwsem);
1932 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
1937 if (sysfs_streq(buf, "\n"))
1938 rc = detach_target(cxlr, pos);
1942 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
1946 if (!is_endpoint_decoder(dev)) {
1951 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
1952 TASK_INTERRUPTIBLE);
1962 #define TARGET_ATTR_RW(n) \
1963 static ssize_t target##n##_show( \
1964 struct device *dev, struct device_attribute *attr, char *buf) \
1966 return show_targetN(to_cxl_region(dev), buf, (n)); \
1968 static ssize_t target##n##_store(struct device *dev, \
1969 struct device_attribute *attr, \
1970 const char *buf, size_t len) \
1972 return store_targetN(to_cxl_region(dev), buf, (n), len); \
1974 static DEVICE_ATTR_RW(target##n)
1993 static struct attribute *target_attrs[] = {
1994 &dev_attr_target0.attr,
1995 &dev_attr_target1.attr,
1996 &dev_attr_target2.attr,
1997 &dev_attr_target3.attr,
1998 &dev_attr_target4.attr,
1999 &dev_attr_target5.attr,
2000 &dev_attr_target6.attr,
2001 &dev_attr_target7.attr,
2002 &dev_attr_target8.attr,
2003 &dev_attr_target9.attr,
2004 &dev_attr_target10.attr,
2005 &dev_attr_target11.attr,
2006 &dev_attr_target12.attr,
2007 &dev_attr_target13.attr,
2008 &dev_attr_target14.attr,
2009 &dev_attr_target15.attr,
2013 static umode_t cxl_region_target_visible(struct kobject *kobj,
2014 struct attribute *a, int n)
2016 struct device *dev = kobj_to_dev(kobj);
2017 struct cxl_region *cxlr = to_cxl_region(dev);
2018 struct cxl_region_params *p = &cxlr->params;
2020 if (n < p->interleave_ways)
2025 static const struct attribute_group cxl_region_target_group = {
2026 .attrs = target_attrs,
2027 .is_visible = cxl_region_target_visible,
2030 static const struct attribute_group *get_cxl_region_target_group(void)
2032 return &cxl_region_target_group;
2035 static const struct attribute_group *region_groups[] = {
2036 &cxl_base_attribute_group,
2038 &cxl_region_target_group,
2042 static void cxl_region_release(struct device *dev)
2044 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
2045 struct cxl_region *cxlr = to_cxl_region(dev);
2046 int id = atomic_read(&cxlrd->region_id);
2049 * Try to reuse the recently idled id rather than the cached
2050 * next id to prevent the region id space from increasing
2054 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
2059 memregion_free(cxlr->id);
2061 put_device(dev->parent);
2065 const struct device_type cxl_region_type = {
2066 .name = "cxl_region",
2067 .release = cxl_region_release,
2068 .groups = region_groups
2071 bool is_cxl_region(struct device *dev)
2073 return dev->type == &cxl_region_type;
2075 EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
2077 static struct cxl_region *to_cxl_region(struct device *dev)
2079 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2080 "not a cxl_region device\n"))
2083 return container_of(dev, struct cxl_region, dev);
2086 static void unregister_region(void *dev)
2088 struct cxl_region *cxlr = to_cxl_region(dev);
2089 struct cxl_region_params *p = &cxlr->params;
2095 * Now that region sysfs is shutdown, the parameter block is now
2096 * read-only, so no need to hold the region rwsem to access the
2097 * region parameters.
2099 for (i = 0; i < p->interleave_ways; i++)
2100 detach_target(cxlr, i);
2102 cxl_region_iomem_release(cxlr);
2106 static struct lock_class_key cxl_region_key;
2108 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2110 struct cxl_region *cxlr;
2113 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2116 return ERR_PTR(-ENOMEM);
2120 device_initialize(dev);
2121 lockdep_set_class(&dev->mutex, &cxl_region_key);
2122 dev->parent = &cxlrd->cxlsd.cxld.dev;
2124 * Keep root decoder pinned through cxl_region_release to fixup
2125 * region id allocations
2127 get_device(dev->parent);
2128 device_set_pm_not_required(dev);
2129 dev->bus = &cxl_bus_type;
2130 dev->type = &cxl_region_type;
2137 * devm_cxl_add_region - Adds a region to a decoder
2138 * @cxlrd: root decoder
2139 * @id: memregion id to create, or memregion_free() on failure
2140 * @mode: mode for the endpoint decoders of this region
2141 * @type: select whether this is an expander or accelerator (type-2 or type-3)
2143 * This is the second step of region initialization. Regions exist within an
2144 * address space which is mapped by a @cxlrd.
2146 * Return: 0 if the region was added to the @cxlrd, else returns negative error
2147 * code. The region will be named "regionZ" where Z is the unique region number.
2149 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2151 enum cxl_decoder_mode mode,
2152 enum cxl_decoder_type type)
2154 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2155 struct cxl_region *cxlr;
2160 case CXL_DECODER_RAM:
2161 case CXL_DECODER_PMEM:
2164 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2165 return ERR_PTR(-EINVAL);
2168 cxlr = cxl_region_alloc(cxlrd, id);
2175 rc = dev_set_name(dev, "region%d", id);
2179 rc = device_add(dev);
2183 rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2187 dev_dbg(port->uport_dev, "%s: created %s\n",
2188 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2196 static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2198 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2201 static ssize_t create_pmem_region_show(struct device *dev,
2202 struct device_attribute *attr, char *buf)
2204 return __create_region_show(to_cxl_root_decoder(dev), buf);
2207 static ssize_t create_ram_region_show(struct device *dev,
2208 struct device_attribute *attr, char *buf)
2210 return __create_region_show(to_cxl_root_decoder(dev), buf);
2213 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2214 enum cxl_decoder_mode mode, int id)
2218 rc = memregion_alloc(GFP_KERNEL);
2222 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2224 return ERR_PTR(-EBUSY);
2227 return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2230 static ssize_t create_pmem_region_store(struct device *dev,
2231 struct device_attribute *attr,
2232 const char *buf, size_t len)
2234 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2235 struct cxl_region *cxlr;
2238 rc = sscanf(buf, "region%d\n", &id);
2242 cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id);
2244 return PTR_ERR(cxlr);
2248 DEVICE_ATTR_RW(create_pmem_region);
2250 static ssize_t create_ram_region_store(struct device *dev,
2251 struct device_attribute *attr,
2252 const char *buf, size_t len)
2254 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2255 struct cxl_region *cxlr;
2258 rc = sscanf(buf, "region%d\n", &id);
2262 cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id);
2264 return PTR_ERR(cxlr);
2268 DEVICE_ATTR_RW(create_ram_region);
2270 static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2273 struct cxl_decoder *cxld = to_cxl_decoder(dev);
2276 rc = down_read_interruptible(&cxl_region_rwsem);
2281 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2283 rc = sysfs_emit(buf, "\n");
2284 up_read(&cxl_region_rwsem);
2288 DEVICE_ATTR_RO(region);
2290 static struct cxl_region *
2291 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2293 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2294 struct device *region_dev;
2296 region_dev = device_find_child_by_name(&cxld->dev, name);
2298 return ERR_PTR(-ENODEV);
2300 return to_cxl_region(region_dev);
2303 static ssize_t delete_region_store(struct device *dev,
2304 struct device_attribute *attr,
2305 const char *buf, size_t len)
2307 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2308 struct cxl_port *port = to_cxl_port(dev->parent);
2309 struct cxl_region *cxlr;
2311 cxlr = cxl_find_region_by_name(cxlrd, buf);
2313 return PTR_ERR(cxlr);
2315 devm_release_action(port->uport_dev, unregister_region, cxlr);
2316 put_device(&cxlr->dev);
2320 DEVICE_ATTR_WO(delete_region);
2322 static void cxl_pmem_region_release(struct device *dev)
2324 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2327 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2328 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2330 put_device(&cxlmd->dev);
2336 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2337 &cxl_base_attribute_group,
2341 const struct device_type cxl_pmem_region_type = {
2342 .name = "cxl_pmem_region",
2343 .release = cxl_pmem_region_release,
2344 .groups = cxl_pmem_region_attribute_groups,
2347 bool is_cxl_pmem_region(struct device *dev)
2349 return dev->type == &cxl_pmem_region_type;
2351 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
2353 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2355 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2356 "not a cxl_pmem_region device\n"))
2358 return container_of(dev, struct cxl_pmem_region, dev);
2360 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
2362 struct cxl_poison_context {
2363 struct cxl_port *port;
2364 enum cxl_decoder_mode mode;
2368 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2369 struct cxl_poison_context *ctx)
2371 struct cxl_dev_state *cxlds = cxlmd->cxlds;
2376 * Collect poison for the remaining unmapped resources
2377 * after poison is collected by committed endpoints.
2379 * Knowing that PMEM must always follow RAM, get poison
2380 * for unmapped resources based on the last decoder's mode:
2381 * ram: scan remains of ram range, then any pmem range
2382 * pmem: scan remains of pmem range
2385 if (ctx->mode == CXL_DECODER_RAM) {
2386 offset = ctx->offset;
2387 length = resource_size(&cxlds->ram_res) - offset;
2388 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2394 if (ctx->mode == CXL_DECODER_PMEM) {
2395 offset = ctx->offset;
2396 length = resource_size(&cxlds->dpa_res) - offset;
2399 } else if (resource_size(&cxlds->pmem_res)) {
2400 offset = cxlds->pmem_res.start;
2401 length = resource_size(&cxlds->pmem_res);
2406 return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2409 static int poison_by_decoder(struct device *dev, void *arg)
2411 struct cxl_poison_context *ctx = arg;
2412 struct cxl_endpoint_decoder *cxled;
2413 struct cxl_memdev *cxlmd;
2417 if (!is_endpoint_decoder(dev))
2420 cxled = to_cxl_endpoint_decoder(dev);
2421 if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2425 * Regions are only created with single mode decoders: pmem or ram.
2426 * Linux does not support mixed mode decoders. This means that
2427 * reading poison per endpoint decoder adheres to the requirement
2428 * that poison reads of pmem and ram must be separated.
2429 * CXL 3.0 Spec 8.2.9.8.4.1
2431 if (cxled->mode == CXL_DECODER_MIXED) {
2432 dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2436 cxlmd = cxled_to_memdev(cxled);
2438 offset = cxled->dpa_res->start - cxled->skip;
2439 length = cxled->skip;
2440 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2441 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2447 offset = cxled->dpa_res->start;
2448 length = cxled->dpa_res->end - offset + 1;
2449 rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2450 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2455 /* Iterate until commit_end is reached */
2456 if (cxled->cxld.id == ctx->port->commit_end) {
2457 ctx->offset = cxled->dpa_res->end + 1;
2458 ctx->mode = cxled->mode;
2465 int cxl_get_poison_by_endpoint(struct cxl_port *port)
2467 struct cxl_poison_context ctx;
2470 rc = down_read_interruptible(&cxl_region_rwsem);
2474 ctx = (struct cxl_poison_context) {
2478 rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2480 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2483 up_read(&cxl_region_rwsem);
2487 static struct lock_class_key cxl_pmem_region_key;
2489 static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr)
2491 struct cxl_region_params *p = &cxlr->params;
2492 struct cxl_nvdimm_bridge *cxl_nvb;
2493 struct cxl_pmem_region *cxlr_pmem;
2497 down_read(&cxl_region_rwsem);
2498 if (p->state != CXL_CONFIG_COMMIT) {
2499 cxlr_pmem = ERR_PTR(-ENXIO);
2503 cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets),
2506 cxlr_pmem = ERR_PTR(-ENOMEM);
2510 cxlr_pmem->hpa_range.start = p->res->start;
2511 cxlr_pmem->hpa_range.end = p->res->end;
2513 /* Snapshot the region configuration underneath the cxl_region_rwsem */
2514 cxlr_pmem->nr_mappings = p->nr_targets;
2515 for (i = 0; i < p->nr_targets; i++) {
2516 struct cxl_endpoint_decoder *cxled = p->targets[i];
2517 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2518 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2521 * Regions never span CXL root devices, so by definition the
2522 * bridge for one device is the same for all.
2525 cxl_nvb = cxl_find_nvdimm_bridge(cxlmd);
2527 cxlr_pmem = ERR_PTR(-ENODEV);
2530 cxlr->cxl_nvb = cxl_nvb;
2533 get_device(&cxlmd->dev);
2534 m->start = cxled->dpa_res->start;
2535 m->size = resource_size(cxled->dpa_res);
2539 dev = &cxlr_pmem->dev;
2540 cxlr_pmem->cxlr = cxlr;
2541 cxlr->cxlr_pmem = cxlr_pmem;
2542 device_initialize(dev);
2543 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2544 device_set_pm_not_required(dev);
2545 dev->parent = &cxlr->dev;
2546 dev->bus = &cxl_bus_type;
2547 dev->type = &cxl_pmem_region_type;
2549 up_read(&cxl_region_rwsem);
2554 static void cxl_dax_region_release(struct device *dev)
2556 struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
2561 static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
2562 &cxl_base_attribute_group,
2566 const struct device_type cxl_dax_region_type = {
2567 .name = "cxl_dax_region",
2568 .release = cxl_dax_region_release,
2569 .groups = cxl_dax_region_attribute_groups,
2572 static bool is_cxl_dax_region(struct device *dev)
2574 return dev->type == &cxl_dax_region_type;
2577 struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
2579 if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
2580 "not a cxl_dax_region device\n"))
2582 return container_of(dev, struct cxl_dax_region, dev);
2584 EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL);
2586 static struct lock_class_key cxl_dax_region_key;
2588 static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
2590 struct cxl_region_params *p = &cxlr->params;
2591 struct cxl_dax_region *cxlr_dax;
2594 down_read(&cxl_region_rwsem);
2595 if (p->state != CXL_CONFIG_COMMIT) {
2596 cxlr_dax = ERR_PTR(-ENXIO);
2600 cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
2602 cxlr_dax = ERR_PTR(-ENOMEM);
2606 cxlr_dax->hpa_range.start = p->res->start;
2607 cxlr_dax->hpa_range.end = p->res->end;
2609 dev = &cxlr_dax->dev;
2610 cxlr_dax->cxlr = cxlr;
2611 device_initialize(dev);
2612 lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
2613 device_set_pm_not_required(dev);
2614 dev->parent = &cxlr->dev;
2615 dev->bus = &cxl_bus_type;
2616 dev->type = &cxl_dax_region_type;
2618 up_read(&cxl_region_rwsem);
2623 static void cxlr_pmem_unregister(void *_cxlr_pmem)
2625 struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
2626 struct cxl_region *cxlr = cxlr_pmem->cxlr;
2627 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2630 * Either the bridge is in ->remove() context under the device_lock(),
2631 * or cxlr_release_nvdimm() is cancelling the bridge's release action
2632 * for @cxlr_pmem and doing it itself (while manually holding the bridge
2635 device_lock_assert(&cxl_nvb->dev);
2636 cxlr->cxlr_pmem = NULL;
2637 cxlr_pmem->cxlr = NULL;
2638 device_unregister(&cxlr_pmem->dev);
2641 static void cxlr_release_nvdimm(void *_cxlr)
2643 struct cxl_region *cxlr = _cxlr;
2644 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2646 device_lock(&cxl_nvb->dev);
2647 if (cxlr->cxlr_pmem)
2648 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
2650 device_unlock(&cxl_nvb->dev);
2651 cxlr->cxl_nvb = NULL;
2652 put_device(&cxl_nvb->dev);
2656 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
2657 * @cxlr: parent CXL region for this pmem region bridge device
2659 * Return: 0 on success negative error code on failure.
2661 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
2663 struct cxl_pmem_region *cxlr_pmem;
2664 struct cxl_nvdimm_bridge *cxl_nvb;
2668 cxlr_pmem = cxl_pmem_region_alloc(cxlr);
2669 if (IS_ERR(cxlr_pmem))
2670 return PTR_ERR(cxlr_pmem);
2671 cxl_nvb = cxlr->cxl_nvb;
2673 dev = &cxlr_pmem->dev;
2674 rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
2678 rc = device_add(dev);
2682 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2685 device_lock(&cxl_nvb->dev);
2686 if (cxl_nvb->dev.driver)
2687 rc = devm_add_action_or_reset(&cxl_nvb->dev,
2688 cxlr_pmem_unregister, cxlr_pmem);
2691 device_unlock(&cxl_nvb->dev);
2696 /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
2697 return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
2702 put_device(&cxl_nvb->dev);
2703 cxlr->cxl_nvb = NULL;
2707 static void cxlr_dax_unregister(void *_cxlr_dax)
2709 struct cxl_dax_region *cxlr_dax = _cxlr_dax;
2711 device_unregister(&cxlr_dax->dev);
2714 static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
2716 struct cxl_dax_region *cxlr_dax;
2720 cxlr_dax = cxl_dax_region_alloc(cxlr);
2721 if (IS_ERR(cxlr_dax))
2722 return PTR_ERR(cxlr_dax);
2724 dev = &cxlr_dax->dev;
2725 rc = dev_set_name(dev, "dax_region%d", cxlr->id);
2729 rc = device_add(dev);
2733 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2736 return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
2743 static int match_root_decoder_by_range(struct device *dev, void *data)
2745 struct range *r1, *r2 = data;
2746 struct cxl_root_decoder *cxlrd;
2748 if (!is_root_decoder(dev))
2751 cxlrd = to_cxl_root_decoder(dev);
2752 r1 = &cxlrd->cxlsd.cxld.hpa_range;
2753 return range_contains(r1, r2);
2756 static int match_region_by_range(struct device *dev, void *data)
2758 struct cxl_region_params *p;
2759 struct cxl_region *cxlr;
2760 struct range *r = data;
2763 if (!is_cxl_region(dev))
2766 cxlr = to_cxl_region(dev);
2769 down_read(&cxl_region_rwsem);
2770 if (p->res && p->res->start == r->start && p->res->end == r->end)
2772 up_read(&cxl_region_rwsem);
2777 /* Establish an empty region covering the given HPA range */
2778 static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
2779 struct cxl_endpoint_decoder *cxled)
2781 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2782 struct cxl_port *port = cxlrd_to_port(cxlrd);
2783 struct range *hpa = &cxled->cxld.hpa_range;
2784 struct cxl_region_params *p;
2785 struct cxl_region *cxlr;
2786 struct resource *res;
2790 cxlr = __create_region(cxlrd, cxled->mode,
2791 atomic_read(&cxlrd->region_id));
2792 } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
2795 dev_err(cxlmd->dev.parent,
2796 "%s:%s: %s failed assign region: %ld\n",
2797 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2798 __func__, PTR_ERR(cxlr));
2802 down_write(&cxl_region_rwsem);
2804 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
2805 dev_err(cxlmd->dev.parent,
2806 "%s:%s: %s autodiscovery interrupted\n",
2807 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2813 set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
2815 res = kmalloc(sizeof(*res), GFP_KERNEL);
2821 *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
2822 dev_name(&cxlr->dev));
2823 rc = insert_resource(cxlrd->res, res);
2826 * Platform-firmware may not have split resources like "System
2827 * RAM" on CXL window boundaries see cxl_region_iomem_release()
2829 dev_warn(cxlmd->dev.parent,
2830 "%s:%s: %s %s cannot insert resource\n",
2831 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2832 __func__, dev_name(&cxlr->dev));
2836 p->interleave_ways = cxled->cxld.interleave_ways;
2837 p->interleave_granularity = cxled->cxld.interleave_granularity;
2838 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2840 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
2844 dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
2845 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
2846 dev_name(&cxlr->dev), p->res, p->interleave_ways,
2847 p->interleave_granularity);
2849 /* ...to match put_device() in cxl_add_to_region() */
2850 get_device(&cxlr->dev);
2851 up_write(&cxl_region_rwsem);
2856 up_write(&cxl_region_rwsem);
2857 devm_release_action(port->uport_dev, unregister_region, cxlr);
2861 int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
2863 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2864 struct range *hpa = &cxled->cxld.hpa_range;
2865 struct cxl_decoder *cxld = &cxled->cxld;
2866 struct device *cxlrd_dev, *region_dev;
2867 struct cxl_root_decoder *cxlrd;
2868 struct cxl_region_params *p;
2869 struct cxl_region *cxlr;
2870 bool attach = false;
2873 cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
2874 match_root_decoder_by_range);
2876 dev_err(cxlmd->dev.parent,
2877 "%s:%s no CXL window for range %#llx:%#llx\n",
2878 dev_name(&cxlmd->dev), dev_name(&cxld->dev),
2879 cxld->hpa_range.start, cxld->hpa_range.end);
2883 cxlrd = to_cxl_root_decoder(cxlrd_dev);
2886 * Ensure that if multiple threads race to construct_region() for @hpa
2887 * one does the construction and the others add to that.
2889 mutex_lock(&cxlrd->range_lock);
2890 region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
2891 match_region_by_range);
2893 cxlr = construct_region(cxlrd, cxled);
2894 region_dev = &cxlr->dev;
2896 cxlr = to_cxl_region(region_dev);
2897 mutex_unlock(&cxlrd->range_lock);
2899 rc = PTR_ERR_OR_ZERO(cxlr);
2903 attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
2905 down_read(&cxl_region_rwsem);
2907 attach = p->state == CXL_CONFIG_COMMIT;
2908 up_read(&cxl_region_rwsem);
2912 * If device_attach() fails the range may still be active via
2913 * the platform-firmware memory map, otherwise the driver for
2914 * regions is local to this file, so driver matching can't fail.
2916 if (device_attach(&cxlr->dev) < 0)
2917 dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
2921 put_device(region_dev);
2923 put_device(cxlrd_dev);
2926 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL);
2928 static int is_system_ram(struct resource *res, void *arg)
2930 struct cxl_region *cxlr = arg;
2931 struct cxl_region_params *p = &cxlr->params;
2933 dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
2937 static int cxl_region_probe(struct device *dev)
2939 struct cxl_region *cxlr = to_cxl_region(dev);
2940 struct cxl_region_params *p = &cxlr->params;
2943 rc = down_read_interruptible(&cxl_region_rwsem);
2945 dev_dbg(&cxlr->dev, "probe interrupted\n");
2949 if (p->state < CXL_CONFIG_COMMIT) {
2950 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
2955 if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
2957 "failed to activate, re-commit region and retry\n");
2963 * From this point on any path that changes the region's state away from
2964 * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
2967 up_read(&cxl_region_rwsem);
2972 switch (cxlr->mode) {
2973 case CXL_DECODER_PMEM:
2974 return devm_cxl_add_pmem_region(cxlr);
2975 case CXL_DECODER_RAM:
2977 * The region can not be manged by CXL if any portion of
2978 * it is already online as 'System RAM'
2980 if (walk_iomem_res_desc(IORES_DESC_NONE,
2981 IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
2982 p->res->start, p->res->end, cxlr,
2985 return devm_cxl_add_dax_region(cxlr);
2987 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
2993 static struct cxl_driver cxl_region_driver = {
2994 .name = "cxl_region",
2995 .probe = cxl_region_probe,
2996 .id = CXL_DEVICE_REGION,
2999 int cxl_region_init(void)
3001 return cxl_driver_register(&cxl_region_driver);
3004 void cxl_region_exit(void)
3006 cxl_driver_unregister(&cxl_region_driver);
3009 MODULE_IMPORT_NS(CXL);
3010 MODULE_IMPORT_NS(DEVMEM);
3011 MODULE_ALIAS_CXL(CXL_DEVICE_REGION);