1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3 #include <linux/memregion.h>
4 #include <linux/genalloc.h>
5 #include <linux/device.h>
6 #include <linux/module.h>
7 #include <linux/slab.h>
8 #include <linux/uuid.h>
9 #include <linux/sort.h>
10 #include <linux/idr.h>
16 * DOC: cxl core region
18 * CXL Regions represent mapped memory capacity in system physical address
19 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
20 * Memory ranges, Regions represent the active mapped capacity by the HDM
21 * Decoder Capability structures throughout the Host Bridges, Switches, and
22 * Endpoints in the topology.
24 * Region configuration has ordering constraints. UUID may be set at any time
25 * but is only visible for persistent regions.
26 * 1. Interleave granularity
32 * All changes to the interleave configuration occur with this lock held
35 static DECLARE_RWSEM(cxl_region_rwsem);
37 static struct cxl_region *to_cxl_region(struct device *dev);
39 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
42 struct cxl_region *cxlr = to_cxl_region(dev);
43 struct cxl_region_params *p = &cxlr->params;
46 rc = down_read_interruptible(&cxl_region_rwsem);
49 if (cxlr->mode != CXL_DECODER_PMEM)
50 rc = sysfs_emit(buf, "\n");
52 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
53 up_read(&cxl_region_rwsem);
58 static int is_dup(struct device *match, void *data)
60 struct cxl_region_params *p;
61 struct cxl_region *cxlr;
64 if (!is_cxl_region(match))
67 lockdep_assert_held(&cxl_region_rwsem);
68 cxlr = to_cxl_region(match);
71 if (uuid_equal(&p->uuid, uuid)) {
72 dev_dbg(match, "already has uuid: %pUb\n", uuid);
79 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
80 const char *buf, size_t len)
82 struct cxl_region *cxlr = to_cxl_region(dev);
83 struct cxl_region_params *p = &cxlr->params;
87 if (len != UUID_STRING_LEN + 1)
90 rc = uuid_parse(buf, &temp);
94 if (uuid_is_null(&temp))
97 rc = down_write_killable(&cxl_region_rwsem);
101 if (uuid_equal(&p->uuid, &temp))
105 if (p->state >= CXL_CONFIG_ACTIVE)
108 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
112 uuid_copy(&p->uuid, &temp);
114 up_write(&cxl_region_rwsem);
120 static DEVICE_ATTR_RW(uuid);
122 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
123 struct cxl_region *cxlr)
125 return xa_load(&port->regions, (unsigned long)cxlr);
128 static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
130 if (!cpu_cache_has_invalidate_memregion()) {
131 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
134 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
138 "Failed to synchronize CPU cache state\n");
143 cpu_cache_invalidate_memregion(IORES_DESC_CXL);
147 static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
149 struct cxl_region_params *p = &cxlr->params;
153 * Before region teardown attempt to flush, and if the flush
154 * fails cancel the region teardown for data consistency
157 rc = cxl_region_invalidate_memregion(cxlr);
161 for (i = count - 1; i >= 0; i--) {
162 struct cxl_endpoint_decoder *cxled = p->targets[i];
163 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
164 struct cxl_port *iter = cxled_to_port(cxled);
165 struct cxl_dev_state *cxlds = cxlmd->cxlds;
171 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
172 iter = to_cxl_port(iter->dev.parent);
174 for (ep = cxl_ep_load(iter, cxlmd); iter;
175 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
176 struct cxl_region_ref *cxl_rr;
177 struct cxl_decoder *cxld;
179 cxl_rr = cxl_rr_load(iter, cxlr);
180 cxld = cxl_rr->decoder;
182 rc = cxld->reset(cxld);
185 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
189 rc = cxled->cxld.reset(&cxled->cxld);
192 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
195 /* all decoders associated with this region have been torn down */
196 clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
201 static int commit_decoder(struct cxl_decoder *cxld)
203 struct cxl_switch_decoder *cxlsd = NULL;
206 return cxld->commit(cxld);
208 if (is_switch_decoder(&cxld->dev))
209 cxlsd = to_cxl_switch_decoder(&cxld->dev);
211 if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
212 "->commit() is required\n"))
217 static int cxl_region_decode_commit(struct cxl_region *cxlr)
219 struct cxl_region_params *p = &cxlr->params;
222 for (i = 0; i < p->nr_targets; i++) {
223 struct cxl_endpoint_decoder *cxled = p->targets[i];
224 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
225 struct cxl_region_ref *cxl_rr;
226 struct cxl_decoder *cxld;
227 struct cxl_port *iter;
230 /* commit bottom up */
231 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
232 iter = to_cxl_port(iter->dev.parent)) {
233 cxl_rr = cxl_rr_load(iter, cxlr);
234 cxld = cxl_rr->decoder;
235 rc = commit_decoder(cxld);
241 /* programming @iter failed, teardown */
242 for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
243 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
244 cxl_rr = cxl_rr_load(iter, cxlr);
245 cxld = cxl_rr->decoder;
250 cxled->cxld.reset(&cxled->cxld);
258 /* undo the targets that were successfully committed */
259 cxl_region_decode_reset(cxlr, i);
263 static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
264 const char *buf, size_t len)
266 struct cxl_region *cxlr = to_cxl_region(dev);
267 struct cxl_region_params *p = &cxlr->params;
271 rc = kstrtobool(buf, &commit);
275 rc = down_write_killable(&cxl_region_rwsem);
279 /* Already in the requested state? */
280 if (commit && p->state >= CXL_CONFIG_COMMIT)
282 if (!commit && p->state < CXL_CONFIG_COMMIT)
285 /* Not ready to commit? */
286 if (commit && p->state < CXL_CONFIG_ACTIVE) {
292 * Invalidate caches before region setup to drop any speculative
293 * consumption of this address space
295 rc = cxl_region_invalidate_memregion(cxlr);
300 rc = cxl_region_decode_commit(cxlr);
302 p->state = CXL_CONFIG_COMMIT;
304 p->state = CXL_CONFIG_RESET_PENDING;
305 up_write(&cxl_region_rwsem);
306 device_release_driver(&cxlr->dev);
307 down_write(&cxl_region_rwsem);
310 * The lock was dropped, so need to revalidate that the reset is
313 if (p->state == CXL_CONFIG_RESET_PENDING) {
314 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
316 * Revert to committed since there may still be active
317 * decoders associated with this region, or move forward
318 * to active to mark the reset successful
321 p->state = CXL_CONFIG_COMMIT;
323 p->state = CXL_CONFIG_ACTIVE;
328 up_write(&cxl_region_rwsem);
335 static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
338 struct cxl_region *cxlr = to_cxl_region(dev);
339 struct cxl_region_params *p = &cxlr->params;
342 rc = down_read_interruptible(&cxl_region_rwsem);
345 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
346 up_read(&cxl_region_rwsem);
350 static DEVICE_ATTR_RW(commit);
352 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
355 struct device *dev = kobj_to_dev(kobj);
356 struct cxl_region *cxlr = to_cxl_region(dev);
359 * Support tooling that expects to find a 'uuid' attribute for all
360 * regions regardless of mode.
362 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
367 static ssize_t interleave_ways_show(struct device *dev,
368 struct device_attribute *attr, char *buf)
370 struct cxl_region *cxlr = to_cxl_region(dev);
371 struct cxl_region_params *p = &cxlr->params;
374 rc = down_read_interruptible(&cxl_region_rwsem);
377 rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
378 up_read(&cxl_region_rwsem);
383 static const struct attribute_group *get_cxl_region_target_group(void);
385 static ssize_t interleave_ways_store(struct device *dev,
386 struct device_attribute *attr,
387 const char *buf, size_t len)
389 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
390 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
391 struct cxl_region *cxlr = to_cxl_region(dev);
392 struct cxl_region_params *p = &cxlr->params;
393 unsigned int val, save;
397 rc = kstrtouint(buf, 0, &val);
401 rc = ways_to_eiw(val, &iw);
406 * Even for x3, x9, and x12 interleaves the region interleave must be a
407 * power of 2 multiple of the host bridge interleave.
409 if (!is_power_of_2(val / cxld->interleave_ways) ||
410 (val % cxld->interleave_ways)) {
411 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
415 rc = down_write_killable(&cxl_region_rwsem);
418 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
423 save = p->interleave_ways;
424 p->interleave_ways = val;
425 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
427 p->interleave_ways = save;
429 up_write(&cxl_region_rwsem);
434 static DEVICE_ATTR_RW(interleave_ways);
436 static ssize_t interleave_granularity_show(struct device *dev,
437 struct device_attribute *attr,
440 struct cxl_region *cxlr = to_cxl_region(dev);
441 struct cxl_region_params *p = &cxlr->params;
444 rc = down_read_interruptible(&cxl_region_rwsem);
447 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
448 up_read(&cxl_region_rwsem);
453 static ssize_t interleave_granularity_store(struct device *dev,
454 struct device_attribute *attr,
455 const char *buf, size_t len)
457 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
458 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
459 struct cxl_region *cxlr = to_cxl_region(dev);
460 struct cxl_region_params *p = &cxlr->params;
464 rc = kstrtoint(buf, 0, &val);
468 rc = granularity_to_eig(val, &ig);
473 * When the host-bridge is interleaved, disallow region granularity !=
474 * root granularity. Regions with a granularity less than the root
475 * interleave result in needing multiple endpoints to support a single
476 * slot in the interleave (possible to support in the future). Regions
477 * with a granularity greater than the root interleave result in invalid
478 * DPA translations (invalid to support).
480 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
483 rc = down_write_killable(&cxl_region_rwsem);
486 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
491 p->interleave_granularity = val;
493 up_write(&cxl_region_rwsem);
498 static DEVICE_ATTR_RW(interleave_granularity);
500 static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
503 struct cxl_region *cxlr = to_cxl_region(dev);
504 struct cxl_region_params *p = &cxlr->params;
505 u64 resource = -1ULL;
508 rc = down_read_interruptible(&cxl_region_rwsem);
512 resource = p->res->start;
513 rc = sysfs_emit(buf, "%#llx\n", resource);
514 up_read(&cxl_region_rwsem);
518 static DEVICE_ATTR_RO(resource);
520 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
523 struct cxl_region *cxlr = to_cxl_region(dev);
525 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
527 static DEVICE_ATTR_RO(mode);
529 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
531 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
532 struct cxl_region_params *p = &cxlr->params;
533 struct resource *res;
536 lockdep_assert_held_write(&cxl_region_rwsem);
538 /* Nothing to do... */
539 if (p->res && resource_size(p->res) == size)
542 /* To change size the old size must be freed first */
546 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
549 /* ways, granularity and uuid (if PMEM) need to be set before HPA */
550 if (!p->interleave_ways || !p->interleave_granularity ||
551 (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
554 div_u64_rem(size, SZ_256M * p->interleave_ways, &remainder);
558 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
559 dev_name(&cxlr->dev));
561 dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n",
567 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
572 static void cxl_region_iomem_release(struct cxl_region *cxlr)
574 struct cxl_region_params *p = &cxlr->params;
576 if (device_is_registered(&cxlr->dev))
577 lockdep_assert_held_write(&cxl_region_rwsem);
580 * Autodiscovered regions may not have been able to insert their
584 remove_resource(p->res);
590 static int free_hpa(struct cxl_region *cxlr)
592 struct cxl_region_params *p = &cxlr->params;
594 lockdep_assert_held_write(&cxl_region_rwsem);
599 if (p->state >= CXL_CONFIG_ACTIVE)
602 cxl_region_iomem_release(cxlr);
603 p->state = CXL_CONFIG_IDLE;
607 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
608 const char *buf, size_t len)
610 struct cxl_region *cxlr = to_cxl_region(dev);
614 rc = kstrtou64(buf, 0, &val);
618 rc = down_write_killable(&cxl_region_rwsem);
623 rc = alloc_hpa(cxlr, val);
626 up_write(&cxl_region_rwsem);
634 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
637 struct cxl_region *cxlr = to_cxl_region(dev);
638 struct cxl_region_params *p = &cxlr->params;
642 rc = down_read_interruptible(&cxl_region_rwsem);
646 size = resource_size(p->res);
647 rc = sysfs_emit(buf, "%#llx\n", size);
648 up_read(&cxl_region_rwsem);
652 static DEVICE_ATTR_RW(size);
654 static struct attribute *cxl_region_attrs[] = {
656 &dev_attr_commit.attr,
657 &dev_attr_interleave_ways.attr,
658 &dev_attr_interleave_granularity.attr,
659 &dev_attr_resource.attr,
665 static const struct attribute_group cxl_region_group = {
666 .attrs = cxl_region_attrs,
667 .is_visible = cxl_region_visible,
670 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
672 struct cxl_region_params *p = &cxlr->params;
673 struct cxl_endpoint_decoder *cxled;
676 rc = down_read_interruptible(&cxl_region_rwsem);
680 if (pos >= p->interleave_ways) {
681 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
687 cxled = p->targets[pos];
689 rc = sysfs_emit(buf, "\n");
691 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
693 up_read(&cxl_region_rwsem);
698 static int match_free_decoder(struct device *dev, void *data)
700 struct cxl_decoder *cxld;
703 if (!is_switch_decoder(dev))
706 cxld = to_cxl_decoder(dev);
708 /* enforce ordered allocation */
720 static struct cxl_decoder *cxl_region_find_decoder(struct cxl_port *port,
721 struct cxl_region *cxlr)
726 dev = device_find_child(&port->dev, &id, match_free_decoder);
730 * This decoder is pinned registered as long as the endpoint decoder is
731 * registered, and endpoint decoder unregistration holds the
732 * cxl_region_rwsem over unregister events, so no need to hold on to
733 * this extra reference.
736 return to_cxl_decoder(dev);
739 static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port,
740 struct cxl_region *cxlr)
742 struct cxl_region_params *p = &cxlr->params;
743 struct cxl_region_ref *cxl_rr, *iter;
747 xa_for_each(&port->regions, index, iter) {
748 struct cxl_region_params *ip = &iter->region->params;
753 if (ip->res->start > p->res->start) {
755 "%s: HPA order violation %s:%pr vs %pr\n",
756 dev_name(&port->dev),
757 dev_name(&iter->region->dev), ip->res, p->res);
758 return ERR_PTR(-EBUSY);
762 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
764 return ERR_PTR(-ENOMEM);
766 cxl_rr->region = cxlr;
767 cxl_rr->nr_targets = 1;
768 xa_init(&cxl_rr->endpoints);
770 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
773 "%s: failed to track region reference: %d\n",
774 dev_name(&port->dev), rc);
782 static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
784 struct cxl_region *cxlr = cxl_rr->region;
785 struct cxl_decoder *cxld = cxl_rr->decoder;
790 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
791 if (cxld->region == cxlr) {
793 put_device(&cxlr->dev);
797 static void free_region_ref(struct cxl_region_ref *cxl_rr)
799 struct cxl_port *port = cxl_rr->port;
800 struct cxl_region *cxlr = cxl_rr->region;
802 cxl_rr_free_decoder(cxl_rr);
803 xa_erase(&port->regions, (unsigned long)cxlr);
804 xa_destroy(&cxl_rr->endpoints);
808 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
809 struct cxl_endpoint_decoder *cxled)
812 struct cxl_port *port = cxl_rr->port;
813 struct cxl_region *cxlr = cxl_rr->region;
814 struct cxl_decoder *cxld = cxl_rr->decoder;
815 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
818 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
827 get_device(&cxlr->dev);
833 static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
834 struct cxl_endpoint_decoder *cxled,
835 struct cxl_region_ref *cxl_rr)
837 struct cxl_decoder *cxld;
839 if (port == cxled_to_port(cxled))
842 cxld = cxl_region_find_decoder(port, cxlr);
844 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
845 dev_name(&port->dev));
850 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
851 dev_name(&port->dev), dev_name(&cxld->dev),
852 dev_name(&cxld->region->dev));
857 * Endpoints should already match the region type, but backstop that
858 * assumption with an assertion. Switch-decoders change mapping-type
859 * based on what is mapped when they are assigned to a region.
861 dev_WARN_ONCE(&cxlr->dev,
862 port == cxled_to_port(cxled) &&
863 cxld->target_type != cxlr->type,
864 "%s:%s mismatch decoder type %d -> %d\n",
865 dev_name(&cxled_to_memdev(cxled)->dev),
866 dev_name(&cxld->dev), cxld->target_type, cxlr->type);
867 cxld->target_type = cxlr->type;
868 cxl_rr->decoder = cxld;
873 * cxl_port_attach_region() - track a region's interest in a port by endpoint
874 * @port: port to add a new region reference 'struct cxl_region_ref'
875 * @cxlr: region to attach to @port
876 * @cxled: endpoint decoder used to create or further pin a region reference
877 * @pos: interleave position of @cxled in @cxlr
879 * The attach event is an opportunity to validate CXL decode setup
880 * constraints and record metadata needed for programming HDM decoders,
881 * in particular decoder target lists.
885 * - validate that there are no other regions with a higher HPA already
886 * associated with @port
887 * - establish a region reference if one is not already present
889 * - additionally allocate a decoder instance that will host @cxlr on
892 * - pin the region reference by the endpoint
893 * - account for how many entries in @port's target list are needed to
894 * cover all of the added endpoints.
896 static int cxl_port_attach_region(struct cxl_port *port,
897 struct cxl_region *cxlr,
898 struct cxl_endpoint_decoder *cxled, int pos)
900 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
901 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
902 struct cxl_region_ref *cxl_rr;
903 bool nr_targets_inc = false;
904 struct cxl_decoder *cxld;
908 lockdep_assert_held_write(&cxl_region_rwsem);
910 cxl_rr = cxl_rr_load(port, cxlr);
912 struct cxl_ep *ep_iter;
916 * Walk the existing endpoints that have been attached to
917 * @cxlr at @port and see if they share the same 'next' port
918 * in the downstream direction. I.e. endpoints that share common
921 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
924 if (ep_iter->next == ep->next) {
931 * New target port, or @port is an endpoint port that always
932 * accounts its own local decode as a target.
934 if (!found || !ep->next) {
935 cxl_rr->nr_targets++;
936 nr_targets_inc = true;
939 cxl_rr = alloc_region_ref(port, cxlr);
940 if (IS_ERR(cxl_rr)) {
942 "%s: failed to allocate region reference\n",
943 dev_name(&port->dev));
944 return PTR_ERR(cxl_rr);
946 nr_targets_inc = true;
948 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
952 cxld = cxl_rr->decoder;
954 rc = cxl_rr_ep_add(cxl_rr, cxled);
957 "%s: failed to track endpoint %s:%s reference\n",
958 dev_name(&port->dev), dev_name(&cxlmd->dev),
959 dev_name(&cxld->dev));
964 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
965 dev_name(port->uport_dev), dev_name(&port->dev),
966 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
967 dev_name(&cxled->cxld.dev), pos,
968 ep ? ep->next ? dev_name(ep->next->uport_dev) :
969 dev_name(&cxlmd->dev) :
971 cxl_rr->nr_eps, cxl_rr->nr_targets);
976 cxl_rr->nr_targets--;
977 if (cxl_rr->nr_eps == 0)
978 free_region_ref(cxl_rr);
982 static void cxl_port_detach_region(struct cxl_port *port,
983 struct cxl_region *cxlr,
984 struct cxl_endpoint_decoder *cxled)
986 struct cxl_region_ref *cxl_rr;
987 struct cxl_ep *ep = NULL;
989 lockdep_assert_held_write(&cxl_region_rwsem);
991 cxl_rr = cxl_rr_load(port, cxlr);
996 * Endpoint ports do not carry cxl_ep references, and they
997 * never target more than one endpoint by definition
999 if (cxl_rr->decoder == &cxled->cxld)
1002 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1004 struct cxl_ep *ep_iter;
1005 unsigned long index;
1009 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1010 if (ep_iter->next == ep->next) {
1016 cxl_rr->nr_targets--;
1019 if (cxl_rr->nr_eps == 0)
1020 free_region_ref(cxl_rr);
1023 static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1024 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1027 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1028 struct cxl_region *cxlr = cxl_rr->region;
1029 struct cxl_region_params *p = &cxlr->params;
1030 struct cxl_endpoint_decoder *cxled_peer;
1031 struct cxl_port *port = cxl_rr->port;
1032 struct cxl_memdev *cxlmd_peer;
1033 struct cxl_ep *ep_peer;
1034 int pos = cxled->pos;
1037 * If this position wants to share a dport with the last endpoint mapped
1038 * then that endpoint, at index 'position - distance', must also be
1039 * mapped by this dport.
1041 if (pos < distance) {
1042 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1043 dev_name(port->uport_dev), dev_name(&port->dev),
1044 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1047 cxled_peer = p->targets[pos - distance];
1048 cxlmd_peer = cxled_to_memdev(cxled_peer);
1049 ep_peer = cxl_ep_load(port, cxlmd_peer);
1050 if (ep->dport != ep_peer->dport) {
1052 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1053 dev_name(port->uport_dev), dev_name(&port->dev),
1054 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1055 dev_name(&cxlmd_peer->dev),
1056 dev_name(&cxled_peer->cxld.dev));
1063 static int cxl_port_setup_targets(struct cxl_port *port,
1064 struct cxl_region *cxlr,
1065 struct cxl_endpoint_decoder *cxled)
1067 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1068 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1069 struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1070 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1071 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1072 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1073 struct cxl_region_params *p = &cxlr->params;
1074 struct cxl_decoder *cxld = cxl_rr->decoder;
1075 struct cxl_switch_decoder *cxlsd;
1080 * While root level decoders support x3, x6, x12, switch level
1081 * decoders only support powers of 2 up to x16.
1083 if (!is_power_of_2(cxl_rr->nr_targets)) {
1084 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1085 dev_name(port->uport_dev), dev_name(&port->dev),
1086 cxl_rr->nr_targets);
1090 cxlsd = to_cxl_switch_decoder(&cxld->dev);
1091 if (cxl_rr->nr_targets_set) {
1095 * Passthrough decoders impose no distance requirements between
1098 if (cxl_rr->nr_targets == 1)
1101 distance = p->nr_targets / cxl_rr->nr_targets;
1102 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1103 if (ep->dport == cxlsd->target[i]) {
1104 rc = check_last_peer(cxled, ep, cxl_rr,
1108 goto out_target_set;
1113 if (is_cxl_root(parent_port)) {
1114 parent_ig = cxlrd->cxlsd.cxld.interleave_granularity;
1115 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1117 * For purposes of address bit routing, use power-of-2 math for
1120 if (!is_power_of_2(parent_iw))
1123 struct cxl_region_ref *parent_rr;
1124 struct cxl_decoder *parent_cxld;
1126 parent_rr = cxl_rr_load(parent_port, cxlr);
1127 parent_cxld = parent_rr->decoder;
1128 parent_ig = parent_cxld->interleave_granularity;
1129 parent_iw = parent_cxld->interleave_ways;
1132 rc = granularity_to_eig(parent_ig, &peig);
1134 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1135 dev_name(parent_port->uport_dev),
1136 dev_name(&parent_port->dev), parent_ig);
1140 rc = ways_to_eiw(parent_iw, &peiw);
1142 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1143 dev_name(parent_port->uport_dev),
1144 dev_name(&parent_port->dev), parent_iw);
1148 iw = cxl_rr->nr_targets;
1149 rc = ways_to_eiw(iw, &eiw);
1151 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1152 dev_name(port->uport_dev), dev_name(&port->dev), iw);
1157 * If @parent_port is masking address bits, pick the next unused address
1158 * bit to route @port's targets.
1160 if (parent_iw > 1 && cxl_rr->nr_targets > 1) {
1161 u32 address_bit = max(peig + peiw, eiw + peig);
1163 eig = address_bit - eiw + 1;
1169 rc = eig_to_granularity(eig, &ig);
1171 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1172 dev_name(port->uport_dev), dev_name(&port->dev),
1177 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1178 if (cxld->interleave_ways != iw ||
1179 cxld->interleave_granularity != ig ||
1180 cxld->hpa_range.start != p->res->start ||
1181 cxld->hpa_range.end != p->res->end ||
1182 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1184 "%s:%s %s expected iw: %d ig: %d %pr\n",
1185 dev_name(port->uport_dev), dev_name(&port->dev),
1186 __func__, iw, ig, p->res);
1188 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1189 dev_name(port->uport_dev), dev_name(&port->dev),
1190 __func__, cxld->interleave_ways,
1191 cxld->interleave_granularity,
1192 (cxld->flags & CXL_DECODER_F_ENABLE) ?
1195 cxld->hpa_range.start, cxld->hpa_range.end);
1199 cxld->interleave_ways = iw;
1200 cxld->interleave_granularity = ig;
1201 cxld->hpa_range = (struct range) {
1202 .start = p->res->start,
1206 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1207 dev_name(&port->dev), iw, ig);
1209 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1211 "%s:%s: targets full trying to add %s:%s at %d\n",
1212 dev_name(port->uport_dev), dev_name(&port->dev),
1213 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1216 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1217 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1218 dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1219 dev_name(port->uport_dev), dev_name(&port->dev),
1220 dev_name(&cxlsd->cxld.dev),
1221 dev_name(ep->dport->dport_dev),
1222 cxl_rr->nr_targets_set);
1226 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1229 cxl_rr->nr_targets_set += inc;
1230 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1231 dev_name(port->uport_dev), dev_name(&port->dev),
1232 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1233 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1238 static void cxl_port_reset_targets(struct cxl_port *port,
1239 struct cxl_region *cxlr)
1241 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1242 struct cxl_decoder *cxld;
1245 * After the last endpoint has been detached the entire cxl_rr may now
1250 cxl_rr->nr_targets_set = 0;
1252 cxld = cxl_rr->decoder;
1253 cxld->hpa_range = (struct range) {
1259 static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1261 struct cxl_region_params *p = &cxlr->params;
1262 struct cxl_endpoint_decoder *cxled;
1263 struct cxl_dev_state *cxlds;
1264 struct cxl_memdev *cxlmd;
1265 struct cxl_port *iter;
1270 * In the auto-discovery case skip automatic teardown since the
1271 * address space is already active
1273 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1276 for (i = 0; i < p->nr_targets; i++) {
1277 cxled = p->targets[i];
1278 cxlmd = cxled_to_memdev(cxled);
1279 cxlds = cxlmd->cxlds;
1284 iter = cxled_to_port(cxled);
1285 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1286 iter = to_cxl_port(iter->dev.parent);
1288 for (ep = cxl_ep_load(iter, cxlmd); iter;
1289 iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1290 cxl_port_reset_targets(iter, cxlr);
1294 static int cxl_region_setup_targets(struct cxl_region *cxlr)
1296 struct cxl_region_params *p = &cxlr->params;
1297 struct cxl_endpoint_decoder *cxled;
1298 struct cxl_dev_state *cxlds;
1299 int i, rc, rch = 0, vh = 0;
1300 struct cxl_memdev *cxlmd;
1301 struct cxl_port *iter;
1304 for (i = 0; i < p->nr_targets; i++) {
1305 cxled = p->targets[i];
1306 cxlmd = cxled_to_memdev(cxled);
1307 cxlds = cxlmd->cxlds;
1309 /* validate that all targets agree on topology */
1317 iter = cxled_to_port(cxled);
1318 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1319 iter = to_cxl_port(iter->dev.parent);
1322 * Descend the topology tree programming / validating
1323 * targets while looking for conflicts.
1325 for (ep = cxl_ep_load(iter, cxlmd); iter;
1326 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1327 rc = cxl_port_setup_targets(iter, cxlr, cxled);
1329 cxl_region_teardown_targets(cxlr);
1336 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1337 cxl_region_teardown_targets(cxlr);
1344 static int cxl_region_validate_position(struct cxl_region *cxlr,
1345 struct cxl_endpoint_decoder *cxled,
1348 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1349 struct cxl_region_params *p = &cxlr->params;
1352 if (pos < 0 || pos >= p->interleave_ways) {
1353 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1354 p->interleave_ways);
1358 if (p->targets[pos] == cxled)
1361 if (p->targets[pos]) {
1362 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1363 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1365 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1366 pos, dev_name(&cxlmd_target->dev),
1367 dev_name(&cxled_target->cxld.dev));
1371 for (i = 0; i < p->interleave_ways; i++) {
1372 struct cxl_endpoint_decoder *cxled_target;
1373 struct cxl_memdev *cxlmd_target;
1375 cxled_target = p->targets[i];
1379 cxlmd_target = cxled_to_memdev(cxled_target);
1380 if (cxlmd_target == cxlmd) {
1382 "%s already specified at position %d via: %s\n",
1383 dev_name(&cxlmd->dev), pos,
1384 dev_name(&cxled_target->cxld.dev));
1392 static int cxl_region_attach_position(struct cxl_region *cxlr,
1393 struct cxl_root_decoder *cxlrd,
1394 struct cxl_endpoint_decoder *cxled,
1395 const struct cxl_dport *dport, int pos)
1397 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1398 struct cxl_port *iter;
1401 if (cxlrd->calc_hb(cxlrd, pos) != dport) {
1402 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1403 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1404 dev_name(&cxlrd->cxlsd.cxld.dev));
1408 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1409 iter = to_cxl_port(iter->dev.parent)) {
1410 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1418 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1419 iter = to_cxl_port(iter->dev.parent))
1420 cxl_port_detach_region(iter, cxlr, cxled);
1424 static int cxl_region_attach_auto(struct cxl_region *cxlr,
1425 struct cxl_endpoint_decoder *cxled, int pos)
1427 struct cxl_region_params *p = &cxlr->params;
1429 if (cxled->state != CXL_DECODER_STATE_AUTO) {
1431 "%s: unable to add decoder to autodetected region\n",
1432 dev_name(&cxled->cxld.dev));
1437 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1438 dev_name(&cxled->cxld.dev), pos);
1442 if (p->nr_targets >= p->interleave_ways) {
1443 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1444 dev_name(&cxled->cxld.dev));
1449 * Temporarily record the endpoint decoder into the target array. Yes,
1450 * this means that userspace can view devices in the wrong position
1451 * before the region activates, and must be careful to understand when
1452 * it might be racing region autodiscovery.
1454 pos = p->nr_targets;
1455 p->targets[pos] = cxled;
1462 static struct cxl_port *next_port(struct cxl_port *port)
1464 if (!port->parent_dport)
1466 return port->parent_dport->port;
1469 static int decoder_match_range(struct device *dev, void *data)
1471 struct cxl_endpoint_decoder *cxled = data;
1472 struct cxl_switch_decoder *cxlsd;
1474 if (!is_switch_decoder(dev))
1477 cxlsd = to_cxl_switch_decoder(dev);
1478 return range_contains(&cxlsd->cxld.hpa_range, &cxled->cxld.hpa_range);
1481 static void find_positions(const struct cxl_switch_decoder *cxlsd,
1482 const struct cxl_port *iter_a,
1483 const struct cxl_port *iter_b, int *a_pos,
1488 for (i = 0, *a_pos = -1, *b_pos = -1; i < cxlsd->nr_targets; i++) {
1489 if (cxlsd->target[i] == iter_a->parent_dport)
1491 else if (cxlsd->target[i] == iter_b->parent_dport)
1493 if (*a_pos >= 0 && *b_pos >= 0)
1498 static int cmp_decode_pos(const void *a, const void *b)
1500 struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1501 struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1502 struct cxl_memdev *cxlmd_a = cxled_to_memdev(cxled_a);
1503 struct cxl_memdev *cxlmd_b = cxled_to_memdev(cxled_b);
1504 struct cxl_port *port_a = cxled_to_port(cxled_a);
1505 struct cxl_port *port_b = cxled_to_port(cxled_b);
1506 struct cxl_port *iter_a, *iter_b, *port = NULL;
1507 struct cxl_switch_decoder *cxlsd;
1512 /* Exit early if any prior sorting failed */
1513 if (cxled_a->pos < 0 || cxled_b->pos < 0)
1517 * Walk up the hierarchy to find a shared port, find the decoder that
1518 * maps the range, compare the relative position of those dport
1521 for (iter_a = port_a; iter_a; iter_a = next_port(iter_a)) {
1522 struct cxl_port *next_a, *next_b;
1524 next_a = next_port(iter_a);
1528 for (iter_b = port_b; iter_b; iter_b = next_port(iter_b)) {
1529 next_b = next_port(iter_b);
1530 if (next_a != next_b)
1541 dev_err(cxlmd_a->dev.parent,
1542 "failed to find shared port with %s\n",
1543 dev_name(cxlmd_b->dev.parent));
1547 dev = device_find_child(&port->dev, cxled_a, decoder_match_range);
1549 struct range *range = &cxled_a->cxld.hpa_range;
1551 dev_err(port->uport_dev,
1552 "failed to find decoder that maps %#llx-%#llx\n",
1553 range->start, range->end);
1557 cxlsd = to_cxl_switch_decoder(dev);
1559 seq = read_seqbegin(&cxlsd->target_lock);
1560 find_positions(cxlsd, iter_a, iter_b, &a_pos, &b_pos);
1561 } while (read_seqretry(&cxlsd->target_lock, seq));
1565 if (a_pos < 0 || b_pos < 0) {
1566 dev_err(port->uport_dev,
1567 "failed to find shared decoder for %s and %s\n",
1568 dev_name(cxlmd_a->dev.parent),
1569 dev_name(cxlmd_b->dev.parent));
1573 dev_dbg(port->uport_dev, "%s comes %s %s\n",
1574 dev_name(cxlmd_a->dev.parent),
1575 a_pos - b_pos < 0 ? "before" : "after",
1576 dev_name(cxlmd_b->dev.parent));
1578 return a_pos - b_pos;
1584 static int cxl_region_sort_targets(struct cxl_region *cxlr)
1586 struct cxl_region_params *p = &cxlr->params;
1589 sort(p->targets, p->nr_targets, sizeof(p->targets[0]), cmp_decode_pos,
1592 for (i = 0; i < p->nr_targets; i++) {
1593 struct cxl_endpoint_decoder *cxled = p->targets[i];
1596 * Record that sorting failed, but still continue to restore
1597 * cxled->pos with its ->targets[] position so that follow-on
1598 * code paths can reliably do p->targets[cxled->pos] to
1599 * self-reference their entry.
1606 dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1610 static int cxl_region_attach(struct cxl_region *cxlr,
1611 struct cxl_endpoint_decoder *cxled, int pos)
1613 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1614 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1615 struct cxl_region_params *p = &cxlr->params;
1616 struct cxl_port *ep_port, *root_port;
1617 struct cxl_dport *dport;
1620 if (cxled->mode != cxlr->mode) {
1621 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1622 dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1626 if (cxled->mode == CXL_DECODER_DEAD) {
1627 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1631 /* all full of members, or interleave config not established? */
1632 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1633 dev_dbg(&cxlr->dev, "region already active\n");
1635 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1636 dev_dbg(&cxlr->dev, "interleave config missing\n");
1640 ep_port = cxled_to_port(cxled);
1641 root_port = cxlrd_to_port(cxlrd);
1642 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1644 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1645 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1646 dev_name(cxlr->dev.parent));
1650 if (cxled->cxld.target_type != cxlr->type) {
1651 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1652 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1653 cxled->cxld.target_type, cxlr->type);
1657 if (!cxled->dpa_res) {
1658 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1659 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1663 if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1664 resource_size(p->res)) {
1666 "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1667 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1668 (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1669 (u64)resource_size(p->res));
1673 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1676 rc = cxl_region_attach_auto(cxlr, cxled, pos);
1680 /* await more targets to arrive... */
1681 if (p->nr_targets < p->interleave_ways)
1685 * All targets are here, which implies all PCI enumeration that
1686 * affects this region has been completed. Walk the topology to
1687 * sort the devices into their relative region decode position.
1689 rc = cxl_region_sort_targets(cxlr);
1693 for (i = 0; i < p->nr_targets; i++) {
1694 cxled = p->targets[i];
1695 ep_port = cxled_to_port(cxled);
1696 dport = cxl_find_dport_by_dev(root_port,
1697 ep_port->host_bridge);
1698 rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1704 rc = cxl_region_setup_targets(cxlr);
1709 * If target setup succeeds in the autodiscovery case
1710 * then the region is already committed.
1712 p->state = CXL_CONFIG_COMMIT;
1717 rc = cxl_region_validate_position(cxlr, cxled, pos);
1721 rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
1725 p->targets[pos] = cxled;
1729 if (p->nr_targets == p->interleave_ways) {
1730 rc = cxl_region_setup_targets(cxlr);
1733 p->state = CXL_CONFIG_ACTIVE;
1736 cxled->cxld.interleave_ways = p->interleave_ways;
1737 cxled->cxld.interleave_granularity = p->interleave_granularity;
1738 cxled->cxld.hpa_range = (struct range) {
1739 .start = p->res->start,
1748 p->targets[pos] = NULL;
1752 static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
1754 struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
1755 struct cxl_region *cxlr = cxled->cxld.region;
1756 struct cxl_region_params *p;
1759 lockdep_assert_held_write(&cxl_region_rwsem);
1765 get_device(&cxlr->dev);
1767 if (p->state > CXL_CONFIG_ACTIVE) {
1769 * TODO: tear down all impacted regions if a device is
1770 * removed out of order
1772 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
1775 p->state = CXL_CONFIG_ACTIVE;
1778 for (iter = ep_port; !is_cxl_root(iter);
1779 iter = to_cxl_port(iter->dev.parent))
1780 cxl_port_detach_region(iter, cxlr, cxled);
1782 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
1783 p->targets[cxled->pos] != cxled) {
1784 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1786 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
1787 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1792 if (p->state == CXL_CONFIG_ACTIVE) {
1793 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
1794 cxl_region_teardown_targets(cxlr);
1796 p->targets[cxled->pos] = NULL;
1798 cxled->cxld.hpa_range = (struct range) {
1803 /* notify the region driver that one of its targets has departed */
1804 up_write(&cxl_region_rwsem);
1805 device_release_driver(&cxlr->dev);
1806 down_write(&cxl_region_rwsem);
1808 put_device(&cxlr->dev);
1812 void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
1814 down_write(&cxl_region_rwsem);
1815 cxled->mode = CXL_DECODER_DEAD;
1816 cxl_region_detach(cxled);
1817 up_write(&cxl_region_rwsem);
1820 static int attach_target(struct cxl_region *cxlr,
1821 struct cxl_endpoint_decoder *cxled, int pos,
1826 if (state == TASK_INTERRUPTIBLE)
1827 rc = down_write_killable(&cxl_region_rwsem);
1829 down_write(&cxl_region_rwsem);
1833 down_read(&cxl_dpa_rwsem);
1834 rc = cxl_region_attach(cxlr, cxled, pos);
1835 up_read(&cxl_dpa_rwsem);
1836 up_write(&cxl_region_rwsem);
1840 static int detach_target(struct cxl_region *cxlr, int pos)
1842 struct cxl_region_params *p = &cxlr->params;
1845 rc = down_write_killable(&cxl_region_rwsem);
1849 if (pos >= p->interleave_ways) {
1850 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1851 p->interleave_ways);
1856 if (!p->targets[pos]) {
1861 rc = cxl_region_detach(p->targets[pos]);
1863 up_write(&cxl_region_rwsem);
1867 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
1872 if (sysfs_streq(buf, "\n"))
1873 rc = detach_target(cxlr, pos);
1877 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
1881 if (!is_endpoint_decoder(dev)) {
1886 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
1887 TASK_INTERRUPTIBLE);
1897 #define TARGET_ATTR_RW(n) \
1898 static ssize_t target##n##_show( \
1899 struct device *dev, struct device_attribute *attr, char *buf) \
1901 return show_targetN(to_cxl_region(dev), buf, (n)); \
1903 static ssize_t target##n##_store(struct device *dev, \
1904 struct device_attribute *attr, \
1905 const char *buf, size_t len) \
1907 return store_targetN(to_cxl_region(dev), buf, (n), len); \
1909 static DEVICE_ATTR_RW(target##n)
1928 static struct attribute *target_attrs[] = {
1929 &dev_attr_target0.attr,
1930 &dev_attr_target1.attr,
1931 &dev_attr_target2.attr,
1932 &dev_attr_target3.attr,
1933 &dev_attr_target4.attr,
1934 &dev_attr_target5.attr,
1935 &dev_attr_target6.attr,
1936 &dev_attr_target7.attr,
1937 &dev_attr_target8.attr,
1938 &dev_attr_target9.attr,
1939 &dev_attr_target10.attr,
1940 &dev_attr_target11.attr,
1941 &dev_attr_target12.attr,
1942 &dev_attr_target13.attr,
1943 &dev_attr_target14.attr,
1944 &dev_attr_target15.attr,
1948 static umode_t cxl_region_target_visible(struct kobject *kobj,
1949 struct attribute *a, int n)
1951 struct device *dev = kobj_to_dev(kobj);
1952 struct cxl_region *cxlr = to_cxl_region(dev);
1953 struct cxl_region_params *p = &cxlr->params;
1955 if (n < p->interleave_ways)
1960 static const struct attribute_group cxl_region_target_group = {
1961 .attrs = target_attrs,
1962 .is_visible = cxl_region_target_visible,
1965 static const struct attribute_group *get_cxl_region_target_group(void)
1967 return &cxl_region_target_group;
1970 static const struct attribute_group *region_groups[] = {
1971 &cxl_base_attribute_group,
1973 &cxl_region_target_group,
1977 static void cxl_region_release(struct device *dev)
1979 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
1980 struct cxl_region *cxlr = to_cxl_region(dev);
1981 int id = atomic_read(&cxlrd->region_id);
1984 * Try to reuse the recently idled id rather than the cached
1985 * next id to prevent the region id space from increasing
1989 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
1994 memregion_free(cxlr->id);
1996 put_device(dev->parent);
2000 const struct device_type cxl_region_type = {
2001 .name = "cxl_region",
2002 .release = cxl_region_release,
2003 .groups = region_groups
2006 bool is_cxl_region(struct device *dev)
2008 return dev->type == &cxl_region_type;
2010 EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
2012 static struct cxl_region *to_cxl_region(struct device *dev)
2014 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2015 "not a cxl_region device\n"))
2018 return container_of(dev, struct cxl_region, dev);
2021 static void unregister_region(void *dev)
2023 struct cxl_region *cxlr = to_cxl_region(dev);
2024 struct cxl_region_params *p = &cxlr->params;
2030 * Now that region sysfs is shutdown, the parameter block is now
2031 * read-only, so no need to hold the region rwsem to access the
2032 * region parameters.
2034 for (i = 0; i < p->interleave_ways; i++)
2035 detach_target(cxlr, i);
2037 cxl_region_iomem_release(cxlr);
2041 static struct lock_class_key cxl_region_key;
2043 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2045 struct cxl_region *cxlr;
2048 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2051 return ERR_PTR(-ENOMEM);
2055 device_initialize(dev);
2056 lockdep_set_class(&dev->mutex, &cxl_region_key);
2057 dev->parent = &cxlrd->cxlsd.cxld.dev;
2059 * Keep root decoder pinned through cxl_region_release to fixup
2060 * region id allocations
2062 get_device(dev->parent);
2063 device_set_pm_not_required(dev);
2064 dev->bus = &cxl_bus_type;
2065 dev->type = &cxl_region_type;
2072 * devm_cxl_add_region - Adds a region to a decoder
2073 * @cxlrd: root decoder
2074 * @id: memregion id to create, or memregion_free() on failure
2075 * @mode: mode for the endpoint decoders of this region
2076 * @type: select whether this is an expander or accelerator (type-2 or type-3)
2078 * This is the second step of region initialization. Regions exist within an
2079 * address space which is mapped by a @cxlrd.
2081 * Return: 0 if the region was added to the @cxlrd, else returns negative error
2082 * code. The region will be named "regionZ" where Z is the unique region number.
2084 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2086 enum cxl_decoder_mode mode,
2087 enum cxl_decoder_type type)
2089 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2090 struct cxl_region *cxlr;
2095 case CXL_DECODER_RAM:
2096 case CXL_DECODER_PMEM:
2099 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2100 return ERR_PTR(-EINVAL);
2103 cxlr = cxl_region_alloc(cxlrd, id);
2110 rc = dev_set_name(dev, "region%d", id);
2114 rc = device_add(dev);
2118 rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2122 dev_dbg(port->uport_dev, "%s: created %s\n",
2123 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2131 static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2133 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2136 static ssize_t create_pmem_region_show(struct device *dev,
2137 struct device_attribute *attr, char *buf)
2139 return __create_region_show(to_cxl_root_decoder(dev), buf);
2142 static ssize_t create_ram_region_show(struct device *dev,
2143 struct device_attribute *attr, char *buf)
2145 return __create_region_show(to_cxl_root_decoder(dev), buf);
2148 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2149 enum cxl_decoder_mode mode, int id)
2153 rc = memregion_alloc(GFP_KERNEL);
2157 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2159 return ERR_PTR(-EBUSY);
2162 return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2165 static ssize_t create_pmem_region_store(struct device *dev,
2166 struct device_attribute *attr,
2167 const char *buf, size_t len)
2169 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2170 struct cxl_region *cxlr;
2173 rc = sscanf(buf, "region%d\n", &id);
2177 cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id);
2179 return PTR_ERR(cxlr);
2183 DEVICE_ATTR_RW(create_pmem_region);
2185 static ssize_t create_ram_region_store(struct device *dev,
2186 struct device_attribute *attr,
2187 const char *buf, size_t len)
2189 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2190 struct cxl_region *cxlr;
2193 rc = sscanf(buf, "region%d\n", &id);
2197 cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id);
2199 return PTR_ERR(cxlr);
2203 DEVICE_ATTR_RW(create_ram_region);
2205 static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2208 struct cxl_decoder *cxld = to_cxl_decoder(dev);
2211 rc = down_read_interruptible(&cxl_region_rwsem);
2216 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2218 rc = sysfs_emit(buf, "\n");
2219 up_read(&cxl_region_rwsem);
2223 DEVICE_ATTR_RO(region);
2225 static struct cxl_region *
2226 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2228 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2229 struct device *region_dev;
2231 region_dev = device_find_child_by_name(&cxld->dev, name);
2233 return ERR_PTR(-ENODEV);
2235 return to_cxl_region(region_dev);
2238 static ssize_t delete_region_store(struct device *dev,
2239 struct device_attribute *attr,
2240 const char *buf, size_t len)
2242 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2243 struct cxl_port *port = to_cxl_port(dev->parent);
2244 struct cxl_region *cxlr;
2246 cxlr = cxl_find_region_by_name(cxlrd, buf);
2248 return PTR_ERR(cxlr);
2250 devm_release_action(port->uport_dev, unregister_region, cxlr);
2251 put_device(&cxlr->dev);
2255 DEVICE_ATTR_WO(delete_region);
2257 static void cxl_pmem_region_release(struct device *dev)
2259 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2262 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2263 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2265 put_device(&cxlmd->dev);
2271 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2272 &cxl_base_attribute_group,
2276 const struct device_type cxl_pmem_region_type = {
2277 .name = "cxl_pmem_region",
2278 .release = cxl_pmem_region_release,
2279 .groups = cxl_pmem_region_attribute_groups,
2282 bool is_cxl_pmem_region(struct device *dev)
2284 return dev->type == &cxl_pmem_region_type;
2286 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
2288 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2290 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2291 "not a cxl_pmem_region device\n"))
2293 return container_of(dev, struct cxl_pmem_region, dev);
2295 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
2297 struct cxl_poison_context {
2298 struct cxl_port *port;
2299 enum cxl_decoder_mode mode;
2303 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2304 struct cxl_poison_context *ctx)
2306 struct cxl_dev_state *cxlds = cxlmd->cxlds;
2311 * Collect poison for the remaining unmapped resources
2312 * after poison is collected by committed endpoints.
2314 * Knowing that PMEM must always follow RAM, get poison
2315 * for unmapped resources based on the last decoder's mode:
2316 * ram: scan remains of ram range, then any pmem range
2317 * pmem: scan remains of pmem range
2320 if (ctx->mode == CXL_DECODER_RAM) {
2321 offset = ctx->offset;
2322 length = resource_size(&cxlds->ram_res) - offset;
2323 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2329 if (ctx->mode == CXL_DECODER_PMEM) {
2330 offset = ctx->offset;
2331 length = resource_size(&cxlds->dpa_res) - offset;
2334 } else if (resource_size(&cxlds->pmem_res)) {
2335 offset = cxlds->pmem_res.start;
2336 length = resource_size(&cxlds->pmem_res);
2341 return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2344 static int poison_by_decoder(struct device *dev, void *arg)
2346 struct cxl_poison_context *ctx = arg;
2347 struct cxl_endpoint_decoder *cxled;
2348 struct cxl_memdev *cxlmd;
2352 if (!is_endpoint_decoder(dev))
2355 cxled = to_cxl_endpoint_decoder(dev);
2356 if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2360 * Regions are only created with single mode decoders: pmem or ram.
2361 * Linux does not support mixed mode decoders. This means that
2362 * reading poison per endpoint decoder adheres to the requirement
2363 * that poison reads of pmem and ram must be separated.
2364 * CXL 3.0 Spec 8.2.9.8.4.1
2366 if (cxled->mode == CXL_DECODER_MIXED) {
2367 dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2371 cxlmd = cxled_to_memdev(cxled);
2373 offset = cxled->dpa_res->start - cxled->skip;
2374 length = cxled->skip;
2375 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2376 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2382 offset = cxled->dpa_res->start;
2383 length = cxled->dpa_res->end - offset + 1;
2384 rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2385 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2390 /* Iterate until commit_end is reached */
2391 if (cxled->cxld.id == ctx->port->commit_end) {
2392 ctx->offset = cxled->dpa_res->end + 1;
2393 ctx->mode = cxled->mode;
2400 int cxl_get_poison_by_endpoint(struct cxl_port *port)
2402 struct cxl_poison_context ctx;
2405 rc = down_read_interruptible(&cxl_region_rwsem);
2409 ctx = (struct cxl_poison_context) {
2413 rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2415 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2418 up_read(&cxl_region_rwsem);
2422 static struct lock_class_key cxl_pmem_region_key;
2424 static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr)
2426 struct cxl_region_params *p = &cxlr->params;
2427 struct cxl_nvdimm_bridge *cxl_nvb;
2428 struct cxl_pmem_region *cxlr_pmem;
2432 down_read(&cxl_region_rwsem);
2433 if (p->state != CXL_CONFIG_COMMIT) {
2434 cxlr_pmem = ERR_PTR(-ENXIO);
2438 cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets),
2441 cxlr_pmem = ERR_PTR(-ENOMEM);
2445 cxlr_pmem->hpa_range.start = p->res->start;
2446 cxlr_pmem->hpa_range.end = p->res->end;
2448 /* Snapshot the region configuration underneath the cxl_region_rwsem */
2449 cxlr_pmem->nr_mappings = p->nr_targets;
2450 for (i = 0; i < p->nr_targets; i++) {
2451 struct cxl_endpoint_decoder *cxled = p->targets[i];
2452 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2453 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2456 * Regions never span CXL root devices, so by definition the
2457 * bridge for one device is the same for all.
2460 cxl_nvb = cxl_find_nvdimm_bridge(cxlmd);
2462 cxlr_pmem = ERR_PTR(-ENODEV);
2465 cxlr->cxl_nvb = cxl_nvb;
2468 get_device(&cxlmd->dev);
2469 m->start = cxled->dpa_res->start;
2470 m->size = resource_size(cxled->dpa_res);
2474 dev = &cxlr_pmem->dev;
2475 cxlr_pmem->cxlr = cxlr;
2476 cxlr->cxlr_pmem = cxlr_pmem;
2477 device_initialize(dev);
2478 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2479 device_set_pm_not_required(dev);
2480 dev->parent = &cxlr->dev;
2481 dev->bus = &cxl_bus_type;
2482 dev->type = &cxl_pmem_region_type;
2484 up_read(&cxl_region_rwsem);
2489 static void cxl_dax_region_release(struct device *dev)
2491 struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
2496 static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
2497 &cxl_base_attribute_group,
2501 const struct device_type cxl_dax_region_type = {
2502 .name = "cxl_dax_region",
2503 .release = cxl_dax_region_release,
2504 .groups = cxl_dax_region_attribute_groups,
2507 static bool is_cxl_dax_region(struct device *dev)
2509 return dev->type == &cxl_dax_region_type;
2512 struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
2514 if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
2515 "not a cxl_dax_region device\n"))
2517 return container_of(dev, struct cxl_dax_region, dev);
2519 EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL);
2521 static struct lock_class_key cxl_dax_region_key;
2523 static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
2525 struct cxl_region_params *p = &cxlr->params;
2526 struct cxl_dax_region *cxlr_dax;
2529 down_read(&cxl_region_rwsem);
2530 if (p->state != CXL_CONFIG_COMMIT) {
2531 cxlr_dax = ERR_PTR(-ENXIO);
2535 cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
2537 cxlr_dax = ERR_PTR(-ENOMEM);
2541 cxlr_dax->hpa_range.start = p->res->start;
2542 cxlr_dax->hpa_range.end = p->res->end;
2544 dev = &cxlr_dax->dev;
2545 cxlr_dax->cxlr = cxlr;
2546 device_initialize(dev);
2547 lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
2548 device_set_pm_not_required(dev);
2549 dev->parent = &cxlr->dev;
2550 dev->bus = &cxl_bus_type;
2551 dev->type = &cxl_dax_region_type;
2553 up_read(&cxl_region_rwsem);
2558 static void cxlr_pmem_unregister(void *_cxlr_pmem)
2560 struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
2561 struct cxl_region *cxlr = cxlr_pmem->cxlr;
2562 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2565 * Either the bridge is in ->remove() context under the device_lock(),
2566 * or cxlr_release_nvdimm() is cancelling the bridge's release action
2567 * for @cxlr_pmem and doing it itself (while manually holding the bridge
2570 device_lock_assert(&cxl_nvb->dev);
2571 cxlr->cxlr_pmem = NULL;
2572 cxlr_pmem->cxlr = NULL;
2573 device_unregister(&cxlr_pmem->dev);
2576 static void cxlr_release_nvdimm(void *_cxlr)
2578 struct cxl_region *cxlr = _cxlr;
2579 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2581 device_lock(&cxl_nvb->dev);
2582 if (cxlr->cxlr_pmem)
2583 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
2585 device_unlock(&cxl_nvb->dev);
2586 cxlr->cxl_nvb = NULL;
2587 put_device(&cxl_nvb->dev);
2591 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
2592 * @cxlr: parent CXL region for this pmem region bridge device
2594 * Return: 0 on success negative error code on failure.
2596 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
2598 struct cxl_pmem_region *cxlr_pmem;
2599 struct cxl_nvdimm_bridge *cxl_nvb;
2603 cxlr_pmem = cxl_pmem_region_alloc(cxlr);
2604 if (IS_ERR(cxlr_pmem))
2605 return PTR_ERR(cxlr_pmem);
2606 cxl_nvb = cxlr->cxl_nvb;
2608 dev = &cxlr_pmem->dev;
2609 rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
2613 rc = device_add(dev);
2617 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2620 device_lock(&cxl_nvb->dev);
2621 if (cxl_nvb->dev.driver)
2622 rc = devm_add_action_or_reset(&cxl_nvb->dev,
2623 cxlr_pmem_unregister, cxlr_pmem);
2626 device_unlock(&cxl_nvb->dev);
2631 /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
2632 return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
2637 put_device(&cxl_nvb->dev);
2638 cxlr->cxl_nvb = NULL;
2642 static void cxlr_dax_unregister(void *_cxlr_dax)
2644 struct cxl_dax_region *cxlr_dax = _cxlr_dax;
2646 device_unregister(&cxlr_dax->dev);
2649 static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
2651 struct cxl_dax_region *cxlr_dax;
2655 cxlr_dax = cxl_dax_region_alloc(cxlr);
2656 if (IS_ERR(cxlr_dax))
2657 return PTR_ERR(cxlr_dax);
2659 dev = &cxlr_dax->dev;
2660 rc = dev_set_name(dev, "dax_region%d", cxlr->id);
2664 rc = device_add(dev);
2668 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2671 return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
2678 static int match_decoder_by_range(struct device *dev, void *data)
2680 struct range *r1, *r2 = data;
2681 struct cxl_root_decoder *cxlrd;
2683 if (!is_root_decoder(dev))
2686 cxlrd = to_cxl_root_decoder(dev);
2687 r1 = &cxlrd->cxlsd.cxld.hpa_range;
2688 return range_contains(r1, r2);
2691 static int match_region_by_range(struct device *dev, void *data)
2693 struct cxl_region_params *p;
2694 struct cxl_region *cxlr;
2695 struct range *r = data;
2698 if (!is_cxl_region(dev))
2701 cxlr = to_cxl_region(dev);
2704 down_read(&cxl_region_rwsem);
2705 if (p->res && p->res->start == r->start && p->res->end == r->end)
2707 up_read(&cxl_region_rwsem);
2712 /* Establish an empty region covering the given HPA range */
2713 static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
2714 struct cxl_endpoint_decoder *cxled)
2716 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2717 struct cxl_port *port = cxlrd_to_port(cxlrd);
2718 struct range *hpa = &cxled->cxld.hpa_range;
2719 struct cxl_region_params *p;
2720 struct cxl_region *cxlr;
2721 struct resource *res;
2725 cxlr = __create_region(cxlrd, cxled->mode,
2726 atomic_read(&cxlrd->region_id));
2727 } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
2730 dev_err(cxlmd->dev.parent,
2731 "%s:%s: %s failed assign region: %ld\n",
2732 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2733 __func__, PTR_ERR(cxlr));
2737 down_write(&cxl_region_rwsem);
2739 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
2740 dev_err(cxlmd->dev.parent,
2741 "%s:%s: %s autodiscovery interrupted\n",
2742 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2748 set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
2750 res = kmalloc(sizeof(*res), GFP_KERNEL);
2756 *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
2757 dev_name(&cxlr->dev));
2758 rc = insert_resource(cxlrd->res, res);
2761 * Platform-firmware may not have split resources like "System
2762 * RAM" on CXL window boundaries see cxl_region_iomem_release()
2764 dev_warn(cxlmd->dev.parent,
2765 "%s:%s: %s %s cannot insert resource\n",
2766 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2767 __func__, dev_name(&cxlr->dev));
2771 p->interleave_ways = cxled->cxld.interleave_ways;
2772 p->interleave_granularity = cxled->cxld.interleave_granularity;
2773 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2775 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
2779 dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
2780 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
2781 dev_name(&cxlr->dev), p->res, p->interleave_ways,
2782 p->interleave_granularity);
2784 /* ...to match put_device() in cxl_add_to_region() */
2785 get_device(&cxlr->dev);
2786 up_write(&cxl_region_rwsem);
2791 up_write(&cxl_region_rwsem);
2792 devm_release_action(port->uport_dev, unregister_region, cxlr);
2796 int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
2798 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2799 struct range *hpa = &cxled->cxld.hpa_range;
2800 struct cxl_decoder *cxld = &cxled->cxld;
2801 struct device *cxlrd_dev, *region_dev;
2802 struct cxl_root_decoder *cxlrd;
2803 struct cxl_region_params *p;
2804 struct cxl_region *cxlr;
2805 bool attach = false;
2808 cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
2809 match_decoder_by_range);
2811 dev_err(cxlmd->dev.parent,
2812 "%s:%s no CXL window for range %#llx:%#llx\n",
2813 dev_name(&cxlmd->dev), dev_name(&cxld->dev),
2814 cxld->hpa_range.start, cxld->hpa_range.end);
2818 cxlrd = to_cxl_root_decoder(cxlrd_dev);
2821 * Ensure that if multiple threads race to construct_region() for @hpa
2822 * one does the construction and the others add to that.
2824 mutex_lock(&cxlrd->range_lock);
2825 region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
2826 match_region_by_range);
2828 cxlr = construct_region(cxlrd, cxled);
2829 region_dev = &cxlr->dev;
2831 cxlr = to_cxl_region(region_dev);
2832 mutex_unlock(&cxlrd->range_lock);
2834 rc = PTR_ERR_OR_ZERO(cxlr);
2838 attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
2840 down_read(&cxl_region_rwsem);
2842 attach = p->state == CXL_CONFIG_COMMIT;
2843 up_read(&cxl_region_rwsem);
2847 * If device_attach() fails the range may still be active via
2848 * the platform-firmware memory map, otherwise the driver for
2849 * regions is local to this file, so driver matching can't fail.
2851 if (device_attach(&cxlr->dev) < 0)
2852 dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
2856 put_device(region_dev);
2858 put_device(cxlrd_dev);
2861 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL);
2863 static int is_system_ram(struct resource *res, void *arg)
2865 struct cxl_region *cxlr = arg;
2866 struct cxl_region_params *p = &cxlr->params;
2868 dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
2872 static int cxl_region_probe(struct device *dev)
2874 struct cxl_region *cxlr = to_cxl_region(dev);
2875 struct cxl_region_params *p = &cxlr->params;
2878 rc = down_read_interruptible(&cxl_region_rwsem);
2880 dev_dbg(&cxlr->dev, "probe interrupted\n");
2884 if (p->state < CXL_CONFIG_COMMIT) {
2885 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
2890 if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
2892 "failed to activate, re-commit region and retry\n");
2898 * From this point on any path that changes the region's state away from
2899 * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
2902 up_read(&cxl_region_rwsem);
2907 switch (cxlr->mode) {
2908 case CXL_DECODER_PMEM:
2909 return devm_cxl_add_pmem_region(cxlr);
2910 case CXL_DECODER_RAM:
2912 * The region can not be manged by CXL if any portion of
2913 * it is already online as 'System RAM'
2915 if (walk_iomem_res_desc(IORES_DESC_NONE,
2916 IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
2917 p->res->start, p->res->end, cxlr,
2920 return devm_cxl_add_dax_region(cxlr);
2922 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
2928 static struct cxl_driver cxl_region_driver = {
2929 .name = "cxl_region",
2930 .probe = cxl_region_probe,
2931 .id = CXL_DEVICE_REGION,
2934 int cxl_region_init(void)
2936 return cxl_driver_register(&cxl_region_driver);
2939 void cxl_region_exit(void)
2941 cxl_driver_unregister(&cxl_region_driver);
2944 MODULE_IMPORT_NS(CXL);
2945 MODULE_IMPORT_NS(DEVMEM);
2946 MODULE_ALIAS_CXL(CXL_DEVICE_REGION);