1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3 #include <linux/seq_file.h>
4 #include <linux/device.h>
5 #include <linux/delay.h>
13 * Compute Express Link Host Managed Device Memory, starting with the
14 * CXL 2.0 specification, is managed by an array of HDM Decoder register
15 * instances per CXL port and per CXL endpoint. Define common helpers
16 * for enumerating these registers and capabilities.
19 DECLARE_RWSEM(cxl_dpa_rwsem);
21 static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
26 rc = cxl_decoder_add_locked(cxld, target_map);
28 put_device(&cxld->dev);
29 dev_err(&port->dev, "Failed to add decoder\n");
33 rc = cxl_decoder_autoremove(&port->dev, cxld);
37 dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));
43 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
44 * single ported host-bridges need not publish a decoder capability when a
45 * passthrough decode can be assumed, i.e. all transactions that the uport sees
46 * are claimed and passed to the single dport. Disable the range until the first
47 * CXL region is enumerated / activated.
49 int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
51 struct cxl_switch_decoder *cxlsd;
52 struct cxl_dport *dport = NULL;
53 int single_port_map[1];
56 cxlsd = cxl_switch_decoder_alloc(port, 1);
58 return PTR_ERR(cxlsd);
60 device_lock_assert(&port->dev);
62 xa_for_each(&port->dports, index, dport)
64 single_port_map[0] = dport->port_id;
66 return add_hdm_decoder(port, &cxlsd->cxld, single_port_map);
68 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL);
70 static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
74 hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
75 cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
76 cxlhdm->target_count =
77 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
78 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
79 cxlhdm->interleave_mask |= GENMASK(11, 8);
80 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))
81 cxlhdm->interleave_mask |= GENMASK(14, 12);
84 static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb,
85 struct cxl_component_regs *regs)
87 struct cxl_register_map map = {
89 .resource = port->component_reg_phys,
91 .max_size = CXL_COMPONENT_REG_BLOCK_SIZE,
94 cxl_probe_component_regs(&port->dev, crb, &map.component_map);
95 if (!map.component_map.hdm_decoder.valid) {
96 dev_dbg(&port->dev, "HDM decoder registers not implemented\n");
97 /* unique error code to indicate no HDM decoder capability */
101 return cxl_map_component_regs(&map, regs, BIT(CXL_CM_CAP_CAP_ID_HDM));
104 static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info)
106 struct cxl_hdm *cxlhdm;
114 cxlhdm = dev_get_drvdata(&info->port->dev);
115 hdm = cxlhdm->regs.hdm_decoder;
121 * If HDM decoders are present and the driver is in control of
122 * Mem_Enable skip DVSEC based emulation
124 if (!info->mem_enabled)
128 * If any decoders are committed already, there should not be any
129 * emulated DVSEC decoders.
131 for (i = 0; i < cxlhdm->decoder_count; i++) {
132 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
133 dev_dbg(&info->port->dev,
134 "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n",
136 FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl),
137 readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)),
138 readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)),
139 readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)),
140 readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i)));
141 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
149 * devm_cxl_setup_hdm - map HDM decoder component registers
150 * @port: cxl_port to map
151 * @info: cached DVSEC range register info
153 struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,
154 struct cxl_endpoint_dvsec_info *info)
156 struct device *dev = &port->dev;
157 struct cxl_hdm *cxlhdm;
161 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);
163 return ERR_PTR(-ENOMEM);
165 dev_set_drvdata(dev, cxlhdm);
167 crb = ioremap(port->component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE);
168 if (!crb && info && info->mem_enabled) {
169 cxlhdm->decoder_count = info->ranges;
172 dev_err(dev, "No component registers mapped\n");
173 return ERR_PTR(-ENXIO);
176 rc = map_hdm_decoder_regs(port, crb, &cxlhdm->regs);
181 parse_hdm_decoder_caps(cxlhdm);
182 if (cxlhdm->decoder_count == 0) {
183 dev_err(dev, "Spec violation. Caps invalid\n");
184 return ERR_PTR(-ENXIO);
188 * Now that the hdm capability is parsed, decide if range
189 * register emulation is needed and fixup cxlhdm accordingly.
191 if (should_emulate_decoders(info)) {
192 dev_dbg(dev, "Fallback map %d range register%s\n", info->ranges,
193 info->ranges > 1 ? "s" : "");
194 cxlhdm->decoder_count = info->ranges;
199 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL);
201 static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth)
203 unsigned long long start = r->start, end = r->end;
205 seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end,
209 void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds)
211 struct resource *p1, *p2;
213 down_read(&cxl_dpa_rwsem);
214 for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {
215 __cxl_dpa_debug(file, p1, 0);
216 for (p2 = p1->child; p2; p2 = p2->sibling)
217 __cxl_dpa_debug(file, p2, 1);
219 up_read(&cxl_dpa_rwsem);
221 EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, CXL);
224 * Must be called in a context that synchronizes against this decoder's
225 * port ->remove() callback (like an endpoint decoder sysfs attribute)
227 static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
229 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
230 struct cxl_port *port = cxled_to_port(cxled);
231 struct cxl_dev_state *cxlds = cxlmd->cxlds;
232 struct resource *res = cxled->dpa_res;
233 resource_size_t skip_start;
235 lockdep_assert_held_write(&cxl_dpa_rwsem);
237 /* save @skip_start, before @res is released */
238 skip_start = res->start - cxled->skip;
239 __release_region(&cxlds->dpa_res, res->start, resource_size(res));
241 __release_region(&cxlds->dpa_res, skip_start, cxled->skip);
243 cxled->dpa_res = NULL;
244 put_device(&cxled->cxld.dev);
248 static void cxl_dpa_release(void *cxled)
250 down_write(&cxl_dpa_rwsem);
251 __cxl_dpa_release(cxled);
252 up_write(&cxl_dpa_rwsem);
256 * Must be called from context that will not race port device
257 * unregistration, like decoder sysfs attribute methods
259 static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
261 struct cxl_port *port = cxled_to_port(cxled);
263 lockdep_assert_held_write(&cxl_dpa_rwsem);
264 devm_remove_action(&port->dev, cxl_dpa_release, cxled);
265 __cxl_dpa_release(cxled);
268 static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
269 resource_size_t base, resource_size_t len,
270 resource_size_t skipped)
272 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
273 struct cxl_port *port = cxled_to_port(cxled);
274 struct cxl_dev_state *cxlds = cxlmd->cxlds;
275 struct device *dev = &port->dev;
276 struct resource *res;
278 lockdep_assert_held_write(&cxl_dpa_rwsem);
281 dev_warn(dev, "decoder%d.%d: empty reservation attempted\n",
282 port->id, cxled->cxld.id);
286 if (cxled->dpa_res) {
287 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
288 port->id, cxled->cxld.id, cxled->dpa_res);
292 if (port->hdm_end + 1 != cxled->cxld.id) {
294 * Assumes alloc and commit order is always in hardware instance
295 * order per expectations from 8.2.5.12.20 Committing Decoder
296 * Programming that enforce decoder[m] committed before
297 * decoder[m+1] commit start.
299 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id,
300 cxled->cxld.id, port->id, port->hdm_end + 1);
305 res = __request_region(&cxlds->dpa_res, base - skipped, skipped,
306 dev_name(&cxled->cxld.dev), 0);
309 "decoder%d.%d: failed to reserve skipped space\n",
310 port->id, cxled->cxld.id);
314 res = __request_region(&cxlds->dpa_res, base, len,
315 dev_name(&cxled->cxld.dev), 0);
317 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
318 port->id, cxled->cxld.id);
320 __release_region(&cxlds->dpa_res, base - skipped,
324 cxled->dpa_res = res;
325 cxled->skip = skipped;
327 if (resource_contains(&cxlds->pmem_res, res))
328 cxled->mode = CXL_DECODER_PMEM;
329 else if (resource_contains(&cxlds->ram_res, res))
330 cxled->mode = CXL_DECODER_RAM;
332 dev_dbg(dev, "decoder%d.%d: %pr mixed\n", port->id,
333 cxled->cxld.id, cxled->dpa_res);
334 cxled->mode = CXL_DECODER_MIXED;
338 get_device(&cxled->cxld.dev);
342 int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
343 resource_size_t base, resource_size_t len,
344 resource_size_t skipped)
346 struct cxl_port *port = cxled_to_port(cxled);
349 down_write(&cxl_dpa_rwsem);
350 rc = __cxl_dpa_reserve(cxled, base, len, skipped);
351 up_write(&cxl_dpa_rwsem);
356 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
358 EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, CXL);
360 resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
362 resource_size_t size = 0;
364 down_read(&cxl_dpa_rwsem);
366 size = resource_size(cxled->dpa_res);
367 up_read(&cxl_dpa_rwsem);
372 resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)
374 resource_size_t base = -1;
376 down_read(&cxl_dpa_rwsem);
378 base = cxled->dpa_res->start;
379 up_read(&cxl_dpa_rwsem);
384 int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
386 struct cxl_port *port = cxled_to_port(cxled);
387 struct device *dev = &cxled->cxld.dev;
390 down_write(&cxl_dpa_rwsem);
391 if (!cxled->dpa_res) {
395 if (cxled->cxld.region) {
396 dev_dbg(dev, "decoder assigned to: %s\n",
397 dev_name(&cxled->cxld.region->dev));
401 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
402 dev_dbg(dev, "decoder enabled\n");
406 if (cxled->cxld.id != port->hdm_end) {
407 dev_dbg(dev, "expected decoder%d.%d\n", port->id,
412 devm_cxl_dpa_release(cxled);
415 up_write(&cxl_dpa_rwsem);
419 int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,
420 enum cxl_decoder_mode mode)
422 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
423 struct cxl_dev_state *cxlds = cxlmd->cxlds;
424 struct device *dev = &cxled->cxld.dev;
428 case CXL_DECODER_RAM:
429 case CXL_DECODER_PMEM:
432 dev_dbg(dev, "unsupported mode: %d\n", mode);
436 down_write(&cxl_dpa_rwsem);
437 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
443 * Only allow modes that are supported by the current partition
446 if (mode == CXL_DECODER_PMEM && !resource_size(&cxlds->pmem_res)) {
447 dev_dbg(dev, "no available pmem capacity\n");
451 if (mode == CXL_DECODER_RAM && !resource_size(&cxlds->ram_res)) {
452 dev_dbg(dev, "no available ram capacity\n");
460 up_write(&cxl_dpa_rwsem);
465 int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
467 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
468 resource_size_t free_ram_start, free_pmem_start;
469 struct cxl_port *port = cxled_to_port(cxled);
470 struct cxl_dev_state *cxlds = cxlmd->cxlds;
471 struct device *dev = &cxled->cxld.dev;
472 resource_size_t start, avail, skip;
473 struct resource *p, *last;
476 down_write(&cxl_dpa_rwsem);
477 if (cxled->cxld.region) {
478 dev_dbg(dev, "decoder attached to %s\n",
479 dev_name(&cxled->cxld.region->dev));
484 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
485 dev_dbg(dev, "decoder enabled\n");
490 for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling)
493 free_ram_start = last->end + 1;
495 free_ram_start = cxlds->ram_res.start;
497 for (p = cxlds->pmem_res.child, last = NULL; p; p = p->sibling)
500 free_pmem_start = last->end + 1;
502 free_pmem_start = cxlds->pmem_res.start;
504 if (cxled->mode == CXL_DECODER_RAM) {
505 start = free_ram_start;
506 avail = cxlds->ram_res.end - start + 1;
508 } else if (cxled->mode == CXL_DECODER_PMEM) {
509 resource_size_t skip_start, skip_end;
511 start = free_pmem_start;
512 avail = cxlds->pmem_res.end - start + 1;
513 skip_start = free_ram_start;
516 * If some pmem is already allocated, then that allocation
517 * already handled the skip.
519 if (cxlds->pmem_res.child &&
520 skip_start == cxlds->pmem_res.child->start)
521 skip_end = skip_start - 1;
523 skip_end = start - 1;
524 skip = skip_end - skip_start + 1;
526 dev_dbg(dev, "mode not set\n");
532 dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size,
533 cxled->mode == CXL_DECODER_RAM ? "ram" : "pmem",
539 rc = __cxl_dpa_reserve(cxled, start, size, skip);
541 up_write(&cxl_dpa_rwsem);
546 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
549 static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
555 * Input validation ensures these warns never fire, but otherwise
556 * suppress unititalized variable usage warnings.
558 if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw),
559 "invalid interleave_ways: %d\n", cxld->interleave_ways))
561 if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig),
562 "invalid interleave_granularity: %d\n",
563 cxld->interleave_granularity))
566 u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);
567 u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK);
568 *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT;
571 static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)
573 u32p_replace_bits(ctrl,
574 !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM),
575 CXL_HDM_DECODER0_CTRL_HOSTONLY);
578 static int cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)
580 struct cxl_dport **t = &cxlsd->target[0];
581 int ways = cxlsd->cxld.interleave_ways;
583 if (dev_WARN_ONCE(&cxlsd->cxld.dev,
584 ways > 8 || ways > cxlsd->nr_targets,
585 "ways: %d overflows targets: %d\n", ways,
589 *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id);
591 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id);
593 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id);
595 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id);
597 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id);
599 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id);
601 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id);
603 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id);
609 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set
610 * committed or error within 10ms, but just be generous with 20ms to account for
611 * clock skew and other marginal behavior
613 #define COMMIT_TIMEOUT_MS 20
614 static int cxld_await_commit(void __iomem *hdm, int id)
619 for (i = 0; i < COMMIT_TIMEOUT_MS; i++) {
620 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
621 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) {
622 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
623 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
626 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
634 static int cxl_decoder_commit(struct cxl_decoder *cxld)
636 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
637 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
638 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
639 int id = cxld->id, rc;
643 if (cxld->flags & CXL_DECODER_F_ENABLE)
646 if (port->commit_end + 1 != id) {
648 "%s: out of order commit, expected decoder%d.%d\n",
649 dev_name(&cxld->dev), port->id, port->commit_end + 1);
653 down_read(&cxl_dpa_rwsem);
654 /* common decoder settings */
655 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
656 cxld_set_interleave(cxld, &ctrl);
657 cxld_set_type(cxld, &ctrl);
658 base = cxld->hpa_range.start;
659 size = range_len(&cxld->hpa_range);
661 writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
662 writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
663 writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
664 writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
666 if (is_switch_decoder(&cxld->dev)) {
667 struct cxl_switch_decoder *cxlsd =
668 to_cxl_switch_decoder(&cxld->dev);
669 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);
670 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);
673 rc = cxlsd_set_targets(cxlsd, &targets);
675 dev_dbg(&port->dev, "%s: target configuration error\n",
676 dev_name(&cxld->dev));
680 writel(upper_32_bits(targets), tl_hi);
681 writel(lower_32_bits(targets), tl_lo);
683 struct cxl_endpoint_decoder *cxled =
684 to_cxl_endpoint_decoder(&cxld->dev);
685 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);
686 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);
688 writel(upper_32_bits(cxled->skip), sk_hi);
689 writel(lower_32_bits(cxled->skip), sk_lo);
692 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
693 up_read(&cxl_dpa_rwsem);
696 rc = cxld_await_commit(hdm, cxld->id);
699 dev_dbg(&port->dev, "%s: error %d committing decoder\n",
700 dev_name(&cxld->dev), rc);
704 cxld->flags |= CXL_DECODER_F_ENABLE;
709 static int cxl_decoder_reset(struct cxl_decoder *cxld)
711 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
712 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
713 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
717 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
720 if (port->commit_end != id) {
722 "%s: out of order reset, expected decoder%d.%d\n",
723 dev_name(&cxld->dev), port->id, port->commit_end);
727 down_read(&cxl_dpa_rwsem);
728 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
729 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
730 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
732 writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
733 writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
734 writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
735 writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
736 up_read(&cxl_dpa_rwsem);
739 cxld->flags &= ~CXL_DECODER_F_ENABLE;
741 /* Userspace is now responsible for reconfiguring this decoder */
742 if (is_endpoint_decoder(&cxld->dev)) {
743 struct cxl_endpoint_decoder *cxled;
745 cxled = to_cxl_endpoint_decoder(&cxld->dev);
746 cxled->state = CXL_DECODER_STATE_MANUAL;
752 static int cxl_setup_hdm_decoder_from_dvsec(
753 struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base,
754 int which, struct cxl_endpoint_dvsec_info *info)
756 struct cxl_endpoint_decoder *cxled;
760 if (!is_cxl_endpoint(port))
763 cxled = to_cxl_endpoint_decoder(&cxld->dev);
764 len = range_len(&info->dvsec_range[which]);
768 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
771 cxld->hpa_range = info->dvsec_range[which];
774 * Set the emulated decoder as locked pending additional support to
775 * change the range registers at run time.
777 cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;
778 port->commit_end = cxld->id;
780 rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0);
783 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
784 port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc);
788 cxled->state = CXL_DECODER_STATE_AUTO;
793 static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
794 int *target_map, void __iomem *hdm, int which,
795 u64 *dpa_base, struct cxl_endpoint_dvsec_info *info)
797 struct cxl_endpoint_decoder *cxled = NULL;
798 u64 size, base, skip, dpa_size, lo, hi;
805 unsigned char target_id[8];
808 if (should_emulate_decoders(info))
809 return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base,
812 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
813 lo = readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
814 hi = readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(which));
815 base = (hi << 32) + lo;
816 lo = readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
817 hi = readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(which));
818 size = (hi << 32) + lo;
819 committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED);
820 cxld->commit = cxl_decoder_commit;
821 cxld->reset = cxl_decoder_reset;
825 if (base == U64_MAX || size == U64_MAX) {
826 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
832 cxled = to_cxl_endpoint_decoder(&cxld->dev);
833 cxld->hpa_range = (struct range) {
835 .end = base + size - 1,
838 /* decoders are enabled if committed */
840 cxld->flags |= CXL_DECODER_F_ENABLE;
841 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
842 cxld->flags |= CXL_DECODER_F_LOCK;
843 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl))
844 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
846 cxld->target_type = CXL_DECODER_DEVMEM;
847 if (cxld->id != port->commit_end + 1) {
849 "decoder%d.%d: Committed out of order\n",
856 "decoder%d.%d: Committed with zero size\n",
860 port->commit_end = cxld->id;
863 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
864 struct cxl_dev_state *cxlds = cxlmd->cxlds;
867 * Default by devtype until a device arrives that needs
870 if (cxlds->type == CXL_DEVTYPE_CLASSMEM)
871 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
873 cxld->target_type = CXL_DECODER_DEVMEM;
875 /* To be overridden by region type at commit time */
876 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
879 if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) &&
880 cxld->target_type == CXL_DECODER_HOSTONLYMEM) {
881 ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY;
882 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
885 rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),
886 &cxld->interleave_ways);
889 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
890 port->id, cxld->id, ctrl);
893 rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl),
894 &cxld->interleave_granularity);
898 dev_dbg(&port->dev, "decoder%d.%d: range: %#llx-%#llx iw: %d ig: %d\n",
899 port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end,
900 cxld->interleave_ways, cxld->interleave_granularity);
903 lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which));
904 hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which));
905 target_list.value = (hi << 32) + lo;
906 for (i = 0; i < cxld->interleave_ways; i++)
907 target_map[i] = target_list.target_id[i];
915 dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder);
918 "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n",
919 port->id, cxld->id, size, cxld->interleave_ways);
922 lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));
923 hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which));
924 skip = (hi << 32) + lo;
925 rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);
928 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
929 port->id, cxld->id, *dpa_base,
930 *dpa_base + dpa_size + skip - 1, rc);
933 *dpa_base += dpa_size + skip;
935 cxled->state = CXL_DECODER_STATE_AUTO;
940 static void cxl_settle_decoders(struct cxl_hdm *cxlhdm)
942 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
950 * Since the register resource was recently claimed via request_region()
951 * be careful about trusting the "not-committed" status until the commit
952 * timeout has elapsed. The commit timeout is 10ms (CXL 2.0
953 * 8.2.5.12.20), but double it to be tolerant of any clock skew between
956 for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {
957 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
958 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)
962 /* ensure that future checks of committed can be trusted */
963 if (committed != cxlhdm->decoder_count)
968 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set
969 * @cxlhdm: Structure to populate with HDM capabilities
970 * @info: cached DVSEC range register info
972 int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
973 struct cxl_endpoint_dvsec_info *info)
975 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
976 struct cxl_port *port = cxlhdm->port;
980 cxl_settle_decoders(cxlhdm);
982 for (i = 0; i < cxlhdm->decoder_count; i++) {
983 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
984 int rc, target_count = cxlhdm->target_count;
985 struct cxl_decoder *cxld;
987 if (is_cxl_endpoint(port)) {
988 struct cxl_endpoint_decoder *cxled;
990 cxled = cxl_endpoint_decoder_alloc(port);
993 "Failed to allocate decoder%d.%d\n",
995 return PTR_ERR(cxled);
999 struct cxl_switch_decoder *cxlsd;
1001 cxlsd = cxl_switch_decoder_alloc(port, target_count);
1002 if (IS_ERR(cxlsd)) {
1003 dev_warn(&port->dev,
1004 "Failed to allocate decoder%d.%d\n",
1006 return PTR_ERR(cxlsd);
1008 cxld = &cxlsd->cxld;
1011 rc = init_hdm_decoder(port, cxld, target_map, hdm, i,
1014 dev_warn(&port->dev,
1015 "Failed to initialize decoder%d.%d\n",
1017 put_device(&cxld->dev);
1020 rc = add_hdm_decoder(port, cxld, target_map);
1022 dev_warn(&port->dev,
1023 "Failed to add decoder%d.%d\n", port->id, i);
1030 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);