From eb4663b07e13bc138aad9e2a93ee9893c7139f51 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Sun, 25 Jun 2023 11:35:20 -0700 Subject: [PATCH] cxl/acpi: Probe RCRB later during RCH downstream port creation The RCRB is extracted already during ACPI CEDT table parsing while the data of this is needed not earlier than dport creation. This implementation comes with drawbacks: During ACPI table scan there is already MMIO access including mapping and unmapping, but only ACPI data should be collected here. The collected data must be transferred through a couple of interfaces until it is finally consumed when creating the dport. This causes complex data structures and function interfaces. Additionally, RCRB parsing will be extended to also extract AER data, it would be much easier do this at a later point during port and dport creation when the data structures are available to hold that data. To simplify all that, probe the RCRB at a later point during RCH downstream port creation. Change ACPI table parser to only extract the base address of either the component registers or the RCRB. Parse and extract the RCRB in devm_cxl_add_rch_dport(). This is in preparation to centralize all RCRB scanning. Signed-off-by: Robert Richter Signed-off-by: Terry Bowman Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20230622205523.85375-2-terry.bowman@amd.com Co-developed-by: Dan Williams Link: https://lore.kernel.org/r/20230622205523.85375-3-terry.bowman@amd.com Signed-off-by: Dan Williams --- drivers/cxl/acpi.c | 51 ++++++++++++++++++------------------------- drivers/cxl/core/core.h | 8 +++++++ drivers/cxl/core/port.c | 21 +++++++++++++----- drivers/cxl/core/regs.c | 15 +++++++++---- drivers/cxl/cxl.h | 12 +++------- drivers/cxl/mem.c | 4 ++-- tools/testing/cxl/Kbuild | 3 ++- tools/testing/cxl/test/cxl.c | 10 --------- tools/testing/cxl/test/mock.c | 34 +++++++++++++++++++++++------ tools/testing/cxl/test/mock.h | 3 --- 10 files changed, 90 insertions(+), 71 deletions(-) diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 7e1765b..70cd9ac 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -372,21 +372,21 @@ static int add_host_bridge_uport(struct device *match, void *arg) return 0; } +/* Note, @dev is used by mock_acpi_table_parse_cedt() */ struct cxl_chbs_context { struct device *dev; unsigned long long uid; - resource_size_t rcrb; - resource_size_t chbcr; + resource_size_t base; u32 cxl_version; }; -static int cxl_get_chbcr(union acpi_subtable_headers *header, void *arg, +static int cxl_get_chbs(union acpi_subtable_headers *header, void *arg, const unsigned long end) { struct cxl_chbs_context *ctx = arg; struct acpi_cedt_chbs *chbs; - if (ctx->chbcr) + if (ctx->base) return 0; chbs = (struct acpi_cedt_chbs *) header; @@ -395,23 +395,16 @@ static int cxl_get_chbcr(union acpi_subtable_headers *header, void *arg, return 0; ctx->cxl_version = chbs->cxl_version; - ctx->rcrb = CXL_RESOURCE_NONE; - ctx->chbcr = CXL_RESOURCE_NONE; + ctx->base = CXL_RESOURCE_NONE; if (!chbs->base) return 0; - if (chbs->cxl_version != ACPI_CEDT_CHBS_VERSION_CXL11) { - ctx->chbcr = chbs->base; - return 0; - } - - if (chbs->length != CXL_RCRB_SIZE) + if (chbs->cxl_version == ACPI_CEDT_CHBS_VERSION_CXL11 && + chbs->length != CXL_RCRB_SIZE) return 0; - ctx->rcrb = chbs->base; - ctx->chbcr = cxl_rcrb_to_component(ctx->dev, chbs->base, - CXL_RCRB_DOWNSTREAM); + ctx->base = chbs->base; return 0; } @@ -443,33 +436,31 @@ static int add_host_bridge_dport(struct device *match, void *arg) .dev = match, .uid = uid, }; - acpi_table_parse_cedt(ACPI_CEDT_TYPE_CHBS, cxl_get_chbcr, &ctx); + acpi_table_parse_cedt(ACPI_CEDT_TYPE_CHBS, cxl_get_chbs, &ctx); - if (!ctx.chbcr) { + if (!ctx.base) { dev_warn(match, "No CHBS found for Host Bridge (UID %lld)\n", uid); return 0; } - if (ctx.rcrb != CXL_RESOURCE_NONE) - dev_dbg(match, "RCRB found for UID %lld: %pa\n", uid, &ctx.rcrb); - - if (ctx.chbcr == CXL_RESOURCE_NONE) { - dev_warn(match, "CHBCR invalid for Host Bridge (UID %lld)\n", + if (ctx.base == CXL_RESOURCE_NONE) { + dev_warn(match, "CHBS invalid for Host Bridge (UID %lld)\n", uid); return 0; } - dev_dbg(match, "CHBCR found: %pa\n", &ctx.chbcr); - pci_root = acpi_pci_find_root(hb->handle); bridge = pci_root->bus->bridge; - if (ctx.cxl_version == ACPI_CEDT_CHBS_VERSION_CXL11) - dport = devm_cxl_add_rch_dport(root_port, bridge, uid, - ctx.chbcr, ctx.rcrb); - else - dport = devm_cxl_add_dport(root_port, bridge, uid, - ctx.chbcr); + + if (ctx.cxl_version == ACPI_CEDT_CHBS_VERSION_CXL11) { + dev_dbg(match, "RCRB found for UID %lld: %pa\n", uid, &ctx.base); + dport = devm_cxl_add_rch_dport(root_port, bridge, uid, ctx.base); + } else { + dev_dbg(match, "CHBCR found for UID %lld: %pa\n", uid, &ctx.base); + dport = devm_cxl_add_dport(root_port, bridge, uid, ctx.base); + } + if (IS_ERR(dport)) return PTR_ERR(dport); diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 27f0968..bd0a578 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -63,6 +63,14 @@ int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size); int cxl_dpa_free(struct cxl_endpoint_decoder *cxled); resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled); resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled); + +enum cxl_rcrb { + CXL_RCRB_DOWNSTREAM, + CXL_RCRB_UPSTREAM, +}; +resource_size_t __rcrb_to_component(struct device *dev, resource_size_t rcrb, + enum cxl_rcrb which); + extern struct rw_semaphore cxl_dpa_rwsem; int cxl_memdev_init(void); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index e7c284c..45f5299a 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -938,12 +938,25 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev, if (!dport) return ERR_PTR(-ENOMEM); + if (rcrb != CXL_RESOURCE_NONE) { + component_reg_phys = __rcrb_to_component(dport_dev, rcrb, + CXL_RCRB_DOWNSTREAM); + if (component_reg_phys == CXL_RESOURCE_NONE) { + dev_warn(dport_dev, "Invalid Component Registers in RCRB"); + return ERR_PTR(-ENXIO); + } + + dport->rch = true; + } + + if (component_reg_phys != CXL_RESOURCE_NONE) + dev_dbg(dport_dev, "Component Registers found for dport: %pa\n", + &component_reg_phys); + dport->dport = dport_dev; dport->port_id = port_id; dport->component_reg_phys = component_reg_phys; dport->port = port; - if (rcrb != CXL_RESOURCE_NONE) - dport->rch = true; dport->rcrb = rcrb; cond_cxl_root_lock(port); @@ -1004,14 +1017,12 @@ EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL); * @port: the cxl_port that references this dport * @dport_dev: firmware or PCI device representing the dport * @port_id: identifier for this dport in a decoder's target list - * @component_reg_phys: optional location of CXL component registers * @rcrb: mandatory location of a Root Complex Register Block * * See CXL 3.0 9.11.8 CXL Devices Attached to an RCH */ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port, struct device *dport_dev, int port_id, - resource_size_t component_reg_phys, resource_size_t rcrb) { struct cxl_dport *dport; @@ -1022,7 +1033,7 @@ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port, } dport = __devm_cxl_add_dport(port, dport_dev, port_id, - component_reg_phys, rcrb); + CXL_RESOURCE_NONE, rcrb); if (IS_ERR(dport)) { dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n", dev_name(&port->dev), PTR_ERR(dport)); diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c index 1476a02..564dd43 100644 --- a/drivers/cxl/core/regs.c +++ b/drivers/cxl/core/regs.c @@ -332,9 +332,8 @@ int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type, } EXPORT_SYMBOL_NS_GPL(cxl_find_regblock, CXL); -resource_size_t cxl_rcrb_to_component(struct device *dev, - resource_size_t rcrb, - enum cxl_rcrb which) +resource_size_t __rcrb_to_component(struct device *dev, resource_size_t rcrb, + enum cxl_rcrb which) { resource_size_t component_reg_phys; void __iomem *addr; @@ -395,4 +394,12 @@ resource_size_t cxl_rcrb_to_component(struct device *dev, return component_reg_phys; } -EXPORT_SYMBOL_NS_GPL(cxl_rcrb_to_component, CXL); + +resource_size_t cxl_rcd_component_reg_phys(struct device *dev, + struct cxl_dport *dport) +{ + if (!dport->rch) + return CXL_RESOURCE_NONE; + return __rcrb_to_component(dev, dport->rcrb, CXL_RCRB_UPSTREAM); +} +EXPORT_SYMBOL_NS_GPL(cxl_rcd_component_reg_phys, CXL); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index f93a285..28888bb 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -262,14 +262,9 @@ int cxl_map_device_regs(struct device *dev, struct cxl_device_regs *regs, enum cxl_regloc_type; int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type, struct cxl_register_map *map); - -enum cxl_rcrb { - CXL_RCRB_DOWNSTREAM, - CXL_RCRB_UPSTREAM, -}; -resource_size_t cxl_rcrb_to_component(struct device *dev, - resource_size_t rcrb, - enum cxl_rcrb which); +struct cxl_dport; +resource_size_t cxl_rcd_component_reg_phys(struct device *dev, + struct cxl_dport *dport); #define CXL_RESOURCE_NONE ((resource_size_t) -1) #define CXL_TARGET_STRLEN 20 @@ -671,7 +666,6 @@ struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port, resource_size_t component_reg_phys); struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port, struct device *dport_dev, int port_id, - resource_size_t component_reg_phys, resource_size_t rcrb); struct cxl_decoder *to_cxl_decoder(struct device *dev); diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index 519edd0..45d4c32 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -72,8 +72,8 @@ static int devm_cxl_add_endpoint(struct device *host, struct cxl_memdev *cxlmd, * typical register locator mechanism. */ if (parent_dport->rch && cxlds->component_reg_phys == CXL_RESOURCE_NONE) - component_reg_phys = cxl_rcrb_to_component( - &cxlmd->dev, parent_dport->rcrb, CXL_RCRB_UPSTREAM); + component_reg_phys = + cxl_rcd_component_reg_phys(&cxlmd->dev, parent_dport); else component_reg_phys = cxlds->component_reg_phys; endpoint = devm_cxl_add_port(host, &cxlmd->dev, component_reg_phys, diff --git a/tools/testing/cxl/Kbuild b/tools/testing/cxl/Kbuild index 6f9347a..8a87d7d 100644 --- a/tools/testing/cxl/Kbuild +++ b/tools/testing/cxl/Kbuild @@ -12,7 +12,8 @@ ldflags-y += --wrap=devm_cxl_enumerate_decoders ldflags-y += --wrap=cxl_await_media_ready ldflags-y += --wrap=cxl_hdm_decode_init ldflags-y += --wrap=cxl_dvsec_rr_decode -ldflags-y += --wrap=cxl_rcrb_to_component +ldflags-y += --wrap=devm_cxl_add_rch_dport +ldflags-y += --wrap=cxl_rcd_component_reg_phys DRIVERS := ../../../drivers CXL_SRC := $(DRIVERS)/cxl diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c index bf00dc5..f5c04787 100644 --- a/tools/testing/cxl/test/cxl.c +++ b/tools/testing/cxl/test/cxl.c @@ -971,15 +971,6 @@ static int mock_cxl_port_enumerate_dports(struct cxl_port *port) return 0; } -resource_size_t mock_cxl_rcrb_to_component(struct device *dev, - resource_size_t rcrb, - enum cxl_rcrb which) -{ - dev_dbg(dev, "rcrb: %pa which: %d\n", &rcrb, which); - - return (resource_size_t) which + 1; -} - static struct cxl_mock_ops cxl_mock_ops = { .is_mock_adev = is_mock_adev, .is_mock_bridge = is_mock_bridge, @@ -988,7 +979,6 @@ static struct cxl_mock_ops cxl_mock_ops = { .is_mock_dev = is_mock_dev, .acpi_table_parse_cedt = mock_acpi_table_parse_cedt, .acpi_evaluate_integer = mock_acpi_evaluate_integer, - .cxl_rcrb_to_component = mock_cxl_rcrb_to_component, .acpi_pci_find_root = mock_acpi_pci_find_root, .devm_cxl_port_enumerate_dports = mock_cxl_port_enumerate_dports, .devm_cxl_setup_hdm = mock_cxl_setup_hdm, diff --git a/tools/testing/cxl/test/mock.c b/tools/testing/cxl/test/mock.c index 2844165..30119a1 100644 --- a/tools/testing/cxl/test/mock.c +++ b/tools/testing/cxl/test/mock.c @@ -259,24 +259,44 @@ int __wrap_cxl_dvsec_rr_decode(struct device *dev, int dvsec, } EXPORT_SYMBOL_NS_GPL(__wrap_cxl_dvsec_rr_decode, CXL); -resource_size_t __wrap_cxl_rcrb_to_component(struct device *dev, - resource_size_t rcrb, - enum cxl_rcrb which) +struct cxl_dport *__wrap_devm_cxl_add_rch_dport(struct cxl_port *port, + struct device *dport_dev, + int port_id, + resource_size_t rcrb) +{ + int index; + struct cxl_dport *dport; + struct cxl_mock_ops *ops = get_cxl_mock_ops(&index); + + if (ops && ops->is_mock_port(dport_dev)) { + dport = devm_cxl_add_dport(port, dport_dev, port_id, + CXL_RESOURCE_NONE); + if (!IS_ERR(dport)) + dport->rch = true; + } else + dport = devm_cxl_add_rch_dport(port, dport_dev, port_id, rcrb); + put_cxl_mock_ops(index); + + return dport; +} +EXPORT_SYMBOL_NS_GPL(__wrap_devm_cxl_add_rch_dport, CXL); + +resource_size_t __wrap_cxl_rcd_component_reg_phys(struct device *dev, + struct cxl_dport *dport) { int index; resource_size_t component_reg_phys; struct cxl_mock_ops *ops = get_cxl_mock_ops(&index); if (ops && ops->is_mock_port(dev)) - component_reg_phys = - ops->cxl_rcrb_to_component(dev, rcrb, which); + component_reg_phys = CXL_RESOURCE_NONE; else - component_reg_phys = cxl_rcrb_to_component(dev, rcrb, which); + component_reg_phys = cxl_rcd_component_reg_phys(dev, dport); put_cxl_mock_ops(index); return component_reg_phys; } -EXPORT_SYMBOL_NS_GPL(__wrap_cxl_rcrb_to_component, CXL); +EXPORT_SYMBOL_NS_GPL(__wrap_cxl_rcd_component_reg_phys, CXL); MODULE_LICENSE("GPL v2"); MODULE_IMPORT_NS(ACPI); diff --git a/tools/testing/cxl/test/mock.h b/tools/testing/cxl/test/mock.h index bef8817..a942237 100644 --- a/tools/testing/cxl/test/mock.h +++ b/tools/testing/cxl/test/mock.h @@ -15,9 +15,6 @@ struct cxl_mock_ops { acpi_string pathname, struct acpi_object_list *arguments, unsigned long long *data); - resource_size_t (*cxl_rcrb_to_component)(struct device *dev, - resource_size_t rcrb, - enum cxl_rcrb which); struct acpi_pci_root *(*acpi_pci_find_root)(acpi_handle handle); bool (*is_mock_bus)(struct pci_bus *bus); bool (*is_mock_port)(struct device *dev); -- 2.7.4