drivers/cdx/cdx.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * CDX bus driver.
   4  *
   5  * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
   6  */
   7
   8 /*
   9  * Architecture Overview
  10  * =====================
  11  * CDX is a Hardware Architecture designed for AMD FPGA devices. It
  12  * consists of sophisticated mechanism for interaction between FPGA,
  13  * Firmware and the APUs (Application CPUs).
  14  *
  15  * Firmware resides on RPU (Realtime CPUs) which interacts with
  16  * the FPGA program manager and the APUs. The RPU provides memory-mapped
  17  * interface (RPU if) which is used to communicate with APUs.
  18  *
  19  * The diagram below shows an overview of the CDX architecture:
  20  *
  21  *          +--------------------------------------+
  22  *          |    Application CPUs (APU)            |
  23  *          |                                      |
  24  *          |                    CDX device drivers|
  25  *          |     Linux OS                |        |
  26  *          |                        CDX bus       |
  27  *          |                             |        |
  28  *          |                     CDX controller   |
  29  *          |                             |        |
  30  *          +-----------------------------|--------+
  31  *                                        | (discover, config,
  32  *                                        |  reset, rescan)
  33  *                                        |
  34  *          +------------------------| RPU if |----+
  35  *          |                             |        |
  36  *          |                             V        |
  37  *          |          Realtime CPUs (RPU)         |
  38  *          |                                      |
  39  *          +--------------------------------------+
  40  *                                |
  41  *          +---------------------|----------------+
  42  *          |  FPGA               |                |
  43  *          |      +-----------------------+       |
  44  *          |      |           |           |       |
  45  *          | +-------+    +-------+   +-------+   |
  46  *          | | dev 1 |    | dev 2 |   | dev 3 |   |
  47  *          | +-------+    +-------+   +-------+   |
  48  *          +--------------------------------------+
  49  *
  50  * The RPU firmware extracts the device information from the loaded FPGA
  51  * image and implements a mechanism that allows the APU drivers to
  52  * enumerate such devices (device personality and resource details) via
  53  * a dedicated communication channel. RPU mediates operations such as
  54  * discover, reset and rescan of the FPGA devices for the APU. This is
  55  * done using memory mapped interface provided by the RPU to APU.
  56  */
  57
  58 #include <linux/init.h>
  59 #include <linux/kernel.h>
  60 #include <linux/of_device.h>
  61 #include <linux/slab.h>
  62 #include <linux/mm.h>
  63 #include <linux/xarray.h>
  64 #include <linux/cdx/cdx_bus.h>
  65 #include <linux/iommu.h>
  66 #include <linux/dma-map-ops.h>
  67 #include "cdx.h"
  68
  69 /* Default DMA mask for devices on a CDX bus */
  70 #define CDX_DEFAULT_DMA_MASK    (~0ULL)
  71 #define MAX_CDX_CONTROLLERS 16
  72
  73 /* CDX controllers registered with the CDX bus */
  74 static DEFINE_XARRAY_ALLOC(cdx_controllers);
  75
  76 /**
  77  * cdx_dev_reset - Reset a CDX device
  78  * @dev: CDX device
  79  *
  80  * Return: -errno on failure, 0 on success.
  81  */
  82 int cdx_dev_reset(struct device *dev)
  83 {
  84         struct cdx_device *cdx_dev = to_cdx_device(dev);
  85         struct cdx_controller *cdx = cdx_dev->cdx;
  86         struct cdx_device_config dev_config = {0};
  87         struct cdx_driver *cdx_drv;
  88         int ret;
  89
  90         cdx_drv = to_cdx_driver(dev->driver);
  91         /* Notify driver that device is being reset */
  92         if (cdx_drv && cdx_drv->reset_prepare)
  93                 cdx_drv->reset_prepare(cdx_dev);
  94
  95         dev_config.type = CDX_DEV_RESET_CONF;
  96         ret = cdx->ops->dev_configure(cdx, cdx_dev->bus_num,
  97                                       cdx_dev->dev_num, &dev_config);
  98         if (ret)
  99                 dev_err(dev, "cdx device reset failed\n");
 100
 101         /* Notify driver that device reset is complete */
 102         if (cdx_drv && cdx_drv->reset_done)
 103                 cdx_drv->reset_done(cdx_dev);
 104
 105         return ret;
 106 }
 107 EXPORT_SYMBOL_GPL(cdx_dev_reset);
 108
 109 /**
 110  * cdx_unregister_device - Unregister a CDX device
 111  * @dev: CDX device
 112  * @data: This is always passed as NULL, and is not used in this API,
 113  *        but is required here as the bus_for_each_dev() API expects
 114  *        the passed function (cdx_unregister_device) to have this
 115  *        as an argument.
 116  *
 117  * Return: 0 on success.
 118  */
 119 static int cdx_unregister_device(struct device *dev,
 120                                  void *data)
 121 {
 122         struct cdx_device *cdx_dev = to_cdx_device(dev);
 123
 124         kfree(cdx_dev->driver_override);
 125         cdx_dev->driver_override = NULL;
 126         /*
 127          * Do not free cdx_dev here as it would be freed in
 128          * cdx_device_release() called from within put_device().
 129          */
 130         device_del(&cdx_dev->dev);
 131         put_device(&cdx_dev->dev);
 132
 133         return 0;
 134 }
 135
 136 static void cdx_unregister_devices(struct bus_type *bus)
 137 {
 138         /* Reset all the devices attached to cdx bus */
 139         bus_for_each_dev(bus, NULL, NULL, cdx_unregister_device);
 140 }
 141
 142 /**
 143  * cdx_match_one_device - Tell if a CDX device structure has a matching
 144  *                        CDX device id structure
 145  * @id: single CDX device id structure to match
 146  * @dev: the CDX device structure to match against
 147  *
 148  * Return: matching cdx_device_id structure or NULL if there is no match.
 149  */
 150 static inline const struct cdx_device_id *
 151 cdx_match_one_device(const struct cdx_device_id *id,
 152                      const struct cdx_device *dev)
 153 {
 154         /* Use vendor ID and device ID for matching */
 155         if ((id->vendor == CDX_ANY_ID || id->vendor == dev->vendor) &&
 156             (id->device == CDX_ANY_ID || id->device == dev->device))
 157                 return id;
 158         return NULL;
 159 }
 160
 161 /**
 162  * cdx_match_id - See if a CDX device matches a given cdx_id table
 163  * @ids: array of CDX device ID structures to search in
 164  * @dev: the CDX device structure to match against.
 165  *
 166  * Used by a driver to check whether a CDX device is in its list of
 167  * supported devices. Returns the matching cdx_device_id structure or
 168  * NULL if there is no match.
 169  *
 170  * Return: matching cdx_device_id structure or NULL if there is no match.
 171  */
 172 static inline const struct cdx_device_id *
 173 cdx_match_id(const struct cdx_device_id *ids, struct cdx_device *dev)
 174 {
 175         if (ids) {
 176                 while (ids->vendor || ids->device) {
 177                         if (cdx_match_one_device(ids, dev))
 178                                 return ids;
 179                         ids++;
 180                 }
 181         }
 182         return NULL;
 183 }
 184
 185 /**
 186  * cdx_bus_match - device to driver matching callback
 187  * @dev: the cdx device to match against
 188  * @drv: the device driver to search for matching cdx device
 189  * structures
 190  *
 191  * Return: true on success, false otherwise.
 192  */
 193 static int cdx_bus_match(struct device *dev, struct device_driver *drv)
 194 {
 195         struct cdx_device *cdx_dev = to_cdx_device(dev);
 196         struct cdx_driver *cdx_drv = to_cdx_driver(drv);
 197         const struct cdx_device_id *found_id = NULL;
 198         const struct cdx_device_id *ids;
 199
 200         ids = cdx_drv->match_id_table;
 201
 202         /* When driver_override is set, only bind to the matching driver */
 203         if (cdx_dev->driver_override && strcmp(cdx_dev->driver_override, drv->name))
 204                 return false;
 205
 206         found_id = cdx_match_id(ids, cdx_dev);
 207         if (!found_id)
 208                 return false;
 209
 210         do {
 211                 /*
 212                  * In case override_only was set, enforce driver_override
 213                  * matching.
 214                  */
 215                 if (!found_id->override_only)
 216                         return true;
 217                 if (cdx_dev->driver_override)
 218                         return true;
 219
 220                 ids = found_id + 1;
 221                 found_id = cdx_match_id(ids, cdx_dev);
 222         } while (found_id);
 223
 224         return false;
 225 }
 226
 227 static int cdx_probe(struct device *dev)
 228 {
 229         struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
 230         struct cdx_device *cdx_dev = to_cdx_device(dev);
 231         int error;
 232
 233         error = cdx_drv->probe(cdx_dev);
 234         if (error) {
 235                 dev_err_probe(dev, error, "%s failed\n", __func__);
 236                 return error;
 237         }
 238
 239         return 0;
 240 }
 241
 242 static void cdx_remove(struct device *dev)
 243 {
 244         struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
 245         struct cdx_device *cdx_dev = to_cdx_device(dev);
 246
 247         if (cdx_drv && cdx_drv->remove)
 248                 cdx_drv->remove(cdx_dev);
 249 }
 250
 251 static void cdx_shutdown(struct device *dev)
 252 {
 253         struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
 254         struct cdx_device *cdx_dev = to_cdx_device(dev);
 255
 256         if (cdx_drv && cdx_drv->shutdown)
 257                 cdx_drv->shutdown(cdx_dev);
 258 }
 259
 260 static int cdx_dma_configure(struct device *dev)
 261 {
 262         struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
 263         struct cdx_device *cdx_dev = to_cdx_device(dev);
 264         u32 input_id = cdx_dev->req_id;
 265         int ret;
 266
 267         ret = of_dma_configure_id(dev, dev->parent->of_node, 0, &input_id);
 268         if (ret && ret != -EPROBE_DEFER) {
 269                 dev_err(dev, "of_dma_configure_id() failed\n");
 270                 return ret;
 271         }
 272
 273         if (!ret && !cdx_drv->driver_managed_dma) {
 274                 ret = iommu_device_use_default_domain(dev);
 275                 if (ret)
 276                         arch_teardown_dma_ops(dev);
 277         }
 278
 279         return 0;
 280 }
 281
 282 static void cdx_dma_cleanup(struct device *dev)
 283 {
 284         struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
 285
 286         if (!cdx_drv->driver_managed_dma)
 287                 iommu_device_unuse_default_domain(dev);
 288 }
 289
 290 /* show configuration fields */
 291 #define cdx_config_attr(field, format_string)   \
 292 static ssize_t  \
 293 field##_show(struct device *dev, struct device_attribute *attr, char *buf)      \
 294 {       \
 295         struct cdx_device *cdx_dev = to_cdx_device(dev);        \
 296         return sysfs_emit(buf, format_string, cdx_dev->field);  \
 297 }       \
 298 static DEVICE_ATTR_RO(field)
 299
 300 cdx_config_attr(vendor, "0x%04x\n");
 301 cdx_config_attr(device, "0x%04x\n");
 302
 303 static ssize_t remove_store(struct device *dev,
 304                             struct device_attribute *attr,
 305                             const char *buf, size_t count)
 306 {
 307         bool val;
 308
 309         if (kstrtobool(buf, &val) < 0)
 310                 return -EINVAL;
 311
 312         if (!val)
 313                 return -EINVAL;
 314
 315         if (device_remove_file_self(dev, attr)) {
 316                 int ret;
 317
 318                 ret = cdx_unregister_device(dev, NULL);
 319                 if (ret)
 320                         return ret;
 321         }
 322
 323         return count;
 324 }
 325 static DEVICE_ATTR_WO(remove);
 326
 327 static ssize_t reset_store(struct device *dev, struct device_attribute *attr,
 328                            const char *buf, size_t count)
 329 {
 330         bool val;
 331         int ret;
 332
 333         if (kstrtobool(buf, &val) < 0)
 334                 return -EINVAL;
 335
 336         if (!val)
 337                 return -EINVAL;
 338
 339         ret = cdx_dev_reset(dev);
 340         if (ret)
 341                 return ret;
 342
 343         return count;
 344 }
 345 static DEVICE_ATTR_WO(reset);
 346
 347 static ssize_t driver_override_store(struct device *dev,
 348                                      struct device_attribute *attr,
 349                                      const char *buf, size_t count)
 350 {
 351         struct cdx_device *cdx_dev = to_cdx_device(dev);
 352         int ret;
 353
 354         if (WARN_ON(dev->bus != &cdx_bus_type))
 355                 return -EINVAL;
 356
 357         ret = driver_set_override(dev, &cdx_dev->driver_override, buf, count);
 358         if (ret)
 359                 return ret;
 360
 361         return count;
 362 }
 363
 364 static ssize_t driver_override_show(struct device *dev,
 365                                     struct device_attribute *attr, char *buf)
 366 {
 367         struct cdx_device *cdx_dev = to_cdx_device(dev);
 368
 369         return sysfs_emit(buf, "%s\n", cdx_dev->driver_override);
 370 }
 371 static DEVICE_ATTR_RW(driver_override);
 372
 373 static struct attribute *cdx_dev_attrs[] = {
 374         &dev_attr_remove.attr,
 375         &dev_attr_reset.attr,
 376         &dev_attr_vendor.attr,
 377         &dev_attr_device.attr,
 378         &dev_attr_driver_override.attr,
 379         NULL,
 380 };
 381 ATTRIBUTE_GROUPS(cdx_dev);
 382
 383 static ssize_t rescan_store(const struct bus_type *bus,
 384                             const char *buf, size_t count)
 385 {
 386         struct cdx_controller *cdx;
 387         unsigned long index;
 388         bool val;
 389
 390         if (kstrtobool(buf, &val) < 0)
 391                 return -EINVAL;
 392
 393         if (!val)
 394                 return -EINVAL;
 395
 396         /* Unregister all the devices on the bus */
 397         cdx_unregister_devices(&cdx_bus_type);
 398
 399         /* Rescan all the devices */
 400         xa_for_each(&cdx_controllers, index, cdx) {
 401                 int ret;
 402
 403                 ret = cdx->ops->scan(cdx);
 404                 if (ret)
 405                         dev_err(cdx->dev, "cdx bus scanning failed\n");
 406         }
 407
 408         return count;
 409 }
 410 static BUS_ATTR_WO(rescan);
 411
 412 static struct attribute *cdx_bus_attrs[] = {
 413         &bus_attr_rescan.attr,
 414         NULL,
 415 };
 416 ATTRIBUTE_GROUPS(cdx_bus);
 417
 418 struct bus_type cdx_bus_type = {
 419         .name           = "cdx",
 420         .match          = cdx_bus_match,
 421         .probe          = cdx_probe,
 422         .remove         = cdx_remove,
 423         .shutdown       = cdx_shutdown,
 424         .dma_configure  = cdx_dma_configure,
 425         .dma_cleanup    = cdx_dma_cleanup,
 426         .bus_groups     = cdx_bus_groups,
 427         .dev_groups     = cdx_dev_groups,
 428 };
 429 EXPORT_SYMBOL_GPL(cdx_bus_type);
 430
 431 int __cdx_driver_register(struct cdx_driver *cdx_driver,
 432                           struct module *owner)
 433 {
 434         int error;
 435
 436         cdx_driver->driver.owner = owner;
 437         cdx_driver->driver.bus = &cdx_bus_type;
 438
 439         error = driver_register(&cdx_driver->driver);
 440         if (error) {
 441                 pr_err("driver_register() failed for %s: %d\n",
 442                        cdx_driver->driver.name, error);
 443                 return error;
 444         }
 445
 446         return 0;
 447 }
 448 EXPORT_SYMBOL_GPL(__cdx_driver_register);
 449
 450 void cdx_driver_unregister(struct cdx_driver *cdx_driver)
 451 {
 452         driver_unregister(&cdx_driver->driver);
 453 }
 454 EXPORT_SYMBOL_GPL(cdx_driver_unregister);
 455
 456 static void cdx_device_release(struct device *dev)
 457 {
 458         struct cdx_device *cdx_dev = to_cdx_device(dev);
 459
 460         kfree(cdx_dev);
 461 }
 462
 463 int cdx_device_add(struct cdx_dev_params *dev_params)
 464 {
 465         struct cdx_controller *cdx = dev_params->cdx;
 466         struct device *parent = cdx->dev;
 467         struct cdx_device *cdx_dev;
 468         int ret;
 469
 470         cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL);
 471         if (!cdx_dev)
 472                 return -ENOMEM;
 473
 474         /* Populate resource */
 475         memcpy(cdx_dev->res, dev_params->res, sizeof(struct resource) *
 476                 dev_params->res_count);
 477         cdx_dev->res_count = dev_params->res_count;
 478
 479         /* Populate CDX dev params */
 480         cdx_dev->req_id = dev_params->req_id;
 481         cdx_dev->vendor = dev_params->vendor;
 482         cdx_dev->device = dev_params->device;
 483         cdx_dev->bus_num = dev_params->bus_num;
 484         cdx_dev->dev_num = dev_params->dev_num;
 485         cdx_dev->cdx = dev_params->cdx;
 486         cdx_dev->dma_mask = CDX_DEFAULT_DMA_MASK;
 487
 488         /* Initialize generic device */
 489         device_initialize(&cdx_dev->dev);
 490         cdx_dev->dev.parent = parent;
 491         cdx_dev->dev.bus = &cdx_bus_type;
 492         cdx_dev->dev.dma_mask = &cdx_dev->dma_mask;
 493         cdx_dev->dev.release = cdx_device_release;
 494
 495         /* Set Name */
 496         dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x",
 497                      ((cdx->id << CDX_CONTROLLER_ID_SHIFT) | (cdx_dev->bus_num & CDX_BUS_NUM_MASK)),
 498                      cdx_dev->dev_num);
 499
 500         ret = device_add(&cdx_dev->dev);
 501         if (ret) {
 502                 dev_err(&cdx_dev->dev,
 503                         "cdx device add failed: %d", ret);
 504                 goto fail;
 505         }
 506
 507         return 0;
 508 fail:
 509         /*
 510          * Do not free cdx_dev here as it would be freed in
 511          * cdx_device_release() called from put_device().
 512          */
 513         put_device(&cdx_dev->dev);
 514
 515         return ret;
 516 }
 517 EXPORT_SYMBOL_GPL(cdx_device_add);
 518
 519 int cdx_register_controller(struct cdx_controller *cdx)
 520 {
 521         int ret;
 522
 523         ret = xa_alloc(&cdx_controllers, &cdx->id, cdx,
 524                        XA_LIMIT(0, MAX_CDX_CONTROLLERS - 1), GFP_KERNEL);
 525         if (ret) {
 526                 dev_err(cdx->dev,
 527                         "No free index available. Maximum controllers already registered\n");
 528                 cdx->id = (u8)MAX_CDX_CONTROLLERS;
 529                 return ret;
 530         }
 531
 532         /* Scan all the devices */
 533         cdx->ops->scan(cdx);
 534
 535         return 0;
 536 }
 537 EXPORT_SYMBOL_GPL(cdx_register_controller);
 538
 539 void cdx_unregister_controller(struct cdx_controller *cdx)
 540 {
 541         if (cdx->id >= MAX_CDX_CONTROLLERS)
 542                 return;
 543
 544         device_for_each_child(cdx->dev, NULL, cdx_unregister_device);
 545         xa_erase(&cdx_controllers, cdx->id);
 546 }
 547 EXPORT_SYMBOL_GPL(cdx_unregister_controller);
 548
 549 static int __init cdx_bus_init(void)
 550 {
 551         return bus_register(&cdx_bus_type);
 552 }
 553 postcore_initcall(cdx_bus_init);