From 38972375ef7bdc7dd989bcb48d5448662a95bca2 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Fri, 29 Jun 2018 15:08:52 -0500 Subject: [PATCH] PCI/IOV: Reset total_VFs limit after detaching PF driver The TotalVFs register in the SR-IOV capability is the hardware limit on the number of VFs. A PF driver can limit the number of VFs further with pci_sriov_set_totalvfs(). When the PF driver is removed, reset any VF limit that was imposed by the driver because that limit may not apply to other drivers. Before 8d85a7a4f2c9 ("PCI/IOV: Allow PF drivers to limit total_VFs to 0"), pci_sriov_set_totalvfs(pdev, 0) meant "we can enable TotalVFs virtual functions", and the nfp driver used that to remove the VF limit when the driver unloads. 8d85a7a4f2c9 broke that because instead of removing the VF limit, pci_sriov_set_totalvfs(pdev, 0) actually sets the limit to zero, and that limit persists even if another driver is loaded. We could fix that by making the nfp driver reset the limit when it unloads, but it seems more robust to do it in the PCI core instead of relying on the driver. The regression scenario is: nfp_pci_probe (driver 1) ... nfp_pci_remove pci_sriov_set_totalvfs(pf->pdev, 0) # limits VFs to 0 ... nfp_pci_probe (driver 2) nfp_rtsym_read_le("nfd_vf_cfg_max_vfs") # no VF limit from firmware Now driver 2 is broken because the VF limit is still 0 from driver 1. Fixes: 8d85a7a4f2c9 ("PCI/IOV: Allow PF drivers to limit total_VFs to 0") Signed-off-by: Jakub Kicinski [bhelgaas: changelog, rename functions] Signed-off-by: Bjorn Helgaas --- drivers/pci/iov.c | 16 ++++++++++++++++ drivers/pci/pci-driver.c | 1 + drivers/pci/pci.h | 4 ++++ 3 files changed, 21 insertions(+) diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index d0d73db..0f04ae6 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -575,6 +575,22 @@ void pci_iov_release(struct pci_dev *dev) } /** + * pci_iov_remove - clean up SR-IOV state after PF driver is detached + * @dev: the PCI device + */ +void pci_iov_remove(struct pci_dev *dev) +{ + struct pci_sriov *iov = dev->sriov; + + if (!dev->is_physfn) + return; + + iov->driver_max_VFs = iov->total_VFs; + if (iov->num_VFs) + pci_warn(dev, "driver left SR-IOV enabled after remove\n"); +} + +/** * pci_iov_update_resource - update a VF BAR * @dev: the PCI device * @resno: the resource number diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index c125d53..6792292 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -445,6 +445,7 @@ static int pci_device_remove(struct device *dev) } pcibios_free_irq(pci_dev); pci_dev->driver = NULL; + pci_iov_remove(pci_dev); } /* Undo the runtime PM settings in local_pci_probe() */ diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index c358e7a0..882f1f9 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -311,6 +311,7 @@ static inline void pci_restore_ats_state(struct pci_dev *dev) #ifdef CONFIG_PCI_IOV int pci_iov_init(struct pci_dev *dev); void pci_iov_release(struct pci_dev *dev); +void pci_iov_remove(struct pci_dev *dev); void pci_iov_update_resource(struct pci_dev *dev, int resno); resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno); void pci_restore_iov_state(struct pci_dev *dev); @@ -325,6 +326,9 @@ static inline void pci_iov_release(struct pci_dev *dev) { } +static inline void pci_iov_remove(struct pci_dev *dev) +{ +} static inline void pci_restore_iov_state(struct pci_dev *dev) { } -- 2.7.4