drivers/iommu/s390-iommu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * IOMMU API for s390 PCI devices
   4  *
   5  * Copyright IBM Corp. 2015
   6  * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
   7  */
   8
   9 #include <linux/pci.h>
  10 #include <linux/iommu.h>
  11 #include <linux/iommu-helper.h>
  12 #include <linux/sizes.h>
  13 #include <asm/pci_dma.h>
  14
  15 /*
  16  * Physically contiguous memory regions can be mapped with 4 KiB alignment,
  17  * we allow all page sizes that are an order of 4KiB (no special large page
  18  * support so far).
  19  */
  20 #define S390_IOMMU_PGSIZES      (~0xFFFUL)
  21
  22 static const struct iommu_ops s390_iommu_ops;
  23
  24 struct s390_domain {
  25         struct iommu_domain     domain;
  26         struct list_head        devices;
  27         unsigned long           *dma_table;
  28         spinlock_t              dma_table_lock;
  29         spinlock_t              list_lock;
  30 };
  31
  32 struct s390_domain_device {
  33         struct list_head        list;
  34         struct zpci_dev         *zdev;
  35 };
  36
  37 static struct s390_domain *to_s390_domain(struct iommu_domain *dom)
  38 {
  39         return container_of(dom, struct s390_domain, domain);
  40 }
  41
  42 static bool s390_iommu_capable(struct device *dev, enum iommu_cap cap)
  43 {
  44         switch (cap) {
  45         case IOMMU_CAP_CACHE_COHERENCY:
  46                 return true;
  47         case IOMMU_CAP_INTR_REMAP:
  48                 return true;
  49         default:
  50                 return false;
  51         }
  52 }
  53
  54 static struct iommu_domain *s390_domain_alloc(unsigned domain_type)
  55 {
  56         struct s390_domain *s390_domain;
  57
  58         if (domain_type != IOMMU_DOMAIN_UNMANAGED)
  59                 return NULL;
  60
  61         s390_domain = kzalloc(sizeof(*s390_domain), GFP_KERNEL);
  62         if (!s390_domain)
  63                 return NULL;
  64
  65         s390_domain->dma_table = dma_alloc_cpu_table();
  66         if (!s390_domain->dma_table) {
  67                 kfree(s390_domain);
  68                 return NULL;
  69         }
  70
  71         spin_lock_init(&s390_domain->dma_table_lock);
  72         spin_lock_init(&s390_domain->list_lock);
  73         INIT_LIST_HEAD(&s390_domain->devices);
  74
  75         return &s390_domain->domain;
  76 }
  77
  78 static void s390_domain_free(struct iommu_domain *domain)
  79 {
  80         struct s390_domain *s390_domain = to_s390_domain(domain);
  81
  82         WARN_ON(!list_empty(&s390_domain->devices));
  83         dma_cleanup_tables(s390_domain->dma_table);
  84         kfree(s390_domain);
  85 }
  86
  87 static void __s390_iommu_detach_device(struct zpci_dev *zdev)
  88 {
  89         struct s390_domain *s390_domain = zdev->s390_domain;
  90         struct s390_domain_device *domain_device, *tmp;
  91         unsigned long flags;
  92
  93         if (!s390_domain)
  94                 return;
  95
  96         spin_lock_irqsave(&s390_domain->list_lock, flags);
  97         list_for_each_entry_safe(domain_device, tmp, &s390_domain->devices,
  98                                  list) {
  99                 if (domain_device->zdev == zdev) {
 100                         list_del(&domain_device->list);
 101                         kfree(domain_device);
 102                         break;
 103                 }
 104         }
 105         spin_unlock_irqrestore(&s390_domain->list_lock, flags);
 106
 107         zpci_unregister_ioat(zdev, 0);
 108         zdev->s390_domain = NULL;
 109         zdev->dma_table = NULL;
 110 }
 111
 112 static int s390_iommu_attach_device(struct iommu_domain *domain,
 113                                     struct device *dev)
 114 {
 115         struct s390_domain *s390_domain = to_s390_domain(domain);
 116         struct zpci_dev *zdev = to_zpci_dev(dev);
 117         struct s390_domain_device *domain_device;
 118         unsigned long flags;
 119         int cc, rc = 0;
 120
 121         if (!zdev)
 122                 return -ENODEV;
 123
 124         domain_device = kzalloc(sizeof(*domain_device), GFP_KERNEL);
 125         if (!domain_device)
 126                 return -ENOMEM;
 127
 128         if (zdev->s390_domain)
 129                 __s390_iommu_detach_device(zdev);
 130         else if (zdev->dma_table)
 131                 zpci_dma_exit_device(zdev);
 132
 133         cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
 134                                 virt_to_phys(s390_domain->dma_table));
 135         if (cc) {
 136                 rc = -EIO;
 137                 goto out_free;
 138         }
 139         zdev->dma_table = s390_domain->dma_table;
 140
 141         spin_lock_irqsave(&s390_domain->list_lock, flags);
 142         /* First device defines the DMA range limits */
 143         if (list_empty(&s390_domain->devices)) {
 144                 domain->geometry.aperture_start = zdev->start_dma;
 145                 domain->geometry.aperture_end = zdev->end_dma;
 146                 domain->geometry.force_aperture = true;
 147         /* Allow only devices with identical DMA range limits */
 148         } else if (domain->geometry.aperture_start != zdev->start_dma ||
 149                    domain->geometry.aperture_end != zdev->end_dma) {
 150                 spin_unlock_irqrestore(&s390_domain->list_lock, flags);
 151                 rc = -EINVAL;
 152                 goto out_unregister;
 153         }
 154         domain_device->zdev = zdev;
 155         zdev->s390_domain = s390_domain;
 156         list_add(&domain_device->list, &s390_domain->devices);
 157         spin_unlock_irqrestore(&s390_domain->list_lock, flags);
 158
 159         return 0;
 160
 161 out_unregister:
 162         zpci_unregister_ioat(zdev, 0);
 163         zdev->dma_table = NULL;
 164 out_free:
 165         kfree(domain_device);
 166
 167         return rc;
 168 }
 169
 170 static void s390_iommu_detach_device(struct iommu_domain *domain,
 171                                      struct device *dev)
 172 {
 173         struct zpci_dev *zdev = to_zpci_dev(dev);
 174
 175         WARN_ON(zdev->s390_domain != to_s390_domain(domain));
 176
 177         __s390_iommu_detach_device(zdev);
 178         zpci_dma_init_device(zdev);
 179 }
 180
 181 static struct iommu_device *s390_iommu_probe_device(struct device *dev)
 182 {
 183         struct zpci_dev *zdev;
 184
 185         if (!dev_is_pci(dev))
 186                 return ERR_PTR(-ENODEV);
 187
 188         zdev = to_zpci_dev(dev);
 189
 190         return &zdev->iommu_dev;
 191 }
 192
 193 static void s390_iommu_release_device(struct device *dev)
 194 {
 195         struct zpci_dev *zdev = to_zpci_dev(dev);
 196
 197         /*
 198          * release_device is expected to detach any domain currently attached
 199          * to the device, but keep it attached to other devices in the group.
 200          */
 201         if (zdev)
 202                 __s390_iommu_detach_device(zdev);
 203 }
 204
 205 static int s390_iommu_update_trans(struct s390_domain *s390_domain,
 206                                    phys_addr_t pa, dma_addr_t dma_addr,
 207                                    size_t size, int flags)
 208 {
 209         struct s390_domain_device *domain_device;
 210         phys_addr_t page_addr = pa & PAGE_MASK;
 211         dma_addr_t start_dma_addr = dma_addr;
 212         unsigned long irq_flags, nr_pages, i;
 213         unsigned long *entry;
 214         int rc = 0;
 215
 216         if (dma_addr < s390_domain->domain.geometry.aperture_start ||
 217             dma_addr + size > s390_domain->domain.geometry.aperture_end)
 218                 return -EINVAL;
 219
 220         nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
 221         if (!nr_pages)
 222                 return 0;
 223
 224         spin_lock_irqsave(&s390_domain->dma_table_lock, irq_flags);
 225         for (i = 0; i < nr_pages; i++) {
 226                 entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr);
 227                 if (!entry) {
 228                         rc = -ENOMEM;
 229                         goto undo_cpu_trans;
 230                 }
 231                 dma_update_cpu_trans(entry, page_addr, flags);
 232                 page_addr += PAGE_SIZE;
 233                 dma_addr += PAGE_SIZE;
 234         }
 235
 236         spin_lock(&s390_domain->list_lock);
 237         list_for_each_entry(domain_device, &s390_domain->devices, list) {
 238                 rc = zpci_refresh_trans((u64) domain_device->zdev->fh << 32,
 239                                         start_dma_addr, nr_pages * PAGE_SIZE);
 240                 if (rc)
 241                         break;
 242         }
 243         spin_unlock(&s390_domain->list_lock);
 244
 245 undo_cpu_trans:
 246         if (rc && ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID)) {
 247                 flags = ZPCI_PTE_INVALID;
 248                 while (i-- > 0) {
 249                         page_addr -= PAGE_SIZE;
 250                         dma_addr -= PAGE_SIZE;
 251                         entry = dma_walk_cpu_trans(s390_domain->dma_table,
 252                                                    dma_addr);
 253                         if (!entry)
 254                                 break;
 255                         dma_update_cpu_trans(entry, page_addr, flags);
 256                 }
 257         }
 258         spin_unlock_irqrestore(&s390_domain->dma_table_lock, irq_flags);
 259
 260         return rc;
 261 }
 262
 263 static int s390_iommu_map(struct iommu_domain *domain, unsigned long iova,
 264                           phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
 265 {
 266         struct s390_domain *s390_domain = to_s390_domain(domain);
 267         int flags = ZPCI_PTE_VALID, rc = 0;
 268
 269         if (!(prot & IOMMU_READ))
 270                 return -EINVAL;
 271
 272         if (!(prot & IOMMU_WRITE))
 273                 flags |= ZPCI_TABLE_PROTECTED;
 274
 275         rc = s390_iommu_update_trans(s390_domain, paddr, iova,
 276                                      size, flags);
 277
 278         return rc;
 279 }
 280
 281 static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain,
 282                                            dma_addr_t iova)
 283 {
 284         struct s390_domain *s390_domain = to_s390_domain(domain);
 285         unsigned long *sto, *pto, *rto, flags;
 286         unsigned int rtx, sx, px;
 287         phys_addr_t phys = 0;
 288
 289         if (iova < domain->geometry.aperture_start ||
 290             iova > domain->geometry.aperture_end)
 291                 return 0;
 292
 293         rtx = calc_rtx(iova);
 294         sx = calc_sx(iova);
 295         px = calc_px(iova);
 296         rto = s390_domain->dma_table;
 297
 298         spin_lock_irqsave(&s390_domain->dma_table_lock, flags);
 299         if (rto && reg_entry_isvalid(rto[rtx])) {
 300                 sto = get_rt_sto(rto[rtx]);
 301                 if (sto && reg_entry_isvalid(sto[sx])) {
 302                         pto = get_st_pto(sto[sx]);
 303                         if (pto && pt_entry_isvalid(pto[px]))
 304                                 phys = pto[px] & ZPCI_PTE_ADDR_MASK;
 305                 }
 306         }
 307         spin_unlock_irqrestore(&s390_domain->dma_table_lock, flags);
 308
 309         return phys;
 310 }
 311
 312 static size_t s390_iommu_unmap(struct iommu_domain *domain,
 313                                unsigned long iova, size_t size,
 314                                struct iommu_iotlb_gather *gather)
 315 {
 316         struct s390_domain *s390_domain = to_s390_domain(domain);
 317         int flags = ZPCI_PTE_INVALID;
 318         phys_addr_t paddr;
 319         int rc;
 320
 321         paddr = s390_iommu_iova_to_phys(domain, iova);
 322         if (!paddr)
 323                 return 0;
 324
 325         rc = s390_iommu_update_trans(s390_domain, paddr, iova,
 326                                      size, flags);
 327         if (rc)
 328                 return 0;
 329
 330         return size;
 331 }
 332
 333 int zpci_init_iommu(struct zpci_dev *zdev)
 334 {
 335         int rc = 0;
 336
 337         rc = iommu_device_sysfs_add(&zdev->iommu_dev, NULL, NULL,
 338                                     "s390-iommu.%08x", zdev->fid);
 339         if (rc)
 340                 goto out_err;
 341
 342         rc = iommu_device_register(&zdev->iommu_dev, &s390_iommu_ops, NULL);
 343         if (rc)
 344                 goto out_sysfs;
 345
 346         return 0;
 347
 348 out_sysfs:
 349         iommu_device_sysfs_remove(&zdev->iommu_dev);
 350
 351 out_err:
 352         return rc;
 353 }
 354
 355 void zpci_destroy_iommu(struct zpci_dev *zdev)
 356 {
 357         iommu_device_unregister(&zdev->iommu_dev);
 358         iommu_device_sysfs_remove(&zdev->iommu_dev);
 359 }
 360
 361 static const struct iommu_ops s390_iommu_ops = {
 362         .capable = s390_iommu_capable,
 363         .domain_alloc = s390_domain_alloc,
 364         .probe_device = s390_iommu_probe_device,
 365         .release_device = s390_iommu_release_device,
 366         .device_group = generic_device_group,
 367         .pgsize_bitmap = S390_IOMMU_PGSIZES,
 368         .default_domain_ops = &(const struct iommu_domain_ops) {
 369                 .attach_dev     = s390_iommu_attach_device,
 370                 .detach_dev     = s390_iommu_detach_device,
 371                 .map            = s390_iommu_map,
 372                 .unmap          = s390_iommu_unmap,
 373                 .iova_to_phys   = s390_iommu_iova_to_phys,
 374                 .free           = s390_domain_free,
 375         }
 376 };