drivers/dma/idxd/init.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #include <linux/init.h>
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/slab.h>
   7 #include <linux/pci.h>
   8 #include <linux/interrupt.h>
   9 #include <linux/delay.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/workqueue.h>
  12 #include <linux/aer.h>
  13 #include <linux/fs.h>
  14 #include <linux/io-64-nonatomic-lo-hi.h>
  15 #include <linux/device.h>
  16 #include <linux/idr.h>
  17 #include <linux/intel-svm.h>
  18 #include <linux/iommu.h>
  19 #include <uapi/linux/idxd.h>
  20 #include <linux/dmaengine.h>
  21 #include "../dmaengine.h"
  22 #include "registers.h"
  23 #include "idxd.h"
  24 #include "perfmon.h"
  25
  26 MODULE_VERSION(IDXD_DRIVER_VERSION);
  27 MODULE_LICENSE("GPL v2");
  28 MODULE_AUTHOR("Intel Corporation");
  29
  30 static bool sva = true;
  31 module_param(sva, bool, 0644);
  32 MODULE_PARM_DESC(sva, "Toggle SVA support on/off");
  33
  34 #define DRV_NAME "idxd"
  35
  36 bool support_enqcmd;
  37 DEFINE_IDA(idxd_ida);
  38
  39 static struct idxd_driver_data idxd_driver_data[] = {
  40         [IDXD_TYPE_DSA] = {
  41                 .name_prefix = "dsa",
  42                 .type = IDXD_TYPE_DSA,
  43                 .compl_size = sizeof(struct dsa_completion_record),
  44                 .align = 32,
  45                 .dev_type = &dsa_device_type,
  46         },
  47         [IDXD_TYPE_IAX] = {
  48                 .name_prefix = "iax",
  49                 .type = IDXD_TYPE_IAX,
  50                 .compl_size = sizeof(struct iax_completion_record),
  51                 .align = 64,
  52                 .dev_type = &iax_device_type,
  53         },
  54 };
  55
  56 static struct pci_device_id idxd_pci_tbl[] = {
  57         /* DSA ver 1.0 platforms */
  58         { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) },
  59
  60         /* IAX ver 1.0 platforms */
  61         { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) },
  62         { 0, }
  63 };
  64 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl);
  65
  66 static int idxd_setup_interrupts(struct idxd_device *idxd)
  67 {
  68         struct pci_dev *pdev = idxd->pdev;
  69         struct device *dev = &pdev->dev;
  70         struct idxd_irq_entry *irq_entry;
  71         int i, msixcnt;
  72         int rc = 0;
  73
  74         msixcnt = pci_msix_vec_count(pdev);
  75         if (msixcnt < 0) {
  76                 dev_err(dev, "Not MSI-X interrupt capable.\n");
  77                 return -ENOSPC;
  78         }
  79
  80         rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX);
  81         if (rc != msixcnt) {
  82                 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc);
  83                 return -ENOSPC;
  84         }
  85         dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt);
  86
  87         /*
  88          * We implement 1 completion list per MSI-X entry except for
  89          * entry 0, which is for errors and others.
  90          */
  91         idxd->irq_entries = kcalloc_node(msixcnt, sizeof(struct idxd_irq_entry),
  92                                          GFP_KERNEL, dev_to_node(dev));
  93         if (!idxd->irq_entries) {
  94                 rc = -ENOMEM;
  95                 goto err_irq_entries;
  96         }
  97
  98         for (i = 0; i < msixcnt; i++) {
  99                 idxd->irq_entries[i].id = i;
 100                 idxd->irq_entries[i].idxd = idxd;
 101                 idxd->irq_entries[i].vector = pci_irq_vector(pdev, i);
 102                 spin_lock_init(&idxd->irq_entries[i].list_lock);
 103         }
 104
 105         irq_entry = &idxd->irq_entries[0];
 106         rc = request_threaded_irq(irq_entry->vector, NULL, idxd_misc_thread,
 107                                   0, "idxd-misc", irq_entry);
 108         if (rc < 0) {
 109                 dev_err(dev, "Failed to allocate misc interrupt.\n");
 110                 goto err_misc_irq;
 111         }
 112
 113         dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", irq_entry->vector);
 114
 115         /* first MSI-X entry is not for wq interrupts */
 116         idxd->num_wq_irqs = msixcnt - 1;
 117
 118         for (i = 1; i < msixcnt; i++) {
 119                 irq_entry = &idxd->irq_entries[i];
 120
 121                 init_llist_head(&idxd->irq_entries[i].pending_llist);
 122                 INIT_LIST_HEAD(&idxd->irq_entries[i].work_list);
 123                 rc = request_threaded_irq(irq_entry->vector, NULL,
 124                                           idxd_wq_thread, 0, "idxd-portal", irq_entry);
 125                 if (rc < 0) {
 126                         dev_err(dev, "Failed to allocate irq %d.\n", irq_entry->vector);
 127                         goto err_wq_irqs;
 128                 }
 129
 130                 dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", i, irq_entry->vector);
 131                 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) {
 132                         /*
 133                          * The MSIX vector enumeration starts at 1 with vector 0 being the
 134                          * misc interrupt that handles non I/O completion events. The
 135                          * interrupt handles are for IMS enumeration on guest. The misc
 136                          * interrupt vector does not require a handle and therefore we start
 137                          * the int_handles at index 0. Since 'i' starts at 1, the first
 138                          * int_handles index will be 0.
 139                          */
 140                         rc = idxd_device_request_int_handle(idxd, i, &idxd->int_handles[i - 1],
 141                                                             IDXD_IRQ_MSIX);
 142                         if (rc < 0) {
 143                                 free_irq(irq_entry->vector, irq_entry);
 144                                 goto err_wq_irqs;
 145                         }
 146                         dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i - 1]);
 147                 }
 148         }
 149
 150         idxd_unmask_error_interrupts(idxd);
 151         idxd_msix_perm_setup(idxd);
 152         return 0;
 153
 154  err_wq_irqs:
 155         while (--i >= 0) {
 156                 irq_entry = &idxd->irq_entries[i];
 157                 free_irq(irq_entry->vector, irq_entry);
 158                 if (i != 0)
 159                         idxd_device_release_int_handle(idxd,
 160                                                        idxd->int_handles[i], IDXD_IRQ_MSIX);
 161         }
 162  err_misc_irq:
 163         /* Disable error interrupt generation */
 164         idxd_mask_error_interrupts(idxd);
 165  err_irq_entries:
 166         pci_free_irq_vectors(pdev);
 167         dev_err(dev, "No usable interrupts\n");
 168         return rc;
 169 }
 170
 171 static void idxd_cleanup_interrupts(struct idxd_device *idxd)
 172 {
 173         struct pci_dev *pdev = idxd->pdev;
 174         struct idxd_irq_entry *irq_entry;
 175         int i, msixcnt;
 176
 177         msixcnt = pci_msix_vec_count(pdev);
 178         if (msixcnt <= 0)
 179                 return;
 180
 181         irq_entry = &idxd->irq_entries[0];
 182         free_irq(irq_entry->vector, irq_entry);
 183
 184         for (i = 1; i < msixcnt; i++) {
 185
 186                 irq_entry = &idxd->irq_entries[i];
 187                 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE))
 188                         idxd_device_release_int_handle(idxd, idxd->int_handles[i],
 189                                                        IDXD_IRQ_MSIX);
 190                 free_irq(irq_entry->vector, irq_entry);
 191         }
 192
 193         idxd_mask_error_interrupts(idxd);
 194         pci_free_irq_vectors(pdev);
 195 }
 196
 197 static int idxd_setup_wqs(struct idxd_device *idxd)
 198 {
 199         struct device *dev = &idxd->pdev->dev;
 200         struct idxd_wq *wq;
 201         int i, rc;
 202
 203         idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *),
 204                                  GFP_KERNEL, dev_to_node(dev));
 205         if (!idxd->wqs)
 206                 return -ENOMEM;
 207
 208         for (i = 0; i < idxd->max_wqs; i++) {
 209                 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev));
 210                 if (!wq) {
 211                         rc = -ENOMEM;
 212                         goto err;
 213                 }
 214
 215                 wq->id = i;
 216                 wq->idxd = idxd;
 217                 device_initialize(&wq->conf_dev);
 218                 wq->conf_dev.parent = &idxd->conf_dev;
 219                 wq->conf_dev.bus = &dsa_bus_type;
 220                 wq->conf_dev.type = &idxd_wq_device_type;
 221                 rc = dev_set_name(&wq->conf_dev, "wq%d.%d", idxd->id, wq->id);
 222                 if (rc < 0) {
 223                         put_device(&wq->conf_dev);
 224                         goto err;
 225                 }
 226
 227                 mutex_init(&wq->wq_lock);
 228                 init_waitqueue_head(&wq->err_queue);
 229                 init_completion(&wq->wq_dead);
 230                 wq->max_xfer_bytes = idxd->max_xfer_bytes;
 231                 wq->max_batch_size = idxd->max_batch_size;
 232                 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev));
 233                 if (!wq->wqcfg) {
 234                         put_device(&wq->conf_dev);
 235                         rc = -ENOMEM;
 236                         goto err;
 237                 }
 238                 idxd->wqs[i] = wq;
 239         }
 240
 241         return 0;
 242
 243  err:
 244         while (--i >= 0)
 245                 put_device(&idxd->wqs[i]->conf_dev);
 246         return rc;
 247 }
 248
 249 static int idxd_setup_engines(struct idxd_device *idxd)
 250 {
 251         struct idxd_engine *engine;
 252         struct device *dev = &idxd->pdev->dev;
 253         int i, rc;
 254
 255         idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *),
 256                                      GFP_KERNEL, dev_to_node(dev));
 257         if (!idxd->engines)
 258                 return -ENOMEM;
 259
 260         for (i = 0; i < idxd->max_engines; i++) {
 261                 engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev));
 262                 if (!engine) {
 263                         rc = -ENOMEM;
 264                         goto err;
 265                 }
 266
 267                 engine->id = i;
 268                 engine->idxd = idxd;
 269                 device_initialize(&engine->conf_dev);
 270                 engine->conf_dev.parent = &idxd->conf_dev;
 271                 engine->conf_dev.bus = &dsa_bus_type;
 272                 engine->conf_dev.type = &idxd_engine_device_type;
 273                 rc = dev_set_name(&engine->conf_dev, "engine%d.%d", idxd->id, engine->id);
 274                 if (rc < 0) {
 275                         put_device(&engine->conf_dev);
 276                         goto err;
 277                 }
 278
 279                 idxd->engines[i] = engine;
 280         }
 281
 282         return 0;
 283
 284  err:
 285         while (--i >= 0)
 286                 put_device(&idxd->engines[i]->conf_dev);
 287         return rc;
 288 }
 289
 290 static int idxd_setup_groups(struct idxd_device *idxd)
 291 {
 292         struct device *dev = &idxd->pdev->dev;
 293         struct idxd_group *group;
 294         int i, rc;
 295
 296         idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *),
 297                                     GFP_KERNEL, dev_to_node(dev));
 298         if (!idxd->groups)
 299                 return -ENOMEM;
 300
 301         for (i = 0; i < idxd->max_groups; i++) {
 302                 group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev));
 303                 if (!group) {
 304                         rc = -ENOMEM;
 305                         goto err;
 306                 }
 307
 308                 group->id = i;
 309                 group->idxd = idxd;
 310                 device_initialize(&group->conf_dev);
 311                 group->conf_dev.parent = &idxd->conf_dev;
 312                 group->conf_dev.bus = &dsa_bus_type;
 313                 group->conf_dev.type = &idxd_group_device_type;
 314                 rc = dev_set_name(&group->conf_dev, "group%d.%d", idxd->id, group->id);
 315                 if (rc < 0) {
 316                         put_device(&group->conf_dev);
 317                         goto err;
 318                 }
 319
 320                 idxd->groups[i] = group;
 321                 group->tc_a = -1;
 322                 group->tc_b = -1;
 323         }
 324
 325         return 0;
 326
 327  err:
 328         while (--i >= 0)
 329                 put_device(&idxd->groups[i]->conf_dev);
 330         return rc;
 331 }
 332
 333 static void idxd_cleanup_internals(struct idxd_device *idxd)
 334 {
 335         int i;
 336
 337         for (i = 0; i < idxd->max_groups; i++)
 338                 put_device(&idxd->groups[i]->conf_dev);
 339         for (i = 0; i < idxd->max_engines; i++)
 340                 put_device(&idxd->engines[i]->conf_dev);
 341         for (i = 0; i < idxd->max_wqs; i++)
 342                 put_device(&idxd->wqs[i]->conf_dev);
 343         destroy_workqueue(idxd->wq);
 344 }
 345
 346 static int idxd_setup_internals(struct idxd_device *idxd)
 347 {
 348         struct device *dev = &idxd->pdev->dev;
 349         int rc, i;
 350
 351         init_waitqueue_head(&idxd->cmd_waitq);
 352
 353         if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) {
 354                 idxd->int_handles = devm_kcalloc(dev, idxd->max_wqs, sizeof(int), GFP_KERNEL);
 355                 if (!idxd->int_handles)
 356                         return -ENOMEM;
 357         }
 358
 359         rc = idxd_setup_wqs(idxd);
 360         if (rc < 0)
 361                 goto err_wqs;
 362
 363         rc = idxd_setup_engines(idxd);
 364         if (rc < 0)
 365                 goto err_engine;
 366
 367         rc = idxd_setup_groups(idxd);
 368         if (rc < 0)
 369                 goto err_group;
 370
 371         idxd->wq = create_workqueue(dev_name(dev));
 372         if (!idxd->wq) {
 373                 rc = -ENOMEM;
 374                 goto err_wkq_create;
 375         }
 376
 377         return 0;
 378
 379  err_wkq_create:
 380         for (i = 0; i < idxd->max_groups; i++)
 381                 put_device(&idxd->groups[i]->conf_dev);
 382  err_group:
 383         for (i = 0; i < idxd->max_engines; i++)
 384                 put_device(&idxd->engines[i]->conf_dev);
 385  err_engine:
 386         for (i = 0; i < idxd->max_wqs; i++)
 387                 put_device(&idxd->wqs[i]->conf_dev);
 388  err_wqs:
 389         kfree(idxd->int_handles);
 390         return rc;
 391 }
 392
 393 static void idxd_read_table_offsets(struct idxd_device *idxd)
 394 {
 395         union offsets_reg offsets;
 396         struct device *dev = &idxd->pdev->dev;
 397
 398         offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET);
 399         offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64));
 400         idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT;
 401         dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset);
 402         idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT;
 403         dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset);
 404         idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT;
 405         dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset);
 406         idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT;
 407         dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset);
 408 }
 409
 410 static void idxd_read_caps(struct idxd_device *idxd)
 411 {
 412         struct device *dev = &idxd->pdev->dev;
 413         int i;
 414
 415         /* reading generic capabilities */
 416         idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET);
 417         dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits);
 418
 419         if (idxd->hw.gen_cap.cmd_cap) {
 420                 idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET);
 421                 dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap);
 422         }
 423
 424         idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift;
 425         dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes);
 426         idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift;
 427         dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size);
 428         if (idxd->hw.gen_cap.config_en)
 429                 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags);
 430
 431         /* reading group capabilities */
 432         idxd->hw.group_cap.bits =
 433                 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET);
 434         dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits);
 435         idxd->max_groups = idxd->hw.group_cap.num_groups;
 436         dev_dbg(dev, "max groups: %u\n", idxd->max_groups);
 437         idxd->max_tokens = idxd->hw.group_cap.total_tokens;
 438         dev_dbg(dev, "max tokens: %u\n", idxd->max_tokens);
 439         idxd->nr_tokens = idxd->max_tokens;
 440
 441         /* read engine capabilities */
 442         idxd->hw.engine_cap.bits =
 443                 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET);
 444         dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits);
 445         idxd->max_engines = idxd->hw.engine_cap.num_engines;
 446         dev_dbg(dev, "max engines: %u\n", idxd->max_engines);
 447
 448         /* read workqueue capabilities */
 449         idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET);
 450         dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits);
 451         idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size;
 452         dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size);
 453         idxd->max_wqs = idxd->hw.wq_cap.num_wqs;
 454         dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs);
 455         idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN);
 456         dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size);
 457
 458         /* reading operation capabilities */
 459         for (i = 0; i < 4; i++) {
 460                 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base +
 461                                 IDXD_OPCAP_OFFSET + i * sizeof(u64));
 462                 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]);
 463         }
 464 }
 465
 466 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data)
 467 {
 468         struct device *dev = &pdev->dev;
 469         struct idxd_device *idxd;
 470         int rc;
 471
 472         idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev));
 473         if (!idxd)
 474                 return NULL;
 475
 476         idxd->pdev = pdev;
 477         idxd->data = data;
 478         idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL);
 479         if (idxd->id < 0)
 480                 return NULL;
 481
 482         device_initialize(&idxd->conf_dev);
 483         idxd->conf_dev.parent = dev;
 484         idxd->conf_dev.bus = &dsa_bus_type;
 485         idxd->conf_dev.type = idxd->data->dev_type;
 486         rc = dev_set_name(&idxd->conf_dev, "%s%d", idxd->data->name_prefix, idxd->id);
 487         if (rc < 0) {
 488                 put_device(&idxd->conf_dev);
 489                 return NULL;
 490         }
 491
 492         spin_lock_init(&idxd->dev_lock);
 493         spin_lock_init(&idxd->cmd_lock);
 494
 495         return idxd;
 496 }
 497
 498 static int idxd_enable_system_pasid(struct idxd_device *idxd)
 499 {
 500         int flags;
 501         unsigned int pasid;
 502         struct iommu_sva *sva;
 503
 504         flags = SVM_FLAG_SUPERVISOR_MODE;
 505
 506         sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags);
 507         if (IS_ERR(sva)) {
 508                 dev_warn(&idxd->pdev->dev,
 509                          "iommu sva bind failed: %ld\n", PTR_ERR(sva));
 510                 return PTR_ERR(sva);
 511         }
 512
 513         pasid = iommu_sva_get_pasid(sva);
 514         if (pasid == IOMMU_PASID_INVALID) {
 515                 iommu_sva_unbind_device(sva);
 516                 return -ENODEV;
 517         }
 518
 519         idxd->sva = sva;
 520         idxd->pasid = pasid;
 521         dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid);
 522         return 0;
 523 }
 524
 525 static void idxd_disable_system_pasid(struct idxd_device *idxd)
 526 {
 527
 528         iommu_sva_unbind_device(idxd->sva);
 529         idxd->sva = NULL;
 530 }
 531
 532 static int idxd_probe(struct idxd_device *idxd)
 533 {
 534         struct pci_dev *pdev = idxd->pdev;
 535         struct device *dev = &pdev->dev;
 536         int rc;
 537
 538         dev_dbg(dev, "%s entered and resetting device\n", __func__);
 539         rc = idxd_device_init_reset(idxd);
 540         if (rc < 0)
 541                 return rc;
 542
 543         dev_dbg(dev, "IDXD reset complete\n");
 544
 545         if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) {
 546                 rc = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA);
 547                 if (rc == 0) {
 548                         rc = idxd_enable_system_pasid(idxd);
 549                         if (rc < 0) {
 550                                 iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
 551                                 dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc);
 552                         } else {
 553                                 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
 554                         }
 555                 } else {
 556                         dev_warn(dev, "Unable to turn on SVA feature.\n");
 557                 }
 558         } else if (!sva) {
 559                 dev_warn(dev, "User forced SVA off via module param.\n");
 560         }
 561
 562         idxd_read_caps(idxd);
 563         idxd_read_table_offsets(idxd);
 564
 565         rc = idxd_setup_internals(idxd);
 566         if (rc)
 567                 goto err;
 568
 569         /* If the configs are readonly, then load them from device */
 570         if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) {
 571                 dev_dbg(dev, "Loading RO device config\n");
 572                 rc = idxd_device_load_config(idxd);
 573                 if (rc < 0)
 574                         goto err_config;
 575         }
 576
 577         rc = idxd_setup_interrupts(idxd);
 578         if (rc)
 579                 goto err_config;
 580
 581         dev_dbg(dev, "IDXD interrupt setup complete.\n");
 582
 583         idxd->major = idxd_cdev_get_major(idxd);
 584
 585         rc = perfmon_pmu_init(idxd);
 586         if (rc < 0)
 587                 dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc);
 588
 589         dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id);
 590         return 0;
 591
 592  err_config:
 593         idxd_cleanup_internals(idxd);
 594  err:
 595         if (device_pasid_enabled(idxd))
 596                 idxd_disable_system_pasid(idxd);
 597         iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
 598         return rc;
 599 }
 600
 601 static void idxd_cleanup(struct idxd_device *idxd)
 602 {
 603         struct device *dev = &idxd->pdev->dev;
 604
 605         perfmon_pmu_remove(idxd);
 606         idxd_cleanup_interrupts(idxd);
 607         idxd_cleanup_internals(idxd);
 608         if (device_pasid_enabled(idxd))
 609                 idxd_disable_system_pasid(idxd);
 610         iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
 611 }
 612
 613 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 614 {
 615         struct device *dev = &pdev->dev;
 616         struct idxd_device *idxd;
 617         struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data;
 618         int rc;
 619
 620         rc = pci_enable_device(pdev);
 621         if (rc)
 622                 return rc;
 623
 624         dev_dbg(dev, "Alloc IDXD context\n");
 625         idxd = idxd_alloc(pdev, data);
 626         if (!idxd) {
 627                 rc = -ENOMEM;
 628                 goto err_idxd_alloc;
 629         }
 630
 631         dev_dbg(dev, "Mapping BARs\n");
 632         idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0);
 633         if (!idxd->reg_base) {
 634                 rc = -ENOMEM;
 635                 goto err_iomap;
 636         }
 637
 638         dev_dbg(dev, "Set DMA masks\n");
 639         rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
 640         if (rc)
 641                 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
 642         if (rc)
 643                 goto err;
 644
 645         rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
 646         if (rc)
 647                 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
 648         if (rc)
 649                 goto err;
 650
 651         dev_dbg(dev, "Set PCI master\n");
 652         pci_set_master(pdev);
 653         pci_set_drvdata(pdev, idxd);
 654
 655         idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET);
 656         rc = idxd_probe(idxd);
 657         if (rc) {
 658                 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n");
 659                 goto err;
 660         }
 661
 662         rc = idxd_register_devices(idxd);
 663         if (rc) {
 664                 dev_err(dev, "IDXD sysfs setup failed\n");
 665                 goto err_dev_register;
 666         }
 667
 668         idxd->state = IDXD_DEV_CONF_READY;
 669
 670         dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n",
 671                  idxd->hw.version);
 672
 673         return 0;
 674
 675  err_dev_register:
 676         idxd_cleanup(idxd);
 677  err:
 678         pci_iounmap(pdev, idxd->reg_base);
 679  err_iomap:
 680         put_device(&idxd->conf_dev);
 681  err_idxd_alloc:
 682         pci_disable_device(pdev);
 683         return rc;
 684 }
 685
 686 static void idxd_flush_pending_llist(struct idxd_irq_entry *ie)
 687 {
 688         struct idxd_desc *desc, *itr;
 689         struct llist_node *head;
 690
 691         head = llist_del_all(&ie->pending_llist);
 692         if (!head)
 693                 return;
 694
 695         llist_for_each_entry_safe(desc, itr, head, llnode) {
 696                 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT);
 697                 idxd_free_desc(desc->wq, desc);
 698         }
 699 }
 700
 701 static void idxd_flush_work_list(struct idxd_irq_entry *ie)
 702 {
 703         struct idxd_desc *desc, *iter;
 704
 705         list_for_each_entry_safe(desc, iter, &ie->work_list, list) {
 706                 list_del(&desc->list);
 707                 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT);
 708                 idxd_free_desc(desc->wq, desc);
 709         }
 710 }
 711
 712 void idxd_wqs_quiesce(struct idxd_device *idxd)
 713 {
 714         struct idxd_wq *wq;
 715         int i;
 716
 717         for (i = 0; i < idxd->max_wqs; i++) {
 718                 wq = idxd->wqs[i];
 719                 if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL)
 720                         idxd_wq_quiesce(wq);
 721         }
 722 }
 723
 724 static void idxd_release_int_handles(struct idxd_device *idxd)
 725 {
 726         struct device *dev = &idxd->pdev->dev;
 727         int i, rc;
 728
 729         for (i = 0; i < idxd->num_wq_irqs; i++) {
 730                 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)) {
 731                         rc = idxd_device_release_int_handle(idxd, idxd->int_handles[i],
 732                                                             IDXD_IRQ_MSIX);
 733                         if (rc < 0)
 734                                 dev_warn(dev, "irq handle %d release failed\n",
 735                                          idxd->int_handles[i]);
 736                         else
 737                                 dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i]);
 738                 }
 739         }
 740 }
 741
 742 static void idxd_shutdown(struct pci_dev *pdev)
 743 {
 744         struct idxd_device *idxd = pci_get_drvdata(pdev);
 745         int rc, i;
 746         struct idxd_irq_entry *irq_entry;
 747         int msixcnt = pci_msix_vec_count(pdev);
 748
 749         rc = idxd_device_disable(idxd);
 750         if (rc)
 751                 dev_err(&pdev->dev, "Disabling device failed\n");
 752
 753         dev_dbg(&pdev->dev, "%s called\n", __func__);
 754         idxd_mask_msix_vectors(idxd);
 755         idxd_mask_error_interrupts(idxd);
 756
 757         for (i = 0; i < msixcnt; i++) {
 758                 irq_entry = &idxd->irq_entries[i];
 759                 synchronize_irq(irq_entry->vector);
 760                 free_irq(irq_entry->vector, irq_entry);
 761                 if (i == 0)
 762                         continue;
 763                 idxd_flush_pending_llist(irq_entry);
 764                 idxd_flush_work_list(irq_entry);
 765         }
 766
 767         idxd_msix_perm_clear(idxd);
 768         idxd_release_int_handles(idxd);
 769         pci_free_irq_vectors(pdev);
 770         pci_iounmap(pdev, idxd->reg_base);
 771         pci_disable_device(pdev);
 772         destroy_workqueue(idxd->wq);
 773 }
 774
 775 static void idxd_remove(struct pci_dev *pdev)
 776 {
 777         struct idxd_device *idxd = pci_get_drvdata(pdev);
 778
 779         dev_dbg(&pdev->dev, "%s called\n", __func__);
 780         idxd_shutdown(pdev);
 781         if (device_pasid_enabled(idxd))
 782                 idxd_disable_system_pasid(idxd);
 783         idxd_unregister_devices(idxd);
 784         perfmon_pmu_remove(idxd);
 785         iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
 786 }
 787
 788 static struct pci_driver idxd_pci_driver = {
 789         .name           = DRV_NAME,
 790         .id_table       = idxd_pci_tbl,
 791         .probe          = idxd_pci_probe,
 792         .remove         = idxd_remove,
 793         .shutdown       = idxd_shutdown,
 794 };
 795
 796 static int __init idxd_init_module(void)
 797 {
 798         int err;
 799
 800         /*
 801          * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in
 802          * enumerating the device. We can not utilize it.
 803          */
 804         if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) {
 805                 pr_warn("idxd driver failed to load without MOVDIR64B.\n");
 806                 return -ENODEV;
 807         }
 808
 809         if (!cpu_feature_enabled(X86_FEATURE_ENQCMD))
 810                 pr_warn("Platform does not have ENQCMD(S) support.\n");
 811         else
 812                 support_enqcmd = true;
 813
 814         perfmon_init();
 815
 816         err = idxd_register_bus_type();
 817         if (err < 0)
 818                 return err;
 819
 820         err = idxd_register_driver();
 821         if (err < 0)
 822                 goto err_idxd_driver_register;
 823
 824         err = idxd_cdev_register();
 825         if (err)
 826                 goto err_cdev_register;
 827
 828         err = pci_register_driver(&idxd_pci_driver);
 829         if (err)
 830                 goto err_pci_register;
 831
 832         return 0;
 833
 834 err_pci_register:
 835         idxd_cdev_remove();
 836 err_cdev_register:
 837         idxd_unregister_driver();
 838 err_idxd_driver_register:
 839         idxd_unregister_bus_type();
 840         return err;
 841 }
 842 module_init(idxd_init_module);
 843
 844 static void __exit idxd_exit_module(void)
 845 {
 846         idxd_unregister_driver();
 847         pci_unregister_driver(&idxd_pci_driver);
 848         idxd_cdev_remove();
 849         idxd_unregister_bus_type();
 850         perfmon_exit();
 851 }
 852 module_exit(idxd_exit_module);