drivers/vfio/vfio_main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * VFIO core
   4  *
   5  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
   6  *     Author: Alex Williamson <alex.williamson@redhat.com>
   7  *
   8  * Derived from original vfio:
   9  * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
  10  * Author: Tom Lyon, pugs@cisco.com
  11  */
  12
  13 #include <linux/cdev.h>
  14 #include <linux/compat.h>
  15 #include <linux/device.h>
  16 #include <linux/fs.h>
  17 #include <linux/idr.h>
  18 #include <linux/iommu.h>
  19 #ifdef CONFIG_HAVE_KVM
  20 #include <linux/kvm_host.h>
  21 #endif
  22 #include <linux/list.h>
  23 #include <linux/miscdevice.h>
  24 #include <linux/module.h>
  25 #include <linux/mutex.h>
  26 #include <linux/pci.h>
  27 #include <linux/rwsem.h>
  28 #include <linux/sched.h>
  29 #include <linux/slab.h>
  30 #include <linux/stat.h>
  31 #include <linux/string.h>
  32 #include <linux/uaccess.h>
  33 #include <linux/vfio.h>
  34 #include <linux/wait.h>
  35 #include <linux/sched/signal.h>
  36 #include <linux/pm_runtime.h>
  37 #include <linux/interval_tree.h>
  38 #include <linux/iova_bitmap.h>
  39 #include <linux/iommufd.h>
  40 #include "vfio.h"
  41
  42 #define DRIVER_VERSION  "0.3"
  43 #define DRIVER_AUTHOR   "Alex Williamson <alex.williamson@redhat.com>"
  44 #define DRIVER_DESC     "VFIO - User Level meta-driver"
  45
  46 static struct vfio {
  47         struct class                    *device_class;
  48         struct ida                      device_ida;
  49 } vfio;
  50
  51 #ifdef CONFIG_VFIO_NOIOMMU
  52 bool vfio_noiommu __read_mostly;
  53 module_param_named(enable_unsafe_noiommu_mode,
  54                    vfio_noiommu, bool, S_IRUGO | S_IWUSR);
  55 MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode.  This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel.  If you do not know what this is for, step away. (default: false)");
  56 #endif
  57
  58 static DEFINE_XARRAY(vfio_device_set_xa);
  59
  60 int vfio_assign_device_set(struct vfio_device *device, void *set_id)
  61 {
  62         unsigned long idx = (unsigned long)set_id;
  63         struct vfio_device_set *new_dev_set;
  64         struct vfio_device_set *dev_set;
  65
  66         if (WARN_ON(!set_id))
  67                 return -EINVAL;
  68
  69         /*
  70          * Atomically acquire a singleton object in the xarray for this set_id
  71          */
  72         xa_lock(&vfio_device_set_xa);
  73         dev_set = xa_load(&vfio_device_set_xa, idx);
  74         if (dev_set)
  75                 goto found_get_ref;
  76         xa_unlock(&vfio_device_set_xa);
  77
  78         new_dev_set = kzalloc(sizeof(*new_dev_set), GFP_KERNEL);
  79         if (!new_dev_set)
  80                 return -ENOMEM;
  81         mutex_init(&new_dev_set->lock);
  82         INIT_LIST_HEAD(&new_dev_set->device_list);
  83         new_dev_set->set_id = set_id;
  84
  85         xa_lock(&vfio_device_set_xa);
  86         dev_set = __xa_cmpxchg(&vfio_device_set_xa, idx, NULL, new_dev_set,
  87                                GFP_KERNEL);
  88         if (!dev_set) {
  89                 dev_set = new_dev_set;
  90                 goto found_get_ref;
  91         }
  92
  93         kfree(new_dev_set);
  94         if (xa_is_err(dev_set)) {
  95                 xa_unlock(&vfio_device_set_xa);
  96                 return xa_err(dev_set);
  97         }
  98
  99 found_get_ref:
 100         dev_set->device_count++;
 101         xa_unlock(&vfio_device_set_xa);
 102         mutex_lock(&dev_set->lock);
 103         device->dev_set = dev_set;
 104         list_add_tail(&device->dev_set_list, &dev_set->device_list);
 105         mutex_unlock(&dev_set->lock);
 106         return 0;
 107 }
 108 EXPORT_SYMBOL_GPL(vfio_assign_device_set);
 109
 110 static void vfio_release_device_set(struct vfio_device *device)
 111 {
 112         struct vfio_device_set *dev_set = device->dev_set;
 113
 114         if (!dev_set)
 115                 return;
 116
 117         mutex_lock(&dev_set->lock);
 118         list_del(&device->dev_set_list);
 119         mutex_unlock(&dev_set->lock);
 120
 121         xa_lock(&vfio_device_set_xa);
 122         if (!--dev_set->device_count) {
 123                 __xa_erase(&vfio_device_set_xa,
 124                            (unsigned long)dev_set->set_id);
 125                 mutex_destroy(&dev_set->lock);
 126                 kfree(dev_set);
 127         }
 128         xa_unlock(&vfio_device_set_xa);
 129 }
 130
 131 unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set)
 132 {
 133         struct vfio_device *cur;
 134         unsigned int open_count = 0;
 135
 136         lockdep_assert_held(&dev_set->lock);
 137
 138         list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
 139                 open_count += cur->open_count;
 140         return open_count;
 141 }
 142 EXPORT_SYMBOL_GPL(vfio_device_set_open_count);
 143
 144 struct vfio_device *
 145 vfio_find_device_in_devset(struct vfio_device_set *dev_set,
 146                            struct device *dev)
 147 {
 148         struct vfio_device *cur;
 149
 150         lockdep_assert_held(&dev_set->lock);
 151
 152         list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
 153                 if (cur->dev == dev)
 154                         return cur;
 155         return NULL;
 156 }
 157 EXPORT_SYMBOL_GPL(vfio_find_device_in_devset);
 158
 159 /*
 160  * Device objects - create, release, get, put, search
 161  */
 162 /* Device reference always implies a group reference */
 163 void vfio_device_put_registration(struct vfio_device *device)
 164 {
 165         if (refcount_dec_and_test(&device->refcount))
 166                 complete(&device->comp);
 167 }
 168
 169 bool vfio_device_try_get_registration(struct vfio_device *device)
 170 {
 171         return refcount_inc_not_zero(&device->refcount);
 172 }
 173
 174 /*
 175  * VFIO driver API
 176  */
 177 /* Release helper called by vfio_put_device() */
 178 static void vfio_device_release(struct device *dev)
 179 {
 180         struct vfio_device *device =
 181                         container_of(dev, struct vfio_device, device);
 182
 183         vfio_release_device_set(device);
 184         ida_free(&vfio.device_ida, device->index);
 185
 186         if (device->ops->release)
 187                 device->ops->release(device);
 188
 189         kvfree(device);
 190 }
 191
 192 static int vfio_init_device(struct vfio_device *device, struct device *dev,
 193                             const struct vfio_device_ops *ops);
 194
 195 /*
 196  * Allocate and initialize vfio_device so it can be registered to vfio
 197  * core.
 198  *
 199  * Drivers should use the wrapper vfio_alloc_device() for allocation.
 200  * @size is the size of the structure to be allocated, including any
 201  * private data used by the driver.
 202  *
 203  * Driver may provide an @init callback to cover device private data.
 204  *
 205  * Use vfio_put_device() to release the structure after success return.
 206  */
 207 struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev,
 208                                        const struct vfio_device_ops *ops)
 209 {
 210         struct vfio_device *device;
 211         int ret;
 212
 213         if (WARN_ON(size < sizeof(struct vfio_device)))
 214                 return ERR_PTR(-EINVAL);
 215
 216         device = kvzalloc(size, GFP_KERNEL);
 217         if (!device)
 218                 return ERR_PTR(-ENOMEM);
 219
 220         ret = vfio_init_device(device, dev, ops);
 221         if (ret)
 222                 goto out_free;
 223         return device;
 224
 225 out_free:
 226         kvfree(device);
 227         return ERR_PTR(ret);
 228 }
 229 EXPORT_SYMBOL_GPL(_vfio_alloc_device);
 230
 231 /*
 232  * Initialize a vfio_device so it can be registered to vfio core.
 233  */
 234 static int vfio_init_device(struct vfio_device *device, struct device *dev,
 235                             const struct vfio_device_ops *ops)
 236 {
 237         int ret;
 238
 239         ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL);
 240         if (ret < 0) {
 241                 dev_dbg(dev, "Error to alloc index\n");
 242                 return ret;
 243         }
 244
 245         device->index = ret;
 246         init_completion(&device->comp);
 247         device->dev = dev;
 248         device->ops = ops;
 249
 250         if (ops->init) {
 251                 ret = ops->init(device);
 252                 if (ret)
 253                         goto out_uninit;
 254         }
 255
 256         device_initialize(&device->device);
 257         device->device.release = vfio_device_release;
 258         device->device.class = vfio.device_class;
 259         device->device.parent = device->dev;
 260         return 0;
 261
 262 out_uninit:
 263         vfio_release_device_set(device);
 264         ida_free(&vfio.device_ida, device->index);
 265         return ret;
 266 }
 267
 268 static int __vfio_register_dev(struct vfio_device *device,
 269                                enum vfio_group_type type)
 270 {
 271         int ret;
 272
 273         if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) &&
 274                     (!device->ops->bind_iommufd ||
 275                      !device->ops->unbind_iommufd ||
 276                      !device->ops->attach_ioas ||
 277                      !device->ops->detach_ioas)))
 278                 return -EINVAL;
 279
 280         /*
 281          * If the driver doesn't specify a set then the device is added to a
 282          * singleton set just for itself.
 283          */
 284         if (!device->dev_set)
 285                 vfio_assign_device_set(device, device);
 286
 287         ret = dev_set_name(&device->device, "vfio%d", device->index);
 288         if (ret)
 289                 return ret;
 290
 291         ret = vfio_device_set_group(device, type);
 292         if (ret)
 293                 return ret;
 294
 295         /*
 296          * VFIO always sets IOMMU_CACHE because we offer no way for userspace to
 297          * restore cache coherency. It has to be checked here because it is only
 298          * valid for cases where we are using iommu groups.
 299          */
 300         if (type == VFIO_IOMMU && !vfio_device_is_noiommu(device) &&
 301             !device_iommu_capable(device->dev, IOMMU_CAP_CACHE_COHERENCY)) {
 302                 ret = -EINVAL;
 303                 goto err_out;
 304         }
 305
 306         ret = vfio_device_add(device);
 307         if (ret)
 308                 goto err_out;
 309
 310         /* Refcounting can't start until the driver calls register */
 311         refcount_set(&device->refcount, 1);
 312
 313         vfio_device_group_register(device);
 314
 315         return 0;
 316 err_out:
 317         vfio_device_remove_group(device);
 318         return ret;
 319 }
 320
 321 int vfio_register_group_dev(struct vfio_device *device)
 322 {
 323         return __vfio_register_dev(device, VFIO_IOMMU);
 324 }
 325 EXPORT_SYMBOL_GPL(vfio_register_group_dev);
 326
 327 /*
 328  * Register a virtual device without IOMMU backing.  The user of this
 329  * device must not be able to directly trigger unmediated DMA.
 330  */
 331 int vfio_register_emulated_iommu_dev(struct vfio_device *device)
 332 {
 333         return __vfio_register_dev(device, VFIO_EMULATED_IOMMU);
 334 }
 335 EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev);
 336
 337 /*
 338  * Decrement the device reference count and wait for the device to be
 339  * removed.  Open file descriptors for the device... */
 340 void vfio_unregister_group_dev(struct vfio_device *device)
 341 {
 342         unsigned int i = 0;
 343         bool interrupted = false;
 344         long rc;
 345
 346         /*
 347          * Prevent new device opened by userspace via the
 348          * VFIO_GROUP_GET_DEVICE_FD in the group path.
 349          */
 350         vfio_device_group_unregister(device);
 351
 352         /*
 353          * Balances vfio_device_add() in register path, also prevents
 354          * new device opened by userspace in the cdev path.
 355          */
 356         vfio_device_del(device);
 357
 358         vfio_device_put_registration(device);
 359         rc = try_wait_for_completion(&device->comp);
 360         while (rc <= 0) {
 361                 if (device->ops->request)
 362                         device->ops->request(device, i++);
 363
 364                 if (interrupted) {
 365                         rc = wait_for_completion_timeout(&device->comp,
 366                                                          HZ * 10);
 367                 } else {
 368                         rc = wait_for_completion_interruptible_timeout(
 369                                 &device->comp, HZ * 10);
 370                         if (rc < 0) {
 371                                 interrupted = true;
 372                                 dev_warn(device->dev,
 373                                          "Device is currently in use, task"
 374                                          " \"%s\" (%d) "
 375                                          "blocked until device is released",
 376                                          current->comm, task_pid_nr(current));
 377                         }
 378                 }
 379         }
 380
 381         /* Balances vfio_device_set_group in register path */
 382         vfio_device_remove_group(device);
 383 }
 384 EXPORT_SYMBOL_GPL(vfio_unregister_group_dev);
 385
 386 #ifdef CONFIG_HAVE_KVM
 387 void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm)
 388 {
 389         void (*pfn)(struct kvm *kvm);
 390         bool (*fn)(struct kvm *kvm);
 391         bool ret;
 392
 393         lockdep_assert_held(&device->dev_set->lock);
 394
 395         if (!kvm)
 396                 return;
 397
 398         pfn = symbol_get(kvm_put_kvm);
 399         if (WARN_ON(!pfn))
 400                 return;
 401
 402         fn = symbol_get(kvm_get_kvm_safe);
 403         if (WARN_ON(!fn)) {
 404                 symbol_put(kvm_put_kvm);
 405                 return;
 406         }
 407
 408         ret = fn(kvm);
 409         symbol_put(kvm_get_kvm_safe);
 410         if (!ret) {
 411                 symbol_put(kvm_put_kvm);
 412                 return;
 413         }
 414
 415         device->put_kvm = pfn;
 416         device->kvm = kvm;
 417 }
 418
 419 void vfio_device_put_kvm(struct vfio_device *device)
 420 {
 421         lockdep_assert_held(&device->dev_set->lock);
 422
 423         if (!device->kvm)
 424                 return;
 425
 426         if (WARN_ON(!device->put_kvm))
 427                 goto clear;
 428
 429         device->put_kvm(device->kvm);
 430         device->put_kvm = NULL;
 431         symbol_put(kvm_put_kvm);
 432
 433 clear:
 434         device->kvm = NULL;
 435 }
 436 #endif
 437
 438 /* true if the vfio_device has open_device() called but not close_device() */
 439 static bool vfio_assert_device_open(struct vfio_device *device)
 440 {
 441         return !WARN_ON_ONCE(!READ_ONCE(device->open_count));
 442 }
 443
 444 struct vfio_device_file *
 445 vfio_allocate_device_file(struct vfio_device *device)
 446 {
 447         struct vfio_device_file *df;
 448
 449         df = kzalloc(sizeof(*df), GFP_KERNEL_ACCOUNT);
 450         if (!df)
 451                 return ERR_PTR(-ENOMEM);
 452
 453         df->device = device;
 454         spin_lock_init(&df->kvm_ref_lock);
 455
 456         return df;
 457 }
 458
 459 static int vfio_df_device_first_open(struct vfio_device_file *df)
 460 {
 461         struct vfio_device *device = df->device;
 462         struct iommufd_ctx *iommufd = df->iommufd;
 463         int ret;
 464
 465         lockdep_assert_held(&device->dev_set->lock);
 466
 467         if (!try_module_get(device->dev->driver->owner))
 468                 return -ENODEV;
 469
 470         if (iommufd)
 471                 ret = vfio_df_iommufd_bind(df);
 472         else
 473                 ret = vfio_device_group_use_iommu(device);
 474         if (ret)
 475                 goto err_module_put;
 476
 477         if (device->ops->open_device) {
 478                 ret = device->ops->open_device(device);
 479                 if (ret)
 480                         goto err_unuse_iommu;
 481         }
 482         return 0;
 483
 484 err_unuse_iommu:
 485         if (iommufd)
 486                 vfio_df_iommufd_unbind(df);
 487         else
 488                 vfio_device_group_unuse_iommu(device);
 489 err_module_put:
 490         module_put(device->dev->driver->owner);
 491         return ret;
 492 }
 493
 494 static void vfio_df_device_last_close(struct vfio_device_file *df)
 495 {
 496         struct vfio_device *device = df->device;
 497         struct iommufd_ctx *iommufd = df->iommufd;
 498
 499         lockdep_assert_held(&device->dev_set->lock);
 500
 501         if (device->ops->close_device)
 502                 device->ops->close_device(device);
 503         if (iommufd)
 504                 vfio_df_iommufd_unbind(df);
 505         else
 506                 vfio_device_group_unuse_iommu(device);
 507         module_put(device->dev->driver->owner);
 508 }
 509
 510 int vfio_df_open(struct vfio_device_file *df)
 511 {
 512         struct vfio_device *device = df->device;
 513         int ret = 0;
 514
 515         lockdep_assert_held(&device->dev_set->lock);
 516
 517         /*
 518          * Only the group path allows the device to be opened multiple
 519          * times.  The device cdev path doesn't have a secure way for it.
 520          */
 521         if (device->open_count != 0 && !df->group)
 522                 return -EINVAL;
 523
 524         device->open_count++;
 525         if (device->open_count == 1) {
 526                 ret = vfio_df_device_first_open(df);
 527                 if (ret)
 528                         device->open_count--;
 529         }
 530
 531         return ret;
 532 }
 533
 534 void vfio_df_close(struct vfio_device_file *df)
 535 {
 536         struct vfio_device *device = df->device;
 537
 538         lockdep_assert_held(&device->dev_set->lock);
 539
 540         vfio_assert_device_open(device);
 541         if (device->open_count == 1)
 542                 vfio_df_device_last_close(df);
 543         device->open_count--;
 544 }
 545
 546 /*
 547  * Wrapper around pm_runtime_resume_and_get().
 548  * Return error code on failure or 0 on success.
 549  */
 550 static inline int vfio_device_pm_runtime_get(struct vfio_device *device)
 551 {
 552         struct device *dev = device->dev;
 553
 554         if (dev->driver && dev->driver->pm) {
 555                 int ret;
 556
 557                 ret = pm_runtime_resume_and_get(dev);
 558                 if (ret) {
 559                         dev_info_ratelimited(dev,
 560                                 "vfio: runtime resume failed %d\n", ret);
 561                         return -EIO;
 562                 }
 563         }
 564
 565         return 0;
 566 }
 567
 568 /*
 569  * Wrapper around pm_runtime_put().
 570  */
 571 static inline void vfio_device_pm_runtime_put(struct vfio_device *device)
 572 {
 573         struct device *dev = device->dev;
 574
 575         if (dev->driver && dev->driver->pm)
 576                 pm_runtime_put(dev);
 577 }
 578
 579 /*
 580  * VFIO Device fd
 581  */
 582 static int vfio_device_fops_release(struct inode *inode, struct file *filep)
 583 {
 584         struct vfio_device_file *df = filep->private_data;
 585         struct vfio_device *device = df->device;
 586
 587         if (df->group)
 588                 vfio_df_group_close(df);
 589         else
 590                 vfio_df_unbind_iommufd(df);
 591
 592         vfio_device_put_registration(device);
 593
 594         kfree(df);
 595
 596         return 0;
 597 }
 598
 599 /*
 600  * vfio_mig_get_next_state - Compute the next step in the FSM
 601  * @cur_fsm - The current state the device is in
 602  * @new_fsm - The target state to reach
 603  * @next_fsm - Pointer to the next step to get to new_fsm
 604  *
 605  * Return 0 upon success, otherwise -errno
 606  * Upon success the next step in the state progression between cur_fsm and
 607  * new_fsm will be set in next_fsm.
 608  *
 609  * This breaks down requests for combination transitions into smaller steps and
 610  * returns the next step to get to new_fsm. The function may need to be called
 611  * multiple times before reaching new_fsm.
 612  *
 613  */
 614 int vfio_mig_get_next_state(struct vfio_device *device,
 615                             enum vfio_device_mig_state cur_fsm,
 616                             enum vfio_device_mig_state new_fsm,
 617                             enum vfio_device_mig_state *next_fsm)
 618 {
 619         enum { VFIO_DEVICE_NUM_STATES = VFIO_DEVICE_STATE_PRE_COPY_P2P + 1 };
 620         /*
 621          * The coding in this table requires the driver to implement the
 622          * following FSM arcs:
 623          *         RESUMING -> STOP
 624          *         STOP -> RESUMING
 625          *         STOP -> STOP_COPY
 626          *         STOP_COPY -> STOP
 627          *
 628          * If P2P is supported then the driver must also implement these FSM
 629          * arcs:
 630          *         RUNNING -> RUNNING_P2P
 631          *         RUNNING_P2P -> RUNNING
 632          *         RUNNING_P2P -> STOP
 633          *         STOP -> RUNNING_P2P
 634          *
 635          * If precopy is supported then the driver must support these additional
 636          * FSM arcs:
 637          *         RUNNING -> PRE_COPY
 638          *         PRE_COPY -> RUNNING
 639          *         PRE_COPY -> STOP_COPY
 640          * However, if precopy and P2P are supported together then the driver
 641          * must support these additional arcs beyond the P2P arcs above:
 642          *         PRE_COPY -> RUNNING
 643          *         PRE_COPY -> PRE_COPY_P2P
 644          *         PRE_COPY_P2P -> PRE_COPY
 645          *         PRE_COPY_P2P -> RUNNING_P2P
 646          *         PRE_COPY_P2P -> STOP_COPY
 647          *         RUNNING -> PRE_COPY
 648          *         RUNNING_P2P -> PRE_COPY_P2P
 649          *
 650          * Without P2P and precopy the driver must implement:
 651          *         RUNNING -> STOP
 652          *         STOP -> RUNNING
 653          *
 654          * The coding will step through multiple states for some combination
 655          * transitions; if all optional features are supported, this means the
 656          * following ones:
 657          *         PRE_COPY -> PRE_COPY_P2P -> STOP_COPY
 658          *         PRE_COPY -> RUNNING -> RUNNING_P2P
 659          *         PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP
 660          *         PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP -> RESUMING
 661          *         PRE_COPY_P2P -> RUNNING_P2P -> RUNNING
 662          *         PRE_COPY_P2P -> RUNNING_P2P -> STOP
 663          *         PRE_COPY_P2P -> RUNNING_P2P -> STOP -> RESUMING
 664          *         RESUMING -> STOP -> RUNNING_P2P
 665          *         RESUMING -> STOP -> RUNNING_P2P -> PRE_COPY_P2P
 666          *         RESUMING -> STOP -> RUNNING_P2P -> RUNNING
 667          *         RESUMING -> STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY
 668          *         RESUMING -> STOP -> STOP_COPY
 669          *         RUNNING -> RUNNING_P2P -> PRE_COPY_P2P
 670          *         RUNNING -> RUNNING_P2P -> STOP
 671          *         RUNNING -> RUNNING_P2P -> STOP -> RESUMING
 672          *         RUNNING -> RUNNING_P2P -> STOP -> STOP_COPY
 673          *         RUNNING_P2P -> RUNNING -> PRE_COPY
 674          *         RUNNING_P2P -> STOP -> RESUMING
 675          *         RUNNING_P2P -> STOP -> STOP_COPY
 676          *         STOP -> RUNNING_P2P -> PRE_COPY_P2P
 677          *         STOP -> RUNNING_P2P -> RUNNING
 678          *         STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY
 679          *         STOP_COPY -> STOP -> RESUMING
 680          *         STOP_COPY -> STOP -> RUNNING_P2P
 681          *         STOP_COPY -> STOP -> RUNNING_P2P -> RUNNING
 682          *
 683          *  The following transitions are blocked:
 684          *         STOP_COPY -> PRE_COPY
 685          *         STOP_COPY -> PRE_COPY_P2P
 686          */
 687         static const u8 vfio_from_fsm_table[VFIO_DEVICE_NUM_STATES][VFIO_DEVICE_NUM_STATES] = {
 688                 [VFIO_DEVICE_STATE_STOP] = {
 689                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 690                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 691                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P,
 692                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 693                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 694                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING,
 695                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 696                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 697                 },
 698                 [VFIO_DEVICE_STATE_RUNNING] = {
 699                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P,
 700                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 701                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 702                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 703                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P,
 704                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 705                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 706                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 707                 },
 708                 [VFIO_DEVICE_STATE_PRE_COPY] = {
 709                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING,
 710                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 711                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 712                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 713                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 714                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING,
 715                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING,
 716                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 717                 },
 718                 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = {
 719                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P,
 720                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 721                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 722                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 723                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 724                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 725                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 726                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 727                 },
 728                 [VFIO_DEVICE_STATE_STOP_COPY] = {
 729                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 730                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP,
 731                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR,
 732                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR,
 733                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 734                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP,
 735                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP,
 736                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 737                 },
 738                 [VFIO_DEVICE_STATE_RESUMING] = {
 739                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 740                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP,
 741                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_STOP,
 742                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_STOP,
 743                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP,
 744                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING,
 745                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP,
 746                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 747                 },
 748                 [VFIO_DEVICE_STATE_RUNNING_P2P] = {
 749                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 750                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 751                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING,
 752                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 753                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP,
 754                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP,
 755                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 756                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 757                 },
 758                 [VFIO_DEVICE_STATE_ERROR] = {
 759                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_ERROR,
 760                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_ERROR,
 761                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR,
 762                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR,
 763                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_ERROR,
 764                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_ERROR,
 765                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_ERROR,
 766                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 767                 },
 768         };
 769
 770         static const unsigned int state_flags_table[VFIO_DEVICE_NUM_STATES] = {
 771                 [VFIO_DEVICE_STATE_STOP] = VFIO_MIGRATION_STOP_COPY,
 772                 [VFIO_DEVICE_STATE_RUNNING] = VFIO_MIGRATION_STOP_COPY,
 773                 [VFIO_DEVICE_STATE_PRE_COPY] =
 774                         VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY,
 775                 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_MIGRATION_STOP_COPY |
 776                                                    VFIO_MIGRATION_P2P |
 777                                                    VFIO_MIGRATION_PRE_COPY,
 778                 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_MIGRATION_STOP_COPY,
 779                 [VFIO_DEVICE_STATE_RESUMING] = VFIO_MIGRATION_STOP_COPY,
 780                 [VFIO_DEVICE_STATE_RUNNING_P2P] =
 781                         VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P,
 782                 [VFIO_DEVICE_STATE_ERROR] = ~0U,
 783         };
 784
 785         if (WARN_ON(cur_fsm >= ARRAY_SIZE(vfio_from_fsm_table) ||
 786                     (state_flags_table[cur_fsm] & device->migration_flags) !=
 787                         state_flags_table[cur_fsm]))
 788                 return -EINVAL;
 789
 790         if (new_fsm >= ARRAY_SIZE(vfio_from_fsm_table) ||
 791            (state_flags_table[new_fsm] & device->migration_flags) !=
 792                         state_flags_table[new_fsm])
 793                 return -EINVAL;
 794
 795         /*
 796          * Arcs touching optional and unsupported states are skipped over. The
 797          * driver will instead see an arc from the original state to the next
 798          * logical state, as per the above comment.
 799          */
 800         *next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm];
 801         while ((state_flags_table[*next_fsm] & device->migration_flags) !=
 802                         state_flags_table[*next_fsm])
 803                 *next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm];
 804
 805         return (*next_fsm != VFIO_DEVICE_STATE_ERROR) ? 0 : -EINVAL;
 806 }
 807 EXPORT_SYMBOL_GPL(vfio_mig_get_next_state);
 808
 809 /*
 810  * Convert the drivers's struct file into a FD number and return it to userspace
 811  */
 812 static int vfio_ioct_mig_return_fd(struct file *filp, void __user *arg,
 813                                    struct vfio_device_feature_mig_state *mig)
 814 {
 815         int ret;
 816         int fd;
 817
 818         fd = get_unused_fd_flags(O_CLOEXEC);
 819         if (fd < 0) {
 820                 ret = fd;
 821                 goto out_fput;
 822         }
 823
 824         mig->data_fd = fd;
 825         if (copy_to_user(arg, mig, sizeof(*mig))) {
 826                 ret = -EFAULT;
 827                 goto out_put_unused;
 828         }
 829         fd_install(fd, filp);
 830         return 0;
 831
 832 out_put_unused:
 833         put_unused_fd(fd);
 834 out_fput:
 835         fput(filp);
 836         return ret;
 837 }
 838
 839 static int
 840 vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device,
 841                                            u32 flags, void __user *arg,
 842                                            size_t argsz)
 843 {
 844         size_t minsz =
 845                 offsetofend(struct vfio_device_feature_mig_state, data_fd);
 846         struct vfio_device_feature_mig_state mig;
 847         struct file *filp = NULL;
 848         int ret;
 849
 850         if (!device->mig_ops)
 851                 return -ENOTTY;
 852
 853         ret = vfio_check_feature(flags, argsz,
 854                                  VFIO_DEVICE_FEATURE_SET |
 855                                  VFIO_DEVICE_FEATURE_GET,
 856                                  sizeof(mig));
 857         if (ret != 1)
 858                 return ret;
 859
 860         if (copy_from_user(&mig, arg, minsz))
 861                 return -EFAULT;
 862
 863         if (flags & VFIO_DEVICE_FEATURE_GET) {
 864                 enum vfio_device_mig_state curr_state;
 865
 866                 ret = device->mig_ops->migration_get_state(device,
 867                                                            &curr_state);
 868                 if (ret)
 869                         return ret;
 870                 mig.device_state = curr_state;
 871                 goto out_copy;
 872         }
 873
 874         /* Handle the VFIO_DEVICE_FEATURE_SET */
 875         filp = device->mig_ops->migration_set_state(device, mig.device_state);
 876         if (IS_ERR(filp) || !filp)
 877                 goto out_copy;
 878
 879         return vfio_ioct_mig_return_fd(filp, arg, &mig);
 880 out_copy:
 881         mig.data_fd = -1;
 882         if (copy_to_user(arg, &mig, sizeof(mig)))
 883                 return -EFAULT;
 884         if (IS_ERR(filp))
 885                 return PTR_ERR(filp);
 886         return 0;
 887 }
 888
 889 static int
 890 vfio_ioctl_device_feature_migration_data_size(struct vfio_device *device,
 891                                               u32 flags, void __user *arg,
 892                                               size_t argsz)
 893 {
 894         struct vfio_device_feature_mig_data_size data_size = {};
 895         unsigned long stop_copy_length;
 896         int ret;
 897
 898         if (!device->mig_ops)
 899                 return -ENOTTY;
 900
 901         ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
 902                                  sizeof(data_size));
 903         if (ret != 1)
 904                 return ret;
 905
 906         ret = device->mig_ops->migration_get_data_size(device, &stop_copy_length);
 907         if (ret)
 908                 return ret;
 909
 910         data_size.stop_copy_length = stop_copy_length;
 911         if (copy_to_user(arg, &data_size, sizeof(data_size)))
 912                 return -EFAULT;
 913
 914         return 0;
 915 }
 916
 917 static int vfio_ioctl_device_feature_migration(struct vfio_device *device,
 918                                                u32 flags, void __user *arg,
 919                                                size_t argsz)
 920 {
 921         struct vfio_device_feature_migration mig = {
 922                 .flags = device->migration_flags,
 923         };
 924         int ret;
 925
 926         if (!device->mig_ops)
 927                 return -ENOTTY;
 928
 929         ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
 930                                  sizeof(mig));
 931         if (ret != 1)
 932                 return ret;
 933         if (copy_to_user(arg, &mig, sizeof(mig)))
 934                 return -EFAULT;
 935         return 0;
 936 }
 937
 938 void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes,
 939                               u32 req_nodes)
 940 {
 941         struct interval_tree_node *prev, *curr, *comb_start, *comb_end;
 942         unsigned long min_gap, curr_gap;
 943
 944         /* Special shortcut when a single range is required */
 945         if (req_nodes == 1) {
 946                 unsigned long last;
 947
 948                 comb_start = interval_tree_iter_first(root, 0, ULONG_MAX);
 949                 curr = comb_start;
 950                 while (curr) {
 951                         last = curr->last;
 952                         prev = curr;
 953                         curr = interval_tree_iter_next(curr, 0, ULONG_MAX);
 954                         if (prev != comb_start)
 955                                 interval_tree_remove(prev, root);
 956                 }
 957                 comb_start->last = last;
 958                 return;
 959         }
 960
 961         /* Combine ranges which have the smallest gap */
 962         while (cur_nodes > req_nodes) {
 963                 prev = NULL;
 964                 min_gap = ULONG_MAX;
 965                 curr = interval_tree_iter_first(root, 0, ULONG_MAX);
 966                 while (curr) {
 967                         if (prev) {
 968                                 curr_gap = curr->start - prev->last;
 969                                 if (curr_gap < min_gap) {
 970                                         min_gap = curr_gap;
 971                                         comb_start = prev;
 972                                         comb_end = curr;
 973                                 }
 974                         }
 975                         prev = curr;
 976                         curr = interval_tree_iter_next(curr, 0, ULONG_MAX);
 977                 }
 978                 comb_start->last = comb_end->last;
 979                 interval_tree_remove(comb_end, root);
 980                 cur_nodes--;
 981         }
 982 }
 983 EXPORT_SYMBOL_GPL(vfio_combine_iova_ranges);
 984
 985 /* Ranges should fit into a single kernel page */
 986 #define LOG_MAX_RANGES \
 987         (PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range))
 988
 989 static int
 990 vfio_ioctl_device_feature_logging_start(struct vfio_device *device,
 991                                         u32 flags, void __user *arg,
 992                                         size_t argsz)
 993 {
 994         size_t minsz =
 995                 offsetofend(struct vfio_device_feature_dma_logging_control,
 996                             ranges);
 997         struct vfio_device_feature_dma_logging_range __user *ranges;
 998         struct vfio_device_feature_dma_logging_control control;
 999         struct vfio_device_feature_dma_logging_range range;
1000         struct rb_root_cached root = RB_ROOT_CACHED;
1001         struct interval_tree_node *nodes;
1002         u64 iova_end;
1003         u32 nnodes;
1004         int i, ret;
1005
1006         if (!device->log_ops)
1007                 return -ENOTTY;
1008
1009         ret = vfio_check_feature(flags, argsz,
1010                                  VFIO_DEVICE_FEATURE_SET,
1011                                  sizeof(control));
1012         if (ret != 1)
1013                 return ret;
1014
1015         if (copy_from_user(&control, arg, minsz))
1016                 return -EFAULT;
1017
1018         nnodes = control.num_ranges;
1019         if (!nnodes)
1020                 return -EINVAL;
1021
1022         if (nnodes > LOG_MAX_RANGES)
1023                 return -E2BIG;
1024
1025         ranges = u64_to_user_ptr(control.ranges);
1026         nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node),
1027                               GFP_KERNEL);
1028         if (!nodes)
1029                 return -ENOMEM;
1030
1031         for (i = 0; i < nnodes; i++) {
1032                 if (copy_from_user(&range, &ranges[i], sizeof(range))) {
1033                         ret = -EFAULT;
1034                         goto end;
1035                 }
1036                 if (!IS_ALIGNED(range.iova, control.page_size) ||
1037                     !IS_ALIGNED(range.length, control.page_size)) {
1038                         ret = -EINVAL;
1039                         goto end;
1040                 }
1041
1042                 if (check_add_overflow(range.iova, range.length, &iova_end) ||
1043                     iova_end > ULONG_MAX) {
1044                         ret = -EOVERFLOW;
1045                         goto end;
1046                 }
1047
1048                 nodes[i].start = range.iova;
1049                 nodes[i].last = range.iova + range.length - 1;
1050                 if (interval_tree_iter_first(&root, nodes[i].start,
1051                                              nodes[i].last)) {
1052                         /* Range overlapping */
1053                         ret = -EINVAL;
1054                         goto end;
1055                 }
1056                 interval_tree_insert(nodes + i, &root);
1057         }
1058
1059         ret = device->log_ops->log_start(device, &root, nnodes,
1060                                          &control.page_size);
1061         if (ret)
1062                 goto end;
1063
1064         if (copy_to_user(arg, &control, sizeof(control))) {
1065                 ret = -EFAULT;
1066                 device->log_ops->log_stop(device);
1067         }
1068
1069 end:
1070         kfree(nodes);
1071         return ret;
1072 }
1073
1074 static int
1075 vfio_ioctl_device_feature_logging_stop(struct vfio_device *device,
1076                                        u32 flags, void __user *arg,
1077                                        size_t argsz)
1078 {
1079         int ret;
1080
1081         if (!device->log_ops)
1082                 return -ENOTTY;
1083
1084         ret = vfio_check_feature(flags, argsz,
1085                                  VFIO_DEVICE_FEATURE_SET, 0);
1086         if (ret != 1)
1087                 return ret;
1088
1089         return device->log_ops->log_stop(device);
1090 }
1091
1092 static int vfio_device_log_read_and_clear(struct iova_bitmap *iter,
1093                                           unsigned long iova, size_t length,
1094                                           void *opaque)
1095 {
1096         struct vfio_device *device = opaque;
1097
1098         return device->log_ops->log_read_and_clear(device, iova, length, iter);
1099 }
1100
1101 static int
1102 vfio_ioctl_device_feature_logging_report(struct vfio_device *device,
1103                                          u32 flags, void __user *arg,
1104                                          size_t argsz)
1105 {
1106         size_t minsz =
1107                 offsetofend(struct vfio_device_feature_dma_logging_report,
1108                             bitmap);
1109         struct vfio_device_feature_dma_logging_report report;
1110         struct iova_bitmap *iter;
1111         u64 iova_end;
1112         int ret;
1113
1114         if (!device->log_ops)
1115                 return -ENOTTY;
1116
1117         ret = vfio_check_feature(flags, argsz,
1118                                  VFIO_DEVICE_FEATURE_GET,
1119                                  sizeof(report));
1120         if (ret != 1)
1121                 return ret;
1122
1123         if (copy_from_user(&report, arg, minsz))
1124                 return -EFAULT;
1125
1126         if (report.page_size < SZ_4K || !is_power_of_2(report.page_size))
1127                 return -EINVAL;
1128
1129         if (check_add_overflow(report.iova, report.length, &iova_end) ||
1130             iova_end > ULONG_MAX)
1131                 return -EOVERFLOW;
1132
1133         iter = iova_bitmap_alloc(report.iova, report.length,
1134                                  report.page_size,
1135                                  u64_to_user_ptr(report.bitmap));
1136         if (IS_ERR(iter))
1137                 return PTR_ERR(iter);
1138
1139         ret = iova_bitmap_for_each(iter, device,
1140                                    vfio_device_log_read_and_clear);
1141
1142         iova_bitmap_free(iter);
1143         return ret;
1144 }
1145
1146 static int vfio_ioctl_device_feature(struct vfio_device *device,
1147                                      struct vfio_device_feature __user *arg)
1148 {
1149         size_t minsz = offsetofend(struct vfio_device_feature, flags);
1150         struct vfio_device_feature feature;
1151
1152         if (copy_from_user(&feature, arg, minsz))
1153                 return -EFAULT;
1154
1155         if (feature.argsz < minsz)
1156                 return -EINVAL;
1157
1158         /* Check unknown flags */
1159         if (feature.flags &
1160             ~(VFIO_DEVICE_FEATURE_MASK | VFIO_DEVICE_FEATURE_SET |
1161               VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_PROBE))
1162                 return -EINVAL;
1163
1164         /* GET & SET are mutually exclusive except with PROBE */
1165         if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) &&
1166             (feature.flags & VFIO_DEVICE_FEATURE_SET) &&
1167             (feature.flags & VFIO_DEVICE_FEATURE_GET))
1168                 return -EINVAL;
1169
1170         switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) {
1171         case VFIO_DEVICE_FEATURE_MIGRATION:
1172                 return vfio_ioctl_device_feature_migration(
1173                         device, feature.flags, arg->data,
1174                         feature.argsz - minsz);
1175         case VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE:
1176                 return vfio_ioctl_device_feature_mig_device_state(
1177                         device, feature.flags, arg->data,
1178                         feature.argsz - minsz);
1179         case VFIO_DEVICE_FEATURE_DMA_LOGGING_START:
1180                 return vfio_ioctl_device_feature_logging_start(
1181                         device, feature.flags, arg->data,
1182                         feature.argsz - minsz);
1183         case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP:
1184                 return vfio_ioctl_device_feature_logging_stop(
1185                         device, feature.flags, arg->data,
1186                         feature.argsz - minsz);
1187         case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT:
1188                 return vfio_ioctl_device_feature_logging_report(
1189                         device, feature.flags, arg->data,
1190                         feature.argsz - minsz);
1191         case VFIO_DEVICE_FEATURE_MIG_DATA_SIZE:
1192                 return vfio_ioctl_device_feature_migration_data_size(
1193                         device, feature.flags, arg->data,
1194                         feature.argsz - minsz);
1195         default:
1196                 if (unlikely(!device->ops->device_feature))
1197                         return -EINVAL;
1198                 return device->ops->device_feature(device, feature.flags,
1199                                                    arg->data,
1200                                                    feature.argsz - minsz);
1201         }
1202 }
1203
1204 static long vfio_device_fops_unl_ioctl(struct file *filep,
1205                                        unsigned int cmd, unsigned long arg)
1206 {
1207         struct vfio_device_file *df = filep->private_data;
1208         struct vfio_device *device = df->device;
1209         void __user *uptr = (void __user *)arg;
1210         int ret;
1211
1212         if (cmd == VFIO_DEVICE_BIND_IOMMUFD)
1213                 return vfio_df_ioctl_bind_iommufd(df, uptr);
1214
1215         /* Paired with smp_store_release() following vfio_df_open() */
1216         if (!smp_load_acquire(&df->access_granted))
1217                 return -EINVAL;
1218
1219         ret = vfio_device_pm_runtime_get(device);
1220         if (ret)
1221                 return ret;
1222
1223         /* cdev only ioctls */
1224         if (IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV) && !df->group) {
1225                 switch (cmd) {
1226                 case VFIO_DEVICE_ATTACH_IOMMUFD_PT:
1227                         ret = vfio_df_ioctl_attach_pt(df, uptr);
1228                         goto out;
1229
1230                 case VFIO_DEVICE_DETACH_IOMMUFD_PT:
1231                         ret = vfio_df_ioctl_detach_pt(df, uptr);
1232                         goto out;
1233                 }
1234         }
1235
1236         switch (cmd) {
1237         case VFIO_DEVICE_FEATURE:
1238                 ret = vfio_ioctl_device_feature(device, uptr);
1239                 break;
1240
1241         default:
1242                 if (unlikely(!device->ops->ioctl))
1243                         ret = -EINVAL;
1244                 else
1245                         ret = device->ops->ioctl(device, cmd, arg);
1246                 break;
1247         }
1248 out:
1249         vfio_device_pm_runtime_put(device);
1250         return ret;
1251 }
1252
1253 static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf,
1254                                      size_t count, loff_t *ppos)
1255 {
1256         struct vfio_device_file *df = filep->private_data;
1257         struct vfio_device *device = df->device;
1258
1259         /* Paired with smp_store_release() following vfio_df_open() */
1260         if (!smp_load_acquire(&df->access_granted))
1261                 return -EINVAL;
1262
1263         if (unlikely(!device->ops->read))
1264                 return -EINVAL;
1265
1266         return device->ops->read(device, buf, count, ppos);
1267 }
1268
1269 static ssize_t vfio_device_fops_write(struct file *filep,
1270                                       const char __user *buf,
1271                                       size_t count, loff_t *ppos)
1272 {
1273         struct vfio_device_file *df = filep->private_data;
1274         struct vfio_device *device = df->device;
1275
1276         /* Paired with smp_store_release() following vfio_df_open() */
1277         if (!smp_load_acquire(&df->access_granted))
1278                 return -EINVAL;
1279
1280         if (unlikely(!device->ops->write))
1281                 return -EINVAL;
1282
1283         return device->ops->write(device, buf, count, ppos);
1284 }
1285
1286 static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma)
1287 {
1288         struct vfio_device_file *df = filep->private_data;
1289         struct vfio_device *device = df->device;
1290
1291         /* Paired with smp_store_release() following vfio_df_open() */
1292         if (!smp_load_acquire(&df->access_granted))
1293                 return -EINVAL;
1294
1295         if (unlikely(!device->ops->mmap))
1296                 return -EINVAL;
1297
1298         return device->ops->mmap(device, vma);
1299 }
1300
1301 const struct file_operations vfio_device_fops = {
1302         .owner          = THIS_MODULE,
1303         .open           = vfio_device_fops_cdev_open,
1304         .release        = vfio_device_fops_release,
1305         .read           = vfio_device_fops_read,
1306         .write          = vfio_device_fops_write,
1307         .unlocked_ioctl = vfio_device_fops_unl_ioctl,
1308         .compat_ioctl   = compat_ptr_ioctl,
1309         .mmap           = vfio_device_fops_mmap,
1310 };
1311
1312 static struct vfio_device *vfio_device_from_file(struct file *file)
1313 {
1314         struct vfio_device_file *df = file->private_data;
1315
1316         if (file->f_op != &vfio_device_fops)
1317                 return NULL;
1318         return df->device;
1319 }
1320
1321 /**
1322  * vfio_file_is_valid - True if the file is valid vfio file
1323  * @file: VFIO group file or VFIO device file
1324  */
1325 bool vfio_file_is_valid(struct file *file)
1326 {
1327         return vfio_group_from_file(file) ||
1328                vfio_device_from_file(file);
1329 }
1330 EXPORT_SYMBOL_GPL(vfio_file_is_valid);
1331
1332 /**
1333  * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file
1334  *        is always CPU cache coherent
1335  * @file: VFIO group file or VFIO device file
1336  *
1337  * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop
1338  * bit in DMA transactions. A return of false indicates that the user has
1339  * rights to access additional instructions such as wbinvd on x86.
1340  */
1341 bool vfio_file_enforced_coherent(struct file *file)
1342 {
1343         struct vfio_device *device;
1344         struct vfio_group *group;
1345
1346         group = vfio_group_from_file(file);
1347         if (group)
1348                 return vfio_group_enforced_coherent(group);
1349
1350         device = vfio_device_from_file(file);
1351         if (device)
1352                 return device_iommu_capable(device->dev,
1353                                             IOMMU_CAP_ENFORCE_CACHE_COHERENCY);
1354
1355         return true;
1356 }
1357 EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent);
1358
1359 static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm)
1360 {
1361         struct vfio_device_file *df = file->private_data;
1362
1363         /*
1364          * The kvm is first recorded in the vfio_device_file, and will
1365          * be propagated to vfio_device::kvm when the file is bound to
1366          * iommufd successfully in the vfio device cdev path.
1367          */
1368         spin_lock(&df->kvm_ref_lock);
1369         df->kvm = kvm;
1370         spin_unlock(&df->kvm_ref_lock);
1371 }
1372
1373 /**
1374  * vfio_file_set_kvm - Link a kvm with VFIO drivers
1375  * @file: VFIO group file or VFIO device file
1376  * @kvm: KVM to link
1377  *
1378  * When a VFIO device is first opened the KVM will be available in
1379  * device->kvm if one was associated with the file.
1380  */
1381 void vfio_file_set_kvm(struct file *file, struct kvm *kvm)
1382 {
1383         struct vfio_group *group;
1384
1385         group = vfio_group_from_file(file);
1386         if (group)
1387                 vfio_group_set_kvm(group, kvm);
1388
1389         if (vfio_device_from_file(file))
1390                 vfio_device_file_set_kvm(file, kvm);
1391 }
1392 EXPORT_SYMBOL_GPL(vfio_file_set_kvm);
1393
1394 /*
1395  * Sub-module support
1396  */
1397 /*
1398  * Helper for managing a buffer of info chain capabilities, allocate or
1399  * reallocate a buffer with additional @size, filling in @id and @version
1400  * of the capability.  A pointer to the new capability is returned.
1401  *
1402  * NB. The chain is based at the head of the buffer, so new entries are
1403  * added to the tail, vfio_info_cap_shift() should be called to fixup the
1404  * next offsets prior to copying to the user buffer.
1405  */
1406 struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps,
1407                                                size_t size, u16 id, u16 version)
1408 {
1409         void *buf;
1410         struct vfio_info_cap_header *header, *tmp;
1411
1412         /* Ensure that the next capability struct will be aligned */
1413         size = ALIGN(size, sizeof(u64));
1414
1415         buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL);
1416         if (!buf) {
1417                 kfree(caps->buf);
1418                 caps->buf = NULL;
1419                 caps->size = 0;
1420                 return ERR_PTR(-ENOMEM);
1421         }
1422
1423         caps->buf = buf;
1424         header = buf + caps->size;
1425
1426         /* Eventually copied to user buffer, zero */
1427         memset(header, 0, size);
1428
1429         header->id = id;
1430         header->version = version;
1431
1432         /* Add to the end of the capability chain */
1433         for (tmp = buf; tmp->next; tmp = buf + tmp->next)
1434                 ; /* nothing */
1435
1436         tmp->next = caps->size;
1437         caps->size += size;
1438
1439         return header;
1440 }
1441 EXPORT_SYMBOL_GPL(vfio_info_cap_add);
1442
1443 void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset)
1444 {
1445         struct vfio_info_cap_header *tmp;
1446         void *buf = (void *)caps->buf;
1447
1448         /* Capability structs should start with proper alignment */
1449         WARN_ON(!IS_ALIGNED(offset, sizeof(u64)));
1450
1451         for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset)
1452                 tmp->next += offset;
1453 }
1454 EXPORT_SYMBOL(vfio_info_cap_shift);
1455
1456 int vfio_info_add_capability(struct vfio_info_cap *caps,
1457                              struct vfio_info_cap_header *cap, size_t size)
1458 {
1459         struct vfio_info_cap_header *header;
1460
1461         header = vfio_info_cap_add(caps, size, cap->id, cap->version);
1462         if (IS_ERR(header))
1463                 return PTR_ERR(header);
1464
1465         memcpy(header + 1, cap + 1, size - sizeof(*header));
1466
1467         return 0;
1468 }
1469 EXPORT_SYMBOL(vfio_info_add_capability);
1470
1471 int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs,
1472                                        int max_irq_type, size_t *data_size)
1473 {
1474         unsigned long minsz;
1475         size_t size;
1476
1477         minsz = offsetofend(struct vfio_irq_set, count);
1478
1479         if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) ||
1480             (hdr->count >= (U32_MAX - hdr->start)) ||
1481             (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
1482                                 VFIO_IRQ_SET_ACTION_TYPE_MASK)))
1483                 return -EINVAL;
1484
1485         if (data_size)
1486                 *data_size = 0;
1487
1488         if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs)
1489                 return -EINVAL;
1490
1491         switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
1492         case VFIO_IRQ_SET_DATA_NONE:
1493                 size = 0;
1494                 break;
1495         case VFIO_IRQ_SET_DATA_BOOL:
1496                 size = sizeof(uint8_t);
1497                 break;
1498         case VFIO_IRQ_SET_DATA_EVENTFD:
1499                 size = sizeof(int32_t);
1500                 break;
1501         default:
1502                 return -EINVAL;
1503         }
1504
1505         if (size) {
1506                 if (hdr->argsz - minsz < hdr->count * size)
1507                         return -EINVAL;
1508
1509                 if (!data_size)
1510                         return -EINVAL;
1511
1512                 *data_size = hdr->count * size;
1513         }
1514
1515         return 0;
1516 }
1517 EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare);
1518
1519 /*
1520  * Pin contiguous user pages and return their associated host pages for local
1521  * domain only.
1522  * @device [in]  : device
1523  * @iova [in]    : starting IOVA of user pages to be pinned.
1524  * @npage [in]   : count of pages to be pinned.  This count should not
1525  *                 be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
1526  * @prot [in]    : protection flags
1527  * @pages[out]   : array of host pages
1528  * Return error or number of pages pinned.
1529  *
1530  * A driver may only call this function if the vfio_device was created
1531  * by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages().
1532  */
1533 int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova,
1534                    int npage, int prot, struct page **pages)
1535 {
1536         /* group->container cannot change while a vfio device is open */
1537         if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device)))
1538                 return -EINVAL;
1539         if (!device->ops->dma_unmap)
1540                 return -EINVAL;
1541         if (vfio_device_has_container(device))
1542                 return vfio_device_container_pin_pages(device, iova,
1543                                                        npage, prot, pages);
1544         if (device->iommufd_access) {
1545                 int ret;
1546
1547                 if (iova > ULONG_MAX)
1548                         return -EINVAL;
1549                 /*
1550                  * VFIO ignores the sub page offset, npages is from the start of
1551                  * a PAGE_SIZE chunk of IOVA. The caller is expected to recover
1552                  * the sub page offset by doing:
1553                  *     pages[0] + (iova % PAGE_SIZE)
1554                  */
1555                 ret = iommufd_access_pin_pages(
1556                         device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE),
1557                         npage * PAGE_SIZE, pages,
1558                         (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0);
1559                 if (ret)
1560                         return ret;
1561                 return npage;
1562         }
1563         return -EINVAL;
1564 }
1565 EXPORT_SYMBOL(vfio_pin_pages);
1566
1567 /*
1568  * Unpin contiguous host pages for local domain only.
1569  * @device [in]  : device
1570  * @iova [in]    : starting address of user pages to be unpinned.
1571  * @npage [in]   : count of pages to be unpinned.  This count should not
1572  *                 be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
1573  */
1574 void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage)
1575 {
1576         if (WARN_ON(!vfio_assert_device_open(device)))
1577                 return;
1578         if (WARN_ON(!device->ops->dma_unmap))
1579                 return;
1580
1581         if (vfio_device_has_container(device)) {
1582                 vfio_device_container_unpin_pages(device, iova, npage);
1583                 return;
1584         }
1585         if (device->iommufd_access) {
1586                 if (WARN_ON(iova > ULONG_MAX))
1587                         return;
1588                 iommufd_access_unpin_pages(device->iommufd_access,
1589                                            ALIGN_DOWN(iova, PAGE_SIZE),
1590                                            npage * PAGE_SIZE);
1591                 return;
1592         }
1593 }
1594 EXPORT_SYMBOL(vfio_unpin_pages);
1595
1596 /*
1597  * This interface allows the CPUs to perform some sort of virtual DMA on
1598  * behalf of the device.
1599  *
1600  * CPUs read/write from/into a range of IOVAs pointing to user space memory
1601  * into/from a kernel buffer.
1602  *
1603  * As the read/write of user space memory is conducted via the CPUs and is
1604  * not a real device DMA, it is not necessary to pin the user space memory.
1605  *
1606  * @device [in]         : VFIO device
1607  * @iova [in]           : base IOVA of a user space buffer
1608  * @data [in]           : pointer to kernel buffer
1609  * @len [in]            : kernel buffer length
1610  * @write               : indicate read or write
1611  * Return error code on failure or 0 on success.
1612  */
1613 int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data,
1614                 size_t len, bool write)
1615 {
1616         if (!data || len <= 0 || !vfio_assert_device_open(device))
1617                 return -EINVAL;
1618
1619         if (vfio_device_has_container(device))
1620                 return vfio_device_container_dma_rw(device, iova,
1621                                                     data, len, write);
1622
1623         if (device->iommufd_access) {
1624                 unsigned int flags = 0;
1625
1626                 if (iova > ULONG_MAX)
1627                         return -EINVAL;
1628
1629                 /* VFIO historically tries to auto-detect a kthread */
1630                 if (!current->mm)
1631                         flags |= IOMMUFD_ACCESS_RW_KTHREAD;
1632                 if (write)
1633                         flags |= IOMMUFD_ACCESS_RW_WRITE;
1634                 return iommufd_access_rw(device->iommufd_access, iova, data,
1635                                          len, flags);
1636         }
1637         return -EINVAL;
1638 }
1639 EXPORT_SYMBOL(vfio_dma_rw);
1640
1641 /*
1642  * Module/class support
1643  */
1644 static int __init vfio_init(void)
1645 {
1646         int ret;
1647
1648         ida_init(&vfio.device_ida);
1649
1650         ret = vfio_group_init();
1651         if (ret)
1652                 return ret;
1653
1654         ret = vfio_virqfd_init();
1655         if (ret)
1656                 goto err_virqfd;
1657
1658         /* /sys/class/vfio-dev/vfioX */
1659         vfio.device_class = class_create("vfio-dev");
1660         if (IS_ERR(vfio.device_class)) {
1661                 ret = PTR_ERR(vfio.device_class);
1662                 goto err_dev_class;
1663         }
1664
1665         ret = vfio_cdev_init(vfio.device_class);
1666         if (ret)
1667                 goto err_alloc_dev_chrdev;
1668
1669         pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
1670         return 0;
1671
1672 err_alloc_dev_chrdev:
1673         class_destroy(vfio.device_class);
1674         vfio.device_class = NULL;
1675 err_dev_class:
1676         vfio_virqfd_exit();
1677 err_virqfd:
1678         vfio_group_cleanup();
1679         return ret;
1680 }
1681
1682 static void __exit vfio_cleanup(void)
1683 {
1684         ida_destroy(&vfio.device_ida);
1685         vfio_cdev_cleanup();
1686         class_destroy(vfio.device_class);
1687         vfio.device_class = NULL;
1688         vfio_virqfd_exit();
1689         vfio_group_cleanup();
1690         xa_destroy(&vfio_device_set_xa);
1691 }
1692
1693 module_init(vfio_init);
1694 module_exit(vfio_cleanup);
1695
1696 MODULE_VERSION(DRIVER_VERSION);
1697 MODULE_LICENSE("GPL v2");
1698 MODULE_AUTHOR(DRIVER_AUTHOR);
1699 MODULE_DESCRIPTION(DRIVER_DESC);
1700 MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce");