drivers/gpu/drm/i915/gvt/kvmgt.c

   1 /*
   2  * KVMGT - the implementation of Intel mediated pass-through framework for KVM
   3  *
   4  * Copyright(c) 2014-2016 Intel Corporation. All rights reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23  * SOFTWARE.
  24  *
  25  * Authors:
  26  *    Kevin Tian <kevin.tian@intel.com>
  27  *    Jike Song <jike.song@intel.com>
  28  *    Xiaoguang Chen <xiaoguang.chen@intel.com>
  29  */
  30
  31 #include <linux/init.h>
  32 #include <linux/device.h>
  33 #include <linux/mm.h>
  34 #include <linux/mmu_context.h>
  35 #include <linux/types.h>
  36 #include <linux/list.h>
  37 #include <linux/rbtree.h>
  38 #include <linux/spinlock.h>
  39 #include <linux/eventfd.h>
  40 #include <linux/uuid.h>
  41 #include <linux/kvm_host.h>
  42 #include <linux/vfio.h>
  43 #include <linux/mdev.h>
  44 #include <linux/debugfs.h>
  45
  46 #include "i915_drv.h"
  47 #include "gvt.h"
  48
  49 static const struct intel_gvt_ops *intel_gvt_ops;
  50
  51 /* helper macros copied from vfio-pci */
  52 #define VFIO_PCI_OFFSET_SHIFT   40
  53 #define VFIO_PCI_OFFSET_TO_INDEX(off)   (off >> VFIO_PCI_OFFSET_SHIFT)
  54 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
  55 #define VFIO_PCI_OFFSET_MASK    (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
  56
  57 #define OPREGION_SIGNATURE "IntelGraphicsMem"
  58
  59 struct vfio_region;
  60 struct intel_vgpu_regops {
  61         size_t (*rw)(struct intel_vgpu *vgpu, char *buf,
  62                         size_t count, loff_t *ppos, bool iswrite);
  63         void (*release)(struct intel_vgpu *vgpu,
  64                         struct vfio_region *region);
  65 };
  66
  67 struct vfio_region {
  68         u32                             type;
  69         u32                             subtype;
  70         size_t                          size;
  71         u32                             flags;
  72         const struct intel_vgpu_regops  *ops;
  73         void                            *data;
  74 };
  75
  76 struct kvmgt_pgfn {
  77         gfn_t gfn;
  78         struct hlist_node hnode;
  79 };
  80
  81 struct kvmgt_guest_info {
  82         struct kvm *kvm;
  83         struct intel_vgpu *vgpu;
  84         struct kvm_page_track_notifier_node track_node;
  85 #define NR_BKT (1 << 18)
  86         struct hlist_head ptable[NR_BKT];
  87 #undef NR_BKT
  88         struct dentry *debugfs_cache_entries;
  89 };
  90
  91 struct gvt_dma {
  92         struct intel_vgpu *vgpu;
  93         struct rb_node gfn_node;
  94         struct rb_node dma_addr_node;
  95         gfn_t gfn;
  96         dma_addr_t dma_addr;
  97         unsigned long size;
  98         struct kref ref;
  99 };
 100
 101 static inline bool handle_valid(unsigned long handle)
 102 {
 103         return !!(handle & ~0xff);
 104 }
 105
 106 static int kvmgt_guest_init(struct mdev_device *mdev);
 107 static void intel_vgpu_release_work(struct work_struct *work);
 108 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info);
 109
 110 static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
 111                 unsigned long size)
 112 {
 113         int total_pages;
 114         int npage;
 115         int ret;
 116
 117         total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE;
 118
 119         for (npage = 0; npage < total_pages; npage++) {
 120                 unsigned long cur_gfn = gfn + npage;
 121
 122                 ret = vfio_unpin_pages(mdev_dev(vgpu->vdev.mdev), &cur_gfn, 1);
 123                 WARN_ON(ret != 1);
 124         }
 125 }
 126
 127 /* Pin a normal or compound guest page for dma. */
 128 static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
 129                 unsigned long size, struct page **page)
 130 {
 131         unsigned long base_pfn = 0;
 132         int total_pages;
 133         int npage;
 134         int ret;
 135
 136         total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE;
 137         /*
 138          * We pin the pages one-by-one to avoid allocating a big arrary
 139          * on stack to hold pfns.
 140          */
 141         for (npage = 0; npage < total_pages; npage++) {
 142                 unsigned long cur_gfn = gfn + npage;
 143                 unsigned long pfn;
 144
 145                 ret = vfio_pin_pages(mdev_dev(vgpu->vdev.mdev), &cur_gfn, 1,
 146                                      IOMMU_READ | IOMMU_WRITE, &pfn);
 147                 if (ret != 1) {
 148                         gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx, ret %d\n",
 149                                      cur_gfn, ret);
 150                         goto err;
 151                 }
 152
 153                 if (!pfn_valid(pfn)) {
 154                         gvt_vgpu_err("pfn 0x%lx is not mem backed\n", pfn);
 155                         npage++;
 156                         ret = -EFAULT;
 157                         goto err;
 158                 }
 159
 160                 if (npage == 0)
 161                         base_pfn = pfn;
 162                 else if (base_pfn + npage != pfn) {
 163                         gvt_vgpu_err("The pages are not continuous\n");
 164                         ret = -EINVAL;
 165                         npage++;
 166                         goto err;
 167                 }
 168         }
 169
 170         *page = pfn_to_page(base_pfn);
 171         return 0;
 172 err:
 173         gvt_unpin_guest_page(vgpu, gfn, npage * PAGE_SIZE);
 174         return ret;
 175 }
 176
 177 static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn,
 178                 dma_addr_t *dma_addr, unsigned long size)
 179 {
 180         struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
 181         struct page *page = NULL;
 182         int ret;
 183
 184         ret = gvt_pin_guest_page(vgpu, gfn, size, &page);
 185         if (ret)
 186                 return ret;
 187
 188         if (!pfn_valid(pfn)) {
 189                 gvt_vgpu_err("pfn 0x%lx is not mem backed\n", pfn);
 190                 vfio_unpin_pages(mdev_dev(vgpu->vdev.mdev), &gfn, 1);
 191                 return -EINVAL;
 192         }
 193
 194         /* Setup DMA mapping. */
 195         *dma_addr = dma_map_page(dev, page, 0, size, PCI_DMA_BIDIRECTIONAL);
 196         ret = dma_mapping_error(dev, *dma_addr);
 197         if (ret) {
 198                 gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n",
 199                              page_to_pfn(page), ret);
 200                 gvt_unpin_guest_page(vgpu, gfn, size);
 201         }
 202
 203         return ret;
 204 }
 205
 206 static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn,
 207                 dma_addr_t dma_addr, unsigned long size)
 208 {
 209         struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
 210
 211         dma_unmap_page(dev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
 212         gvt_unpin_guest_page(vgpu, gfn, size);
 213 }
 214
 215 static struct gvt_dma *__gvt_cache_find_dma_addr(struct intel_vgpu *vgpu,
 216                 dma_addr_t dma_addr)
 217 {
 218         struct rb_node *node = vgpu->vdev.dma_addr_cache.rb_node;
 219         struct gvt_dma *itr;
 220
 221         while (node) {
 222                 itr = rb_entry(node, struct gvt_dma, dma_addr_node);
 223
 224                 if (dma_addr < itr->dma_addr)
 225                         node = node->rb_left;
 226                 else if (dma_addr > itr->dma_addr)
 227                         node = node->rb_right;
 228                 else
 229                         return itr;
 230         }
 231         return NULL;
 232 }
 233
 234 static struct gvt_dma *__gvt_cache_find_gfn(struct intel_vgpu *vgpu, gfn_t gfn)
 235 {
 236         struct rb_node *node = vgpu->vdev.gfn_cache.rb_node;
 237         struct gvt_dma *itr;
 238
 239         while (node) {
 240                 itr = rb_entry(node, struct gvt_dma, gfn_node);
 241
 242                 if (gfn < itr->gfn)
 243                         node = node->rb_left;
 244                 else if (gfn > itr->gfn)
 245                         node = node->rb_right;
 246                 else
 247                         return itr;
 248         }
 249         return NULL;
 250 }
 251
 252 static int __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn,
 253                 dma_addr_t dma_addr, unsigned long size)
 254 {
 255         struct gvt_dma *new, *itr;
 256         struct rb_node **link, *parent = NULL;
 257
 258         new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL);
 259         if (!new)
 260                 return -ENOMEM;
 261
 262         new->vgpu = vgpu;
 263         new->gfn = gfn;
 264         new->dma_addr = dma_addr;
 265         new->size = size;
 266         kref_init(&new->ref);
 267
 268         /* gfn_cache maps gfn to struct gvt_dma. */
 269         link = &vgpu->vdev.gfn_cache.rb_node;
 270         while (*link) {
 271                 parent = *link;
 272                 itr = rb_entry(parent, struct gvt_dma, gfn_node);
 273
 274                 if (gfn < itr->gfn)
 275                         link = &parent->rb_left;
 276                 else
 277                         link = &parent->rb_right;
 278         }
 279         rb_link_node(&new->gfn_node, parent, link);
 280         rb_insert_color(&new->gfn_node, &vgpu->vdev.gfn_cache);
 281
 282         /* dma_addr_cache maps dma addr to struct gvt_dma. */
 283         parent = NULL;
 284         link = &vgpu->vdev.dma_addr_cache.rb_node;
 285         while (*link) {
 286                 parent = *link;
 287                 itr = rb_entry(parent, struct gvt_dma, dma_addr_node);
 288
 289                 if (dma_addr < itr->dma_addr)
 290                         link = &parent->rb_left;
 291                 else
 292                         link = &parent->rb_right;
 293         }
 294         rb_link_node(&new->dma_addr_node, parent, link);
 295         rb_insert_color(&new->dma_addr_node, &vgpu->vdev.dma_addr_cache);
 296
 297         vgpu->vdev.nr_cache_entries++;
 298         return 0;
 299 }
 300
 301 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu,
 302                                 struct gvt_dma *entry)
 303 {
 304         rb_erase(&entry->gfn_node, &vgpu->vdev.gfn_cache);
 305         rb_erase(&entry->dma_addr_node, &vgpu->vdev.dma_addr_cache);
 306         kfree(entry);
 307         vgpu->vdev.nr_cache_entries--;
 308 }
 309
 310 static void gvt_cache_destroy(struct intel_vgpu *vgpu)
 311 {
 312         struct gvt_dma *dma;
 313         struct rb_node *node = NULL;
 314
 315         for (;;) {
 316                 mutex_lock(&vgpu->vdev.cache_lock);
 317                 node = rb_first(&vgpu->vdev.gfn_cache);
 318                 if (!node) {
 319                         mutex_unlock(&vgpu->vdev.cache_lock);
 320                         break;
 321                 }
 322                 dma = rb_entry(node, struct gvt_dma, gfn_node);
 323                 gvt_dma_unmap_page(vgpu, dma->gfn, dma->dma_addr, dma->size);
 324                 __gvt_cache_remove_entry(vgpu, dma);
 325                 mutex_unlock(&vgpu->vdev.cache_lock);
 326         }
 327 }
 328
 329 static void gvt_cache_init(struct intel_vgpu *vgpu)
 330 {
 331         vgpu->vdev.gfn_cache = RB_ROOT;
 332         vgpu->vdev.dma_addr_cache = RB_ROOT;
 333         vgpu->vdev.nr_cache_entries = 0;
 334         mutex_init(&vgpu->vdev.cache_lock);
 335 }
 336
 337 static void kvmgt_protect_table_init(struct kvmgt_guest_info *info)
 338 {
 339         hash_init(info->ptable);
 340 }
 341
 342 static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info)
 343 {
 344         struct kvmgt_pgfn *p;
 345         struct hlist_node *tmp;
 346         int i;
 347
 348         hash_for_each_safe(info->ptable, i, tmp, p, hnode) {
 349                 hash_del(&p->hnode);
 350                 kfree(p);
 351         }
 352 }
 353
 354 static struct kvmgt_pgfn *
 355 __kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn)
 356 {
 357         struct kvmgt_pgfn *p, *res = NULL;
 358
 359         hash_for_each_possible(info->ptable, p, hnode, gfn) {
 360                 if (gfn == p->gfn) {
 361                         res = p;
 362                         break;
 363                 }
 364         }
 365
 366         return res;
 367 }
 368
 369 static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info,
 370                                 gfn_t gfn)
 371 {
 372         struct kvmgt_pgfn *p;
 373
 374         p = __kvmgt_protect_table_find(info, gfn);
 375         return !!p;
 376 }
 377
 378 static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn)
 379 {
 380         struct kvmgt_pgfn *p;
 381
 382         if (kvmgt_gfn_is_write_protected(info, gfn))
 383                 return;
 384
 385         p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC);
 386         if (WARN(!p, "gfn: 0x%llx\n", gfn))
 387                 return;
 388
 389         p->gfn = gfn;
 390         hash_add(info->ptable, &p->hnode, gfn);
 391 }
 392
 393 static void kvmgt_protect_table_del(struct kvmgt_guest_info *info,
 394                                 gfn_t gfn)
 395 {
 396         struct kvmgt_pgfn *p;
 397
 398         p = __kvmgt_protect_table_find(info, gfn);
 399         if (p) {
 400                 hash_del(&p->hnode);
 401                 kfree(p);
 402         }
 403 }
 404
 405 static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf,
 406                 size_t count, loff_t *ppos, bool iswrite)
 407 {
 408         unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -
 409                         VFIO_PCI_NUM_REGIONS;
 410         void *base = vgpu->vdev.region[i].data;
 411         loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
 412
 413         if (pos >= vgpu->vdev.region[i].size || iswrite) {
 414                 gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n");
 415                 return -EINVAL;
 416         }
 417         count = min(count, (size_t)(vgpu->vdev.region[i].size - pos));
 418         memcpy(buf, base + pos, count);
 419
 420         return count;
 421 }
 422
 423 static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu,
 424                 struct vfio_region *region)
 425 {
 426 }
 427
 428 static const struct intel_vgpu_regops intel_vgpu_regops_opregion = {
 429         .rw = intel_vgpu_reg_rw_opregion,
 430         .release = intel_vgpu_reg_release_opregion,
 431 };
 432
 433 static int intel_vgpu_register_reg(struct intel_vgpu *vgpu,
 434                 unsigned int type, unsigned int subtype,
 435                 const struct intel_vgpu_regops *ops,
 436                 size_t size, u32 flags, void *data)
 437 {
 438         struct vfio_region *region;
 439
 440         region = krealloc(vgpu->vdev.region,
 441                         (vgpu->vdev.num_regions + 1) * sizeof(*region),
 442                         GFP_KERNEL);
 443         if (!region)
 444                 return -ENOMEM;
 445
 446         vgpu->vdev.region = region;
 447         vgpu->vdev.region[vgpu->vdev.num_regions].type = type;
 448         vgpu->vdev.region[vgpu->vdev.num_regions].subtype = subtype;
 449         vgpu->vdev.region[vgpu->vdev.num_regions].ops = ops;
 450         vgpu->vdev.region[vgpu->vdev.num_regions].size = size;
 451         vgpu->vdev.region[vgpu->vdev.num_regions].flags = flags;
 452         vgpu->vdev.region[vgpu->vdev.num_regions].data = data;
 453         vgpu->vdev.num_regions++;
 454         return 0;
 455 }
 456
 457 static int kvmgt_get_vfio_device(void *p_vgpu)
 458 {
 459         struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
 460
 461         vgpu->vdev.vfio_device = vfio_device_get_from_dev(
 462                 mdev_dev(vgpu->vdev.mdev));
 463         if (!vgpu->vdev.vfio_device) {
 464                 gvt_vgpu_err("failed to get vfio device\n");
 465                 return -ENODEV;
 466         }
 467         return 0;
 468 }
 469
 470
 471 static int kvmgt_set_opregion(void *p_vgpu)
 472 {
 473         struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
 474         void *base;
 475         int ret;
 476
 477         /* Each vgpu has its own opregion, although VFIO would create another
 478          * one later. This one is used to expose opregion to VFIO. And the
 479          * other one created by VFIO later, is used by guest actually.
 480          */
 481         base = vgpu_opregion(vgpu)->va;
 482         if (!base)
 483                 return -ENOMEM;
 484
 485         if (memcmp(base, OPREGION_SIGNATURE, 16)) {
 486                 memunmap(base);
 487                 return -EINVAL;
 488         }
 489
 490         ret = intel_vgpu_register_reg(vgpu,
 491                         PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
 492                         VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION,
 493                         &intel_vgpu_regops_opregion, OPREGION_SIZE,
 494                         VFIO_REGION_INFO_FLAG_READ, base);
 495
 496         return ret;
 497 }
 498
 499 static void kvmgt_put_vfio_device(void *vgpu)
 500 {
 501         if (WARN_ON(!((struct intel_vgpu *)vgpu)->vdev.vfio_device))
 502                 return;
 503
 504         vfio_device_put(((struct intel_vgpu *)vgpu)->vdev.vfio_device);
 505 }
 506
 507 static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev)
 508 {
 509         struct intel_vgpu *vgpu = NULL;
 510         struct intel_vgpu_type *type;
 511         struct device *pdev;
 512         void *gvt;
 513         int ret;
 514
 515         pdev = mdev_parent_dev(mdev);
 516         gvt = kdev_to_i915(pdev)->gvt;
 517
 518         type = intel_gvt_ops->gvt_find_vgpu_type(gvt, kobject_name(kobj));
 519         if (!type) {
 520                 gvt_vgpu_err("failed to find type %s to create\n",
 521                                                 kobject_name(kobj));
 522                 ret = -EINVAL;
 523                 goto out;
 524         }
 525
 526         vgpu = intel_gvt_ops->vgpu_create(gvt, type);
 527         if (IS_ERR_OR_NULL(vgpu)) {
 528                 ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu);
 529                 gvt_err("failed to create intel vgpu: %d\n", ret);
 530                 goto out;
 531         }
 532
 533         INIT_WORK(&vgpu->vdev.release_work, intel_vgpu_release_work);
 534
 535         vgpu->vdev.mdev = mdev;
 536         mdev_set_drvdata(mdev, vgpu);
 537
 538         gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n",
 539                      dev_name(mdev_dev(mdev)));
 540         ret = 0;
 541
 542 out:
 543         return ret;
 544 }
 545
 546 static int intel_vgpu_remove(struct mdev_device *mdev)
 547 {
 548         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 549
 550         if (handle_valid(vgpu->handle))
 551                 return -EBUSY;
 552
 553         intel_gvt_ops->vgpu_destroy(vgpu);
 554         return 0;
 555 }
 556
 557 static int intel_vgpu_iommu_notifier(struct notifier_block *nb,
 558                                      unsigned long action, void *data)
 559 {
 560         struct intel_vgpu *vgpu = container_of(nb,
 561                                         struct intel_vgpu,
 562                                         vdev.iommu_notifier);
 563
 564         if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) {
 565                 struct vfio_iommu_type1_dma_unmap *unmap = data;
 566                 struct gvt_dma *entry;
 567                 unsigned long iov_pfn, end_iov_pfn;
 568
 569                 iov_pfn = unmap->iova >> PAGE_SHIFT;
 570                 end_iov_pfn = iov_pfn + unmap->size / PAGE_SIZE;
 571
 572                 mutex_lock(&vgpu->vdev.cache_lock);
 573                 for (; iov_pfn < end_iov_pfn; iov_pfn++) {
 574                         entry = __gvt_cache_find_gfn(vgpu, iov_pfn);
 575                         if (!entry)
 576                                 continue;
 577
 578                         gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr,
 579                                            entry->size);
 580                         __gvt_cache_remove_entry(vgpu, entry);
 581                 }
 582                 mutex_unlock(&vgpu->vdev.cache_lock);
 583         }
 584
 585         return NOTIFY_OK;
 586 }
 587
 588 static int intel_vgpu_group_notifier(struct notifier_block *nb,
 589                                      unsigned long action, void *data)
 590 {
 591         struct intel_vgpu *vgpu = container_of(nb,
 592                                         struct intel_vgpu,
 593                                         vdev.group_notifier);
 594
 595         /* the only action we care about */
 596         if (action == VFIO_GROUP_NOTIFY_SET_KVM) {
 597                 vgpu->vdev.kvm = data;
 598
 599                 if (!data)
 600                         schedule_work(&vgpu->vdev.release_work);
 601         }
 602
 603         return NOTIFY_OK;
 604 }
 605
 606 static int intel_vgpu_open(struct mdev_device *mdev)
 607 {
 608         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 609         unsigned long events;
 610         int ret;
 611
 612         vgpu->vdev.iommu_notifier.notifier_call = intel_vgpu_iommu_notifier;
 613         vgpu->vdev.group_notifier.notifier_call = intel_vgpu_group_notifier;
 614
 615         events = VFIO_IOMMU_NOTIFY_DMA_UNMAP;
 616         ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, &events,
 617                                 &vgpu->vdev.iommu_notifier);
 618         if (ret != 0) {
 619                 gvt_vgpu_err("vfio_register_notifier for iommu failed: %d\n",
 620                         ret);
 621                 goto out;
 622         }
 623
 624         events = VFIO_GROUP_NOTIFY_SET_KVM;
 625         ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, &events,
 626                                 &vgpu->vdev.group_notifier);
 627         if (ret != 0) {
 628                 gvt_vgpu_err("vfio_register_notifier for group failed: %d\n",
 629                         ret);
 630                 goto undo_iommu;
 631         }
 632
 633         ret = kvmgt_guest_init(mdev);
 634         if (ret)
 635                 goto undo_group;
 636
 637         intel_gvt_ops->vgpu_activate(vgpu);
 638
 639         atomic_set(&vgpu->vdev.released, 0);
 640         return ret;
 641
 642 undo_group:
 643         vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
 644                                         &vgpu->vdev.group_notifier);
 645
 646 undo_iommu:
 647         vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
 648                                         &vgpu->vdev.iommu_notifier);
 649 out:
 650         return ret;
 651 }
 652
 653 static void intel_vgpu_release_msi_eventfd_ctx(struct intel_vgpu *vgpu)
 654 {
 655         struct eventfd_ctx *trigger;
 656
 657         trigger = vgpu->vdev.msi_trigger;
 658         if (trigger) {
 659                 eventfd_ctx_put(trigger);
 660                 vgpu->vdev.msi_trigger = NULL;
 661         }
 662 }
 663
 664 static void __intel_vgpu_release(struct intel_vgpu *vgpu)
 665 {
 666         struct kvmgt_guest_info *info;
 667         int ret;
 668
 669         if (!handle_valid(vgpu->handle))
 670                 return;
 671
 672         if (atomic_cmpxchg(&vgpu->vdev.released, 0, 1))
 673                 return;
 674
 675         intel_gvt_ops->vgpu_deactivate(vgpu);
 676
 677         ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_IOMMU_NOTIFY,
 678                                         &vgpu->vdev.iommu_notifier);
 679         WARN(ret, "vfio_unregister_notifier for iommu failed: %d\n", ret);
 680
 681         ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_GROUP_NOTIFY,
 682                                         &vgpu->vdev.group_notifier);
 683         WARN(ret, "vfio_unregister_notifier for group failed: %d\n", ret);
 684
 685         info = (struct kvmgt_guest_info *)vgpu->handle;
 686         kvmgt_guest_exit(info);
 687
 688         intel_vgpu_release_msi_eventfd_ctx(vgpu);
 689
 690         vgpu->vdev.kvm = NULL;
 691         vgpu->handle = 0;
 692 }
 693
 694 static void intel_vgpu_release(struct mdev_device *mdev)
 695 {
 696         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 697
 698         __intel_vgpu_release(vgpu);
 699 }
 700
 701 static void intel_vgpu_release_work(struct work_struct *work)
 702 {
 703         struct intel_vgpu *vgpu = container_of(work, struct intel_vgpu,
 704                                         vdev.release_work);
 705
 706         __intel_vgpu_release(vgpu);
 707 }
 708
 709 static uint64_t intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar)
 710 {
 711         u32 start_lo, start_hi;
 712         u32 mem_type;
 713
 714         start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
 715                         PCI_BASE_ADDRESS_MEM_MASK;
 716         mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
 717                         PCI_BASE_ADDRESS_MEM_TYPE_MASK;
 718
 719         switch (mem_type) {
 720         case PCI_BASE_ADDRESS_MEM_TYPE_64:
 721                 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space
 722                                                 + bar + 4));
 723                 break;
 724         case PCI_BASE_ADDRESS_MEM_TYPE_32:
 725         case PCI_BASE_ADDRESS_MEM_TYPE_1M:
 726                 /* 1M mem BAR treated as 32-bit BAR */
 727         default:
 728                 /* mem unknown type treated as 32-bit BAR */
 729                 start_hi = 0;
 730                 break;
 731         }
 732
 733         return ((u64)start_hi << 32) | start_lo;
 734 }
 735
 736 static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, uint64_t off,
 737                              void *buf, unsigned int count, bool is_write)
 738 {
 739         uint64_t bar_start = intel_vgpu_get_bar_addr(vgpu, bar);
 740         int ret;
 741
 742         if (is_write)
 743                 ret = intel_gvt_ops->emulate_mmio_write(vgpu,
 744                                         bar_start + off, buf, count);
 745         else
 746                 ret = intel_gvt_ops->emulate_mmio_read(vgpu,
 747                                         bar_start + off, buf, count);
 748         return ret;
 749 }
 750
 751 static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, uint64_t off)
 752 {
 753         return off >= vgpu_aperture_offset(vgpu) &&
 754                off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu);
 755 }
 756
 757 static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t off,
 758                 void *buf, unsigned long count, bool is_write)
 759 {
 760         void *aperture_va;
 761
 762         if (!intel_vgpu_in_aperture(vgpu, off) ||
 763             !intel_vgpu_in_aperture(vgpu, off + count)) {
 764                 gvt_vgpu_err("Invalid aperture offset %llu\n", off);
 765                 return -EINVAL;
 766         }
 767
 768         aperture_va = io_mapping_map_wc(&vgpu->gvt->dev_priv->ggtt.iomap,
 769                                         ALIGN_DOWN(off, PAGE_SIZE),
 770                                         count + offset_in_page(off));
 771         if (!aperture_va)
 772                 return -EIO;
 773
 774         if (is_write)
 775                 memcpy(aperture_va + offset_in_page(off), buf, count);
 776         else
 777                 memcpy(buf, aperture_va + offset_in_page(off), count);
 778
 779         io_mapping_unmap(aperture_va);
 780
 781         return 0;
 782 }
 783
 784 static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf,
 785                         size_t count, loff_t *ppos, bool is_write)
 786 {
 787         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 788         unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 789         uint64_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
 790         int ret = -EINVAL;
 791
 792
 793         if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) {
 794                 gvt_vgpu_err("invalid index: %u\n", index);
 795                 return -EINVAL;
 796         }
 797
 798         switch (index) {
 799         case VFIO_PCI_CONFIG_REGION_INDEX:
 800                 if (is_write)
 801                         ret = intel_gvt_ops->emulate_cfg_write(vgpu, pos,
 802                                                 buf, count);
 803                 else
 804                         ret = intel_gvt_ops->emulate_cfg_read(vgpu, pos,
 805                                                 buf, count);
 806                 break;
 807         case VFIO_PCI_BAR0_REGION_INDEX:
 808                 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos,
 809                                         buf, count, is_write);
 810                 break;
 811         case VFIO_PCI_BAR2_REGION_INDEX:
 812                 ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write);
 813                 break;
 814         case VFIO_PCI_BAR1_REGION_INDEX:
 815         case VFIO_PCI_BAR3_REGION_INDEX:
 816         case VFIO_PCI_BAR4_REGION_INDEX:
 817         case VFIO_PCI_BAR5_REGION_INDEX:
 818         case VFIO_PCI_VGA_REGION_INDEX:
 819         case VFIO_PCI_ROM_REGION_INDEX:
 820                 break;
 821         default:
 822                 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions)
 823                         return -EINVAL;
 824
 825                 index -= VFIO_PCI_NUM_REGIONS;
 826                 return vgpu->vdev.region[index].ops->rw(vgpu, buf, count,
 827                                 ppos, is_write);
 828         }
 829
 830         return ret == 0 ? count : ret;
 831 }
 832
 833 static bool gtt_entry(struct mdev_device *mdev, loff_t *ppos)
 834 {
 835         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 836         unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 837         struct intel_gvt *gvt = vgpu->gvt;
 838         int offset;
 839
 840         /* Only allow MMIO GGTT entry access */
 841         if (index != PCI_BASE_ADDRESS_0)
 842                 return false;
 843
 844         offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) -
 845                 intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0);
 846
 847         return (offset >= gvt->device_info.gtt_start_offset &&
 848                 offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ?
 849                         true : false;
 850 }
 851
 852 static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf,
 853                         size_t count, loff_t *ppos)
 854 {
 855         unsigned int done = 0;
 856         int ret;
 857
 858         while (count) {
 859                 size_t filled;
 860
 861                 /* Only support GGTT entry 8 bytes read */
 862                 if (count >= 8 && !(*ppos % 8) &&
 863                         gtt_entry(mdev, ppos)) {
 864                         u64 val;
 865
 866                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 867                                         ppos, false);
 868                         if (ret <= 0)
 869                                 goto read_err;
 870
 871                         if (copy_to_user(buf, &val, sizeof(val)))
 872                                 goto read_err;
 873
 874                         filled = 8;
 875                 } else if (count >= 4 && !(*ppos % 4)) {
 876                         u32 val;
 877
 878                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 879                                         ppos, false);
 880                         if (ret <= 0)
 881                                 goto read_err;
 882
 883                         if (copy_to_user(buf, &val, sizeof(val)))
 884                                 goto read_err;
 885
 886                         filled = 4;
 887                 } else if (count >= 2 && !(*ppos % 2)) {
 888                         u16 val;
 889
 890                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 891                                         ppos, false);
 892                         if (ret <= 0)
 893                                 goto read_err;
 894
 895                         if (copy_to_user(buf, &val, sizeof(val)))
 896                                 goto read_err;
 897
 898                         filled = 2;
 899                 } else {
 900                         u8 val;
 901
 902                         ret = intel_vgpu_rw(mdev, &val, sizeof(val), ppos,
 903                                         false);
 904                         if (ret <= 0)
 905                                 goto read_err;
 906
 907                         if (copy_to_user(buf, &val, sizeof(val)))
 908                                 goto read_err;
 909
 910                         filled = 1;
 911                 }
 912
 913                 count -= filled;
 914                 done += filled;
 915                 *ppos += filled;
 916                 buf += filled;
 917         }
 918
 919         return done;
 920
 921 read_err:
 922         return -EFAULT;
 923 }
 924
 925 static ssize_t intel_vgpu_write(struct mdev_device *mdev,
 926                                 const char __user *buf,
 927                                 size_t count, loff_t *ppos)
 928 {
 929         unsigned int done = 0;
 930         int ret;
 931
 932         while (count) {
 933                 size_t filled;
 934
 935                 /* Only support GGTT entry 8 bytes write */
 936                 if (count >= 8 && !(*ppos % 8) &&
 937                         gtt_entry(mdev, ppos)) {
 938                         u64 val;
 939
 940                         if (copy_from_user(&val, buf, sizeof(val)))
 941                                 goto write_err;
 942
 943                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 944                                         ppos, true);
 945                         if (ret <= 0)
 946                                 goto write_err;
 947
 948                         filled = 8;
 949                 } else if (count >= 4 && !(*ppos % 4)) {
 950                         u32 val;
 951
 952                         if (copy_from_user(&val, buf, sizeof(val)))
 953                                 goto write_err;
 954
 955                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 956                                         ppos, true);
 957                         if (ret <= 0)
 958                                 goto write_err;
 959
 960                         filled = 4;
 961                 } else if (count >= 2 && !(*ppos % 2)) {
 962                         u16 val;
 963
 964                         if (copy_from_user(&val, buf, sizeof(val)))
 965                                 goto write_err;
 966
 967                         ret = intel_vgpu_rw(mdev, (char *)&val,
 968                                         sizeof(val), ppos, true);
 969                         if (ret <= 0)
 970                                 goto write_err;
 971
 972                         filled = 2;
 973                 } else {
 974                         u8 val;
 975
 976                         if (copy_from_user(&val, buf, sizeof(val)))
 977                                 goto write_err;
 978
 979                         ret = intel_vgpu_rw(mdev, &val, sizeof(val),
 980                                         ppos, true);
 981                         if (ret <= 0)
 982                                 goto write_err;
 983
 984                         filled = 1;
 985                 }
 986
 987                 count -= filled;
 988                 done += filled;
 989                 *ppos += filled;
 990                 buf += filled;
 991         }
 992
 993         return done;
 994 write_err:
 995         return -EFAULT;
 996 }
 997
 998 static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma)
 999 {
1000         unsigned int index;
1001         u64 virtaddr;
1002         unsigned long req_size, pgoff = 0;
1003         pgprot_t pg_prot;
1004         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
1005
1006         index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1007         if (index >= VFIO_PCI_ROM_REGION_INDEX)
1008                 return -EINVAL;
1009
1010         if (vma->vm_end < vma->vm_start)
1011                 return -EINVAL;
1012         if ((vma->vm_flags & VM_SHARED) == 0)
1013                 return -EINVAL;
1014         if (index != VFIO_PCI_BAR2_REGION_INDEX)
1015                 return -EINVAL;
1016
1017         pg_prot = vma->vm_page_prot;
1018         virtaddr = vma->vm_start;
1019         req_size = vma->vm_end - vma->vm_start;
1020         pgoff = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT;
1021
1022         return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot);
1023 }
1024
1025 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type)
1026 {
1027         if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX)
1028                 return 1;
1029
1030         return 0;
1031 }
1032
1033 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu,
1034                         unsigned int index, unsigned int start,
1035                         unsigned int count, uint32_t flags,
1036                         void *data)
1037 {
1038         return 0;
1039 }
1040
1041 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu,
1042                         unsigned int index, unsigned int start,
1043                         unsigned int count, uint32_t flags, void *data)
1044 {
1045         return 0;
1046 }
1047
1048 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu,
1049                 unsigned int index, unsigned int start, unsigned int count,
1050                 uint32_t flags, void *data)
1051 {
1052         return 0;
1053 }
1054
1055 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu,
1056                 unsigned int index, unsigned int start, unsigned int count,
1057                 uint32_t flags, void *data)
1058 {
1059         struct eventfd_ctx *trigger;
1060
1061         if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
1062                 int fd = *(int *)data;
1063
1064                 trigger = eventfd_ctx_fdget(fd);
1065                 if (IS_ERR(trigger)) {
1066                         gvt_vgpu_err("eventfd_ctx_fdget failed\n");
1067                         return PTR_ERR(trigger);
1068                 }
1069                 vgpu->vdev.msi_trigger = trigger;
1070         } else if ((flags & VFIO_IRQ_SET_DATA_NONE) && !count)
1071                 intel_vgpu_release_msi_eventfd_ctx(vgpu);
1072
1073         return 0;
1074 }
1075
1076 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, uint32_t flags,
1077                 unsigned int index, unsigned int start, unsigned int count,
1078                 void *data)
1079 {
1080         int (*func)(struct intel_vgpu *vgpu, unsigned int index,
1081                         unsigned int start, unsigned int count, uint32_t flags,
1082                         void *data) = NULL;
1083
1084         switch (index) {
1085         case VFIO_PCI_INTX_IRQ_INDEX:
1086                 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1087                 case VFIO_IRQ_SET_ACTION_MASK:
1088                         func = intel_vgpu_set_intx_mask;
1089                         break;
1090                 case VFIO_IRQ_SET_ACTION_UNMASK:
1091                         func = intel_vgpu_set_intx_unmask;
1092                         break;
1093                 case VFIO_IRQ_SET_ACTION_TRIGGER:
1094                         func = intel_vgpu_set_intx_trigger;
1095                         break;
1096                 }
1097                 break;
1098         case VFIO_PCI_MSI_IRQ_INDEX:
1099                 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1100                 case VFIO_IRQ_SET_ACTION_MASK:
1101                 case VFIO_IRQ_SET_ACTION_UNMASK:
1102                         /* XXX Need masking support exported */
1103                         break;
1104                 case VFIO_IRQ_SET_ACTION_TRIGGER:
1105                         func = intel_vgpu_set_msi_trigger;
1106                         break;
1107                 }
1108                 break;
1109         }
1110
1111         if (!func)
1112                 return -ENOTTY;
1113
1114         return func(vgpu, index, start, count, flags, data);
1115 }
1116
1117 static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
1118                              unsigned long arg)
1119 {
1120         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
1121         unsigned long minsz;
1122
1123         gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd);
1124
1125         if (cmd == VFIO_DEVICE_GET_INFO) {
1126                 struct vfio_device_info info;
1127
1128                 minsz = offsetofend(struct vfio_device_info, num_irqs);
1129
1130                 if (copy_from_user(&info, (void __user *)arg, minsz))
1131                         return -EFAULT;
1132
1133                 if (info.argsz < minsz)
1134                         return -EINVAL;
1135
1136                 info.flags = VFIO_DEVICE_FLAGS_PCI;
1137                 info.flags |= VFIO_DEVICE_FLAGS_RESET;
1138                 info.num_regions = VFIO_PCI_NUM_REGIONS +
1139                                 vgpu->vdev.num_regions;
1140                 info.num_irqs = VFIO_PCI_NUM_IRQS;
1141
1142                 return copy_to_user((void __user *)arg, &info, minsz) ?
1143                         -EFAULT : 0;
1144
1145         } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
1146                 struct vfio_region_info info;
1147                 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
1148                 int i, ret;
1149                 struct vfio_region_info_cap_sparse_mmap *sparse = NULL;
1150                 size_t size;
1151                 int nr_areas = 1;
1152                 int cap_type_id;
1153
1154                 minsz = offsetofend(struct vfio_region_info, offset);
1155
1156                 if (copy_from_user(&info, (void __user *)arg, minsz))
1157                         return -EFAULT;
1158
1159                 if (info.argsz < minsz)
1160                         return -EINVAL;
1161
1162                 switch (info.index) {
1163                 case VFIO_PCI_CONFIG_REGION_INDEX:
1164                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1165                         info.size = vgpu->gvt->device_info.cfg_space_size;
1166                         info.flags = VFIO_REGION_INFO_FLAG_READ |
1167                                      VFIO_REGION_INFO_FLAG_WRITE;
1168                         break;
1169                 case VFIO_PCI_BAR0_REGION_INDEX:
1170                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1171                         info.size = vgpu->cfg_space.bar[info.index].size;
1172                         if (!info.size) {
1173                                 info.flags = 0;
1174                                 break;
1175                         }
1176
1177                         info.flags = VFIO_REGION_INFO_FLAG_READ |
1178                                      VFIO_REGION_INFO_FLAG_WRITE;
1179                         break;
1180                 case VFIO_PCI_BAR1_REGION_INDEX:
1181                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1182                         info.size = 0;
1183                         info.flags = 0;
1184                         break;
1185                 case VFIO_PCI_BAR2_REGION_INDEX:
1186                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1187                         info.flags = VFIO_REGION_INFO_FLAG_CAPS |
1188                                         VFIO_REGION_INFO_FLAG_MMAP |
1189                                         VFIO_REGION_INFO_FLAG_READ |
1190                                         VFIO_REGION_INFO_FLAG_WRITE;
1191                         info.size = gvt_aperture_sz(vgpu->gvt);
1192
1193                         size = sizeof(*sparse) +
1194                                         (nr_areas * sizeof(*sparse->areas));
1195                         sparse = kzalloc(size, GFP_KERNEL);
1196                         if (!sparse)
1197                                 return -ENOMEM;
1198
1199                         sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1200                         sparse->header.version = 1;
1201                         sparse->nr_areas = nr_areas;
1202                         cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1203                         sparse->areas[0].offset =
1204                                         PAGE_ALIGN(vgpu_aperture_offset(vgpu));
1205                         sparse->areas[0].size = vgpu_aperture_sz(vgpu);
1206                         break;
1207
1208                 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1209                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1210                         info.size = 0;
1211                         info.flags = 0;
1212
1213                         gvt_dbg_core("get region info bar:%d\n", info.index);
1214                         break;
1215
1216                 case VFIO_PCI_ROM_REGION_INDEX:
1217                 case VFIO_PCI_VGA_REGION_INDEX:
1218                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1219                         info.size = 0;
1220                         info.flags = 0;
1221
1222                         gvt_dbg_core("get region info index:%d\n", info.index);
1223                         break;
1224                 default:
1225                         {
1226                                 struct vfio_region_info_cap_type cap_type = {
1227                                         .header.id = VFIO_REGION_INFO_CAP_TYPE,
1228                                         .header.version = 1 };
1229
1230                                 if (info.index >= VFIO_PCI_NUM_REGIONS +
1231                                                 vgpu->vdev.num_regions)
1232                                         return -EINVAL;
1233
1234                                 i = info.index - VFIO_PCI_NUM_REGIONS;
1235
1236                                 info.offset =
1237                                         VFIO_PCI_INDEX_TO_OFFSET(info.index);
1238                                 info.size = vgpu->vdev.region[i].size;
1239                                 info.flags = vgpu->vdev.region[i].flags;
1240
1241                                 cap_type.type = vgpu->vdev.region[i].type;
1242                                 cap_type.subtype = vgpu->vdev.region[i].subtype;
1243
1244                                 ret = vfio_info_add_capability(&caps,
1245                                                         &cap_type.header,
1246                                                         sizeof(cap_type));
1247                                 if (ret)
1248                                         return ret;
1249                         }
1250                 }
1251
1252                 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) {
1253                         switch (cap_type_id) {
1254                         case VFIO_REGION_INFO_CAP_SPARSE_MMAP:
1255                                 ret = vfio_info_add_capability(&caps,
1256                                         &sparse->header, sizeof(*sparse) +
1257                                         (sparse->nr_areas *
1258                                                 sizeof(*sparse->areas)));
1259                                 kfree(sparse);
1260                                 if (ret)
1261                                         return ret;
1262                                 break;
1263                         default:
1264                                 return -EINVAL;
1265                         }
1266                 }
1267
1268                 if (caps.size) {
1269                         info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
1270                         if (info.argsz < sizeof(info) + caps.size) {
1271                                 info.argsz = sizeof(info) + caps.size;
1272                                 info.cap_offset = 0;
1273                         } else {
1274                                 vfio_info_cap_shift(&caps, sizeof(info));
1275                                 if (copy_to_user((void __user *)arg +
1276                                                   sizeof(info), caps.buf,
1277                                                   caps.size)) {
1278                                         kfree(caps.buf);
1279                                         return -EFAULT;
1280                                 }
1281                                 info.cap_offset = sizeof(info);
1282                         }
1283
1284                         kfree(caps.buf);
1285                 }
1286
1287                 return copy_to_user((void __user *)arg, &info, minsz) ?
1288                         -EFAULT : 0;
1289         } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
1290                 struct vfio_irq_info info;
1291
1292                 minsz = offsetofend(struct vfio_irq_info, count);
1293
1294                 if (copy_from_user(&info, (void __user *)arg, minsz))
1295                         return -EFAULT;
1296
1297                 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
1298                         return -EINVAL;
1299
1300                 switch (info.index) {
1301                 case VFIO_PCI_INTX_IRQ_INDEX:
1302                 case VFIO_PCI_MSI_IRQ_INDEX:
1303                         break;
1304                 default:
1305                         return -EINVAL;
1306                 }
1307
1308                 info.flags = VFIO_IRQ_INFO_EVENTFD;
1309
1310                 info.count = intel_vgpu_get_irq_count(vgpu, info.index);
1311
1312                 if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
1313                         info.flags |= (VFIO_IRQ_INFO_MASKABLE |
1314                                        VFIO_IRQ_INFO_AUTOMASKED);
1315                 else
1316                         info.flags |= VFIO_IRQ_INFO_NORESIZE;
1317
1318                 return copy_to_user((void __user *)arg, &info, minsz) ?
1319                         -EFAULT : 0;
1320         } else if (cmd == VFIO_DEVICE_SET_IRQS) {
1321                 struct vfio_irq_set hdr;
1322                 u8 *data = NULL;
1323                 int ret = 0;
1324                 size_t data_size = 0;
1325
1326                 minsz = offsetofend(struct vfio_irq_set, count);
1327
1328                 if (copy_from_user(&hdr, (void __user *)arg, minsz))
1329                         return -EFAULT;
1330
1331                 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) {
1332                         int max = intel_vgpu_get_irq_count(vgpu, hdr.index);
1333
1334                         ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
1335                                                 VFIO_PCI_NUM_IRQS, &data_size);
1336                         if (ret) {
1337                                 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n");
1338                                 return -EINVAL;
1339                         }
1340                         if (data_size) {
1341                                 data = memdup_user((void __user *)(arg + minsz),
1342                                                    data_size);
1343                                 if (IS_ERR(data))
1344                                         return PTR_ERR(data);
1345                         }
1346                 }
1347
1348                 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index,
1349                                         hdr.start, hdr.count, data);
1350                 kfree(data);
1351
1352                 return ret;
1353         } else if (cmd == VFIO_DEVICE_RESET) {
1354                 intel_gvt_ops->vgpu_reset(vgpu);
1355                 return 0;
1356         } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) {
1357                 struct vfio_device_gfx_plane_info dmabuf;
1358                 int ret = 0;
1359
1360                 minsz = offsetofend(struct vfio_device_gfx_plane_info,
1361                                     dmabuf_id);
1362                 if (copy_from_user(&dmabuf, (void __user *)arg, minsz))
1363                         return -EFAULT;
1364                 if (dmabuf.argsz < minsz)
1365                         return -EINVAL;
1366
1367                 ret = intel_gvt_ops->vgpu_query_plane(vgpu, &dmabuf);
1368                 if (ret != 0)
1369                         return ret;
1370
1371                 return copy_to_user((void __user *)arg, &dmabuf, minsz) ?
1372                                                                 -EFAULT : 0;
1373         } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) {
1374                 __u32 dmabuf_id;
1375                 __s32 dmabuf_fd;
1376
1377                 if (get_user(dmabuf_id, (__u32 __user *)arg))
1378                         return -EFAULT;
1379
1380                 dmabuf_fd = intel_gvt_ops->vgpu_get_dmabuf(vgpu, dmabuf_id);
1381                 return dmabuf_fd;
1382
1383         }
1384
1385         return -ENOTTY;
1386 }
1387
1388 static ssize_t
1389 vgpu_id_show(struct device *dev, struct device_attribute *attr,
1390              char *buf)
1391 {
1392         struct mdev_device *mdev = mdev_from_dev(dev);
1393
1394         if (mdev) {
1395                 struct intel_vgpu *vgpu = (struct intel_vgpu *)
1396                         mdev_get_drvdata(mdev);
1397                 return sprintf(buf, "%d\n", vgpu->id);
1398         }
1399         return sprintf(buf, "\n");
1400 }
1401
1402 static ssize_t
1403 hw_id_show(struct device *dev, struct device_attribute *attr,
1404            char *buf)
1405 {
1406         struct mdev_device *mdev = mdev_from_dev(dev);
1407
1408         if (mdev) {
1409                 struct intel_vgpu *vgpu = (struct intel_vgpu *)
1410                         mdev_get_drvdata(mdev);
1411                 return sprintf(buf, "%u\n",
1412                                vgpu->submission.shadow_ctx->hw_id);
1413         }
1414         return sprintf(buf, "\n");
1415 }
1416
1417 static DEVICE_ATTR_RO(vgpu_id);
1418 static DEVICE_ATTR_RO(hw_id);
1419
1420 static struct attribute *intel_vgpu_attrs[] = {
1421         &dev_attr_vgpu_id.attr,
1422         &dev_attr_hw_id.attr,
1423         NULL
1424 };
1425
1426 static const struct attribute_group intel_vgpu_group = {
1427         .name = "intel_vgpu",
1428         .attrs = intel_vgpu_attrs,
1429 };
1430
1431 static const struct attribute_group *intel_vgpu_groups[] = {
1432         &intel_vgpu_group,
1433         NULL,
1434 };
1435
1436 static struct mdev_parent_ops intel_vgpu_ops = {
1437         .mdev_attr_groups       = intel_vgpu_groups,
1438         .create                 = intel_vgpu_create,
1439         .remove                 = intel_vgpu_remove,
1440
1441         .open                   = intel_vgpu_open,
1442         .release                = intel_vgpu_release,
1443
1444         .read                   = intel_vgpu_read,
1445         .write                  = intel_vgpu_write,
1446         .mmap                   = intel_vgpu_mmap,
1447         .ioctl                  = intel_vgpu_ioctl,
1448 };
1449
1450 static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops)
1451 {
1452         struct attribute **kvm_type_attrs;
1453         struct attribute_group **kvm_vgpu_type_groups;
1454
1455         intel_gvt_ops = ops;
1456         if (!intel_gvt_ops->get_gvt_attrs(&kvm_type_attrs,
1457                         &kvm_vgpu_type_groups))
1458                 return -EFAULT;
1459         intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups;
1460
1461         return mdev_register_device(dev, &intel_vgpu_ops);
1462 }
1463
1464 static void kvmgt_host_exit(struct device *dev, void *gvt)
1465 {
1466         mdev_unregister_device(dev);
1467 }
1468
1469 static int kvmgt_page_track_add(unsigned long handle, u64 gfn)
1470 {
1471         struct kvmgt_guest_info *info;
1472         struct kvm *kvm;
1473         struct kvm_memory_slot *slot;
1474         int idx;
1475
1476         if (!handle_valid(handle))
1477                 return -ESRCH;
1478
1479         info = (struct kvmgt_guest_info *)handle;
1480         kvm = info->kvm;
1481
1482         idx = srcu_read_lock(&kvm->srcu);
1483         slot = gfn_to_memslot(kvm, gfn);
1484         if (!slot) {
1485                 srcu_read_unlock(&kvm->srcu, idx);
1486                 return -EINVAL;
1487         }
1488
1489         spin_lock(&kvm->mmu_lock);
1490
1491         if (kvmgt_gfn_is_write_protected(info, gfn))
1492                 goto out;
1493
1494         kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
1495         kvmgt_protect_table_add(info, gfn);
1496
1497 out:
1498         spin_unlock(&kvm->mmu_lock);
1499         srcu_read_unlock(&kvm->srcu, idx);
1500         return 0;
1501 }
1502
1503 static int kvmgt_page_track_remove(unsigned long handle, u64 gfn)
1504 {
1505         struct kvmgt_guest_info *info;
1506         struct kvm *kvm;
1507         struct kvm_memory_slot *slot;
1508         int idx;
1509
1510         if (!handle_valid(handle))
1511                 return 0;
1512
1513         info = (struct kvmgt_guest_info *)handle;
1514         kvm = info->kvm;
1515
1516         idx = srcu_read_lock(&kvm->srcu);
1517         slot = gfn_to_memslot(kvm, gfn);
1518         if (!slot) {
1519                 srcu_read_unlock(&kvm->srcu, idx);
1520                 return -EINVAL;
1521         }
1522
1523         spin_lock(&kvm->mmu_lock);
1524
1525         if (!kvmgt_gfn_is_write_protected(info, gfn))
1526                 goto out;
1527
1528         kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
1529         kvmgt_protect_table_del(info, gfn);
1530
1531 out:
1532         spin_unlock(&kvm->mmu_lock);
1533         srcu_read_unlock(&kvm->srcu, idx);
1534         return 0;
1535 }
1536
1537 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1538                 const u8 *val, int len,
1539                 struct kvm_page_track_notifier_node *node)
1540 {
1541         struct kvmgt_guest_info *info = container_of(node,
1542                                         struct kvmgt_guest_info, track_node);
1543
1544         if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa)))
1545                 intel_gvt_ops->write_protect_handler(info->vgpu, gpa,
1546                                                      (void *)val, len);
1547 }
1548
1549 static void kvmgt_page_track_flush_slot(struct kvm *kvm,
1550                 struct kvm_memory_slot *slot,
1551                 struct kvm_page_track_notifier_node *node)
1552 {
1553         int i;
1554         gfn_t gfn;
1555         struct kvmgt_guest_info *info = container_of(node,
1556                                         struct kvmgt_guest_info, track_node);
1557
1558         spin_lock(&kvm->mmu_lock);
1559         for (i = 0; i < slot->npages; i++) {
1560                 gfn = slot->base_gfn + i;
1561                 if (kvmgt_gfn_is_write_protected(info, gfn)) {
1562                         kvm_slot_page_track_remove_page(kvm, slot, gfn,
1563                                                 KVM_PAGE_TRACK_WRITE);
1564                         kvmgt_protect_table_del(info, gfn);
1565                 }
1566         }
1567         spin_unlock(&kvm->mmu_lock);
1568 }
1569
1570 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu, struct kvm *kvm)
1571 {
1572         struct intel_vgpu *itr;
1573         struct kvmgt_guest_info *info;
1574         int id;
1575         bool ret = false;
1576
1577         mutex_lock(&vgpu->gvt->lock);
1578         for_each_active_vgpu(vgpu->gvt, itr, id) {
1579                 if (!handle_valid(itr->handle))
1580                         continue;
1581
1582                 info = (struct kvmgt_guest_info *)itr->handle;
1583                 if (kvm && kvm == info->kvm) {
1584                         ret = true;
1585                         goto out;
1586                 }
1587         }
1588 out:
1589         mutex_unlock(&vgpu->gvt->lock);
1590         return ret;
1591 }
1592
1593 static int kvmgt_guest_init(struct mdev_device *mdev)
1594 {
1595         struct kvmgt_guest_info *info;
1596         struct intel_vgpu *vgpu;
1597         struct kvm *kvm;
1598
1599         vgpu = mdev_get_drvdata(mdev);
1600         if (handle_valid(vgpu->handle))
1601                 return -EEXIST;
1602
1603         kvm = vgpu->vdev.kvm;
1604         if (!kvm || kvm->mm != current->mm) {
1605                 gvt_vgpu_err("KVM is required to use Intel vGPU\n");
1606                 return -ESRCH;
1607         }
1608
1609         if (__kvmgt_vgpu_exist(vgpu, kvm))
1610                 return -EEXIST;
1611
1612         info = vzalloc(sizeof(struct kvmgt_guest_info));
1613         if (!info)
1614                 return -ENOMEM;
1615
1616         vgpu->handle = (unsigned long)info;
1617         info->vgpu = vgpu;
1618         info->kvm = kvm;
1619         kvm_get_kvm(info->kvm);
1620
1621         kvmgt_protect_table_init(info);
1622         gvt_cache_init(vgpu);
1623
1624         mutex_init(&vgpu->dmabuf_lock);
1625         init_completion(&vgpu->vblank_done);
1626
1627         info->track_node.track_write = kvmgt_page_track_write;
1628         info->track_node.track_flush_slot = kvmgt_page_track_flush_slot;
1629         kvm_page_track_register_notifier(kvm, &info->track_node);
1630
1631         info->debugfs_cache_entries = debugfs_create_ulong(
1632                                                 "kvmgt_nr_cache_entries",
1633                                                 0444, vgpu->debugfs,
1634                                                 &vgpu->vdev.nr_cache_entries);
1635         if (!info->debugfs_cache_entries)
1636                 gvt_vgpu_err("Cannot create kvmgt debugfs entry\n");
1637
1638         return 0;
1639 }
1640
1641 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info)
1642 {
1643         debugfs_remove(info->debugfs_cache_entries);
1644
1645         kvm_page_track_unregister_notifier(info->kvm, &info->track_node);
1646         kvm_put_kvm(info->kvm);
1647         kvmgt_protect_table_destroy(info);
1648         gvt_cache_destroy(info->vgpu);
1649         vfree(info);
1650
1651         return true;
1652 }
1653
1654 static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle)
1655 {
1656         /* nothing to do here */
1657         return 0;
1658 }
1659
1660 static void kvmgt_detach_vgpu(unsigned long handle)
1661 {
1662         /* nothing to do here */
1663 }
1664
1665 static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data)
1666 {
1667         struct kvmgt_guest_info *info;
1668         struct intel_vgpu *vgpu;
1669
1670         if (!handle_valid(handle))
1671                 return -ESRCH;
1672
1673         info = (struct kvmgt_guest_info *)handle;
1674         vgpu = info->vgpu;
1675
1676         /*
1677          * When guest is poweroff, msi_trigger is set to NULL, but vgpu's
1678          * config and mmio register isn't restored to default during guest
1679          * poweroff. If this vgpu is still used in next vm, this vgpu's pipe
1680          * may be enabled, then once this vgpu is active, it will get inject
1681          * vblank interrupt request. But msi_trigger is null until msi is
1682          * enabled by guest. so if msi_trigger is null, success is still
1683          * returned and don't inject interrupt into guest.
1684          */
1685         if (vgpu->vdev.msi_trigger == NULL)
1686                 return 0;
1687
1688         if (eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1)
1689                 return 0;
1690
1691         return -EFAULT;
1692 }
1693
1694 static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn)
1695 {
1696         struct kvmgt_guest_info *info;
1697         kvm_pfn_t pfn;
1698
1699         if (!handle_valid(handle))
1700                 return INTEL_GVT_INVALID_ADDR;
1701
1702         info = (struct kvmgt_guest_info *)handle;
1703
1704         pfn = gfn_to_pfn(info->kvm, gfn);
1705         if (is_error_noslot_pfn(pfn))
1706                 return INTEL_GVT_INVALID_ADDR;
1707
1708         return pfn;
1709 }
1710
1711 int kvmgt_dma_map_guest_page(unsigned long handle, unsigned long gfn,
1712                 unsigned long size, dma_addr_t *dma_addr)
1713 {
1714         struct kvmgt_guest_info *info;
1715         struct intel_vgpu *vgpu;
1716         struct gvt_dma *entry;
1717         int ret;
1718
1719         if (!handle_valid(handle))
1720                 return -EINVAL;
1721
1722         info = (struct kvmgt_guest_info *)handle;
1723         vgpu = info->vgpu;
1724
1725         mutex_lock(&info->vgpu->vdev.cache_lock);
1726
1727         entry = __gvt_cache_find_gfn(info->vgpu, gfn);
1728         if (!entry) {
1729                 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);
1730                 if (ret)
1731                         goto err_unlock;
1732
1733                 ret = __gvt_cache_add(info->vgpu, gfn, *dma_addr, size);
1734                 if (ret)
1735                         goto err_unmap;
1736         } else {
1737                 kref_get(&entry->ref);
1738                 *dma_addr = entry->dma_addr;
1739         }
1740
1741         mutex_unlock(&info->vgpu->vdev.cache_lock);
1742         return 0;
1743
1744 err_unmap:
1745         gvt_dma_unmap_page(vgpu, gfn, *dma_addr, size);
1746 err_unlock:
1747         mutex_unlock(&info->vgpu->vdev.cache_lock);
1748         return ret;
1749 }
1750
1751 static void __gvt_dma_release(struct kref *ref)
1752 {
1753         struct gvt_dma *entry = container_of(ref, typeof(*entry), ref);
1754
1755         gvt_dma_unmap_page(entry->vgpu, entry->gfn, entry->dma_addr,
1756                            entry->size);
1757         __gvt_cache_remove_entry(entry->vgpu, entry);
1758 }
1759
1760 void kvmgt_dma_unmap_guest_page(unsigned long handle, dma_addr_t dma_addr)
1761 {
1762         struct kvmgt_guest_info *info;
1763         struct gvt_dma *entry;
1764
1765         if (!handle_valid(handle))
1766                 return;
1767
1768         info = (struct kvmgt_guest_info *)handle;
1769
1770         mutex_lock(&info->vgpu->vdev.cache_lock);
1771         entry = __gvt_cache_find_dma_addr(info->vgpu, dma_addr);
1772         if (entry)
1773                 kref_put(&entry->ref, __gvt_dma_release);
1774         mutex_unlock(&info->vgpu->vdev.cache_lock);
1775 }
1776
1777 static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa,
1778                         void *buf, unsigned long len, bool write)
1779 {
1780         struct kvmgt_guest_info *info;
1781         struct kvm *kvm;
1782         int idx, ret;
1783         bool kthread = current->mm == NULL;
1784
1785         if (!handle_valid(handle))
1786                 return -ESRCH;
1787
1788         info = (struct kvmgt_guest_info *)handle;
1789         kvm = info->kvm;
1790
1791         if (kthread)
1792                 use_mm(kvm->mm);
1793
1794         idx = srcu_read_lock(&kvm->srcu);
1795         ret = write ? kvm_write_guest(kvm, gpa, buf, len) :
1796                       kvm_read_guest(kvm, gpa, buf, len);
1797         srcu_read_unlock(&kvm->srcu, idx);
1798
1799         if (kthread)
1800                 unuse_mm(kvm->mm);
1801
1802         return ret;
1803 }
1804
1805 static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa,
1806                         void *buf, unsigned long len)
1807 {
1808         return kvmgt_rw_gpa(handle, gpa, buf, len, false);
1809 }
1810
1811 static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa,
1812                         void *buf, unsigned long len)
1813 {
1814         return kvmgt_rw_gpa(handle, gpa, buf, len, true);
1815 }
1816
1817 static unsigned long kvmgt_virt_to_pfn(void *addr)
1818 {
1819         return PFN_DOWN(__pa(addr));
1820 }
1821
1822 static bool kvmgt_is_valid_gfn(unsigned long handle, unsigned long gfn)
1823 {
1824         struct kvmgt_guest_info *info;
1825         struct kvm *kvm;
1826
1827         if (!handle_valid(handle))
1828                 return false;
1829
1830         info = (struct kvmgt_guest_info *)handle;
1831         kvm = info->kvm;
1832
1833         return kvm_is_visible_gfn(kvm, gfn);
1834
1835 }
1836
1837 struct intel_gvt_mpt kvmgt_mpt = {
1838         .host_init = kvmgt_host_init,
1839         .host_exit = kvmgt_host_exit,
1840         .attach_vgpu = kvmgt_attach_vgpu,
1841         .detach_vgpu = kvmgt_detach_vgpu,
1842         .inject_msi = kvmgt_inject_msi,
1843         .from_virt_to_mfn = kvmgt_virt_to_pfn,
1844         .enable_page_track = kvmgt_page_track_add,
1845         .disable_page_track = kvmgt_page_track_remove,
1846         .read_gpa = kvmgt_read_gpa,
1847         .write_gpa = kvmgt_write_gpa,
1848         .gfn_to_mfn = kvmgt_gfn_to_pfn,
1849         .dma_map_guest_page = kvmgt_dma_map_guest_page,
1850         .dma_unmap_guest_page = kvmgt_dma_unmap_guest_page,
1851         .set_opregion = kvmgt_set_opregion,
1852         .get_vfio_device = kvmgt_get_vfio_device,
1853         .put_vfio_device = kvmgt_put_vfio_device,
1854         .is_valid_gfn = kvmgt_is_valid_gfn,
1855 };
1856 EXPORT_SYMBOL_GPL(kvmgt_mpt);
1857
1858 static int __init kvmgt_init(void)
1859 {
1860         return 0;
1861 }
1862
1863 static void __exit kvmgt_exit(void)
1864 {
1865 }
1866
1867 module_init(kvmgt_init);
1868 module_exit(kvmgt_exit);
1869
1870 MODULE_LICENSE("GPL and additional rights");
1871 MODULE_AUTHOR("Intel Corporation");