drivers/pci/endpoint/functions/pci-epf-vntb.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Endpoint Function Driver to implement Non-Transparent Bridge functionality
   4  * Between PCI RC and EP
   5  *
   6  * Copyright (C) 2020 Texas Instruments
   7  * Copyright (C) 2022 NXP
   8  *
   9  * Based on pci-epf-ntb.c
  10  * Author: Frank Li <Frank.Li@nxp.com>
  11  * Author: Kishon Vijay Abraham I <kishon@ti.com>
  12  */
  13
  14 /*
  15  * +------------+         +---------------------------------------+
  16  * |            |         |                                       |
  17  * +------------+         |                        +--------------+
  18  * | NTB        |         |                        | NTB          |
  19  * | NetDev     |         |                        | NetDev       |
  20  * +------------+         |                        +--------------+
  21  * | NTB        |         |                        | NTB          |
  22  * | Transfer   |         |                        | Transfer     |
  23  * +------------+         |                        +--------------+
  24  * |            |         |                        |              |
  25  * |  PCI NTB   |         |                        |              |
  26  * |    EPF     |         |                        |              |
  27  * |   Driver   |         |                        | PCI Virtual  |
  28  * |            |         +---------------+        | NTB Driver   |
  29  * |            |         | PCI EP NTB    |<------>|              |
  30  * |            |         |  FN Driver    |        |              |
  31  * +------------+         +---------------+        +--------------+
  32  * |            |         |               |        |              |
  33  * |  PCI Bus   | <-----> |  PCI EP Bus   |        |  Virtual PCI |
  34  * |            |  PCI    |               |        |     Bus      |
  35  * +------------+         +---------------+--------+--------------+
  36  * PCIe Root Port                        PCI EP
  37  */
  38
  39 #include <linux/delay.h>
  40 #include <linux/io.h>
  41 #include <linux/module.h>
  42 #include <linux/slab.h>
  43
  44 #include <linux/pci-epc.h>
  45 #include <linux/pci-epf.h>
  46 #include <linux/ntb.h>
  47
  48 static struct workqueue_struct *kpcintb_workqueue;
  49
  50 #define COMMAND_CONFIGURE_DOORBELL      1
  51 #define COMMAND_TEARDOWN_DOORBELL       2
  52 #define COMMAND_CONFIGURE_MW            3
  53 #define COMMAND_TEARDOWN_MW             4
  54 #define COMMAND_LINK_UP                 5
  55 #define COMMAND_LINK_DOWN               6
  56
  57 #define COMMAND_STATUS_OK               1
  58 #define COMMAND_STATUS_ERROR            2
  59
  60 #define LINK_STATUS_UP                  BIT(0)
  61
  62 #define SPAD_COUNT                      64
  63 #define DB_COUNT                        4
  64 #define NTB_MW_OFFSET                   2
  65 #define DB_COUNT_MASK                   GENMASK(15, 0)
  66 #define MSIX_ENABLE                     BIT(16)
  67 #define MAX_DB_COUNT                    32
  68 #define MAX_MW                          4
  69
  70 enum epf_ntb_bar {
  71         BAR_CONFIG,
  72         BAR_DB,
  73         BAR_MW0,
  74         BAR_MW1,
  75         BAR_MW2,
  76 };
  77
  78 /*
  79  * +--------------------------------------------------+ Base
  80  * |                                                  |
  81  * |                                                  |
  82  * |                                                  |
  83  * |          Common Control Register                 |
  84  * |                                                  |
  85  * |                                                  |
  86  * |                                                  |
  87  * +-----------------------+--------------------------+ Base+span_offset
  88  * |                       |                          |
  89  * |    Peer Span Space    |    Span Space            |
  90  * |                       |                          |
  91  * |                       |                          |
  92  * +-----------------------+--------------------------+ Base+span_offset
  93  * |                       |                          |     +span_count * 4
  94  * |                       |                          |
  95  * |     Span Space        |   Peer Span Space        |
  96  * |                       |                          |
  97  * +-----------------------+--------------------------+
  98  *       Virtual PCI             PCIe Endpoint
  99  *       NTB Driver               NTB Driver
 100  */
 101 struct epf_ntb_ctrl {
 102         u32 command;
 103         u32 argument;
 104         u16 command_status;
 105         u16 link_status;
 106         u32 topology;
 107         u64 addr;
 108         u64 size;
 109         u32 num_mws;
 110         u32 reserved;
 111         u32 spad_offset;
 112         u32 spad_count;
 113         u32 db_entry_size;
 114         u32 db_data[MAX_DB_COUNT];
 115         u32 db_offset[MAX_DB_COUNT];
 116 } __packed;
 117
 118 struct epf_ntb {
 119         struct ntb_dev ntb;
 120         struct pci_epf *epf;
 121         struct config_group group;
 122
 123         u32 num_mws;
 124         u32 db_count;
 125         u32 spad_count;
 126         u64 mws_size[MAX_MW];
 127         u64 db;
 128         u32 vbus_number;
 129         u16 vntb_pid;
 130         u16 vntb_vid;
 131
 132         bool linkup;
 133         u32 spad_size;
 134
 135         enum pci_barno epf_ntb_bar[6];
 136
 137         struct epf_ntb_ctrl *reg;
 138
 139         phys_addr_t epf_db_phy;
 140         void __iomem *epf_db;
 141
 142         phys_addr_t vpci_mw_phy[MAX_MW];
 143         void __iomem *vpci_mw_addr[MAX_MW];
 144
 145         struct delayed_work cmd_handler;
 146 };
 147
 148 #define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group)
 149 #define ntb_ndev(__ntb) container_of(__ntb, struct epf_ntb, ntb)
 150
 151 static struct pci_epf_header epf_ntb_header = {
 152         .vendorid       = PCI_ANY_ID,
 153         .deviceid       = PCI_ANY_ID,
 154         .baseclass_code = PCI_BASE_CLASS_MEMORY,
 155         .interrupt_pin  = PCI_INTERRUPT_INTA,
 156 };
 157
 158 /**
 159  * epf_ntb_link_up() - Raise link_up interrupt to Virtual Host (VHOST)
 160  * @ntb: NTB device that facilitates communication between HOST and VHOST
 161  * @link_up: true or false indicating Link is UP or Down
 162  *
 163  * Once NTB function in HOST invoke ntb_link_enable(),
 164  * this NTB function driver will trigger a link event to VHOST.
 165  *
 166  * Returns: Zero for success, or an error code in case of failure
 167  */
 168 static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up)
 169 {
 170         if (link_up)
 171                 ntb->reg->link_status |= LINK_STATUS_UP;
 172         else
 173                 ntb->reg->link_status &= ~LINK_STATUS_UP;
 174
 175         ntb_link_event(&ntb->ntb);
 176         return 0;
 177 }
 178
 179 /**
 180  * epf_ntb_configure_mw() - Configure the Outbound Address Space for VHOST
 181  *   to access the memory window of HOST
 182  * @ntb: NTB device that facilitates communication between HOST and VHOST
 183  * @mw: Index of the memory window (either 0, 1, 2 or 3)
 184  *
 185  *                          EP Outbound Window
 186  * +--------+              +-----------+
 187  * |        |              |           |
 188  * |        |              |           |
 189  * |        |              |           |
 190  * |        |              |           |
 191  * |        |              +-----------+
 192  * | Virtual|              | Memory Win|
 193  * | NTB    | -----------> |           |
 194  * | Driver |              |           |
 195  * |        |              +-----------+
 196  * |        |              |           |
 197  * |        |              |           |
 198  * +--------+              +-----------+
 199  *  VHOST                   PCI EP
 200  *
 201  * Returns: Zero for success, or an error code in case of failure
 202  */
 203 static int epf_ntb_configure_mw(struct epf_ntb *ntb, u32 mw)
 204 {
 205         phys_addr_t phys_addr;
 206         u8 func_no, vfunc_no;
 207         u64 addr, size;
 208         int ret = 0;
 209
 210         phys_addr = ntb->vpci_mw_phy[mw];
 211         addr = ntb->reg->addr;
 212         size = ntb->reg->size;
 213
 214         func_no = ntb->epf->func_no;
 215         vfunc_no = ntb->epf->vfunc_no;
 216
 217         ret = pci_epc_map_addr(ntb->epf->epc, func_no, vfunc_no, phys_addr, addr, size);
 218         if (ret)
 219                 dev_err(&ntb->epf->epc->dev,
 220                         "Failed to map memory window %d address\n", mw);
 221         return ret;
 222 }
 223
 224 /**
 225  * epf_ntb_teardown_mw() - Teardown the configured OB ATU
 226  * @ntb: NTB device that facilitates communication between HOST and VHOST
 227  * @mw: Index of the memory window (either 0, 1, 2 or 3)
 228  *
 229  * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using
 230  * pci_epc_unmap_addr()
 231  */
 232 static void epf_ntb_teardown_mw(struct epf_ntb *ntb, u32 mw)
 233 {
 234         pci_epc_unmap_addr(ntb->epf->epc,
 235                            ntb->epf->func_no,
 236                            ntb->epf->vfunc_no,
 237                            ntb->vpci_mw_phy[mw]);
 238 }
 239
 240 /**
 241  * epf_ntb_cmd_handler() - Handle commands provided by the NTB HOST
 242  * @work: work_struct for the epf_ntb_epc
 243  *
 244  * Workqueue function that gets invoked for the two epf_ntb_epc
 245  * periodically (once every 5ms) to see if it has received any commands
 246  * from NTB HOST. The HOST can send commands to configure doorbell or
 247  * configure memory window or to update link status.
 248  */
 249 static void epf_ntb_cmd_handler(struct work_struct *work)
 250 {
 251         struct epf_ntb_ctrl *ctrl;
 252         u32 command, argument;
 253         struct epf_ntb *ntb;
 254         struct device *dev;
 255         int ret;
 256         int i;
 257
 258         ntb = container_of(work, struct epf_ntb, cmd_handler.work);
 259
 260         for (i = 1; i < ntb->db_count; i++) {
 261                 if (readl(ntb->epf_db + i * 4)) {
 262                         if (readl(ntb->epf_db + i * 4))
 263                                 ntb->db |= 1 << (i - 1);
 264
 265                         ntb_db_event(&ntb->ntb, i);
 266                         writel(0, ntb->epf_db + i * 4);
 267                 }
 268         }
 269
 270         ctrl = ntb->reg;
 271         command = ctrl->command;
 272         if (!command)
 273                 goto reset_handler;
 274         argument = ctrl->argument;
 275
 276         ctrl->command = 0;
 277         ctrl->argument = 0;
 278
 279         ctrl = ntb->reg;
 280         dev = &ntb->epf->dev;
 281
 282         switch (command) {
 283         case COMMAND_CONFIGURE_DOORBELL:
 284                 ctrl->command_status = COMMAND_STATUS_OK;
 285                 break;
 286         case COMMAND_TEARDOWN_DOORBELL:
 287                 ctrl->command_status = COMMAND_STATUS_OK;
 288                 break;
 289         case COMMAND_CONFIGURE_MW:
 290                 ret = epf_ntb_configure_mw(ntb, argument);
 291                 if (ret < 0)
 292                         ctrl->command_status = COMMAND_STATUS_ERROR;
 293                 else
 294                         ctrl->command_status = COMMAND_STATUS_OK;
 295                 break;
 296         case COMMAND_TEARDOWN_MW:
 297                 epf_ntb_teardown_mw(ntb, argument);
 298                 ctrl->command_status = COMMAND_STATUS_OK;
 299                 break;
 300         case COMMAND_LINK_UP:
 301                 ntb->linkup = true;
 302                 ret = epf_ntb_link_up(ntb, true);
 303                 if (ret < 0)
 304                         ctrl->command_status = COMMAND_STATUS_ERROR;
 305                 else
 306                         ctrl->command_status = COMMAND_STATUS_OK;
 307                 goto reset_handler;
 308         case COMMAND_LINK_DOWN:
 309                 ntb->linkup = false;
 310                 ret = epf_ntb_link_up(ntb, false);
 311                 if (ret < 0)
 312                         ctrl->command_status = COMMAND_STATUS_ERROR;
 313                 else
 314                         ctrl->command_status = COMMAND_STATUS_OK;
 315                 break;
 316         default:
 317                 dev_err(dev, "UNKNOWN command: %d\n", command);
 318                 break;
 319         }
 320
 321 reset_handler:
 322         queue_delayed_work(kpcintb_workqueue, &ntb->cmd_handler,
 323                            msecs_to_jiffies(5));
 324 }
 325
 326 /**
 327  * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR
 328  * @ntb: EPC associated with one of the HOST which holds peer's outbound
 329  *       address.
 330  *
 331  * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and
 332  * self scratchpad region (removes inbound ATU configuration). While BAR0 is
 333  * the default self scratchpad BAR, an NTB could have other BARs for self
 334  * scratchpad (because of reserved BARs). This function can get the exact BAR
 335  * used for self scratchpad from epf_ntb_bar[BAR_CONFIG].
 336  *
 337  * Please note the self scratchpad region and config region is combined to
 338  * a single region and mapped using the same BAR. Also note VHOST's peer
 339  * scratchpad is HOST's self scratchpad.
 340  *
 341  * Returns: void
 342  */
 343 static void epf_ntb_config_sspad_bar_clear(struct epf_ntb *ntb)
 344 {
 345         struct pci_epf_bar *epf_bar;
 346         enum pci_barno barno;
 347
 348         barno = ntb->epf_ntb_bar[BAR_CONFIG];
 349         epf_bar = &ntb->epf->bar[barno];
 350
 351         pci_epc_clear_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);
 352 }
 353
 354 /**
 355  * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR
 356  * @ntb: NTB device that facilitates communication between HOST and VHOST
 357  *
 358  * Map BAR0 of EP CONTROLLER which contains the VHOST's config and
 359  * self scratchpad region.
 360  *
 361  * Please note the self scratchpad region and config region is combined to
 362  * a single region and mapped using the same BAR.
 363  *
 364  * Returns: Zero for success, or an error code in case of failure
 365  */
 366 static int epf_ntb_config_sspad_bar_set(struct epf_ntb *ntb)
 367 {
 368         struct pci_epf_bar *epf_bar;
 369         enum pci_barno barno;
 370         u8 func_no, vfunc_no;
 371         struct device *dev;
 372         int ret;
 373
 374         dev = &ntb->epf->dev;
 375         func_no = ntb->epf->func_no;
 376         vfunc_no = ntb->epf->vfunc_no;
 377         barno = ntb->epf_ntb_bar[BAR_CONFIG];
 378         epf_bar = &ntb->epf->bar[barno];
 379
 380         ret = pci_epc_set_bar(ntb->epf->epc, func_no, vfunc_no, epf_bar);
 381         if (ret) {
 382                 dev_err(dev, "inft: Config/Status/SPAD BAR set failed\n");
 383                 return ret;
 384         }
 385         return 0;
 386 }
 387
 388 /**
 389  * epf_ntb_config_spad_bar_free() - Free the physical memory associated with
 390  *   config + scratchpad region
 391  * @ntb: NTB device that facilitates communication between HOST and VHOST
 392  */
 393 static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)
 394 {
 395         enum pci_barno barno;
 396
 397         barno = ntb->epf_ntb_bar[BAR_CONFIG];
 398         pci_epf_free_space(ntb->epf, ntb->reg, barno, 0);
 399 }
 400
 401 /**
 402  * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad
 403  *   region
 404  * @ntb: NTB device that facilitates communication between HOST and VHOST
 405  *
 406  * Allocate the Local Memory mentioned in the above diagram. The size of
 407  * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION
 408  * is obtained from "spad-count" configfs entry.
 409  *
 410  * Returns: Zero for success, or an error code in case of failure
 411  */
 412 static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb)
 413 {
 414         size_t align;
 415         enum pci_barno barno;
 416         struct epf_ntb_ctrl *ctrl;
 417         u32 spad_size, ctrl_size;
 418         u64 size;
 419         struct pci_epf *epf = ntb->epf;
 420         struct device *dev = &epf->dev;
 421         u32 spad_count;
 422         void *base;
 423         int i;
 424         const struct pci_epc_features *epc_features = pci_epc_get_features(epf->epc,
 425                                                                 epf->func_no,
 426                                                                 epf->vfunc_no);
 427         barno = ntb->epf_ntb_bar[BAR_CONFIG];
 428         size = epc_features->bar_fixed_size[barno];
 429         align = epc_features->align;
 430
 431         if ((!IS_ALIGNED(size, align)))
 432                 return -EINVAL;
 433
 434         spad_count = ntb->spad_count;
 435
 436         ctrl_size = sizeof(struct epf_ntb_ctrl);
 437         spad_size = 2 * spad_count * 4;
 438
 439         if (!align) {
 440                 ctrl_size = roundup_pow_of_two(ctrl_size);
 441                 spad_size = roundup_pow_of_two(spad_size);
 442         } else {
 443                 ctrl_size = ALIGN(ctrl_size, align);
 444                 spad_size = ALIGN(spad_size, align);
 445         }
 446
 447         if (!size)
 448                 size = ctrl_size + spad_size;
 449         else if (size < ctrl_size + spad_size)
 450                 return -EINVAL;
 451
 452         base = pci_epf_alloc_space(epf, size, barno, align, 0);
 453         if (!base) {
 454                 dev_err(dev, "Config/Status/SPAD alloc region fail\n");
 455                 return -ENOMEM;
 456         }
 457
 458         ntb->reg = base;
 459
 460         ctrl = ntb->reg;
 461         ctrl->spad_offset = ctrl_size;
 462
 463         ctrl->spad_count = spad_count;
 464         ctrl->num_mws = ntb->num_mws;
 465         ntb->spad_size = spad_size;
 466
 467         ctrl->db_entry_size = 4;
 468
 469         for (i = 0; i < ntb->db_count; i++) {
 470                 ntb->reg->db_data[i] = 1 + i;
 471                 ntb->reg->db_offset[i] = 0;
 472         }
 473
 474         return 0;
 475 }
 476
 477 /**
 478  * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capability
 479  * @ntb: NTB device that facilitates communication between HOST and VHOST
 480  *
 481  * Configure MSI/MSI-X capability for each interface with number of
 482  * interrupts equal to "db_count" configfs entry.
 483  *
 484  * Returns: Zero for success, or an error code in case of failure
 485  */
 486 static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)
 487 {
 488         const struct pci_epc_features *epc_features;
 489         struct device *dev;
 490         u32 db_count;
 491         int ret;
 492
 493         dev = &ntb->epf->dev;
 494
 495         epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);
 496
 497         if (!(epc_features->msix_capable || epc_features->msi_capable)) {
 498                 dev_err(dev, "MSI or MSI-X is required for doorbell\n");
 499                 return -EINVAL;
 500         }
 501
 502         db_count = ntb->db_count;
 503         if (db_count > MAX_DB_COUNT) {
 504                 dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT);
 505                 return -EINVAL;
 506         }
 507
 508         ntb->db_count = db_count;
 509
 510         if (epc_features->msi_capable) {
 511                 ret = pci_epc_set_msi(ntb->epf->epc,
 512                                       ntb->epf->func_no,
 513                                       ntb->epf->vfunc_no,
 514                                       16);
 515                 if (ret) {
 516                         dev_err(dev, "MSI configuration failed\n");
 517                         return ret;
 518                 }
 519         }
 520
 521         return 0;
 522 }
 523
 524 /**
 525  * epf_ntb_db_bar_init() - Configure Doorbell window BARs
 526  * @ntb: NTB device that facilitates communication between HOST and VHOST
 527  *
 528  * Returns: Zero for success, or an error code in case of failure
 529  */
 530 static int epf_ntb_db_bar_init(struct epf_ntb *ntb)
 531 {
 532         const struct pci_epc_features *epc_features;
 533         u32 align;
 534         struct device *dev = &ntb->epf->dev;
 535         int ret;
 536         struct pci_epf_bar *epf_bar;
 537         void __iomem *mw_addr;
 538         enum pci_barno barno;
 539         size_t size = 4 * ntb->db_count;
 540
 541         epc_features = pci_epc_get_features(ntb->epf->epc,
 542                                             ntb->epf->func_no,
 543                                             ntb->epf->vfunc_no);
 544         align = epc_features->align;
 545
 546         if (size < 128)
 547                 size = 128;
 548
 549         if (align)
 550                 size = ALIGN(size, align);
 551         else
 552                 size = roundup_pow_of_two(size);
 553
 554         barno = ntb->epf_ntb_bar[BAR_DB];
 555
 556         mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, align, 0);
 557         if (!mw_addr) {
 558                 dev_err(dev, "Failed to allocate OB address\n");
 559                 return -ENOMEM;
 560         }
 561
 562         ntb->epf_db = mw_addr;
 563
 564         epf_bar = &ntb->epf->bar[barno];
 565
 566         ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);
 567         if (ret) {
 568                 dev_err(dev, "Doorbell BAR set failed\n");
 569                         goto err_alloc_peer_mem;
 570         }
 571         return ret;
 572
 573 err_alloc_peer_mem:
 574         pci_epc_mem_free_addr(ntb->epf->epc, epf_bar->phys_addr, mw_addr, epf_bar->size);
 575         return -1;
 576 }
 577
 578 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws);
 579
 580 /**
 581  * epf_ntb_db_bar_clear() - Clear doorbell BAR and free memory
 582  *   allocated in peer's outbound address space
 583  * @ntb: NTB device that facilitates communication between HOST and VHOST
 584  */
 585 static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)
 586 {
 587         enum pci_barno barno;
 588
 589         barno = ntb->epf_ntb_bar[BAR_DB];
 590         pci_epf_free_space(ntb->epf, ntb->epf_db, barno, 0);
 591         pci_epc_clear_bar(ntb->epf->epc,
 592                           ntb->epf->func_no,
 593                           ntb->epf->vfunc_no,
 594                           &ntb->epf->bar[barno]);
 595 }
 596
 597 /**
 598  * epf_ntb_mw_bar_init() - Configure Memory window BARs
 599  * @ntb: NTB device that facilitates communication between HOST and VHOST
 600  *
 601  * Returns: Zero for success, or an error code in case of failure
 602  */
 603 static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
 604 {
 605         int ret = 0;
 606         int i;
 607         u64 size;
 608         enum pci_barno barno;
 609         struct device *dev = &ntb->epf->dev;
 610
 611         for (i = 0; i < ntb->num_mws; i++) {
 612                 size = ntb->mws_size[i];
 613                 barno = ntb->epf_ntb_bar[BAR_MW0 + i];
 614
 615                 ntb->epf->bar[barno].barno = barno;
 616                 ntb->epf->bar[barno].size = size;
 617                 ntb->epf->bar[barno].addr = NULL;
 618                 ntb->epf->bar[barno].phys_addr = 0;
 619                 ntb->epf->bar[barno].flags |= upper_32_bits(size) ?
 620                                 PCI_BASE_ADDRESS_MEM_TYPE_64 :
 621                                 PCI_BASE_ADDRESS_MEM_TYPE_32;
 622
 623                 ret = pci_epc_set_bar(ntb->epf->epc,
 624                                       ntb->epf->func_no,
 625                                       ntb->epf->vfunc_no,
 626                                       &ntb->epf->bar[barno]);
 627                 if (ret) {
 628                         dev_err(dev, "MW set failed\n");
 629                         goto err_alloc_mem;
 630                 }
 631
 632                 /* Allocate EPC outbound memory windows to vpci vntb device */
 633                 ntb->vpci_mw_addr[i] = pci_epc_mem_alloc_addr(ntb->epf->epc,
 634                                                               &ntb->vpci_mw_phy[i],
 635                                                               size);
 636                 if (!ntb->vpci_mw_addr[i]) {
 637                         ret = -ENOMEM;
 638                         dev_err(dev, "Failed to allocate source address\n");
 639                         goto err_set_bar;
 640                 }
 641         }
 642
 643         return ret;
 644
 645 err_set_bar:
 646         pci_epc_clear_bar(ntb->epf->epc,
 647                           ntb->epf->func_no,
 648                           ntb->epf->vfunc_no,
 649                           &ntb->epf->bar[barno]);
 650 err_alloc_mem:
 651         epf_ntb_mw_bar_clear(ntb, i);
 652         return ret;
 653 }
 654
 655 /**
 656  * epf_ntb_mw_bar_clear() - Clear Memory window BARs
 657  * @ntb: NTB device that facilitates communication between HOST and VHOST
 658  */
 659 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws)
 660 {
 661         enum pci_barno barno;
 662         int i;
 663
 664         for (i = 0; i < num_mws; i++) {
 665                 barno = ntb->epf_ntb_bar[BAR_MW0 + i];
 666                 pci_epc_clear_bar(ntb->epf->epc,
 667                                   ntb->epf->func_no,
 668                                   ntb->epf->vfunc_no,
 669                                   &ntb->epf->bar[barno]);
 670
 671                 pci_epc_mem_free_addr(ntb->epf->epc,
 672                                       ntb->vpci_mw_phy[i],
 673                                       ntb->vpci_mw_addr[i],
 674                                       ntb->mws_size[i]);
 675         }
 676 }
 677
 678 /**
 679  * epf_ntb_epc_destroy() - Cleanup NTB EPC interface
 680  * @ntb: NTB device that facilitates communication between HOST and VHOST
 681  *
 682  * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces
 683  */
 684 static void epf_ntb_epc_destroy(struct epf_ntb *ntb)
 685 {
 686         pci_epc_remove_epf(ntb->epf->epc, ntb->epf, 0);
 687         pci_epc_put(ntb->epf->epc);
 688 }
 689
 690 /**
 691  * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB
 692  * constructs (scratchpad region, doorbell, memorywindow)
 693  * @ntb: NTB device that facilitates communication between HOST and VHOST
 694  *
 695  * Returns: Zero for success, or an error code in case of failure
 696  */
 697 static int epf_ntb_init_epc_bar(struct epf_ntb *ntb)
 698 {
 699         const struct pci_epc_features *epc_features;
 700         enum pci_barno barno;
 701         enum epf_ntb_bar bar;
 702         struct device *dev;
 703         u32 num_mws;
 704         int i;
 705
 706         barno = BAR_0;
 707         num_mws = ntb->num_mws;
 708         dev = &ntb->epf->dev;
 709         epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);
 710
 711         /* These are required BARs which are mandatory for NTB functionality */
 712         for (bar = BAR_CONFIG; bar <= BAR_MW0; bar++, barno++) {
 713                 barno = pci_epc_get_next_free_bar(epc_features, barno);
 714                 if (barno < 0) {
 715                         dev_err(dev, "Fail to get NTB function BAR\n");
 716                         return barno;
 717                 }
 718                 ntb->epf_ntb_bar[bar] = barno;
 719         }
 720
 721         /* These are optional BARs which don't impact NTB functionality */
 722         for (bar = BAR_MW1, i = 1; i < num_mws; bar++, barno++, i++) {
 723                 barno = pci_epc_get_next_free_bar(epc_features, barno);
 724                 if (barno < 0) {
 725                         ntb->num_mws = i;
 726                         dev_dbg(dev, "BAR not available for > MW%d\n", i + 1);
 727                 }
 728                 ntb->epf_ntb_bar[bar] = barno;
 729         }
 730
 731         return 0;
 732 }
 733
 734 /**
 735  * epf_ntb_epc_init() - Initialize NTB interface
 736  * @ntb: NTB device that facilitates communication between HOST and VHOST
 737  *
 738  * Wrapper to initialize a particular EPC interface and start the workqueue
 739  * to check for commands from HOST. This function will write to the
 740  * EP controller HW for configuring it.
 741  *
 742  * Returns: Zero for success, or an error code in case of failure
 743  */
 744 static int epf_ntb_epc_init(struct epf_ntb *ntb)
 745 {
 746         u8 func_no, vfunc_no;
 747         struct pci_epc *epc;
 748         struct pci_epf *epf;
 749         struct device *dev;
 750         int ret;
 751
 752         epf = ntb->epf;
 753         dev = &epf->dev;
 754         epc = epf->epc;
 755         func_no = ntb->epf->func_no;
 756         vfunc_no = ntb->epf->vfunc_no;
 757
 758         ret = epf_ntb_config_sspad_bar_set(ntb);
 759         if (ret) {
 760                 dev_err(dev, "Config/self SPAD BAR init failed");
 761                 return ret;
 762         }
 763
 764         ret = epf_ntb_configure_interrupt(ntb);
 765         if (ret) {
 766                 dev_err(dev, "Interrupt configuration failed\n");
 767                 goto err_config_interrupt;
 768         }
 769
 770         ret = epf_ntb_db_bar_init(ntb);
 771         if (ret) {
 772                 dev_err(dev, "DB BAR init failed\n");
 773                 goto err_db_bar_init;
 774         }
 775
 776         ret = epf_ntb_mw_bar_init(ntb);
 777         if (ret) {
 778                 dev_err(dev, "MW BAR init failed\n");
 779                 goto err_mw_bar_init;
 780         }
 781
 782         if (vfunc_no <= 1) {
 783                 ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header);
 784                 if (ret) {
 785                         dev_err(dev, "Configuration header write failed\n");
 786                         goto err_write_header;
 787                 }
 788         }
 789
 790         INIT_DELAYED_WORK(&ntb->cmd_handler, epf_ntb_cmd_handler);
 791         queue_work(kpcintb_workqueue, &ntb->cmd_handler.work);
 792
 793         return 0;
 794
 795 err_write_header:
 796         epf_ntb_mw_bar_clear(ntb, ntb->num_mws);
 797 err_mw_bar_init:
 798         epf_ntb_db_bar_clear(ntb);
 799 err_db_bar_init:
 800 err_config_interrupt:
 801         epf_ntb_config_sspad_bar_clear(ntb);
 802
 803         return ret;
 804 }
 805
 806
 807 /**
 808  * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces
 809  * @ntb: NTB device that facilitates communication between HOST and VHOST
 810  *
 811  * Wrapper to cleanup all NTB interfaces.
 812  */
 813 static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)
 814 {
 815         epf_ntb_db_bar_clear(ntb);
 816         epf_ntb_mw_bar_clear(ntb, ntb->num_mws);
 817 }
 818
 819 #define EPF_NTB_R(_name)                                                \
 820 static ssize_t epf_ntb_##_name##_show(struct config_item *item,         \
 821                                       char *page)                       \
 822 {                                                                       \
 823         struct config_group *group = to_config_group(item);             \
 824         struct epf_ntb *ntb = to_epf_ntb(group);                        \
 825                                                                         \
 826         return sprintf(page, "%d\n", ntb->_name);                       \
 827 }
 828
 829 #define EPF_NTB_W(_name)                                                \
 830 static ssize_t epf_ntb_##_name##_store(struct config_item *item,        \
 831                                        const char *page, size_t len)    \
 832 {                                                                       \
 833         struct config_group *group = to_config_group(item);             \
 834         struct epf_ntb *ntb = to_epf_ntb(group);                        \
 835         u32 val;                                                        \
 836         int ret;                                                        \
 837                                                                         \
 838         ret = kstrtou32(page, 0, &val);                                 \
 839         if (ret)                                                        \
 840                 return ret;                                             \
 841                                                                         \
 842         ntb->_name = val;                                               \
 843                                                                         \
 844         return len;                                                     \
 845 }
 846
 847 #define EPF_NTB_MW_R(_name)                                             \
 848 static ssize_t epf_ntb_##_name##_show(struct config_item *item,         \
 849                                       char *page)                       \
 850 {                                                                       \
 851         struct config_group *group = to_config_group(item);             \
 852         struct epf_ntb *ntb = to_epf_ntb(group);                        \
 853         struct device *dev = &ntb->epf->dev;                            \
 854         int win_no;                                                     \
 855                                                                         \
 856         if (sscanf(#_name, "mw%d", &win_no) != 1)                       \
 857                 return -EINVAL;                                         \
 858                                                                         \
 859         if (win_no <= 0 || win_no > ntb->num_mws) {                     \
 860                 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
 861                 return -EINVAL;                                         \
 862         }                                                               \
 863                                                                         \
 864         return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]);      \
 865 }
 866
 867 #define EPF_NTB_MW_W(_name)                                             \
 868 static ssize_t epf_ntb_##_name##_store(struct config_item *item,        \
 869                                        const char *page, size_t len)    \
 870 {                                                                       \
 871         struct config_group *group = to_config_group(item);             \
 872         struct epf_ntb *ntb = to_epf_ntb(group);                        \
 873         struct device *dev = &ntb->epf->dev;                            \
 874         int win_no;                                                     \
 875         u64 val;                                                        \
 876         int ret;                                                        \
 877                                                                         \
 878         ret = kstrtou64(page, 0, &val);                                 \
 879         if (ret)                                                        \
 880                 return ret;                                             \
 881                                                                         \
 882         if (sscanf(#_name, "mw%d", &win_no) != 1)                       \
 883                 return -EINVAL;                                         \
 884                                                                         \
 885         if (win_no <= 0 || win_no > ntb->num_mws) {                     \
 886                 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
 887                 return -EINVAL;                                         \
 888         }                                                               \
 889                                                                         \
 890         ntb->mws_size[win_no - 1] = val;                                \
 891                                                                         \
 892         return len;                                                     \
 893 }
 894
 895 static ssize_t epf_ntb_num_mws_store(struct config_item *item,
 896                                      const char *page, size_t len)
 897 {
 898         struct config_group *group = to_config_group(item);
 899         struct epf_ntb *ntb = to_epf_ntb(group);
 900         u32 val;
 901         int ret;
 902
 903         ret = kstrtou32(page, 0, &val);
 904         if (ret)
 905                 return ret;
 906
 907         if (val > MAX_MW)
 908                 return -EINVAL;
 909
 910         ntb->num_mws = val;
 911
 912         return len;
 913 }
 914
 915 EPF_NTB_R(spad_count)
 916 EPF_NTB_W(spad_count)
 917 EPF_NTB_R(db_count)
 918 EPF_NTB_W(db_count)
 919 EPF_NTB_R(num_mws)
 920 EPF_NTB_R(vbus_number)
 921 EPF_NTB_W(vbus_number)
 922 EPF_NTB_R(vntb_pid)
 923 EPF_NTB_W(vntb_pid)
 924 EPF_NTB_R(vntb_vid)
 925 EPF_NTB_W(vntb_vid)
 926 EPF_NTB_MW_R(mw1)
 927 EPF_NTB_MW_W(mw1)
 928 EPF_NTB_MW_R(mw2)
 929 EPF_NTB_MW_W(mw2)
 930 EPF_NTB_MW_R(mw3)
 931 EPF_NTB_MW_W(mw3)
 932 EPF_NTB_MW_R(mw4)
 933 EPF_NTB_MW_W(mw4)
 934
 935 CONFIGFS_ATTR(epf_ntb_, spad_count);
 936 CONFIGFS_ATTR(epf_ntb_, db_count);
 937 CONFIGFS_ATTR(epf_ntb_, num_mws);
 938 CONFIGFS_ATTR(epf_ntb_, mw1);
 939 CONFIGFS_ATTR(epf_ntb_, mw2);
 940 CONFIGFS_ATTR(epf_ntb_, mw3);
 941 CONFIGFS_ATTR(epf_ntb_, mw4);
 942 CONFIGFS_ATTR(epf_ntb_, vbus_number);
 943 CONFIGFS_ATTR(epf_ntb_, vntb_pid);
 944 CONFIGFS_ATTR(epf_ntb_, vntb_vid);
 945
 946 static struct configfs_attribute *epf_ntb_attrs[] = {
 947         &epf_ntb_attr_spad_count,
 948         &epf_ntb_attr_db_count,
 949         &epf_ntb_attr_num_mws,
 950         &epf_ntb_attr_mw1,
 951         &epf_ntb_attr_mw2,
 952         &epf_ntb_attr_mw3,
 953         &epf_ntb_attr_mw4,
 954         &epf_ntb_attr_vbus_number,
 955         &epf_ntb_attr_vntb_pid,
 956         &epf_ntb_attr_vntb_vid,
 957         NULL,
 958 };
 959
 960 static const struct config_item_type ntb_group_type = {
 961         .ct_attrs       = epf_ntb_attrs,
 962         .ct_owner       = THIS_MODULE,
 963 };
 964
 965 /**
 966  * epf_ntb_add_cfs() - Add configfs directory specific to NTB
 967  * @epf: NTB endpoint function device
 968  * @group: A pointer to the config_group structure referencing a group of
 969  *         config_items of a specific type that belong to a specific sub-system.
 970  *
 971  * Add configfs directory specific to NTB. This directory will hold
 972  * NTB specific properties like db_count, spad_count, num_mws etc.,
 973  *
 974  * Returns: Pointer to config_group
 975  */
 976 static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf,
 977                                             struct config_group *group)
 978 {
 979         struct epf_ntb *ntb = epf_get_drvdata(epf);
 980         struct config_group *ntb_group = &ntb->group;
 981         struct device *dev = &epf->dev;
 982
 983         config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type);
 984
 985         return ntb_group;
 986 }
 987
 988 /*==== virtual PCI bus driver, which only load virtual NTB PCI driver ====*/
 989
 990 static u32 pci_space[] = {
 991         0xffffffff,     /*DeviceID, Vendor ID*/
 992         0,              /*Status, Command*/
 993         0xffffffff,     /*Class code, subclass, prog if, revision id*/
 994         0x40,           /*bist, header type, latency Timer, cache line size*/
 995         0,              /*BAR 0*/
 996         0,              /*BAR 1*/
 997         0,              /*BAR 2*/
 998         0,              /*BAR 3*/
 999         0,              /*BAR 4*/
1000         0,              /*BAR 5*/
1001         0,              /*Cardbus cis point*/
1002         0,              /*Subsystem ID Subystem vendor id*/
1003         0,              /*ROM Base Address*/
1004         0,              /*Reserved, Cap. Point*/
1005         0,              /*Reserved,*/
1006         0,              /*Max Lat, Min Gnt, interrupt pin, interrupt line*/
1007 };
1008
1009 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val)
1010 {
1011         if (devfn == 0) {
1012                 memcpy(val, ((u8 *)pci_space) + where, size);
1013                 return PCIBIOS_SUCCESSFUL;
1014         }
1015         return PCIBIOS_DEVICE_NOT_FOUND;
1016 }
1017
1018 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val)
1019 {
1020         return 0;
1021 }
1022
1023 static struct pci_ops vpci_ops = {
1024         .read = pci_read,
1025         .write = pci_write,
1026 };
1027
1028 static int vpci_scan_bus(void *sysdata)
1029 {
1030         struct pci_bus *vpci_bus;
1031         struct epf_ntb *ndev = sysdata;
1032
1033         vpci_bus = pci_scan_bus(ndev->vbus_number, &vpci_ops, sysdata);
1034         if (vpci_bus)
1035                 pr_err("create pci bus\n");
1036
1037         pci_bus_add_devices(vpci_bus);
1038
1039         return 0;
1040 }
1041
1042 /*==================== Virtual PCIe NTB driver ==========================*/
1043
1044 static int vntb_epf_mw_count(struct ntb_dev *ntb, int pidx)
1045 {
1046         struct epf_ntb *ndev = ntb_ndev(ntb);
1047
1048         return ndev->num_mws;
1049 }
1050
1051 static int vntb_epf_spad_count(struct ntb_dev *ntb)
1052 {
1053         return ntb_ndev(ntb)->spad_count;
1054 }
1055
1056 static int vntb_epf_peer_mw_count(struct ntb_dev *ntb)
1057 {
1058         return ntb_ndev(ntb)->num_mws;
1059 }
1060
1061 static u64 vntb_epf_db_valid_mask(struct ntb_dev *ntb)
1062 {
1063         return BIT_ULL(ntb_ndev(ntb)->db_count) - 1;
1064 }
1065
1066 static int vntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1067 {
1068         return 0;
1069 }
1070
1071 static int vntb_epf_mw_set_trans(struct ntb_dev *ndev, int pidx, int idx,
1072                 dma_addr_t addr, resource_size_t size)
1073 {
1074         struct epf_ntb *ntb = ntb_ndev(ndev);
1075         struct pci_epf_bar *epf_bar;
1076         enum pci_barno barno;
1077         int ret;
1078         struct device *dev;
1079
1080         dev = &ntb->ntb.dev;
1081         barno = ntb->epf_ntb_bar[BAR_MW0 + idx];
1082         epf_bar = &ntb->epf->bar[barno];
1083         epf_bar->phys_addr = addr;
1084         epf_bar->barno = barno;
1085         epf_bar->size = size;
1086
1087         ret = pci_epc_set_bar(ntb->epf->epc, 0, 0, epf_bar);
1088         if (ret) {
1089                 dev_err(dev, "failure set mw trans\n");
1090                 return ret;
1091         }
1092         return 0;
1093 }
1094
1095 static int vntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx)
1096 {
1097         return 0;
1098 }
1099
1100 static int vntb_epf_peer_mw_get_addr(struct ntb_dev *ndev, int idx,
1101                                 phys_addr_t *base, resource_size_t *size)
1102 {
1103
1104         struct epf_ntb *ntb = ntb_ndev(ndev);
1105
1106         if (base)
1107                 *base = ntb->vpci_mw_phy[idx];
1108
1109         if (size)
1110                 *size = ntb->mws_size[idx];
1111
1112         return 0;
1113 }
1114
1115 static int vntb_epf_link_enable(struct ntb_dev *ntb,
1116                         enum ntb_speed max_speed,
1117                         enum ntb_width max_width)
1118 {
1119         return 0;
1120 }
1121
1122 static u32 vntb_epf_spad_read(struct ntb_dev *ndev, int idx)
1123 {
1124         struct epf_ntb *ntb = ntb_ndev(ndev);
1125         int off = ntb->reg->spad_offset, ct = ntb->reg->spad_count * 4;
1126         u32 val;
1127         void __iomem *base = ntb->reg;
1128
1129         val = readl(base + off + ct + idx * 4);
1130         return val;
1131 }
1132
1133 static int vntb_epf_spad_write(struct ntb_dev *ndev, int idx, u32 val)
1134 {
1135         struct epf_ntb *ntb = ntb_ndev(ndev);
1136         struct epf_ntb_ctrl *ctrl = ntb->reg;
1137         int off = ctrl->spad_offset, ct = ctrl->spad_count * 4;
1138         void __iomem *base = ntb->reg;
1139
1140         writel(val, base + off + ct + idx * 4);
1141         return 0;
1142 }
1143
1144 static u32 vntb_epf_peer_spad_read(struct ntb_dev *ndev, int pidx, int idx)
1145 {
1146         struct epf_ntb *ntb = ntb_ndev(ndev);
1147         struct epf_ntb_ctrl *ctrl = ntb->reg;
1148         int off = ctrl->spad_offset;
1149         void __iomem *base = ntb->reg;
1150         u32 val;
1151
1152         val = readl(base + off + idx * 4);
1153         return val;
1154 }
1155
1156 static int vntb_epf_peer_spad_write(struct ntb_dev *ndev, int pidx, int idx, u32 val)
1157 {
1158         struct epf_ntb *ntb = ntb_ndev(ndev);
1159         struct epf_ntb_ctrl *ctrl = ntb->reg;
1160         int off = ctrl->spad_offset;
1161         void __iomem *base = ntb->reg;
1162
1163         writel(val, base + off + idx * 4);
1164         return 0;
1165 }
1166
1167 static int vntb_epf_peer_db_set(struct ntb_dev *ndev, u64 db_bits)
1168 {
1169         u32 interrupt_num = ffs(db_bits) + 1;
1170         struct epf_ntb *ntb = ntb_ndev(ndev);
1171         u8 func_no, vfunc_no;
1172         int ret;
1173
1174         func_no = ntb->epf->func_no;
1175         vfunc_no = ntb->epf->vfunc_no;
1176
1177         ret = pci_epc_raise_irq(ntb->epf->epc,
1178                                 func_no,
1179                                 vfunc_no,
1180                                 PCI_EPC_IRQ_MSI,
1181                                 interrupt_num + 1);
1182         if (ret)
1183                 dev_err(&ntb->ntb.dev, "Failed to raise IRQ\n");
1184
1185         return ret;
1186 }
1187
1188 static u64 vntb_epf_db_read(struct ntb_dev *ndev)
1189 {
1190         struct epf_ntb *ntb = ntb_ndev(ndev);
1191
1192         return ntb->db;
1193 }
1194
1195 static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx,
1196                         resource_size_t *addr_align,
1197                         resource_size_t *size_align,
1198                         resource_size_t *size_max)
1199 {
1200         struct epf_ntb *ntb = ntb_ndev(ndev);
1201
1202         if (addr_align)
1203                 *addr_align = SZ_4K;
1204
1205         if (size_align)
1206                 *size_align = 1;
1207
1208         if (size_max)
1209                 *size_max = ntb->mws_size[idx];
1210
1211         return 0;
1212 }
1213
1214 static u64 vntb_epf_link_is_up(struct ntb_dev *ndev,
1215                         enum ntb_speed *speed,
1216                         enum ntb_width *width)
1217 {
1218         struct epf_ntb *ntb = ntb_ndev(ndev);
1219
1220         return ntb->reg->link_status;
1221 }
1222
1223 static int vntb_epf_db_clear_mask(struct ntb_dev *ndev, u64 db_bits)
1224 {
1225         return 0;
1226 }
1227
1228 static int vntb_epf_db_clear(struct ntb_dev *ndev, u64 db_bits)
1229 {
1230         struct epf_ntb *ntb = ntb_ndev(ndev);
1231
1232         ntb->db &= ~db_bits;
1233         return 0;
1234 }
1235
1236 static int vntb_epf_link_disable(struct ntb_dev *ntb)
1237 {
1238         return 0;
1239 }
1240
1241 static const struct ntb_dev_ops vntb_epf_ops = {
1242         .mw_count               = vntb_epf_mw_count,
1243         .spad_count             = vntb_epf_spad_count,
1244         .peer_mw_count          = vntb_epf_peer_mw_count,
1245         .db_valid_mask          = vntb_epf_db_valid_mask,
1246         .db_set_mask            = vntb_epf_db_set_mask,
1247         .mw_set_trans           = vntb_epf_mw_set_trans,
1248         .mw_clear_trans         = vntb_epf_mw_clear_trans,
1249         .peer_mw_get_addr       = vntb_epf_peer_mw_get_addr,
1250         .link_enable            = vntb_epf_link_enable,
1251         .spad_read              = vntb_epf_spad_read,
1252         .spad_write             = vntb_epf_spad_write,
1253         .peer_spad_read         = vntb_epf_peer_spad_read,
1254         .peer_spad_write        = vntb_epf_peer_spad_write,
1255         .peer_db_set            = vntb_epf_peer_db_set,
1256         .db_read                = vntb_epf_db_read,
1257         .mw_get_align           = vntb_epf_mw_get_align,
1258         .link_is_up             = vntb_epf_link_is_up,
1259         .db_clear_mask          = vntb_epf_db_clear_mask,
1260         .db_clear               = vntb_epf_db_clear,
1261         .link_disable           = vntb_epf_link_disable,
1262 };
1263
1264 static int pci_vntb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1265 {
1266         int ret;
1267         struct epf_ntb *ndev = (struct epf_ntb *)pdev->sysdata;
1268         struct device *dev = &pdev->dev;
1269
1270         ndev->ntb.pdev = pdev;
1271         ndev->ntb.topo = NTB_TOPO_NONE;
1272         ndev->ntb.ops =  &vntb_epf_ops;
1273
1274         ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
1275         if (ret) {
1276                 dev_err(dev, "Cannot set DMA mask\n");
1277                 return -EINVAL;
1278         }
1279
1280         ret = ntb_register_device(&ndev->ntb);
1281         if (ret) {
1282                 dev_err(dev, "Failed to register NTB device\n");
1283                 goto err_register_dev;
1284         }
1285
1286         dev_dbg(dev, "PCI Virtual NTB driver loaded\n");
1287         return 0;
1288
1289 err_register_dev:
1290         return -EINVAL;
1291 }
1292
1293 static struct pci_device_id pci_vntb_table[] = {
1294         {
1295                 PCI_DEVICE(0xffff, 0xffff),
1296         },
1297         {},
1298 };
1299
1300 static struct pci_driver vntb_pci_driver = {
1301         .name           = "pci-vntb",
1302         .id_table       = pci_vntb_table,
1303         .probe          = pci_vntb_probe,
1304 };
1305
1306 /* ============ PCIe EPF Driver Bind ====================*/
1307
1308 /**
1309  * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality
1310  * @epf: NTB endpoint function device
1311  *
1312  * Initialize both the endpoint controllers associated with NTB function device.
1313  * Invoked when a primary interface or secondary interface is bound to EPC
1314  * device. This function will succeed only when EPC is bound to both the
1315  * interfaces.
1316  *
1317  * Returns: Zero for success, or an error code in case of failure
1318  */
1319 static int epf_ntb_bind(struct pci_epf *epf)
1320 {
1321         struct epf_ntb *ntb = epf_get_drvdata(epf);
1322         struct device *dev = &epf->dev;
1323         int ret;
1324
1325         if (!epf->epc) {
1326                 dev_dbg(dev, "PRIMARY EPC interface not yet bound\n");
1327                 return 0;
1328         }
1329
1330         ret = epf_ntb_init_epc_bar(ntb);
1331         if (ret) {
1332                 dev_err(dev, "Failed to create NTB EPC\n");
1333                 goto err_bar_init;
1334         }
1335
1336         ret = epf_ntb_config_spad_bar_alloc(ntb);
1337         if (ret) {
1338                 dev_err(dev, "Failed to allocate BAR memory\n");
1339                 goto err_bar_alloc;
1340         }
1341
1342         ret = epf_ntb_epc_init(ntb);
1343         if (ret) {
1344                 dev_err(dev, "Failed to initialize EPC\n");
1345                 goto err_bar_alloc;
1346         }
1347
1348         epf_set_drvdata(epf, ntb);
1349
1350         pci_space[0] = (ntb->vntb_pid << 16) | ntb->vntb_vid;
1351         pci_vntb_table[0].vendor = ntb->vntb_vid;
1352         pci_vntb_table[0].device = ntb->vntb_pid;
1353
1354         ret = pci_register_driver(&vntb_pci_driver);
1355         if (ret) {
1356                 dev_err(dev, "failure register vntb pci driver\n");
1357                 goto err_bar_alloc;
1358         }
1359
1360         vpci_scan_bus(ntb);
1361
1362         return 0;
1363
1364 err_bar_alloc:
1365         epf_ntb_config_spad_bar_free(ntb);
1366
1367 err_bar_init:
1368         epf_ntb_epc_destroy(ntb);
1369
1370         return ret;
1371 }
1372
1373 /**
1374  * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind()
1375  * @epf: NTB endpoint function device
1376  *
1377  * Cleanup the initialization from epf_ntb_bind()
1378  */
1379 static void epf_ntb_unbind(struct pci_epf *epf)
1380 {
1381         struct epf_ntb *ntb = epf_get_drvdata(epf);
1382
1383         epf_ntb_epc_cleanup(ntb);
1384         epf_ntb_config_spad_bar_free(ntb);
1385         epf_ntb_epc_destroy(ntb);
1386
1387         pci_unregister_driver(&vntb_pci_driver);
1388 }
1389
1390 // EPF driver probe
1391 static struct pci_epf_ops epf_ntb_ops = {
1392         .bind   = epf_ntb_bind,
1393         .unbind = epf_ntb_unbind,
1394         .add_cfs = epf_ntb_add_cfs,
1395 };
1396
1397 /**
1398  * epf_ntb_probe() - Probe NTB function driver
1399  * @epf: NTB endpoint function device
1400  *
1401  * Probe NTB function driver when endpoint function bus detects a NTB
1402  * endpoint function.
1403  *
1404  * Returns: Zero for success, or an error code in case of failure
1405  */
1406 static int epf_ntb_probe(struct pci_epf *epf)
1407 {
1408         struct epf_ntb *ntb;
1409         struct device *dev;
1410
1411         dev = &epf->dev;
1412
1413         ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL);
1414         if (!ntb)
1415                 return -ENOMEM;
1416
1417         epf->header = &epf_ntb_header;
1418         ntb->epf = epf;
1419         ntb->vbus_number = 0xff;
1420         epf_set_drvdata(epf, ntb);
1421
1422         dev_info(dev, "pci-ep epf driver loaded\n");
1423         return 0;
1424 }
1425
1426 static const struct pci_epf_device_id epf_ntb_ids[] = {
1427         {
1428                 .name = "pci_epf_vntb",
1429         },
1430         {},
1431 };
1432
1433 static struct pci_epf_driver epf_ntb_driver = {
1434         .driver.name    = "pci_epf_vntb",
1435         .probe          = epf_ntb_probe,
1436         .id_table       = epf_ntb_ids,
1437         .ops            = &epf_ntb_ops,
1438         .owner          = THIS_MODULE,
1439 };
1440
1441 static int __init epf_ntb_init(void)
1442 {
1443         int ret;
1444
1445         kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM |
1446                                             WQ_HIGHPRI, 0);
1447         ret = pci_epf_register_driver(&epf_ntb_driver);
1448         if (ret) {
1449                 destroy_workqueue(kpcintb_workqueue);
1450                 pr_err("Failed to register pci epf ntb driver --> %d\n", ret);
1451                 return ret;
1452         }
1453
1454         return 0;
1455 }
1456 module_init(epf_ntb_init);
1457
1458 static void __exit epf_ntb_exit(void)
1459 {
1460         pci_epf_unregister_driver(&epf_ntb_driver);
1461         destroy_workqueue(kpcintb_workqueue);
1462 }
1463 module_exit(epf_ntb_exit);
1464
1465 MODULE_DESCRIPTION("PCI EPF NTB DRIVER");
1466 MODULE_AUTHOR("Frank Li <Frank.li@nxp.com>");
1467 MODULE_LICENSE("GPL v2");