drivers/vdpa/pds/vdpa_dev.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */
   3
   4 #include <linux/pci.h>
   5 #include <linux/vdpa.h>
   6 #include <uapi/linux/vdpa.h>
   7 #include <linux/virtio_pci_modern.h>
   8
   9 #include <linux/pds/pds_common.h>
  10 #include <linux/pds/pds_core_if.h>
  11 #include <linux/pds/pds_adminq.h>
  12 #include <linux/pds/pds_auxbus.h>
  13
  14 #include "vdpa_dev.h"
  15 #include "aux_drv.h"
  16 #include "cmds.h"
  17 #include "debugfs.h"
  18
  19 static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev);
  20
  21 static struct pds_vdpa_device *vdpa_to_pdsv(struct vdpa_device *vdpa_dev)
  22 {
  23         return container_of(vdpa_dev, struct pds_vdpa_device, vdpa_dev);
  24 }
  25
  26 static int pds_vdpa_notify_handler(struct notifier_block *nb,
  27                                    unsigned long ecode,
  28                                    void *data)
  29 {
  30         struct pds_vdpa_device *pdsv = container_of(nb, struct pds_vdpa_device, nb);
  31         struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev;
  32
  33         dev_dbg(dev, "%s: event code %lu\n", __func__, ecode);
  34
  35         if (ecode == PDS_EVENT_RESET || ecode == PDS_EVENT_LINK_CHANGE) {
  36                 if (pdsv->config_cb.callback)
  37                         pdsv->config_cb.callback(pdsv->config_cb.private);
  38         }
  39
  40         return 0;
  41 }
  42
  43 static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv)
  44 {
  45         struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev;
  46         struct notifier_block *nb = &pdsv->nb;
  47         int err;
  48
  49         if (!nb->notifier_call) {
  50                 nb->notifier_call = pds_vdpa_notify_handler;
  51                 err = pdsc_register_notify(nb);
  52                 if (err) {
  53                         nb->notifier_call = NULL;
  54                         dev_err(dev, "failed to register pds event handler: %ps\n",
  55                                 ERR_PTR(err));
  56                         return -EINVAL;
  57                 }
  58                 dev_dbg(dev, "pds event handler registered\n");
  59         }
  60
  61         return 0;
  62 }
  63
  64 static void pds_vdpa_unregister_event_handler(struct pds_vdpa_device *pdsv)
  65 {
  66         if (pdsv->nb.notifier_call) {
  67                 pdsc_unregister_notify(&pdsv->nb);
  68                 pdsv->nb.notifier_call = NULL;
  69         }
  70 }
  71
  72 static int pds_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,
  73                                    u64 desc_addr, u64 driver_addr, u64 device_addr)
  74 {
  75         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
  76
  77         pdsv->vqs[qid].desc_addr = desc_addr;
  78         pdsv->vqs[qid].avail_addr = driver_addr;
  79         pdsv->vqs[qid].used_addr = device_addr;
  80
  81         return 0;
  82 }
  83
  84 static void pds_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, u32 num)
  85 {
  86         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
  87
  88         pdsv->vqs[qid].q_len = num;
  89 }
  90
  91 static void pds_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid)
  92 {
  93         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
  94
  95         iowrite16(qid, pdsv->vqs[qid].notify);
  96 }
  97
  98 static void pds_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid,
  99                                struct vdpa_callback *cb)
 100 {
 101         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 102
 103         pdsv->vqs[qid].event_cb = *cb;
 104 }
 105
 106 static irqreturn_t pds_vdpa_isr(int irq, void *data)
 107 {
 108         struct pds_vdpa_vq_info *vq;
 109
 110         vq = data;
 111         if (vq->event_cb.callback)
 112                 vq->event_cb.callback(vq->event_cb.private);
 113
 114         return IRQ_HANDLED;
 115 }
 116
 117 static void pds_vdpa_release_irq(struct pds_vdpa_device *pdsv, int qid)
 118 {
 119         if (pdsv->vqs[qid].irq == VIRTIO_MSI_NO_VECTOR)
 120                 return;
 121
 122         free_irq(pdsv->vqs[qid].irq, &pdsv->vqs[qid]);
 123         pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR;
 124 }
 125
 126 static void pds_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool ready)
 127 {
 128         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 129         struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev;
 130         struct device *dev = &pdsv->vdpa_dev.dev;
 131         u64 driver_features;
 132         u16 invert_idx = 0;
 133         int irq;
 134         int err;
 135
 136         dev_dbg(dev, "%s: qid %d ready %d => %d\n",
 137                 __func__, qid, pdsv->vqs[qid].ready, ready);
 138         if (ready == pdsv->vqs[qid].ready)
 139                 return;
 140
 141         driver_features = pds_vdpa_get_driver_features(vdpa_dev);
 142         if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
 143                 invert_idx = PDS_VDPA_PACKED_INVERT_IDX;
 144
 145         if (ready) {
 146                 irq = pci_irq_vector(pdev, qid);
 147                 snprintf(pdsv->vqs[qid].irq_name, sizeof(pdsv->vqs[qid].irq_name),
 148                          "vdpa-%s-%d", dev_name(dev), qid);
 149
 150                 err = request_irq(irq, pds_vdpa_isr, 0,
 151                                   pdsv->vqs[qid].irq_name, &pdsv->vqs[qid]);
 152                 if (err) {
 153                         dev_err(dev, "%s: no irq for qid %d: %pe\n",
 154                                 __func__, qid, ERR_PTR(err));
 155                         return;
 156                 }
 157                 pdsv->vqs[qid].irq = irq;
 158
 159                 /* Pass vq setup info to DSC using adminq to gather up and
 160                  * send all info at once so FW can do its full set up in
 161                  * one easy operation
 162                  */
 163                 err = pds_vdpa_cmd_init_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]);
 164                 if (err) {
 165                         dev_err(dev, "Failed to init vq %d: %pe\n",
 166                                 qid, ERR_PTR(err));
 167                         pds_vdpa_release_irq(pdsv, qid);
 168                         ready = false;
 169                 }
 170         } else {
 171                 err = pds_vdpa_cmd_reset_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]);
 172                 if (err)
 173                         dev_err(dev, "%s: reset_vq failed qid %d: %pe\n",
 174                                 __func__, qid, ERR_PTR(err));
 175                 pds_vdpa_release_irq(pdsv, qid);
 176         }
 177
 178         pdsv->vqs[qid].ready = ready;
 179 }
 180
 181 static bool pds_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid)
 182 {
 183         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 184
 185         return pdsv->vqs[qid].ready;
 186 }
 187
 188 static int pds_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
 189                                  const struct vdpa_vq_state *state)
 190 {
 191         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 192         struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
 193         struct device *dev = &padev->aux_dev.dev;
 194         u64 driver_features;
 195         u16 avail;
 196         u16 used;
 197
 198         if (pdsv->vqs[qid].ready) {
 199                 dev_err(dev, "Setting device position is denied while vq is enabled\n");
 200                 return -EINVAL;
 201         }
 202
 203         driver_features = pds_vdpa_get_driver_features(vdpa_dev);
 204         if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
 205                 avail = state->packed.last_avail_idx |
 206                         (state->packed.last_avail_counter << 15);
 207                 used = state->packed.last_used_idx |
 208                        (state->packed.last_used_counter << 15);
 209
 210                 /* The avail and used index are stored with the packed wrap
 211                  * counter bit inverted.  This way, in case set_vq_state is
 212                  * not called, the initial value can be set to zero prior to
 213                  * feature negotiation, and it is good for both packed and
 214                  * split vq.
 215                  */
 216                 avail ^= PDS_VDPA_PACKED_INVERT_IDX;
 217                 used ^= PDS_VDPA_PACKED_INVERT_IDX;
 218         } else {
 219                 avail = state->split.avail_index;
 220                 /* state->split does not provide a used_index:
 221                  * the vq will be set to "empty" here, and the vq will read
 222                  * the current used index the next time the vq is kicked.
 223                  */
 224                 used = avail;
 225         }
 226
 227         if (used != avail) {
 228                 dev_dbg(dev, "Setting used equal to avail, for interoperability\n");
 229                 used = avail;
 230         }
 231
 232         pdsv->vqs[qid].avail_idx = avail;
 233         pdsv->vqs[qid].used_idx = used;
 234
 235         return 0;
 236 }
 237
 238 static int pds_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
 239                                  struct vdpa_vq_state *state)
 240 {
 241         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 242         struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
 243         struct device *dev = &padev->aux_dev.dev;
 244         u64 driver_features;
 245         u16 avail;
 246         u16 used;
 247
 248         if (pdsv->vqs[qid].ready) {
 249                 dev_err(dev, "Getting device position is denied while vq is enabled\n");
 250                 return -EINVAL;
 251         }
 252
 253         avail = pdsv->vqs[qid].avail_idx;
 254         used = pdsv->vqs[qid].used_idx;
 255
 256         driver_features = pds_vdpa_get_driver_features(vdpa_dev);
 257         if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
 258                 avail ^= PDS_VDPA_PACKED_INVERT_IDX;
 259                 used ^= PDS_VDPA_PACKED_INVERT_IDX;
 260
 261                 state->packed.last_avail_idx = avail & 0x7fff;
 262                 state->packed.last_avail_counter = avail >> 15;
 263                 state->packed.last_used_idx = used & 0x7fff;
 264                 state->packed.last_used_counter = used >> 15;
 265         } else {
 266                 state->split.avail_index = avail;
 267                 /* state->split does not provide a used_index. */
 268         }
 269
 270         return 0;
 271 }
 272
 273 static struct vdpa_notification_area
 274 pds_vdpa_get_vq_notification(struct vdpa_device *vdpa_dev, u16 qid)
 275 {
 276         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 277         struct virtio_pci_modern_device *vd_mdev;
 278         struct vdpa_notification_area area;
 279
 280         area.addr = pdsv->vqs[qid].notify_pa;
 281
 282         vd_mdev = &pdsv->vdpa_aux->vd_mdev;
 283         if (!vd_mdev->notify_offset_multiplier)
 284                 area.size = PDS_PAGE_SIZE;
 285         else
 286                 area.size = vd_mdev->notify_offset_multiplier;
 287
 288         return area;
 289 }
 290
 291 static int pds_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev, u16 qid)
 292 {
 293         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 294
 295         return pdsv->vqs[qid].irq;
 296 }
 297
 298 static u32 pds_vdpa_get_vq_align(struct vdpa_device *vdpa_dev)
 299 {
 300         return PDS_PAGE_SIZE;
 301 }
 302
 303 static u32 pds_vdpa_get_vq_group(struct vdpa_device *vdpa_dev, u16 idx)
 304 {
 305         return 0;
 306 }
 307
 308 static u64 pds_vdpa_get_device_features(struct vdpa_device *vdpa_dev)
 309 {
 310         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 311
 312         return pdsv->supported_features;
 313 }
 314
 315 static int pds_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features)
 316 {
 317         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 318         struct device *dev = &pdsv->vdpa_dev.dev;
 319         u64 driver_features;
 320         u64 nego_features;
 321         u64 missing;
 322
 323         if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)) && features) {
 324                 dev_err(dev, "VIRTIO_F_ACCESS_PLATFORM is not negotiated\n");
 325                 return -EOPNOTSUPP;
 326         }
 327
 328         pdsv->req_features = features;
 329
 330         /* Check for valid feature bits */
 331         nego_features = features & le64_to_cpu(pdsv->vdpa_aux->ident.hw_features);
 332         missing = pdsv->req_features & ~nego_features;
 333         if (missing) {
 334                 dev_err(dev, "Can't support all requested features in %#llx, missing %#llx features\n",
 335                         pdsv->req_features, missing);
 336                 return -EOPNOTSUPP;
 337         }
 338
 339         driver_features = pds_vdpa_get_driver_features(vdpa_dev);
 340         dev_dbg(dev, "%s: %#llx => %#llx\n",
 341                 __func__, driver_features, nego_features);
 342
 343         if (driver_features == nego_features)
 344                 return 0;
 345
 346         vp_modern_set_features(&pdsv->vdpa_aux->vd_mdev, nego_features);
 347
 348         return 0;
 349 }
 350
 351 static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev)
 352 {
 353         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 354
 355         return vp_modern_get_driver_features(&pdsv->vdpa_aux->vd_mdev);
 356 }
 357
 358 static void pds_vdpa_set_config_cb(struct vdpa_device *vdpa_dev,
 359                                    struct vdpa_callback *cb)
 360 {
 361         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 362
 363         pdsv->config_cb.callback = cb->callback;
 364         pdsv->config_cb.private = cb->private;
 365 }
 366
 367 static u16 pds_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)
 368 {
 369         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 370
 371         /* qemu has assert() that vq_num_max <= VIRTQUEUE_MAX_SIZE (1024) */
 372         return min_t(u16, 1024, BIT(le16_to_cpu(pdsv->vdpa_aux->ident.max_qlen)));
 373 }
 374
 375 static u32 pds_vdpa_get_device_id(struct vdpa_device *vdpa_dev)
 376 {
 377         return VIRTIO_ID_NET;
 378 }
 379
 380 static u32 pds_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev)
 381 {
 382         return PCI_VENDOR_ID_PENSANDO;
 383 }
 384
 385 static u8 pds_vdpa_get_status(struct vdpa_device *vdpa_dev)
 386 {
 387         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 388
 389         return vp_modern_get_status(&pdsv->vdpa_aux->vd_mdev);
 390 }
 391
 392 static void pds_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
 393 {
 394         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 395         struct device *dev = &pdsv->vdpa_dev.dev;
 396         u8 old_status;
 397         int i;
 398
 399         old_status = pds_vdpa_get_status(vdpa_dev);
 400         dev_dbg(dev, "%s: old %#x new %#x\n", __func__, old_status, status);
 401
 402         pds_vdpa_cmd_set_status(pdsv, status);
 403
 404         /* Note: still working with FW on the need for this reset cmd */
 405         if (status == 0) {
 406                 pds_vdpa_cmd_reset(pdsv);
 407
 408                 for (i = 0; i < pdsv->num_vqs; i++) {
 409                         pdsv->vqs[i].avail_idx = 0;
 410                         pdsv->vqs[i].used_idx = 0;
 411                 }
 412         }
 413
 414         if (status & ~old_status & VIRTIO_CONFIG_S_FEATURES_OK) {
 415                 for (i = 0; i < pdsv->num_vqs; i++) {
 416                         pdsv->vqs[i].notify =
 417                                 vp_modern_map_vq_notify(&pdsv->vdpa_aux->vd_mdev,
 418                                                         i, &pdsv->vqs[i].notify_pa);
 419                 }
 420         }
 421 }
 422
 423 static int pds_vdpa_reset(struct vdpa_device *vdpa_dev)
 424 {
 425         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 426         struct device *dev;
 427         int err = 0;
 428         u8 status;
 429         int i;
 430
 431         dev = &pdsv->vdpa_aux->padev->aux_dev.dev;
 432         status = pds_vdpa_get_status(vdpa_dev);
 433
 434         if (status == 0)
 435                 return 0;
 436
 437         if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
 438                 /* Reset the vqs */
 439                 for (i = 0; i < pdsv->num_vqs && !err; i++) {
 440                         err = pds_vdpa_cmd_reset_vq(pdsv, i, 0, &pdsv->vqs[i]);
 441                         if (err)
 442                                 dev_err(dev, "%s: reset_vq failed qid %d: %pe\n",
 443                                         __func__, i, ERR_PTR(err));
 444                         pds_vdpa_release_irq(pdsv, i);
 445                         memset(&pdsv->vqs[i], 0, sizeof(pdsv->vqs[0]));
 446                         pdsv->vqs[i].ready = false;
 447                 }
 448         }
 449
 450         pds_vdpa_set_status(vdpa_dev, 0);
 451
 452         return 0;
 453 }
 454
 455 static size_t pds_vdpa_get_config_size(struct vdpa_device *vdpa_dev)
 456 {
 457         return sizeof(struct virtio_net_config);
 458 }
 459
 460 static void pds_vdpa_get_config(struct vdpa_device *vdpa_dev,
 461                                 unsigned int offset,
 462                                 void *buf, unsigned int len)
 463 {
 464         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 465         void __iomem *device;
 466
 467         if (offset + len > sizeof(struct virtio_net_config)) {
 468                 WARN(true, "%s: bad read, offset %d len %d\n", __func__, offset, len);
 469                 return;
 470         }
 471
 472         device = pdsv->vdpa_aux->vd_mdev.device;
 473         memcpy_fromio(buf, device + offset, len);
 474 }
 475
 476 static void pds_vdpa_set_config(struct vdpa_device *vdpa_dev,
 477                                 unsigned int offset, const void *buf,
 478                                 unsigned int len)
 479 {
 480         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 481         void __iomem *device;
 482
 483         if (offset + len > sizeof(struct virtio_net_config)) {
 484                 WARN(true, "%s: bad read, offset %d len %d\n", __func__, offset, len);
 485                 return;
 486         }
 487
 488         device = pdsv->vdpa_aux->vd_mdev.device;
 489         memcpy_toio(device + offset, buf, len);
 490 }
 491
 492 static const struct vdpa_config_ops pds_vdpa_ops = {
 493         .set_vq_address         = pds_vdpa_set_vq_address,
 494         .set_vq_num             = pds_vdpa_set_vq_num,
 495         .kick_vq                = pds_vdpa_kick_vq,
 496         .set_vq_cb              = pds_vdpa_set_vq_cb,
 497         .set_vq_ready           = pds_vdpa_set_vq_ready,
 498         .get_vq_ready           = pds_vdpa_get_vq_ready,
 499         .set_vq_state           = pds_vdpa_set_vq_state,
 500         .get_vq_state           = pds_vdpa_get_vq_state,
 501         .get_vq_notification    = pds_vdpa_get_vq_notification,
 502         .get_vq_irq             = pds_vdpa_get_vq_irq,
 503         .get_vq_align           = pds_vdpa_get_vq_align,
 504         .get_vq_group           = pds_vdpa_get_vq_group,
 505
 506         .get_device_features    = pds_vdpa_get_device_features,
 507         .set_driver_features    = pds_vdpa_set_driver_features,
 508         .get_driver_features    = pds_vdpa_get_driver_features,
 509         .set_config_cb          = pds_vdpa_set_config_cb,
 510         .get_vq_num_max         = pds_vdpa_get_vq_num_max,
 511         .get_device_id          = pds_vdpa_get_device_id,
 512         .get_vendor_id          = pds_vdpa_get_vendor_id,
 513         .get_status             = pds_vdpa_get_status,
 514         .set_status             = pds_vdpa_set_status,
 515         .reset                  = pds_vdpa_reset,
 516         .get_config_size        = pds_vdpa_get_config_size,
 517         .get_config             = pds_vdpa_get_config,
 518         .set_config             = pds_vdpa_set_config,
 519 };
 520 static struct virtio_device_id pds_vdpa_id_table[] = {
 521         {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
 522         {0},
 523 };
 524
 525 static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
 526                             const struct vdpa_dev_set_config *add_config)
 527 {
 528         struct pds_vdpa_aux *vdpa_aux;
 529         struct pds_vdpa_device *pdsv;
 530         struct vdpa_mgmt_dev *mgmt;
 531         u16 fw_max_vqs, vq_pairs;
 532         struct device *dma_dev;
 533         struct pci_dev *pdev;
 534         struct device *dev;
 535         u8 mac[ETH_ALEN];
 536         int err;
 537         int i;
 538
 539         vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);
 540         dev = &vdpa_aux->padev->aux_dev.dev;
 541         mgmt = &vdpa_aux->vdpa_mdev;
 542
 543         if (vdpa_aux->pdsv) {
 544                 dev_warn(dev, "Multiple vDPA devices on a VF is not supported.\n");
 545                 return -EOPNOTSUPP;
 546         }
 547
 548         pdsv = vdpa_alloc_device(struct pds_vdpa_device, vdpa_dev,
 549                                  dev, &pds_vdpa_ops, 1, 1, name, false);
 550         if (IS_ERR(pdsv)) {
 551                 dev_err(dev, "Failed to allocate vDPA structure: %pe\n", pdsv);
 552                 return PTR_ERR(pdsv);
 553         }
 554
 555         vdpa_aux->pdsv = pdsv;
 556         pdsv->vdpa_aux = vdpa_aux;
 557
 558         pdev = vdpa_aux->padev->vf_pdev;
 559         dma_dev = &pdev->dev;
 560         pdsv->vdpa_dev.dma_dev = dma_dev;
 561
 562         pdsv->supported_features = mgmt->supported_features;
 563
 564         if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
 565                 u64 unsupp_features =
 566                         add_config->device_features & ~mgmt->supported_features;
 567
 568                 if (unsupp_features) {
 569                         dev_err(dev, "Unsupported features: %#llx\n", unsupp_features);
 570                         err = -EOPNOTSUPP;
 571                         goto err_unmap;
 572                 }
 573
 574                 pdsv->supported_features = add_config->device_features;
 575         }
 576
 577         err = pds_vdpa_cmd_reset(pdsv);
 578         if (err) {
 579                 dev_err(dev, "Failed to reset hw: %pe\n", ERR_PTR(err));
 580                 goto err_unmap;
 581         }
 582
 583         err = pds_vdpa_init_hw(pdsv);
 584         if (err) {
 585                 dev_err(dev, "Failed to init hw: %pe\n", ERR_PTR(err));
 586                 goto err_unmap;
 587         }
 588
 589         fw_max_vqs = le16_to_cpu(pdsv->vdpa_aux->ident.max_vqs);
 590         vq_pairs = fw_max_vqs / 2;
 591
 592         /* Make sure we have the queues being requested */
 593         if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MAX_VQP))
 594                 vq_pairs = add_config->net.max_vq_pairs;
 595
 596         pdsv->num_vqs = 2 * vq_pairs;
 597         if (pdsv->supported_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))
 598                 pdsv->num_vqs++;
 599
 600         if (pdsv->num_vqs > fw_max_vqs) {
 601                 dev_err(dev, "%s: queue count requested %u greater than max %u\n",
 602                         __func__, pdsv->num_vqs, fw_max_vqs);
 603                 err = -ENOSPC;
 604                 goto err_unmap;
 605         }
 606
 607         if (pdsv->num_vqs != fw_max_vqs) {
 608                 err = pds_vdpa_cmd_set_max_vq_pairs(pdsv, vq_pairs);
 609                 if (err) {
 610                         dev_err(dev, "Failed to set max_vq_pairs: %pe\n",
 611                                 ERR_PTR(err));
 612                         goto err_unmap;
 613                 }
 614         }
 615
 616         /* Set a mac, either from the user config if provided
 617          * or set a random mac if default is 00:..:00
 618          */
 619         if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR)) {
 620                 ether_addr_copy(mac, add_config->net.mac);
 621                 pds_vdpa_cmd_set_mac(pdsv, mac);
 622         } else {
 623                 struct virtio_net_config __iomem *vc;
 624
 625                 vc = pdsv->vdpa_aux->vd_mdev.device;
 626                 memcpy_fromio(mac, vc->mac, sizeof(mac));
 627                 if (is_zero_ether_addr(mac)) {
 628                         eth_random_addr(mac);
 629                         dev_info(dev, "setting random mac %pM\n", mac);
 630                         pds_vdpa_cmd_set_mac(pdsv, mac);
 631                 }
 632         }
 633
 634         for (i = 0; i < pdsv->num_vqs; i++) {
 635                 pdsv->vqs[i].qid = i;
 636                 pdsv->vqs[i].pdsv = pdsv;
 637                 pdsv->vqs[i].irq = VIRTIO_MSI_NO_VECTOR;
 638                 pdsv->vqs[i].notify = vp_modern_map_vq_notify(&pdsv->vdpa_aux->vd_mdev,
 639                                                               i, &pdsv->vqs[i].notify_pa);
 640         }
 641
 642         pdsv->vdpa_dev.mdev = &vdpa_aux->vdpa_mdev;
 643
 644         err = pds_vdpa_register_event_handler(pdsv);
 645         if (err) {
 646                 dev_err(dev, "Failed to register for PDS events: %pe\n", ERR_PTR(err));
 647                 goto err_unmap;
 648         }
 649
 650         /* We use the _vdpa_register_device() call rather than the
 651          * vdpa_register_device() to avoid a deadlock because our
 652          * dev_add() is called with the vdpa_dev_lock already set
 653          * by vdpa_nl_cmd_dev_add_set_doit()
 654          */
 655         err = _vdpa_register_device(&pdsv->vdpa_dev, pdsv->num_vqs);
 656         if (err) {
 657                 dev_err(dev, "Failed to register to vDPA bus: %pe\n", ERR_PTR(err));
 658                 goto err_unevent;
 659         }
 660
 661         pds_vdpa_debugfs_add_vdpadev(vdpa_aux);
 662
 663         return 0;
 664
 665 err_unevent:
 666         pds_vdpa_unregister_event_handler(pdsv);
 667 err_unmap:
 668         put_device(&pdsv->vdpa_dev.dev);
 669         vdpa_aux->pdsv = NULL;
 670         return err;
 671 }
 672
 673 static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
 674                              struct vdpa_device *vdpa_dev)
 675 {
 676         struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 677         struct pds_vdpa_aux *vdpa_aux;
 678
 679         pds_vdpa_unregister_event_handler(pdsv);
 680
 681         vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);
 682         _vdpa_unregister_device(vdpa_dev);
 683
 684         pds_vdpa_cmd_reset(vdpa_aux->pdsv);
 685         pds_vdpa_debugfs_reset_vdpadev(vdpa_aux);
 686
 687         vdpa_aux->pdsv = NULL;
 688
 689         dev_info(&vdpa_aux->padev->aux_dev.dev, "Removed vdpa device\n");
 690 }
 691
 692 static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = {
 693         .dev_add = pds_vdpa_dev_add,
 694         .dev_del = pds_vdpa_dev_del
 695 };
 696
 697 int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)
 698 {
 699         union pds_core_adminq_cmd cmd = {
 700                 .vdpa_ident.opcode = PDS_VDPA_CMD_IDENT,
 701                 .vdpa_ident.vf_id = cpu_to_le16(vdpa_aux->vf_id),
 702         };
 703         union pds_core_adminq_comp comp = {};
 704         struct vdpa_mgmt_dev *mgmt;
 705         struct pci_dev *pf_pdev;
 706         struct device *pf_dev;
 707         struct pci_dev *pdev;
 708         dma_addr_t ident_pa;
 709         struct device *dev;
 710         u16 dev_intrs;
 711         u16 max_vqs;
 712         int err;
 713
 714         dev = &vdpa_aux->padev->aux_dev.dev;
 715         pdev = vdpa_aux->padev->vf_pdev;
 716         mgmt = &vdpa_aux->vdpa_mdev;
 717
 718         /* Get resource info through the PF's adminq.  It is a block of info,
 719          * so we need to map some memory for PF to make available to the
 720          * firmware for writing the data.
 721          */
 722         pf_pdev = pci_physfn(vdpa_aux->padev->vf_pdev);
 723         pf_dev = &pf_pdev->dev;
 724         ident_pa = dma_map_single(pf_dev, &vdpa_aux->ident,
 725                                   sizeof(vdpa_aux->ident), DMA_FROM_DEVICE);
 726         if (dma_mapping_error(pf_dev, ident_pa)) {
 727                 dev_err(dev, "Failed to map ident space\n");
 728                 return -ENOMEM;
 729         }
 730
 731         cmd.vdpa_ident.ident_pa = cpu_to_le64(ident_pa);
 732         cmd.vdpa_ident.len = cpu_to_le32(sizeof(vdpa_aux->ident));
 733         err = pds_client_adminq_cmd(vdpa_aux->padev, &cmd,
 734                                     sizeof(cmd.vdpa_ident), &comp, 0);
 735         dma_unmap_single(pf_dev, ident_pa,
 736                          sizeof(vdpa_aux->ident), DMA_FROM_DEVICE);
 737         if (err) {
 738                 dev_err(dev, "Failed to ident hw, status %d: %pe\n",
 739                         comp.status, ERR_PTR(err));
 740                 return err;
 741         }
 742
 743         max_vqs = le16_to_cpu(vdpa_aux->ident.max_vqs);
 744         dev_intrs = pci_msix_vec_count(pdev);
 745         dev_dbg(dev, "ident.max_vqs %d dev_intrs %d\n", max_vqs, dev_intrs);
 746
 747         max_vqs = min_t(u16, dev_intrs, max_vqs);
 748         mgmt->max_supported_vqs = min_t(u16, PDS_VDPA_MAX_QUEUES, max_vqs);
 749         vdpa_aux->nintrs = mgmt->max_supported_vqs;
 750
 751         mgmt->ops = &pds_vdpa_mgmt_dev_ops;
 752         mgmt->id_table = pds_vdpa_id_table;
 753         mgmt->device = dev;
 754         mgmt->supported_features = le64_to_cpu(vdpa_aux->ident.hw_features);
 755         mgmt->config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR);
 756         mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP);
 757         mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_FEATURES);
 758
 759         err = pci_alloc_irq_vectors(pdev, vdpa_aux->nintrs, vdpa_aux->nintrs,
 760                                     PCI_IRQ_MSIX);
 761         if (err < 0) {
 762                 dev_err(dev, "Couldn't get %d msix vectors: %pe\n",
 763                         vdpa_aux->nintrs, ERR_PTR(err));
 764                 return err;
 765         }
 766         vdpa_aux->nintrs = err;
 767
 768         return 0;
 769 }