drivers/net/ethernet/google/gve/gve_main.c

   1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
   2 /* Google virtual Ethernet (gve) driver
   3  *
   4  * Copyright (C) 2015-2019 Google, Inc.
   5  */
   6
   7 #include <linux/cpumask.h>
   8 #include <linux/etherdevice.h>
   9 #include <linux/interrupt.h>
  10 #include <linux/module.h>
  11 #include <linux/pci.h>
  12 #include <linux/sched.h>
  13 #include <linux/timer.h>
  14 #include <linux/workqueue.h>
  15 #include <net/sch_generic.h>
  16 #include "gve.h"
  17 #include "gve_adminq.h"
  18 #include "gve_register.h"
  19
  20 #define GVE_DEFAULT_RX_COPYBREAK        (256)
  21
  22 #define DEFAULT_MSG_LEVEL       (NETIF_MSG_DRV | NETIF_MSG_LINK)
  23 #define GVE_VERSION             "1.0.0"
  24 #define GVE_VERSION_PREFIX      "GVE-"
  25
  26 const char gve_version_str[] = GVE_VERSION;
  27 static const char gve_version_prefix[] = GVE_VERSION_PREFIX;
  28
  29 static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s)
  30 {
  31         struct gve_priv *priv = netdev_priv(dev);
  32         unsigned int start;
  33         int ring;
  34
  35         if (priv->rx) {
  36                 for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) {
  37                         do {
  38                                 start =
  39                                   u64_stats_fetch_begin(&priv->rx[ring].statss);
  40                                 s->rx_packets += priv->rx[ring].rpackets;
  41                                 s->rx_bytes += priv->rx[ring].rbytes;
  42                         } while (u64_stats_fetch_retry(&priv->rx[ring].statss,
  43                                                        start));
  44                 }
  45         }
  46         if (priv->tx) {
  47                 for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) {
  48                         do {
  49                                 start =
  50                                   u64_stats_fetch_begin(&priv->tx[ring].statss);
  51                                 s->tx_packets += priv->tx[ring].pkt_done;
  52                                 s->tx_bytes += priv->tx[ring].bytes_done;
  53                         } while (u64_stats_fetch_retry(&priv->tx[ring].statss,
  54                                                        start));
  55                 }
  56         }
  57 }
  58
  59 static int gve_alloc_counter_array(struct gve_priv *priv)
  60 {
  61         priv->counter_array =
  62                 dma_alloc_coherent(&priv->pdev->dev,
  63                                    priv->num_event_counters *
  64                                    sizeof(*priv->counter_array),
  65                                    &priv->counter_array_bus, GFP_KERNEL);
  66         if (!priv->counter_array)
  67                 return -ENOMEM;
  68
  69         return 0;
  70 }
  71
  72 static void gve_free_counter_array(struct gve_priv *priv)
  73 {
  74         dma_free_coherent(&priv->pdev->dev,
  75                           priv->num_event_counters *
  76                           sizeof(*priv->counter_array),
  77                           priv->counter_array, priv->counter_array_bus);
  78         priv->counter_array = NULL;
  79 }
  80
  81 /* NIC requests to report stats */
  82 static void gve_stats_report_task(struct work_struct *work)
  83 {
  84         struct gve_priv *priv = container_of(work, struct gve_priv,
  85                                              stats_report_task);
  86         if (gve_get_do_report_stats(priv)) {
  87                 gve_handle_report_stats(priv);
  88                 gve_clear_do_report_stats(priv);
  89         }
  90 }
  91
  92 static void gve_stats_report_schedule(struct gve_priv *priv)
  93 {
  94         if (!gve_get_probe_in_progress(priv) &&
  95             !gve_get_reset_in_progress(priv)) {
  96                 gve_set_do_report_stats(priv);
  97                 queue_work(priv->gve_wq, &priv->stats_report_task);
  98         }
  99 }
 100
 101 static void gve_stats_report_timer(struct timer_list *t)
 102 {
 103         struct gve_priv *priv = from_timer(priv, t, stats_report_timer);
 104
 105         mod_timer(&priv->stats_report_timer,
 106                   round_jiffies(jiffies +
 107                   msecs_to_jiffies(priv->stats_report_timer_period)));
 108         gve_stats_report_schedule(priv);
 109 }
 110
 111 static int gve_alloc_stats_report(struct gve_priv *priv)
 112 {
 113         int tx_stats_num, rx_stats_num;
 114
 115         tx_stats_num = (GVE_TX_STATS_REPORT_NUM + NIC_TX_STATS_REPORT_NUM) *
 116                        priv->tx_cfg.num_queues;
 117         rx_stats_num = (GVE_RX_STATS_REPORT_NUM + NIC_RX_STATS_REPORT_NUM) *
 118                        priv->rx_cfg.num_queues;
 119         priv->stats_report_len = struct_size(priv->stats_report, stats,
 120                                              tx_stats_num + rx_stats_num);
 121         priv->stats_report =
 122                 dma_alloc_coherent(&priv->pdev->dev, priv->stats_report_len,
 123                                    &priv->stats_report_bus, GFP_KERNEL);
 124         if (!priv->stats_report)
 125                 return -ENOMEM;
 126         /* Set up timer for the report-stats task */
 127         timer_setup(&priv->stats_report_timer, gve_stats_report_timer, 0);
 128         priv->stats_report_timer_period = GVE_STATS_REPORT_TIMER_PERIOD;
 129         return 0;
 130 }
 131
 132 static void gve_free_stats_report(struct gve_priv *priv)
 133 {
 134         del_timer_sync(&priv->stats_report_timer);
 135         dma_free_coherent(&priv->pdev->dev, priv->stats_report_len,
 136                           priv->stats_report, priv->stats_report_bus);
 137         priv->stats_report = NULL;
 138 }
 139
 140 static irqreturn_t gve_mgmnt_intr(int irq, void *arg)
 141 {
 142         struct gve_priv *priv = arg;
 143
 144         queue_work(priv->gve_wq, &priv->service_task);
 145         return IRQ_HANDLED;
 146 }
 147
 148 static irqreturn_t gve_intr(int irq, void *arg)
 149 {
 150         struct gve_notify_block *block = arg;
 151         struct gve_priv *priv = block->priv;
 152
 153         iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block));
 154         napi_schedule_irqoff(&block->napi);
 155         return IRQ_HANDLED;
 156 }
 157
 158 static int gve_napi_poll(struct napi_struct *napi, int budget)
 159 {
 160         struct gve_notify_block *block;
 161         __be32 __iomem *irq_doorbell;
 162         bool reschedule = false;
 163         struct gve_priv *priv;
 164
 165         block = container_of(napi, struct gve_notify_block, napi);
 166         priv = block->priv;
 167
 168         if (block->tx)
 169                 reschedule |= gve_tx_poll(block, budget);
 170         if (block->rx)
 171                 reschedule |= gve_rx_poll(block, budget);
 172
 173         if (reschedule)
 174                 return budget;
 175
 176         napi_complete(napi);
 177         irq_doorbell = gve_irq_doorbell(priv, block);
 178         iowrite32be(GVE_IRQ_ACK | GVE_IRQ_EVENT, irq_doorbell);
 179
 180         /* Double check we have no extra work.
 181          * Ensure unmask synchronizes with checking for work.
 182          */
 183         dma_rmb();
 184         if (block->tx)
 185                 reschedule |= gve_tx_poll(block, -1);
 186         if (block->rx)
 187                 reschedule |= gve_rx_poll(block, -1);
 188         if (reschedule && napi_reschedule(napi))
 189                 iowrite32be(GVE_IRQ_MASK, irq_doorbell);
 190
 191         return 0;
 192 }
 193
 194 static int gve_alloc_notify_blocks(struct gve_priv *priv)
 195 {
 196         int num_vecs_requested = priv->num_ntfy_blks + 1;
 197         char *name = priv->dev->name;
 198         unsigned int active_cpus;
 199         int vecs_enabled;
 200         int i, j;
 201         int err;
 202
 203         priv->msix_vectors = kvzalloc(num_vecs_requested *
 204                                       sizeof(*priv->msix_vectors), GFP_KERNEL);
 205         if (!priv->msix_vectors)
 206                 return -ENOMEM;
 207         for (i = 0; i < num_vecs_requested; i++)
 208                 priv->msix_vectors[i].entry = i;
 209         vecs_enabled = pci_enable_msix_range(priv->pdev, priv->msix_vectors,
 210                                              GVE_MIN_MSIX, num_vecs_requested);
 211         if (vecs_enabled < 0) {
 212                 dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n",
 213                         GVE_MIN_MSIX, vecs_enabled);
 214                 err = vecs_enabled;
 215                 goto abort_with_msix_vectors;
 216         }
 217         if (vecs_enabled != num_vecs_requested) {
 218                 int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1;
 219                 int vecs_per_type = new_num_ntfy_blks / 2;
 220                 int vecs_left = new_num_ntfy_blks % 2;
 221
 222                 priv->num_ntfy_blks = new_num_ntfy_blks;
 223                 priv->tx_cfg.max_queues = min_t(int, priv->tx_cfg.max_queues,
 224                                                 vecs_per_type);
 225                 priv->rx_cfg.max_queues = min_t(int, priv->rx_cfg.max_queues,
 226                                                 vecs_per_type + vecs_left);
 227                 dev_err(&priv->pdev->dev,
 228                         "Could not enable desired msix, only enabled %d, adjusting tx max queues to %d, and rx max queues to %d\n",
 229                         vecs_enabled, priv->tx_cfg.max_queues,
 230                         priv->rx_cfg.max_queues);
 231                 if (priv->tx_cfg.num_queues > priv->tx_cfg.max_queues)
 232                         priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
 233                 if (priv->rx_cfg.num_queues > priv->rx_cfg.max_queues)
 234                         priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
 235         }
 236         /* Half the notification blocks go to TX and half to RX */
 237         active_cpus = min_t(int, priv->num_ntfy_blks / 2, num_online_cpus());
 238
 239         /* Setup Management Vector  - the last vector */
 240         snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), "%s-mgmnt",
 241                  name);
 242         err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
 243                           gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
 244         if (err) {
 245                 dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
 246                 goto abort_with_msix_enabled;
 247         }
 248         priv->ntfy_blocks =
 249                 dma_alloc_coherent(&priv->pdev->dev,
 250                                    priv->num_ntfy_blks *
 251                                    sizeof(*priv->ntfy_blocks),
 252                                    &priv->ntfy_block_bus, GFP_KERNEL);
 253         if (!priv->ntfy_blocks) {
 254                 err = -ENOMEM;
 255                 goto abort_with_mgmt_vector;
 256         }
 257         /* Setup the other blocks - the first n-1 vectors */
 258         for (i = 0; i < priv->num_ntfy_blks; i++) {
 259                 struct gve_notify_block *block = &priv->ntfy_blocks[i];
 260                 int msix_idx = i;
 261
 262                 snprintf(block->name, sizeof(block->name), "%s-ntfy-block.%d",
 263                          name, i);
 264                 block->priv = priv;
 265                 err = request_irq(priv->msix_vectors[msix_idx].vector,
 266                                   gve_intr, 0, block->name, block);
 267                 if (err) {
 268                         dev_err(&priv->pdev->dev,
 269                                 "Failed to receive msix vector %d\n", i);
 270                         goto abort_with_some_ntfy_blocks;
 271                 }
 272                 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
 273                                       get_cpu_mask(i % active_cpus));
 274         }
 275         return 0;
 276 abort_with_some_ntfy_blocks:
 277         for (j = 0; j < i; j++) {
 278                 struct gve_notify_block *block = &priv->ntfy_blocks[j];
 279                 int msix_idx = j;
 280
 281                 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
 282                                       NULL);
 283                 free_irq(priv->msix_vectors[msix_idx].vector, block);
 284         }
 285         dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
 286                           sizeof(*priv->ntfy_blocks),
 287                           priv->ntfy_blocks, priv->ntfy_block_bus);
 288         priv->ntfy_blocks = NULL;
 289 abort_with_mgmt_vector:
 290         free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
 291 abort_with_msix_enabled:
 292         pci_disable_msix(priv->pdev);
 293 abort_with_msix_vectors:
 294         kvfree(priv->msix_vectors);
 295         priv->msix_vectors = NULL;
 296         return err;
 297 }
 298
 299 static void gve_free_notify_blocks(struct gve_priv *priv)
 300 {
 301         int i;
 302
 303         /* Free the irqs */
 304         for (i = 0; i < priv->num_ntfy_blks; i++) {
 305                 struct gve_notify_block *block = &priv->ntfy_blocks[i];
 306                 int msix_idx = i;
 307
 308                 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
 309                                       NULL);
 310                 free_irq(priv->msix_vectors[msix_idx].vector, block);
 311         }
 312         dma_free_coherent(&priv->pdev->dev,
 313                           priv->num_ntfy_blks * sizeof(*priv->ntfy_blocks),
 314                           priv->ntfy_blocks, priv->ntfy_block_bus);
 315         priv->ntfy_blocks = NULL;
 316         free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
 317         pci_disable_msix(priv->pdev);
 318         kvfree(priv->msix_vectors);
 319         priv->msix_vectors = NULL;
 320 }
 321
 322 static int gve_setup_device_resources(struct gve_priv *priv)
 323 {
 324         int err;
 325
 326         err = gve_alloc_counter_array(priv);
 327         if (err)
 328                 return err;
 329         err = gve_alloc_notify_blocks(priv);
 330         if (err)
 331                 goto abort_with_counter;
 332         err = gve_alloc_stats_report(priv);
 333         if (err)
 334                 goto abort_with_ntfy_blocks;
 335         err = gve_adminq_configure_device_resources(priv,
 336                                                     priv->counter_array_bus,
 337                                                     priv->num_event_counters,
 338                                                     priv->ntfy_block_bus,
 339                                                     priv->num_ntfy_blks);
 340         if (unlikely(err)) {
 341                 dev_err(&priv->pdev->dev,
 342                         "could not setup device_resources: err=%d\n", err);
 343                 err = -ENXIO;
 344                 goto abort_with_stats_report;
 345         }
 346         err = gve_adminq_report_stats(priv, priv->stats_report_len,
 347                                       priv->stats_report_bus,
 348                                       GVE_STATS_REPORT_TIMER_PERIOD);
 349         if (err)
 350                 dev_err(&priv->pdev->dev,
 351                         "Failed to report stats: err=%d\n", err);
 352         gve_set_device_resources_ok(priv);
 353         return 0;
 354 abort_with_stats_report:
 355         gve_free_stats_report(priv);
 356 abort_with_ntfy_blocks:
 357         gve_free_notify_blocks(priv);
 358 abort_with_counter:
 359         gve_free_counter_array(priv);
 360         return err;
 361 }
 362
 363 static void gve_trigger_reset(struct gve_priv *priv);
 364
 365 static void gve_teardown_device_resources(struct gve_priv *priv)
 366 {
 367         int err;
 368
 369         /* Tell device its resources are being freed */
 370         if (gve_get_device_resources_ok(priv)) {
 371                 /* detach the stats report */
 372                 err = gve_adminq_report_stats(priv, 0, 0x0, GVE_STATS_REPORT_TIMER_PERIOD);
 373                 if (err) {
 374                         dev_err(&priv->pdev->dev,
 375                                 "Failed to detach stats report: err=%d\n", err);
 376                         gve_trigger_reset(priv);
 377                 }
 378                 err = gve_adminq_deconfigure_device_resources(priv);
 379                 if (err) {
 380                         dev_err(&priv->pdev->dev,
 381                                 "Could not deconfigure device resources: err=%d\n",
 382                                 err);
 383                         gve_trigger_reset(priv);
 384                 }
 385         }
 386         gve_free_counter_array(priv);
 387         gve_free_notify_blocks(priv);
 388         gve_free_stats_report(priv);
 389         gve_clear_device_resources_ok(priv);
 390 }
 391
 392 static void gve_add_napi(struct gve_priv *priv, int ntfy_idx)
 393 {
 394         struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 395
 396         netif_napi_add(priv->dev, &block->napi, gve_napi_poll,
 397                        NAPI_POLL_WEIGHT);
 398 }
 399
 400 static void gve_remove_napi(struct gve_priv *priv, int ntfy_idx)
 401 {
 402         struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 403
 404         netif_napi_del(&block->napi);
 405 }
 406
 407 static int gve_register_qpls(struct gve_priv *priv)
 408 {
 409         int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
 410         int err;
 411         int i;
 412
 413         for (i = 0; i < num_qpls; i++) {
 414                 err = gve_adminq_register_page_list(priv, &priv->qpls[i]);
 415                 if (err) {
 416                         netif_err(priv, drv, priv->dev,
 417                                   "failed to register queue page list %d\n",
 418                                   priv->qpls[i].id);
 419                         /* This failure will trigger a reset - no need to clean
 420                          * up
 421                          */
 422                         return err;
 423                 }
 424         }
 425         return 0;
 426 }
 427
 428 static int gve_unregister_qpls(struct gve_priv *priv)
 429 {
 430         int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
 431         int err;
 432         int i;
 433
 434         for (i = 0; i < num_qpls; i++) {
 435                 err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id);
 436                 /* This failure will trigger a reset - no need to clean up */
 437                 if (err) {
 438                         netif_err(priv, drv, priv->dev,
 439                                   "Failed to unregister queue page list %d\n",
 440                                   priv->qpls[i].id);
 441                         return err;
 442                 }
 443         }
 444         return 0;
 445 }
 446
 447 static int gve_create_rings(struct gve_priv *priv)
 448 {
 449         int err;
 450         int i;
 451
 452         err = gve_adminq_create_tx_queues(priv, priv->tx_cfg.num_queues);
 453         if (err) {
 454                 netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n",
 455                           priv->tx_cfg.num_queues);
 456                 /* This failure will trigger a reset - no need to clean
 457                  * up
 458                  */
 459                 return err;
 460         }
 461         netif_dbg(priv, drv, priv->dev, "created %d tx queues\n",
 462                   priv->tx_cfg.num_queues);
 463
 464         err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues);
 465         if (err) {
 466                 netif_err(priv, drv, priv->dev, "failed to create %d rx queues\n",
 467                           priv->rx_cfg.num_queues);
 468                 /* This failure will trigger a reset - no need to clean
 469                  * up
 470                  */
 471                 return err;
 472         }
 473         netif_dbg(priv, drv, priv->dev, "created %d rx queues\n",
 474                   priv->rx_cfg.num_queues);
 475
 476         /* Rx data ring has been prefilled with packet buffers at queue
 477          * allocation time.
 478          * Write the doorbell to provide descriptor slots and packet buffers
 479          * to the NIC.
 480          */
 481         for (i = 0; i < priv->rx_cfg.num_queues; i++)
 482                 gve_rx_write_doorbell(priv, &priv->rx[i]);
 483
 484         return 0;
 485 }
 486
 487 static int gve_alloc_rings(struct gve_priv *priv)
 488 {
 489         int ntfy_idx;
 490         int err;
 491         int i;
 492
 493         /* Setup tx rings */
 494         priv->tx = kvzalloc(priv->tx_cfg.num_queues * sizeof(*priv->tx),
 495                             GFP_KERNEL);
 496         if (!priv->tx)
 497                 return -ENOMEM;
 498         err = gve_tx_alloc_rings(priv);
 499         if (err)
 500                 goto free_tx;
 501         /* Setup rx rings */
 502         priv->rx = kvzalloc(priv->rx_cfg.num_queues * sizeof(*priv->rx),
 503                             GFP_KERNEL);
 504         if (!priv->rx) {
 505                 err = -ENOMEM;
 506                 goto free_tx_queue;
 507         }
 508         err = gve_rx_alloc_rings(priv);
 509         if (err)
 510                 goto free_rx;
 511         /* Add tx napi & init sync stats*/
 512         for (i = 0; i < priv->tx_cfg.num_queues; i++) {
 513                 u64_stats_init(&priv->tx[i].statss);
 514                 ntfy_idx = gve_tx_idx_to_ntfy(priv, i);
 515                 gve_add_napi(priv, ntfy_idx);
 516         }
 517         /* Add rx napi  & init sync stats*/
 518         for (i = 0; i < priv->rx_cfg.num_queues; i++) {
 519                 u64_stats_init(&priv->rx[i].statss);
 520                 ntfy_idx = gve_rx_idx_to_ntfy(priv, i);
 521                 gve_add_napi(priv, ntfy_idx);
 522         }
 523
 524         return 0;
 525
 526 free_rx:
 527         kvfree(priv->rx);
 528         priv->rx = NULL;
 529 free_tx_queue:
 530         gve_tx_free_rings(priv);
 531 free_tx:
 532         kvfree(priv->tx);
 533         priv->tx = NULL;
 534         return err;
 535 }
 536
 537 static int gve_destroy_rings(struct gve_priv *priv)
 538 {
 539         int err;
 540
 541         err = gve_adminq_destroy_tx_queues(priv, priv->tx_cfg.num_queues);
 542         if (err) {
 543                 netif_err(priv, drv, priv->dev,
 544                           "failed to destroy tx queues\n");
 545                 /* This failure will trigger a reset - no need to clean up */
 546                 return err;
 547         }
 548         netif_dbg(priv, drv, priv->dev, "destroyed tx queues\n");
 549         err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues);
 550         if (err) {
 551                 netif_err(priv, drv, priv->dev,
 552                           "failed to destroy rx queues\n");
 553                 /* This failure will trigger a reset - no need to clean up */
 554                 return err;
 555         }
 556         netif_dbg(priv, drv, priv->dev, "destroyed rx queues\n");
 557         return 0;
 558 }
 559
 560 static void gve_free_rings(struct gve_priv *priv)
 561 {
 562         int ntfy_idx;
 563         int i;
 564
 565         if (priv->tx) {
 566                 for (i = 0; i < priv->tx_cfg.num_queues; i++) {
 567                         ntfy_idx = gve_tx_idx_to_ntfy(priv, i);
 568                         gve_remove_napi(priv, ntfy_idx);
 569                 }
 570                 gve_tx_free_rings(priv);
 571                 kvfree(priv->tx);
 572                 priv->tx = NULL;
 573         }
 574         if (priv->rx) {
 575                 for (i = 0; i < priv->rx_cfg.num_queues; i++) {
 576                         ntfy_idx = gve_rx_idx_to_ntfy(priv, i);
 577                         gve_remove_napi(priv, ntfy_idx);
 578                 }
 579                 gve_rx_free_rings(priv);
 580                 kvfree(priv->rx);
 581                 priv->rx = NULL;
 582         }
 583 }
 584
 585 int gve_alloc_page(struct gve_priv *priv, struct device *dev,
 586                    struct page **page, dma_addr_t *dma,
 587                    enum dma_data_direction dir)
 588 {
 589         *page = alloc_page(GFP_KERNEL);
 590         if (!*page) {
 591                 priv->page_alloc_fail++;
 592                 return -ENOMEM;
 593         }
 594         *dma = dma_map_page(dev, *page, 0, PAGE_SIZE, dir);
 595         if (dma_mapping_error(dev, *dma)) {
 596                 priv->dma_mapping_error++;
 597                 put_page(*page);
 598                 return -ENOMEM;
 599         }
 600         return 0;
 601 }
 602
 603 static int gve_alloc_queue_page_list(struct gve_priv *priv, u32 id,
 604                                      int pages)
 605 {
 606         struct gve_queue_page_list *qpl = &priv->qpls[id];
 607         int err;
 608         int i;
 609
 610         if (pages + priv->num_registered_pages > priv->max_registered_pages) {
 611                 netif_err(priv, drv, priv->dev,
 612                           "Reached max number of registered pages %llu > %llu\n",
 613                           pages + priv->num_registered_pages,
 614                           priv->max_registered_pages);
 615                 return -EINVAL;
 616         }
 617
 618         qpl->id = id;
 619         qpl->num_entries = 0;
 620         qpl->pages = kvzalloc(pages * sizeof(*qpl->pages), GFP_KERNEL);
 621         /* caller handles clean up */
 622         if (!qpl->pages)
 623                 return -ENOMEM;
 624         qpl->page_buses = kvzalloc(pages * sizeof(*qpl->page_buses),
 625                                    GFP_KERNEL);
 626         /* caller handles clean up */
 627         if (!qpl->page_buses)
 628                 return -ENOMEM;
 629
 630         for (i = 0; i < pages; i++) {
 631                 err = gve_alloc_page(priv, &priv->pdev->dev, &qpl->pages[i],
 632                                      &qpl->page_buses[i],
 633                                      gve_qpl_dma_dir(priv, id));
 634                 /* caller handles clean up */
 635                 if (err)
 636                         return -ENOMEM;
 637                 qpl->num_entries++;
 638         }
 639         priv->num_registered_pages += pages;
 640
 641         return 0;
 642 }
 643
 644 void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma,
 645                    enum dma_data_direction dir)
 646 {
 647         if (!dma_mapping_error(dev, dma))
 648                 dma_unmap_page(dev, dma, PAGE_SIZE, dir);
 649         if (page)
 650                 put_page(page);
 651 }
 652
 653 static void gve_free_queue_page_list(struct gve_priv *priv,
 654                                      int id)
 655 {
 656         struct gve_queue_page_list *qpl = &priv->qpls[id];
 657         int i;
 658
 659         if (!qpl->pages)
 660                 return;
 661         if (!qpl->page_buses)
 662                 goto free_pages;
 663
 664         for (i = 0; i < qpl->num_entries; i++)
 665                 gve_free_page(&priv->pdev->dev, qpl->pages[i],
 666                               qpl->page_buses[i], gve_qpl_dma_dir(priv, id));
 667
 668         kvfree(qpl->page_buses);
 669 free_pages:
 670         kvfree(qpl->pages);
 671         priv->num_registered_pages -= qpl->num_entries;
 672 }
 673
 674 static int gve_alloc_qpls(struct gve_priv *priv)
 675 {
 676         int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
 677         int i, j;
 678         int err;
 679
 680         priv->qpls = kvzalloc(num_qpls * sizeof(*priv->qpls), GFP_KERNEL);
 681         if (!priv->qpls)
 682                 return -ENOMEM;
 683
 684         for (i = 0; i < gve_num_tx_qpls(priv); i++) {
 685                 err = gve_alloc_queue_page_list(priv, i,
 686                                                 priv->tx_pages_per_qpl);
 687                 if (err)
 688                         goto free_qpls;
 689         }
 690         for (; i < num_qpls; i++) {
 691                 err = gve_alloc_queue_page_list(priv, i,
 692                                                 priv->rx_pages_per_qpl);
 693                 if (err)
 694                         goto free_qpls;
 695         }
 696
 697         priv->qpl_cfg.qpl_map_size = BITS_TO_LONGS(num_qpls) *
 698                                      sizeof(unsigned long) * BITS_PER_BYTE;
 699         priv->qpl_cfg.qpl_id_map = kvzalloc(BITS_TO_LONGS(num_qpls) *
 700                                             sizeof(unsigned long), GFP_KERNEL);
 701         if (!priv->qpl_cfg.qpl_id_map) {
 702                 err = -ENOMEM;
 703                 goto free_qpls;
 704         }
 705
 706         return 0;
 707
 708 free_qpls:
 709         for (j = 0; j <= i; j++)
 710                 gve_free_queue_page_list(priv, j);
 711         kvfree(priv->qpls);
 712         return err;
 713 }
 714
 715 static void gve_free_qpls(struct gve_priv *priv)
 716 {
 717         int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
 718         int i;
 719
 720         kvfree(priv->qpl_cfg.qpl_id_map);
 721
 722         for (i = 0; i < num_qpls; i++)
 723                 gve_free_queue_page_list(priv, i);
 724
 725         kvfree(priv->qpls);
 726 }
 727
 728 /* Use this to schedule a reset when the device is capable of continuing
 729  * to handle other requests in its current state. If it is not, do a reset
 730  * in thread instead.
 731  */
 732 void gve_schedule_reset(struct gve_priv *priv)
 733 {
 734         gve_set_do_reset(priv);
 735         queue_work(priv->gve_wq, &priv->service_task);
 736 }
 737
 738 static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up);
 739 static int gve_reset_recovery(struct gve_priv *priv, bool was_up);
 740 static void gve_turndown(struct gve_priv *priv);
 741 static void gve_turnup(struct gve_priv *priv);
 742
 743 static int gve_open(struct net_device *dev)
 744 {
 745         struct gve_priv *priv = netdev_priv(dev);
 746         int err;
 747
 748         err = gve_alloc_qpls(priv);
 749         if (err)
 750                 return err;
 751         err = gve_alloc_rings(priv);
 752         if (err)
 753                 goto free_qpls;
 754
 755         err = netif_set_real_num_tx_queues(dev, priv->tx_cfg.num_queues);
 756         if (err)
 757                 goto free_rings;
 758         err = netif_set_real_num_rx_queues(dev, priv->rx_cfg.num_queues);
 759         if (err)
 760                 goto free_rings;
 761
 762         err = gve_register_qpls(priv);
 763         if (err)
 764                 goto reset;
 765         err = gve_create_rings(priv);
 766         if (err)
 767                 goto reset;
 768         gve_set_device_rings_ok(priv);
 769
 770         if (gve_get_report_stats(priv))
 771                 mod_timer(&priv->stats_report_timer,
 772                           round_jiffies(jiffies +
 773                                 msecs_to_jiffies(priv->stats_report_timer_period)));
 774
 775         gve_turnup(priv);
 776         queue_work(priv->gve_wq, &priv->service_task);
 777         priv->interface_up_cnt++;
 778         return 0;
 779
 780 free_rings:
 781         gve_free_rings(priv);
 782 free_qpls:
 783         gve_free_qpls(priv);
 784         return err;
 785
 786 reset:
 787         /* This must have been called from a reset due to the rtnl lock
 788          * so just return at this point.
 789          */
 790         if (gve_get_reset_in_progress(priv))
 791                 return err;
 792         /* Otherwise reset before returning */
 793         gve_reset_and_teardown(priv, true);
 794         /* if this fails there is nothing we can do so just ignore the return */
 795         gve_reset_recovery(priv, false);
 796         /* return the original error */
 797         return err;
 798 }
 799
 800 static int gve_close(struct net_device *dev)
 801 {
 802         struct gve_priv *priv = netdev_priv(dev);
 803         int err;
 804
 805         netif_carrier_off(dev);
 806         if (gve_get_device_rings_ok(priv)) {
 807                 gve_turndown(priv);
 808                 err = gve_destroy_rings(priv);
 809                 if (err)
 810                         goto err;
 811                 err = gve_unregister_qpls(priv);
 812                 if (err)
 813                         goto err;
 814                 gve_clear_device_rings_ok(priv);
 815         }
 816         del_timer_sync(&priv->stats_report_timer);
 817
 818         gve_free_rings(priv);
 819         gve_free_qpls(priv);
 820         priv->interface_down_cnt++;
 821         return 0;
 822
 823 err:
 824         /* This must have been called from a reset due to the rtnl lock
 825          * so just return at this point.
 826          */
 827         if (gve_get_reset_in_progress(priv))
 828                 return err;
 829         /* Otherwise reset before returning */
 830         gve_reset_and_teardown(priv, true);
 831         return gve_reset_recovery(priv, false);
 832 }
 833
 834 int gve_adjust_queues(struct gve_priv *priv,
 835                       struct gve_queue_config new_rx_config,
 836                       struct gve_queue_config new_tx_config)
 837 {
 838         int err;
 839
 840         if (netif_carrier_ok(priv->dev)) {
 841                 /* To make this process as simple as possible we teardown the
 842                  * device, set the new configuration, and then bring the device
 843                  * up again.
 844                  */
 845                 err = gve_close(priv->dev);
 846                 /* we have already tried to reset in close,
 847                  * just fail at this point
 848                  */
 849                 if (err)
 850                         return err;
 851                 priv->tx_cfg = new_tx_config;
 852                 priv->rx_cfg = new_rx_config;
 853
 854                 err = gve_open(priv->dev);
 855                 if (err)
 856                         goto err;
 857
 858                 return 0;
 859         }
 860         /* Set the config for the next up. */
 861         priv->tx_cfg = new_tx_config;
 862         priv->rx_cfg = new_rx_config;
 863
 864         return 0;
 865 err:
 866         netif_err(priv, drv, priv->dev,
 867                   "Adjust queues failed! !!! DISABLING ALL QUEUES !!!\n");
 868         gve_turndown(priv);
 869         return err;
 870 }
 871
 872 static void gve_turndown(struct gve_priv *priv)
 873 {
 874         int idx;
 875
 876         if (netif_carrier_ok(priv->dev))
 877                 netif_carrier_off(priv->dev);
 878
 879         if (!gve_get_napi_enabled(priv))
 880                 return;
 881
 882         /* Disable napi to prevent more work from coming in */
 883         for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
 884                 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
 885                 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 886
 887                 napi_disable(&block->napi);
 888         }
 889         for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) {
 890                 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
 891                 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 892
 893                 napi_disable(&block->napi);
 894         }
 895
 896         /* Stop tx queues */
 897         netif_tx_disable(priv->dev);
 898
 899         gve_clear_napi_enabled(priv);
 900         gve_clear_report_stats(priv);
 901 }
 902
 903 static void gve_turnup(struct gve_priv *priv)
 904 {
 905         int idx;
 906
 907         /* Start the tx queues */
 908         netif_tx_start_all_queues(priv->dev);
 909
 910         /* Enable napi and unmask interrupts for all queues */
 911         for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
 912                 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
 913                 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 914
 915                 napi_enable(&block->napi);
 916                 iowrite32be(0, gve_irq_doorbell(priv, block));
 917         }
 918         for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) {
 919                 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
 920                 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 921
 922                 napi_enable(&block->napi);
 923                 iowrite32be(0, gve_irq_doorbell(priv, block));
 924         }
 925
 926         gve_set_napi_enabled(priv);
 927 }
 928
 929 static void gve_tx_timeout(struct net_device *dev, unsigned int txqueue)
 930 {
 931         struct gve_priv *priv = netdev_priv(dev);
 932
 933         gve_schedule_reset(priv);
 934         priv->tx_timeo_cnt++;
 935 }
 936
 937 static const struct net_device_ops gve_netdev_ops = {
 938         .ndo_start_xmit         =       gve_tx,
 939         .ndo_open               =       gve_open,
 940         .ndo_stop               =       gve_close,
 941         .ndo_get_stats64        =       gve_get_stats,
 942         .ndo_tx_timeout         =       gve_tx_timeout,
 943 };
 944
 945 static void gve_handle_status(struct gve_priv *priv, u32 status)
 946 {
 947         if (GVE_DEVICE_STATUS_RESET_MASK & status) {
 948                 dev_info(&priv->pdev->dev, "Device requested reset.\n");
 949                 gve_set_do_reset(priv);
 950         }
 951         if (GVE_DEVICE_STATUS_REPORT_STATS_MASK & status) {
 952                 priv->stats_report_trigger_cnt++;
 953                 gve_set_do_report_stats(priv);
 954         }
 955 }
 956
 957 static void gve_handle_reset(struct gve_priv *priv)
 958 {
 959         /* A service task will be scheduled at the end of probe to catch any
 960          * resets that need to happen, and we don't want to reset until
 961          * probe is done.
 962          */
 963         if (gve_get_probe_in_progress(priv))
 964                 return;
 965
 966         if (gve_get_do_reset(priv)) {
 967                 rtnl_lock();
 968                 gve_reset(priv, false);
 969                 rtnl_unlock();
 970         }
 971 }
 972
 973 void gve_handle_report_stats(struct gve_priv *priv)
 974 {
 975         int idx, stats_idx = 0, tx_bytes;
 976         unsigned int start = 0;
 977         struct stats *stats = priv->stats_report->stats;
 978
 979         if (!gve_get_report_stats(priv))
 980                 return;
 981
 982         be64_add_cpu(&priv->stats_report->written_count, 1);
 983         /* tx stats */
 984         if (priv->tx) {
 985                 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
 986                         do {
 987                                 start = u64_stats_fetch_begin(&priv->tx[idx].statss);
 988                                 tx_bytes = priv->tx[idx].bytes_done;
 989                         } while (u64_stats_fetch_retry(&priv->tx[idx].statss, start));
 990                         stats[stats_idx++] = (struct stats) {
 991                                 .stat_name = cpu_to_be32(TX_WAKE_CNT),
 992                                 .value = cpu_to_be64(priv->tx[idx].wake_queue),
 993                                 .queue_id = cpu_to_be32(idx),
 994                         };
 995                         stats[stats_idx++] = (struct stats) {
 996                                 .stat_name = cpu_to_be32(TX_STOP_CNT),
 997                                 .value = cpu_to_be64(priv->tx[idx].stop_queue),
 998                                 .queue_id = cpu_to_be32(idx),
 999                         };
1000                         stats[stats_idx++] = (struct stats) {
1001                                 .stat_name = cpu_to_be32(TX_FRAMES_SENT),
1002                                 .value = cpu_to_be64(priv->tx[idx].req),
1003                                 .queue_id = cpu_to_be32(idx),
1004                         };
1005                         stats[stats_idx++] = (struct stats) {
1006                                 .stat_name = cpu_to_be32(TX_BYTES_SENT),
1007                                 .value = cpu_to_be64(tx_bytes),
1008                                 .queue_id = cpu_to_be32(idx),
1009                         };
1010                         stats[stats_idx++] = (struct stats) {
1011                                 .stat_name = cpu_to_be32(TX_LAST_COMPLETION_PROCESSED),
1012                                 .value = cpu_to_be64(priv->tx[idx].done),
1013                                 .queue_id = cpu_to_be32(idx),
1014                         };
1015                 }
1016         }
1017         /* rx stats */
1018         if (priv->rx) {
1019                 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) {
1020                         stats[stats_idx++] = (struct stats) {
1021                                 .stat_name = cpu_to_be32(RX_NEXT_EXPECTED_SEQUENCE),
1022                                 .value = cpu_to_be64(priv->rx[idx].desc.seqno),
1023                                 .queue_id = cpu_to_be32(idx),
1024                         };
1025                         stats[stats_idx++] = (struct stats) {
1026                                 .stat_name = cpu_to_be32(RX_BUFFERS_POSTED),
1027                                 .value = cpu_to_be64(priv->rx[0].fill_cnt),
1028                                 .queue_id = cpu_to_be32(idx),
1029                         };
1030                 }
1031         }
1032 }
1033
1034 static void gve_handle_link_status(struct gve_priv *priv, bool link_status)
1035 {
1036         if (!gve_get_napi_enabled(priv))
1037                 return;
1038
1039         if (link_status == netif_carrier_ok(priv->dev))
1040                 return;
1041
1042         if (link_status) {
1043                 netdev_info(priv->dev, "Device link is up.\n");
1044                 netif_carrier_on(priv->dev);
1045         } else {
1046                 netdev_info(priv->dev, "Device link is down.\n");
1047                 netif_carrier_off(priv->dev);
1048         }
1049 }
1050
1051 /* Handle NIC status register changes, reset requests and report stats */
1052 static void gve_service_task(struct work_struct *work)
1053 {
1054         struct gve_priv *priv = container_of(work, struct gve_priv,
1055                                              service_task);
1056         u32 status = ioread32be(&priv->reg_bar0->device_status);
1057
1058         gve_handle_status(priv, status);
1059
1060         gve_handle_reset(priv);
1061         gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
1062 }
1063
1064 static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
1065 {
1066         int num_ntfy;
1067         int err;
1068
1069         /* Set up the adminq */
1070         err = gve_adminq_alloc(&priv->pdev->dev, priv);
1071         if (err) {
1072                 dev_err(&priv->pdev->dev,
1073                         "Failed to alloc admin queue: err=%d\n", err);
1074                 return err;
1075         }
1076
1077         if (skip_describe_device)
1078                 goto setup_device;
1079
1080         /* Get the initial information we need from the device */
1081         err = gve_adminq_describe_device(priv);
1082         if (err) {
1083                 dev_err(&priv->pdev->dev,
1084                         "Could not get device information: err=%d\n", err);
1085                 goto err;
1086         }
1087         if (priv->dev->max_mtu > PAGE_SIZE) {
1088                 priv->dev->max_mtu = PAGE_SIZE;
1089                 err = gve_adminq_set_mtu(priv, priv->dev->mtu);
1090                 if (err) {
1091                         dev_err(&priv->pdev->dev, "Could not set mtu");
1092                         goto err;
1093                 }
1094         }
1095         priv->dev->mtu = priv->dev->max_mtu;
1096         num_ntfy = pci_msix_vec_count(priv->pdev);
1097         if (num_ntfy <= 0) {
1098                 dev_err(&priv->pdev->dev,
1099                         "could not count MSI-x vectors: err=%d\n", num_ntfy);
1100                 err = num_ntfy;
1101                 goto err;
1102         } else if (num_ntfy < GVE_MIN_MSIX) {
1103                 dev_err(&priv->pdev->dev, "gve needs at least %d MSI-x vectors, but only has %d\n",
1104                         GVE_MIN_MSIX, num_ntfy);
1105                 err = -EINVAL;
1106                 goto err;
1107         }
1108
1109         priv->num_registered_pages = 0;
1110         priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
1111         /* gvnic has one Notification Block per MSI-x vector, except for the
1112          * management vector
1113          */
1114         priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1;
1115         priv->mgmt_msix_idx = priv->num_ntfy_blks;
1116
1117         priv->tx_cfg.max_queues =
1118                 min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2);
1119         priv->rx_cfg.max_queues =
1120                 min_t(int, priv->rx_cfg.max_queues, priv->num_ntfy_blks / 2);
1121
1122         priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
1123         priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
1124         if (priv->default_num_queues > 0) {
1125                 priv->tx_cfg.num_queues = min_t(int, priv->default_num_queues,
1126                                                 priv->tx_cfg.num_queues);
1127                 priv->rx_cfg.num_queues = min_t(int, priv->default_num_queues,
1128                                                 priv->rx_cfg.num_queues);
1129         }
1130
1131         dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n",
1132                  priv->tx_cfg.num_queues, priv->rx_cfg.num_queues);
1133         dev_info(&priv->pdev->dev, "Max TX queues %d, Max RX queues %d\n",
1134                  priv->tx_cfg.max_queues, priv->rx_cfg.max_queues);
1135
1136 setup_device:
1137         err = gve_setup_device_resources(priv);
1138         if (!err)
1139                 return 0;
1140 err:
1141         gve_adminq_free(&priv->pdev->dev, priv);
1142         return err;
1143 }
1144
1145 static void gve_teardown_priv_resources(struct gve_priv *priv)
1146 {
1147         gve_teardown_device_resources(priv);
1148         gve_adminq_free(&priv->pdev->dev, priv);
1149 }
1150
1151 static void gve_trigger_reset(struct gve_priv *priv)
1152 {
1153         /* Reset the device by releasing the AQ */
1154         gve_adminq_release(priv);
1155 }
1156
1157 static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up)
1158 {
1159         gve_trigger_reset(priv);
1160         /* With the reset having already happened, close cannot fail */
1161         if (was_up)
1162                 gve_close(priv->dev);
1163         gve_teardown_priv_resources(priv);
1164 }
1165
1166 static int gve_reset_recovery(struct gve_priv *priv, bool was_up)
1167 {
1168         int err;
1169
1170         err = gve_init_priv(priv, true);
1171         if (err)
1172                 goto err;
1173         if (was_up) {
1174                 err = gve_open(priv->dev);
1175                 if (err)
1176                         goto err;
1177         }
1178         return 0;
1179 err:
1180         dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n");
1181         gve_turndown(priv);
1182         return err;
1183 }
1184
1185 int gve_reset(struct gve_priv *priv, bool attempt_teardown)
1186 {
1187         bool was_up = netif_carrier_ok(priv->dev);
1188         int err;
1189
1190         dev_info(&priv->pdev->dev, "Performing reset\n");
1191         gve_clear_do_reset(priv);
1192         gve_set_reset_in_progress(priv);
1193         /* If we aren't attempting to teardown normally, just go turndown and
1194          * reset right away.
1195          */
1196         if (!attempt_teardown) {
1197                 gve_turndown(priv);
1198                 gve_reset_and_teardown(priv, was_up);
1199         } else {
1200                 /* Otherwise attempt to close normally */
1201                 if (was_up) {
1202                         err = gve_close(priv->dev);
1203                         /* If that fails reset as we did above */
1204                         if (err)
1205                                 gve_reset_and_teardown(priv, was_up);
1206                 }
1207                 /* Clean up any remaining resources */
1208                 gve_teardown_priv_resources(priv);
1209         }
1210
1211         /* Set it all back up */
1212         err = gve_reset_recovery(priv, was_up);
1213         gve_clear_reset_in_progress(priv);
1214         priv->reset_cnt++;
1215         priv->interface_up_cnt = 0;
1216         priv->interface_down_cnt = 0;
1217         priv->stats_report_trigger_cnt = 0;
1218         return err;
1219 }
1220
1221 static void gve_write_version(u8 __iomem *driver_version_register)
1222 {
1223         const char *c = gve_version_prefix;
1224
1225         while (*c) {
1226                 writeb(*c, driver_version_register);
1227                 c++;
1228         }
1229
1230         c = gve_version_str;
1231         while (*c) {
1232                 writeb(*c, driver_version_register);
1233                 c++;
1234         }
1235         writeb('\n', driver_version_register);
1236 }
1237
1238 static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1239 {
1240         int max_tx_queues, max_rx_queues;
1241         struct net_device *dev;
1242         __be32 __iomem *db_bar;
1243         struct gve_registers __iomem *reg_bar;
1244         struct gve_priv *priv;
1245         int err;
1246
1247         err = pci_enable_device(pdev);
1248         if (err)
1249                 return -ENXIO;
1250
1251         err = pci_request_regions(pdev, "gvnic-cfg");
1252         if (err)
1253                 goto abort_with_enabled;
1254
1255         pci_set_master(pdev);
1256
1257         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1258         if (err) {
1259                 dev_err(&pdev->dev, "Failed to set dma mask: err=%d\n", err);
1260                 goto abort_with_pci_region;
1261         }
1262
1263         err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1264         if (err) {
1265                 dev_err(&pdev->dev,
1266                         "Failed to set consistent dma mask: err=%d\n", err);
1267                 goto abort_with_pci_region;
1268         }
1269
1270         reg_bar = pci_iomap(pdev, GVE_REGISTER_BAR, 0);
1271         if (!reg_bar) {
1272                 dev_err(&pdev->dev, "Failed to map pci bar!\n");
1273                 err = -ENOMEM;
1274                 goto abort_with_pci_region;
1275         }
1276
1277         db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0);
1278         if (!db_bar) {
1279                 dev_err(&pdev->dev, "Failed to map doorbell bar!\n");
1280                 err = -ENOMEM;
1281                 goto abort_with_reg_bar;
1282         }
1283
1284         gve_write_version(&reg_bar->driver_version);
1285         /* Get max queues to alloc etherdev */
1286         max_rx_queues = ioread32be(&reg_bar->max_tx_queues);
1287         max_tx_queues = ioread32be(&reg_bar->max_rx_queues);
1288         /* Alloc and setup the netdev and priv */
1289         dev = alloc_etherdev_mqs(sizeof(*priv), max_tx_queues, max_rx_queues);
1290         if (!dev) {
1291                 dev_err(&pdev->dev, "could not allocate netdev\n");
1292                 goto abort_with_db_bar;
1293         }
1294         SET_NETDEV_DEV(dev, &pdev->dev);
1295         pci_set_drvdata(pdev, dev);
1296         dev->ethtool_ops = &gve_ethtool_ops;
1297         dev->netdev_ops = &gve_netdev_ops;
1298         /* advertise features */
1299         dev->hw_features = NETIF_F_HIGHDMA;
1300         dev->hw_features |= NETIF_F_SG;
1301         dev->hw_features |= NETIF_F_HW_CSUM;
1302         dev->hw_features |= NETIF_F_TSO;
1303         dev->hw_features |= NETIF_F_TSO6;
1304         dev->hw_features |= NETIF_F_TSO_ECN;
1305         dev->hw_features |= NETIF_F_RXCSUM;
1306         dev->hw_features |= NETIF_F_RXHASH;
1307         dev->features = dev->hw_features;
1308         dev->watchdog_timeo = 5 * HZ;
1309         dev->min_mtu = ETH_MIN_MTU;
1310         netif_carrier_off(dev);
1311
1312         priv = netdev_priv(dev);
1313         priv->dev = dev;
1314         priv->pdev = pdev;
1315         priv->msg_enable = DEFAULT_MSG_LEVEL;
1316         priv->reg_bar0 = reg_bar;
1317         priv->db_bar2 = db_bar;
1318         priv->service_task_flags = 0x0;
1319         priv->state_flags = 0x0;
1320         priv->ethtool_flags = 0x0;
1321
1322         gve_set_probe_in_progress(priv);
1323         priv->gve_wq = alloc_ordered_workqueue("gve", 0);
1324         if (!priv->gve_wq) {
1325                 dev_err(&pdev->dev, "Could not allocate workqueue");
1326                 err = -ENOMEM;
1327                 goto abort_with_netdev;
1328         }
1329         INIT_WORK(&priv->service_task, gve_service_task);
1330         INIT_WORK(&priv->stats_report_task, gve_stats_report_task);
1331         priv->tx_cfg.max_queues = max_tx_queues;
1332         priv->rx_cfg.max_queues = max_rx_queues;
1333
1334         err = gve_init_priv(priv, false);
1335         if (err)
1336                 goto abort_with_wq;
1337
1338         err = register_netdev(dev);
1339         if (err)
1340                 goto abort_with_wq;
1341
1342         dev_info(&pdev->dev, "GVE version %s\n", gve_version_str);
1343         gve_clear_probe_in_progress(priv);
1344         queue_work(priv->gve_wq, &priv->service_task);
1345         return 0;
1346
1347 abort_with_wq:
1348         destroy_workqueue(priv->gve_wq);
1349
1350 abort_with_netdev:
1351         free_netdev(dev);
1352
1353 abort_with_db_bar:
1354         pci_iounmap(pdev, db_bar);
1355
1356 abort_with_reg_bar:
1357         pci_iounmap(pdev, reg_bar);
1358
1359 abort_with_pci_region:
1360         pci_release_regions(pdev);
1361
1362 abort_with_enabled:
1363         pci_disable_device(pdev);
1364         return -ENXIO;
1365 }
1366
1367 static void gve_remove(struct pci_dev *pdev)
1368 {
1369         struct net_device *netdev = pci_get_drvdata(pdev);
1370         struct gve_priv *priv = netdev_priv(netdev);
1371         __be32 __iomem *db_bar = priv->db_bar2;
1372         void __iomem *reg_bar = priv->reg_bar0;
1373
1374         unregister_netdev(netdev);
1375         gve_teardown_priv_resources(priv);
1376         destroy_workqueue(priv->gve_wq);
1377         free_netdev(netdev);
1378         pci_iounmap(pdev, db_bar);
1379         pci_iounmap(pdev, reg_bar);
1380         pci_release_regions(pdev);
1381         pci_disable_device(pdev);
1382 }
1383
1384 static const struct pci_device_id gve_id_table[] = {
1385         { PCI_DEVICE(PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC) },
1386         { }
1387 };
1388
1389 static struct pci_driver gvnic_driver = {
1390         .name           = "gvnic",
1391         .id_table       = gve_id_table,
1392         .probe          = gve_probe,
1393         .remove         = gve_remove,
1394 };
1395
1396 module_pci_driver(gvnic_driver);
1397
1398 MODULE_DEVICE_TABLE(pci, gve_id_table);
1399 MODULE_AUTHOR("Google, Inc.");
1400 MODULE_DESCRIPTION("gVNIC Driver");
1401 MODULE_LICENSE("Dual MIT/GPL");
1402 MODULE_VERSION(GVE_VERSION);