drivers/net/ethernet/ibm/ibmvnic.c

   1 /**************************************************************************/
   2 /*                                                                        */
   3 /*  IBM System i and System p Virtual NIC Device Driver                   */
   4 /*  Copyright (C) 2014 IBM Corp.                                          */
   5 /*  Santiago Leon (santi_leon@yahoo.com)                                  */
   6 /*  Thomas Falcon (tlfalcon@linux.vnet.ibm.com)                           */
   7 /*  John Allen (jallen@linux.vnet.ibm.com)                                */
   8 /*                                                                        */
   9 /*  This program is free software; you can redistribute it and/or modify  */
  10 /*  it under the terms of the GNU General Public License as published by  */
  11 /*  the Free Software Foundation; either version 2 of the License, or     */
  12 /*  (at your option) any later version.                                   */
  13 /*                                                                        */
  14 /*  This program is distributed in the hope that it will be useful,       */
  15 /*  but WITHOUT ANY WARRANTY; without even the implied warranty of        */
  16 /*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         */
  17 /*  GNU General Public License for more details.                          */
  18 /*                                                                        */
  19 /*  You should have received a copy of the GNU General Public License     */
  20 /*  along with this program.                                              */
  21 /*                                                                        */
  22 /* This module contains the implementation of a virtual ethernet device   */
  23 /* for use with IBM i/p Series LPAR Linux. It utilizes the logical LAN    */
  24 /* option of the RS/6000 Platform Architecture to interface with virtual  */
  25 /* ethernet NICs that are presented to the partition by the hypervisor.   */
  26 /*                                                                         */
  27 /* Messages are passed between the VNIC driver and the VNIC server using  */
  28 /* Command/Response Queues (CRQs) and sub CRQs (sCRQs). CRQs are used to  */
  29 /* issue and receive commands that initiate communication with the server */
  30 /* on driver initialization. Sub CRQs (sCRQs) are similar to CRQs, but    */
  31 /* are used by the driver to notify the server that a packet is           */
  32 /* ready for transmission or that a buffer has been added to receive a    */
  33 /* packet. Subsequently, sCRQs are used by the server to notify the       */
  34 /* driver that a packet transmission has been completed or that a packet  */
  35 /* has been received and placed in a waiting buffer.                      */
  36 /*                                                                        */
  37 /* In lieu of a more conventional "on-the-fly" DMA mapping strategy in    */
  38 /* which skbs are DMA mapped and immediately unmapped when the transmit   */
  39 /* or receive has been completed, the VNIC driver is required to use      */
  40 /* "long term mapping". This entails that large, continuous DMA mapped    */
  41 /* buffers are allocated on driver initialization and these buffers are   */
  42 /* then continuously reused to pass skbs to and from the VNIC server.     */
  43 /*                                                                        */
  44 /**************************************************************************/
  45
  46 #include <linux/module.h>
  47 #include <linux/moduleparam.h>
  48 #include <linux/types.h>
  49 #include <linux/errno.h>
  50 #include <linux/completion.h>
  51 #include <linux/ioport.h>
  52 #include <linux/dma-mapping.h>
  53 #include <linux/kernel.h>
  54 #include <linux/netdevice.h>
  55 #include <linux/etherdevice.h>
  56 #include <linux/skbuff.h>
  57 #include <linux/init.h>
  58 #include <linux/delay.h>
  59 #include <linux/mm.h>
  60 #include <linux/ethtool.h>
  61 #include <linux/proc_fs.h>
  62 #include <linux/in.h>
  63 #include <linux/ip.h>
  64 #include <linux/ipv6.h>
  65 #include <linux/irq.h>
  66 #include <linux/kthread.h>
  67 #include <linux/seq_file.h>
  68 #include <linux/interrupt.h>
  69 #include <net/net_namespace.h>
  70 #include <asm/hvcall.h>
  71 #include <linux/atomic.h>
  72 #include <asm/vio.h>
  73 #include <asm/iommu.h>
  74 #include <linux/uaccess.h>
  75 #include <asm/firmware.h>
  76 #include <linux/workqueue.h>
  77 #include <linux/if_vlan.h>
  78 #include <linux/utsname.h>
  79
  80 #include "ibmvnic.h"
  81
  82 static const char ibmvnic_driver_name[] = "ibmvnic";
  83 static const char ibmvnic_driver_string[] = "IBM System i/p Virtual NIC Driver";
  84
  85 MODULE_AUTHOR("Santiago Leon");
  86 MODULE_DESCRIPTION("IBM System i/p Virtual NIC Driver");
  87 MODULE_LICENSE("GPL");
  88 MODULE_VERSION(IBMVNIC_DRIVER_VERSION);
  89
  90 static int ibmvnic_version = IBMVNIC_INITIAL_VERSION;
  91 static int ibmvnic_remove(struct vio_dev *);
  92 static void release_sub_crqs(struct ibmvnic_adapter *);
  93 static int ibmvnic_reset_crq(struct ibmvnic_adapter *);
  94 static int ibmvnic_send_crq_init(struct ibmvnic_adapter *);
  95 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *);
  96 static int ibmvnic_send_crq(struct ibmvnic_adapter *, union ibmvnic_crq *);
  97 static int send_subcrq(struct ibmvnic_adapter *adapter, u64 remote_handle,
  98                        union sub_crq *sub_crq);
  99 static int send_subcrq_indirect(struct ibmvnic_adapter *, u64, u64, u64);
 100 static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance);
 101 static int enable_scrq_irq(struct ibmvnic_adapter *,
 102                            struct ibmvnic_sub_crq_queue *);
 103 static int disable_scrq_irq(struct ibmvnic_adapter *,
 104                             struct ibmvnic_sub_crq_queue *);
 105 static int pending_scrq(struct ibmvnic_adapter *,
 106                         struct ibmvnic_sub_crq_queue *);
 107 static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *,
 108                                         struct ibmvnic_sub_crq_queue *);
 109 static int ibmvnic_poll(struct napi_struct *napi, int data);
 110 static void send_map_query(struct ibmvnic_adapter *adapter);
 111 static void send_request_map(struct ibmvnic_adapter *, dma_addr_t, __be32, u8);
 112 static void send_request_unmap(struct ibmvnic_adapter *, u8);
 113 static void send_login(struct ibmvnic_adapter *adapter);
 114 static void send_cap_queries(struct ibmvnic_adapter *adapter);
 115 static int init_sub_crqs(struct ibmvnic_adapter *);
 116 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter);
 117 static int ibmvnic_init(struct ibmvnic_adapter *);
 118 static void release_crq_queue(struct ibmvnic_adapter *);
 119 static int __ibmvnic_set_mac(struct net_device *netdev, struct sockaddr *p);
 120
 121 struct ibmvnic_stat {
 122         char name[ETH_GSTRING_LEN];
 123         int offset;
 124 };
 125
 126 #define IBMVNIC_STAT_OFF(stat) (offsetof(struct ibmvnic_adapter, stats) + \
 127                              offsetof(struct ibmvnic_statistics, stat))
 128 #define IBMVNIC_GET_STAT(a, off) (*((u64 *)(((unsigned long)(a)) + off)))
 129
 130 static const struct ibmvnic_stat ibmvnic_stats[] = {
 131         {"rx_packets", IBMVNIC_STAT_OFF(rx_packets)},
 132         {"rx_bytes", IBMVNIC_STAT_OFF(rx_bytes)},
 133         {"tx_packets", IBMVNIC_STAT_OFF(tx_packets)},
 134         {"tx_bytes", IBMVNIC_STAT_OFF(tx_bytes)},
 135         {"ucast_tx_packets", IBMVNIC_STAT_OFF(ucast_tx_packets)},
 136         {"ucast_rx_packets", IBMVNIC_STAT_OFF(ucast_rx_packets)},
 137         {"mcast_tx_packets", IBMVNIC_STAT_OFF(mcast_tx_packets)},
 138         {"mcast_rx_packets", IBMVNIC_STAT_OFF(mcast_rx_packets)},
 139         {"bcast_tx_packets", IBMVNIC_STAT_OFF(bcast_tx_packets)},
 140         {"bcast_rx_packets", IBMVNIC_STAT_OFF(bcast_rx_packets)},
 141         {"align_errors", IBMVNIC_STAT_OFF(align_errors)},
 142         {"fcs_errors", IBMVNIC_STAT_OFF(fcs_errors)},
 143         {"single_collision_frames", IBMVNIC_STAT_OFF(single_collision_frames)},
 144         {"multi_collision_frames", IBMVNIC_STAT_OFF(multi_collision_frames)},
 145         {"sqe_test_errors", IBMVNIC_STAT_OFF(sqe_test_errors)},
 146         {"deferred_tx", IBMVNIC_STAT_OFF(deferred_tx)},
 147         {"late_collisions", IBMVNIC_STAT_OFF(late_collisions)},
 148         {"excess_collisions", IBMVNIC_STAT_OFF(excess_collisions)},
 149         {"internal_mac_tx_errors", IBMVNIC_STAT_OFF(internal_mac_tx_errors)},
 150         {"carrier_sense", IBMVNIC_STAT_OFF(carrier_sense)},
 151         {"too_long_frames", IBMVNIC_STAT_OFF(too_long_frames)},
 152         {"internal_mac_rx_errors", IBMVNIC_STAT_OFF(internal_mac_rx_errors)},
 153 };
 154
 155 static long h_reg_sub_crq(unsigned long unit_address, unsigned long token,
 156                           unsigned long length, unsigned long *number,
 157                           unsigned long *irq)
 158 {
 159         unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
 160         long rc;
 161
 162         rc = plpar_hcall(H_REG_SUB_CRQ, retbuf, unit_address, token, length);
 163         *number = retbuf[0];
 164         *irq = retbuf[1];
 165
 166         return rc;
 167 }
 168
 169 static int alloc_long_term_buff(struct ibmvnic_adapter *adapter,
 170                                 struct ibmvnic_long_term_buff *ltb, int size)
 171 {
 172         struct device *dev = &adapter->vdev->dev;
 173
 174         ltb->size = size;
 175         ltb->buff = dma_alloc_coherent(dev, ltb->size, &ltb->addr,
 176                                        GFP_KERNEL);
 177
 178         if (!ltb->buff) {
 179                 dev_err(dev, "Couldn't alloc long term buffer\n");
 180                 return -ENOMEM;
 181         }
 182         ltb->map_id = adapter->map_id;
 183         adapter->map_id++;
 184
 185         init_completion(&adapter->fw_done);
 186         send_request_map(adapter, ltb->addr,
 187                          ltb->size, ltb->map_id);
 188         wait_for_completion(&adapter->fw_done);
 189
 190         if (adapter->fw_done_rc) {
 191                 dev_err(dev, "Couldn't map long term buffer,rc = %d\n",
 192                         adapter->fw_done_rc);
 193                 return -1;
 194         }
 195         return 0;
 196 }
 197
 198 static void free_long_term_buff(struct ibmvnic_adapter *adapter,
 199                                 struct ibmvnic_long_term_buff *ltb)
 200 {
 201         struct device *dev = &adapter->vdev->dev;
 202
 203         if (!ltb->buff)
 204                 return;
 205
 206         if (adapter->reset_reason != VNIC_RESET_FAILOVER &&
 207             adapter->reset_reason != VNIC_RESET_MOBILITY)
 208                 send_request_unmap(adapter, ltb->map_id);
 209         dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr);
 210 }
 211
 212 static int reset_long_term_buff(struct ibmvnic_adapter *adapter,
 213                                 struct ibmvnic_long_term_buff *ltb)
 214 {
 215         memset(ltb->buff, 0, ltb->size);
 216
 217         init_completion(&adapter->fw_done);
 218         send_request_map(adapter, ltb->addr, ltb->size, ltb->map_id);
 219         wait_for_completion(&adapter->fw_done);
 220
 221         if (adapter->fw_done_rc) {
 222                 dev_info(&adapter->vdev->dev,
 223                          "Reset failed, attempting to free and reallocate buffer\n");
 224                 free_long_term_buff(adapter, ltb);
 225                 return alloc_long_term_buff(adapter, ltb, ltb->size);
 226         }
 227         return 0;
 228 }
 229
 230 static void deactivate_rx_pools(struct ibmvnic_adapter *adapter)
 231 {
 232         int i;
 233
 234         for (i = 0; i < be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
 235              i++)
 236                 adapter->rx_pool[i].active = 0;
 237 }
 238
 239 static void replenish_rx_pool(struct ibmvnic_adapter *adapter,
 240                               struct ibmvnic_rx_pool *pool)
 241 {
 242         int count = pool->size - atomic_read(&pool->available);
 243         struct device *dev = &adapter->vdev->dev;
 244         int buffers_added = 0;
 245         unsigned long lpar_rc;
 246         union sub_crq sub_crq;
 247         struct sk_buff *skb;
 248         unsigned int offset;
 249         dma_addr_t dma_addr;
 250         unsigned char *dst;
 251         u64 *handle_array;
 252         int shift = 0;
 253         int index;
 254         int i;
 255
 256         if (!pool->active)
 257                 return;
 258
 259         handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +
 260                                       be32_to_cpu(adapter->login_rsp_buf->
 261                                       off_rxadd_subcrqs));
 262
 263         for (i = 0; i < count; ++i) {
 264                 skb = alloc_skb(pool->buff_size, GFP_ATOMIC);
 265                 if (!skb) {
 266                         dev_err(dev, "Couldn't replenish rx buff\n");
 267                         adapter->replenish_no_mem++;
 268                         break;
 269                 }
 270
 271                 index = pool->free_map[pool->next_free];
 272
 273                 if (pool->rx_buff[index].skb)
 274                         dev_err(dev, "Inconsistent free_map!\n");
 275
 276                 /* Copy the skb to the long term mapped DMA buffer */
 277                 offset = index * pool->buff_size;
 278                 dst = pool->long_term_buff.buff + offset;
 279                 memset(dst, 0, pool->buff_size);
 280                 dma_addr = pool->long_term_buff.addr + offset;
 281                 pool->rx_buff[index].data = dst;
 282
 283                 pool->free_map[pool->next_free] = IBMVNIC_INVALID_MAP;
 284                 pool->rx_buff[index].dma = dma_addr;
 285                 pool->rx_buff[index].skb = skb;
 286                 pool->rx_buff[index].pool_index = pool->index;
 287                 pool->rx_buff[index].size = pool->buff_size;
 288
 289                 memset(&sub_crq, 0, sizeof(sub_crq));
 290                 sub_crq.rx_add.first = IBMVNIC_CRQ_CMD;
 291                 sub_crq.rx_add.correlator =
 292                     cpu_to_be64((u64)&pool->rx_buff[index]);
 293                 sub_crq.rx_add.ioba = cpu_to_be32(dma_addr);
 294                 sub_crq.rx_add.map_id = pool->long_term_buff.map_id;
 295
 296                 /* The length field of the sCRQ is defined to be 24 bits so the
 297                  * buffer size needs to be left shifted by a byte before it is
 298                  * converted to big endian to prevent the last byte from being
 299                  * truncated.
 300                  */
 301 #ifdef __LITTLE_ENDIAN__
 302                 shift = 8;
 303 #endif
 304                 sub_crq.rx_add.len = cpu_to_be32(pool->buff_size << shift);
 305
 306                 lpar_rc = send_subcrq(adapter, handle_array[pool->index],
 307                                       &sub_crq);
 308                 if (lpar_rc != H_SUCCESS)
 309                         goto failure;
 310
 311                 buffers_added++;
 312                 adapter->replenish_add_buff_success++;
 313                 pool->next_free = (pool->next_free + 1) % pool->size;
 314         }
 315         atomic_add(buffers_added, &pool->available);
 316         return;
 317
 318 failure:
 319         dev_info(dev, "replenish pools failure\n");
 320         pool->free_map[pool->next_free] = index;
 321         pool->rx_buff[index].skb = NULL;
 322         if (!dma_mapping_error(dev, dma_addr))
 323                 dma_unmap_single(dev, dma_addr, pool->buff_size,
 324                                  DMA_FROM_DEVICE);
 325
 326         dev_kfree_skb_any(skb);
 327         adapter->replenish_add_buff_failure++;
 328         atomic_add(buffers_added, &pool->available);
 329
 330         if (lpar_rc == H_CLOSED) {
 331                 /* Disable buffer pool replenishment and report carrier off if
 332                  * queue is closed. Firmware guarantees that a signal will
 333                  * be sent to the driver, triggering a reset.
 334                  */
 335                 deactivate_rx_pools(adapter);
 336                 netif_carrier_off(adapter->netdev);
 337         }
 338 }
 339
 340 static void replenish_pools(struct ibmvnic_adapter *adapter)
 341 {
 342         int i;
 343
 344         adapter->replenish_task_cycles++;
 345         for (i = 0; i < be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
 346              i++) {
 347                 if (adapter->rx_pool[i].active)
 348                         replenish_rx_pool(adapter, &adapter->rx_pool[i]);
 349         }
 350 }
 351
 352 static void release_stats_buffers(struct ibmvnic_adapter *adapter)
 353 {
 354         kfree(adapter->tx_stats_buffers);
 355         kfree(adapter->rx_stats_buffers);
 356 }
 357
 358 static int init_stats_buffers(struct ibmvnic_adapter *adapter)
 359 {
 360         adapter->tx_stats_buffers =
 361                                 kcalloc(adapter->req_tx_queues,
 362                                         sizeof(struct ibmvnic_tx_queue_stats),
 363                                         GFP_KERNEL);
 364         if (!adapter->tx_stats_buffers)
 365                 return -ENOMEM;
 366
 367         adapter->rx_stats_buffers =
 368                                 kcalloc(adapter->req_rx_queues,
 369                                         sizeof(struct ibmvnic_rx_queue_stats),
 370                                         GFP_KERNEL);
 371         if (!adapter->rx_stats_buffers)
 372                 return -ENOMEM;
 373
 374         return 0;
 375 }
 376
 377 static void release_stats_token(struct ibmvnic_adapter *adapter)
 378 {
 379         struct device *dev = &adapter->vdev->dev;
 380
 381         if (!adapter->stats_token)
 382                 return;
 383
 384         dma_unmap_single(dev, adapter->stats_token,
 385                          sizeof(struct ibmvnic_statistics),
 386                          DMA_FROM_DEVICE);
 387         adapter->stats_token = 0;
 388 }
 389
 390 static int init_stats_token(struct ibmvnic_adapter *adapter)
 391 {
 392         struct device *dev = &adapter->vdev->dev;
 393         dma_addr_t stok;
 394
 395         stok = dma_map_single(dev, &adapter->stats,
 396                               sizeof(struct ibmvnic_statistics),
 397                               DMA_FROM_DEVICE);
 398         if (dma_mapping_error(dev, stok)) {
 399                 dev_err(dev, "Couldn't map stats buffer\n");
 400                 return -1;
 401         }
 402
 403         adapter->stats_token = stok;
 404         netdev_dbg(adapter->netdev, "Stats token initialized (%llx)\n", stok);
 405         return 0;
 406 }
 407
 408 static int reset_rx_pools(struct ibmvnic_adapter *adapter)
 409 {
 410         struct ibmvnic_rx_pool *rx_pool;
 411         int rx_scrqs;
 412         int i, j, rc;
 413
 414         rx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
 415         for (i = 0; i < rx_scrqs; i++) {
 416                 rx_pool = &adapter->rx_pool[i];
 417
 418                 netdev_dbg(adapter->netdev, "Re-setting rx_pool[%d]\n", i);
 419
 420                 rc = reset_long_term_buff(adapter, &rx_pool->long_term_buff);
 421                 if (rc)
 422                         return rc;
 423
 424                 for (j = 0; j < rx_pool->size; j++)
 425                         rx_pool->free_map[j] = j;
 426
 427                 memset(rx_pool->rx_buff, 0,
 428                        rx_pool->size * sizeof(struct ibmvnic_rx_buff));
 429
 430                 atomic_set(&rx_pool->available, 0);
 431                 rx_pool->next_alloc = 0;
 432                 rx_pool->next_free = 0;
 433                 rx_pool->active = 1;
 434         }
 435
 436         return 0;
 437 }
 438
 439 static void release_rx_pools(struct ibmvnic_adapter *adapter)
 440 {
 441         struct ibmvnic_rx_pool *rx_pool;
 442         int rx_scrqs;
 443         int i, j;
 444
 445         if (!adapter->rx_pool)
 446                 return;
 447
 448         rx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
 449         for (i = 0; i < rx_scrqs; i++) {
 450                 rx_pool = &adapter->rx_pool[i];
 451
 452                 netdev_dbg(adapter->netdev, "Releasing rx_pool[%d]\n", i);
 453
 454                 kfree(rx_pool->free_map);
 455                 free_long_term_buff(adapter, &rx_pool->long_term_buff);
 456
 457                 if (!rx_pool->rx_buff)
 458                         continue;
 459
 460                 for (j = 0; j < rx_pool->size; j++) {
 461                         if (rx_pool->rx_buff[j].skb) {
 462                                 dev_kfree_skb_any(rx_pool->rx_buff[i].skb);
 463                                 rx_pool->rx_buff[i].skb = NULL;
 464                         }
 465                 }
 466
 467                 kfree(rx_pool->rx_buff);
 468         }
 469
 470         kfree(adapter->rx_pool);
 471         adapter->rx_pool = NULL;
 472 }
 473
 474 static int init_rx_pools(struct net_device *netdev)
 475 {
 476         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
 477         struct device *dev = &adapter->vdev->dev;
 478         struct ibmvnic_rx_pool *rx_pool;
 479         int rxadd_subcrqs;
 480         u64 *size_array;
 481         int i, j;
 482
 483         rxadd_subcrqs =
 484                 be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
 485         size_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +
 486                 be32_to_cpu(adapter->login_rsp_buf->off_rxadd_buff_size));
 487
 488         adapter->rx_pool = kcalloc(rxadd_subcrqs,
 489                                    sizeof(struct ibmvnic_rx_pool),
 490                                    GFP_KERNEL);
 491         if (!adapter->rx_pool) {
 492                 dev_err(dev, "Failed to allocate rx pools\n");
 493                 return -1;
 494         }
 495
 496         for (i = 0; i < rxadd_subcrqs; i++) {
 497                 rx_pool = &adapter->rx_pool[i];
 498
 499                 netdev_dbg(adapter->netdev,
 500                            "Initializing rx_pool[%d], %lld buffs, %lld bytes each\n",
 501                            i, adapter->req_rx_add_entries_per_subcrq,
 502                            be64_to_cpu(size_array[i]));
 503
 504                 rx_pool->size = adapter->req_rx_add_entries_per_subcrq;
 505                 rx_pool->index = i;
 506                 rx_pool->buff_size = be64_to_cpu(size_array[i]);
 507                 rx_pool->active = 1;
 508
 509                 rx_pool->free_map = kcalloc(rx_pool->size, sizeof(int),
 510                                             GFP_KERNEL);
 511                 if (!rx_pool->free_map) {
 512                         release_rx_pools(adapter);
 513                         return -1;
 514                 }
 515
 516                 rx_pool->rx_buff = kcalloc(rx_pool->size,
 517                                            sizeof(struct ibmvnic_rx_buff),
 518                                            GFP_KERNEL);
 519                 if (!rx_pool->rx_buff) {
 520                         dev_err(dev, "Couldn't alloc rx buffers\n");
 521                         release_rx_pools(adapter);
 522                         return -1;
 523                 }
 524
 525                 if (alloc_long_term_buff(adapter, &rx_pool->long_term_buff,
 526                                          rx_pool->size * rx_pool->buff_size)) {
 527                         release_rx_pools(adapter);
 528                         return -1;
 529                 }
 530
 531                 for (j = 0; j < rx_pool->size; ++j)
 532                         rx_pool->free_map[j] = j;
 533
 534                 atomic_set(&rx_pool->available, 0);
 535                 rx_pool->next_alloc = 0;
 536                 rx_pool->next_free = 0;
 537         }
 538
 539         return 0;
 540 }
 541
 542 static int reset_tx_pools(struct ibmvnic_adapter *adapter)
 543 {
 544         struct ibmvnic_tx_pool *tx_pool;
 545         int tx_scrqs;
 546         int i, j, rc;
 547
 548         tx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);
 549         for (i = 0; i < tx_scrqs; i++) {
 550                 netdev_dbg(adapter->netdev, "Re-setting tx_pool[%d]\n", i);
 551
 552                 tx_pool = &adapter->tx_pool[i];
 553
 554                 rc = reset_long_term_buff(adapter, &tx_pool->long_term_buff);
 555                 if (rc)
 556                         return rc;
 557
 558                 rc = reset_long_term_buff(adapter, &tx_pool->tso_ltb);
 559                 if (rc)
 560                         return rc;
 561
 562                 memset(tx_pool->tx_buff, 0,
 563                        adapter->req_tx_entries_per_subcrq *
 564                        sizeof(struct ibmvnic_tx_buff));
 565
 566                 for (j = 0; j < adapter->req_tx_entries_per_subcrq; j++)
 567                         tx_pool->free_map[j] = j;
 568
 569                 tx_pool->consumer_index = 0;
 570                 tx_pool->producer_index = 0;
 571                 tx_pool->tso_index = 0;
 572         }
 573
 574         return 0;
 575 }
 576
 577 static void release_vpd_data(struct ibmvnic_adapter *adapter)
 578 {
 579         if (!adapter->vpd)
 580                 return;
 581
 582         kfree(adapter->vpd->buff);
 583         kfree(adapter->vpd);
 584 }
 585
 586 static void release_tx_pools(struct ibmvnic_adapter *adapter)
 587 {
 588         struct ibmvnic_tx_pool *tx_pool;
 589         int i, tx_scrqs;
 590
 591         if (!adapter->tx_pool)
 592                 return;
 593
 594         tx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);
 595         for (i = 0; i < tx_scrqs; i++) {
 596                 netdev_dbg(adapter->netdev, "Releasing tx_pool[%d]\n", i);
 597                 tx_pool = &adapter->tx_pool[i];
 598                 kfree(tx_pool->tx_buff);
 599                 free_long_term_buff(adapter, &tx_pool->long_term_buff);
 600                 free_long_term_buff(adapter, &tx_pool->tso_ltb);
 601                 kfree(tx_pool->free_map);
 602         }
 603
 604         kfree(adapter->tx_pool);
 605         adapter->tx_pool = NULL;
 606 }
 607
 608 static int init_tx_pools(struct net_device *netdev)
 609 {
 610         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
 611         struct device *dev = &adapter->vdev->dev;
 612         struct ibmvnic_tx_pool *tx_pool;
 613         int tx_subcrqs;
 614         int i, j;
 615
 616         tx_subcrqs = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);
 617         adapter->tx_pool = kcalloc(tx_subcrqs,
 618                                    sizeof(struct ibmvnic_tx_pool), GFP_KERNEL);
 619         if (!adapter->tx_pool)
 620                 return -1;
 621
 622         for (i = 0; i < tx_subcrqs; i++) {
 623                 tx_pool = &adapter->tx_pool[i];
 624
 625                 netdev_dbg(adapter->netdev,
 626                            "Initializing tx_pool[%d], %lld buffs\n",
 627                            i, adapter->req_tx_entries_per_subcrq);
 628
 629                 tx_pool->tx_buff = kcalloc(adapter->req_tx_entries_per_subcrq,
 630                                            sizeof(struct ibmvnic_tx_buff),
 631                                            GFP_KERNEL);
 632                 if (!tx_pool->tx_buff) {
 633                         dev_err(dev, "tx pool buffer allocation failed\n");
 634                         release_tx_pools(adapter);
 635                         return -1;
 636                 }
 637
 638                 if (alloc_long_term_buff(adapter, &tx_pool->long_term_buff,
 639                                          adapter->req_tx_entries_per_subcrq *
 640                                          adapter->req_mtu)) {
 641                         release_tx_pools(adapter);
 642                         return -1;
 643                 }
 644
 645                 /* alloc TSO ltb */
 646                 if (alloc_long_term_buff(adapter, &tx_pool->tso_ltb,
 647                                          IBMVNIC_TSO_BUFS *
 648                                          IBMVNIC_TSO_BUF_SZ)) {
 649                         release_tx_pools(adapter);
 650                         return -1;
 651                 }
 652
 653                 tx_pool->tso_index = 0;
 654
 655                 tx_pool->free_map = kcalloc(adapter->req_tx_entries_per_subcrq,
 656                                             sizeof(int), GFP_KERNEL);
 657                 if (!tx_pool->free_map) {
 658                         release_tx_pools(adapter);
 659                         return -1;
 660                 }
 661
 662                 for (j = 0; j < adapter->req_tx_entries_per_subcrq; j++)
 663                         tx_pool->free_map[j] = j;
 664
 665                 tx_pool->consumer_index = 0;
 666                 tx_pool->producer_index = 0;
 667         }
 668
 669         return 0;
 670 }
 671
 672 static void release_error_buffers(struct ibmvnic_adapter *adapter)
 673 {
 674         struct device *dev = &adapter->vdev->dev;
 675         struct ibmvnic_error_buff *error_buff, *tmp;
 676         unsigned long flags;
 677
 678         spin_lock_irqsave(&adapter->error_list_lock, flags);
 679         list_for_each_entry_safe(error_buff, tmp, &adapter->errors, list) {
 680                 list_del(&error_buff->list);
 681                 dma_unmap_single(dev, error_buff->dma, error_buff->len,
 682                                  DMA_FROM_DEVICE);
 683                 kfree(error_buff->buff);
 684                 kfree(error_buff);
 685         }
 686         spin_unlock_irqrestore(&adapter->error_list_lock, flags);
 687 }
 688
 689 static void ibmvnic_napi_enable(struct ibmvnic_adapter *adapter)
 690 {
 691         int i;
 692
 693         if (adapter->napi_enabled)
 694                 return;
 695
 696         for (i = 0; i < adapter->req_rx_queues; i++)
 697                 napi_enable(&adapter->napi[i]);
 698
 699         adapter->napi_enabled = true;
 700 }
 701
 702 static void ibmvnic_napi_disable(struct ibmvnic_adapter *adapter)
 703 {
 704         int i;
 705
 706         if (!adapter->napi_enabled)
 707                 return;
 708
 709         for (i = 0; i < adapter->req_rx_queues; i++) {
 710                 netdev_dbg(adapter->netdev, "Disabling napi[%d]\n", i);
 711                 napi_disable(&adapter->napi[i]);
 712         }
 713
 714         adapter->napi_enabled = false;
 715 }
 716
 717 static int ibmvnic_login(struct net_device *netdev)
 718 {
 719         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
 720         unsigned long timeout = msecs_to_jiffies(30000);
 721         struct device *dev = &adapter->vdev->dev;
 722         int rc;
 723
 724         do {
 725                 if (adapter->renegotiate) {
 726                         adapter->renegotiate = false;
 727                         release_sub_crqs(adapter);
 728
 729                         reinit_completion(&adapter->init_done);
 730                         send_cap_queries(adapter);
 731                         if (!wait_for_completion_timeout(&adapter->init_done,
 732                                                          timeout)) {
 733                                 dev_err(dev, "Capabilities query timeout\n");
 734                                 return -1;
 735                         }
 736                         rc = init_sub_crqs(adapter);
 737                         if (rc) {
 738                                 dev_err(dev,
 739                                         "Initialization of SCRQ's failed\n");
 740                                 return -1;
 741                         }
 742                         rc = init_sub_crq_irqs(adapter);
 743                         if (rc) {
 744                                 dev_err(dev,
 745                                         "Initialization of SCRQ's irqs failed\n");
 746                                 return -1;
 747                         }
 748                 }
 749
 750                 reinit_completion(&adapter->init_done);
 751                 send_login(adapter);
 752                 if (!wait_for_completion_timeout(&adapter->init_done,
 753                                                  timeout)) {
 754                         dev_err(dev, "Login timeout\n");
 755                         return -1;
 756                 }
 757         } while (adapter->renegotiate);
 758
 759         return 0;
 760 }
 761
 762 static void release_resources(struct ibmvnic_adapter *adapter)
 763 {
 764         int i;
 765
 766         release_vpd_data(adapter);
 767
 768         release_tx_pools(adapter);
 769         release_rx_pools(adapter);
 770
 771         release_stats_token(adapter);
 772         release_stats_buffers(adapter);
 773         release_error_buffers(adapter);
 774
 775         if (adapter->napi) {
 776                 for (i = 0; i < adapter->req_rx_queues; i++) {
 777                         if (&adapter->napi[i]) {
 778                                 netdev_dbg(adapter->netdev,
 779                                            "Releasing napi[%d]\n", i);
 780                                 netif_napi_del(&adapter->napi[i]);
 781                         }
 782                 }
 783         }
 784 }
 785
 786 static int set_link_state(struct ibmvnic_adapter *adapter, u8 link_state)
 787 {
 788         struct net_device *netdev = adapter->netdev;
 789         unsigned long timeout = msecs_to_jiffies(30000);
 790         union ibmvnic_crq crq;
 791         bool resend;
 792         int rc;
 793
 794         netdev_dbg(netdev, "setting link state %d\n", link_state);
 795
 796         memset(&crq, 0, sizeof(crq));
 797         crq.logical_link_state.first = IBMVNIC_CRQ_CMD;
 798         crq.logical_link_state.cmd = LOGICAL_LINK_STATE;
 799         crq.logical_link_state.link_state = link_state;
 800
 801         do {
 802                 resend = false;
 803
 804                 reinit_completion(&adapter->init_done);
 805                 rc = ibmvnic_send_crq(adapter, &crq);
 806                 if (rc) {
 807                         netdev_err(netdev, "Failed to set link state\n");
 808                         return rc;
 809                 }
 810
 811                 if (!wait_for_completion_timeout(&adapter->init_done,
 812                                                  timeout)) {
 813                         netdev_err(netdev, "timeout setting link state\n");
 814                         return -1;
 815                 }
 816
 817                 if (adapter->init_done_rc == 1) {
 818                         /* Partuial success, delay and re-send */
 819                         mdelay(1000);
 820                         resend = true;
 821                 }
 822         } while (resend);
 823
 824         return 0;
 825 }
 826
 827 static int set_real_num_queues(struct net_device *netdev)
 828 {
 829         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
 830         int rc;
 831
 832         netdev_dbg(netdev, "Setting real tx/rx queues (%llx/%llx)\n",
 833                    adapter->req_tx_queues, adapter->req_rx_queues);
 834
 835         rc = netif_set_real_num_tx_queues(netdev, adapter->req_tx_queues);
 836         if (rc) {
 837                 netdev_err(netdev, "failed to set the number of tx queues\n");
 838                 return rc;
 839         }
 840
 841         rc = netif_set_real_num_rx_queues(netdev, adapter->req_rx_queues);
 842         if (rc)
 843                 netdev_err(netdev, "failed to set the number of rx queues\n");
 844
 845         return rc;
 846 }
 847
 848 static int ibmvnic_get_vpd(struct ibmvnic_adapter *adapter)
 849 {
 850         struct device *dev = &adapter->vdev->dev;
 851         union ibmvnic_crq crq;
 852         int len = 0;
 853
 854         if (adapter->vpd->buff)
 855                 len = adapter->vpd->len;
 856
 857         reinit_completion(&adapter->fw_done);
 858         crq.get_vpd_size.first = IBMVNIC_CRQ_CMD;
 859         crq.get_vpd_size.cmd = GET_VPD_SIZE;
 860         ibmvnic_send_crq(adapter, &crq);
 861         wait_for_completion(&adapter->fw_done);
 862
 863         if (!adapter->vpd->len)
 864                 return -ENODATA;
 865
 866         if (!adapter->vpd->buff)
 867                 adapter->vpd->buff = kzalloc(adapter->vpd->len, GFP_KERNEL);
 868         else if (adapter->vpd->len != len)
 869                 adapter->vpd->buff =
 870                         krealloc(adapter->vpd->buff,
 871                                  adapter->vpd->len, GFP_KERNEL);
 872
 873         if (!adapter->vpd->buff) {
 874                 dev_err(dev, "Could allocate VPD buffer\n");
 875                 return -ENOMEM;
 876         }
 877
 878         adapter->vpd->dma_addr =
 879                 dma_map_single(dev, adapter->vpd->buff, adapter->vpd->len,
 880                                DMA_FROM_DEVICE);
 881         if (dma_mapping_error(dev, adapter->vpd->dma_addr)) {
 882                 dev_err(dev, "Could not map VPD buffer\n");
 883                 kfree(adapter->vpd->buff);
 884                 return -ENOMEM;
 885         }
 886
 887         reinit_completion(&adapter->fw_done);
 888         crq.get_vpd.first = IBMVNIC_CRQ_CMD;
 889         crq.get_vpd.cmd = GET_VPD;
 890         crq.get_vpd.ioba = cpu_to_be32(adapter->vpd->dma_addr);
 891         crq.get_vpd.len = cpu_to_be32((u32)adapter->vpd->len);
 892         ibmvnic_send_crq(adapter, &crq);
 893         wait_for_completion(&adapter->fw_done);
 894
 895         return 0;
 896 }
 897
 898 static int init_resources(struct ibmvnic_adapter *adapter)
 899 {
 900         struct net_device *netdev = adapter->netdev;
 901         int i, rc;
 902
 903         rc = set_real_num_queues(netdev);
 904         if (rc)
 905                 return rc;
 906
 907         rc = init_stats_buffers(adapter);
 908         if (rc)
 909                 return rc;
 910
 911         rc = init_stats_token(adapter);
 912         if (rc)
 913                 return rc;
 914
 915         adapter->vpd = kzalloc(sizeof(*adapter->vpd), GFP_KERNEL);
 916         if (!adapter->vpd)
 917                 return -ENOMEM;
 918
 919         adapter->map_id = 1;
 920         adapter->napi = kcalloc(adapter->req_rx_queues,
 921                                 sizeof(struct napi_struct), GFP_KERNEL);
 922         if (!adapter->napi)
 923                 return -ENOMEM;
 924
 925         for (i = 0; i < adapter->req_rx_queues; i++) {
 926                 netdev_dbg(netdev, "Adding napi[%d]\n", i);
 927                 netif_napi_add(netdev, &adapter->napi[i], ibmvnic_poll,
 928                                NAPI_POLL_WEIGHT);
 929         }
 930
 931         send_map_query(adapter);
 932
 933         rc = init_rx_pools(netdev);
 934         if (rc)
 935                 return rc;
 936
 937         rc = init_tx_pools(netdev);
 938         return rc;
 939 }
 940
 941 static int __ibmvnic_open(struct net_device *netdev)
 942 {
 943         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
 944         enum vnic_state prev_state = adapter->state;
 945         int i, rc;
 946
 947         adapter->state = VNIC_OPENING;
 948         replenish_pools(adapter);
 949         ibmvnic_napi_enable(adapter);
 950
 951         /* We're ready to receive frames, enable the sub-crq interrupts and
 952          * set the logical link state to up
 953          */
 954         for (i = 0; i < adapter->req_rx_queues; i++) {
 955                 netdev_dbg(netdev, "Enabling rx_scrq[%d] irq\n", i);
 956                 if (prev_state == VNIC_CLOSED)
 957                         enable_irq(adapter->rx_scrq[i]->irq);
 958                 else
 959                         enable_scrq_irq(adapter, adapter->rx_scrq[i]);
 960         }
 961
 962         for (i = 0; i < adapter->req_tx_queues; i++) {
 963                 netdev_dbg(netdev, "Enabling tx_scrq[%d] irq\n", i);
 964                 if (prev_state == VNIC_CLOSED)
 965                         enable_irq(adapter->tx_scrq[i]->irq);
 966                 else
 967                         enable_scrq_irq(adapter, adapter->tx_scrq[i]);
 968         }
 969
 970         rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_UP);
 971         if (rc) {
 972                 for (i = 0; i < adapter->req_rx_queues; i++)
 973                         napi_disable(&adapter->napi[i]);
 974                 release_resources(adapter);
 975                 return rc;
 976         }
 977
 978         netif_tx_start_all_queues(netdev);
 979
 980         if (prev_state == VNIC_CLOSED) {
 981                 for (i = 0; i < adapter->req_rx_queues; i++)
 982                         napi_schedule(&adapter->napi[i]);
 983         }
 984
 985         adapter->state = VNIC_OPEN;
 986         return rc;
 987 }
 988
 989 static int ibmvnic_open(struct net_device *netdev)
 990 {
 991         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
 992         int rc, vpd;
 993
 994         mutex_lock(&adapter->reset_lock);
 995
 996         if (adapter->mac_change_pending) {
 997                 __ibmvnic_set_mac(netdev, &adapter->desired.mac);
 998                 adapter->mac_change_pending = false;
 999         }
1000
1001         if (adapter->state != VNIC_CLOSED) {
1002                 rc = ibmvnic_login(netdev);
1003                 if (rc) {
1004                         mutex_unlock(&adapter->reset_lock);
1005                         return rc;
1006                 }
1007
1008                 rc = init_resources(adapter);
1009                 if (rc) {
1010                         netdev_err(netdev, "failed to initialize resources\n");
1011                         release_resources(adapter);
1012                         mutex_unlock(&adapter->reset_lock);
1013                         return rc;
1014                 }
1015         }
1016
1017         rc = __ibmvnic_open(netdev);
1018         netif_carrier_on(netdev);
1019
1020         /* Vital Product Data (VPD) */
1021         vpd = ibmvnic_get_vpd(adapter);
1022         if (vpd)
1023                 netdev_err(netdev, "failed to initialize Vital Product Data (VPD)\n");
1024
1025         mutex_unlock(&adapter->reset_lock);
1026
1027         return rc;
1028 }
1029
1030 static void clean_tx_pools(struct ibmvnic_adapter *adapter)
1031 {
1032         struct ibmvnic_tx_pool *tx_pool;
1033         u64 tx_entries;
1034         int tx_scrqs;
1035         int i, j;
1036
1037         if (!adapter->tx_pool)
1038                 return;
1039
1040         tx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);
1041         tx_entries = adapter->req_tx_entries_per_subcrq;
1042
1043         /* Free any remaining skbs in the tx buffer pools */
1044         for (i = 0; i < tx_scrqs; i++) {
1045                 tx_pool = &adapter->tx_pool[i];
1046                 if (!tx_pool)
1047                         continue;
1048
1049                 netdev_dbg(adapter->netdev, "Cleaning tx_pool[%d]\n", i);
1050                 for (j = 0; j < tx_entries; j++) {
1051                         if (tx_pool->tx_buff[j].skb) {
1052                                 dev_kfree_skb_any(tx_pool->tx_buff[j].skb);
1053                                 tx_pool->tx_buff[j].skb = NULL;
1054                         }
1055                 }
1056         }
1057 }
1058
1059 static int __ibmvnic_close(struct net_device *netdev)
1060 {
1061         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1062         int rc = 0;
1063         int i;
1064
1065         adapter->state = VNIC_CLOSING;
1066
1067         /* ensure that transmissions are stopped if called by do_reset */
1068         if (adapter->resetting)
1069                 netif_tx_disable(netdev);
1070         else
1071                 netif_tx_stop_all_queues(netdev);
1072
1073         ibmvnic_napi_disable(adapter);
1074
1075         if (adapter->tx_scrq) {
1076                 for (i = 0; i < adapter->req_tx_queues; i++)
1077                         if (adapter->tx_scrq[i]->irq) {
1078                                 netdev_dbg(adapter->netdev,
1079                                            "Disabling tx_scrq[%d] irq\n", i);
1080                                 disable_irq(adapter->tx_scrq[i]->irq);
1081                         }
1082         }
1083
1084         rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);
1085         if (rc)
1086                 return rc;
1087
1088         if (adapter->rx_scrq) {
1089                 for (i = 0; i < adapter->req_rx_queues; i++) {
1090                         int retries = 10;
1091
1092                         while (pending_scrq(adapter, adapter->rx_scrq[i])) {
1093                                 retries--;
1094                                 mdelay(100);
1095
1096                                 if (retries == 0)
1097                                         break;
1098                         }
1099
1100                         if (adapter->rx_scrq[i]->irq) {
1101                                 netdev_dbg(adapter->netdev,
1102                                            "Disabling rx_scrq[%d] irq\n", i);
1103                                 disable_irq(adapter->rx_scrq[i]->irq);
1104                         }
1105                 }
1106         }
1107
1108         clean_tx_pools(adapter);
1109         adapter->state = VNIC_CLOSED;
1110         return rc;
1111 }
1112
1113 static int ibmvnic_close(struct net_device *netdev)
1114 {
1115         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1116         int rc;
1117
1118         mutex_lock(&adapter->reset_lock);
1119         rc = __ibmvnic_close(netdev);
1120         mutex_unlock(&adapter->reset_lock);
1121
1122         return rc;
1123 }
1124
1125 /**
1126  * build_hdr_data - creates L2/L3/L4 header data buffer
1127  * @hdr_field - bitfield determining needed headers
1128  * @skb - socket buffer
1129  * @hdr_len - array of header lengths
1130  * @tot_len - total length of data
1131  *
1132  * Reads hdr_field to determine which headers are needed by firmware.
1133  * Builds a buffer containing these headers.  Saves individual header
1134  * lengths and total buffer length to be used to build descriptors.
1135  */
1136 static int build_hdr_data(u8 hdr_field, struct sk_buff *skb,
1137                           int *hdr_len, u8 *hdr_data)
1138 {
1139         int len = 0;
1140         u8 *hdr;
1141
1142         hdr_len[0] = sizeof(struct ethhdr);
1143
1144         if (skb->protocol == htons(ETH_P_IP)) {
1145                 hdr_len[1] = ip_hdr(skb)->ihl * 4;
1146                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1147                         hdr_len[2] = tcp_hdrlen(skb);
1148                 else if (ip_hdr(skb)->protocol == IPPROTO_UDP)
1149                         hdr_len[2] = sizeof(struct udphdr);
1150         } else if (skb->protocol == htons(ETH_P_IPV6)) {
1151                 hdr_len[1] = sizeof(struct ipv6hdr);
1152                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1153                         hdr_len[2] = tcp_hdrlen(skb);
1154                 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)
1155                         hdr_len[2] = sizeof(struct udphdr);
1156         }
1157
1158         memset(hdr_data, 0, 120);
1159         if ((hdr_field >> 6) & 1) {
1160                 hdr = skb_mac_header(skb);
1161                 memcpy(hdr_data, hdr, hdr_len[0]);
1162                 len += hdr_len[0];
1163         }
1164
1165         if ((hdr_field >> 5) & 1) {
1166                 hdr = skb_network_header(skb);
1167                 memcpy(hdr_data + len, hdr, hdr_len[1]);
1168                 len += hdr_len[1];
1169         }
1170
1171         if ((hdr_field >> 4) & 1) {
1172                 hdr = skb_transport_header(skb);
1173                 memcpy(hdr_data + len, hdr, hdr_len[2]);
1174                 len += hdr_len[2];
1175         }
1176         return len;
1177 }
1178
1179 /**
1180  * create_hdr_descs - create header and header extension descriptors
1181  * @hdr_field - bitfield determining needed headers
1182  * @data - buffer containing header data
1183  * @len - length of data buffer
1184  * @hdr_len - array of individual header lengths
1185  * @scrq_arr - descriptor array
1186  *
1187  * Creates header and, if needed, header extension descriptors and
1188  * places them in a descriptor array, scrq_arr
1189  */
1190
1191 static int create_hdr_descs(u8 hdr_field, u8 *hdr_data, int len, int *hdr_len,
1192                             union sub_crq *scrq_arr)
1193 {
1194         union sub_crq hdr_desc;
1195         int tmp_len = len;
1196         int num_descs = 0;
1197         u8 *data, *cur;
1198         int tmp;
1199
1200         while (tmp_len > 0) {
1201                 cur = hdr_data + len - tmp_len;
1202
1203                 memset(&hdr_desc, 0, sizeof(hdr_desc));
1204                 if (cur != hdr_data) {
1205                         data = hdr_desc.hdr_ext.data;
1206                         tmp = tmp_len > 29 ? 29 : tmp_len;
1207                         hdr_desc.hdr_ext.first = IBMVNIC_CRQ_CMD;
1208                         hdr_desc.hdr_ext.type = IBMVNIC_HDR_EXT_DESC;
1209                         hdr_desc.hdr_ext.len = tmp;
1210                 } else {
1211                         data = hdr_desc.hdr.data;
1212                         tmp = tmp_len > 24 ? 24 : tmp_len;
1213                         hdr_desc.hdr.first = IBMVNIC_CRQ_CMD;
1214                         hdr_desc.hdr.type = IBMVNIC_HDR_DESC;
1215                         hdr_desc.hdr.len = tmp;
1216                         hdr_desc.hdr.l2_len = (u8)hdr_len[0];
1217                         hdr_desc.hdr.l3_len = cpu_to_be16((u16)hdr_len[1]);
1218                         hdr_desc.hdr.l4_len = (u8)hdr_len[2];
1219                         hdr_desc.hdr.flag = hdr_field << 1;
1220                 }
1221                 memcpy(data, cur, tmp);
1222                 tmp_len -= tmp;
1223                 *scrq_arr = hdr_desc;
1224                 scrq_arr++;
1225                 num_descs++;
1226         }
1227
1228         return num_descs;
1229 }
1230
1231 /**
1232  * build_hdr_descs_arr - build a header descriptor array
1233  * @skb - socket buffer
1234  * @num_entries - number of descriptors to be sent
1235  * @subcrq - first TX descriptor
1236  * @hdr_field - bit field determining which headers will be sent
1237  *
1238  * This function will build a TX descriptor array with applicable
1239  * L2/L3/L4 packet header descriptors to be sent by send_subcrq_indirect.
1240  */
1241
1242 static void build_hdr_descs_arr(struct ibmvnic_tx_buff *txbuff,
1243                                 int *num_entries, u8 hdr_field)
1244 {
1245         int hdr_len[3] = {0, 0, 0};
1246         int tot_len;
1247         u8 *hdr_data = txbuff->hdr_data;
1248
1249         tot_len = build_hdr_data(hdr_field, txbuff->skb, hdr_len,
1250                                  txbuff->hdr_data);
1251         *num_entries += create_hdr_descs(hdr_field, hdr_data, tot_len, hdr_len,
1252                          txbuff->indir_arr + 1);
1253 }
1254
1255 static int ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev)
1256 {
1257         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1258         int queue_num = skb_get_queue_mapping(skb);
1259         u8 *hdrs = (u8 *)&adapter->tx_rx_desc_req;
1260         struct device *dev = &adapter->vdev->dev;
1261         struct ibmvnic_tx_buff *tx_buff = NULL;
1262         struct ibmvnic_sub_crq_queue *tx_scrq;
1263         struct ibmvnic_tx_pool *tx_pool;
1264         unsigned int tx_send_failed = 0;
1265         unsigned int tx_map_failed = 0;
1266         unsigned int tx_dropped = 0;
1267         unsigned int tx_packets = 0;
1268         unsigned int tx_bytes = 0;
1269         dma_addr_t data_dma_addr;
1270         struct netdev_queue *txq;
1271         unsigned long lpar_rc;
1272         union sub_crq tx_crq;
1273         unsigned int offset;
1274         int num_entries = 1;
1275         unsigned char *dst;
1276         u64 *handle_array;
1277         int index = 0;
1278         int ret = 0;
1279
1280         if (adapter->resetting) {
1281                 if (!netif_subqueue_stopped(netdev, skb))
1282                         netif_stop_subqueue(netdev, queue_num);
1283                 dev_kfree_skb_any(skb);
1284
1285                 tx_send_failed++;
1286                 tx_dropped++;
1287                 ret = NETDEV_TX_OK;
1288                 goto out;
1289         }
1290
1291         tx_pool = &adapter->tx_pool[queue_num];
1292         tx_scrq = adapter->tx_scrq[queue_num];
1293         txq = netdev_get_tx_queue(netdev, skb_get_queue_mapping(skb));
1294         handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +
1295                 be32_to_cpu(adapter->login_rsp_buf->off_txsubm_subcrqs));
1296
1297         index = tx_pool->free_map[tx_pool->consumer_index];
1298
1299         if (skb_is_gso(skb)) {
1300                 offset = tx_pool->tso_index * IBMVNIC_TSO_BUF_SZ;
1301                 dst = tx_pool->tso_ltb.buff + offset;
1302                 memset(dst, 0, IBMVNIC_TSO_BUF_SZ);
1303                 data_dma_addr = tx_pool->tso_ltb.addr + offset;
1304                 tx_pool->tso_index++;
1305                 if (tx_pool->tso_index == IBMVNIC_TSO_BUFS)
1306                         tx_pool->tso_index = 0;
1307         } else {
1308                 offset = index * adapter->req_mtu;
1309                 dst = tx_pool->long_term_buff.buff + offset;
1310                 memset(dst, 0, adapter->req_mtu);
1311                 data_dma_addr = tx_pool->long_term_buff.addr + offset;
1312         }
1313
1314         if (skb_shinfo(skb)->nr_frags) {
1315                 int cur, i;
1316
1317                 /* Copy the head */
1318                 skb_copy_from_linear_data(skb, dst, skb_headlen(skb));
1319                 cur = skb_headlen(skb);
1320
1321                 /* Copy the frags */
1322                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1323                         const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1324
1325                         memcpy(dst + cur,
1326                                page_address(skb_frag_page(frag)) +
1327                                frag->page_offset, skb_frag_size(frag));
1328                         cur += skb_frag_size(frag);
1329                 }
1330         } else {
1331                 skb_copy_from_linear_data(skb, dst, skb->len);
1332         }
1333
1334         tx_pool->consumer_index =
1335             (tx_pool->consumer_index + 1) %
1336                 adapter->req_tx_entries_per_subcrq;
1337
1338         tx_buff = &tx_pool->tx_buff[index];
1339         tx_buff->skb = skb;
1340         tx_buff->data_dma[0] = data_dma_addr;
1341         tx_buff->data_len[0] = skb->len;
1342         tx_buff->index = index;
1343         tx_buff->pool_index = queue_num;
1344         tx_buff->last_frag = true;
1345
1346         memset(&tx_crq, 0, sizeof(tx_crq));
1347         tx_crq.v1.first = IBMVNIC_CRQ_CMD;
1348         tx_crq.v1.type = IBMVNIC_TX_DESC;
1349         tx_crq.v1.n_crq_elem = 1;
1350         tx_crq.v1.n_sge = 1;
1351         tx_crq.v1.flags1 = IBMVNIC_TX_COMP_NEEDED;
1352         tx_crq.v1.correlator = cpu_to_be32(index);
1353         if (skb_is_gso(skb))
1354                 tx_crq.v1.dma_reg = cpu_to_be16(tx_pool->tso_ltb.map_id);
1355         else
1356                 tx_crq.v1.dma_reg = cpu_to_be16(tx_pool->long_term_buff.map_id);
1357         tx_crq.v1.sge_len = cpu_to_be32(skb->len);
1358         tx_crq.v1.ioba = cpu_to_be64(data_dma_addr);
1359
1360         if (adapter->vlan_header_insertion) {
1361                 tx_crq.v1.flags2 |= IBMVNIC_TX_VLAN_INSERT;
1362                 tx_crq.v1.vlan_id = cpu_to_be16(skb->vlan_tci);
1363         }
1364
1365         if (skb->protocol == htons(ETH_P_IP)) {
1366                 if (ip_hdr(skb)->version == 4)
1367                         tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_IPV4;
1368                 else if (ip_hdr(skb)->version == 6)
1369                         tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_IPV6;
1370
1371                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1372                         tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_TCP;
1373                 else if (ip_hdr(skb)->protocol != IPPROTO_TCP)
1374                         tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_UDP;
1375         }
1376
1377         if (skb->ip_summed == CHECKSUM_PARTIAL) {
1378                 tx_crq.v1.flags1 |= IBMVNIC_TX_CHKSUM_OFFLOAD;
1379                 hdrs += 2;
1380         }
1381         if (skb_is_gso(skb)) {
1382                 tx_crq.v1.flags1 |= IBMVNIC_TX_LSO;
1383                 tx_crq.v1.mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
1384                 hdrs += 2;
1385         }
1386         /* determine if l2/3/4 headers are sent to firmware */
1387         if ((*hdrs >> 7) & 1 &&
1388             (skb->protocol == htons(ETH_P_IP) ||
1389              skb->protocol == htons(ETH_P_IPV6))) {
1390                 build_hdr_descs_arr(tx_buff, &num_entries, *hdrs);
1391                 tx_crq.v1.n_crq_elem = num_entries;
1392                 tx_buff->indir_arr[0] = tx_crq;
1393                 tx_buff->indir_dma = dma_map_single(dev, tx_buff->indir_arr,
1394                                                     sizeof(tx_buff->indir_arr),
1395                                                     DMA_TO_DEVICE);
1396                 if (dma_mapping_error(dev, tx_buff->indir_dma)) {
1397                         dev_kfree_skb_any(skb);
1398                         tx_buff->skb = NULL;
1399                         if (!firmware_has_feature(FW_FEATURE_CMO))
1400                                 dev_err(dev, "tx: unable to map descriptor array\n");
1401                         tx_map_failed++;
1402                         tx_dropped++;
1403                         ret = NETDEV_TX_OK;
1404                         goto out;
1405                 }
1406                 lpar_rc = send_subcrq_indirect(adapter, handle_array[queue_num],
1407                                                (u64)tx_buff->indir_dma,
1408                                                (u64)num_entries);
1409         } else {
1410                 lpar_rc = send_subcrq(adapter, handle_array[queue_num],
1411                                       &tx_crq);
1412         }
1413         if (lpar_rc != H_SUCCESS) {
1414                 dev_err(dev, "tx failed with code %ld\n", lpar_rc);
1415
1416                 if (tx_pool->consumer_index == 0)
1417                         tx_pool->consumer_index =
1418                                 adapter->req_tx_entries_per_subcrq - 1;
1419                 else
1420                         tx_pool->consumer_index--;
1421
1422                 dev_kfree_skb_any(skb);
1423                 tx_buff->skb = NULL;
1424
1425                 if (lpar_rc == H_CLOSED) {
1426                         /* Disable TX and report carrier off if queue is closed.
1427                          * Firmware guarantees that a signal will be sent to the
1428                          * driver, triggering a reset or some other action.
1429                          */
1430                         netif_tx_stop_all_queues(netdev);
1431                         netif_carrier_off(netdev);
1432                 }
1433
1434                 tx_send_failed++;
1435                 tx_dropped++;
1436                 ret = NETDEV_TX_OK;
1437                 goto out;
1438         }
1439
1440         if (atomic_inc_return(&tx_scrq->used)
1441                                         >= adapter->req_tx_entries_per_subcrq) {
1442                 netdev_info(netdev, "Stopping queue %d\n", queue_num);
1443                 netif_stop_subqueue(netdev, queue_num);
1444         }
1445
1446         tx_packets++;
1447         tx_bytes += skb->len;
1448         txq->trans_start = jiffies;
1449         ret = NETDEV_TX_OK;
1450
1451 out:
1452         netdev->stats.tx_dropped += tx_dropped;
1453         netdev->stats.tx_bytes += tx_bytes;
1454         netdev->stats.tx_packets += tx_packets;
1455         adapter->tx_send_failed += tx_send_failed;
1456         adapter->tx_map_failed += tx_map_failed;
1457         adapter->tx_stats_buffers[queue_num].packets += tx_packets;
1458         adapter->tx_stats_buffers[queue_num].bytes += tx_bytes;
1459         adapter->tx_stats_buffers[queue_num].dropped_packets += tx_dropped;
1460
1461         return ret;
1462 }
1463
1464 static void ibmvnic_set_multi(struct net_device *netdev)
1465 {
1466         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1467         struct netdev_hw_addr *ha;
1468         union ibmvnic_crq crq;
1469
1470         memset(&crq, 0, sizeof(crq));
1471         crq.request_capability.first = IBMVNIC_CRQ_CMD;
1472         crq.request_capability.cmd = REQUEST_CAPABILITY;
1473
1474         if (netdev->flags & IFF_PROMISC) {
1475                 if (!adapter->promisc_supported)
1476                         return;
1477         } else {
1478                 if (netdev->flags & IFF_ALLMULTI) {
1479                         /* Accept all multicast */
1480                         memset(&crq, 0, sizeof(crq));
1481                         crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;
1482                         crq.multicast_ctrl.cmd = MULTICAST_CTRL;
1483                         crq.multicast_ctrl.flags = IBMVNIC_ENABLE_ALL;
1484                         ibmvnic_send_crq(adapter, &crq);
1485                 } else if (netdev_mc_empty(netdev)) {
1486                         /* Reject all multicast */
1487                         memset(&crq, 0, sizeof(crq));
1488                         crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;
1489                         crq.multicast_ctrl.cmd = MULTICAST_CTRL;
1490                         crq.multicast_ctrl.flags = IBMVNIC_DISABLE_ALL;
1491                         ibmvnic_send_crq(adapter, &crq);
1492                 } else {
1493                         /* Accept one or more multicast(s) */
1494                         netdev_for_each_mc_addr(ha, netdev) {
1495                                 memset(&crq, 0, sizeof(crq));
1496                                 crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;
1497                                 crq.multicast_ctrl.cmd = MULTICAST_CTRL;
1498                                 crq.multicast_ctrl.flags = IBMVNIC_ENABLE_MC;
1499                                 ether_addr_copy(&crq.multicast_ctrl.mac_addr[0],
1500                                                 ha->addr);
1501                                 ibmvnic_send_crq(adapter, &crq);
1502                         }
1503                 }
1504         }
1505 }
1506
1507 static int __ibmvnic_set_mac(struct net_device *netdev, struct sockaddr *p)
1508 {
1509         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1510         struct sockaddr *addr = p;
1511         union ibmvnic_crq crq;
1512
1513         if (!is_valid_ether_addr(addr->sa_data))
1514                 return -EADDRNOTAVAIL;
1515
1516         memset(&crq, 0, sizeof(crq));
1517         crq.change_mac_addr.first = IBMVNIC_CRQ_CMD;
1518         crq.change_mac_addr.cmd = CHANGE_MAC_ADDR;
1519         ether_addr_copy(&crq.change_mac_addr.mac_addr[0], addr->sa_data);
1520         ibmvnic_send_crq(adapter, &crq);
1521         /* netdev->dev_addr is changed in handle_change_mac_rsp function */
1522         return 0;
1523 }
1524
1525 static int ibmvnic_set_mac(struct net_device *netdev, void *p)
1526 {
1527         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1528         struct sockaddr *addr = p;
1529
1530         if (adapter->state != VNIC_OPEN) {
1531                 memcpy(&adapter->desired.mac, addr, sizeof(struct sockaddr));
1532                 adapter->mac_change_pending = true;
1533                 return 0;
1534         }
1535
1536         __ibmvnic_set_mac(netdev, addr);
1537
1538         return 0;
1539 }
1540
1541 /**
1542  * do_reset returns zero if we are able to keep processing reset events, or
1543  * non-zero if we hit a fatal error and must halt.
1544  */
1545 static int do_reset(struct ibmvnic_adapter *adapter,
1546                     struct ibmvnic_rwi *rwi, u32 reset_state)
1547 {
1548         struct net_device *netdev = adapter->netdev;
1549         int i, rc;
1550
1551         netdev_dbg(adapter->netdev, "Re-setting driver (%d)\n",
1552                    rwi->reset_reason);
1553
1554         netif_carrier_off(netdev);
1555         adapter->reset_reason = rwi->reset_reason;
1556
1557         if (rwi->reset_reason == VNIC_RESET_MOBILITY) {
1558                 rc = ibmvnic_reenable_crq_queue(adapter);
1559                 if (rc)
1560                         return 0;
1561         }
1562
1563         rc = __ibmvnic_close(netdev);
1564         if (rc)
1565                 return rc;
1566
1567         if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM ||
1568             adapter->wait_for_reset) {
1569                 release_resources(adapter);
1570                 release_sub_crqs(adapter);
1571                 release_crq_queue(adapter);
1572         }
1573
1574         if (adapter->reset_reason != VNIC_RESET_NON_FATAL) {
1575                 /* remove the closed state so when we call open it appears
1576                  * we are coming from the probed state.
1577                  */
1578                 adapter->state = VNIC_PROBED;
1579
1580                 rc = ibmvnic_init(adapter);
1581                 if (rc)
1582                         return IBMVNIC_INIT_FAILED;
1583
1584                 /* If the adapter was in PROBE state prior to the reset,
1585                  * exit here.
1586                  */
1587                 if (reset_state == VNIC_PROBED)
1588                         return 0;
1589
1590                 rc = ibmvnic_login(netdev);
1591                 if (rc) {
1592                         adapter->state = VNIC_PROBED;
1593                         return 0;
1594                 }
1595
1596                 if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM ||
1597                     adapter->wait_for_reset) {
1598                         rc = init_resources(adapter);
1599                         if (rc)
1600                                 return rc;
1601                 } else {
1602                         rc = reset_tx_pools(adapter);
1603                         if (rc)
1604                                 return rc;
1605
1606                         rc = reset_rx_pools(adapter);
1607                         if (rc)
1608                                 return rc;
1609
1610                         if (reset_state == VNIC_CLOSED)
1611                                 return 0;
1612                 }
1613         }
1614
1615         rc = __ibmvnic_open(netdev);
1616         if (rc) {
1617                 if (list_empty(&adapter->rwi_list))
1618                         adapter->state = VNIC_CLOSED;
1619                 else
1620                         adapter->state = reset_state;
1621
1622                 return 0;
1623         }
1624
1625         netif_carrier_on(netdev);
1626
1627         /* kick napi */
1628         for (i = 0; i < adapter->req_rx_queues; i++)
1629                 napi_schedule(&adapter->napi[i]);
1630
1631         if (adapter->reset_reason != VNIC_RESET_FAILOVER)
1632                 netdev_notify_peers(netdev);
1633
1634         return 0;
1635 }
1636
1637 static struct ibmvnic_rwi *get_next_rwi(struct ibmvnic_adapter *adapter)
1638 {
1639         struct ibmvnic_rwi *rwi;
1640
1641         mutex_lock(&adapter->rwi_lock);
1642
1643         if (!list_empty(&adapter->rwi_list)) {
1644                 rwi = list_first_entry(&adapter->rwi_list, struct ibmvnic_rwi,
1645                                        list);
1646                 list_del(&rwi->list);
1647         } else {
1648                 rwi = NULL;
1649         }
1650
1651         mutex_unlock(&adapter->rwi_lock);
1652         return rwi;
1653 }
1654
1655 static void free_all_rwi(struct ibmvnic_adapter *adapter)
1656 {
1657         struct ibmvnic_rwi *rwi;
1658
1659         rwi = get_next_rwi(adapter);
1660         while (rwi) {
1661                 kfree(rwi);
1662                 rwi = get_next_rwi(adapter);
1663         }
1664 }
1665
1666 static void __ibmvnic_reset(struct work_struct *work)
1667 {
1668         struct ibmvnic_rwi *rwi;
1669         struct ibmvnic_adapter *adapter;
1670         struct net_device *netdev;
1671         u32 reset_state;
1672         int rc = 0;
1673
1674         adapter = container_of(work, struct ibmvnic_adapter, ibmvnic_reset);
1675         netdev = adapter->netdev;
1676
1677         mutex_lock(&adapter->reset_lock);
1678         adapter->resetting = true;
1679         reset_state = adapter->state;
1680
1681         rwi = get_next_rwi(adapter);
1682         while (rwi) {
1683                 rc = do_reset(adapter, rwi, reset_state);
1684                 kfree(rwi);
1685                 if (rc && rc != IBMVNIC_INIT_FAILED)
1686                         break;
1687
1688                 rwi = get_next_rwi(adapter);
1689         }
1690
1691         if (adapter->wait_for_reset) {
1692                 adapter->wait_for_reset = false;
1693                 adapter->reset_done_rc = rc;
1694                 complete(&adapter->reset_done);
1695         }
1696
1697         if (rc) {
1698                 netdev_dbg(adapter->netdev, "Reset failed\n");
1699                 free_all_rwi(adapter);
1700                 mutex_unlock(&adapter->reset_lock);
1701                 return;
1702         }
1703
1704         adapter->resetting = false;
1705         mutex_unlock(&adapter->reset_lock);
1706 }
1707
1708 static void ibmvnic_reset(struct ibmvnic_adapter *adapter,
1709                           enum ibmvnic_reset_reason reason)
1710 {
1711         struct ibmvnic_rwi *rwi, *tmp;
1712         struct net_device *netdev = adapter->netdev;
1713         struct list_head *entry;
1714
1715         if (adapter->state == VNIC_REMOVING ||
1716             adapter->state == VNIC_REMOVED) {
1717                 netdev_dbg(netdev, "Adapter removing, skipping reset\n");
1718                 return;
1719         }
1720
1721         if (adapter->state == VNIC_PROBING) {
1722                 netdev_warn(netdev, "Adapter reset during probe\n");
1723                 adapter->init_done_rc = EAGAIN;
1724                 return;
1725         }
1726
1727         mutex_lock(&adapter->rwi_lock);
1728
1729         list_for_each(entry, &adapter->rwi_list) {
1730                 tmp = list_entry(entry, struct ibmvnic_rwi, list);
1731                 if (tmp->reset_reason == reason) {
1732                         netdev_dbg(netdev, "Skipping matching reset\n");
1733                         mutex_unlock(&adapter->rwi_lock);
1734                         return;
1735                 }
1736         }
1737
1738         rwi = kzalloc(sizeof(*rwi), GFP_KERNEL);
1739         if (!rwi) {
1740                 mutex_unlock(&adapter->rwi_lock);
1741                 ibmvnic_close(netdev);
1742                 return;
1743         }
1744
1745         rwi->reset_reason = reason;
1746         list_add_tail(&rwi->list, &adapter->rwi_list);
1747         mutex_unlock(&adapter->rwi_lock);
1748
1749         netdev_dbg(adapter->netdev, "Scheduling reset (reason %d)\n", reason);
1750         schedule_work(&adapter->ibmvnic_reset);
1751 }
1752
1753 static void ibmvnic_tx_timeout(struct net_device *dev)
1754 {
1755         struct ibmvnic_adapter *adapter = netdev_priv(dev);
1756
1757         ibmvnic_reset(adapter, VNIC_RESET_TIMEOUT);
1758 }
1759
1760 static void remove_buff_from_pool(struct ibmvnic_adapter *adapter,
1761                                   struct ibmvnic_rx_buff *rx_buff)
1762 {
1763         struct ibmvnic_rx_pool *pool = &adapter->rx_pool[rx_buff->pool_index];
1764
1765         rx_buff->skb = NULL;
1766
1767         pool->free_map[pool->next_alloc] = (int)(rx_buff - pool->rx_buff);
1768         pool->next_alloc = (pool->next_alloc + 1) % pool->size;
1769
1770         atomic_dec(&pool->available);
1771 }
1772
1773 static int ibmvnic_poll(struct napi_struct *napi, int budget)
1774 {
1775         struct net_device *netdev = napi->dev;
1776         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1777         int scrq_num = (int)(napi - adapter->napi);
1778         int frames_processed = 0;
1779
1780 restart_poll:
1781         while (frames_processed < budget) {
1782                 struct sk_buff *skb;
1783                 struct ibmvnic_rx_buff *rx_buff;
1784                 union sub_crq *next;
1785                 u32 length;
1786                 u16 offset;
1787                 u8 flags = 0;
1788
1789                 if (unlikely(adapter->resetting)) {
1790                         enable_scrq_irq(adapter, adapter->rx_scrq[scrq_num]);
1791                         napi_complete_done(napi, frames_processed);
1792                         return frames_processed;
1793                 }
1794
1795                 if (!pending_scrq(adapter, adapter->rx_scrq[scrq_num]))
1796                         break;
1797                 next = ibmvnic_next_scrq(adapter, adapter->rx_scrq[scrq_num]);
1798                 rx_buff =
1799                     (struct ibmvnic_rx_buff *)be64_to_cpu(next->
1800                                                           rx_comp.correlator);
1801                 /* do error checking */
1802                 if (next->rx_comp.rc) {
1803                         netdev_dbg(netdev, "rx buffer returned with rc %x\n",
1804                                    be16_to_cpu(next->rx_comp.rc));
1805                         /* free the entry */
1806                         next->rx_comp.first = 0;
1807                         remove_buff_from_pool(adapter, rx_buff);
1808                         continue;
1809                 }
1810
1811                 length = be32_to_cpu(next->rx_comp.len);
1812                 offset = be16_to_cpu(next->rx_comp.off_frame_data);
1813                 flags = next->rx_comp.flags;
1814                 skb = rx_buff->skb;
1815                 skb_copy_to_linear_data(skb, rx_buff->data + offset,
1816                                         length);
1817
1818                 /* VLAN Header has been stripped by the system firmware and
1819                  * needs to be inserted by the driver
1820                  */
1821                 if (adapter->rx_vlan_header_insertion &&
1822                     (flags & IBMVNIC_VLAN_STRIPPED))
1823                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1824                                                ntohs(next->rx_comp.vlan_tci));
1825
1826                 /* free the entry */
1827                 next->rx_comp.first = 0;
1828                 remove_buff_from_pool(adapter, rx_buff);
1829
1830                 skb_put(skb, length);
1831                 skb->protocol = eth_type_trans(skb, netdev);
1832                 skb_record_rx_queue(skb, scrq_num);
1833
1834                 if (flags & IBMVNIC_IP_CHKSUM_GOOD &&
1835                     flags & IBMVNIC_TCP_UDP_CHKSUM_GOOD) {
1836                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1837                 }
1838
1839                 length = skb->len;
1840                 napi_gro_receive(napi, skb); /* send it up */
1841                 netdev->stats.rx_packets++;
1842                 netdev->stats.rx_bytes += length;
1843                 adapter->rx_stats_buffers[scrq_num].packets++;
1844                 adapter->rx_stats_buffers[scrq_num].bytes += length;
1845                 frames_processed++;
1846         }
1847
1848         if (adapter->state != VNIC_CLOSING)
1849                 replenish_rx_pool(adapter, &adapter->rx_pool[scrq_num]);
1850
1851         if (frames_processed < budget) {
1852                 enable_scrq_irq(adapter, adapter->rx_scrq[scrq_num]);
1853                 napi_complete_done(napi, frames_processed);
1854                 if (pending_scrq(adapter, adapter->rx_scrq[scrq_num]) &&
1855                     napi_reschedule(napi)) {
1856                         disable_scrq_irq(adapter, adapter->rx_scrq[scrq_num]);
1857                         goto restart_poll;
1858                 }
1859         }
1860         return frames_processed;
1861 }
1862
1863 #ifdef CONFIG_NET_POLL_CONTROLLER
1864 static void ibmvnic_netpoll_controller(struct net_device *dev)
1865 {
1866         struct ibmvnic_adapter *adapter = netdev_priv(dev);
1867         int i;
1868
1869         replenish_pools(netdev_priv(dev));
1870         for (i = 0; i < adapter->req_rx_queues; i++)
1871                 ibmvnic_interrupt_rx(adapter->rx_scrq[i]->irq,
1872                                      adapter->rx_scrq[i]);
1873 }
1874 #endif
1875
1876 static int wait_for_reset(struct ibmvnic_adapter *adapter)
1877 {
1878         adapter->fallback.mtu = adapter->req_mtu;
1879         adapter->fallback.rx_queues = adapter->req_rx_queues;
1880         adapter->fallback.tx_queues = adapter->req_tx_queues;
1881         adapter->fallback.rx_entries = adapter->req_rx_add_entries_per_subcrq;
1882         adapter->fallback.tx_entries = adapter->req_tx_entries_per_subcrq;
1883
1884         init_completion(&adapter->reset_done);
1885         ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM);
1886         adapter->wait_for_reset = true;
1887         wait_for_completion(&adapter->reset_done);
1888
1889         if (adapter->reset_done_rc) {
1890                 adapter->desired.mtu = adapter->fallback.mtu;
1891                 adapter->desired.rx_queues = adapter->fallback.rx_queues;
1892                 adapter->desired.tx_queues = adapter->fallback.tx_queues;
1893                 adapter->desired.rx_entries = adapter->fallback.rx_entries;
1894                 adapter->desired.tx_entries = adapter->fallback.tx_entries;
1895
1896                 init_completion(&adapter->reset_done);
1897                 ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM);
1898                 wait_for_completion(&adapter->reset_done);
1899         }
1900         adapter->wait_for_reset = false;
1901
1902         return adapter->reset_done_rc;
1903 }
1904
1905 static int ibmvnic_change_mtu(struct net_device *netdev, int new_mtu)
1906 {
1907         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1908
1909         adapter->desired.mtu = new_mtu + ETH_HLEN;
1910
1911         return wait_for_reset(adapter);
1912 }
1913
1914 static const struct net_device_ops ibmvnic_netdev_ops = {
1915         .ndo_open               = ibmvnic_open,
1916         .ndo_stop               = ibmvnic_close,
1917         .ndo_start_xmit         = ibmvnic_xmit,
1918         .ndo_set_rx_mode        = ibmvnic_set_multi,
1919         .ndo_set_mac_address    = ibmvnic_set_mac,
1920         .ndo_validate_addr      = eth_validate_addr,
1921         .ndo_tx_timeout         = ibmvnic_tx_timeout,
1922 #ifdef CONFIG_NET_POLL_CONTROLLER
1923         .ndo_poll_controller    = ibmvnic_netpoll_controller,
1924 #endif
1925         .ndo_change_mtu         = ibmvnic_change_mtu,
1926 };
1927
1928 /* ethtool functions */
1929
1930 static int ibmvnic_get_link_ksettings(struct net_device *netdev,
1931                                       struct ethtool_link_ksettings *cmd)
1932 {
1933         u32 supported, advertising;
1934
1935         supported = (SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg |
1936                           SUPPORTED_FIBRE);
1937         advertising = (ADVERTISED_1000baseT_Full | ADVERTISED_Autoneg |
1938                             ADVERTISED_FIBRE);
1939         cmd->base.speed = SPEED_1000;
1940         cmd->base.duplex = DUPLEX_FULL;
1941         cmd->base.port = PORT_FIBRE;
1942         cmd->base.phy_address = 0;
1943         cmd->base.autoneg = AUTONEG_ENABLE;
1944
1945         ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
1946                                                 supported);
1947         ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
1948                                                 advertising);
1949
1950         return 0;
1951 }
1952
1953 static void ibmvnic_get_drvinfo(struct net_device *netdev,
1954                                 struct ethtool_drvinfo *info)
1955 {
1956         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1957
1958         strlcpy(info->driver, ibmvnic_driver_name, sizeof(info->driver));
1959         strlcpy(info->version, IBMVNIC_DRIVER_VERSION, sizeof(info->version));
1960         strlcpy(info->fw_version, adapter->fw_version,
1961                 sizeof(info->fw_version));
1962 }
1963
1964 static u32 ibmvnic_get_msglevel(struct net_device *netdev)
1965 {
1966         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1967
1968         return adapter->msg_enable;
1969 }
1970
1971 static void ibmvnic_set_msglevel(struct net_device *netdev, u32 data)
1972 {
1973         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1974
1975         adapter->msg_enable = data;
1976 }
1977
1978 static u32 ibmvnic_get_link(struct net_device *netdev)
1979 {
1980         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1981
1982         /* Don't need to send a query because we request a logical link up at
1983          * init and then we wait for link state indications
1984          */
1985         return adapter->logical_link_state;
1986 }
1987
1988 static void ibmvnic_get_ringparam(struct net_device *netdev,
1989                                   struct ethtool_ringparam *ring)
1990 {
1991         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
1992
1993         ring->rx_max_pending = adapter->max_rx_add_entries_per_subcrq;
1994         ring->tx_max_pending = adapter->max_tx_entries_per_subcrq;
1995         ring->rx_mini_max_pending = 0;
1996         ring->rx_jumbo_max_pending = 0;
1997         ring->rx_pending = adapter->req_rx_add_entries_per_subcrq;
1998         ring->tx_pending = adapter->req_tx_entries_per_subcrq;
1999         ring->rx_mini_pending = 0;
2000         ring->rx_jumbo_pending = 0;
2001 }
2002
2003 static int ibmvnic_set_ringparam(struct net_device *netdev,
2004                                  struct ethtool_ringparam *ring)
2005 {
2006         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2007
2008         if (ring->rx_pending > adapter->max_rx_add_entries_per_subcrq  ||
2009             ring->tx_pending > adapter->max_tx_entries_per_subcrq) {
2010                 netdev_err(netdev, "Invalid request.\n");
2011                 netdev_err(netdev, "Max tx buffers = %llu\n",
2012                            adapter->max_rx_add_entries_per_subcrq);
2013                 netdev_err(netdev, "Max rx buffers = %llu\n",
2014                            adapter->max_tx_entries_per_subcrq);
2015                 return -EINVAL;
2016         }
2017
2018         adapter->desired.rx_entries = ring->rx_pending;
2019         adapter->desired.tx_entries = ring->tx_pending;
2020
2021         return wait_for_reset(adapter);
2022 }
2023
2024 static void ibmvnic_get_channels(struct net_device *netdev,
2025                                  struct ethtool_channels *channels)
2026 {
2027         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2028
2029         channels->max_rx = adapter->max_rx_queues;
2030         channels->max_tx = adapter->max_tx_queues;
2031         channels->max_other = 0;
2032         channels->max_combined = 0;
2033         channels->rx_count = adapter->req_rx_queues;
2034         channels->tx_count = adapter->req_tx_queues;
2035         channels->other_count = 0;
2036         channels->combined_count = 0;
2037 }
2038
2039 static int ibmvnic_set_channels(struct net_device *netdev,
2040                                 struct ethtool_channels *channels)
2041 {
2042         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
2043
2044         adapter->desired.rx_queues = channels->rx_count;
2045         adapter->desired.tx_queues = channels->tx_count;
2046
2047         return wait_for_reset(adapter);
2048 }
2049
2050 static void ibmvnic_get_strings(struct net_device *dev, u32 stringset, u8 *data)
2051 {
2052         struct ibmvnic_adapter *adapter = netdev_priv(dev);
2053         int i;
2054
2055         if (stringset != ETH_SS_STATS)
2056                 return;
2057
2058         for (i = 0; i < ARRAY_SIZE(ibmvnic_stats); i++, data += ETH_GSTRING_LEN)
2059                 memcpy(data, ibmvnic_stats[i].name, ETH_GSTRING_LEN);
2060
2061         for (i = 0; i < adapter->req_tx_queues; i++) {
2062                 snprintf(data, ETH_GSTRING_LEN, "tx%d_packets", i);
2063                 data += ETH_GSTRING_LEN;
2064
2065                 snprintf(data, ETH_GSTRING_LEN, "tx%d_bytes", i);
2066                 data += ETH_GSTRING_LEN;
2067
2068                 snprintf(data, ETH_GSTRING_LEN, "tx%d_dropped_packets", i);
2069                 data += ETH_GSTRING_LEN;
2070         }
2071
2072         for (i = 0; i < adapter->req_rx_queues; i++) {
2073                 snprintf(data, ETH_GSTRING_LEN, "rx%d_packets", i);
2074                 data += ETH_GSTRING_LEN;
2075
2076                 snprintf(data, ETH_GSTRING_LEN, "rx%d_bytes", i);
2077                 data += ETH_GSTRING_LEN;
2078
2079                 snprintf(data, ETH_GSTRING_LEN, "rx%d_interrupts", i);
2080                 data += ETH_GSTRING_LEN;
2081         }
2082 }
2083
2084 static int ibmvnic_get_sset_count(struct net_device *dev, int sset)
2085 {
2086         struct ibmvnic_adapter *adapter = netdev_priv(dev);
2087
2088         switch (sset) {
2089         case ETH_SS_STATS:
2090                 return ARRAY_SIZE(ibmvnic_stats) +
2091                        adapter->req_tx_queues * NUM_TX_STATS +
2092                        adapter->req_rx_queues * NUM_RX_STATS;
2093         default:
2094                 return -EOPNOTSUPP;
2095         }
2096 }
2097
2098 static void ibmvnic_get_ethtool_stats(struct net_device *dev,
2099                                       struct ethtool_stats *stats, u64 *data)
2100 {
2101         struct ibmvnic_adapter *adapter = netdev_priv(dev);
2102         union ibmvnic_crq crq;
2103         int i, j;
2104
2105         memset(&crq, 0, sizeof(crq));
2106         crq.request_statistics.first = IBMVNIC_CRQ_CMD;
2107         crq.request_statistics.cmd = REQUEST_STATISTICS;
2108         crq.request_statistics.ioba = cpu_to_be32(adapter->stats_token);
2109         crq.request_statistics.len =
2110             cpu_to_be32(sizeof(struct ibmvnic_statistics));
2111
2112         /* Wait for data to be written */
2113         init_completion(&adapter->stats_done);
2114         ibmvnic_send_crq(adapter, &crq);
2115         wait_for_completion(&adapter->stats_done);
2116
2117         for (i = 0; i < ARRAY_SIZE(ibmvnic_stats); i++)
2118                 data[i] = be64_to_cpu(IBMVNIC_GET_STAT(adapter,
2119                                                 ibmvnic_stats[i].offset));
2120
2121         for (j = 0; j < adapter->req_tx_queues; j++) {
2122                 data[i] = adapter->tx_stats_buffers[j].packets;
2123                 i++;
2124                 data[i] = adapter->tx_stats_buffers[j].bytes;
2125                 i++;
2126                 data[i] = adapter->tx_stats_buffers[j].dropped_packets;
2127                 i++;
2128         }
2129
2130         for (j = 0; j < adapter->req_rx_queues; j++) {
2131                 data[i] = adapter->rx_stats_buffers[j].packets;
2132                 i++;
2133                 data[i] = adapter->rx_stats_buffers[j].bytes;
2134                 i++;
2135                 data[i] = adapter->rx_stats_buffers[j].interrupts;
2136                 i++;
2137         }
2138 }
2139
2140 static const struct ethtool_ops ibmvnic_ethtool_ops = {
2141         .get_drvinfo            = ibmvnic_get_drvinfo,
2142         .get_msglevel           = ibmvnic_get_msglevel,
2143         .set_msglevel           = ibmvnic_set_msglevel,
2144         .get_link               = ibmvnic_get_link,
2145         .get_ringparam          = ibmvnic_get_ringparam,
2146         .set_ringparam          = ibmvnic_set_ringparam,
2147         .get_channels           = ibmvnic_get_channels,
2148         .set_channels           = ibmvnic_set_channels,
2149         .get_strings            = ibmvnic_get_strings,
2150         .get_sset_count         = ibmvnic_get_sset_count,
2151         .get_ethtool_stats      = ibmvnic_get_ethtool_stats,
2152         .get_link_ksettings     = ibmvnic_get_link_ksettings,
2153 };
2154
2155 /* Routines for managing CRQs/sCRQs  */
2156
2157 static int reset_one_sub_crq_queue(struct ibmvnic_adapter *adapter,
2158                                    struct ibmvnic_sub_crq_queue *scrq)
2159 {
2160         int rc;
2161
2162         if (scrq->irq) {
2163                 free_irq(scrq->irq, scrq);
2164                 irq_dispose_mapping(scrq->irq);
2165                 scrq->irq = 0;
2166         }
2167
2168         memset(scrq->msgs, 0, 4 * PAGE_SIZE);
2169         scrq->cur = 0;
2170
2171         rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,
2172                            4 * PAGE_SIZE, &scrq->crq_num, &scrq->hw_irq);
2173         return rc;
2174 }
2175
2176 static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter)
2177 {
2178         int i, rc;
2179
2180         for (i = 0; i < adapter->req_tx_queues; i++) {
2181                 netdev_dbg(adapter->netdev, "Re-setting tx_scrq[%d]\n", i);
2182                 rc = reset_one_sub_crq_queue(adapter, adapter->tx_scrq[i]);
2183                 if (rc)
2184                         return rc;
2185         }
2186
2187         for (i = 0; i < adapter->req_rx_queues; i++) {
2188                 netdev_dbg(adapter->netdev, "Re-setting rx_scrq[%d]\n", i);
2189                 rc = reset_one_sub_crq_queue(adapter, adapter->rx_scrq[i]);
2190                 if (rc)
2191                         return rc;
2192         }
2193
2194         return rc;
2195 }
2196
2197 static void release_sub_crq_queue(struct ibmvnic_adapter *adapter,
2198                                   struct ibmvnic_sub_crq_queue *scrq)
2199 {
2200         struct device *dev = &adapter->vdev->dev;
2201         long rc;
2202
2203         netdev_dbg(adapter->netdev, "Releasing sub-CRQ\n");
2204
2205         /* Close the sub-crqs */
2206         do {
2207                 rc = plpar_hcall_norets(H_FREE_SUB_CRQ,
2208                                         adapter->vdev->unit_address,
2209                                         scrq->crq_num);
2210         } while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
2211
2212         if (rc) {
2213                 netdev_err(adapter->netdev,
2214                            "Failed to release sub-CRQ %16lx, rc = %ld\n",
2215                            scrq->crq_num, rc);
2216         }
2217
2218         dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE,
2219                          DMA_BIDIRECTIONAL);
2220         free_pages((unsigned long)scrq->msgs, 2);
2221         kfree(scrq);
2222 }
2223
2224 static struct ibmvnic_sub_crq_queue *init_sub_crq_queue(struct ibmvnic_adapter
2225                                                         *adapter)
2226 {
2227         struct device *dev = &adapter->vdev->dev;
2228         struct ibmvnic_sub_crq_queue *scrq;
2229         int rc;
2230
2231         scrq = kzalloc(sizeof(*scrq), GFP_KERNEL);
2232         if (!scrq)
2233                 return NULL;
2234
2235         scrq->msgs =
2236                 (union sub_crq *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 2);
2237         if (!scrq->msgs) {
2238                 dev_warn(dev, "Couldn't allocate crq queue messages page\n");
2239                 goto zero_page_failed;
2240         }
2241
2242         scrq->msg_token = dma_map_single(dev, scrq->msgs, 4 * PAGE_SIZE,
2243                                          DMA_BIDIRECTIONAL);
2244         if (dma_mapping_error(dev, scrq->msg_token)) {
2245                 dev_warn(dev, "Couldn't map crq queue messages page\n");
2246                 goto map_failed;
2247         }
2248
2249         rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,
2250                            4 * PAGE_SIZE, &scrq->crq_num, &scrq->hw_irq);
2251
2252         if (rc == H_RESOURCE)
2253                 rc = ibmvnic_reset_crq(adapter);
2254
2255         if (rc == H_CLOSED) {
2256                 dev_warn(dev, "Partner adapter not ready, waiting.\n");
2257         } else if (rc) {
2258                 dev_warn(dev, "Error %d registering sub-crq\n", rc);
2259                 goto reg_failed;
2260         }
2261
2262         scrq->adapter = adapter;
2263         scrq->size = 4 * PAGE_SIZE / sizeof(*scrq->msgs);
2264         spin_lock_init(&scrq->lock);
2265
2266         netdev_dbg(adapter->netdev,
2267                    "sub-crq initialized, num %lx, hw_irq=%lx, irq=%x\n",
2268                    scrq->crq_num, scrq->hw_irq, scrq->irq);
2269
2270         return scrq;
2271
2272 reg_failed:
2273         dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE,
2274                          DMA_BIDIRECTIONAL);
2275 map_failed:
2276         free_pages((unsigned long)scrq->msgs, 2);
2277 zero_page_failed:
2278         kfree(scrq);
2279
2280         return NULL;
2281 }
2282
2283 static void release_sub_crqs(struct ibmvnic_adapter *adapter)
2284 {
2285         int i;
2286
2287         if (adapter->tx_scrq) {
2288                 for (i = 0; i < adapter->req_tx_queues; i++) {
2289                         if (!adapter->tx_scrq[i])
2290                                 continue;
2291
2292                         netdev_dbg(adapter->netdev, "Releasing tx_scrq[%d]\n",
2293                                    i);
2294                         if (adapter->tx_scrq[i]->irq) {
2295                                 free_irq(adapter->tx_scrq[i]->irq,
2296                                          adapter->tx_scrq[i]);
2297                                 irq_dispose_mapping(adapter->tx_scrq[i]->irq);
2298                                 adapter->tx_scrq[i]->irq = 0;
2299                         }
2300
2301                         release_sub_crq_queue(adapter, adapter->tx_scrq[i]);
2302                 }
2303
2304                 kfree(adapter->tx_scrq);
2305                 adapter->tx_scrq = NULL;
2306         }
2307
2308         if (adapter->rx_scrq) {
2309                 for (i = 0; i < adapter->req_rx_queues; i++) {
2310                         if (!adapter->rx_scrq[i])
2311                                 continue;
2312
2313                         netdev_dbg(adapter->netdev, "Releasing rx_scrq[%d]\n",
2314                                    i);
2315                         if (adapter->rx_scrq[i]->irq) {
2316                                 free_irq(adapter->rx_scrq[i]->irq,
2317                                          adapter->rx_scrq[i]);
2318                                 irq_dispose_mapping(adapter->rx_scrq[i]->irq);
2319                                 adapter->rx_scrq[i]->irq = 0;
2320                         }
2321
2322                         release_sub_crq_queue(adapter, adapter->rx_scrq[i]);
2323                 }
2324
2325                 kfree(adapter->rx_scrq);
2326                 adapter->rx_scrq = NULL;
2327         }
2328 }
2329
2330 static int disable_scrq_irq(struct ibmvnic_adapter *adapter,
2331                             struct ibmvnic_sub_crq_queue *scrq)
2332 {
2333         struct device *dev = &adapter->vdev->dev;
2334         unsigned long rc;
2335
2336         rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,
2337                                 H_DISABLE_VIO_INTERRUPT, scrq->hw_irq, 0, 0);
2338         if (rc)
2339                 dev_err(dev, "Couldn't disable scrq irq 0x%lx. rc=%ld\n",
2340                         scrq->hw_irq, rc);
2341         return rc;
2342 }
2343
2344 static int enable_scrq_irq(struct ibmvnic_adapter *adapter,
2345                            struct ibmvnic_sub_crq_queue *scrq)
2346 {
2347         struct device *dev = &adapter->vdev->dev;
2348         unsigned long rc;
2349
2350         if (scrq->hw_irq > 0x100000000ULL) {
2351                 dev_err(dev, "bad hw_irq = %lx\n", scrq->hw_irq);
2352                 return 1;
2353         }
2354
2355         rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,
2356                                 H_ENABLE_VIO_INTERRUPT, scrq->hw_irq, 0, 0);
2357         if (rc)
2358                 dev_err(dev, "Couldn't enable scrq irq 0x%lx. rc=%ld\n",
2359                         scrq->hw_irq, rc);
2360         return rc;
2361 }
2362
2363 static int ibmvnic_complete_tx(struct ibmvnic_adapter *adapter,
2364                                struct ibmvnic_sub_crq_queue *scrq)
2365 {
2366         struct device *dev = &adapter->vdev->dev;
2367         struct ibmvnic_tx_buff *txbuff;
2368         union sub_crq *next;
2369         int index;
2370         int i, j;
2371         u8 first;
2372
2373 restart_loop:
2374         while (pending_scrq(adapter, scrq)) {
2375                 unsigned int pool = scrq->pool_index;
2376
2377                 next = ibmvnic_next_scrq(adapter, scrq);
2378                 for (i = 0; i < next->tx_comp.num_comps; i++) {
2379                         if (next->tx_comp.rcs[i]) {
2380                                 dev_err(dev, "tx error %x\n",
2381                                         next->tx_comp.rcs[i]);
2382                                 continue;
2383                         }
2384                         index = be32_to_cpu(next->tx_comp.correlators[i]);
2385                         txbuff = &adapter->tx_pool[pool].tx_buff[index];
2386
2387                         for (j = 0; j < IBMVNIC_MAX_FRAGS_PER_CRQ; j++) {
2388                                 if (!txbuff->data_dma[j])
2389                                         continue;
2390
2391                                 txbuff->data_dma[j] = 0;
2392                         }
2393                         /* if sub_crq was sent indirectly */
2394                         first = txbuff->indir_arr[0].generic.first;
2395                         if (first == IBMVNIC_CRQ_CMD) {
2396                                 dma_unmap_single(dev, txbuff->indir_dma,
2397                                                  sizeof(txbuff->indir_arr),
2398                                                  DMA_TO_DEVICE);
2399                         }
2400
2401                         if (txbuff->last_frag) {
2402                                 dev_kfree_skb_any(txbuff->skb);
2403                                 txbuff->skb = NULL;
2404                         }
2405
2406                         adapter->tx_pool[pool].free_map[adapter->tx_pool[pool].
2407                                                      producer_index] = index;
2408                         adapter->tx_pool[pool].producer_index =
2409                             (adapter->tx_pool[pool].producer_index + 1) %
2410                             adapter->req_tx_entries_per_subcrq;
2411                 }
2412                 /* remove tx_comp scrq*/
2413                 next->tx_comp.first = 0;
2414
2415                 if (atomic_sub_return(next->tx_comp.num_comps, &scrq->used) <=
2416                     (adapter->req_tx_entries_per_subcrq / 2) &&
2417                     __netif_subqueue_stopped(adapter->netdev,
2418                                              scrq->pool_index)) {
2419                         netif_wake_subqueue(adapter->netdev, scrq->pool_index);
2420                         netdev_info(adapter->netdev, "Started queue %d\n",
2421                                     scrq->pool_index);
2422                 }
2423         }
2424
2425         enable_scrq_irq(adapter, scrq);
2426
2427         if (pending_scrq(adapter, scrq)) {
2428                 disable_scrq_irq(adapter, scrq);
2429                 goto restart_loop;
2430         }
2431
2432         return 0;
2433 }
2434
2435 static irqreturn_t ibmvnic_interrupt_tx(int irq, void *instance)
2436 {
2437         struct ibmvnic_sub_crq_queue *scrq = instance;
2438         struct ibmvnic_adapter *adapter = scrq->adapter;
2439
2440         disable_scrq_irq(adapter, scrq);
2441         ibmvnic_complete_tx(adapter, scrq);
2442
2443         return IRQ_HANDLED;
2444 }
2445
2446 static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance)
2447 {
2448         struct ibmvnic_sub_crq_queue *scrq = instance;
2449         struct ibmvnic_adapter *adapter = scrq->adapter;
2450
2451         adapter->rx_stats_buffers[scrq->scrq_num].interrupts++;
2452
2453         if (napi_schedule_prep(&adapter->napi[scrq->scrq_num])) {
2454                 disable_scrq_irq(adapter, scrq);
2455                 __napi_schedule(&adapter->napi[scrq->scrq_num]);
2456         }
2457
2458         return IRQ_HANDLED;
2459 }
2460
2461 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter)
2462 {
2463         struct device *dev = &adapter->vdev->dev;
2464         struct ibmvnic_sub_crq_queue *scrq;
2465         int i = 0, j = 0;
2466         int rc = 0;
2467
2468         for (i = 0; i < adapter->req_tx_queues; i++) {
2469                 netdev_dbg(adapter->netdev, "Initializing tx_scrq[%d] irq\n",
2470                            i);
2471                 scrq = adapter->tx_scrq[i];
2472                 scrq->irq = irq_create_mapping(NULL, scrq->hw_irq);
2473
2474                 if (!scrq->irq) {
2475                         rc = -EINVAL;
2476                         dev_err(dev, "Error mapping irq\n");
2477                         goto req_tx_irq_failed;
2478                 }
2479
2480                 rc = request_irq(scrq->irq, ibmvnic_interrupt_tx,
2481                                  0, "ibmvnic_tx", scrq);
2482
2483                 if (rc) {
2484                         dev_err(dev, "Couldn't register tx irq 0x%x. rc=%d\n",
2485                                 scrq->irq, rc);
2486                         irq_dispose_mapping(scrq->irq);
2487                         goto req_rx_irq_failed;
2488                 }
2489         }
2490
2491         for (i = 0; i < adapter->req_rx_queues; i++) {
2492                 netdev_dbg(adapter->netdev, "Initializing rx_scrq[%d] irq\n",
2493                            i);
2494                 scrq = adapter->rx_scrq[i];
2495                 scrq->irq = irq_create_mapping(NULL, scrq->hw_irq);
2496                 if (!scrq->irq) {
2497                         rc = -EINVAL;
2498                         dev_err(dev, "Error mapping irq\n");
2499                         goto req_rx_irq_failed;
2500                 }
2501                 rc = request_irq(scrq->irq, ibmvnic_interrupt_rx,
2502                                  0, "ibmvnic_rx", scrq);
2503                 if (rc) {
2504                         dev_err(dev, "Couldn't register rx irq 0x%x. rc=%d\n",
2505                                 scrq->irq, rc);
2506                         irq_dispose_mapping(scrq->irq);
2507                         goto req_rx_irq_failed;
2508                 }
2509         }
2510         return rc;
2511
2512 req_rx_irq_failed:
2513         for (j = 0; j < i; j++) {
2514                 free_irq(adapter->rx_scrq[j]->irq, adapter->rx_scrq[j]);
2515                 irq_dispose_mapping(adapter->rx_scrq[j]->irq);
2516         }
2517         i = adapter->req_tx_queues;
2518 req_tx_irq_failed:
2519         for (j = 0; j < i; j++) {
2520                 free_irq(adapter->tx_scrq[j]->irq, adapter->tx_scrq[j]);
2521                 irq_dispose_mapping(adapter->rx_scrq[j]->irq);
2522         }
2523         release_sub_crqs(adapter);
2524         return rc;
2525 }
2526
2527 static int init_sub_crqs(struct ibmvnic_adapter *adapter)
2528 {
2529         struct device *dev = &adapter->vdev->dev;
2530         struct ibmvnic_sub_crq_queue **allqueues;
2531         int registered_queues = 0;
2532         int total_queues;
2533         int more = 0;
2534         int i;
2535
2536         total_queues = adapter->req_tx_queues + adapter->req_rx_queues;
2537
2538         allqueues = kcalloc(total_queues, sizeof(*allqueues), GFP_KERNEL);
2539         if (!allqueues)
2540                 return -1;
2541
2542         for (i = 0; i < total_queues; i++) {
2543                 allqueues[i] = init_sub_crq_queue(adapter);
2544                 if (!allqueues[i]) {
2545                         dev_warn(dev, "Couldn't allocate all sub-crqs\n");
2546                         break;
2547                 }
2548                 registered_queues++;
2549         }
2550
2551         /* Make sure we were able to register the minimum number of queues */
2552         if (registered_queues <
2553             adapter->min_tx_queues + adapter->min_rx_queues) {
2554                 dev_err(dev, "Fatal: Couldn't init  min number of sub-crqs\n");
2555                 goto tx_failed;
2556         }
2557
2558         /* Distribute the failed allocated queues*/
2559         for (i = 0; i < total_queues - registered_queues + more ; i++) {
2560                 netdev_dbg(adapter->netdev, "Reducing number of queues\n");
2561                 switch (i % 3) {
2562                 case 0:
2563                         if (adapter->req_rx_queues > adapter->min_rx_queues)
2564                                 adapter->req_rx_queues--;
2565                         else
2566                                 more++;
2567                         break;
2568                 case 1:
2569                         if (adapter->req_tx_queues > adapter->min_tx_queues)
2570                                 adapter->req_tx_queues--;
2571                         else
2572                                 more++;
2573                         break;
2574                 }
2575         }
2576
2577         adapter->tx_scrq = kcalloc(adapter->req_tx_queues,
2578                                    sizeof(*adapter->tx_scrq), GFP_KERNEL);
2579         if (!adapter->tx_scrq)
2580                 goto tx_failed;
2581
2582         for (i = 0; i < adapter->req_tx_queues; i++) {
2583                 adapter->tx_scrq[i] = allqueues[i];
2584                 adapter->tx_scrq[i]->pool_index = i;
2585         }
2586
2587         adapter->rx_scrq = kcalloc(adapter->req_rx_queues,
2588                                    sizeof(*adapter->rx_scrq), GFP_KERNEL);
2589         if (!adapter->rx_scrq)
2590                 goto rx_failed;
2591
2592         for (i = 0; i < adapter->req_rx_queues; i++) {
2593                 adapter->rx_scrq[i] = allqueues[i + adapter->req_tx_queues];
2594                 adapter->rx_scrq[i]->scrq_num = i;
2595         }
2596
2597         kfree(allqueues);
2598         return 0;
2599
2600 rx_failed:
2601         kfree(adapter->tx_scrq);
2602         adapter->tx_scrq = NULL;
2603 tx_failed:
2604         for (i = 0; i < registered_queues; i++)
2605                 release_sub_crq_queue(adapter, allqueues[i]);
2606         kfree(allqueues);
2607         return -1;
2608 }
2609
2610 static void ibmvnic_send_req_caps(struct ibmvnic_adapter *adapter, int retry)
2611 {
2612         struct device *dev = &adapter->vdev->dev;
2613         union ibmvnic_crq crq;
2614         int max_entries;
2615
2616         if (!retry) {
2617                 /* Sub-CRQ entries are 32 byte long */
2618                 int entries_page = 4 * PAGE_SIZE / (sizeof(u64) * 4);
2619
2620                 if (adapter->min_tx_entries_per_subcrq > entries_page ||
2621                     adapter->min_rx_add_entries_per_subcrq > entries_page) {
2622                         dev_err(dev, "Fatal, invalid entries per sub-crq\n");
2623                         return;
2624                 }
2625
2626                 if (adapter->desired.mtu)
2627                         adapter->req_mtu = adapter->desired.mtu;
2628                 else
2629                         adapter->req_mtu = adapter->netdev->mtu + ETH_HLEN;
2630
2631                 if (!adapter->desired.tx_entries)
2632                         adapter->desired.tx_entries =
2633                                         adapter->max_tx_entries_per_subcrq;
2634                 if (!adapter->desired.rx_entries)
2635                         adapter->desired.rx_entries =
2636                                         adapter->max_rx_add_entries_per_subcrq;
2637
2638                 max_entries = IBMVNIC_MAX_LTB_SIZE /
2639                               (adapter->req_mtu + IBMVNIC_BUFFER_HLEN);
2640
2641                 if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) *
2642                         adapter->desired.tx_entries > IBMVNIC_MAX_LTB_SIZE) {
2643                         adapter->desired.tx_entries = max_entries;
2644                 }
2645
2646                 if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) *
2647                         adapter->desired.rx_entries > IBMVNIC_MAX_LTB_SIZE) {
2648                         adapter->desired.rx_entries = max_entries;
2649                 }
2650
2651                 if (adapter->desired.tx_entries)
2652                         adapter->req_tx_entries_per_subcrq =
2653                                         adapter->desired.tx_entries;
2654                 else
2655                         adapter->req_tx_entries_per_subcrq =
2656                                         adapter->max_tx_entries_per_subcrq;
2657
2658                 if (adapter->desired.rx_entries)
2659                         adapter->req_rx_add_entries_per_subcrq =
2660                                         adapter->desired.rx_entries;
2661                 else
2662                         adapter->req_rx_add_entries_per_subcrq =
2663                                         adapter->max_rx_add_entries_per_subcrq;
2664
2665                 if (adapter->desired.tx_queues)
2666                         adapter->req_tx_queues =
2667                                         adapter->desired.tx_queues;
2668                 else
2669                         adapter->req_tx_queues =
2670                                         adapter->opt_tx_comp_sub_queues;
2671
2672                 if (adapter->desired.rx_queues)
2673                         adapter->req_rx_queues =
2674                                         adapter->desired.rx_queues;
2675                 else
2676                         adapter->req_rx_queues =
2677                                         adapter->opt_rx_comp_queues;
2678
2679                 adapter->req_rx_add_queues = adapter->max_rx_add_queues;
2680         }
2681
2682         memset(&crq, 0, sizeof(crq));
2683         crq.request_capability.first = IBMVNIC_CRQ_CMD;
2684         crq.request_capability.cmd = REQUEST_CAPABILITY;
2685
2686         crq.request_capability.capability = cpu_to_be16(REQ_TX_QUEUES);
2687         crq.request_capability.number = cpu_to_be64(adapter->req_tx_queues);
2688         atomic_inc(&adapter->running_cap_crqs);
2689         ibmvnic_send_crq(adapter, &crq);
2690
2691         crq.request_capability.capability = cpu_to_be16(REQ_RX_QUEUES);
2692         crq.request_capability.number = cpu_to_be64(adapter->req_rx_queues);
2693         atomic_inc(&adapter->running_cap_crqs);
2694         ibmvnic_send_crq(adapter, &crq);
2695
2696         crq.request_capability.capability = cpu_to_be16(REQ_RX_ADD_QUEUES);
2697         crq.request_capability.number = cpu_to_be64(adapter->req_rx_add_queues);
2698         atomic_inc(&adapter->running_cap_crqs);
2699         ibmvnic_send_crq(adapter, &crq);
2700
2701         crq.request_capability.capability =
2702             cpu_to_be16(REQ_TX_ENTRIES_PER_SUBCRQ);
2703         crq.request_capability.number =
2704             cpu_to_be64(adapter->req_tx_entries_per_subcrq);
2705         atomic_inc(&adapter->running_cap_crqs);
2706         ibmvnic_send_crq(adapter, &crq);
2707
2708         crq.request_capability.capability =
2709             cpu_to_be16(REQ_RX_ADD_ENTRIES_PER_SUBCRQ);
2710         crq.request_capability.number =
2711             cpu_to_be64(adapter->req_rx_add_entries_per_subcrq);
2712         atomic_inc(&adapter->running_cap_crqs);
2713         ibmvnic_send_crq(adapter, &crq);
2714
2715         crq.request_capability.capability = cpu_to_be16(REQ_MTU);
2716         crq.request_capability.number = cpu_to_be64(adapter->req_mtu);
2717         atomic_inc(&adapter->running_cap_crqs);
2718         ibmvnic_send_crq(adapter, &crq);
2719
2720         if (adapter->netdev->flags & IFF_PROMISC) {
2721                 if (adapter->promisc_supported) {
2722                         crq.request_capability.capability =
2723                             cpu_to_be16(PROMISC_REQUESTED);
2724                         crq.request_capability.number = cpu_to_be64(1);
2725                         atomic_inc(&adapter->running_cap_crqs);
2726                         ibmvnic_send_crq(adapter, &crq);
2727                 }
2728         } else {
2729                 crq.request_capability.capability =
2730                     cpu_to_be16(PROMISC_REQUESTED);
2731                 crq.request_capability.number = cpu_to_be64(0);
2732                 atomic_inc(&adapter->running_cap_crqs);
2733                 ibmvnic_send_crq(adapter, &crq);
2734         }
2735 }
2736
2737 static int pending_scrq(struct ibmvnic_adapter *adapter,
2738                         struct ibmvnic_sub_crq_queue *scrq)
2739 {
2740         union sub_crq *entry = &scrq->msgs[scrq->cur];
2741
2742         if (entry->generic.first & IBMVNIC_CRQ_CMD_RSP)
2743                 return 1;
2744         else
2745                 return 0;
2746 }
2747
2748 static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *adapter,
2749                                         struct ibmvnic_sub_crq_queue *scrq)
2750 {
2751         union sub_crq *entry;
2752         unsigned long flags;
2753
2754         spin_lock_irqsave(&scrq->lock, flags);
2755         entry = &scrq->msgs[scrq->cur];
2756         if (entry->generic.first & IBMVNIC_CRQ_CMD_RSP) {
2757                 if (++scrq->cur == scrq->size)
2758                         scrq->cur = 0;
2759         } else {
2760                 entry = NULL;
2761         }
2762         spin_unlock_irqrestore(&scrq->lock, flags);
2763
2764         return entry;
2765 }
2766
2767 static union ibmvnic_crq *ibmvnic_next_crq(struct ibmvnic_adapter *adapter)
2768 {
2769         struct ibmvnic_crq_queue *queue = &adapter->crq;
2770         union ibmvnic_crq *crq;
2771
2772         crq = &queue->msgs[queue->cur];
2773         if (crq->generic.first & IBMVNIC_CRQ_CMD_RSP) {
2774                 if (++queue->cur == queue->size)
2775                         queue->cur = 0;
2776         } else {
2777                 crq = NULL;
2778         }
2779
2780         return crq;
2781 }
2782
2783 static int send_subcrq(struct ibmvnic_adapter *adapter, u64 remote_handle,
2784                        union sub_crq *sub_crq)
2785 {
2786         unsigned int ua = adapter->vdev->unit_address;
2787         struct device *dev = &adapter->vdev->dev;
2788         u64 *u64_crq = (u64 *)sub_crq;
2789         int rc;
2790
2791         netdev_dbg(adapter->netdev,
2792                    "Sending sCRQ %016lx: %016lx %016lx %016lx %016lx\n",
2793                    (unsigned long int)cpu_to_be64(remote_handle),
2794                    (unsigned long int)cpu_to_be64(u64_crq[0]),
2795                    (unsigned long int)cpu_to_be64(u64_crq[1]),
2796                    (unsigned long int)cpu_to_be64(u64_crq[2]),
2797                    (unsigned long int)cpu_to_be64(u64_crq[3]));
2798
2799         /* Make sure the hypervisor sees the complete request */
2800         mb();
2801
2802         rc = plpar_hcall_norets(H_SEND_SUB_CRQ, ua,
2803                                 cpu_to_be64(remote_handle),
2804                                 cpu_to_be64(u64_crq[0]),
2805                                 cpu_to_be64(u64_crq[1]),
2806                                 cpu_to_be64(u64_crq[2]),
2807                                 cpu_to_be64(u64_crq[3]));
2808
2809         if (rc) {
2810                 if (rc == H_CLOSED)
2811                         dev_warn(dev, "CRQ Queue closed\n");
2812                 dev_err(dev, "Send error (rc=%d)\n", rc);
2813         }
2814
2815         return rc;
2816 }
2817
2818 static int send_subcrq_indirect(struct ibmvnic_adapter *adapter,
2819                                 u64 remote_handle, u64 ioba, u64 num_entries)
2820 {
2821         unsigned int ua = adapter->vdev->unit_address;
2822         struct device *dev = &adapter->vdev->dev;
2823         int rc;
2824
2825         /* Make sure the hypervisor sees the complete request */
2826         mb();
2827         rc = plpar_hcall_norets(H_SEND_SUB_CRQ_INDIRECT, ua,
2828                                 cpu_to_be64(remote_handle),
2829                                 ioba, num_entries);
2830
2831         if (rc) {
2832                 if (rc == H_CLOSED)
2833                         dev_warn(dev, "CRQ Queue closed\n");
2834                 dev_err(dev, "Send (indirect) error (rc=%d)\n", rc);
2835         }
2836
2837         return rc;
2838 }
2839
2840 static int ibmvnic_send_crq(struct ibmvnic_adapter *adapter,
2841                             union ibmvnic_crq *crq)
2842 {
2843         unsigned int ua = adapter->vdev->unit_address;
2844         struct device *dev = &adapter->vdev->dev;
2845         u64 *u64_crq = (u64 *)crq;
2846         int rc;
2847
2848         netdev_dbg(adapter->netdev, "Sending CRQ: %016lx %016lx\n",
2849                    (unsigned long int)cpu_to_be64(u64_crq[0]),
2850                    (unsigned long int)cpu_to_be64(u64_crq[1]));
2851
2852         /* Make sure the hypervisor sees the complete request */
2853         mb();
2854
2855         rc = plpar_hcall_norets(H_SEND_CRQ, ua,
2856                                 cpu_to_be64(u64_crq[0]),
2857                                 cpu_to_be64(u64_crq[1]));
2858
2859         if (rc) {
2860                 if (rc == H_CLOSED)
2861                         dev_warn(dev, "CRQ Queue closed\n");
2862                 dev_warn(dev, "Send error (rc=%d)\n", rc);
2863         }
2864
2865         return rc;
2866 }
2867
2868 static int ibmvnic_send_crq_init(struct ibmvnic_adapter *adapter)
2869 {
2870         union ibmvnic_crq crq;
2871
2872         memset(&crq, 0, sizeof(crq));
2873         crq.generic.first = IBMVNIC_CRQ_INIT_CMD;
2874         crq.generic.cmd = IBMVNIC_CRQ_INIT;
2875         netdev_dbg(adapter->netdev, "Sending CRQ init\n");
2876
2877         return ibmvnic_send_crq(adapter, &crq);
2878 }
2879
2880 static int send_version_xchg(struct ibmvnic_adapter *adapter)
2881 {
2882         union ibmvnic_crq crq;
2883
2884         memset(&crq, 0, sizeof(crq));
2885         crq.version_exchange.first = IBMVNIC_CRQ_CMD;
2886         crq.version_exchange.cmd = VERSION_EXCHANGE;
2887         crq.version_exchange.version = cpu_to_be16(ibmvnic_version);
2888
2889         return ibmvnic_send_crq(adapter, &crq);
2890 }
2891
2892 struct vnic_login_client_data {
2893         u8      type;
2894         __be16  len;
2895         char    name;
2896 } __packed;
2897
2898 static int vnic_client_data_len(struct ibmvnic_adapter *adapter)
2899 {
2900         int len;
2901
2902         /* Calculate the amount of buffer space needed for the
2903          * vnic client data in the login buffer. There are four entries,
2904          * OS name, LPAR name, device name, and a null last entry.
2905          */
2906         len = 4 * sizeof(struct vnic_login_client_data);
2907         len += 6; /* "Linux" plus NULL */
2908         len += strlen(utsname()->nodename) + 1;
2909         len += strlen(adapter->netdev->name) + 1;
2910
2911         return len;
2912 }
2913
2914 static void vnic_add_client_data(struct ibmvnic_adapter *adapter,
2915                                  struct vnic_login_client_data *vlcd)
2916 {
2917         const char *os_name = "Linux";
2918         int len;
2919
2920         /* Type 1 - LPAR OS */
2921         vlcd->type = 1;
2922         len = strlen(os_name) + 1;
2923         vlcd->len = cpu_to_be16(len);
2924         strncpy(&vlcd->name, os_name, len);
2925         vlcd = (struct vnic_login_client_data *)((char *)&vlcd->name + len);
2926
2927         /* Type 2 - LPAR name */
2928         vlcd->type = 2;
2929         len = strlen(utsname()->nodename) + 1;
2930         vlcd->len = cpu_to_be16(len);
2931         strncpy(&vlcd->name, utsname()->nodename, len);
2932         vlcd = (struct vnic_login_client_data *)((char *)&vlcd->name + len);
2933
2934         /* Type 3 - device name */
2935         vlcd->type = 3;
2936         len = strlen(adapter->netdev->name) + 1;
2937         vlcd->len = cpu_to_be16(len);
2938         strncpy(&vlcd->name, adapter->netdev->name, len);
2939 }
2940
2941 static void send_login(struct ibmvnic_adapter *adapter)
2942 {
2943         struct ibmvnic_login_rsp_buffer *login_rsp_buffer;
2944         struct ibmvnic_login_buffer *login_buffer;
2945         struct device *dev = &adapter->vdev->dev;
2946         dma_addr_t rsp_buffer_token;
2947         dma_addr_t buffer_token;
2948         size_t rsp_buffer_size;
2949         union ibmvnic_crq crq;
2950         size_t buffer_size;
2951         __be64 *tx_list_p;
2952         __be64 *rx_list_p;
2953         int client_data_len;
2954         struct vnic_login_client_data *vlcd;
2955         int i;
2956
2957         client_data_len = vnic_client_data_len(adapter);
2958
2959         buffer_size =
2960             sizeof(struct ibmvnic_login_buffer) +
2961             sizeof(u64) * (adapter->req_tx_queues + adapter->req_rx_queues) +
2962             client_data_len;
2963
2964         login_buffer = kzalloc(buffer_size, GFP_ATOMIC);
2965         if (!login_buffer)
2966                 goto buf_alloc_failed;
2967
2968         buffer_token = dma_map_single(dev, login_buffer, buffer_size,
2969                                       DMA_TO_DEVICE);
2970         if (dma_mapping_error(dev, buffer_token)) {
2971                 dev_err(dev, "Couldn't map login buffer\n");
2972                 goto buf_map_failed;
2973         }
2974
2975         rsp_buffer_size = sizeof(struct ibmvnic_login_rsp_buffer) +
2976                           sizeof(u64) * adapter->req_tx_queues +
2977                           sizeof(u64) * adapter->req_rx_queues +
2978                           sizeof(u64) * adapter->req_rx_queues +
2979                           sizeof(u8) * IBMVNIC_TX_DESC_VERSIONS;
2980
2981         login_rsp_buffer = kmalloc(rsp_buffer_size, GFP_ATOMIC);
2982         if (!login_rsp_buffer)
2983                 goto buf_rsp_alloc_failed;
2984
2985         rsp_buffer_token = dma_map_single(dev, login_rsp_buffer,
2986                                           rsp_buffer_size, DMA_FROM_DEVICE);
2987         if (dma_mapping_error(dev, rsp_buffer_token)) {
2988                 dev_err(dev, "Couldn't map login rsp buffer\n");
2989                 goto buf_rsp_map_failed;
2990         }
2991
2992         adapter->login_buf = login_buffer;
2993         adapter->login_buf_token = buffer_token;
2994         adapter->login_buf_sz = buffer_size;
2995         adapter->login_rsp_buf = login_rsp_buffer;
2996         adapter->login_rsp_buf_token = rsp_buffer_token;
2997         adapter->login_rsp_buf_sz = rsp_buffer_size;
2998
2999         login_buffer->len = cpu_to_be32(buffer_size);
3000         login_buffer->version = cpu_to_be32(INITIAL_VERSION_LB);
3001         login_buffer->num_txcomp_subcrqs = cpu_to_be32(adapter->req_tx_queues);
3002         login_buffer->off_txcomp_subcrqs =
3003             cpu_to_be32(sizeof(struct ibmvnic_login_buffer));
3004         login_buffer->num_rxcomp_subcrqs = cpu_to_be32(adapter->req_rx_queues);
3005         login_buffer->off_rxcomp_subcrqs =
3006             cpu_to_be32(sizeof(struct ibmvnic_login_buffer) +
3007                         sizeof(u64) * adapter->req_tx_queues);
3008         login_buffer->login_rsp_ioba = cpu_to_be32(rsp_buffer_token);
3009         login_buffer->login_rsp_len = cpu_to_be32(rsp_buffer_size);
3010
3011         tx_list_p = (__be64 *)((char *)login_buffer +
3012                                       sizeof(struct ibmvnic_login_buffer));
3013         rx_list_p = (__be64 *)((char *)login_buffer +
3014                                       sizeof(struct ibmvnic_login_buffer) +
3015                                       sizeof(u64) * adapter->req_tx_queues);
3016
3017         for (i = 0; i < adapter->req_tx_queues; i++) {
3018                 if (adapter->tx_scrq[i]) {
3019                         tx_list_p[i] = cpu_to_be64(adapter->tx_scrq[i]->
3020                                                    crq_num);
3021                 }
3022         }
3023
3024         for (i = 0; i < adapter->req_rx_queues; i++) {
3025                 if (adapter->rx_scrq[i]) {
3026                         rx_list_p[i] = cpu_to_be64(adapter->rx_scrq[i]->
3027                                                    crq_num);
3028                 }
3029         }
3030
3031         /* Insert vNIC login client data */
3032         vlcd = (struct vnic_login_client_data *)
3033                 ((char *)rx_list_p + (sizeof(u64) * adapter->req_rx_queues));
3034         login_buffer->client_data_offset =
3035                         cpu_to_be32((char *)vlcd - (char *)login_buffer);
3036         login_buffer->client_data_len = cpu_to_be32(client_data_len);
3037
3038         vnic_add_client_data(adapter, vlcd);
3039
3040         netdev_dbg(adapter->netdev, "Login Buffer:\n");
3041         for (i = 0; i < (adapter->login_buf_sz - 1) / 8 + 1; i++) {
3042                 netdev_dbg(adapter->netdev, "%016lx\n",
3043                            ((unsigned long int *)(adapter->login_buf))[i]);
3044         }
3045
3046         memset(&crq, 0, sizeof(crq));
3047         crq.login.first = IBMVNIC_CRQ_CMD;
3048         crq.login.cmd = LOGIN;
3049         crq.login.ioba = cpu_to_be32(buffer_token);
3050         crq.login.len = cpu_to_be32(buffer_size);
3051         ibmvnic_send_crq(adapter, &crq);
3052
3053         return;
3054
3055 buf_rsp_map_failed:
3056         kfree(login_rsp_buffer);
3057 buf_rsp_alloc_failed:
3058         dma_unmap_single(dev, buffer_token, buffer_size, DMA_TO_DEVICE);
3059 buf_map_failed:
3060         kfree(login_buffer);
3061 buf_alloc_failed:
3062         return;
3063 }
3064
3065 static void send_request_map(struct ibmvnic_adapter *adapter, dma_addr_t addr,
3066                              u32 len, u8 map_id)
3067 {
3068         union ibmvnic_crq crq;
3069
3070         memset(&crq, 0, sizeof(crq));
3071         crq.request_map.first = IBMVNIC_CRQ_CMD;
3072         crq.request_map.cmd = REQUEST_MAP;
3073         crq.request_map.map_id = map_id;
3074         crq.request_map.ioba = cpu_to_be32(addr);
3075         crq.request_map.len = cpu_to_be32(len);
3076         ibmvnic_send_crq(adapter, &crq);
3077 }
3078
3079 static void send_request_unmap(struct ibmvnic_adapter *adapter, u8 map_id)
3080 {
3081         union ibmvnic_crq crq;
3082
3083         memset(&crq, 0, sizeof(crq));
3084         crq.request_unmap.first = IBMVNIC_CRQ_CMD;
3085         crq.request_unmap.cmd = REQUEST_UNMAP;
3086         crq.request_unmap.map_id = map_id;
3087         ibmvnic_send_crq(adapter, &crq);
3088 }
3089
3090 static void send_map_query(struct ibmvnic_adapter *adapter)
3091 {
3092         union ibmvnic_crq crq;
3093
3094         memset(&crq, 0, sizeof(crq));
3095         crq.query_map.first = IBMVNIC_CRQ_CMD;
3096         crq.query_map.cmd = QUERY_MAP;
3097         ibmvnic_send_crq(adapter, &crq);
3098 }
3099
3100 /* Send a series of CRQs requesting various capabilities of the VNIC server */
3101 static void send_cap_queries(struct ibmvnic_adapter *adapter)
3102 {
3103         union ibmvnic_crq crq;
3104
3105         atomic_set(&adapter->running_cap_crqs, 0);
3106         memset(&crq, 0, sizeof(crq));
3107         crq.query_capability.first = IBMVNIC_CRQ_CMD;
3108         crq.query_capability.cmd = QUERY_CAPABILITY;
3109
3110         crq.query_capability.capability = cpu_to_be16(MIN_TX_QUEUES);
3111         atomic_inc(&adapter->running_cap_crqs);
3112         ibmvnic_send_crq(adapter, &crq);
3113
3114         crq.query_capability.capability = cpu_to_be16(MIN_RX_QUEUES);
3115         atomic_inc(&adapter->running_cap_crqs);
3116         ibmvnic_send_crq(adapter, &crq);
3117
3118         crq.query_capability.capability = cpu_to_be16(MIN_RX_ADD_QUEUES);
3119         atomic_inc(&adapter->running_cap_crqs);
3120         ibmvnic_send_crq(adapter, &crq);
3121
3122         crq.query_capability.capability = cpu_to_be16(MAX_TX_QUEUES);
3123         atomic_inc(&adapter->running_cap_crqs);
3124         ibmvnic_send_crq(adapter, &crq);
3125
3126         crq.query_capability.capability = cpu_to_be16(MAX_RX_QUEUES);
3127         atomic_inc(&adapter->running_cap_crqs);
3128         ibmvnic_send_crq(adapter, &crq);
3129
3130         crq.query_capability.capability = cpu_to_be16(MAX_RX_ADD_QUEUES);
3131         atomic_inc(&adapter->running_cap_crqs);
3132         ibmvnic_send_crq(adapter, &crq);
3133
3134         crq.query_capability.capability =
3135             cpu_to_be16(MIN_TX_ENTRIES_PER_SUBCRQ);
3136         atomic_inc(&adapter->running_cap_crqs);
3137         ibmvnic_send_crq(adapter, &crq);
3138
3139         crq.query_capability.capability =
3140             cpu_to_be16(MIN_RX_ADD_ENTRIES_PER_SUBCRQ);
3141         atomic_inc(&adapter->running_cap_crqs);
3142         ibmvnic_send_crq(adapter, &crq);
3143
3144         crq.query_capability.capability =
3145             cpu_to_be16(MAX_TX_ENTRIES_PER_SUBCRQ);
3146         atomic_inc(&adapter->running_cap_crqs);
3147         ibmvnic_send_crq(adapter, &crq);
3148
3149         crq.query_capability.capability =
3150             cpu_to_be16(MAX_RX_ADD_ENTRIES_PER_SUBCRQ);
3151         atomic_inc(&adapter->running_cap_crqs);
3152         ibmvnic_send_crq(adapter, &crq);
3153
3154         crq.query_capability.capability = cpu_to_be16(TCP_IP_OFFLOAD);
3155         atomic_inc(&adapter->running_cap_crqs);
3156         ibmvnic_send_crq(adapter, &crq);
3157
3158         crq.query_capability.capability = cpu_to_be16(PROMISC_SUPPORTED);
3159         atomic_inc(&adapter->running_cap_crqs);
3160         ibmvnic_send_crq(adapter, &crq);
3161
3162         crq.query_capability.capability = cpu_to_be16(MIN_MTU);
3163         atomic_inc(&adapter->running_cap_crqs);
3164         ibmvnic_send_crq(adapter, &crq);
3165
3166         crq.query_capability.capability = cpu_to_be16(MAX_MTU);
3167         atomic_inc(&adapter->running_cap_crqs);
3168         ibmvnic_send_crq(adapter, &crq);
3169
3170         crq.query_capability.capability = cpu_to_be16(MAX_MULTICAST_FILTERS);
3171         atomic_inc(&adapter->running_cap_crqs);
3172         ibmvnic_send_crq(adapter, &crq);
3173
3174         crq.query_capability.capability = cpu_to_be16(VLAN_HEADER_INSERTION);
3175         atomic_inc(&adapter->running_cap_crqs);
3176         ibmvnic_send_crq(adapter, &crq);
3177
3178         crq.query_capability.capability = cpu_to_be16(RX_VLAN_HEADER_INSERTION);
3179         atomic_inc(&adapter->running_cap_crqs);
3180         ibmvnic_send_crq(adapter, &crq);
3181
3182         crq.query_capability.capability = cpu_to_be16(MAX_TX_SG_ENTRIES);
3183         atomic_inc(&adapter->running_cap_crqs);
3184         ibmvnic_send_crq(adapter, &crq);
3185
3186         crq.query_capability.capability = cpu_to_be16(RX_SG_SUPPORTED);
3187         atomic_inc(&adapter->running_cap_crqs);
3188         ibmvnic_send_crq(adapter, &crq);
3189
3190         crq.query_capability.capability = cpu_to_be16(OPT_TX_COMP_SUB_QUEUES);
3191         atomic_inc(&adapter->running_cap_crqs);
3192         ibmvnic_send_crq(adapter, &crq);
3193
3194         crq.query_capability.capability = cpu_to_be16(OPT_RX_COMP_QUEUES);
3195         atomic_inc(&adapter->running_cap_crqs);
3196         ibmvnic_send_crq(adapter, &crq);
3197
3198         crq.query_capability.capability =
3199                         cpu_to_be16(OPT_RX_BUFADD_Q_PER_RX_COMP_Q);
3200         atomic_inc(&adapter->running_cap_crqs);
3201         ibmvnic_send_crq(adapter, &crq);
3202
3203         crq.query_capability.capability =
3204                         cpu_to_be16(OPT_TX_ENTRIES_PER_SUBCRQ);
3205         atomic_inc(&adapter->running_cap_crqs);
3206         ibmvnic_send_crq(adapter, &crq);
3207
3208         crq.query_capability.capability =
3209                         cpu_to_be16(OPT_RXBA_ENTRIES_PER_SUBCRQ);
3210         atomic_inc(&adapter->running_cap_crqs);
3211         ibmvnic_send_crq(adapter, &crq);
3212
3213         crq.query_capability.capability = cpu_to_be16(TX_RX_DESC_REQ);
3214         atomic_inc(&adapter->running_cap_crqs);
3215         ibmvnic_send_crq(adapter, &crq);
3216 }
3217
3218 static void handle_vpd_size_rsp(union ibmvnic_crq *crq,
3219                                 struct ibmvnic_adapter *adapter)
3220 {
3221         struct device *dev = &adapter->vdev->dev;
3222
3223         if (crq->get_vpd_size_rsp.rc.code) {
3224                 dev_err(dev, "Error retrieving VPD size, rc=%x\n",
3225                         crq->get_vpd_size_rsp.rc.code);
3226                 complete(&adapter->fw_done);
3227                 return;
3228         }
3229
3230         adapter->vpd->len = be64_to_cpu(crq->get_vpd_size_rsp.len);
3231         complete(&adapter->fw_done);
3232 }
3233
3234 static void handle_vpd_rsp(union ibmvnic_crq *crq,
3235                            struct ibmvnic_adapter *adapter)
3236 {
3237         struct device *dev = &adapter->vdev->dev;
3238         unsigned char *substr = NULL, *ptr = NULL;
3239         u8 fw_level_len = 0;
3240
3241         memset(adapter->fw_version, 0, 32);
3242
3243         dma_unmap_single(dev, adapter->vpd->dma_addr, adapter->vpd->len,
3244                          DMA_FROM_DEVICE);
3245
3246         if (crq->get_vpd_rsp.rc.code) {
3247                 dev_err(dev, "Error retrieving VPD from device, rc=%x\n",
3248                         crq->get_vpd_rsp.rc.code);
3249                 goto complete;
3250         }
3251
3252         /* get the position of the firmware version info
3253          * located after the ASCII 'RM' substring in the buffer
3254          */
3255         substr = strnstr(adapter->vpd->buff, "RM", adapter->vpd->len);
3256         if (!substr) {
3257                 dev_info(dev, "No FW level provided by VPD\n");
3258                 goto complete;
3259         }
3260
3261         /* get length of firmware level ASCII substring */
3262         if ((substr + 2) < (adapter->vpd->buff + adapter->vpd->len)) {
3263                 fw_level_len = *(substr + 2);
3264         } else {
3265                 dev_info(dev, "Length of FW substr extrapolated VDP buff\n");
3266                 goto complete;
3267         }
3268
3269         /* copy firmware version string from vpd into adapter */
3270         if ((substr + 3 + fw_level_len) <
3271             (adapter->vpd->buff + adapter->vpd->len)) {
3272                 ptr = strncpy((char *)adapter->fw_version,
3273                               substr + 3, fw_level_len);
3274
3275                 if (!ptr)
3276                         dev_err(dev, "Failed to isolate FW level string\n");
3277         } else {
3278                 dev_info(dev, "FW substr extrapolated VPD buff\n");
3279         }
3280
3281 complete:
3282         complete(&adapter->fw_done);
3283 }
3284
3285 static void handle_query_ip_offload_rsp(struct ibmvnic_adapter *adapter)
3286 {
3287         struct device *dev = &adapter->vdev->dev;
3288         struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf;
3289         union ibmvnic_crq crq;
3290         int i;
3291
3292         dma_unmap_single(dev, adapter->ip_offload_tok,
3293                          sizeof(adapter->ip_offload_buf), DMA_FROM_DEVICE);
3294
3295         netdev_dbg(adapter->netdev, "Query IP Offload Buffer:\n");
3296         for (i = 0; i < (sizeof(adapter->ip_offload_buf) - 1) / 8 + 1; i++)
3297                 netdev_dbg(adapter->netdev, "%016lx\n",
3298                            ((unsigned long int *)(buf))[i]);
3299
3300         netdev_dbg(adapter->netdev, "ipv4_chksum = %d\n", buf->ipv4_chksum);
3301         netdev_dbg(adapter->netdev, "ipv6_chksum = %d\n", buf->ipv6_chksum);
3302         netdev_dbg(adapter->netdev, "tcp_ipv4_chksum = %d\n",
3303                    buf->tcp_ipv4_chksum);
3304         netdev_dbg(adapter->netdev, "tcp_ipv6_chksum = %d\n",
3305                    buf->tcp_ipv6_chksum);
3306         netdev_dbg(adapter->netdev, "udp_ipv4_chksum = %d\n",
3307                    buf->udp_ipv4_chksum);
3308         netdev_dbg(adapter->netdev, "udp_ipv6_chksum = %d\n",
3309                    buf->udp_ipv6_chksum);
3310         netdev_dbg(adapter->netdev, "large_tx_ipv4 = %d\n",
3311                    buf->large_tx_ipv4);
3312         netdev_dbg(adapter->netdev, "large_tx_ipv6 = %d\n",
3313                    buf->large_tx_ipv6);
3314         netdev_dbg(adapter->netdev, "large_rx_ipv4 = %d\n",
3315                    buf->large_rx_ipv4);
3316         netdev_dbg(adapter->netdev, "large_rx_ipv6 = %d\n",
3317                    buf->large_rx_ipv6);
3318         netdev_dbg(adapter->netdev, "max_ipv4_hdr_sz = %d\n",
3319                    buf->max_ipv4_header_size);
3320         netdev_dbg(adapter->netdev, "max_ipv6_hdr_sz = %d\n",
3321                    buf->max_ipv6_header_size);
3322         netdev_dbg(adapter->netdev, "max_tcp_hdr_size = %d\n",
3323                    buf->max_tcp_header_size);
3324         netdev_dbg(adapter->netdev, "max_udp_hdr_size = %d\n",
3325                    buf->max_udp_header_size);
3326         netdev_dbg(adapter->netdev, "max_large_tx_size = %d\n",
3327                    buf->max_large_tx_size);
3328         netdev_dbg(adapter->netdev, "max_large_rx_size = %d\n",
3329                    buf->max_large_rx_size);
3330         netdev_dbg(adapter->netdev, "ipv6_ext_hdr = %d\n",
3331                    buf->ipv6_extension_header);
3332         netdev_dbg(adapter->netdev, "tcp_pseudosum_req = %d\n",
3333                    buf->tcp_pseudosum_req);
3334         netdev_dbg(adapter->netdev, "num_ipv6_ext_hd = %d\n",
3335                    buf->num_ipv6_ext_headers);
3336         netdev_dbg(adapter->netdev, "off_ipv6_ext_hd = %d\n",
3337                    buf->off_ipv6_ext_headers);
3338
3339         adapter->ip_offload_ctrl_tok =
3340             dma_map_single(dev, &adapter->ip_offload_ctrl,
3341                            sizeof(adapter->ip_offload_ctrl), DMA_TO_DEVICE);
3342
3343         if (dma_mapping_error(dev, adapter->ip_offload_ctrl_tok)) {
3344                 dev_err(dev, "Couldn't map ip offload control buffer\n");
3345                 return;
3346         }
3347
3348         adapter->ip_offload_ctrl.version = cpu_to_be32(INITIAL_VERSION_IOB);
3349         adapter->ip_offload_ctrl.tcp_ipv4_chksum = buf->tcp_ipv4_chksum;
3350         adapter->ip_offload_ctrl.udp_ipv4_chksum = buf->udp_ipv4_chksum;
3351         adapter->ip_offload_ctrl.tcp_ipv6_chksum = buf->tcp_ipv6_chksum;
3352         adapter->ip_offload_ctrl.udp_ipv6_chksum = buf->udp_ipv6_chksum;
3353         adapter->ip_offload_ctrl.large_tx_ipv4 = buf->large_tx_ipv4;
3354         adapter->ip_offload_ctrl.large_tx_ipv6 = buf->large_tx_ipv6;
3355
3356         /* large_rx disabled for now, additional features needed */
3357         adapter->ip_offload_ctrl.large_rx_ipv4 = 0;
3358         adapter->ip_offload_ctrl.large_rx_ipv6 = 0;
3359
3360         adapter->netdev->features = NETIF_F_SG | NETIF_F_GSO;
3361
3362         if (buf->tcp_ipv4_chksum || buf->udp_ipv4_chksum)
3363                 adapter->netdev->features |= NETIF_F_IP_CSUM;
3364
3365         if (buf->tcp_ipv6_chksum || buf->udp_ipv6_chksum)
3366                 adapter->netdev->features |= NETIF_F_IPV6_CSUM;
3367
3368         if ((adapter->netdev->features &
3369             (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)))
3370                 adapter->netdev->features |= NETIF_F_RXCSUM;
3371
3372         if (buf->large_tx_ipv4)
3373                 adapter->netdev->features |= NETIF_F_TSO;
3374         if (buf->large_tx_ipv6)
3375                 adapter->netdev->features |= NETIF_F_TSO6;
3376
3377         adapter->netdev->hw_features |= adapter->netdev->features;
3378
3379         memset(&crq, 0, sizeof(crq));
3380         crq.control_ip_offload.first = IBMVNIC_CRQ_CMD;
3381         crq.control_ip_offload.cmd = CONTROL_IP_OFFLOAD;
3382         crq.control_ip_offload.len =
3383             cpu_to_be32(sizeof(adapter->ip_offload_ctrl));
3384         crq.control_ip_offload.ioba = cpu_to_be32(adapter->ip_offload_ctrl_tok);
3385         ibmvnic_send_crq(adapter, &crq);
3386 }
3387
3388 static void handle_error_info_rsp(union ibmvnic_crq *crq,
3389                                   struct ibmvnic_adapter *adapter)
3390 {
3391         struct device *dev = &adapter->vdev->dev;
3392         struct ibmvnic_error_buff *error_buff, *tmp;
3393         unsigned long flags;
3394         bool found = false;
3395         int i;
3396
3397         if (!crq->request_error_rsp.rc.code) {
3398                 dev_info(dev, "Request Error Rsp returned with rc=%x\n",
3399                          crq->request_error_rsp.rc.code);
3400                 return;
3401         }
3402
3403         spin_lock_irqsave(&adapter->error_list_lock, flags);
3404         list_for_each_entry_safe(error_buff, tmp, &adapter->errors, list)
3405                 if (error_buff->error_id == crq->request_error_rsp.error_id) {
3406                         found = true;
3407                         list_del(&error_buff->list);
3408                         break;
3409                 }
3410         spin_unlock_irqrestore(&adapter->error_list_lock, flags);
3411
3412         if (!found) {
3413                 dev_err(dev, "Couldn't find error id %x\n",
3414                         be32_to_cpu(crq->request_error_rsp.error_id));
3415                 return;
3416         }
3417
3418         dev_err(dev, "Detailed info for error id %x:",
3419                 be32_to_cpu(crq->request_error_rsp.error_id));
3420
3421         for (i = 0; i < error_buff->len; i++) {
3422                 pr_cont("%02x", (int)error_buff->buff[i]);
3423                 if (i % 8 == 7)
3424                         pr_cont(" ");
3425         }
3426         pr_cont("\n");
3427
3428         dma_unmap_single(dev, error_buff->dma, error_buff->len,
3429                          DMA_FROM_DEVICE);
3430         kfree(error_buff->buff);
3431         kfree(error_buff);
3432 }
3433
3434 static void request_error_information(struct ibmvnic_adapter *adapter,
3435                                       union ibmvnic_crq *err_crq)
3436 {
3437         struct device *dev = &adapter->vdev->dev;
3438         struct net_device *netdev = adapter->netdev;
3439         struct ibmvnic_error_buff *error_buff;
3440         unsigned long timeout = msecs_to_jiffies(30000);
3441         union ibmvnic_crq crq;
3442         unsigned long flags;
3443         int rc, detail_len;
3444
3445         error_buff = kmalloc(sizeof(*error_buff), GFP_ATOMIC);
3446         if (!error_buff)
3447                 return;
3448
3449         detail_len = be32_to_cpu(err_crq->error_indication.detail_error_sz);
3450         error_buff->buff = kmalloc(detail_len, GFP_ATOMIC);
3451         if (!error_buff->buff) {
3452                 kfree(error_buff);
3453                 return;
3454         }
3455
3456         error_buff->dma = dma_map_single(dev, error_buff->buff, detail_len,
3457                                          DMA_FROM_DEVICE);
3458         if (dma_mapping_error(dev, error_buff->dma)) {
3459                 netdev_err(netdev, "Couldn't map error buffer\n");
3460                 kfree(error_buff->buff);
3461                 kfree(error_buff);
3462                 return;
3463         }
3464
3465         error_buff->len = detail_len;
3466         error_buff->error_id = err_crq->error_indication.error_id;
3467
3468         spin_lock_irqsave(&adapter->error_list_lock, flags);
3469         list_add_tail(&error_buff->list, &adapter->errors);
3470         spin_unlock_irqrestore(&adapter->error_list_lock, flags);
3471
3472         memset(&crq, 0, sizeof(crq));
3473         crq.request_error_info.first = IBMVNIC_CRQ_CMD;
3474         crq.request_error_info.cmd = REQUEST_ERROR_INFO;
3475         crq.request_error_info.ioba = cpu_to_be32(error_buff->dma);
3476         crq.request_error_info.len = cpu_to_be32(detail_len);
3477         crq.request_error_info.error_id = err_crq->error_indication.error_id;
3478
3479         rc = ibmvnic_send_crq(adapter, &crq);
3480         if (rc) {
3481                 netdev_err(netdev, "failed to request error information\n");
3482                 goto err_info_fail;
3483         }
3484
3485         if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {
3486                 netdev_err(netdev, "timeout waiting for error information\n");
3487                 goto err_info_fail;
3488         }
3489
3490         return;
3491
3492 err_info_fail:
3493         spin_lock_irqsave(&adapter->error_list_lock, flags);
3494         list_del(&error_buff->list);
3495         spin_unlock_irqrestore(&adapter->error_list_lock, flags);
3496
3497         kfree(error_buff->buff);
3498         kfree(error_buff);
3499 }
3500
3501 static void handle_error_indication(union ibmvnic_crq *crq,
3502                                     struct ibmvnic_adapter *adapter)
3503 {
3504         struct device *dev = &adapter->vdev->dev;
3505
3506         dev_err(dev, "Firmware reports %serror id %x, cause %d\n",
3507                 crq->error_indication.flags
3508                         & IBMVNIC_FATAL_ERROR ? "FATAL " : "",
3509                 be32_to_cpu(crq->error_indication.error_id),
3510                 be16_to_cpu(crq->error_indication.error_cause));
3511
3512         if (be32_to_cpu(crq->error_indication.error_id))
3513                 request_error_information(adapter, crq);
3514
3515         if (crq->error_indication.flags & IBMVNIC_FATAL_ERROR)
3516                 ibmvnic_reset(adapter, VNIC_RESET_FATAL);
3517         else
3518                 ibmvnic_reset(adapter, VNIC_RESET_NON_FATAL);
3519 }
3520
3521 static void handle_change_mac_rsp(union ibmvnic_crq *crq,
3522                                   struct ibmvnic_adapter *adapter)
3523 {
3524         struct net_device *netdev = adapter->netdev;
3525         struct device *dev = &adapter->vdev->dev;
3526         long rc;
3527
3528         rc = crq->change_mac_addr_rsp.rc.code;
3529         if (rc) {
3530                 dev_err(dev, "Error %ld in CHANGE_MAC_ADDR_RSP\n", rc);
3531                 return;
3532         }
3533         memcpy(netdev->dev_addr, &crq->change_mac_addr_rsp.mac_addr[0],
3534                ETH_ALEN);
3535 }
3536
3537 static void handle_request_cap_rsp(union ibmvnic_crq *crq,
3538                                    struct ibmvnic_adapter *adapter)
3539 {
3540         struct device *dev = &adapter->vdev->dev;
3541         u64 *req_value;
3542         char *name;
3543
3544         atomic_dec(&adapter->running_cap_crqs);
3545         switch (be16_to_cpu(crq->request_capability_rsp.capability)) {
3546         case REQ_TX_QUEUES:
3547                 req_value = &adapter->req_tx_queues;
3548                 name = "tx";
3549                 break;
3550         case REQ_RX_QUEUES:
3551                 req_value = &adapter->req_rx_queues;
3552                 name = "rx";
3553                 break;
3554         case REQ_RX_ADD_QUEUES:
3555                 req_value = &adapter->req_rx_add_queues;
3556                 name = "rx_add";
3557                 break;
3558         case REQ_TX_ENTRIES_PER_SUBCRQ:
3559                 req_value = &adapter->req_tx_entries_per_subcrq;
3560                 name = "tx_entries_per_subcrq";
3561                 break;
3562         case REQ_RX_ADD_ENTRIES_PER_SUBCRQ:
3563                 req_value = &adapter->req_rx_add_entries_per_subcrq;
3564                 name = "rx_add_entries_per_subcrq";
3565                 break;
3566         case REQ_MTU:
3567                 req_value = &adapter->req_mtu;
3568                 name = "mtu";
3569                 break;
3570         case PROMISC_REQUESTED:
3571                 req_value = &adapter->promisc;
3572                 name = "promisc";
3573                 break;
3574         default:
3575                 dev_err(dev, "Got invalid cap request rsp %d\n",
3576                         crq->request_capability.capability);
3577                 return;
3578         }
3579
3580         switch (crq->request_capability_rsp.rc.code) {
3581         case SUCCESS:
3582                 break;
3583         case PARTIALSUCCESS:
3584                 dev_info(dev, "req=%lld, rsp=%ld in %s queue, retrying.\n",
3585                          *req_value,
3586                          (long int)be64_to_cpu(crq->request_capability_rsp.
3587                                                number), name);
3588                 *req_value = be64_to_cpu(crq->request_capability_rsp.number);
3589                 ibmvnic_send_req_caps(adapter, 1);
3590                 return;
3591         default:
3592                 dev_err(dev, "Error %d in request cap rsp\n",
3593                         crq->request_capability_rsp.rc.code);
3594                 return;
3595         }
3596
3597         /* Done receiving requested capabilities, query IP offload support */
3598         if (atomic_read(&adapter->running_cap_crqs) == 0) {
3599                 union ibmvnic_crq newcrq;
3600                 int buf_sz = sizeof(struct ibmvnic_query_ip_offload_buffer);
3601                 struct ibmvnic_query_ip_offload_buffer *ip_offload_buf =
3602                     &adapter->ip_offload_buf;
3603
3604                 adapter->wait_capability = false;
3605                 adapter->ip_offload_tok = dma_map_single(dev, ip_offload_buf,
3606                                                          buf_sz,
3607                                                          DMA_FROM_DEVICE);
3608
3609                 if (dma_mapping_error(dev, adapter->ip_offload_tok)) {
3610                         if (!firmware_has_feature(FW_FEATURE_CMO))
3611                                 dev_err(dev, "Couldn't map offload buffer\n");
3612                         return;
3613                 }
3614
3615                 memset(&newcrq, 0, sizeof(newcrq));
3616                 newcrq.query_ip_offload.first = IBMVNIC_CRQ_CMD;
3617                 newcrq.query_ip_offload.cmd = QUERY_IP_OFFLOAD;
3618                 newcrq.query_ip_offload.len = cpu_to_be32(buf_sz);
3619                 newcrq.query_ip_offload.ioba =
3620                     cpu_to_be32(adapter->ip_offload_tok);
3621
3622                 ibmvnic_send_crq(adapter, &newcrq);
3623         }
3624 }
3625
3626 static int handle_login_rsp(union ibmvnic_crq *login_rsp_crq,
3627                             struct ibmvnic_adapter *adapter)
3628 {
3629         struct device *dev = &adapter->vdev->dev;
3630         struct net_device *netdev = adapter->netdev;
3631         struct ibmvnic_login_rsp_buffer *login_rsp = adapter->login_rsp_buf;
3632         struct ibmvnic_login_buffer *login = adapter->login_buf;
3633         int i;
3634
3635         dma_unmap_single(dev, adapter->login_buf_token, adapter->login_buf_sz,
3636                          DMA_BIDIRECTIONAL);
3637         dma_unmap_single(dev, adapter->login_rsp_buf_token,
3638                          adapter->login_rsp_buf_sz, DMA_BIDIRECTIONAL);
3639
3640         /* If the number of queues requested can't be allocated by the
3641          * server, the login response will return with code 1. We will need
3642          * to resend the login buffer with fewer queues requested.
3643          */
3644         if (login_rsp_crq->generic.rc.code) {
3645                 adapter->renegotiate = true;
3646                 complete(&adapter->init_done);
3647                 return 0;
3648         }
3649
3650         netdev->mtu = adapter->req_mtu - ETH_HLEN;
3651
3652         netdev_dbg(adapter->netdev, "Login Response Buffer:\n");
3653         for (i = 0; i < (adapter->login_rsp_buf_sz - 1) / 8 + 1; i++) {
3654                 netdev_dbg(adapter->netdev, "%016lx\n",
3655                            ((unsigned long int *)(adapter->login_rsp_buf))[i]);
3656         }
3657
3658         /* Sanity checks */
3659         if (login->num_txcomp_subcrqs != login_rsp->num_txsubm_subcrqs ||
3660             (be32_to_cpu(login->num_rxcomp_subcrqs) *
3661              adapter->req_rx_add_queues !=
3662              be32_to_cpu(login_rsp->num_rxadd_subcrqs))) {
3663                 dev_err(dev, "FATAL: Inconsistent login and login rsp\n");
3664                 ibmvnic_remove(adapter->vdev);
3665                 return -EIO;
3666         }
3667         complete(&adapter->init_done);
3668
3669         return 0;
3670 }
3671
3672 static void handle_request_unmap_rsp(union ibmvnic_crq *crq,
3673                                      struct ibmvnic_adapter *adapter)
3674 {
3675         struct device *dev = &adapter->vdev->dev;
3676         long rc;
3677
3678         rc = crq->request_unmap_rsp.rc.code;
3679         if (rc)
3680                 dev_err(dev, "Error %ld in REQUEST_UNMAP_RSP\n", rc);
3681 }
3682
3683 static void handle_query_map_rsp(union ibmvnic_crq *crq,
3684                                  struct ibmvnic_adapter *adapter)
3685 {
3686         struct net_device *netdev = adapter->netdev;
3687         struct device *dev = &adapter->vdev->dev;
3688         long rc;
3689
3690         rc = crq->query_map_rsp.rc.code;
3691         if (rc) {
3692                 dev_err(dev, "Error %ld in QUERY_MAP_RSP\n", rc);
3693                 return;
3694         }
3695         netdev_dbg(netdev, "page_size = %d\ntot_pages = %d\nfree_pages = %d\n",
3696                    crq->query_map_rsp.page_size, crq->query_map_rsp.tot_pages,
3697                    crq->query_map_rsp.free_pages);
3698 }
3699
3700 static void handle_query_cap_rsp(union ibmvnic_crq *crq,
3701                                  struct ibmvnic_adapter *adapter)
3702 {
3703         struct net_device *netdev = adapter->netdev;
3704         struct device *dev = &adapter->vdev->dev;
3705         long rc;
3706
3707         atomic_dec(&adapter->running_cap_crqs);
3708         netdev_dbg(netdev, "Outstanding queries: %d\n",
3709                    atomic_read(&adapter->running_cap_crqs));
3710         rc = crq->query_capability.rc.code;
3711         if (rc) {
3712                 dev_err(dev, "Error %ld in QUERY_CAP_RSP\n", rc);
3713                 goto out;
3714         }
3715
3716         switch (be16_to_cpu(crq->query_capability.capability)) {
3717         case MIN_TX_QUEUES:
3718                 adapter->min_tx_queues =
3719                     be64_to_cpu(crq->query_capability.number);
3720                 netdev_dbg(netdev, "min_tx_queues = %lld\n",
3721                            adapter->min_tx_queues);
3722                 break;
3723         case MIN_RX_QUEUES:
3724                 adapter->min_rx_queues =
3725                     be64_to_cpu(crq->query_capability.number);
3726                 netdev_dbg(netdev, "min_rx_queues = %lld\n",
3727                            adapter->min_rx_queues);
3728                 break;
3729         case MIN_RX_ADD_QUEUES:
3730                 adapter->min_rx_add_queues =
3731                     be64_to_cpu(crq->query_capability.number);
3732                 netdev_dbg(netdev, "min_rx_add_queues = %lld\n",
3733                            adapter->min_rx_add_queues);
3734                 break;
3735         case MAX_TX_QUEUES:
3736                 adapter->max_tx_queues =
3737                     be64_to_cpu(crq->query_capability.number);
3738                 netdev_dbg(netdev, "max_tx_queues = %lld\n",
3739                            adapter->max_tx_queues);
3740                 break;
3741         case MAX_RX_QUEUES:
3742                 adapter->max_rx_queues =
3743                     be64_to_cpu(crq->query_capability.number);
3744                 netdev_dbg(netdev, "max_rx_queues = %lld\n",
3745                            adapter->max_rx_queues);
3746                 break;
3747         case MAX_RX_ADD_QUEUES:
3748                 adapter->max_rx_add_queues =
3749                     be64_to_cpu(crq->query_capability.number);
3750                 netdev_dbg(netdev, "max_rx_add_queues = %lld\n",
3751                            adapter->max_rx_add_queues);
3752                 break;
3753         case MIN_TX_ENTRIES_PER_SUBCRQ:
3754                 adapter->min_tx_entries_per_subcrq =
3755                     be64_to_cpu(crq->query_capability.number);
3756                 netdev_dbg(netdev, "min_tx_entries_per_subcrq = %lld\n",
3757                            adapter->min_tx_entries_per_subcrq);
3758                 break;
3759         case MIN_RX_ADD_ENTRIES_PER_SUBCRQ:
3760                 adapter->min_rx_add_entries_per_subcrq =
3761                     be64_to_cpu(crq->query_capability.number);
3762                 netdev_dbg(netdev, "min_rx_add_entrs_per_subcrq = %lld\n",
3763                            adapter->min_rx_add_entries_per_subcrq);
3764                 break;
3765         case MAX_TX_ENTRIES_PER_SUBCRQ:
3766                 adapter->max_tx_entries_per_subcrq =
3767                     be64_to_cpu(crq->query_capability.number);
3768                 netdev_dbg(netdev, "max_tx_entries_per_subcrq = %lld\n",
3769                            adapter->max_tx_entries_per_subcrq);
3770                 break;
3771         case MAX_RX_ADD_ENTRIES_PER_SUBCRQ:
3772                 adapter->max_rx_add_entries_per_subcrq =
3773                     be64_to_cpu(crq->query_capability.number);
3774                 netdev_dbg(netdev, "max_rx_add_entrs_per_subcrq = %lld\n",
3775                            adapter->max_rx_add_entries_per_subcrq);
3776                 break;
3777         case TCP_IP_OFFLOAD:
3778                 adapter->tcp_ip_offload =
3779                     be64_to_cpu(crq->query_capability.number);
3780                 netdev_dbg(netdev, "tcp_ip_offload = %lld\n",
3781                            adapter->tcp_ip_offload);
3782                 break;
3783         case PROMISC_SUPPORTED:
3784                 adapter->promisc_supported =
3785                     be64_to_cpu(crq->query_capability.number);
3786                 netdev_dbg(netdev, "promisc_supported = %lld\n",
3787                            adapter->promisc_supported);
3788                 break;
3789         case MIN_MTU:
3790                 adapter->min_mtu = be64_to_cpu(crq->query_capability.number);
3791                 netdev->min_mtu = adapter->min_mtu - ETH_HLEN;
3792                 netdev_dbg(netdev, "min_mtu = %lld\n", adapter->min_mtu);
3793                 break;
3794         case MAX_MTU:
3795                 adapter->max_mtu = be64_to_cpu(crq->query_capability.number);
3796                 netdev->max_mtu = adapter->max_mtu - ETH_HLEN;
3797                 netdev_dbg(netdev, "max_mtu = %lld\n", adapter->max_mtu);
3798                 break;
3799         case MAX_MULTICAST_FILTERS:
3800                 adapter->max_multicast_filters =
3801                     be64_to_cpu(crq->query_capability.number);
3802                 netdev_dbg(netdev, "max_multicast_filters = %lld\n",
3803                            adapter->max_multicast_filters);
3804                 break;
3805         case VLAN_HEADER_INSERTION:
3806                 adapter->vlan_header_insertion =
3807                     be64_to_cpu(crq->query_capability.number);
3808                 if (adapter->vlan_header_insertion)
3809                         netdev->features |= NETIF_F_HW_VLAN_STAG_TX;
3810                 netdev_dbg(netdev, "vlan_header_insertion = %lld\n",
3811                            adapter->vlan_header_insertion);
3812                 break;
3813         case RX_VLAN_HEADER_INSERTION:
3814                 adapter->rx_vlan_header_insertion =
3815                     be64_to_cpu(crq->query_capability.number);
3816                 netdev_dbg(netdev, "rx_vlan_header_insertion = %lld\n",
3817                            adapter->rx_vlan_header_insertion);
3818                 break;
3819         case MAX_TX_SG_ENTRIES:
3820                 adapter->max_tx_sg_entries =
3821                     be64_to_cpu(crq->query_capability.number);
3822                 netdev_dbg(netdev, "max_tx_sg_entries = %lld\n",
3823                            adapter->max_tx_sg_entries);
3824                 break;
3825         case RX_SG_SUPPORTED:
3826                 adapter->rx_sg_supported =
3827                     be64_to_cpu(crq->query_capability.number);
3828                 netdev_dbg(netdev, "rx_sg_supported = %lld\n",
3829                            adapter->rx_sg_supported);
3830                 break;
3831         case OPT_TX_COMP_SUB_QUEUES:
3832                 adapter->opt_tx_comp_sub_queues =
3833                     be64_to_cpu(crq->query_capability.number);
3834                 netdev_dbg(netdev, "opt_tx_comp_sub_queues = %lld\n",
3835                            adapter->opt_tx_comp_sub_queues);
3836                 break;
3837         case OPT_RX_COMP_QUEUES:
3838                 adapter->opt_rx_comp_queues =
3839                     be64_to_cpu(crq->query_capability.number);
3840                 netdev_dbg(netdev, "opt_rx_comp_queues = %lld\n",
3841                            adapter->opt_rx_comp_queues);
3842                 break;
3843         case OPT_RX_BUFADD_Q_PER_RX_COMP_Q:
3844                 adapter->opt_rx_bufadd_q_per_rx_comp_q =
3845                     be64_to_cpu(crq->query_capability.number);
3846                 netdev_dbg(netdev, "opt_rx_bufadd_q_per_rx_comp_q = %lld\n",
3847                            adapter->opt_rx_bufadd_q_per_rx_comp_q);
3848                 break;
3849         case OPT_TX_ENTRIES_PER_SUBCRQ:
3850                 adapter->opt_tx_entries_per_subcrq =
3851                     be64_to_cpu(crq->query_capability.number);
3852                 netdev_dbg(netdev, "opt_tx_entries_per_subcrq = %lld\n",
3853                            adapter->opt_tx_entries_per_subcrq);
3854                 break;
3855         case OPT_RXBA_ENTRIES_PER_SUBCRQ:
3856                 adapter->opt_rxba_entries_per_subcrq =
3857                     be64_to_cpu(crq->query_capability.number);
3858                 netdev_dbg(netdev, "opt_rxba_entries_per_subcrq = %lld\n",
3859                            adapter->opt_rxba_entries_per_subcrq);
3860                 break;
3861         case TX_RX_DESC_REQ:
3862                 adapter->tx_rx_desc_req = crq->query_capability.number;
3863                 netdev_dbg(netdev, "tx_rx_desc_req = %llx\n",
3864                            adapter->tx_rx_desc_req);
3865                 break;
3866
3867         default:
3868                 netdev_err(netdev, "Got invalid cap rsp %d\n",
3869                            crq->query_capability.capability);
3870         }
3871
3872 out:
3873         if (atomic_read(&adapter->running_cap_crqs) == 0) {
3874                 adapter->wait_capability = false;
3875                 ibmvnic_send_req_caps(adapter, 0);
3876         }
3877 }
3878
3879 static void ibmvnic_handle_crq(union ibmvnic_crq *crq,
3880                                struct ibmvnic_adapter *adapter)
3881 {
3882         struct ibmvnic_generic_crq *gen_crq = &crq->generic;
3883         struct net_device *netdev = adapter->netdev;
3884         struct device *dev = &adapter->vdev->dev;
3885         u64 *u64_crq = (u64 *)crq;
3886         long rc;
3887
3888         netdev_dbg(netdev, "Handling CRQ: %016lx %016lx\n",
3889                    (unsigned long int)cpu_to_be64(u64_crq[0]),
3890                    (unsigned long int)cpu_to_be64(u64_crq[1]));
3891         switch (gen_crq->first) {
3892         case IBMVNIC_CRQ_INIT_RSP:
3893                 switch (gen_crq->cmd) {
3894                 case IBMVNIC_CRQ_INIT:
3895                         dev_info(dev, "Partner initialized\n");
3896                         adapter->from_passive_init = true;
3897                         complete(&adapter->init_done);
3898                         break;
3899                 case IBMVNIC_CRQ_INIT_COMPLETE:
3900                         dev_info(dev, "Partner initialization complete\n");
3901                         send_version_xchg(adapter);
3902                         break;
3903                 default:
3904                         dev_err(dev, "Unknown crq cmd: %d\n", gen_crq->cmd);
3905                 }
3906                 return;
3907         case IBMVNIC_CRQ_XPORT_EVENT:
3908                 netif_carrier_off(netdev);
3909                 if (gen_crq->cmd == IBMVNIC_PARTITION_MIGRATED) {
3910                         dev_info(dev, "Migrated, re-enabling adapter\n");
3911                         ibmvnic_reset(adapter, VNIC_RESET_MOBILITY);
3912                 } else if (gen_crq->cmd == IBMVNIC_DEVICE_FAILOVER) {
3913                         dev_info(dev, "Backing device failover detected\n");
3914                         ibmvnic_reset(adapter, VNIC_RESET_FAILOVER);
3915                 } else {
3916                         /* The adapter lost the connection */
3917                         dev_err(dev, "Virtual Adapter failed (rc=%d)\n",
3918                                 gen_crq->cmd);
3919                         ibmvnic_reset(adapter, VNIC_RESET_FATAL);
3920                 }
3921                 return;
3922         case IBMVNIC_CRQ_CMD_RSP:
3923                 break;
3924         default:
3925                 dev_err(dev, "Got an invalid msg type 0x%02x\n",
3926                         gen_crq->first);
3927                 return;
3928         }
3929
3930         switch (gen_crq->cmd) {
3931         case VERSION_EXCHANGE_RSP:
3932                 rc = crq->version_exchange_rsp.rc.code;
3933                 if (rc) {
3934                         dev_err(dev, "Error %ld in VERSION_EXCHG_RSP\n", rc);
3935                         break;
3936                 }
3937                 dev_info(dev, "Partner protocol version is %d\n",
3938                          crq->version_exchange_rsp.version);
3939                 if (be16_to_cpu(crq->version_exchange_rsp.version) <
3940                     ibmvnic_version)
3941                         ibmvnic_version =
3942                             be16_to_cpu(crq->version_exchange_rsp.version);
3943                 send_cap_queries(adapter);
3944                 break;
3945         case QUERY_CAPABILITY_RSP:
3946                 handle_query_cap_rsp(crq, adapter);
3947                 break;
3948         case QUERY_MAP_RSP:
3949                 handle_query_map_rsp(crq, adapter);
3950                 break;
3951         case REQUEST_MAP_RSP:
3952                 adapter->fw_done_rc = crq->request_map_rsp.rc.code;
3953                 complete(&adapter->fw_done);
3954                 break;
3955         case REQUEST_UNMAP_RSP:
3956                 handle_request_unmap_rsp(crq, adapter);
3957                 break;
3958         case REQUEST_CAPABILITY_RSP:
3959                 handle_request_cap_rsp(crq, adapter);
3960                 break;
3961         case LOGIN_RSP:
3962                 netdev_dbg(netdev, "Got Login Response\n");
3963                 handle_login_rsp(crq, adapter);
3964                 break;
3965         case LOGICAL_LINK_STATE_RSP:
3966                 netdev_dbg(netdev,
3967                            "Got Logical Link State Response, state: %d rc: %d\n",
3968                            crq->logical_link_state_rsp.link_state,
3969                            crq->logical_link_state_rsp.rc.code);
3970                 adapter->logical_link_state =
3971                     crq->logical_link_state_rsp.link_state;
3972                 adapter->init_done_rc = crq->logical_link_state_rsp.rc.code;
3973                 complete(&adapter->init_done);
3974                 break;
3975         case LINK_STATE_INDICATION:
3976                 netdev_dbg(netdev, "Got Logical Link State Indication\n");
3977                 adapter->phys_link_state =
3978                     crq->link_state_indication.phys_link_state;
3979                 adapter->logical_link_state =
3980                     crq->link_state_indication.logical_link_state;
3981                 break;
3982         case CHANGE_MAC_ADDR_RSP:
3983                 netdev_dbg(netdev, "Got MAC address change Response\n");
3984                 handle_change_mac_rsp(crq, adapter);
3985                 break;
3986         case ERROR_INDICATION:
3987                 netdev_dbg(netdev, "Got Error Indication\n");
3988                 handle_error_indication(crq, adapter);
3989                 break;
3990         case REQUEST_ERROR_RSP:
3991                 netdev_dbg(netdev, "Got Error Detail Response\n");
3992                 handle_error_info_rsp(crq, adapter);
3993                 break;
3994         case REQUEST_STATISTICS_RSP:
3995                 netdev_dbg(netdev, "Got Statistics Response\n");
3996                 complete(&adapter->stats_done);
3997                 break;
3998         case QUERY_IP_OFFLOAD_RSP:
3999                 netdev_dbg(netdev, "Got Query IP offload Response\n");
4000                 handle_query_ip_offload_rsp(adapter);
4001                 break;
4002         case MULTICAST_CTRL_RSP:
4003                 netdev_dbg(netdev, "Got multicast control Response\n");
4004                 break;
4005         case CONTROL_IP_OFFLOAD_RSP:
4006                 netdev_dbg(netdev, "Got Control IP offload Response\n");
4007                 dma_unmap_single(dev, adapter->ip_offload_ctrl_tok,
4008                                  sizeof(adapter->ip_offload_ctrl),
4009                                  DMA_TO_DEVICE);
4010                 complete(&adapter->init_done);
4011                 break;
4012         case COLLECT_FW_TRACE_RSP:
4013                 netdev_dbg(netdev, "Got Collect firmware trace Response\n");
4014                 complete(&adapter->fw_done);
4015                 break;
4016         case GET_VPD_SIZE_RSP:
4017                 handle_vpd_size_rsp(crq, adapter);
4018                 break;
4019         case GET_VPD_RSP:
4020                 handle_vpd_rsp(crq, adapter);
4021                 break;
4022         default:
4023                 netdev_err(netdev, "Got an invalid cmd type 0x%02x\n",
4024                            gen_crq->cmd);
4025         }
4026 }
4027
4028 static irqreturn_t ibmvnic_interrupt(int irq, void *instance)
4029 {
4030         struct ibmvnic_adapter *adapter = instance;
4031
4032         tasklet_schedule(&adapter->tasklet);
4033         return IRQ_HANDLED;
4034 }
4035
4036 static void ibmvnic_tasklet(void *data)
4037 {
4038         struct ibmvnic_adapter *adapter = data;
4039         struct ibmvnic_crq_queue *queue = &adapter->crq;
4040         union ibmvnic_crq *crq;
4041         unsigned long flags;
4042         bool done = false;
4043
4044         spin_lock_irqsave(&queue->lock, flags);
4045         while (!done) {
4046                 /* Pull all the valid messages off the CRQ */
4047                 while ((crq = ibmvnic_next_crq(adapter)) != NULL) {
4048                         ibmvnic_handle_crq(crq, adapter);
4049                         crq->generic.first = 0;
4050                 }
4051
4052                 /* remain in tasklet until all
4053                  * capabilities responses are received
4054                  */
4055                 if (!adapter->wait_capability)
4056                         done = true;
4057         }
4058         /* if capabilities CRQ's were sent in this tasklet, the following
4059          * tasklet must wait until all responses are received
4060          */
4061         if (atomic_read(&adapter->running_cap_crqs) != 0)
4062                 adapter->wait_capability = true;
4063         spin_unlock_irqrestore(&queue->lock, flags);
4064 }
4065
4066 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *adapter)
4067 {
4068         struct vio_dev *vdev = adapter->vdev;
4069         int rc;
4070
4071         do {
4072                 rc = plpar_hcall_norets(H_ENABLE_CRQ, vdev->unit_address);
4073         } while (rc == H_IN_PROGRESS || rc == H_BUSY || H_IS_LONG_BUSY(rc));
4074
4075         if (rc)
4076                 dev_err(&vdev->dev, "Error enabling adapter (rc=%d)\n", rc);
4077
4078         return rc;
4079 }
4080
4081 static int ibmvnic_reset_crq(struct ibmvnic_adapter *adapter)
4082 {
4083         struct ibmvnic_crq_queue *crq = &adapter->crq;
4084         struct device *dev = &adapter->vdev->dev;
4085         struct vio_dev *vdev = adapter->vdev;
4086         int rc;
4087
4088         /* Close the CRQ */
4089         do {
4090                 rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
4091         } while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
4092
4093         /* Clean out the queue */
4094         memset(crq->msgs, 0, PAGE_SIZE);
4095         crq->cur = 0;
4096
4097         /* And re-open it again */
4098         rc = plpar_hcall_norets(H_REG_CRQ, vdev->unit_address,
4099                                 crq->msg_token, PAGE_SIZE);
4100
4101         if (rc == H_CLOSED)
4102                 /* Adapter is good, but other end is not ready */
4103                 dev_warn(dev, "Partner adapter not ready\n");
4104         else if (rc != 0)
4105                 dev_warn(dev, "Couldn't register crq (rc=%d)\n", rc);
4106
4107         return rc;
4108 }
4109
4110 static void release_crq_queue(struct ibmvnic_adapter *adapter)
4111 {
4112         struct ibmvnic_crq_queue *crq = &adapter->crq;
4113         struct vio_dev *vdev = adapter->vdev;
4114         long rc;
4115
4116         if (!crq->msgs)
4117                 return;
4118
4119         netdev_dbg(adapter->netdev, "Releasing CRQ\n");
4120         free_irq(vdev->irq, adapter);
4121         tasklet_kill(&adapter->tasklet);
4122         do {
4123                 rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
4124         } while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
4125
4126         dma_unmap_single(&vdev->dev, crq->msg_token, PAGE_SIZE,
4127                          DMA_BIDIRECTIONAL);
4128         free_page((unsigned long)crq->msgs);
4129         crq->msgs = NULL;
4130 }
4131
4132 static int init_crq_queue(struct ibmvnic_adapter *adapter)
4133 {
4134         struct ibmvnic_crq_queue *crq = &adapter->crq;
4135         struct device *dev = &adapter->vdev->dev;
4136         struct vio_dev *vdev = adapter->vdev;
4137         int rc, retrc = -ENOMEM;
4138
4139         if (crq->msgs)
4140                 return 0;
4141
4142         crq->msgs = (union ibmvnic_crq *)get_zeroed_page(GFP_KERNEL);
4143         /* Should we allocate more than one page? */
4144
4145         if (!crq->msgs)
4146                 return -ENOMEM;
4147
4148         crq->size = PAGE_SIZE / sizeof(*crq->msgs);
4149         crq->msg_token = dma_map_single(dev, crq->msgs, PAGE_SIZE,
4150                                         DMA_BIDIRECTIONAL);
4151         if (dma_mapping_error(dev, crq->msg_token))
4152                 goto map_failed;
4153
4154         rc = plpar_hcall_norets(H_REG_CRQ, vdev->unit_address,
4155                                 crq->msg_token, PAGE_SIZE);
4156
4157         if (rc == H_RESOURCE)
4158                 /* maybe kexecing and resource is busy. try a reset */
4159                 rc = ibmvnic_reset_crq(adapter);
4160         retrc = rc;
4161
4162         if (rc == H_CLOSED) {
4163                 dev_warn(dev, "Partner adapter not ready\n");
4164         } else if (rc) {
4165                 dev_warn(dev, "Error %d opening adapter\n", rc);
4166                 goto reg_crq_failed;
4167         }
4168
4169         retrc = 0;
4170
4171         tasklet_init(&adapter->tasklet, (void *)ibmvnic_tasklet,
4172                      (unsigned long)adapter);
4173
4174         netdev_dbg(adapter->netdev, "registering irq 0x%x\n", vdev->irq);
4175         rc = request_irq(vdev->irq, ibmvnic_interrupt, 0, IBMVNIC_NAME,
4176                          adapter);
4177         if (rc) {
4178                 dev_err(dev, "Couldn't register irq 0x%x. rc=%d\n",
4179                         vdev->irq, rc);
4180                 goto req_irq_failed;
4181         }
4182
4183         rc = vio_enable_interrupts(vdev);
4184         if (rc) {
4185                 dev_err(dev, "Error %d enabling interrupts\n", rc);
4186                 goto req_irq_failed;
4187         }
4188
4189         crq->cur = 0;
4190         spin_lock_init(&crq->lock);
4191
4192         return retrc;
4193
4194 req_irq_failed:
4195         tasklet_kill(&adapter->tasklet);
4196         do {
4197                 rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
4198         } while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
4199 reg_crq_failed:
4200         dma_unmap_single(dev, crq->msg_token, PAGE_SIZE, DMA_BIDIRECTIONAL);
4201 map_failed:
4202         free_page((unsigned long)crq->msgs);
4203         crq->msgs = NULL;
4204         return retrc;
4205 }
4206
4207 static int ibmvnic_init(struct ibmvnic_adapter *adapter)
4208 {
4209         struct device *dev = &adapter->vdev->dev;
4210         unsigned long timeout = msecs_to_jiffies(30000);
4211         int rc;
4212
4213         if (adapter->resetting && !adapter->wait_for_reset) {
4214                 rc = ibmvnic_reset_crq(adapter);
4215                 if (!rc)
4216                         rc = vio_enable_interrupts(adapter->vdev);
4217         } else {
4218                 rc = init_crq_queue(adapter);
4219         }
4220
4221         if (rc) {
4222                 dev_err(dev, "Couldn't initialize crq. rc=%d\n", rc);
4223                 return rc;
4224         }
4225
4226         adapter->from_passive_init = false;
4227
4228         init_completion(&adapter->init_done);
4229         adapter->init_done_rc = 0;
4230         ibmvnic_send_crq_init(adapter);
4231         if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {
4232                 dev_err(dev, "Initialization sequence timed out\n");
4233                 return -1;
4234         }
4235
4236         if (adapter->init_done_rc) {
4237                 release_crq_queue(adapter);
4238                 return adapter->init_done_rc;
4239         }
4240
4241         if (adapter->from_passive_init) {
4242                 adapter->state = VNIC_OPEN;
4243                 adapter->from_passive_init = false;
4244                 return -1;
4245         }
4246
4247         if (adapter->resetting && !adapter->wait_for_reset)
4248                 rc = reset_sub_crq_queues(adapter);
4249         else
4250                 rc = init_sub_crqs(adapter);
4251         if (rc) {
4252                 dev_err(dev, "Initialization of sub crqs failed\n");
4253                 release_crq_queue(adapter);
4254                 return rc;
4255         }
4256
4257         rc = init_sub_crq_irqs(adapter);
4258         if (rc) {
4259                 dev_err(dev, "Failed to initialize sub crq irqs\n");
4260                 release_crq_queue(adapter);
4261         }
4262
4263         return rc;
4264 }
4265
4266 static struct device_attribute dev_attr_failover;
4267
4268 static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)
4269 {
4270         struct ibmvnic_adapter *adapter;
4271         struct net_device *netdev;
4272         unsigned char *mac_addr_p;
4273         int rc;
4274
4275         dev_dbg(&dev->dev, "entering ibmvnic_probe for UA 0x%x\n",
4276                 dev->unit_address);
4277
4278         mac_addr_p = (unsigned char *)vio_get_attribute(dev,
4279                                                         VETH_MAC_ADDR, NULL);
4280         if (!mac_addr_p) {
4281                 dev_err(&dev->dev,
4282                         "(%s:%3.3d) ERROR: Can't find MAC_ADDR attribute\n",
4283                         __FILE__, __LINE__);
4284                 return 0;
4285         }
4286
4287         netdev = alloc_etherdev_mq(sizeof(struct ibmvnic_adapter),
4288                                    IBMVNIC_MAX_TX_QUEUES);
4289         if (!netdev)
4290                 return -ENOMEM;
4291
4292         adapter = netdev_priv(netdev);
4293         adapter->state = VNIC_PROBING;
4294         dev_set_drvdata(&dev->dev, netdev);
4295         adapter->vdev = dev;
4296         adapter->netdev = netdev;
4297
4298         ether_addr_copy(adapter->mac_addr, mac_addr_p);
4299         ether_addr_copy(netdev->dev_addr, adapter->mac_addr);
4300         netdev->irq = dev->irq;
4301         netdev->netdev_ops = &ibmvnic_netdev_ops;
4302         netdev->ethtool_ops = &ibmvnic_ethtool_ops;
4303         SET_NETDEV_DEV(netdev, &dev->dev);
4304
4305         spin_lock_init(&adapter->stats_lock);
4306
4307         INIT_LIST_HEAD(&adapter->errors);
4308         spin_lock_init(&adapter->error_list_lock);
4309
4310         INIT_WORK(&adapter->ibmvnic_reset, __ibmvnic_reset);
4311         INIT_LIST_HEAD(&adapter->rwi_list);
4312         mutex_init(&adapter->reset_lock);
4313         mutex_init(&adapter->rwi_lock);
4314         adapter->resetting = false;
4315
4316         adapter->mac_change_pending = false;
4317
4318         do {
4319                 rc = ibmvnic_init(adapter);
4320                 if (rc && rc != EAGAIN)
4321                         goto ibmvnic_init_fail;
4322         } while (rc == EAGAIN);
4323
4324         netdev->mtu = adapter->req_mtu - ETH_HLEN;
4325         netdev->min_mtu = adapter->min_mtu - ETH_HLEN;
4326         netdev->max_mtu = adapter->max_mtu - ETH_HLEN;
4327
4328         rc = device_create_file(&dev->dev, &dev_attr_failover);
4329         if (rc)
4330                 goto ibmvnic_init_fail;
4331
4332         netif_carrier_off(netdev);
4333         rc = register_netdev(netdev);
4334         if (rc) {
4335                 dev_err(&dev->dev, "failed to register netdev rc=%d\n", rc);
4336                 goto ibmvnic_register_fail;
4337         }
4338         dev_info(&dev->dev, "ibmvnic registered\n");
4339
4340         adapter->state = VNIC_PROBED;
4341
4342         adapter->wait_for_reset = false;
4343
4344         return 0;
4345
4346 ibmvnic_register_fail:
4347         device_remove_file(&dev->dev, &dev_attr_failover);
4348
4349 ibmvnic_init_fail:
4350         release_sub_crqs(adapter);
4351         release_crq_queue(adapter);
4352         free_netdev(netdev);
4353
4354         return rc;
4355 }
4356
4357 static int ibmvnic_remove(struct vio_dev *dev)
4358 {
4359         struct net_device *netdev = dev_get_drvdata(&dev->dev);
4360         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
4361
4362         adapter->state = VNIC_REMOVING;
4363         unregister_netdev(netdev);
4364         mutex_lock(&adapter->reset_lock);
4365
4366         release_resources(adapter);
4367         release_sub_crqs(adapter);
4368         release_crq_queue(adapter);
4369
4370         adapter->state = VNIC_REMOVED;
4371
4372         mutex_unlock(&adapter->reset_lock);
4373         device_remove_file(&dev->dev, &dev_attr_failover);
4374         free_netdev(netdev);
4375         dev_set_drvdata(&dev->dev, NULL);
4376
4377         return 0;
4378 }
4379
4380 static ssize_t failover_store(struct device *dev, struct device_attribute *attr,
4381                               const char *buf, size_t count)
4382 {
4383         struct net_device *netdev = dev_get_drvdata(dev);
4384         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
4385         unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
4386         __be64 session_token;
4387         long rc;
4388
4389         if (!sysfs_streq(buf, "1"))
4390                 return -EINVAL;
4391
4392         rc = plpar_hcall(H_VIOCTL, retbuf, adapter->vdev->unit_address,
4393                          H_GET_SESSION_TOKEN, 0, 0, 0);
4394         if (rc) {
4395                 netdev_err(netdev, "Couldn't retrieve session token, rc %ld\n",
4396                            rc);
4397                 return -EINVAL;
4398         }
4399
4400         session_token = (__be64)retbuf[0];
4401         netdev_dbg(netdev, "Initiating client failover, session id %llx\n",
4402                    be64_to_cpu(session_token));
4403         rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,
4404                                 H_SESSION_ERR_DETECTED, session_token, 0, 0);
4405         if (rc) {
4406                 netdev_err(netdev, "Client initiated failover failed, rc %ld\n",
4407                            rc);
4408                 return -EINVAL;
4409         }
4410
4411         return count;
4412 }
4413
4414 static DEVICE_ATTR(failover, 0200, NULL, failover_store);
4415
4416 static unsigned long ibmvnic_get_desired_dma(struct vio_dev *vdev)
4417 {
4418         struct net_device *netdev = dev_get_drvdata(&vdev->dev);
4419         struct ibmvnic_adapter *adapter;
4420         struct iommu_table *tbl;
4421         unsigned long ret = 0;
4422         int i;
4423
4424         tbl = get_iommu_table_base(&vdev->dev);
4425
4426         /* netdev inits at probe time along with the structures we need below*/
4427         if (!netdev)
4428                 return IOMMU_PAGE_ALIGN(IBMVNIC_IO_ENTITLEMENT_DEFAULT, tbl);
4429
4430         adapter = netdev_priv(netdev);
4431
4432         ret += PAGE_SIZE; /* the crq message queue */
4433         ret += IOMMU_PAGE_ALIGN(sizeof(struct ibmvnic_statistics), tbl);
4434
4435         for (i = 0; i < adapter->req_tx_queues + adapter->req_rx_queues; i++)
4436                 ret += 4 * PAGE_SIZE; /* the scrq message queue */
4437
4438         for (i = 0; i < be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
4439              i++)
4440                 ret += adapter->rx_pool[i].size *
4441                     IOMMU_PAGE_ALIGN(adapter->rx_pool[i].buff_size, tbl);
4442
4443         return ret;
4444 }
4445
4446 static int ibmvnic_resume(struct device *dev)
4447 {
4448         struct net_device *netdev = dev_get_drvdata(dev);
4449         struct ibmvnic_adapter *adapter = netdev_priv(netdev);
4450
4451         if (adapter->state != VNIC_OPEN)
4452                 return 0;
4453
4454         tasklet_schedule(&adapter->tasklet);
4455
4456         return 0;
4457 }
4458
4459 static const struct vio_device_id ibmvnic_device_table[] = {
4460         {"network", "IBM,vnic"},
4461         {"", "" }
4462 };
4463 MODULE_DEVICE_TABLE(vio, ibmvnic_device_table);
4464
4465 static const struct dev_pm_ops ibmvnic_pm_ops = {
4466         .resume = ibmvnic_resume
4467 };
4468
4469 static struct vio_driver ibmvnic_driver = {
4470         .id_table       = ibmvnic_device_table,
4471         .probe          = ibmvnic_probe,
4472         .remove         = ibmvnic_remove,
4473         .get_desired_dma = ibmvnic_get_desired_dma,
4474         .name           = ibmvnic_driver_name,
4475         .pm             = &ibmvnic_pm_ops,
4476 };
4477
4478 /* module functions */
4479 static int __init ibmvnic_module_init(void)
4480 {
4481         pr_info("%s: %s %s\n", ibmvnic_driver_name, ibmvnic_driver_string,
4482                 IBMVNIC_DRIVER_VERSION);
4483
4484         return vio_register_driver(&ibmvnic_driver);
4485 }
4486
4487 static void __exit ibmvnic_module_exit(void)
4488 {
4489         vio_unregister_driver(&ibmvnic_driver);
4490 }
4491
4492 module_init(ibmvnic_module_init);
4493 module_exit(ibmvnic_module_exit);