drivers/net/ethernet/sfc/nic.c

   1 /****************************************************************************
   2  * Driver for Solarflare Solarstorm network controllers and boards
   3  * Copyright 2005-2006 Fen Systems Ltd.
   4  * Copyright 2006-2011 Solarflare Communications Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation, incorporated herein by reference.
   9  */
  10
  11 #include <linux/bitops.h>
  12 #include <linux/delay.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/pci.h>
  15 #include <linux/module.h>
  16 #include <linux/seq_file.h>
  17 #include "net_driver.h"
  18 #include "bitfield.h"
  19 #include "efx.h"
  20 #include "nic.h"
  21 #include "regs.h"
  22 #include "io.h"
  23 #include "workarounds.h"
  24
  25 /**************************************************************************
  26  *
  27  * Configurable values
  28  *
  29  **************************************************************************
  30  */
  31
  32 /* This is set to 16 for a good reason.  In summary, if larger than
  33  * 16, the descriptor cache holds more than a default socket
  34  * buffer's worth of packets (for UDP we can only have at most one
  35  * socket buffer's worth outstanding).  This combined with the fact
  36  * that we only get 1 TX event per descriptor cache means the NIC
  37  * goes idle.
  38  */
  39 #define TX_DC_ENTRIES 16
  40 #define TX_DC_ENTRIES_ORDER 1
  41
  42 #define RX_DC_ENTRIES 64
  43 #define RX_DC_ENTRIES_ORDER 3
  44
  45 /* If EFX_MAX_INT_ERRORS internal errors occur within
  46  * EFX_INT_ERROR_EXPIRE seconds, we consider the NIC broken and
  47  * disable it.
  48  */
  49 #define EFX_INT_ERROR_EXPIRE 3600
  50 #define EFX_MAX_INT_ERRORS 5
  51
  52 /* Depth of RX flush request fifo */
  53 #define EFX_RX_FLUSH_COUNT 4
  54
  55 /* Driver generated events */
  56 #define _EFX_CHANNEL_MAGIC_TEST         0x000101
  57 #define _EFX_CHANNEL_MAGIC_FILL         0x000102
  58 #define _EFX_CHANNEL_MAGIC_RX_DRAIN     0x000103
  59 #define _EFX_CHANNEL_MAGIC_TX_DRAIN     0x000104
  60
  61 #define _EFX_CHANNEL_MAGIC(_code, _data)        ((_code) << 8 | (_data))
  62 #define _EFX_CHANNEL_MAGIC_CODE(_magic)         ((_magic) >> 8)
  63
  64 #define EFX_CHANNEL_MAGIC_TEST(_channel)                                \
  65         _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_TEST, (_channel)->channel)
  66 #define EFX_CHANNEL_MAGIC_FILL(_rx_queue)                               \
  67         _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_FILL,                     \
  68                            efx_rx_queue_index(_rx_queue))
  69 #define EFX_CHANNEL_MAGIC_RX_DRAIN(_rx_queue)                           \
  70         _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_RX_DRAIN,                 \
  71                            efx_rx_queue_index(_rx_queue))
  72 #define EFX_CHANNEL_MAGIC_TX_DRAIN(_tx_queue)                           \
  73         _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_TX_DRAIN,                 \
  74                            (_tx_queue)->queue)
  75
  76 /**************************************************************************
  77  *
  78  * Solarstorm hardware access
  79  *
  80  **************************************************************************/
  81
  82 static inline void efx_write_buf_tbl(struct efx_nic *efx, efx_qword_t *value,
  83                                      unsigned int index)
  84 {
  85         efx_sram_writeq(efx, efx->membase + efx->type->buf_tbl_base,
  86                         value, index);
  87 }
  88
  89 /* Read the current event from the event queue */
  90 static inline efx_qword_t *efx_event(struct efx_channel *channel,
  91                                      unsigned int index)
  92 {
  93         return ((efx_qword_t *) (channel->eventq.addr)) +
  94                 (index & channel->eventq_mask);
  95 }
  96
  97 /* See if an event is present
  98  *
  99  * We check both the high and low dword of the event for all ones.  We
 100  * wrote all ones when we cleared the event, and no valid event can
 101  * have all ones in either its high or low dwords.  This approach is
 102  * robust against reordering.
 103  *
 104  * Note that using a single 64-bit comparison is incorrect; even
 105  * though the CPU read will be atomic, the DMA write may not be.
 106  */
 107 static inline int efx_event_present(efx_qword_t *event)
 108 {
 109         return !(EFX_DWORD_IS_ALL_ONES(event->dword[0]) |
 110                   EFX_DWORD_IS_ALL_ONES(event->dword[1]));
 111 }
 112
 113 static bool efx_masked_compare_oword(const efx_oword_t *a, const efx_oword_t *b,
 114                                      const efx_oword_t *mask)
 115 {
 116         return ((a->u64[0] ^ b->u64[0]) & mask->u64[0]) ||
 117                 ((a->u64[1] ^ b->u64[1]) & mask->u64[1]);
 118 }
 119
 120 int efx_nic_test_registers(struct efx_nic *efx,
 121                            const struct efx_nic_register_test *regs,
 122                            size_t n_regs)
 123 {
 124         unsigned address = 0, i, j;
 125         efx_oword_t mask, imask, original, reg, buf;
 126
 127         for (i = 0; i < n_regs; ++i) {
 128                 address = regs[i].address;
 129                 mask = imask = regs[i].mask;
 130                 EFX_INVERT_OWORD(imask);
 131
 132                 efx_reado(efx, &original, address);
 133
 134                 /* bit sweep on and off */
 135                 for (j = 0; j < 128; j++) {
 136                         if (!EFX_EXTRACT_OWORD32(mask, j, j))
 137                                 continue;
 138
 139                         /* Test this testable bit can be set in isolation */
 140                         EFX_AND_OWORD(reg, original, mask);
 141                         EFX_SET_OWORD32(reg, j, j, 1);
 142
 143                         efx_writeo(efx, &reg, address);
 144                         efx_reado(efx, &buf, address);
 145
 146                         if (efx_masked_compare_oword(&reg, &buf, &mask))
 147                                 goto fail;
 148
 149                         /* Test this testable bit can be cleared in isolation */
 150                         EFX_OR_OWORD(reg, original, mask);
 151                         EFX_SET_OWORD32(reg, j, j, 0);
 152
 153                         efx_writeo(efx, &reg, address);
 154                         efx_reado(efx, &buf, address);
 155
 156                         if (efx_masked_compare_oword(&reg, &buf, &mask))
 157                                 goto fail;
 158                 }
 159
 160                 efx_writeo(efx, &original, address);
 161         }
 162
 163         return 0;
 164
 165 fail:
 166         netif_err(efx, hw, efx->net_dev,
 167                   "wrote "EFX_OWORD_FMT" read "EFX_OWORD_FMT
 168                   " at address 0x%x mask "EFX_OWORD_FMT"\n", EFX_OWORD_VAL(reg),
 169                   EFX_OWORD_VAL(buf), address, EFX_OWORD_VAL(mask));
 170         return -EIO;
 171 }
 172
 173 /**************************************************************************
 174  *
 175  * Special buffer handling
 176  * Special buffers are used for event queues and the TX and RX
 177  * descriptor rings.
 178  *
 179  *************************************************************************/
 180
 181 /*
 182  * Initialise a special buffer
 183  *
 184  * This will define a buffer (previously allocated via
 185  * efx_alloc_special_buffer()) in the buffer table, allowing
 186  * it to be used for event queues, descriptor rings etc.
 187  */
 188 static void
 189 efx_init_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 190 {
 191         efx_qword_t buf_desc;
 192         unsigned int index;
 193         dma_addr_t dma_addr;
 194         int i;
 195
 196         EFX_BUG_ON_PARANOID(!buffer->addr);
 197
 198         /* Write buffer descriptors to NIC */
 199         for (i = 0; i < buffer->entries; i++) {
 200                 index = buffer->index + i;
 201                 dma_addr = buffer->dma_addr + (i * EFX_BUF_SIZE);
 202                 netif_dbg(efx, probe, efx->net_dev,
 203                           "mapping special buffer %d at %llx\n",
 204                           index, (unsigned long long)dma_addr);
 205                 EFX_POPULATE_QWORD_3(buf_desc,
 206                                      FRF_AZ_BUF_ADR_REGION, 0,
 207                                      FRF_AZ_BUF_ADR_FBUF, dma_addr >> 12,
 208                                      FRF_AZ_BUF_OWNER_ID_FBUF, 0);
 209                 efx_write_buf_tbl(efx, &buf_desc, index);
 210         }
 211 }
 212
 213 /* Unmaps a buffer and clears the buffer table entries */
 214 static void
 215 efx_fini_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 216 {
 217         efx_oword_t buf_tbl_upd;
 218         unsigned int start = buffer->index;
 219         unsigned int end = (buffer->index + buffer->entries - 1);
 220
 221         if (!buffer->entries)
 222                 return;
 223
 224         netif_dbg(efx, hw, efx->net_dev, "unmapping special buffers %d-%d\n",
 225                   buffer->index, buffer->index + buffer->entries - 1);
 226
 227         EFX_POPULATE_OWORD_4(buf_tbl_upd,
 228                              FRF_AZ_BUF_UPD_CMD, 0,
 229                              FRF_AZ_BUF_CLR_CMD, 1,
 230                              FRF_AZ_BUF_CLR_END_ID, end,
 231                              FRF_AZ_BUF_CLR_START_ID, start);
 232         efx_writeo(efx, &buf_tbl_upd, FR_AZ_BUF_TBL_UPD);
 233 }
 234
 235 /*
 236  * Allocate a new special buffer
 237  *
 238  * This allocates memory for a new buffer, clears it and allocates a
 239  * new buffer ID range.  It does not write into the buffer table.
 240  *
 241  * This call will allocate 4KB buffers, since 8KB buffers can't be
 242  * used for event queues and descriptor rings.
 243  */
 244 static int efx_alloc_special_buffer(struct efx_nic *efx,
 245                                     struct efx_special_buffer *buffer,
 246                                     unsigned int len)
 247 {
 248         len = ALIGN(len, EFX_BUF_SIZE);
 249
 250         buffer->addr = dma_alloc_coherent(&efx->pci_dev->dev, len,
 251                                           &buffer->dma_addr, GFP_KERNEL);
 252         if (!buffer->addr)
 253                 return -ENOMEM;
 254         buffer->len = len;
 255         buffer->entries = len / EFX_BUF_SIZE;
 256         BUG_ON(buffer->dma_addr & (EFX_BUF_SIZE - 1));
 257
 258         /* All zeros is a potentially valid event so memset to 0xff */
 259         memset(buffer->addr, 0xff, len);
 260
 261         /* Select new buffer ID */
 262         buffer->index = efx->next_buffer_table;
 263         efx->next_buffer_table += buffer->entries;
 264 #ifdef CONFIG_SFC_SRIOV
 265         BUG_ON(efx_sriov_enabled(efx) &&
 266                efx->vf_buftbl_base < efx->next_buffer_table);
 267 #endif
 268
 269         netif_dbg(efx, probe, efx->net_dev,
 270                   "allocating special buffers %d-%d at %llx+%x "
 271                   "(virt %p phys %llx)\n", buffer->index,
 272                   buffer->index + buffer->entries - 1,
 273                   (u64)buffer->dma_addr, len,
 274                   buffer->addr, (u64)virt_to_phys(buffer->addr));
 275
 276         return 0;
 277 }
 278
 279 static void
 280 efx_free_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 281 {
 282         if (!buffer->addr)
 283                 return;
 284
 285         netif_dbg(efx, hw, efx->net_dev,
 286                   "deallocating special buffers %d-%d at %llx+%x "
 287                   "(virt %p phys %llx)\n", buffer->index,
 288                   buffer->index + buffer->entries - 1,
 289                   (u64)buffer->dma_addr, buffer->len,
 290                   buffer->addr, (u64)virt_to_phys(buffer->addr));
 291
 292         dma_free_coherent(&efx->pci_dev->dev, buffer->len, buffer->addr,
 293                           buffer->dma_addr);
 294         buffer->addr = NULL;
 295         buffer->entries = 0;
 296 }
 297
 298 /**************************************************************************
 299  *
 300  * Generic buffer handling
 301  * These buffers are used for interrupt status, MAC stats, etc.
 302  *
 303  **************************************************************************/
 304
 305 int efx_nic_alloc_buffer(struct efx_nic *efx, struct efx_buffer *buffer,
 306                          unsigned int len)
 307 {
 308         buffer->addr = dma_alloc_coherent(&efx->pci_dev->dev, len,
 309                                           &buffer->dma_addr, GFP_ATOMIC);
 310         if (!buffer->addr)
 311                 return -ENOMEM;
 312         buffer->len = len;
 313         memset(buffer->addr, 0, len);
 314         return 0;
 315 }
 316
 317 void efx_nic_free_buffer(struct efx_nic *efx, struct efx_buffer *buffer)
 318 {
 319         if (buffer->addr) {
 320                 dma_free_coherent(&efx->pci_dev->dev, buffer->len,
 321                                   buffer->addr, buffer->dma_addr);
 322                 buffer->addr = NULL;
 323         }
 324 }
 325
 326 /**************************************************************************
 327  *
 328  * TX path
 329  *
 330  **************************************************************************/
 331
 332 /* Returns a pointer to the specified transmit descriptor in the TX
 333  * descriptor queue belonging to the specified channel.
 334  */
 335 static inline efx_qword_t *
 336 efx_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index)
 337 {
 338         return ((efx_qword_t *) (tx_queue->txd.addr)) + index;
 339 }
 340
 341 /* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */
 342 static inline void efx_notify_tx_desc(struct efx_tx_queue *tx_queue)
 343 {
 344         unsigned write_ptr;
 345         efx_dword_t reg;
 346
 347         write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
 348         EFX_POPULATE_DWORD_1(reg, FRF_AZ_TX_DESC_WPTR_DWORD, write_ptr);
 349         efx_writed_page(tx_queue->efx, &reg,
 350                         FR_AZ_TX_DESC_UPD_DWORD_P0, tx_queue->queue);
 351 }
 352
 353 /* Write pointer and first descriptor for TX descriptor ring */
 354 static inline void efx_push_tx_desc(struct efx_tx_queue *tx_queue,
 355                                     const efx_qword_t *txd)
 356 {
 357         unsigned write_ptr;
 358         efx_oword_t reg;
 359
 360         BUILD_BUG_ON(FRF_AZ_TX_DESC_LBN != 0);
 361         BUILD_BUG_ON(FR_AA_TX_DESC_UPD_KER != FR_BZ_TX_DESC_UPD_P0);
 362
 363         write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
 364         EFX_POPULATE_OWORD_2(reg, FRF_AZ_TX_DESC_PUSH_CMD, true,
 365                              FRF_AZ_TX_DESC_WPTR, write_ptr);
 366         reg.qword[0] = *txd;
 367         efx_writeo_page(tx_queue->efx, &reg,
 368                         FR_BZ_TX_DESC_UPD_P0, tx_queue->queue);
 369 }
 370
 371 static inline bool
 372 efx_may_push_tx_desc(struct efx_tx_queue *tx_queue, unsigned int write_count)
 373 {
 374         unsigned empty_read_count = ACCESS_ONCE(tx_queue->empty_read_count);
 375
 376         if (empty_read_count == 0)
 377                 return false;
 378
 379         tx_queue->empty_read_count = 0;
 380         return ((empty_read_count ^ write_count) & ~EFX_EMPTY_COUNT_VALID) == 0;
 381 }
 382
 383 /* For each entry inserted into the software descriptor ring, create a
 384  * descriptor in the hardware TX descriptor ring (in host memory), and
 385  * write a doorbell.
 386  */
 387 void efx_nic_push_buffers(struct efx_tx_queue *tx_queue)
 388 {
 389
 390         struct efx_tx_buffer *buffer;
 391         efx_qword_t *txd;
 392         unsigned write_ptr;
 393         unsigned old_write_count = tx_queue->write_count;
 394
 395         BUG_ON(tx_queue->write_count == tx_queue->insert_count);
 396
 397         do {
 398                 write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
 399                 buffer = &tx_queue->buffer[write_ptr];
 400                 txd = efx_tx_desc(tx_queue, write_ptr);
 401                 ++tx_queue->write_count;
 402
 403                 /* Create TX descriptor ring entry */
 404                 BUILD_BUG_ON(EFX_TX_BUF_CONT != 1);
 405                 EFX_POPULATE_QWORD_4(*txd,
 406                                      FSF_AZ_TX_KER_CONT,
 407                                      buffer->flags & EFX_TX_BUF_CONT,
 408                                      FSF_AZ_TX_KER_BYTE_COUNT, buffer->len,
 409                                      FSF_AZ_TX_KER_BUF_REGION, 0,
 410                                      FSF_AZ_TX_KER_BUF_ADDR, buffer->dma_addr);
 411         } while (tx_queue->write_count != tx_queue->insert_count);
 412
 413         wmb(); /* Ensure descriptors are written before they are fetched */
 414
 415         if (efx_may_push_tx_desc(tx_queue, old_write_count)) {
 416                 txd = efx_tx_desc(tx_queue,
 417                                   old_write_count & tx_queue->ptr_mask);
 418                 efx_push_tx_desc(tx_queue, txd);
 419                 ++tx_queue->pushes;
 420         } else {
 421                 efx_notify_tx_desc(tx_queue);
 422         }
 423 }
 424
 425 /* Allocate hardware resources for a TX queue */
 426 int efx_nic_probe_tx(struct efx_tx_queue *tx_queue)
 427 {
 428         struct efx_nic *efx = tx_queue->efx;
 429         unsigned entries;
 430
 431         entries = tx_queue->ptr_mask + 1;
 432         return efx_alloc_special_buffer(efx, &tx_queue->txd,
 433                                         entries * sizeof(efx_qword_t));
 434 }
 435
 436 void efx_nic_init_tx(struct efx_tx_queue *tx_queue)
 437 {
 438         struct efx_nic *efx = tx_queue->efx;
 439         efx_oword_t reg;
 440
 441         /* Pin TX descriptor ring */
 442         efx_init_special_buffer(efx, &tx_queue->txd);
 443
 444         /* Push TX descriptor ring to card */
 445         EFX_POPULATE_OWORD_10(reg,
 446                               FRF_AZ_TX_DESCQ_EN, 1,
 447                               FRF_AZ_TX_ISCSI_DDIG_EN, 0,
 448                               FRF_AZ_TX_ISCSI_HDIG_EN, 0,
 449                               FRF_AZ_TX_DESCQ_BUF_BASE_ID, tx_queue->txd.index,
 450                               FRF_AZ_TX_DESCQ_EVQ_ID,
 451                               tx_queue->channel->channel,
 452                               FRF_AZ_TX_DESCQ_OWNER_ID, 0,
 453                               FRF_AZ_TX_DESCQ_LABEL, tx_queue->queue,
 454                               FRF_AZ_TX_DESCQ_SIZE,
 455                               __ffs(tx_queue->txd.entries),
 456                               FRF_AZ_TX_DESCQ_TYPE, 0,
 457                               FRF_BZ_TX_NON_IP_DROP_DIS, 1);
 458
 459         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
 460                 int csum = tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD;
 461                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_IP_CHKSM_DIS, !csum);
 462                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_TCP_CHKSM_DIS,
 463                                     !csum);
 464         }
 465
 466         efx_writeo_table(efx, &reg, efx->type->txd_ptr_tbl_base,
 467                          tx_queue->queue);
 468
 469         if (efx_nic_rev(efx) < EFX_REV_FALCON_B0) {
 470                 /* Only 128 bits in this register */
 471                 BUILD_BUG_ON(EFX_MAX_TX_QUEUES > 128);
 472
 473                 efx_reado(efx, &reg, FR_AA_TX_CHKSM_CFG);
 474                 if (tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD)
 475                         __clear_bit_le(tx_queue->queue, &reg);
 476                 else
 477                         __set_bit_le(tx_queue->queue, &reg);
 478                 efx_writeo(efx, &reg, FR_AA_TX_CHKSM_CFG);
 479         }
 480
 481         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
 482                 EFX_POPULATE_OWORD_1(reg,
 483                                      FRF_BZ_TX_PACE,
 484                                      (tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI) ?
 485                                      FFE_BZ_TX_PACE_OFF :
 486                                      FFE_BZ_TX_PACE_RESERVED);
 487                 efx_writeo_table(efx, &reg, FR_BZ_TX_PACE_TBL,
 488                                  tx_queue->queue);
 489         }
 490 }
 491
 492 static void efx_flush_tx_queue(struct efx_tx_queue *tx_queue)
 493 {
 494         struct efx_nic *efx = tx_queue->efx;
 495         efx_oword_t tx_flush_descq;
 496
 497         EFX_POPULATE_OWORD_2(tx_flush_descq,
 498                              FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
 499                              FRF_AZ_TX_FLUSH_DESCQ, tx_queue->queue);
 500         efx_writeo(efx, &tx_flush_descq, FR_AZ_TX_FLUSH_DESCQ);
 501 }
 502
 503 void efx_nic_fini_tx(struct efx_tx_queue *tx_queue)
 504 {
 505         struct efx_nic *efx = tx_queue->efx;
 506         efx_oword_t tx_desc_ptr;
 507
 508         /* Remove TX descriptor ring from card */
 509         EFX_ZERO_OWORD(tx_desc_ptr);
 510         efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base,
 511                          tx_queue->queue);
 512
 513         /* Unpin TX descriptor ring */
 514         efx_fini_special_buffer(efx, &tx_queue->txd);
 515 }
 516
 517 /* Free buffers backing TX queue */
 518 void efx_nic_remove_tx(struct efx_tx_queue *tx_queue)
 519 {
 520         efx_free_special_buffer(tx_queue->efx, &tx_queue->txd);
 521 }
 522
 523 /**************************************************************************
 524  *
 525  * RX path
 526  *
 527  **************************************************************************/
 528
 529 /* Returns a pointer to the specified descriptor in the RX descriptor queue */
 530 static inline efx_qword_t *
 531 efx_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index)
 532 {
 533         return ((efx_qword_t *) (rx_queue->rxd.addr)) + index;
 534 }
 535
 536 /* This creates an entry in the RX descriptor queue */
 537 static inline void
 538 efx_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned index)
 539 {
 540         struct efx_rx_buffer *rx_buf;
 541         efx_qword_t *rxd;
 542
 543         rxd = efx_rx_desc(rx_queue, index);
 544         rx_buf = efx_rx_buffer(rx_queue, index);
 545         EFX_POPULATE_QWORD_3(*rxd,
 546                              FSF_AZ_RX_KER_BUF_SIZE,
 547                              rx_buf->len -
 548                              rx_queue->efx->type->rx_buffer_padding,
 549                              FSF_AZ_RX_KER_BUF_REGION, 0,
 550                              FSF_AZ_RX_KER_BUF_ADDR, rx_buf->dma_addr);
 551 }
 552
 553 /* This writes to the RX_DESC_WPTR register for the specified receive
 554  * descriptor ring.
 555  */
 556 void efx_nic_notify_rx_desc(struct efx_rx_queue *rx_queue)
 557 {
 558         struct efx_nic *efx = rx_queue->efx;
 559         efx_dword_t reg;
 560         unsigned write_ptr;
 561
 562         while (rx_queue->notified_count != rx_queue->added_count) {
 563                 efx_build_rx_desc(
 564                         rx_queue,
 565                         rx_queue->notified_count & rx_queue->ptr_mask);
 566                 ++rx_queue->notified_count;
 567         }
 568
 569         wmb();
 570         write_ptr = rx_queue->added_count & rx_queue->ptr_mask;
 571         EFX_POPULATE_DWORD_1(reg, FRF_AZ_RX_DESC_WPTR_DWORD, write_ptr);
 572         efx_writed_page(efx, &reg, FR_AZ_RX_DESC_UPD_DWORD_P0,
 573                         efx_rx_queue_index(rx_queue));
 574 }
 575
 576 int efx_nic_probe_rx(struct efx_rx_queue *rx_queue)
 577 {
 578         struct efx_nic *efx = rx_queue->efx;
 579         unsigned entries;
 580
 581         entries = rx_queue->ptr_mask + 1;
 582         return efx_alloc_special_buffer(efx, &rx_queue->rxd,
 583                                         entries * sizeof(efx_qword_t));
 584 }
 585
 586 void efx_nic_init_rx(struct efx_rx_queue *rx_queue)
 587 {
 588         efx_oword_t rx_desc_ptr;
 589         struct efx_nic *efx = rx_queue->efx;
 590         bool is_b0 = efx_nic_rev(efx) >= EFX_REV_FALCON_B0;
 591         bool iscsi_digest_en = is_b0;
 592
 593         netif_dbg(efx, hw, efx->net_dev,
 594                   "RX queue %d ring in special buffers %d-%d\n",
 595                   efx_rx_queue_index(rx_queue), rx_queue->rxd.index,
 596                   rx_queue->rxd.index + rx_queue->rxd.entries - 1);
 597
 598         /* Pin RX descriptor ring */
 599         efx_init_special_buffer(efx, &rx_queue->rxd);
 600
 601         /* Push RX descriptor ring to card */
 602         EFX_POPULATE_OWORD_10(rx_desc_ptr,
 603                               FRF_AZ_RX_ISCSI_DDIG_EN, iscsi_digest_en,
 604                               FRF_AZ_RX_ISCSI_HDIG_EN, iscsi_digest_en,
 605                               FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index,
 606                               FRF_AZ_RX_DESCQ_EVQ_ID,
 607                               efx_rx_queue_channel(rx_queue)->channel,
 608                               FRF_AZ_RX_DESCQ_OWNER_ID, 0,
 609                               FRF_AZ_RX_DESCQ_LABEL,
 610                               efx_rx_queue_index(rx_queue),
 611                               FRF_AZ_RX_DESCQ_SIZE,
 612                               __ffs(rx_queue->rxd.entries),
 613                               FRF_AZ_RX_DESCQ_TYPE, 0 /* kernel queue */ ,
 614                               /* For >=B0 this is scatter so disable */
 615                               FRF_AZ_RX_DESCQ_JUMBO, !is_b0,
 616                               FRF_AZ_RX_DESCQ_EN, 1);
 617         efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
 618                          efx_rx_queue_index(rx_queue));
 619 }
 620
 621 static void efx_flush_rx_queue(struct efx_rx_queue *rx_queue)
 622 {
 623         struct efx_nic *efx = rx_queue->efx;
 624         efx_oword_t rx_flush_descq;
 625
 626         EFX_POPULATE_OWORD_2(rx_flush_descq,
 627                              FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
 628                              FRF_AZ_RX_FLUSH_DESCQ,
 629                              efx_rx_queue_index(rx_queue));
 630         efx_writeo(efx, &rx_flush_descq, FR_AZ_RX_FLUSH_DESCQ);
 631 }
 632
 633 void efx_nic_fini_rx(struct efx_rx_queue *rx_queue)
 634 {
 635         efx_oword_t rx_desc_ptr;
 636         struct efx_nic *efx = rx_queue->efx;
 637
 638         /* Remove RX descriptor ring from card */
 639         EFX_ZERO_OWORD(rx_desc_ptr);
 640         efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
 641                          efx_rx_queue_index(rx_queue));
 642
 643         /* Unpin RX descriptor ring */
 644         efx_fini_special_buffer(efx, &rx_queue->rxd);
 645 }
 646
 647 /* Free buffers backing RX queue */
 648 void efx_nic_remove_rx(struct efx_rx_queue *rx_queue)
 649 {
 650         efx_free_special_buffer(rx_queue->efx, &rx_queue->rxd);
 651 }
 652
 653 /**************************************************************************
 654  *
 655  * Flush handling
 656  *
 657  **************************************************************************/
 658
 659 /* efx_nic_flush_queues() must be woken up when all flushes are completed,
 660  * or more RX flushes can be kicked off.
 661  */
 662 static bool efx_flush_wake(struct efx_nic *efx)
 663 {
 664         /* Ensure that all updates are visible to efx_nic_flush_queues() */
 665         smp_mb();
 666
 667         return (atomic_read(&efx->drain_pending) == 0 ||
 668                 (atomic_read(&efx->rxq_flush_outstanding) < EFX_RX_FLUSH_COUNT
 669                  && atomic_read(&efx->rxq_flush_pending) > 0));
 670 }
 671
 672 /* Flush all the transmit queues, and continue flushing receive queues until
 673  * they're all flushed. Wait for the DRAIN events to be recieved so that there
 674  * are no more RX and TX events left on any channel. */
 675 int efx_nic_flush_queues(struct efx_nic *efx)
 676 {
 677         unsigned timeout = msecs_to_jiffies(5000); /* 5s for all flushes and drains */
 678         struct efx_channel *channel;
 679         struct efx_rx_queue *rx_queue;
 680         struct efx_tx_queue *tx_queue;
 681         int rc = 0;
 682
 683         efx->fc_disable++;
 684         efx->type->prepare_flush(efx);
 685
 686         efx_for_each_channel(channel, efx) {
 687                 efx_for_each_channel_tx_queue(tx_queue, channel) {
 688                         atomic_inc(&efx->drain_pending);
 689                         efx_flush_tx_queue(tx_queue);
 690                 }
 691                 efx_for_each_channel_rx_queue(rx_queue, channel) {
 692                         atomic_inc(&efx->drain_pending);
 693                         rx_queue->flush_pending = true;
 694                         atomic_inc(&efx->rxq_flush_pending);
 695                 }
 696         }
 697
 698         while (timeout && atomic_read(&efx->drain_pending) > 0) {
 699                 /* If SRIOV is enabled, then offload receive queue flushing to
 700                  * the firmware (though we will still have to poll for
 701                  * completion). If that fails, fall back to the old scheme.
 702                  */
 703                 if (efx_sriov_enabled(efx)) {
 704                         rc = efx_mcdi_flush_rxqs(efx);
 705                         if (!rc)
 706                                 goto wait;
 707                 }
 708
 709                 /* The hardware supports four concurrent rx flushes, each of
 710                  * which may need to be retried if there is an outstanding
 711                  * descriptor fetch
 712                  */
 713                 efx_for_each_channel(channel, efx) {
 714                         efx_for_each_channel_rx_queue(rx_queue, channel) {
 715                                 if (atomic_read(&efx->rxq_flush_outstanding) >=
 716                                     EFX_RX_FLUSH_COUNT)
 717                                         break;
 718
 719                                 if (rx_queue->flush_pending) {
 720                                         rx_queue->flush_pending = false;
 721                                         atomic_dec(&efx->rxq_flush_pending);
 722                                         atomic_inc(&efx->rxq_flush_outstanding);
 723                                         efx_flush_rx_queue(rx_queue);
 724                                 }
 725                         }
 726                 }
 727
 728         wait:
 729                 timeout = wait_event_timeout(efx->flush_wq, efx_flush_wake(efx),
 730                                              timeout);
 731         }
 732
 733         if (atomic_read(&efx->drain_pending)) {
 734                 netif_err(efx, hw, efx->net_dev, "failed to flush %d queues "
 735                           "(rx %d+%d)\n", atomic_read(&efx->drain_pending),
 736                           atomic_read(&efx->rxq_flush_outstanding),
 737                           atomic_read(&efx->rxq_flush_pending));
 738                 rc = -ETIMEDOUT;
 739
 740                 atomic_set(&efx->drain_pending, 0);
 741                 atomic_set(&efx->rxq_flush_pending, 0);
 742                 atomic_set(&efx->rxq_flush_outstanding, 0);
 743         }
 744
 745         efx->fc_disable--;
 746
 747         return rc;
 748 }
 749
 750 /**************************************************************************
 751  *
 752  * Event queue processing
 753  * Event queues are processed by per-channel tasklets.
 754  *
 755  **************************************************************************/
 756
 757 /* Update a channel's event queue's read pointer (RPTR) register
 758  *
 759  * This writes the EVQ_RPTR_REG register for the specified channel's
 760  * event queue.
 761  */
 762 void efx_nic_eventq_read_ack(struct efx_channel *channel)
 763 {
 764         efx_dword_t reg;
 765         struct efx_nic *efx = channel->efx;
 766
 767         EFX_POPULATE_DWORD_1(reg, FRF_AZ_EVQ_RPTR,
 768                              channel->eventq_read_ptr & channel->eventq_mask);
 769         efx_writed_table(efx, &reg, efx->type->evq_rptr_tbl_base,
 770                          channel->channel);
 771 }
 772
 773 /* Use HW to insert a SW defined event */
 774 void efx_generate_event(struct efx_nic *efx, unsigned int evq,
 775                         efx_qword_t *event)
 776 {
 777         efx_oword_t drv_ev_reg;
 778
 779         BUILD_BUG_ON(FRF_AZ_DRV_EV_DATA_LBN != 0 ||
 780                      FRF_AZ_DRV_EV_DATA_WIDTH != 64);
 781         drv_ev_reg.u32[0] = event->u32[0];
 782         drv_ev_reg.u32[1] = event->u32[1];
 783         drv_ev_reg.u32[2] = 0;
 784         drv_ev_reg.u32[3] = 0;
 785         EFX_SET_OWORD_FIELD(drv_ev_reg, FRF_AZ_DRV_EV_QID, evq);
 786         efx_writeo(efx, &drv_ev_reg, FR_AZ_DRV_EV);
 787 }
 788
 789 static void efx_magic_event(struct efx_channel *channel, u32 magic)
 790 {
 791         efx_qword_t event;
 792
 793         EFX_POPULATE_QWORD_2(event, FSF_AZ_EV_CODE,
 794                              FSE_AZ_EV_CODE_DRV_GEN_EV,
 795                              FSF_AZ_DRV_GEN_EV_MAGIC, magic);
 796         efx_generate_event(channel->efx, channel->channel, &event);
 797 }
 798
 799 /* Handle a transmit completion event
 800  *
 801  * The NIC batches TX completion events; the message we receive is of
 802  * the form "complete all TX events up to this index".
 803  */
 804 static int
 805 efx_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)
 806 {
 807         unsigned int tx_ev_desc_ptr;
 808         unsigned int tx_ev_q_label;
 809         struct efx_tx_queue *tx_queue;
 810         struct efx_nic *efx = channel->efx;
 811         int tx_packets = 0;
 812
 813         if (unlikely(ACCESS_ONCE(efx->reset_pending)))
 814                 return 0;
 815
 816         if (likely(EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_COMP))) {
 817                 /* Transmit completion */
 818                 tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR);
 819                 tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
 820                 tx_queue = efx_channel_get_tx_queue(
 821                         channel, tx_ev_q_label % EFX_TXQ_TYPES);
 822                 tx_packets = ((tx_ev_desc_ptr - tx_queue->read_count) &
 823                               tx_queue->ptr_mask);
 824                 efx_xmit_done(tx_queue, tx_ev_desc_ptr);
 825         } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) {
 826                 /* Rewrite the FIFO write pointer */
 827                 tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
 828                 tx_queue = efx_channel_get_tx_queue(
 829                         channel, tx_ev_q_label % EFX_TXQ_TYPES);
 830
 831                 netif_tx_lock(efx->net_dev);
 832                 efx_notify_tx_desc(tx_queue);
 833                 netif_tx_unlock(efx->net_dev);
 834         } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_PKT_ERR) &&
 835                    EFX_WORKAROUND_10727(efx)) {
 836                 efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
 837         } else {
 838                 netif_err(efx, tx_err, efx->net_dev,
 839                           "channel %d unexpected TX event "
 840                           EFX_QWORD_FMT"\n", channel->channel,
 841                           EFX_QWORD_VAL(*event));
 842         }
 843
 844         return tx_packets;
 845 }
 846
 847 /* Detect errors included in the rx_evt_pkt_ok bit. */
 848 static u16 efx_handle_rx_not_ok(struct efx_rx_queue *rx_queue,
 849                                 const efx_qword_t *event)
 850 {
 851         struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
 852         struct efx_nic *efx = rx_queue->efx;
 853         bool rx_ev_buf_owner_id_err, rx_ev_ip_hdr_chksum_err;
 854         bool rx_ev_tcp_udp_chksum_err, rx_ev_eth_crc_err;
 855         bool rx_ev_frm_trunc, rx_ev_drib_nib, rx_ev_tobe_disc;
 856         bool rx_ev_other_err, rx_ev_pause_frm;
 857         bool rx_ev_hdr_type, rx_ev_mcast_pkt;
 858         unsigned rx_ev_pkt_type;
 859
 860         rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
 861         rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
 862         rx_ev_tobe_disc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_TOBE_DISC);
 863         rx_ev_pkt_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_TYPE);
 864         rx_ev_buf_owner_id_err = EFX_QWORD_FIELD(*event,
 865                                                  FSF_AZ_RX_EV_BUF_OWNER_ID_ERR);
 866         rx_ev_ip_hdr_chksum_err = EFX_QWORD_FIELD(*event,
 867                                                   FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR);
 868         rx_ev_tcp_udp_chksum_err = EFX_QWORD_FIELD(*event,
 869                                                    FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR);
 870         rx_ev_eth_crc_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_ETH_CRC_ERR);
 871         rx_ev_frm_trunc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_FRM_TRUNC);
 872         rx_ev_drib_nib = ((efx_nic_rev(efx) >= EFX_REV_FALCON_B0) ?
 873                           0 : EFX_QWORD_FIELD(*event, FSF_AA_RX_EV_DRIB_NIB));
 874         rx_ev_pause_frm = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PAUSE_FRM_ERR);
 875
 876         /* Every error apart from tobe_disc and pause_frm */
 877         rx_ev_other_err = (rx_ev_drib_nib | rx_ev_tcp_udp_chksum_err |
 878                            rx_ev_buf_owner_id_err | rx_ev_eth_crc_err |
 879                            rx_ev_frm_trunc | rx_ev_ip_hdr_chksum_err);
 880
 881         /* Count errors that are not in MAC stats.  Ignore expected
 882          * checksum errors during self-test. */
 883         if (rx_ev_frm_trunc)
 884                 ++channel->n_rx_frm_trunc;
 885         else if (rx_ev_tobe_disc)
 886                 ++channel->n_rx_tobe_disc;
 887         else if (!efx->loopback_selftest) {
 888                 if (rx_ev_ip_hdr_chksum_err)
 889                         ++channel->n_rx_ip_hdr_chksum_err;
 890                 else if (rx_ev_tcp_udp_chksum_err)
 891                         ++channel->n_rx_tcp_udp_chksum_err;
 892         }
 893
 894         /* TOBE_DISC is expected on unicast mismatches; don't print out an
 895          * error message.  FRM_TRUNC indicates RXDP dropped the packet due
 896          * to a FIFO overflow.
 897          */
 898 #ifdef DEBUG
 899         if (rx_ev_other_err && net_ratelimit()) {
 900                 netif_dbg(efx, rx_err, efx->net_dev,
 901                           " RX queue %d unexpected RX event "
 902                           EFX_QWORD_FMT "%s%s%s%s%s%s%s%s\n",
 903                           efx_rx_queue_index(rx_queue), EFX_QWORD_VAL(*event),
 904                           rx_ev_buf_owner_id_err ? " [OWNER_ID_ERR]" : "",
 905                           rx_ev_ip_hdr_chksum_err ?
 906                           " [IP_HDR_CHKSUM_ERR]" : "",
 907                           rx_ev_tcp_udp_chksum_err ?
 908                           " [TCP_UDP_CHKSUM_ERR]" : "",
 909                           rx_ev_eth_crc_err ? " [ETH_CRC_ERR]" : "",
 910                           rx_ev_frm_trunc ? " [FRM_TRUNC]" : "",
 911                           rx_ev_drib_nib ? " [DRIB_NIB]" : "",
 912                           rx_ev_tobe_disc ? " [TOBE_DISC]" : "",
 913                           rx_ev_pause_frm ? " [PAUSE]" : "");
 914         }
 915 #endif
 916
 917         /* The frame must be discarded if any of these are true. */
 918         return (rx_ev_eth_crc_err | rx_ev_frm_trunc | rx_ev_drib_nib |
 919                 rx_ev_tobe_disc | rx_ev_pause_frm) ?
 920                 EFX_RX_PKT_DISCARD : 0;
 921 }
 922
 923 /* Handle receive events that are not in-order. */
 924 static void
 925 efx_handle_rx_bad_index(struct efx_rx_queue *rx_queue, unsigned index)
 926 {
 927         struct efx_nic *efx = rx_queue->efx;
 928         unsigned expected, dropped;
 929
 930         expected = rx_queue->removed_count & rx_queue->ptr_mask;
 931         dropped = (index - expected) & rx_queue->ptr_mask;
 932         netif_info(efx, rx_err, efx->net_dev,
 933                    "dropped %d events (index=%d expected=%d)\n",
 934                    dropped, index, expected);
 935
 936         efx_schedule_reset(efx, EFX_WORKAROUND_5676(efx) ?
 937                            RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE);
 938 }
 939
 940 /* Handle a packet received event
 941  *
 942  * The NIC gives a "discard" flag if it's a unicast packet with the
 943  * wrong destination address
 944  * Also "is multicast" and "matches multicast filter" flags can be used to
 945  * discard non-matching multicast packets.
 946  */
 947 static void
 948 efx_handle_rx_event(struct efx_channel *channel, const efx_qword_t *event)
 949 {
 950         unsigned int rx_ev_desc_ptr, rx_ev_byte_cnt;
 951         unsigned int rx_ev_hdr_type, rx_ev_mcast_pkt;
 952         unsigned expected_ptr;
 953         bool rx_ev_pkt_ok;
 954         u16 flags;
 955         struct efx_rx_queue *rx_queue;
 956         struct efx_nic *efx = channel->efx;
 957
 958         if (unlikely(ACCESS_ONCE(efx->reset_pending)))
 959                 return;
 960
 961         /* Basic packet information */
 962         rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT);
 963         rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK);
 964         rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
 965         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT));
 966         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP) != 1);
 967         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_Q_LABEL) !=
 968                 channel->channel);
 969
 970         rx_queue = efx_channel_get_rx_queue(channel);
 971
 972         rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_DESC_PTR);
 973         expected_ptr = rx_queue->removed_count & rx_queue->ptr_mask;
 974         if (unlikely(rx_ev_desc_ptr != expected_ptr))
 975                 efx_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr);
 976
 977         if (likely(rx_ev_pkt_ok)) {
 978                 /* If packet is marked as OK and packet type is TCP/IP or
 979                  * UDP/IP, then we can rely on the hardware checksum.
 980                  */
 981                 flags = (rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_TCP ||
 982                          rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_UDP) ?
 983                         EFX_RX_PKT_CSUMMED : 0;
 984         } else {
 985                 flags = efx_handle_rx_not_ok(rx_queue, event);
 986         }
 987
 988         /* Detect multicast packets that didn't match the filter */
 989         rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
 990         if (rx_ev_mcast_pkt) {
 991                 unsigned int rx_ev_mcast_hash_match =
 992                         EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_HASH_MATCH);
 993
 994                 if (unlikely(!rx_ev_mcast_hash_match)) {
 995                         ++channel->n_rx_mcast_mismatch;
 996                         flags |= EFX_RX_PKT_DISCARD;
 997                 }
 998         }
 999
1000         channel->irq_mod_score += 2;
1001
1002         /* Handle received packet */
1003         efx_rx_packet(rx_queue, rx_ev_desc_ptr, rx_ev_byte_cnt, flags);
1004 }
1005
1006 /* If this flush done event corresponds to a &struct efx_tx_queue, then
1007  * send an %EFX_CHANNEL_MAGIC_TX_DRAIN event to drain the event queue
1008  * of all transmit completions.
1009  */
1010 static void
1011 efx_handle_tx_flush_done(struct efx_nic *efx, efx_qword_t *event)
1012 {
1013         struct efx_tx_queue *tx_queue;
1014         int qid;
1015
1016         qid = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
1017         if (qid < EFX_TXQ_TYPES * efx->n_tx_channels) {
1018                 tx_queue = efx_get_tx_queue(efx, qid / EFX_TXQ_TYPES,
1019                                             qid % EFX_TXQ_TYPES);
1020
1021                 efx_magic_event(tx_queue->channel,
1022                                 EFX_CHANNEL_MAGIC_TX_DRAIN(tx_queue));
1023         }
1024 }
1025
1026 /* If this flush done event corresponds to a &struct efx_rx_queue: If the flush
1027  * was succesful then send an %EFX_CHANNEL_MAGIC_RX_DRAIN, otherwise add
1028  * the RX queue back to the mask of RX queues in need of flushing.
1029  */
1030 static void
1031 efx_handle_rx_flush_done(struct efx_nic *efx, efx_qword_t *event)
1032 {
1033         struct efx_channel *channel;
1034         struct efx_rx_queue *rx_queue;
1035         int qid;
1036         bool failed;
1037
1038         qid = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
1039         failed = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
1040         if (qid >= efx->n_channels)
1041                 return;
1042         channel = efx_get_channel(efx, qid);
1043         if (!efx_channel_has_rx_queue(channel))
1044                 return;
1045         rx_queue = efx_channel_get_rx_queue(channel);
1046
1047         if (failed) {
1048                 netif_info(efx, hw, efx->net_dev,
1049                            "RXQ %d flush retry\n", qid);
1050                 rx_queue->flush_pending = true;
1051                 atomic_inc(&efx->rxq_flush_pending);
1052         } else {
1053                 efx_magic_event(efx_rx_queue_channel(rx_queue),
1054                                 EFX_CHANNEL_MAGIC_RX_DRAIN(rx_queue));
1055         }
1056         atomic_dec(&efx->rxq_flush_outstanding);
1057         if (efx_flush_wake(efx))
1058                 wake_up(&efx->flush_wq);
1059 }
1060
1061 static void
1062 efx_handle_drain_event(struct efx_channel *channel)
1063 {
1064         struct efx_nic *efx = channel->efx;
1065
1066         WARN_ON(atomic_read(&efx->drain_pending) == 0);
1067         atomic_dec(&efx->drain_pending);
1068         if (efx_flush_wake(efx))
1069                 wake_up(&efx->flush_wq);
1070 }
1071
1072 static void
1073 efx_handle_generated_event(struct efx_channel *channel, efx_qword_t *event)
1074 {
1075         struct efx_nic *efx = channel->efx;
1076         struct efx_rx_queue *rx_queue =
1077                 efx_channel_has_rx_queue(channel) ?
1078                 efx_channel_get_rx_queue(channel) : NULL;
1079         unsigned magic, code;
1080
1081         magic = EFX_QWORD_FIELD(*event, FSF_AZ_DRV_GEN_EV_MAGIC);
1082         code = _EFX_CHANNEL_MAGIC_CODE(magic);
1083
1084         if (magic == EFX_CHANNEL_MAGIC_TEST(channel)) {
1085                 channel->event_test_cpu = raw_smp_processor_id();
1086         } else if (rx_queue && magic == EFX_CHANNEL_MAGIC_FILL(rx_queue)) {
1087                 /* The queue must be empty, so we won't receive any rx
1088                  * events, so efx_process_channel() won't refill the
1089                  * queue. Refill it here */
1090                 efx_fast_push_rx_descriptors(rx_queue);
1091         } else if (rx_queue && magic == EFX_CHANNEL_MAGIC_RX_DRAIN(rx_queue)) {
1092                 rx_queue->enabled = false;
1093                 efx_handle_drain_event(channel);
1094         } else if (code == _EFX_CHANNEL_MAGIC_TX_DRAIN) {
1095                 efx_handle_drain_event(channel);
1096         } else {
1097                 netif_dbg(efx, hw, efx->net_dev, "channel %d received "
1098                           "generated event "EFX_QWORD_FMT"\n",
1099                           channel->channel, EFX_QWORD_VAL(*event));
1100         }
1101 }
1102
1103 static void
1104 efx_handle_driver_event(struct efx_channel *channel, efx_qword_t *event)
1105 {
1106         struct efx_nic *efx = channel->efx;
1107         unsigned int ev_sub_code;
1108         unsigned int ev_sub_data;
1109
1110         ev_sub_code = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBCODE);
1111         ev_sub_data = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
1112
1113         switch (ev_sub_code) {
1114         case FSE_AZ_TX_DESCQ_FLS_DONE_EV:
1115                 netif_vdbg(efx, hw, efx->net_dev, "channel %d TXQ %d flushed\n",
1116                            channel->channel, ev_sub_data);
1117                 efx_handle_tx_flush_done(efx, event);
1118                 efx_sriov_tx_flush_done(efx, event);
1119                 break;
1120         case FSE_AZ_RX_DESCQ_FLS_DONE_EV:
1121                 netif_vdbg(efx, hw, efx->net_dev, "channel %d RXQ %d flushed\n",
1122                            channel->channel, ev_sub_data);
1123                 efx_handle_rx_flush_done(efx, event);
1124                 efx_sriov_rx_flush_done(efx, event);
1125                 break;
1126         case FSE_AZ_EVQ_INIT_DONE_EV:
1127                 netif_dbg(efx, hw, efx->net_dev,
1128                           "channel %d EVQ %d initialised\n",
1129                           channel->channel, ev_sub_data);
1130                 break;
1131         case FSE_AZ_SRM_UPD_DONE_EV:
1132                 netif_vdbg(efx, hw, efx->net_dev,
1133                            "channel %d SRAM update done\n", channel->channel);
1134                 break;
1135         case FSE_AZ_WAKE_UP_EV:
1136                 netif_vdbg(efx, hw, efx->net_dev,
1137                            "channel %d RXQ %d wakeup event\n",
1138                            channel->channel, ev_sub_data);
1139                 break;
1140         case FSE_AZ_TIMER_EV:
1141                 netif_vdbg(efx, hw, efx->net_dev,
1142                            "channel %d RX queue %d timer expired\n",
1143                            channel->channel, ev_sub_data);
1144                 break;
1145         case FSE_AA_RX_RECOVER_EV:
1146                 netif_err(efx, rx_err, efx->net_dev,
1147                           "channel %d seen DRIVER RX_RESET event. "
1148                         "Resetting.\n", channel->channel);
1149                 atomic_inc(&efx->rx_reset);
1150                 efx_schedule_reset(efx,
1151                                    EFX_WORKAROUND_6555(efx) ?
1152                                    RESET_TYPE_RX_RECOVERY :
1153                                    RESET_TYPE_DISABLE);
1154                 break;
1155         case FSE_BZ_RX_DSC_ERROR_EV:
1156                 if (ev_sub_data < EFX_VI_BASE) {
1157                         netif_err(efx, rx_err, efx->net_dev,
1158                                   "RX DMA Q %d reports descriptor fetch error."
1159                                   " RX Q %d is disabled.\n", ev_sub_data,
1160                                   ev_sub_data);
1161                         efx_schedule_reset(efx, RESET_TYPE_RX_DESC_FETCH);
1162                 } else
1163                         efx_sriov_desc_fetch_err(efx, ev_sub_data);
1164                 break;
1165         case FSE_BZ_TX_DSC_ERROR_EV:
1166                 if (ev_sub_data < EFX_VI_BASE) {
1167                         netif_err(efx, tx_err, efx->net_dev,
1168                                   "TX DMA Q %d reports descriptor fetch error."
1169                                   " TX Q %d is disabled.\n", ev_sub_data,
1170                                   ev_sub_data);
1171                         efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
1172                 } else
1173                         efx_sriov_desc_fetch_err(efx, ev_sub_data);
1174                 break;
1175         default:
1176                 netif_vdbg(efx, hw, efx->net_dev,
1177                            "channel %d unknown driver event code %d "
1178                            "data %04x\n", channel->channel, ev_sub_code,
1179                            ev_sub_data);
1180                 break;
1181         }
1182 }
1183
1184 int efx_nic_process_eventq(struct efx_channel *channel, int budget)
1185 {
1186         struct efx_nic *efx = channel->efx;
1187         unsigned int read_ptr;
1188         efx_qword_t event, *p_event;
1189         int ev_code;
1190         int tx_packets = 0;
1191         int spent = 0;
1192
1193         read_ptr = channel->eventq_read_ptr;
1194
1195         for (;;) {
1196                 p_event = efx_event(channel, read_ptr);
1197                 event = *p_event;
1198
1199                 if (!efx_event_present(&event))
1200                         /* End of events */
1201                         break;
1202
1203                 netif_vdbg(channel->efx, intr, channel->efx->net_dev,
1204                            "channel %d event is "EFX_QWORD_FMT"\n",
1205                            channel->channel, EFX_QWORD_VAL(event));
1206
1207                 /* Clear this event by marking it all ones */
1208                 EFX_SET_QWORD(*p_event);
1209
1210                 ++read_ptr;
1211
1212                 ev_code = EFX_QWORD_FIELD(event, FSF_AZ_EV_CODE);
1213
1214                 switch (ev_code) {
1215                 case FSE_AZ_EV_CODE_RX_EV:
1216                         efx_handle_rx_event(channel, &event);
1217                         if (++spent == budget)
1218                                 goto out;
1219                         break;
1220                 case FSE_AZ_EV_CODE_TX_EV:
1221                         tx_packets += efx_handle_tx_event(channel, &event);
1222                         if (tx_packets > efx->txq_entries) {
1223                                 spent = budget;
1224                                 goto out;
1225                         }
1226                         break;
1227                 case FSE_AZ_EV_CODE_DRV_GEN_EV:
1228                         efx_handle_generated_event(channel, &event);
1229                         break;
1230                 case FSE_AZ_EV_CODE_DRIVER_EV:
1231                         efx_handle_driver_event(channel, &event);
1232                         break;
1233                 case FSE_CZ_EV_CODE_USER_EV:
1234                         efx_sriov_event(channel, &event);
1235                         break;
1236                 case FSE_CZ_EV_CODE_MCDI_EV:
1237                         efx_mcdi_process_event(channel, &event);
1238                         break;
1239                 case FSE_AZ_EV_CODE_GLOBAL_EV:
1240                         if (efx->type->handle_global_event &&
1241                             efx->type->handle_global_event(channel, &event))
1242                                 break;
1243                         /* else fall through */
1244                 default:
1245                         netif_err(channel->efx, hw, channel->efx->net_dev,
1246                                   "channel %d unknown event type %d (data "
1247                                   EFX_QWORD_FMT ")\n", channel->channel,
1248                                   ev_code, EFX_QWORD_VAL(event));
1249                 }
1250         }
1251
1252 out:
1253         channel->eventq_read_ptr = read_ptr;
1254         return spent;
1255 }
1256
1257 /* Check whether an event is present in the eventq at the current
1258  * read pointer.  Only useful for self-test.
1259  */
1260 bool efx_nic_event_present(struct efx_channel *channel)
1261 {
1262         return efx_event_present(efx_event(channel, channel->eventq_read_ptr));
1263 }
1264
1265 /* Allocate buffer table entries for event queue */
1266 int efx_nic_probe_eventq(struct efx_channel *channel)
1267 {
1268         struct efx_nic *efx = channel->efx;
1269         unsigned entries;
1270
1271         entries = channel->eventq_mask + 1;
1272         return efx_alloc_special_buffer(efx, &channel->eventq,
1273                                         entries * sizeof(efx_qword_t));
1274 }
1275
1276 void efx_nic_init_eventq(struct efx_channel *channel)
1277 {
1278         efx_oword_t reg;
1279         struct efx_nic *efx = channel->efx;
1280
1281         netif_dbg(efx, hw, efx->net_dev,
1282                   "channel %d event queue in special buffers %d-%d\n",
1283                   channel->channel, channel->eventq.index,
1284                   channel->eventq.index + channel->eventq.entries - 1);
1285
1286         if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
1287                 EFX_POPULATE_OWORD_3(reg,
1288                                      FRF_CZ_TIMER_Q_EN, 1,
1289                                      FRF_CZ_HOST_NOTIFY_MODE, 0,
1290                                      FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS);
1291                 efx_writeo_table(efx, &reg, FR_BZ_TIMER_TBL, channel->channel);
1292         }
1293
1294         /* Pin event queue buffer */
1295         efx_init_special_buffer(efx, &channel->eventq);
1296
1297         /* Fill event queue with all ones (i.e. empty events) */
1298         memset(channel->eventq.addr, 0xff, channel->eventq.len);
1299
1300         /* Push event queue to card */
1301         EFX_POPULATE_OWORD_3(reg,
1302                              FRF_AZ_EVQ_EN, 1,
1303                              FRF_AZ_EVQ_SIZE, __ffs(channel->eventq.entries),
1304                              FRF_AZ_EVQ_BUF_BASE_ID, channel->eventq.index);
1305         efx_writeo_table(efx, &reg, efx->type->evq_ptr_tbl_base,
1306                          channel->channel);
1307
1308         efx->type->push_irq_moderation(channel);
1309 }
1310
1311 void efx_nic_fini_eventq(struct efx_channel *channel)
1312 {
1313         efx_oword_t reg;
1314         struct efx_nic *efx = channel->efx;
1315
1316         /* Remove event queue from card */
1317         EFX_ZERO_OWORD(reg);
1318         efx_writeo_table(efx, &reg, efx->type->evq_ptr_tbl_base,
1319                          channel->channel);
1320         if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0)
1321                 efx_writeo_table(efx, &reg, FR_BZ_TIMER_TBL, channel->channel);
1322
1323         /* Unpin event queue */
1324         efx_fini_special_buffer(efx, &channel->eventq);
1325 }
1326
1327 /* Free buffers backing event queue */
1328 void efx_nic_remove_eventq(struct efx_channel *channel)
1329 {
1330         efx_free_special_buffer(channel->efx, &channel->eventq);
1331 }
1332
1333
1334 void efx_nic_event_test_start(struct efx_channel *channel)
1335 {
1336         channel->event_test_cpu = -1;
1337         smp_wmb();
1338         efx_magic_event(channel, EFX_CHANNEL_MAGIC_TEST(channel));
1339 }
1340
1341 void efx_nic_generate_fill_event(struct efx_rx_queue *rx_queue)
1342 {
1343         efx_magic_event(efx_rx_queue_channel(rx_queue),
1344                         EFX_CHANNEL_MAGIC_FILL(rx_queue));
1345 }
1346
1347 /**************************************************************************
1348  *
1349  * Hardware interrupts
1350  * The hardware interrupt handler does very little work; all the event
1351  * queue processing is carried out by per-channel tasklets.
1352  *
1353  **************************************************************************/
1354
1355 /* Enable/disable/generate interrupts */
1356 static inline void efx_nic_interrupts(struct efx_nic *efx,
1357                                       bool enabled, bool force)
1358 {
1359         efx_oword_t int_en_reg_ker;
1360
1361         EFX_POPULATE_OWORD_3(int_en_reg_ker,
1362                              FRF_AZ_KER_INT_LEVE_SEL, efx->irq_level,
1363                              FRF_AZ_KER_INT_KER, force,
1364                              FRF_AZ_DRV_INT_EN_KER, enabled);
1365         efx_writeo(efx, &int_en_reg_ker, FR_AZ_INT_EN_KER);
1366 }
1367
1368 void efx_nic_enable_interrupts(struct efx_nic *efx)
1369 {
1370         EFX_ZERO_OWORD(*((efx_oword_t *) efx->irq_status.addr));
1371         wmb(); /* Ensure interrupt vector is clear before interrupts enabled */
1372
1373         efx_nic_interrupts(efx, true, false);
1374 }
1375
1376 void efx_nic_disable_interrupts(struct efx_nic *efx)
1377 {
1378         /* Disable interrupts */
1379         efx_nic_interrupts(efx, false, false);
1380 }
1381
1382 /* Generate a test interrupt
1383  * Interrupt must already have been enabled, otherwise nasty things
1384  * may happen.
1385  */
1386 void efx_nic_irq_test_start(struct efx_nic *efx)
1387 {
1388         efx->last_irq_cpu = -1;
1389         smp_wmb();
1390         efx_nic_interrupts(efx, true, true);
1391 }
1392
1393 /* Process a fatal interrupt
1394  * Disable bus mastering ASAP and schedule a reset
1395  */
1396 irqreturn_t efx_nic_fatal_interrupt(struct efx_nic *efx)
1397 {
1398         struct falcon_nic_data *nic_data = efx->nic_data;
1399         efx_oword_t *int_ker = efx->irq_status.addr;
1400         efx_oword_t fatal_intr;
1401         int error, mem_perr;
1402
1403         efx_reado(efx, &fatal_intr, FR_AZ_FATAL_INTR_KER);
1404         error = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_FATAL_INTR);
1405
1406         netif_err(efx, hw, efx->net_dev, "SYSTEM ERROR "EFX_OWORD_FMT" status "
1407                   EFX_OWORD_FMT ": %s\n", EFX_OWORD_VAL(*int_ker),
1408                   EFX_OWORD_VAL(fatal_intr),
1409                   error ? "disabling bus mastering" : "no recognised error");
1410
1411         /* If this is a memory parity error dump which blocks are offending */
1412         mem_perr = (EFX_OWORD_FIELD(fatal_intr, FRF_AZ_MEM_PERR_INT_KER) ||
1413                     EFX_OWORD_FIELD(fatal_intr, FRF_AZ_SRM_PERR_INT_KER));
1414         if (mem_perr) {
1415                 efx_oword_t reg;
1416                 efx_reado(efx, &reg, FR_AZ_MEM_STAT);
1417                 netif_err(efx, hw, efx->net_dev,
1418                           "SYSTEM ERROR: memory parity error "EFX_OWORD_FMT"\n",
1419                           EFX_OWORD_VAL(reg));
1420         }
1421
1422         /* Disable both devices */
1423         pci_clear_master(efx->pci_dev);
1424         if (efx_nic_is_dual_func(efx))
1425                 pci_clear_master(nic_data->pci_dev2);
1426         efx_nic_disable_interrupts(efx);
1427
1428         /* Count errors and reset or disable the NIC accordingly */
1429         if (efx->int_error_count == 0 ||
1430             time_after(jiffies, efx->int_error_expire)) {
1431                 efx->int_error_count = 0;
1432                 efx->int_error_expire =
1433                         jiffies + EFX_INT_ERROR_EXPIRE * HZ;
1434         }
1435         if (++efx->int_error_count < EFX_MAX_INT_ERRORS) {
1436                 netif_err(efx, hw, efx->net_dev,
1437                           "SYSTEM ERROR - reset scheduled\n");
1438                 efx_schedule_reset(efx, RESET_TYPE_INT_ERROR);
1439         } else {
1440                 netif_err(efx, hw, efx->net_dev,
1441                           "SYSTEM ERROR - max number of errors seen."
1442                           "NIC will be disabled\n");
1443                 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
1444         }
1445
1446         return IRQ_HANDLED;
1447 }
1448
1449 /* Handle a legacy interrupt
1450  * Acknowledges the interrupt and schedule event queue processing.
1451  */
1452 static irqreturn_t efx_legacy_interrupt(int irq, void *dev_id)
1453 {
1454         struct efx_nic *efx = dev_id;
1455         efx_oword_t *int_ker = efx->irq_status.addr;
1456         irqreturn_t result = IRQ_NONE;
1457         struct efx_channel *channel;
1458         efx_dword_t reg;
1459         u32 queues;
1460         int syserr;
1461
1462         /* Could this be ours?  If interrupts are disabled then the
1463          * channel state may not be valid.
1464          */
1465         if (!efx->legacy_irq_enabled)
1466                 return result;
1467
1468         /* Read the ISR which also ACKs the interrupts */
1469         efx_readd(efx, &reg, FR_BZ_INT_ISR0);
1470         queues = EFX_EXTRACT_DWORD(reg, 0, 31);
1471
1472         /* Handle non-event-queue sources */
1473         if (queues & (1U << efx->irq_level)) {
1474                 syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
1475                 if (unlikely(syserr))
1476                         return efx_nic_fatal_interrupt(efx);
1477                 efx->last_irq_cpu = raw_smp_processor_id();
1478         }
1479
1480         if (queues != 0) {
1481                 if (EFX_WORKAROUND_15783(efx))
1482                         efx->irq_zero_count = 0;
1483
1484                 /* Schedule processing of any interrupting queues */
1485                 efx_for_each_channel(channel, efx) {
1486                         if (queues & 1)
1487                                 efx_schedule_channel_irq(channel);
1488                         queues >>= 1;
1489                 }
1490                 result = IRQ_HANDLED;
1491
1492         } else if (EFX_WORKAROUND_15783(efx)) {
1493                 efx_qword_t *event;
1494
1495                 /* We can't return IRQ_HANDLED more than once on seeing ISR=0
1496                  * because this might be a shared interrupt. */
1497                 if (efx->irq_zero_count++ == 0)
1498                         result = IRQ_HANDLED;
1499
1500                 /* Ensure we schedule or rearm all event queues */
1501                 efx_for_each_channel(channel, efx) {
1502                         event = efx_event(channel, channel->eventq_read_ptr);
1503                         if (efx_event_present(event))
1504                                 efx_schedule_channel_irq(channel);
1505                         else
1506                                 efx_nic_eventq_read_ack(channel);
1507                 }
1508         }
1509
1510         if (result == IRQ_HANDLED)
1511                 netif_vdbg(efx, intr, efx->net_dev,
1512                            "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n",
1513                            irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg));
1514
1515         return result;
1516 }
1517
1518 /* Handle an MSI interrupt
1519  *
1520  * Handle an MSI hardware interrupt.  This routine schedules event
1521  * queue processing.  No interrupt acknowledgement cycle is necessary.
1522  * Also, we never need to check that the interrupt is for us, since
1523  * MSI interrupts cannot be shared.
1524  */
1525 static irqreturn_t efx_msi_interrupt(int irq, void *dev_id)
1526 {
1527         struct efx_channel *channel = *(struct efx_channel **)dev_id;
1528         struct efx_nic *efx = channel->efx;
1529         efx_oword_t *int_ker = efx->irq_status.addr;
1530         int syserr;
1531
1532         netif_vdbg(efx, intr, efx->net_dev,
1533                    "IRQ %d on CPU %d status " EFX_OWORD_FMT "\n",
1534                    irq, raw_smp_processor_id(), EFX_OWORD_VAL(*int_ker));
1535
1536         /* Handle non-event-queue sources */
1537         if (channel->channel == efx->irq_level) {
1538                 syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
1539                 if (unlikely(syserr))
1540                         return efx_nic_fatal_interrupt(efx);
1541                 efx->last_irq_cpu = raw_smp_processor_id();
1542         }
1543
1544         /* Schedule processing of the channel */
1545         efx_schedule_channel_irq(channel);
1546
1547         return IRQ_HANDLED;
1548 }
1549
1550
1551 /* Setup RSS indirection table.
1552  * This maps from the hash value of the packet to RXQ
1553  */
1554 void efx_nic_push_rx_indir_table(struct efx_nic *efx)
1555 {
1556         size_t i = 0;
1557         efx_dword_t dword;
1558
1559         if (efx_nic_rev(efx) < EFX_REV_FALCON_B0)
1560                 return;
1561
1562         BUILD_BUG_ON(ARRAY_SIZE(efx->rx_indir_table) !=
1563                      FR_BZ_RX_INDIRECTION_TBL_ROWS);
1564
1565         for (i = 0; i < FR_BZ_RX_INDIRECTION_TBL_ROWS; i++) {
1566                 EFX_POPULATE_DWORD_1(dword, FRF_BZ_IT_QUEUE,
1567                                      efx->rx_indir_table[i]);
1568                 efx_writed_table(efx, &dword, FR_BZ_RX_INDIRECTION_TBL, i);
1569         }
1570 }
1571
1572 /* Hook interrupt handler(s)
1573  * Try MSI and then legacy interrupts.
1574  */
1575 int efx_nic_init_interrupt(struct efx_nic *efx)
1576 {
1577         struct efx_channel *channel;
1578         int rc;
1579
1580         if (!EFX_INT_MODE_USE_MSI(efx)) {
1581                 irq_handler_t handler;
1582                 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1583                         handler = efx_legacy_interrupt;
1584                 else
1585                         handler = falcon_legacy_interrupt_a1;
1586
1587                 rc = request_irq(efx->legacy_irq, handler, IRQF_SHARED,
1588                                  efx->name, efx);
1589                 if (rc) {
1590                         netif_err(efx, drv, efx->net_dev,
1591                                   "failed to hook legacy IRQ %d\n",
1592                                   efx->pci_dev->irq);
1593                         goto fail1;
1594                 }
1595                 return 0;
1596         }
1597
1598         /* Hook MSI or MSI-X interrupt */
1599         efx_for_each_channel(channel, efx) {
1600                 rc = request_irq(channel->irq, efx_msi_interrupt,
1601                                  IRQF_PROBE_SHARED, /* Not shared */
1602                                  efx->channel_name[channel->channel],
1603                                  &efx->channel[channel->channel]);
1604                 if (rc) {
1605                         netif_err(efx, drv, efx->net_dev,
1606                                   "failed to hook IRQ %d\n", channel->irq);
1607                         goto fail2;
1608                 }
1609         }
1610
1611         return 0;
1612
1613  fail2:
1614         efx_for_each_channel(channel, efx)
1615                 free_irq(channel->irq, &efx->channel[channel->channel]);
1616  fail1:
1617         return rc;
1618 }
1619
1620 void efx_nic_fini_interrupt(struct efx_nic *efx)
1621 {
1622         struct efx_channel *channel;
1623         efx_oword_t reg;
1624
1625         /* Disable MSI/MSI-X interrupts */
1626         efx_for_each_channel(channel, efx) {
1627                 if (channel->irq)
1628                         free_irq(channel->irq, &efx->channel[channel->channel]);
1629         }
1630
1631         /* ACK legacy interrupt */
1632         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1633                 efx_reado(efx, &reg, FR_BZ_INT_ISR0);
1634         else
1635                 falcon_irq_ack_a1(efx);
1636
1637         /* Disable legacy interrupt */
1638         if (efx->legacy_irq)
1639                 free_irq(efx->legacy_irq, efx);
1640 }
1641
1642 /* Looks at available SRAM resources and works out how many queues we
1643  * can support, and where things like descriptor caches should live.
1644  *
1645  * SRAM is split up as follows:
1646  * 0                          buftbl entries for channels
1647  * efx->vf_buftbl_base        buftbl entries for SR-IOV
1648  * efx->rx_dc_base            RX descriptor caches
1649  * efx->tx_dc_base            TX descriptor caches
1650  */
1651 void efx_nic_dimension_resources(struct efx_nic *efx, unsigned sram_lim_qw)
1652 {
1653         unsigned vi_count, buftbl_min;
1654
1655         /* Account for the buffer table entries backing the datapath channels
1656          * and the descriptor caches for those channels.
1657          */
1658         buftbl_min = ((efx->n_rx_channels * EFX_MAX_DMAQ_SIZE +
1659                        efx->n_tx_channels * EFX_TXQ_TYPES * EFX_MAX_DMAQ_SIZE +
1660                        efx->n_channels * EFX_MAX_EVQ_SIZE)
1661                       * sizeof(efx_qword_t) / EFX_BUF_SIZE);
1662         vi_count = max(efx->n_channels, efx->n_tx_channels * EFX_TXQ_TYPES);
1663
1664 #ifdef CONFIG_SFC_SRIOV
1665         if (efx_sriov_wanted(efx)) {
1666                 unsigned vi_dc_entries, buftbl_free, entries_per_vf, vf_limit;
1667
1668                 efx->vf_buftbl_base = buftbl_min;
1669
1670                 vi_dc_entries = RX_DC_ENTRIES + TX_DC_ENTRIES;
1671                 vi_count = max(vi_count, EFX_VI_BASE);
1672                 buftbl_free = (sram_lim_qw - buftbl_min -
1673                                vi_count * vi_dc_entries);
1674
1675                 entries_per_vf = ((vi_dc_entries + EFX_VF_BUFTBL_PER_VI) *
1676                                   efx_vf_size(efx));
1677                 vf_limit = min(buftbl_free / entries_per_vf,
1678                                (1024U - EFX_VI_BASE) >> efx->vi_scale);
1679
1680                 if (efx->vf_count > vf_limit) {
1681                         netif_err(efx, probe, efx->net_dev,
1682                                   "Reducing VF count from from %d to %d\n",
1683                                   efx->vf_count, vf_limit);
1684                         efx->vf_count = vf_limit;
1685                 }
1686                 vi_count += efx->vf_count * efx_vf_size(efx);
1687         }
1688 #endif
1689
1690         efx->tx_dc_base = sram_lim_qw - vi_count * TX_DC_ENTRIES;
1691         efx->rx_dc_base = efx->tx_dc_base - vi_count * RX_DC_ENTRIES;
1692 }
1693
1694 u32 efx_nic_fpga_ver(struct efx_nic *efx)
1695 {
1696         efx_oword_t altera_build;
1697         efx_reado(efx, &altera_build, FR_AZ_ALTERA_BUILD);
1698         return EFX_OWORD_FIELD(altera_build, FRF_AZ_ALTERA_BUILD_VER);
1699 }
1700
1701 void efx_nic_init_common(struct efx_nic *efx)
1702 {
1703         efx_oword_t temp;
1704
1705         /* Set positions of descriptor caches in SRAM. */
1706         EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_TX_DC_BASE_ADR, efx->tx_dc_base);
1707         efx_writeo(efx, &temp, FR_AZ_SRM_TX_DC_CFG);
1708         EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_RX_DC_BASE_ADR, efx->rx_dc_base);
1709         efx_writeo(efx, &temp, FR_AZ_SRM_RX_DC_CFG);
1710
1711         /* Set TX descriptor cache size. */
1712         BUILD_BUG_ON(TX_DC_ENTRIES != (8 << TX_DC_ENTRIES_ORDER));
1713         EFX_POPULATE_OWORD_1(temp, FRF_AZ_TX_DC_SIZE, TX_DC_ENTRIES_ORDER);
1714         efx_writeo(efx, &temp, FR_AZ_TX_DC_CFG);
1715
1716         /* Set RX descriptor cache size.  Set low watermark to size-8, as
1717          * this allows most efficient prefetching.
1718          */
1719         BUILD_BUG_ON(RX_DC_ENTRIES != (8 << RX_DC_ENTRIES_ORDER));
1720         EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_SIZE, RX_DC_ENTRIES_ORDER);
1721         efx_writeo(efx, &temp, FR_AZ_RX_DC_CFG);
1722         EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_PF_LWM, RX_DC_ENTRIES - 8);
1723         efx_writeo(efx, &temp, FR_AZ_RX_DC_PF_WM);
1724
1725         /* Program INT_KER address */
1726         EFX_POPULATE_OWORD_2(temp,
1727                              FRF_AZ_NORM_INT_VEC_DIS_KER,
1728                              EFX_INT_MODE_USE_MSI(efx),
1729                              FRF_AZ_INT_ADR_KER, efx->irq_status.dma_addr);
1730         efx_writeo(efx, &temp, FR_AZ_INT_ADR_KER);
1731
1732         if (EFX_WORKAROUND_17213(efx) && !EFX_INT_MODE_USE_MSI(efx))
1733                 /* Use an interrupt level unused by event queues */
1734                 efx->irq_level = 0x1f;
1735         else
1736                 /* Use a valid MSI-X vector */
1737                 efx->irq_level = 0;
1738
1739         /* Enable all the genuinely fatal interrupts.  (They are still
1740          * masked by the overall interrupt mask, controlled by
1741          * falcon_interrupts()).
1742          *
1743          * Note: All other fatal interrupts are enabled
1744          */
1745         EFX_POPULATE_OWORD_3(temp,
1746                              FRF_AZ_ILL_ADR_INT_KER_EN, 1,
1747                              FRF_AZ_RBUF_OWN_INT_KER_EN, 1,
1748                              FRF_AZ_TBUF_OWN_INT_KER_EN, 1);
1749         if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0)
1750                 EFX_SET_OWORD_FIELD(temp, FRF_CZ_SRAM_PERR_INT_P_KER_EN, 1);
1751         EFX_INVERT_OWORD(temp);
1752         efx_writeo(efx, &temp, FR_AZ_FATAL_INTR_KER);
1753
1754         efx_nic_push_rx_indir_table(efx);
1755
1756         /* Disable the ugly timer-based TX DMA backoff and allow TX DMA to be
1757          * controlled by the RX FIFO fill level. Set arbitration to one pkt/Q.
1758          */
1759         efx_reado(efx, &temp, FR_AZ_TX_RESERVED);
1760         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER, 0xfe);
1761         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER_EN, 1);
1762         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
1763         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PUSH_EN, 1);
1764         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_DIS_NON_IP_EV, 1);
1765         /* Enable SW_EV to inherit in char driver - assume harmless here */
1766         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_SOFT_EVT_EN, 1);
1767         /* Prefetch threshold 2 => fetch when descriptor cache half empty */
1768         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_THRESHOLD, 2);
1769         /* Disable hardware watchdog which can misfire */
1770         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
1771         /* Squash TX of packets of 16 bytes or less */
1772         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1773                 EFX_SET_OWORD_FIELD(temp, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
1774         efx_writeo(efx, &temp, FR_AZ_TX_RESERVED);
1775
1776         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
1777                 EFX_POPULATE_OWORD_4(temp,
1778                                      /* Default values */
1779                                      FRF_BZ_TX_PACE_SB_NOT_AF, 0x15,
1780                                      FRF_BZ_TX_PACE_SB_AF, 0xb,
1781                                      FRF_BZ_TX_PACE_FB_BASE, 0,
1782                                      /* Allow large pace values in the
1783                                       * fast bin. */
1784                                      FRF_BZ_TX_PACE_BIN_TH,
1785                                      FFE_BZ_TX_PACE_RESERVED);
1786                 efx_writeo(efx, &temp, FR_BZ_TX_PACE);
1787         }
1788 }
1789
1790 /* Register dump */
1791
1792 #define REGISTER_REVISION_A     1
1793 #define REGISTER_REVISION_B     2
1794 #define REGISTER_REVISION_C     3
1795 #define REGISTER_REVISION_Z     3       /* latest revision */
1796
1797 struct efx_nic_reg {
1798         u32 offset:24;
1799         u32 min_revision:2, max_revision:2;
1800 };
1801
1802 #define REGISTER(name, min_rev, max_rev) {                              \
1803         FR_ ## min_rev ## max_rev ## _ ## name,                         \
1804         REGISTER_REVISION_ ## min_rev, REGISTER_REVISION_ ## max_rev    \
1805 }
1806 #define REGISTER_AA(name) REGISTER(name, A, A)
1807 #define REGISTER_AB(name) REGISTER(name, A, B)
1808 #define REGISTER_AZ(name) REGISTER(name, A, Z)
1809 #define REGISTER_BB(name) REGISTER(name, B, B)
1810 #define REGISTER_BZ(name) REGISTER(name, B, Z)
1811 #define REGISTER_CZ(name) REGISTER(name, C, Z)
1812
1813 static const struct efx_nic_reg efx_nic_regs[] = {
1814         REGISTER_AZ(ADR_REGION),
1815         REGISTER_AZ(INT_EN_KER),
1816         REGISTER_BZ(INT_EN_CHAR),
1817         REGISTER_AZ(INT_ADR_KER),
1818         REGISTER_BZ(INT_ADR_CHAR),
1819         /* INT_ACK_KER is WO */
1820         /* INT_ISR0 is RC */
1821         REGISTER_AZ(HW_INIT),
1822         REGISTER_CZ(USR_EV_CFG),
1823         REGISTER_AB(EE_SPI_HCMD),
1824         REGISTER_AB(EE_SPI_HADR),
1825         REGISTER_AB(EE_SPI_HDATA),
1826         REGISTER_AB(EE_BASE_PAGE),
1827         REGISTER_AB(EE_VPD_CFG0),
1828         /* EE_VPD_SW_CNTL and EE_VPD_SW_DATA are not used */
1829         /* PMBX_DBG_IADDR and PBMX_DBG_IDATA are indirect */
1830         /* PCIE_CORE_INDIRECT is indirect */
1831         REGISTER_AB(NIC_STAT),
1832         REGISTER_AB(GPIO_CTL),
1833         REGISTER_AB(GLB_CTL),
1834         /* FATAL_INTR_KER and FATAL_INTR_CHAR are partly RC */
1835         REGISTER_BZ(DP_CTRL),
1836         REGISTER_AZ(MEM_STAT),
1837         REGISTER_AZ(CS_DEBUG),
1838         REGISTER_AZ(ALTERA_BUILD),
1839         REGISTER_AZ(CSR_SPARE),
1840         REGISTER_AB(PCIE_SD_CTL0123),
1841         REGISTER_AB(PCIE_SD_CTL45),
1842         REGISTER_AB(PCIE_PCS_CTL_STAT),
1843         /* DEBUG_DATA_OUT is not used */
1844         /* DRV_EV is WO */
1845         REGISTER_AZ(EVQ_CTL),
1846         REGISTER_AZ(EVQ_CNT1),
1847         REGISTER_AZ(EVQ_CNT2),
1848         REGISTER_AZ(BUF_TBL_CFG),
1849         REGISTER_AZ(SRM_RX_DC_CFG),
1850         REGISTER_AZ(SRM_TX_DC_CFG),
1851         REGISTER_AZ(SRM_CFG),
1852         /* BUF_TBL_UPD is WO */
1853         REGISTER_AZ(SRM_UPD_EVQ),
1854         REGISTER_AZ(SRAM_PARITY),
1855         REGISTER_AZ(RX_CFG),
1856         REGISTER_BZ(RX_FILTER_CTL),
1857         /* RX_FLUSH_DESCQ is WO */
1858         REGISTER_AZ(RX_DC_CFG),
1859         REGISTER_AZ(RX_DC_PF_WM),
1860         REGISTER_BZ(RX_RSS_TKEY),
1861         /* RX_NODESC_DROP is RC */
1862         REGISTER_AA(RX_SELF_RST),
1863         /* RX_DEBUG, RX_PUSH_DROP are not used */
1864         REGISTER_CZ(RX_RSS_IPV6_REG1),
1865         REGISTER_CZ(RX_RSS_IPV6_REG2),
1866         REGISTER_CZ(RX_RSS_IPV6_REG3),
1867         /* TX_FLUSH_DESCQ is WO */
1868         REGISTER_AZ(TX_DC_CFG),
1869         REGISTER_AA(TX_CHKSM_CFG),
1870         REGISTER_AZ(TX_CFG),
1871         /* TX_PUSH_DROP is not used */
1872         REGISTER_AZ(TX_RESERVED),
1873         REGISTER_BZ(TX_PACE),
1874         /* TX_PACE_DROP_QID is RC */
1875         REGISTER_BB(TX_VLAN),
1876         REGISTER_BZ(TX_IPFIL_PORTEN),
1877         REGISTER_AB(MD_TXD),
1878         REGISTER_AB(MD_RXD),
1879         REGISTER_AB(MD_CS),
1880         REGISTER_AB(MD_PHY_ADR),
1881         REGISTER_AB(MD_ID),
1882         /* MD_STAT is RC */
1883         REGISTER_AB(MAC_STAT_DMA),
1884         REGISTER_AB(MAC_CTRL),
1885         REGISTER_BB(GEN_MODE),
1886         REGISTER_AB(MAC_MC_HASH_REG0),
1887         REGISTER_AB(MAC_MC_HASH_REG1),
1888         REGISTER_AB(GM_CFG1),
1889         REGISTER_AB(GM_CFG2),
1890         /* GM_IPG and GM_HD are not used */
1891         REGISTER_AB(GM_MAX_FLEN),
1892         /* GM_TEST is not used */
1893         REGISTER_AB(GM_ADR1),
1894         REGISTER_AB(GM_ADR2),
1895         REGISTER_AB(GMF_CFG0),
1896         REGISTER_AB(GMF_CFG1),
1897         REGISTER_AB(GMF_CFG2),
1898         REGISTER_AB(GMF_CFG3),
1899         REGISTER_AB(GMF_CFG4),
1900         REGISTER_AB(GMF_CFG5),
1901         REGISTER_BB(TX_SRC_MAC_CTL),
1902         REGISTER_AB(XM_ADR_LO),
1903         REGISTER_AB(XM_ADR_HI),
1904         REGISTER_AB(XM_GLB_CFG),
1905         REGISTER_AB(XM_TX_CFG),
1906         REGISTER_AB(XM_RX_CFG),
1907         REGISTER_AB(XM_MGT_INT_MASK),
1908         REGISTER_AB(XM_FC),
1909         REGISTER_AB(XM_PAUSE_TIME),
1910         REGISTER_AB(XM_TX_PARAM),
1911         REGISTER_AB(XM_RX_PARAM),
1912         /* XM_MGT_INT_MSK (note no 'A') is RC */
1913         REGISTER_AB(XX_PWR_RST),
1914         REGISTER_AB(XX_SD_CTL),
1915         REGISTER_AB(XX_TXDRV_CTL),
1916         /* XX_PRBS_CTL, XX_PRBS_CHK and XX_PRBS_ERR are not used */
1917         /* XX_CORE_STAT is partly RC */
1918 };
1919
1920 struct efx_nic_reg_table {
1921         u32 offset:24;
1922         u32 min_revision:2, max_revision:2;
1923         u32 step:6, rows:21;
1924 };
1925
1926 #define REGISTER_TABLE_DIMENSIONS(_, offset, min_rev, max_rev, step, rows) { \
1927         offset,                                                         \
1928         REGISTER_REVISION_ ## min_rev, REGISTER_REVISION_ ## max_rev,   \
1929         step, rows                                                      \
1930 }
1931 #define REGISTER_TABLE(name, min_rev, max_rev)                          \
1932         REGISTER_TABLE_DIMENSIONS(                                      \
1933                 name, FR_ ## min_rev ## max_rev ## _ ## name,           \
1934                 min_rev, max_rev,                                       \
1935                 FR_ ## min_rev ## max_rev ## _ ## name ## _STEP,        \
1936                 FR_ ## min_rev ## max_rev ## _ ## name ## _ROWS)
1937 #define REGISTER_TABLE_AA(name) REGISTER_TABLE(name, A, A)
1938 #define REGISTER_TABLE_AZ(name) REGISTER_TABLE(name, A, Z)
1939 #define REGISTER_TABLE_BB(name) REGISTER_TABLE(name, B, B)
1940 #define REGISTER_TABLE_BZ(name) REGISTER_TABLE(name, B, Z)
1941 #define REGISTER_TABLE_BB_CZ(name)                                      \
1942         REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, B, B,           \
1943                                   FR_BZ_ ## name ## _STEP,              \
1944                                   FR_BB_ ## name ## _ROWS),             \
1945         REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, C, Z,           \
1946                                   FR_BZ_ ## name ## _STEP,              \
1947                                   FR_CZ_ ## name ## _ROWS)
1948 #define REGISTER_TABLE_CZ(name) REGISTER_TABLE(name, C, Z)
1949
1950 static const struct efx_nic_reg_table efx_nic_reg_tables[] = {
1951         /* DRIVER is not used */
1952         /* EVQ_RPTR, TIMER_COMMAND, USR_EV and {RX,TX}_DESC_UPD are WO */
1953         REGISTER_TABLE_BB(TX_IPFIL_TBL),
1954         REGISTER_TABLE_BB(TX_SRC_MAC_TBL),
1955         REGISTER_TABLE_AA(RX_DESC_PTR_TBL_KER),
1956         REGISTER_TABLE_BB_CZ(RX_DESC_PTR_TBL),
1957         REGISTER_TABLE_AA(TX_DESC_PTR_TBL_KER),
1958         REGISTER_TABLE_BB_CZ(TX_DESC_PTR_TBL),
1959         REGISTER_TABLE_AA(EVQ_PTR_TBL_KER),
1960         REGISTER_TABLE_BB_CZ(EVQ_PTR_TBL),
1961         /* We can't reasonably read all of the buffer table (up to 8MB!).
1962          * However this driver will only use a few entries.  Reading
1963          * 1K entries allows for some expansion of queue count and
1964          * size before we need to change the version. */
1965         REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL_KER, FR_AA_BUF_FULL_TBL_KER,
1966                                   A, A, 8, 1024),
1967         REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL, FR_BZ_BUF_FULL_TBL,
1968                                   B, Z, 8, 1024),
1969         REGISTER_TABLE_CZ(RX_MAC_FILTER_TBL0),
1970         REGISTER_TABLE_BB_CZ(TIMER_TBL),
1971         REGISTER_TABLE_BB_CZ(TX_PACE_TBL),
1972         REGISTER_TABLE_BZ(RX_INDIRECTION_TBL),
1973         /* TX_FILTER_TBL0 is huge and not used by this driver */
1974         REGISTER_TABLE_CZ(TX_MAC_FILTER_TBL0),
1975         REGISTER_TABLE_CZ(MC_TREG_SMEM),
1976         /* MSIX_PBA_TABLE is not mapped */
1977         /* SRM_DBG is not mapped (and is redundant with BUF_FLL_TBL) */
1978         REGISTER_TABLE_BZ(RX_FILTER_TBL0),
1979 };
1980
1981 size_t efx_nic_get_regs_len(struct efx_nic *efx)
1982 {
1983         const struct efx_nic_reg *reg;
1984         const struct efx_nic_reg_table *table;
1985         size_t len = 0;
1986
1987         for (reg = efx_nic_regs;
1988              reg < efx_nic_regs + ARRAY_SIZE(efx_nic_regs);
1989              reg++)
1990                 if (efx->type->revision >= reg->min_revision &&
1991                     efx->type->revision <= reg->max_revision)
1992                         len += sizeof(efx_oword_t);
1993
1994         for (table = efx_nic_reg_tables;
1995              table < efx_nic_reg_tables + ARRAY_SIZE(efx_nic_reg_tables);
1996              table++)
1997                 if (efx->type->revision >= table->min_revision &&
1998                     efx->type->revision <= table->max_revision)
1999                         len += table->rows * min_t(size_t, table->step, 16);
2000
2001         return len;
2002 }
2003
2004 void efx_nic_get_regs(struct efx_nic *efx, void *buf)
2005 {
2006         const struct efx_nic_reg *reg;
2007         const struct efx_nic_reg_table *table;
2008
2009         for (reg = efx_nic_regs;
2010              reg < efx_nic_regs + ARRAY_SIZE(efx_nic_regs);
2011              reg++) {
2012                 if (efx->type->revision >= reg->min_revision &&
2013                     efx->type->revision <= reg->max_revision) {
2014                         efx_reado(efx, (efx_oword_t *)buf, reg->offset);
2015                         buf += sizeof(efx_oword_t);
2016                 }
2017         }
2018
2019         for (table = efx_nic_reg_tables;
2020              table < efx_nic_reg_tables + ARRAY_SIZE(efx_nic_reg_tables);
2021              table++) {
2022                 size_t size, i;
2023
2024                 if (!(efx->type->revision >= table->min_revision &&
2025                       efx->type->revision <= table->max_revision))
2026                         continue;
2027
2028                 size = min_t(size_t, table->step, 16);
2029
2030                 for (i = 0; i < table->rows; i++) {
2031                         switch (table->step) {
2032                         case 4: /* 32-bit register or SRAM */
2033                                 efx_readd_table(efx, buf, table->offset, i);
2034                                 break;
2035                         case 8: /* 64-bit SRAM */
2036                                 efx_sram_readq(efx,
2037                                                efx->membase + table->offset,
2038                                                buf, i);
2039                                 break;
2040                         case 16: /* 128-bit register */
2041                                 efx_reado_table(efx, buf, table->offset, i);
2042                                 break;
2043                         case 32: /* 128-bit register, interleaved */
2044                                 efx_reado_table(efx, buf, table->offset, 2 * i);
2045                                 break;
2046                         default:
2047                                 WARN_ON(1);
2048                                 return;
2049                         }
2050                         buf += size;
2051                 }
2052         }
2053 }