drivers/net/ethernet/pensando/ionic/ionic_dev.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/types.h>
   6 #include <linux/errno.h>
   7 #include <linux/io.h>
   8 #include <linux/slab.h>
   9 #include <linux/etherdevice.h>
  10 #include "ionic.h"
  11 #include "ionic_dev.h"
  12 #include "ionic_lif.h"
  13
  14 static void ionic_watchdog_cb(struct timer_list *t)
  15 {
  16         struct ionic *ionic = from_timer(ionic, t, watchdog_timer);
  17         struct ionic_lif *lif = ionic->lif;
  18         struct ionic_deferred_work *work;
  19         int hb;
  20
  21         mod_timer(&ionic->watchdog_timer,
  22                   round_jiffies(jiffies + ionic->watchdog_period));
  23
  24         if (!lif)
  25                 return;
  26
  27         hb = ionic_heartbeat_check(ionic);
  28         dev_dbg(ionic->dev, "%s: hb %d running %d UP %d\n",
  29                 __func__, hb, netif_running(lif->netdev),
  30                 test_bit(IONIC_LIF_F_UP, lif->state));
  31
  32         if (hb >= 0 &&
  33             !test_bit(IONIC_LIF_F_FW_RESET, lif->state))
  34                 ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
  35
  36         if (test_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED, lif->state) &&
  37             !test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
  38                 work = kzalloc(sizeof(*work), GFP_ATOMIC);
  39                 if (!work) {
  40                         netdev_err(lif->netdev, "rxmode change dropped\n");
  41                         return;
  42                 }
  43
  44                 work->type = IONIC_DW_TYPE_RX_MODE;
  45                 netdev_dbg(lif->netdev, "deferred: rx_mode\n");
  46                 ionic_lif_deferred_enqueue(&lif->deferred, work);
  47         }
  48 }
  49
  50 static void ionic_watchdog_init(struct ionic *ionic)
  51 {
  52         struct ionic_dev *idev = &ionic->idev;
  53
  54         timer_setup(&ionic->watchdog_timer, ionic_watchdog_cb, 0);
  55         ionic->watchdog_period = IONIC_WATCHDOG_SECS * HZ;
  56
  57         /* set times to ensure the first check will proceed */
  58         atomic_long_set(&idev->last_check_time, jiffies - 2 * HZ);
  59         idev->last_hb_time = jiffies - 2 * ionic->watchdog_period;
  60         /* init as ready, so no transition if the first check succeeds */
  61         idev->last_fw_hb = 0;
  62         idev->fw_hb_ready = true;
  63         idev->fw_status_ready = true;
  64         idev->fw_generation = IONIC_FW_STS_F_GENERATION &
  65                               ioread8(&idev->dev_info_regs->fw_status);
  66 }
  67
  68 void ionic_init_devinfo(struct ionic *ionic)
  69 {
  70         struct ionic_dev *idev = &ionic->idev;
  71
  72         idev->dev_info.asic_type = ioread8(&idev->dev_info_regs->asic_type);
  73         idev->dev_info.asic_rev = ioread8(&idev->dev_info_regs->asic_rev);
  74
  75         memcpy_fromio(idev->dev_info.fw_version,
  76                       idev->dev_info_regs->fw_version,
  77                       IONIC_DEVINFO_FWVERS_BUFLEN);
  78
  79         memcpy_fromio(idev->dev_info.serial_num,
  80                       idev->dev_info_regs->serial_num,
  81                       IONIC_DEVINFO_SERIAL_BUFLEN);
  82
  83         idev->dev_info.fw_version[IONIC_DEVINFO_FWVERS_BUFLEN] = 0;
  84         idev->dev_info.serial_num[IONIC_DEVINFO_SERIAL_BUFLEN] = 0;
  85
  86         dev_dbg(ionic->dev, "fw_version %s\n", idev->dev_info.fw_version);
  87 }
  88
  89 int ionic_dev_setup(struct ionic *ionic)
  90 {
  91         struct ionic_dev_bar *bar = ionic->bars;
  92         unsigned int num_bars = ionic->num_bars;
  93         struct ionic_dev *idev = &ionic->idev;
  94         struct device *dev = ionic->dev;
  95         u32 sig;
  96
  97         /* BAR0: dev_cmd and interrupts */
  98         if (num_bars < 1) {
  99                 dev_err(dev, "No bars found, aborting\n");
 100                 return -EFAULT;
 101         }
 102
 103         if (bar->len < IONIC_BAR0_SIZE) {
 104                 dev_err(dev, "Resource bar size %lu too small, aborting\n",
 105                         bar->len);
 106                 return -EFAULT;
 107         }
 108
 109         idev->dev_info_regs = bar->vaddr + IONIC_BAR0_DEV_INFO_REGS_OFFSET;
 110         idev->dev_cmd_regs = bar->vaddr + IONIC_BAR0_DEV_CMD_REGS_OFFSET;
 111         idev->intr_status = bar->vaddr + IONIC_BAR0_INTR_STATUS_OFFSET;
 112         idev->intr_ctrl = bar->vaddr + IONIC_BAR0_INTR_CTRL_OFFSET;
 113
 114         idev->hwstamp_regs = &idev->dev_info_regs->hwstamp;
 115
 116         sig = ioread32(&idev->dev_info_regs->signature);
 117         if (sig != IONIC_DEV_INFO_SIGNATURE) {
 118                 dev_err(dev, "Incompatible firmware signature %x", sig);
 119                 return -EFAULT;
 120         }
 121
 122         ionic_init_devinfo(ionic);
 123
 124         /* BAR1: doorbells */
 125         bar++;
 126         if (num_bars < 2) {
 127                 dev_err(dev, "Doorbell bar missing, aborting\n");
 128                 return -EFAULT;
 129         }
 130
 131         ionic_watchdog_init(ionic);
 132
 133         idev->db_pages = bar->vaddr;
 134         idev->phy_db_pages = bar->bus_addr;
 135
 136         return 0;
 137 }
 138
 139 /* Devcmd Interface */
 140 bool ionic_is_fw_running(struct ionic_dev *idev)
 141 {
 142         u8 fw_status = ioread8(&idev->dev_info_regs->fw_status);
 143
 144         /* firmware is useful only if the running bit is set and
 145          * fw_status != 0xff (bad PCI read)
 146          */
 147         return (fw_status != 0xff) && (fw_status & IONIC_FW_STS_F_RUNNING);
 148 }
 149
 150 int ionic_heartbeat_check(struct ionic *ionic)
 151 {
 152         unsigned long check_time, last_check_time;
 153         struct ionic_dev *idev = &ionic->idev;
 154         struct ionic_lif *lif = ionic->lif;
 155         bool fw_status_ready = true;
 156         bool fw_hb_ready;
 157         u8 fw_generation;
 158         u8 fw_status;
 159         u32 fw_hb;
 160
 161         /* wait a least one second before testing again */
 162         check_time = jiffies;
 163         last_check_time = atomic_long_read(&idev->last_check_time);
 164 do_check_time:
 165         if (time_before(check_time, last_check_time + HZ))
 166                 return 0;
 167         if (!atomic_long_try_cmpxchg_relaxed(&idev->last_check_time,
 168                                              &last_check_time, check_time)) {
 169                 /* if called concurrently, only the first should proceed. */
 170                 dev_dbg(ionic->dev, "%s: do_check_time again\n", __func__);
 171                 goto do_check_time;
 172         }
 173
 174         fw_status = ioread8(&idev->dev_info_regs->fw_status);
 175
 176         /* If fw_status is not ready don't bother with the generation */
 177         if (!ionic_is_fw_running(idev)) {
 178                 fw_status_ready = false;
 179         } else {
 180                 fw_generation = fw_status & IONIC_FW_STS_F_GENERATION;
 181                 if (idev->fw_generation != fw_generation) {
 182                         dev_info(ionic->dev, "FW generation 0x%02x -> 0x%02x\n",
 183                                  idev->fw_generation, fw_generation);
 184
 185                         idev->fw_generation = fw_generation;
 186
 187                         /* If the generation changed, the fw status is not
 188                          * ready so we need to trigger a fw-down cycle.  After
 189                          * the down, the next watchdog will see the fw is up
 190                          * and the generation value stable, so will trigger
 191                          * the fw-up activity.
 192                          *
 193                          * If we had already moved to FW_RESET from a RESET event,
 194                          * it is possible that we never saw the fw_status go to 0,
 195                          * so we fake the current idev->fw_status_ready here to
 196                          * force the transition and get FW up again.
 197                          */
 198                         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
 199                                 idev->fw_status_ready = false;  /* go to running */
 200                         else
 201                                 fw_status_ready = false;        /* go to down */
 202                 }
 203         }
 204
 205         /* is this a transition? */
 206         if (fw_status_ready != idev->fw_status_ready) {
 207                 bool trigger = false;
 208
 209                 if (!fw_status_ready && lif &&
 210                     !test_bit(IONIC_LIF_F_FW_RESET, lif->state) &&
 211                     !test_and_set_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
 212                         dev_info(ionic->dev, "FW stopped 0x%02x\n", fw_status);
 213                         trigger = true;
 214
 215                 } else if (fw_status_ready && lif &&
 216                            test_bit(IONIC_LIF_F_FW_RESET, lif->state) &&
 217                            !test_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
 218                         dev_info(ionic->dev, "FW running 0x%02x\n", fw_status);
 219                         trigger = true;
 220                 }
 221
 222                 if (trigger) {
 223                         struct ionic_deferred_work *work;
 224
 225                         idev->fw_status_ready = fw_status_ready;
 226
 227                         work = kzalloc(sizeof(*work), GFP_ATOMIC);
 228                         if (work) {
 229                                 work->type = IONIC_DW_TYPE_LIF_RESET;
 230                                 work->fw_status = fw_status_ready;
 231                                 ionic_lif_deferred_enqueue(&lif->deferred, work);
 232                         }
 233                 }
 234         }
 235
 236         if (!idev->fw_status_ready)
 237                 return -ENXIO;
 238
 239         /* Because of some variability in the actual FW heartbeat, we
 240          * wait longer than the DEVCMD_TIMEOUT before checking again.
 241          */
 242         last_check_time = idev->last_hb_time;
 243         if (time_before(check_time, last_check_time + DEVCMD_TIMEOUT * 2 * HZ))
 244                 return 0;
 245
 246         fw_hb = ioread32(&idev->dev_info_regs->fw_heartbeat);
 247         fw_hb_ready = fw_hb != idev->last_fw_hb;
 248
 249         /* early FW version had no heartbeat, so fake it */
 250         if (!fw_hb_ready && !fw_hb)
 251                 fw_hb_ready = true;
 252
 253         dev_dbg(ionic->dev, "%s: fw_hb %u last_fw_hb %u ready %u\n",
 254                 __func__, fw_hb, idev->last_fw_hb, fw_hb_ready);
 255
 256         idev->last_fw_hb = fw_hb;
 257
 258         /* log a transition */
 259         if (fw_hb_ready != idev->fw_hb_ready) {
 260                 idev->fw_hb_ready = fw_hb_ready;
 261                 if (!fw_hb_ready)
 262                         dev_info(ionic->dev, "FW heartbeat stalled at %d\n", fw_hb);
 263                 else
 264                         dev_info(ionic->dev, "FW heartbeat restored at %d\n", fw_hb);
 265         }
 266
 267         if (!fw_hb_ready)
 268                 return -ENXIO;
 269
 270         idev->last_hb_time = check_time;
 271
 272         return 0;
 273 }
 274
 275 u8 ionic_dev_cmd_status(struct ionic_dev *idev)
 276 {
 277         return ioread8(&idev->dev_cmd_regs->comp.comp.status);
 278 }
 279
 280 bool ionic_dev_cmd_done(struct ionic_dev *idev)
 281 {
 282         return ioread32(&idev->dev_cmd_regs->done) & IONIC_DEV_CMD_DONE;
 283 }
 284
 285 void ionic_dev_cmd_comp(struct ionic_dev *idev, union ionic_dev_cmd_comp *comp)
 286 {
 287         memcpy_fromio(comp, &idev->dev_cmd_regs->comp, sizeof(*comp));
 288 }
 289
 290 void ionic_dev_cmd_go(struct ionic_dev *idev, union ionic_dev_cmd *cmd)
 291 {
 292         memcpy_toio(&idev->dev_cmd_regs->cmd, cmd, sizeof(*cmd));
 293         iowrite32(0, &idev->dev_cmd_regs->done);
 294         iowrite32(1, &idev->dev_cmd_regs->doorbell);
 295 }
 296
 297 /* Device commands */
 298 void ionic_dev_cmd_identify(struct ionic_dev *idev, u8 ver)
 299 {
 300         union ionic_dev_cmd cmd = {
 301                 .identify.opcode = IONIC_CMD_IDENTIFY,
 302                 .identify.ver = ver,
 303         };
 304
 305         ionic_dev_cmd_go(idev, &cmd);
 306 }
 307
 308 void ionic_dev_cmd_init(struct ionic_dev *idev)
 309 {
 310         union ionic_dev_cmd cmd = {
 311                 .init.opcode = IONIC_CMD_INIT,
 312                 .init.type = 0,
 313         };
 314
 315         ionic_dev_cmd_go(idev, &cmd);
 316 }
 317
 318 void ionic_dev_cmd_reset(struct ionic_dev *idev)
 319 {
 320         union ionic_dev_cmd cmd = {
 321                 .reset.opcode = IONIC_CMD_RESET,
 322         };
 323
 324         ionic_dev_cmd_go(idev, &cmd);
 325 }
 326
 327 /* Port commands */
 328 void ionic_dev_cmd_port_identify(struct ionic_dev *idev)
 329 {
 330         union ionic_dev_cmd cmd = {
 331                 .port_init.opcode = IONIC_CMD_PORT_IDENTIFY,
 332                 .port_init.index = 0,
 333         };
 334
 335         ionic_dev_cmd_go(idev, &cmd);
 336 }
 337
 338 void ionic_dev_cmd_port_init(struct ionic_dev *idev)
 339 {
 340         union ionic_dev_cmd cmd = {
 341                 .port_init.opcode = IONIC_CMD_PORT_INIT,
 342                 .port_init.index = 0,
 343                 .port_init.info_pa = cpu_to_le64(idev->port_info_pa),
 344         };
 345
 346         ionic_dev_cmd_go(idev, &cmd);
 347 }
 348
 349 void ionic_dev_cmd_port_reset(struct ionic_dev *idev)
 350 {
 351         union ionic_dev_cmd cmd = {
 352                 .port_reset.opcode = IONIC_CMD_PORT_RESET,
 353                 .port_reset.index = 0,
 354         };
 355
 356         ionic_dev_cmd_go(idev, &cmd);
 357 }
 358
 359 void ionic_dev_cmd_port_state(struct ionic_dev *idev, u8 state)
 360 {
 361         union ionic_dev_cmd cmd = {
 362                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 363                 .port_setattr.index = 0,
 364                 .port_setattr.attr = IONIC_PORT_ATTR_STATE,
 365                 .port_setattr.state = state,
 366         };
 367
 368         ionic_dev_cmd_go(idev, &cmd);
 369 }
 370
 371 void ionic_dev_cmd_port_speed(struct ionic_dev *idev, u32 speed)
 372 {
 373         union ionic_dev_cmd cmd = {
 374                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 375                 .port_setattr.index = 0,
 376                 .port_setattr.attr = IONIC_PORT_ATTR_SPEED,
 377                 .port_setattr.speed = cpu_to_le32(speed),
 378         };
 379
 380         ionic_dev_cmd_go(idev, &cmd);
 381 }
 382
 383 void ionic_dev_cmd_port_autoneg(struct ionic_dev *idev, u8 an_enable)
 384 {
 385         union ionic_dev_cmd cmd = {
 386                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 387                 .port_setattr.index = 0,
 388                 .port_setattr.attr = IONIC_PORT_ATTR_AUTONEG,
 389                 .port_setattr.an_enable = an_enable,
 390         };
 391
 392         ionic_dev_cmd_go(idev, &cmd);
 393 }
 394
 395 void ionic_dev_cmd_port_fec(struct ionic_dev *idev, u8 fec_type)
 396 {
 397         union ionic_dev_cmd cmd = {
 398                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 399                 .port_setattr.index = 0,
 400                 .port_setattr.attr = IONIC_PORT_ATTR_FEC,
 401                 .port_setattr.fec_type = fec_type,
 402         };
 403
 404         ionic_dev_cmd_go(idev, &cmd);
 405 }
 406
 407 void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type)
 408 {
 409         union ionic_dev_cmd cmd = {
 410                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 411                 .port_setattr.index = 0,
 412                 .port_setattr.attr = IONIC_PORT_ATTR_PAUSE,
 413                 .port_setattr.pause_type = pause_type,
 414         };
 415
 416         ionic_dev_cmd_go(idev, &cmd);
 417 }
 418
 419 /* VF commands */
 420 int ionic_set_vf_config(struct ionic *ionic, int vf, u8 attr, u8 *data)
 421 {
 422         union ionic_dev_cmd cmd = {
 423                 .vf_setattr.opcode = IONIC_CMD_VF_SETATTR,
 424                 .vf_setattr.attr = attr,
 425                 .vf_setattr.vf_index = cpu_to_le16(vf),
 426         };
 427         int err;
 428
 429         switch (attr) {
 430         case IONIC_VF_ATTR_SPOOFCHK:
 431                 cmd.vf_setattr.spoofchk = *data;
 432                 dev_dbg(ionic->dev, "%s: vf %d spoof %d\n",
 433                         __func__, vf, *data);
 434                 break;
 435         case IONIC_VF_ATTR_TRUST:
 436                 cmd.vf_setattr.trust = *data;
 437                 dev_dbg(ionic->dev, "%s: vf %d trust %d\n",
 438                         __func__, vf, *data);
 439                 break;
 440         case IONIC_VF_ATTR_LINKSTATE:
 441                 cmd.vf_setattr.linkstate = *data;
 442                 dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n",
 443                         __func__, vf, *data);
 444                 break;
 445         case IONIC_VF_ATTR_MAC:
 446                 ether_addr_copy(cmd.vf_setattr.macaddr, data);
 447                 dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n",
 448                         __func__, vf, data);
 449                 break;
 450         case IONIC_VF_ATTR_VLAN:
 451                 cmd.vf_setattr.vlanid = cpu_to_le16(*(u16 *)data);
 452                 dev_dbg(ionic->dev, "%s: vf %d vlan %d\n",
 453                         __func__, vf, *(u16 *)data);
 454                 break;
 455         case IONIC_VF_ATTR_RATE:
 456                 cmd.vf_setattr.maxrate = cpu_to_le32(*(u32 *)data);
 457                 dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n",
 458                         __func__, vf, *(u32 *)data);
 459                 break;
 460         case IONIC_VF_ATTR_STATSADDR:
 461                 cmd.vf_setattr.stats_pa = cpu_to_le64(*(u64 *)data);
 462                 dev_dbg(ionic->dev, "%s: vf %d stats_pa 0x%08llx\n",
 463                         __func__, vf, *(u64 *)data);
 464                 break;
 465         default:
 466                 return -EINVAL;
 467         }
 468
 469         mutex_lock(&ionic->dev_cmd_lock);
 470         ionic_dev_cmd_go(&ionic->idev, &cmd);
 471         err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
 472         mutex_unlock(&ionic->dev_cmd_lock);
 473
 474         return err;
 475 }
 476
 477 int ionic_dev_cmd_vf_getattr(struct ionic *ionic, int vf, u8 attr,
 478                              struct ionic_vf_getattr_comp *comp)
 479 {
 480         union ionic_dev_cmd cmd = {
 481                 .vf_getattr.opcode = IONIC_CMD_VF_GETATTR,
 482                 .vf_getattr.attr = attr,
 483                 .vf_getattr.vf_index = cpu_to_le16(vf),
 484         };
 485         int err;
 486
 487         if (vf >= ionic->num_vfs)
 488                 return -EINVAL;
 489
 490         switch (attr) {
 491         case IONIC_VF_ATTR_SPOOFCHK:
 492         case IONIC_VF_ATTR_TRUST:
 493         case IONIC_VF_ATTR_LINKSTATE:
 494         case IONIC_VF_ATTR_MAC:
 495         case IONIC_VF_ATTR_VLAN:
 496         case IONIC_VF_ATTR_RATE:
 497                 break;
 498         case IONIC_VF_ATTR_STATSADDR:
 499         default:
 500                 return -EINVAL;
 501         }
 502
 503         mutex_lock(&ionic->dev_cmd_lock);
 504         ionic_dev_cmd_go(&ionic->idev, &cmd);
 505         err = ionic_dev_cmd_wait_nomsg(ionic, DEVCMD_TIMEOUT);
 506         memcpy_fromio(comp, &ionic->idev.dev_cmd_regs->comp.vf_getattr,
 507                       sizeof(*comp));
 508         mutex_unlock(&ionic->dev_cmd_lock);
 509
 510         if (err && comp->status != IONIC_RC_ENOSUPP)
 511                 ionic_dev_cmd_dev_err_print(ionic, cmd.vf_getattr.opcode,
 512                                             comp->status, err);
 513
 514         return err;
 515 }
 516
 517 /* LIF commands */
 518 void ionic_dev_cmd_queue_identify(struct ionic_dev *idev,
 519                                   u16 lif_type, u8 qtype, u8 qver)
 520 {
 521         union ionic_dev_cmd cmd = {
 522                 .q_identify.opcode = IONIC_CMD_Q_IDENTIFY,
 523                 .q_identify.lif_type = cpu_to_le16(lif_type),
 524                 .q_identify.type = qtype,
 525                 .q_identify.ver = qver,
 526         };
 527
 528         ionic_dev_cmd_go(idev, &cmd);
 529 }
 530
 531 void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, u8 type, u8 ver)
 532 {
 533         union ionic_dev_cmd cmd = {
 534                 .lif_identify.opcode = IONIC_CMD_LIF_IDENTIFY,
 535                 .lif_identify.type = type,
 536                 .lif_identify.ver = ver,
 537         };
 538
 539         ionic_dev_cmd_go(idev, &cmd);
 540 }
 541
 542 void ionic_dev_cmd_lif_init(struct ionic_dev *idev, u16 lif_index,
 543                             dma_addr_t info_pa)
 544 {
 545         union ionic_dev_cmd cmd = {
 546                 .lif_init.opcode = IONIC_CMD_LIF_INIT,
 547                 .lif_init.index = cpu_to_le16(lif_index),
 548                 .lif_init.info_pa = cpu_to_le64(info_pa),
 549         };
 550
 551         ionic_dev_cmd_go(idev, &cmd);
 552 }
 553
 554 void ionic_dev_cmd_lif_reset(struct ionic_dev *idev, u16 lif_index)
 555 {
 556         union ionic_dev_cmd cmd = {
 557                 .lif_init.opcode = IONIC_CMD_LIF_RESET,
 558                 .lif_init.index = cpu_to_le16(lif_index),
 559         };
 560
 561         ionic_dev_cmd_go(idev, &cmd);
 562 }
 563
 564 void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq,
 565                                u16 lif_index, u16 intr_index)
 566 {
 567         struct ionic_queue *q = &qcq->q;
 568         struct ionic_cq *cq = &qcq->cq;
 569
 570         union ionic_dev_cmd cmd = {
 571                 .q_init.opcode = IONIC_CMD_Q_INIT,
 572                 .q_init.lif_index = cpu_to_le16(lif_index),
 573                 .q_init.type = q->type,
 574                 .q_init.ver = qcq->q.lif->qtype_info[q->type].version,
 575                 .q_init.index = cpu_to_le32(q->index),
 576                 .q_init.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 577                                             IONIC_QINIT_F_ENA),
 578                 .q_init.pid = cpu_to_le16(q->pid),
 579                 .q_init.intr_index = cpu_to_le16(intr_index),
 580                 .q_init.ring_size = ilog2(q->num_descs),
 581                 .q_init.ring_base = cpu_to_le64(q->base_pa),
 582                 .q_init.cq_ring_base = cpu_to_le64(cq->base_pa),
 583         };
 584
 585         ionic_dev_cmd_go(idev, &cmd);
 586 }
 587
 588 int ionic_db_page_num(struct ionic_lif *lif, int pid)
 589 {
 590         return (lif->hw_index * lif->dbid_count) + pid;
 591 }
 592
 593 int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq,
 594                   struct ionic_intr_info *intr,
 595                   unsigned int num_descs, size_t desc_size)
 596 {
 597         unsigned int ring_size;
 598
 599         if (desc_size == 0 || !is_power_of_2(num_descs))
 600                 return -EINVAL;
 601
 602         ring_size = ilog2(num_descs);
 603         if (ring_size < 2 || ring_size > 16)
 604                 return -EINVAL;
 605
 606         cq->lif = lif;
 607         cq->bound_intr = intr;
 608         cq->num_descs = num_descs;
 609         cq->desc_size = desc_size;
 610         cq->tail_idx = 0;
 611         cq->done_color = 1;
 612
 613         return 0;
 614 }
 615
 616 void ionic_cq_map(struct ionic_cq *cq, void *base, dma_addr_t base_pa)
 617 {
 618         struct ionic_cq_info *cur;
 619         unsigned int i;
 620
 621         cq->base = base;
 622         cq->base_pa = base_pa;
 623
 624         for (i = 0, cur = cq->info; i < cq->num_descs; i++, cur++)
 625                 cur->cq_desc = base + (i * cq->desc_size);
 626 }
 627
 628 void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q)
 629 {
 630         cq->bound_q = q;
 631 }
 632
 633 unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do,
 634                               ionic_cq_cb cb, ionic_cq_done_cb done_cb,
 635                               void *done_arg)
 636 {
 637         struct ionic_cq_info *cq_info;
 638         unsigned int work_done = 0;
 639
 640         if (work_to_do == 0)
 641                 return 0;
 642
 643         cq_info = &cq->info[cq->tail_idx];
 644         while (cb(cq, cq_info)) {
 645                 if (cq->tail_idx == cq->num_descs - 1)
 646                         cq->done_color = !cq->done_color;
 647                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
 648                 cq_info = &cq->info[cq->tail_idx];
 649
 650                 if (++work_done >= work_to_do)
 651                         break;
 652         }
 653
 654         if (work_done && done_cb)
 655                 done_cb(done_arg);
 656
 657         return work_done;
 658 }
 659
 660 int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev,
 661                  struct ionic_queue *q, unsigned int index, const char *name,
 662                  unsigned int num_descs, size_t desc_size,
 663                  size_t sg_desc_size, unsigned int pid)
 664 {
 665         unsigned int ring_size;
 666
 667         if (desc_size == 0 || !is_power_of_2(num_descs))
 668                 return -EINVAL;
 669
 670         ring_size = ilog2(num_descs);
 671         if (ring_size < 2 || ring_size > 16)
 672                 return -EINVAL;
 673
 674         q->lif = lif;
 675         q->idev = idev;
 676         q->index = index;
 677         q->num_descs = num_descs;
 678         q->desc_size = desc_size;
 679         q->sg_desc_size = sg_desc_size;
 680         q->tail_idx = 0;
 681         q->head_idx = 0;
 682         q->pid = pid;
 683
 684         snprintf(q->name, sizeof(q->name), "L%d-%s%u", lif->index, name, index);
 685
 686         return 0;
 687 }
 688
 689 void ionic_q_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
 690 {
 691         struct ionic_desc_info *cur;
 692         unsigned int i;
 693
 694         q->base = base;
 695         q->base_pa = base_pa;
 696
 697         for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
 698                 cur->desc = base + (i * q->desc_size);
 699 }
 700
 701 void ionic_q_sg_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
 702 {
 703         struct ionic_desc_info *cur;
 704         unsigned int i;
 705
 706         q->sg_base = base;
 707         q->sg_base_pa = base_pa;
 708
 709         for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
 710                 cur->sg_desc = base + (i * q->sg_desc_size);
 711 }
 712
 713 void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb,
 714                   void *cb_arg)
 715 {
 716         struct ionic_desc_info *desc_info;
 717         struct ionic_lif *lif = q->lif;
 718         struct device *dev = q->dev;
 719
 720         desc_info = &q->info[q->head_idx];
 721         desc_info->cb = cb;
 722         desc_info->cb_arg = cb_arg;
 723
 724         q->head_idx = (q->head_idx + 1) & (q->num_descs - 1);
 725
 726         dev_dbg(dev, "lif=%d qname=%s qid=%d qtype=%d p_index=%d ringdb=%d\n",
 727                 q->lif->index, q->name, q->hw_type, q->hw_index,
 728                 q->head_idx, ring_doorbell);
 729
 730         if (ring_doorbell)
 731                 ionic_dbell_ring(lif->kern_dbpage, q->hw_type,
 732                                  q->dbval | q->head_idx);
 733 }
 734
 735 static bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos)
 736 {
 737         unsigned int mask, tail, head;
 738
 739         mask = q->num_descs - 1;
 740         tail = q->tail_idx;
 741         head = q->head_idx;
 742
 743         return ((pos - tail) & mask) < ((head - tail) & mask);
 744 }
 745
 746 void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info,
 747                      unsigned int stop_index)
 748 {
 749         struct ionic_desc_info *desc_info;
 750         ionic_desc_cb cb;
 751         void *cb_arg;
 752         u16 index;
 753
 754         /* check for empty queue */
 755         if (q->tail_idx == q->head_idx)
 756                 return;
 757
 758         /* stop index must be for a descriptor that is not yet completed */
 759         if (unlikely(!ionic_q_is_posted(q, stop_index)))
 760                 dev_err(q->dev,
 761                         "ionic stop is not posted %s stop %u tail %u head %u\n",
 762                         q->name, stop_index, q->tail_idx, q->head_idx);
 763
 764         do {
 765                 desc_info = &q->info[q->tail_idx];
 766                 index = q->tail_idx;
 767                 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 768
 769                 cb = desc_info->cb;
 770                 cb_arg = desc_info->cb_arg;
 771
 772                 desc_info->cb = NULL;
 773                 desc_info->cb_arg = NULL;
 774
 775                 if (cb)
 776                         cb(q, desc_info, cq_info, cb_arg);
 777         } while (index != stop_index);
 778 }