drivers/net/wireless/ath/ath10k/ce.c

   1 /*
   2  * Copyright (c) 2005-2011 Atheros Communications Inc.
   3  * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
   4  * Copyright (c) 2018 The Linux Foundation. All rights reserved.
   5  *
   6  * Permission to use, copy, modify, and/or distribute this software for any
   7  * purpose with or without fee is hereby granted, provided that the above
   8  * copyright notice and this permission notice appear in all copies.
   9  *
  10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  17  */
  18
  19 #include "hif.h"
  20 #include "ce.h"
  21 #include "debug.h"
  22
  23 /*
  24  * Support for Copy Engine hardware, which is mainly used for
  25  * communication between Host and Target over a PCIe interconnect.
  26  */
  27
  28 /*
  29  * A single CopyEngine (CE) comprises two "rings":
  30  *   a source ring
  31  *   a destination ring
  32  *
  33  * Each ring consists of a number of descriptors which specify
  34  * an address, length, and meta-data.
  35  *
  36  * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)
  37  * controls one ring and the other side controls the other ring.
  38  * The source side chooses when to initiate a transfer and it
  39  * chooses what to send (buffer address, length). The destination
  40  * side keeps a supply of "anonymous receive buffers" available and
  41  * it handles incoming data as it arrives (when the destination
  42  * receives an interrupt).
  43  *
  44  * The sender may send a simple buffer (address/length) or it may
  45  * send a small list of buffers.  When a small list is sent, hardware
  46  * "gathers" these and they end up in a single destination buffer
  47  * with a single interrupt.
  48  *
  49  * There are several "contexts" managed by this layer -- more, it
  50  * may seem -- than should be needed. These are provided mainly for
  51  * maximum flexibility and especially to facilitate a simpler HIF
  52  * implementation. There are per-CopyEngine recv, send, and watermark
  53  * contexts. These are supplied by the caller when a recv, send,
  54  * or watermark handler is established and they are echoed back to
  55  * the caller when the respective callbacks are invoked. There is
  56  * also a per-transfer context supplied by the caller when a buffer
  57  * (or sendlist) is sent and when a buffer is enqueued for recv.
  58  * These per-transfer contexts are echoed back to the caller when
  59  * the buffer is sent/received.
  60  */
  61
  62 static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
  63                                         struct ath10k_ce_pipe *ce_state)
  64 {
  65         u32 ce_id = ce_state->id;
  66         u32 addr = 0;
  67
  68         switch (ce_id) {
  69         case 0:
  70                 addr = 0x00032000;
  71                 break;
  72         case 3:
  73                 addr = 0x0003200C;
  74                 break;
  75         case 4:
  76                 addr = 0x00032010;
  77                 break;
  78         case 5:
  79                 addr = 0x00032014;
  80                 break;
  81         case 7:
  82                 addr = 0x0003201C;
  83                 break;
  84         default:
  85                 ath10k_warn(ar, "invalid CE id: %d", ce_id);
  86                 break;
  87         }
  88         return addr;
  89 }
  90
  91 static inline u32 shadow_dst_wr_ind_addr(struct ath10k *ar,
  92                                          struct ath10k_ce_pipe *ce_state)
  93 {
  94         u32 ce_id = ce_state->id;
  95         u32 addr = 0;
  96
  97         switch (ce_id) {
  98         case 1:
  99                 addr = 0x00032034;
 100                 break;
 101         case 2:
 102                 addr = 0x00032038;
 103                 break;
 104         case 5:
 105                 addr = 0x00032044;
 106                 break;
 107         case 7:
 108                 addr = 0x0003204C;
 109                 break;
 110         case 8:
 111                 addr = 0x00032050;
 112                 break;
 113         case 9:
 114                 addr = 0x00032054;
 115                 break;
 116         case 10:
 117                 addr = 0x00032058;
 118                 break;
 119         case 11:
 120                 addr = 0x0003205C;
 121                 break;
 122         default:
 123                 ath10k_warn(ar, "invalid CE id: %d", ce_id);
 124                 break;
 125         }
 126
 127         return addr;
 128 }
 129
 130 static inline unsigned int
 131 ath10k_set_ring_byte(unsigned int offset,
 132                      struct ath10k_hw_ce_regs_addr_map *addr_map)
 133 {
 134         return ((offset << addr_map->lsb) & addr_map->mask);
 135 }
 136
 137 static inline unsigned int
 138 ath10k_get_ring_byte(unsigned int offset,
 139                      struct ath10k_hw_ce_regs_addr_map *addr_map)
 140 {
 141         return ((offset & addr_map->mask) >> (addr_map->lsb));
 142 }
 143
 144 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
 145 {
 146         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 147
 148         return ce->bus_ops->read32(ar, offset);
 149 }
 150
 151 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)
 152 {
 153         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 154
 155         ce->bus_ops->write32(ar, offset, value);
 156 }
 157
 158 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
 159                                                        u32 ce_ctrl_addr,
 160                                                        unsigned int n)
 161 {
 162         ath10k_ce_write32(ar, ce_ctrl_addr +
 163                           ar->hw_ce_regs->dst_wr_index_addr, n);
 164 }
 165
 166 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
 167                                                       u32 ce_ctrl_addr)
 168 {
 169         return ath10k_ce_read32(ar, ce_ctrl_addr +
 170                                 ar->hw_ce_regs->dst_wr_index_addr);
 171 }
 172
 173 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
 174                                                       u32 ce_ctrl_addr,
 175                                                       unsigned int n)
 176 {
 177         ath10k_ce_write32(ar, ce_ctrl_addr +
 178                           ar->hw_ce_regs->sr_wr_index_addr, n);
 179 }
 180
 181 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
 182                                                      u32 ce_ctrl_addr)
 183 {
 184         return ath10k_ce_read32(ar, ce_ctrl_addr +
 185                                 ar->hw_ce_regs->sr_wr_index_addr);
 186 }
 187
 188 static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,
 189                                                          u32 ce_id)
 190 {
 191         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 192
 193         return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;
 194 }
 195
 196 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
 197                                                     u32 ce_ctrl_addr)
 198 {
 199         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 200         u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
 201         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 202         u32 index;
 203
 204         if (ar->hw_params.rri_on_ddr &&
 205             (ce_state->attr_flags & CE_ATTR_DIS_INTR))
 206                 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);
 207         else
 208                 index = ath10k_ce_read32(ar, ce_ctrl_addr +
 209                                          ar->hw_ce_regs->current_srri_addr);
 210
 211         return index;
 212 }
 213
 214 static inline void
 215 ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,
 216                                           struct ath10k_ce_pipe *ce_state,
 217                                           unsigned int value)
 218 {
 219         ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);
 220 }
 221
 222 static inline void
 223 ath10k_ce_shadow_dest_ring_write_index_set(struct ath10k *ar,
 224                                            struct ath10k_ce_pipe *ce_state,
 225                                            unsigned int value)
 226 {
 227         ath10k_ce_write32(ar, shadow_dst_wr_ind_addr(ar, ce_state), value);
 228 }
 229
 230 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
 231                                                     u32 ce_id,
 232                                                     u64 addr)
 233 {
 234         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 235         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 236         u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
 237         u32 addr_lo = lower_32_bits(addr);
 238
 239         ath10k_ce_write32(ar, ce_ctrl_addr +
 240                           ar->hw_ce_regs->sr_base_addr_lo, addr_lo);
 241
 242         if (ce_state->ops->ce_set_src_ring_base_addr_hi) {
 243                 ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr,
 244                                                             addr);
 245         }
 246 }
 247
 248 static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar,
 249                                                 u32 ce_ctrl_addr,
 250                                                 u64 addr)
 251 {
 252         u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
 253
 254         ath10k_ce_write32(ar, ce_ctrl_addr +
 255                           ar->hw_ce_regs->sr_base_addr_hi, addr_hi);
 256 }
 257
 258 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
 259                                                u32 ce_ctrl_addr,
 260                                                unsigned int n)
 261 {
 262         ath10k_ce_write32(ar, ce_ctrl_addr +
 263                           ar->hw_ce_regs->sr_size_addr, n);
 264 }
 265
 266 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
 267                                                u32 ce_ctrl_addr,
 268                                                unsigned int n)
 269 {
 270         struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 271
 272         u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 273                                           ctrl_regs->addr);
 274
 275         ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 276                           (ctrl1_addr &  ~(ctrl_regs->dmax->mask)) |
 277                           ath10k_set_ring_byte(n, ctrl_regs->dmax));
 278 }
 279
 280 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
 281                                                     u32 ce_ctrl_addr,
 282                                                     unsigned int n)
 283 {
 284         struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 285
 286         u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 287                                           ctrl_regs->addr);
 288
 289         ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 290                           (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
 291                           ath10k_set_ring_byte(n, ctrl_regs->src_ring));
 292 }
 293
 294 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
 295                                                      u32 ce_ctrl_addr,
 296                                                      unsigned int n)
 297 {
 298         struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 299
 300         u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 301                                           ctrl_regs->addr);
 302
 303         ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 304                           (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
 305                           ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
 306 }
 307
 308 static inline
 309         u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)
 310 {
 311         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 312
 313         return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &
 314                 CE_DDR_RRI_MASK;
 315 }
 316
 317 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
 318                                                      u32 ce_ctrl_addr)
 319 {
 320         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 321         u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
 322         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 323         u32 index;
 324
 325         if (ar->hw_params.rri_on_ddr &&
 326             (ce_state->attr_flags & CE_ATTR_DIS_INTR))
 327                 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);
 328         else
 329                 index = ath10k_ce_read32(ar, ce_ctrl_addr +
 330                                          ar->hw_ce_regs->current_drri_addr);
 331
 332         return index;
 333 }
 334
 335 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
 336                                                      u32 ce_id,
 337                                                      u64 addr)
 338 {
 339         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 340         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 341         u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
 342         u32 addr_lo = lower_32_bits(addr);
 343
 344         ath10k_ce_write32(ar, ce_ctrl_addr +
 345                           ar->hw_ce_regs->dr_base_addr_lo, addr_lo);
 346
 347         if (ce_state->ops->ce_set_dest_ring_base_addr_hi) {
 348                 ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr,
 349                                                              addr);
 350         }
 351 }
 352
 353 static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar,
 354                                                  u32 ce_ctrl_addr,
 355                                                  u64 addr)
 356 {
 357         u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
 358         u32 reg_value;
 359
 360         reg_value = ath10k_ce_read32(ar, ce_ctrl_addr +
 361                                      ar->hw_ce_regs->dr_base_addr_hi);
 362         reg_value &= ~CE_DESC_ADDR_HI_MASK;
 363         reg_value |= addr_hi;
 364         ath10k_ce_write32(ar, ce_ctrl_addr +
 365                           ar->hw_ce_regs->dr_base_addr_hi, reg_value);
 366 }
 367
 368 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
 369                                                 u32 ce_ctrl_addr,
 370                                                 unsigned int n)
 371 {
 372         ath10k_ce_write32(ar, ce_ctrl_addr +
 373                           ar->hw_ce_regs->dr_size_addr, n);
 374 }
 375
 376 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
 377                                                    u32 ce_ctrl_addr,
 378                                                    unsigned int n)
 379 {
 380         struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
 381         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
 382
 383         ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
 384                           (addr & ~(srcr_wm->wm_high->mask)) |
 385                           (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
 386 }
 387
 388 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
 389                                                   u32 ce_ctrl_addr,
 390                                                   unsigned int n)
 391 {
 392         struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
 393         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
 394
 395         ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
 396                           (addr & ~(srcr_wm->wm_low->mask)) |
 397                           (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
 398 }
 399
 400 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
 401                                                     u32 ce_ctrl_addr,
 402                                                     unsigned int n)
 403 {
 404         struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
 405         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
 406
 407         ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
 408                           (addr & ~(dstr_wm->wm_high->mask)) |
 409                           (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
 410 }
 411
 412 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
 413                                                    u32 ce_ctrl_addr,
 414                                                    unsigned int n)
 415 {
 416         struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
 417         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
 418
 419         ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
 420                           (addr & ~(dstr_wm->wm_low->mask)) |
 421                           (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
 422 }
 423
 424 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
 425                                                         u32 ce_ctrl_addr)
 426 {
 427         struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
 428
 429         u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 430                                             ar->hw_ce_regs->host_ie_addr);
 431
 432         ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 433                           host_ie_addr | host_ie->copy_complete->mask);
 434 }
 435
 436 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
 437                                                         u32 ce_ctrl_addr)
 438 {
 439         struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
 440
 441         u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 442                                             ar->hw_ce_regs->host_ie_addr);
 443
 444         ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 445                           host_ie_addr & ~(host_ie->copy_complete->mask));
 446 }
 447
 448 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
 449                                                     u32 ce_ctrl_addr)
 450 {
 451         struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
 452
 453         u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 454                                             ar->hw_ce_regs->host_ie_addr);
 455
 456         ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 457                           host_ie_addr & ~(wm_regs->wm_mask));
 458 }
 459
 460 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar,
 461                                                u32 ce_ctrl_addr)
 462 {
 463         struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
 464
 465         u32 misc_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 466                                             ar->hw_ce_regs->misc_ie_addr);
 467
 468         ath10k_ce_write32(ar,
 469                           ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
 470                           misc_ie_addr | misc_regs->err_mask);
 471 }
 472
 473 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
 474                                                 u32 ce_ctrl_addr)
 475 {
 476         struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
 477
 478         u32 misc_ie_addr = ath10k_ce_read32(ar,
 479                         ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);
 480
 481         ath10k_ce_write32(ar,
 482                           ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
 483                           misc_ie_addr & ~(misc_regs->err_mask));
 484 }
 485
 486 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
 487                                                      u32 ce_ctrl_addr,
 488                                                      unsigned int mask)
 489 {
 490         struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
 491
 492         ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
 493 }
 494
 495 /*
 496  * Guts of ath10k_ce_send.
 497  * The caller takes responsibility for any needed locking.
 498  */
 499 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
 500                                   void *per_transfer_context,
 501                                   dma_addr_t buffer,
 502                                   unsigned int nbytes,
 503                                   unsigned int transfer_id,
 504                                   unsigned int flags)
 505 {
 506         struct ath10k *ar = ce_state->ar;
 507         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
 508         struct ce_desc *desc, sdesc;
 509         unsigned int nentries_mask = src_ring->nentries_mask;
 510         unsigned int sw_index = src_ring->sw_index;
 511         unsigned int write_index = src_ring->write_index;
 512         u32 ctrl_addr = ce_state->ctrl_addr;
 513         u32 desc_flags = 0;
 514         int ret = 0;
 515
 516         if (nbytes > ce_state->src_sz_max)
 517                 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
 518                             __func__, nbytes, ce_state->src_sz_max);
 519
 520         if (unlikely(CE_RING_DELTA(nentries_mask,
 521                                    write_index, sw_index - 1) <= 0)) {
 522                 ret = -ENOSR;
 523                 goto exit;
 524         }
 525
 526         desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
 527                                    write_index);
 528
 529         desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
 530
 531         if (flags & CE_SEND_FLAG_GATHER)
 532                 desc_flags |= CE_DESC_FLAGS_GATHER;
 533         if (flags & CE_SEND_FLAG_BYTE_SWAP)
 534                 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
 535
 536         sdesc.addr   = __cpu_to_le32(buffer);
 537         sdesc.nbytes = __cpu_to_le16(nbytes);
 538         sdesc.flags  = __cpu_to_le16(desc_flags);
 539
 540         *desc = sdesc;
 541
 542         src_ring->per_transfer_context[write_index] = per_transfer_context;
 543
 544         /* Update Source Ring Write Index */
 545         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 546
 547         /* WORKAROUND */
 548         if (!(flags & CE_SEND_FLAG_GATHER))
 549                 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
 550
 551         src_ring->write_index = write_index;
 552 exit:
 553         return ret;
 554 }
 555
 556 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
 557                                      void *per_transfer_context,
 558                                      dma_addr_t buffer,
 559                                      unsigned int nbytes,
 560                                      unsigned int transfer_id,
 561                                      unsigned int flags)
 562 {
 563         struct ath10k *ar = ce_state->ar;
 564         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
 565         struct ce_desc_64 *desc, sdesc;
 566         unsigned int nentries_mask = src_ring->nentries_mask;
 567         unsigned int sw_index;
 568         unsigned int write_index = src_ring->write_index;
 569         u32 ctrl_addr = ce_state->ctrl_addr;
 570         __le32 *addr;
 571         u32 desc_flags = 0;
 572         int ret = 0;
 573
 574         if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
 575                 return -ESHUTDOWN;
 576
 577         if (nbytes > ce_state->src_sz_max)
 578                 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
 579                             __func__, nbytes, ce_state->src_sz_max);
 580
 581         if (ar->hw_params.rri_on_ddr)
 582                 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);
 583         else
 584                 sw_index = src_ring->sw_index;
 585
 586         if (unlikely(CE_RING_DELTA(nentries_mask,
 587                                    write_index, sw_index - 1) <= 0)) {
 588                 ret = -ENOSR;
 589                 goto exit;
 590         }
 591
 592         desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
 593                                       write_index);
 594
 595         desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
 596
 597         if (flags & CE_SEND_FLAG_GATHER)
 598                 desc_flags |= CE_DESC_FLAGS_GATHER;
 599
 600         if (flags & CE_SEND_FLAG_BYTE_SWAP)
 601                 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
 602
 603         addr = (__le32 *)&sdesc.addr;
 604
 605         flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK;
 606         addr[0] = __cpu_to_le32(buffer);
 607         addr[1] = __cpu_to_le32(flags);
 608         if (flags & CE_SEND_FLAG_GATHER)
 609                 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);
 610         else
 611                 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));
 612
 613         sdesc.nbytes = __cpu_to_le16(nbytes);
 614         sdesc.flags  = __cpu_to_le16(desc_flags);
 615
 616         *desc = sdesc;
 617
 618         src_ring->per_transfer_context[write_index] = per_transfer_context;
 619
 620         /* Update Source Ring Write Index */
 621         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 622
 623         if (!(flags & CE_SEND_FLAG_GATHER)) {
 624                 if (ar->hw_params.shadow_reg_support)
 625                         ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,
 626                                                                   write_index);
 627                 else
 628                         ath10k_ce_src_ring_write_index_set(ar, ctrl_addr,
 629                                                            write_index);
 630         }
 631
 632         src_ring->write_index = write_index;
 633 exit:
 634         return ret;
 635 }
 636
 637 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
 638                           void *per_transfer_context,
 639                           dma_addr_t buffer,
 640                           unsigned int nbytes,
 641                           unsigned int transfer_id,
 642                           unsigned int flags)
 643 {
 644         return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
 645                                     buffer, nbytes, transfer_id, flags);
 646 }
 647 EXPORT_SYMBOL(ath10k_ce_send_nolock);
 648
 649 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
 650 {
 651         struct ath10k *ar = pipe->ar;
 652         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 653         struct ath10k_ce_ring *src_ring = pipe->src_ring;
 654         u32 ctrl_addr = pipe->ctrl_addr;
 655
 656         lockdep_assert_held(&ce->ce_lock);
 657
 658         /*
 659          * This function must be called only if there is an incomplete
 660          * scatter-gather transfer (before index register is updated)
 661          * that needs to be cleaned up.
 662          */
 663         if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
 664                 return;
 665
 666         if (WARN_ON_ONCE(src_ring->write_index ==
 667                          ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
 668                 return;
 669
 670         src_ring->write_index--;
 671         src_ring->write_index &= src_ring->nentries_mask;
 672
 673         src_ring->per_transfer_context[src_ring->write_index] = NULL;
 674 }
 675 EXPORT_SYMBOL(__ath10k_ce_send_revert);
 676
 677 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
 678                    void *per_transfer_context,
 679                    dma_addr_t buffer,
 680                    unsigned int nbytes,
 681                    unsigned int transfer_id,
 682                    unsigned int flags)
 683 {
 684         struct ath10k *ar = ce_state->ar;
 685         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 686         int ret;
 687
 688         spin_lock_bh(&ce->ce_lock);
 689         ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
 690                                     buffer, nbytes, transfer_id, flags);
 691         spin_unlock_bh(&ce->ce_lock);
 692
 693         return ret;
 694 }
 695 EXPORT_SYMBOL(ath10k_ce_send);
 696
 697 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
 698 {
 699         struct ath10k *ar = pipe->ar;
 700         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 701         int delta;
 702
 703         spin_lock_bh(&ce->ce_lock);
 704         delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
 705                               pipe->src_ring->write_index,
 706                               pipe->src_ring->sw_index - 1);
 707         spin_unlock_bh(&ce->ce_lock);
 708
 709         return delta;
 710 }
 711 EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);
 712
 713 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
 714 {
 715         struct ath10k *ar = pipe->ar;
 716         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 717         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 718         unsigned int nentries_mask = dest_ring->nentries_mask;
 719         unsigned int write_index = dest_ring->write_index;
 720         unsigned int sw_index = dest_ring->sw_index;
 721
 722         lockdep_assert_held(&ce->ce_lock);
 723
 724         return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
 725 }
 726 EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);
 727
 728 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
 729                                    dma_addr_t paddr)
 730 {
 731         struct ath10k *ar = pipe->ar;
 732         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 733         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 734         unsigned int nentries_mask = dest_ring->nentries_mask;
 735         unsigned int write_index = dest_ring->write_index;
 736         unsigned int sw_index = dest_ring->sw_index;
 737         struct ce_desc *base = dest_ring->base_addr_owner_space;
 738         struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
 739         u32 ctrl_addr = pipe->ctrl_addr;
 740
 741         lockdep_assert_held(&ce->ce_lock);
 742
 743         if ((pipe->id != 5) &&
 744             CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
 745                 return -ENOSPC;
 746
 747         desc->addr = __cpu_to_le32(paddr);
 748         desc->nbytes = 0;
 749
 750         dest_ring->per_transfer_context[write_index] = ctx;
 751         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 752         ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 753         dest_ring->write_index = write_index;
 754
 755         return 0;
 756 }
 757
 758 static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,
 759                                       void *ctx,
 760                                       dma_addr_t paddr)
 761 {
 762         struct ath10k *ar = pipe->ar;
 763         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 764         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 765         unsigned int nentries_mask = dest_ring->nentries_mask;
 766         unsigned int write_index = dest_ring->write_index;
 767         unsigned int sw_index = dest_ring->sw_index;
 768         struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
 769         struct ce_desc_64 *desc =
 770                         CE_DEST_RING_TO_DESC_64(base, write_index);
 771         u32 ctrl_addr = pipe->ctrl_addr;
 772
 773         lockdep_assert_held(&ce->ce_lock);
 774
 775         if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
 776                 return -ENOSPC;
 777
 778         desc->addr = __cpu_to_le64(paddr);
 779         desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK);
 780
 781         desc->nbytes = 0;
 782
 783         dest_ring->per_transfer_context[write_index] = ctx;
 784         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 785         ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 786         dest_ring->write_index = write_index;
 787
 788         return 0;
 789 }
 790
 791 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
 792 {
 793         struct ath10k *ar = pipe->ar;
 794         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 795         unsigned int nentries_mask = dest_ring->nentries_mask;
 796         unsigned int write_index = dest_ring->write_index;
 797         u32 ctrl_addr = pipe->ctrl_addr;
 798         u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
 799
 800         /* Prevent CE ring stuck issue that will occur when ring is full.
 801          * Make sure that write index is 1 less than read index.
 802          */
 803         if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)
 804                 nentries -= 1;
 805
 806         write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
 807         ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 808         dest_ring->write_index = write_index;
 809 }
 810 EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);
 811
 812 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
 813                           dma_addr_t paddr)
 814 {
 815         struct ath10k *ar = pipe->ar;
 816         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 817         int ret;
 818
 819         spin_lock_bh(&ce->ce_lock);
 820         ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);
 821         spin_unlock_bh(&ce->ce_lock);
 822
 823         return ret;
 824 }
 825 EXPORT_SYMBOL(ath10k_ce_rx_post_buf);
 826
 827 /*
 828  * Guts of ath10k_ce_completed_recv_next.
 829  * The caller takes responsibility for any necessary locking.
 830  */
 831 static int
 832          _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
 833                                                void **per_transfer_contextp,
 834                                                unsigned int *nbytesp)
 835 {
 836         struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
 837         unsigned int nentries_mask = dest_ring->nentries_mask;
 838         unsigned int sw_index = dest_ring->sw_index;
 839
 840         struct ce_desc *base = dest_ring->base_addr_owner_space;
 841         struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
 842         struct ce_desc sdesc;
 843         u16 nbytes;
 844
 845         /* Copy in one go for performance reasons */
 846         sdesc = *desc;
 847
 848         nbytes = __le16_to_cpu(sdesc.nbytes);
 849         if (nbytes == 0) {
 850                 /*
 851                  * This closes a relatively unusual race where the Host
 852                  * sees the updated DRRI before the update to the
 853                  * corresponding descriptor has completed. We treat this
 854                  * as a descriptor that is not yet done.
 855                  */
 856                 return -EIO;
 857         }
 858
 859         desc->nbytes = 0;
 860
 861         /* Return data from completed destination descriptor */
 862         *nbytesp = nbytes;
 863
 864         if (per_transfer_contextp)
 865                 *per_transfer_contextp =
 866                         dest_ring->per_transfer_context[sw_index];
 867
 868         /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
 869          * So update transfer context all CEs except CE5.
 870          */
 871         if (ce_state->id != 5)
 872                 dest_ring->per_transfer_context[sw_index] = NULL;
 873
 874         /* Update sw_index */
 875         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 876         dest_ring->sw_index = sw_index;
 877
 878         return 0;
 879 }
 880
 881 static int
 882 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,
 883                                          void **per_transfer_contextp,
 884                                          unsigned int *nbytesp)
 885 {
 886         struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
 887         unsigned int nentries_mask = dest_ring->nentries_mask;
 888         unsigned int sw_index = dest_ring->sw_index;
 889         struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
 890         struct ce_desc_64 *desc =
 891                 CE_DEST_RING_TO_DESC_64(base, sw_index);
 892         struct ce_desc_64 sdesc;
 893         u16 nbytes;
 894
 895         /* Copy in one go for performance reasons */
 896         sdesc = *desc;
 897
 898         nbytes = __le16_to_cpu(sdesc.nbytes);
 899         if (nbytes == 0) {
 900                 /* This closes a relatively unusual race where the Host
 901                  * sees the updated DRRI before the update to the
 902                  * corresponding descriptor has completed. We treat this
 903                  * as a descriptor that is not yet done.
 904                  */
 905                 return -EIO;
 906         }
 907
 908         desc->nbytes = 0;
 909
 910         /* Return data from completed destination descriptor */
 911         *nbytesp = nbytes;
 912
 913         if (per_transfer_contextp)
 914                 *per_transfer_contextp =
 915                         dest_ring->per_transfer_context[sw_index];
 916
 917         /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
 918          * So update transfer context all CEs except CE5.
 919          */
 920         if (ce_state->id != 5)
 921                 dest_ring->per_transfer_context[sw_index] = NULL;
 922
 923         /* Update sw_index */
 924         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 925         dest_ring->sw_index = sw_index;
 926
 927         return 0;
 928 }
 929
 930 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
 931                                          void **per_transfer_ctx,
 932                                          unsigned int *nbytesp)
 933 {
 934         return ce_state->ops->ce_completed_recv_next_nolock(ce_state,
 935                                                             per_transfer_ctx,
 936                                                             nbytesp);
 937 }
 938 EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);
 939
 940 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
 941                                   void **per_transfer_contextp,
 942                                   unsigned int *nbytesp)
 943 {
 944         struct ath10k *ar = ce_state->ar;
 945         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 946         int ret;
 947
 948         spin_lock_bh(&ce->ce_lock);
 949         ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,
 950                                                    per_transfer_contextp,
 951                                                    nbytesp);
 952
 953         spin_unlock_bh(&ce->ce_lock);
 954
 955         return ret;
 956 }
 957 EXPORT_SYMBOL(ath10k_ce_completed_recv_next);
 958
 959 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
 960                                        void **per_transfer_contextp,
 961                                        dma_addr_t *bufferp)
 962 {
 963         struct ath10k_ce_ring *dest_ring;
 964         unsigned int nentries_mask;
 965         unsigned int sw_index;
 966         unsigned int write_index;
 967         int ret;
 968         struct ath10k *ar;
 969         struct ath10k_ce *ce;
 970
 971         dest_ring = ce_state->dest_ring;
 972
 973         if (!dest_ring)
 974                 return -EIO;
 975
 976         ar = ce_state->ar;
 977         ce = ath10k_ce_priv(ar);
 978
 979         spin_lock_bh(&ce->ce_lock);
 980
 981         nentries_mask = dest_ring->nentries_mask;
 982         sw_index = dest_ring->sw_index;
 983         write_index = dest_ring->write_index;
 984         if (write_index != sw_index) {
 985                 struct ce_desc *base = dest_ring->base_addr_owner_space;
 986                 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
 987
 988                 /* Return data from completed destination descriptor */
 989                 *bufferp = __le32_to_cpu(desc->addr);
 990
 991                 if (per_transfer_contextp)
 992                         *per_transfer_contextp =
 993                                 dest_ring->per_transfer_context[sw_index];
 994
 995                 /* sanity */
 996                 dest_ring->per_transfer_context[sw_index] = NULL;
 997                 desc->nbytes = 0;
 998
 999                 /* Update sw_index */
1000                 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1001                 dest_ring->sw_index = sw_index;
1002                 ret = 0;
1003         } else {
1004                 ret = -EIO;
1005         }
1006
1007         spin_unlock_bh(&ce->ce_lock);
1008
1009         return ret;
1010 }
1011
1012 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,
1013                                           void **per_transfer_contextp,
1014                                           dma_addr_t *bufferp)
1015 {
1016         struct ath10k_ce_ring *dest_ring;
1017         unsigned int nentries_mask;
1018         unsigned int sw_index;
1019         unsigned int write_index;
1020         int ret;
1021         struct ath10k *ar;
1022         struct ath10k_ce *ce;
1023
1024         dest_ring = ce_state->dest_ring;
1025
1026         if (!dest_ring)
1027                 return -EIO;
1028
1029         ar = ce_state->ar;
1030         ce = ath10k_ce_priv(ar);
1031
1032         spin_lock_bh(&ce->ce_lock);
1033
1034         nentries_mask = dest_ring->nentries_mask;
1035         sw_index = dest_ring->sw_index;
1036         write_index = dest_ring->write_index;
1037         if (write_index != sw_index) {
1038                 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
1039                 struct ce_desc_64 *desc =
1040                         CE_DEST_RING_TO_DESC_64(base, sw_index);
1041
1042                 /* Return data from completed destination descriptor */
1043                 *bufferp = __le64_to_cpu(desc->addr);
1044
1045                 if (per_transfer_contextp)
1046                         *per_transfer_contextp =
1047                                 dest_ring->per_transfer_context[sw_index];
1048
1049                 /* sanity */
1050                 dest_ring->per_transfer_context[sw_index] = NULL;
1051                 desc->nbytes = 0;
1052
1053                 /* Update sw_index */
1054                 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1055                 dest_ring->sw_index = sw_index;
1056                 ret = 0;
1057         } else {
1058                 ret = -EIO;
1059         }
1060
1061         spin_unlock_bh(&ce->ce_lock);
1062
1063         return ret;
1064 }
1065
1066 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
1067                                void **per_transfer_contextp,
1068                                dma_addr_t *bufferp)
1069 {
1070         return ce_state->ops->ce_revoke_recv_next(ce_state,
1071                                                   per_transfer_contextp,
1072                                                   bufferp);
1073 }
1074 EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);
1075
1076 /*
1077  * Guts of ath10k_ce_completed_send_next.
1078  * The caller takes responsibility for any necessary locking.
1079  */
1080 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1081                                          void **per_transfer_contextp)
1082 {
1083         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1084         u32 ctrl_addr = ce_state->ctrl_addr;
1085         struct ath10k *ar = ce_state->ar;
1086         unsigned int nentries_mask = src_ring->nentries_mask;
1087         unsigned int sw_index = src_ring->sw_index;
1088         unsigned int read_index;
1089         struct ce_desc *desc;
1090
1091         if (src_ring->hw_index == sw_index) {
1092                 /*
1093                  * The SW completion index has caught up with the cached
1094                  * version of the HW completion index.
1095                  * Update the cached HW completion index to see whether
1096                  * the SW has really caught up to the HW, or if the cached
1097                  * value of the HW index has become stale.
1098                  */
1099
1100                 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1101                 if (read_index == 0xffffffff)
1102                         return -ENODEV;
1103
1104                 read_index &= nentries_mask;
1105                 src_ring->hw_index = read_index;
1106         }
1107
1108         if (ar->hw_params.rri_on_ddr)
1109                 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1110         else
1111                 read_index = src_ring->hw_index;
1112
1113         if (read_index == sw_index)
1114                 return -EIO;
1115
1116         if (per_transfer_contextp)
1117                 *per_transfer_contextp =
1118                         src_ring->per_transfer_context[sw_index];
1119
1120         /* sanity */
1121         src_ring->per_transfer_context[sw_index] = NULL;
1122         desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
1123                                    sw_index);
1124         desc->nbytes = 0;
1125
1126         /* Update sw_index */
1127         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1128         src_ring->sw_index = sw_index;
1129
1130         return 0;
1131 }
1132 EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);
1133
1134 static void ath10k_ce_extract_desc_data(struct ath10k *ar,
1135                                         struct ath10k_ce_ring *src_ring,
1136                                         u32 sw_index,
1137                                         dma_addr_t *bufferp,
1138                                         u32 *nbytesp,
1139                                         u32 *transfer_idp)
1140 {
1141                 struct ce_desc *base = src_ring->base_addr_owner_space;
1142                 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
1143
1144                 /* Return data from completed source descriptor */
1145                 *bufferp = __le32_to_cpu(desc->addr);
1146                 *nbytesp = __le16_to_cpu(desc->nbytes);
1147                 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1148                                    CE_DESC_FLAGS_META_DATA);
1149 }
1150
1151 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,
1152                                            struct ath10k_ce_ring *src_ring,
1153                                            u32 sw_index,
1154                                            dma_addr_t *bufferp,
1155                                            u32 *nbytesp,
1156                                            u32 *transfer_idp)
1157 {
1158                 struct ce_desc_64 *base = src_ring->base_addr_owner_space;
1159                 struct ce_desc_64 *desc =
1160                         CE_SRC_RING_TO_DESC_64(base, sw_index);
1161
1162                 /* Return data from completed source descriptor */
1163                 *bufferp = __le64_to_cpu(desc->addr);
1164                 *nbytesp = __le16_to_cpu(desc->nbytes);
1165                 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1166                                    CE_DESC_FLAGS_META_DATA);
1167 }
1168
1169 /* NB: Modeled after ath10k_ce_completed_send_next */
1170 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
1171                                void **per_transfer_contextp,
1172                                dma_addr_t *bufferp,
1173                                unsigned int *nbytesp,
1174                                unsigned int *transfer_idp)
1175 {
1176         struct ath10k_ce_ring *src_ring;
1177         unsigned int nentries_mask;
1178         unsigned int sw_index;
1179         unsigned int write_index;
1180         int ret;
1181         struct ath10k *ar;
1182         struct ath10k_ce *ce;
1183
1184         src_ring = ce_state->src_ring;
1185
1186         if (!src_ring)
1187                 return -EIO;
1188
1189         ar = ce_state->ar;
1190         ce = ath10k_ce_priv(ar);
1191
1192         spin_lock_bh(&ce->ce_lock);
1193
1194         nentries_mask = src_ring->nentries_mask;
1195         sw_index = src_ring->sw_index;
1196         write_index = src_ring->write_index;
1197
1198         if (write_index != sw_index) {
1199                 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,
1200                                                     bufferp, nbytesp,
1201                                                     transfer_idp);
1202
1203                 if (per_transfer_contextp)
1204                         *per_transfer_contextp =
1205                                 src_ring->per_transfer_context[sw_index];
1206
1207                 /* sanity */
1208                 src_ring->per_transfer_context[sw_index] = NULL;
1209
1210                 /* Update sw_index */
1211                 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1212                 src_ring->sw_index = sw_index;
1213                 ret = 0;
1214         } else {
1215                 ret = -EIO;
1216         }
1217
1218         spin_unlock_bh(&ce->ce_lock);
1219
1220         return ret;
1221 }
1222 EXPORT_SYMBOL(ath10k_ce_cancel_send_next);
1223
1224 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
1225                                   void **per_transfer_contextp)
1226 {
1227         struct ath10k *ar = ce_state->ar;
1228         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1229         int ret;
1230
1231         spin_lock_bh(&ce->ce_lock);
1232         ret = ath10k_ce_completed_send_next_nolock(ce_state,
1233                                                    per_transfer_contextp);
1234         spin_unlock_bh(&ce->ce_lock);
1235
1236         return ret;
1237 }
1238 EXPORT_SYMBOL(ath10k_ce_completed_send_next);
1239
1240 /*
1241  * Guts of interrupt handler for per-engine interrupts on a particular CE.
1242  *
1243  * Invokes registered callbacks for recv_complete,
1244  * send_complete, and watermarks.
1245  */
1246 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
1247 {
1248         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1249         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1250         struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
1251         u32 ctrl_addr = ce_state->ctrl_addr;
1252
1253         spin_lock_bh(&ce->ce_lock);
1254
1255         /* Clear the copy-complete interrupts that will be handled here. */
1256         ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
1257                                           wm_regs->cc_mask);
1258
1259         spin_unlock_bh(&ce->ce_lock);
1260
1261         if (ce_state->recv_cb)
1262                 ce_state->recv_cb(ce_state);
1263
1264         if (ce_state->send_cb)
1265                 ce_state->send_cb(ce_state);
1266
1267         spin_lock_bh(&ce->ce_lock);
1268
1269         /*
1270          * Misc CE interrupts are not being handled, but still need
1271          * to be cleared.
1272          */
1273         ath10k_ce_engine_int_status_clear(ar, ctrl_addr, wm_regs->wm_mask);
1274
1275         spin_unlock_bh(&ce->ce_lock);
1276 }
1277 EXPORT_SYMBOL(ath10k_ce_per_engine_service);
1278
1279 /*
1280  * Handler for per-engine interrupts on ALL active CEs.
1281  * This is used in cases where the system is sharing a
1282  * single interrput for all CEs
1283  */
1284
1285 void ath10k_ce_per_engine_service_any(struct ath10k *ar)
1286 {
1287         int ce_id;
1288         u32 intr_summary;
1289
1290         intr_summary = ath10k_ce_interrupt_summary(ar);
1291
1292         for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
1293                 if (intr_summary & (1 << ce_id))
1294                         intr_summary &= ~(1 << ce_id);
1295                 else
1296                         /* no intr pending on this CE */
1297                         continue;
1298
1299                 ath10k_ce_per_engine_service(ar, ce_id);
1300         }
1301 }
1302 EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);
1303
1304 /*
1305  * Adjust interrupts for the copy complete handler.
1306  * If it's needed for either send or recv, then unmask
1307  * this interrupt; otherwise, mask it.
1308  *
1309  * Called with ce_lock held.
1310  */
1311 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
1312 {
1313         u32 ctrl_addr = ce_state->ctrl_addr;
1314         struct ath10k *ar = ce_state->ar;
1315         bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
1316
1317         if ((!disable_copy_compl_intr) &&
1318             (ce_state->send_cb || ce_state->recv_cb))
1319                 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
1320         else
1321                 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1322
1323         ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1324 }
1325
1326 int ath10k_ce_disable_interrupts(struct ath10k *ar)
1327 {
1328         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1329         struct ath10k_ce_pipe *ce_state;
1330         u32 ctrl_addr;
1331         int ce_id;
1332
1333         for (ce_id = 0; ce_id < CE_COUNT; ce_id++) {
1334                 ce_state  = &ce->ce_states[ce_id];
1335                 if (ce_state->attr_flags & CE_ATTR_POLL)
1336                         continue;
1337
1338                 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1339
1340                 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1341                 ath10k_ce_error_intr_disable(ar, ctrl_addr);
1342                 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1343         }
1344
1345         return 0;
1346 }
1347 EXPORT_SYMBOL(ath10k_ce_disable_interrupts);
1348
1349 void ath10k_ce_enable_interrupts(struct ath10k *ar)
1350 {
1351         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1352         int ce_id;
1353         struct ath10k_ce_pipe *ce_state;
1354
1355         /* Enable interrupts for copy engine that
1356          * are not using polling mode.
1357          */
1358         for (ce_id = 0; ce_id < CE_COUNT; ce_id++) {
1359                 ce_state  = &ce->ce_states[ce_id];
1360                 if (ce_state->attr_flags & CE_ATTR_POLL)
1361                         continue;
1362
1363                 ath10k_ce_per_engine_handler_adjust(ce_state);
1364         }
1365 }
1366 EXPORT_SYMBOL(ath10k_ce_enable_interrupts);
1367
1368 static int ath10k_ce_init_src_ring(struct ath10k *ar,
1369                                    unsigned int ce_id,
1370                                    const struct ce_attr *attr)
1371 {
1372         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1373         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1374         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1375         u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1376
1377         nentries = roundup_pow_of_two(attr->src_nentries);
1378
1379         if (ar->hw_params.target_64bit)
1380                 memset(src_ring->base_addr_owner_space, 0,
1381                        nentries * sizeof(struct ce_desc_64));
1382         else
1383                 memset(src_ring->base_addr_owner_space, 0,
1384                        nentries * sizeof(struct ce_desc));
1385
1386         src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1387         src_ring->sw_index &= src_ring->nentries_mask;
1388         src_ring->hw_index = src_ring->sw_index;
1389
1390         src_ring->write_index =
1391                 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
1392         src_ring->write_index &= src_ring->nentries_mask;
1393
1394         ath10k_ce_src_ring_base_addr_set(ar, ce_id,
1395                                          src_ring->base_addr_ce_space);
1396         ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
1397         ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
1398         ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
1399         ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
1400         ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
1401
1402         ath10k_dbg(ar, ATH10K_DBG_BOOT,
1403                    "boot init ce src ring id %d entries %d base_addr %pK\n",
1404                    ce_id, nentries, src_ring->base_addr_owner_space);
1405
1406         return 0;
1407 }
1408
1409 static int ath10k_ce_init_dest_ring(struct ath10k *ar,
1410                                     unsigned int ce_id,
1411                                     const struct ce_attr *attr)
1412 {
1413         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1414         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1415         struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
1416         u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1417
1418         nentries = roundup_pow_of_two(attr->dest_nentries);
1419
1420         if (ar->hw_params.target_64bit)
1421                 memset(dest_ring->base_addr_owner_space, 0,
1422                        nentries * sizeof(struct ce_desc_64));
1423         else
1424                 memset(dest_ring->base_addr_owner_space, 0,
1425                        nentries * sizeof(struct ce_desc));
1426
1427         dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
1428         dest_ring->sw_index &= dest_ring->nentries_mask;
1429         dest_ring->write_index =
1430                 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
1431         dest_ring->write_index &= dest_ring->nentries_mask;
1432
1433         ath10k_ce_dest_ring_base_addr_set(ar, ce_id,
1434                                           dest_ring->base_addr_ce_space);
1435         ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
1436         ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
1437         ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
1438         ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
1439
1440         ath10k_dbg(ar, ATH10K_DBG_BOOT,
1441                    "boot ce dest ring id %d entries %d base_addr %pK\n",
1442                    ce_id, nentries, dest_ring->base_addr_owner_space);
1443
1444         return 0;
1445 }
1446
1447 static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,
1448                                        struct ath10k_ce_ring *src_ring,
1449                                        u32 nentries)
1450 {
1451         src_ring->shadow_base_unaligned = kcalloc(nentries,
1452                                                   sizeof(struct ce_desc_64),
1453                                                   GFP_KERNEL);
1454         if (!src_ring->shadow_base_unaligned)
1455                 return -ENOMEM;
1456
1457         src_ring->shadow_base = (struct ce_desc_64 *)
1458                         PTR_ALIGN(src_ring->shadow_base_unaligned,
1459                                   CE_DESC_RING_ALIGN);
1460         return 0;
1461 }
1462
1463 static struct ath10k_ce_ring *
1464 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
1465                          const struct ce_attr *attr)
1466 {
1467         struct ath10k_ce_ring *src_ring;
1468         u32 nentries = attr->src_nentries;
1469         dma_addr_t base_addr;
1470         int ret;
1471
1472         nentries = roundup_pow_of_two(nentries);
1473
1474         src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1475                                        nentries), GFP_KERNEL);
1476         if (src_ring == NULL)
1477                 return ERR_PTR(-ENOMEM);
1478
1479         src_ring->nentries = nentries;
1480         src_ring->nentries_mask = nentries - 1;
1481
1482         /*
1483          * Legacy platforms that do not support cache
1484          * coherent DMA are unsupported
1485          */
1486         src_ring->base_addr_owner_space_unaligned =
1487                 dma_alloc_coherent(ar->dev,
1488                                    (nentries * sizeof(struct ce_desc) +
1489                                     CE_DESC_RING_ALIGN),
1490                                    &base_addr, GFP_KERNEL);
1491         if (!src_ring->base_addr_owner_space_unaligned) {
1492                 kfree(src_ring);
1493                 return ERR_PTR(-ENOMEM);
1494         }
1495
1496         src_ring->base_addr_ce_space_unaligned = base_addr;
1497
1498         src_ring->base_addr_owner_space =
1499                         PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1500                                   CE_DESC_RING_ALIGN);
1501         src_ring->base_addr_ce_space =
1502                         ALIGN(src_ring->base_addr_ce_space_unaligned,
1503                               CE_DESC_RING_ALIGN);
1504
1505         if (ar->hw_params.shadow_reg_support) {
1506                 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1507                 if (ret) {
1508                         dma_free_coherent(ar->dev,
1509                                           (nentries * sizeof(struct ce_desc_64) +
1510                                            CE_DESC_RING_ALIGN),
1511                                           src_ring->base_addr_owner_space_unaligned,
1512                                           base_addr);
1513                         kfree(src_ring);
1514                         return ERR_PTR(ret);
1515                 }
1516         }
1517
1518         return src_ring;
1519 }
1520
1521 static struct ath10k_ce_ring *
1522 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
1523                             const struct ce_attr *attr)
1524 {
1525         struct ath10k_ce_ring *src_ring;
1526         u32 nentries = attr->src_nentries;
1527         dma_addr_t base_addr;
1528         int ret;
1529
1530         nentries = roundup_pow_of_two(nentries);
1531
1532         src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1533                                        nentries), GFP_KERNEL);
1534         if (!src_ring)
1535                 return ERR_PTR(-ENOMEM);
1536
1537         src_ring->nentries = nentries;
1538         src_ring->nentries_mask = nentries - 1;
1539
1540         /* Legacy platforms that do not support cache
1541          * coherent DMA are unsupported
1542          */
1543         src_ring->base_addr_owner_space_unaligned =
1544                 dma_alloc_coherent(ar->dev,
1545                                    (nentries * sizeof(struct ce_desc_64) +
1546                                     CE_DESC_RING_ALIGN),
1547                                    &base_addr, GFP_KERNEL);
1548         if (!src_ring->base_addr_owner_space_unaligned) {
1549                 kfree(src_ring);
1550                 return ERR_PTR(-ENOMEM);
1551         }
1552
1553         src_ring->base_addr_ce_space_unaligned = base_addr;
1554
1555         src_ring->base_addr_owner_space =
1556                         PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1557                                   CE_DESC_RING_ALIGN);
1558         src_ring->base_addr_ce_space =
1559                         ALIGN(src_ring->base_addr_ce_space_unaligned,
1560                               CE_DESC_RING_ALIGN);
1561
1562         if (ar->hw_params.shadow_reg_support) {
1563                 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1564                 if (ret) {
1565                         dma_free_coherent(ar->dev,
1566                                           (nentries * sizeof(struct ce_desc_64) +
1567                                            CE_DESC_RING_ALIGN),
1568                                           src_ring->base_addr_owner_space_unaligned,
1569                                           base_addr);
1570                         kfree(src_ring);
1571                         return ERR_PTR(ret);
1572                 }
1573         }
1574
1575         return src_ring;
1576 }
1577
1578 static struct ath10k_ce_ring *
1579 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
1580                           const struct ce_attr *attr)
1581 {
1582         struct ath10k_ce_ring *dest_ring;
1583         u32 nentries;
1584         dma_addr_t base_addr;
1585
1586         nentries = roundup_pow_of_two(attr->dest_nentries);
1587
1588         dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1589                                         nentries), GFP_KERNEL);
1590         if (dest_ring == NULL)
1591                 return ERR_PTR(-ENOMEM);
1592
1593         dest_ring->nentries = nentries;
1594         dest_ring->nentries_mask = nentries - 1;
1595
1596         /*
1597          * Legacy platforms that do not support cache
1598          * coherent DMA are unsupported
1599          */
1600         dest_ring->base_addr_owner_space_unaligned =
1601                 dma_alloc_coherent(ar->dev,
1602                                    (nentries * sizeof(struct ce_desc) +
1603                                     CE_DESC_RING_ALIGN),
1604                                    &base_addr, GFP_KERNEL);
1605         if (!dest_ring->base_addr_owner_space_unaligned) {
1606                 kfree(dest_ring);
1607                 return ERR_PTR(-ENOMEM);
1608         }
1609
1610         dest_ring->base_addr_ce_space_unaligned = base_addr;
1611
1612         dest_ring->base_addr_owner_space =
1613                         PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1614                                   CE_DESC_RING_ALIGN);
1615         dest_ring->base_addr_ce_space =
1616                                 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1617                                       CE_DESC_RING_ALIGN);
1618
1619         return dest_ring;
1620 }
1621
1622 static struct ath10k_ce_ring *
1623 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,
1624                              const struct ce_attr *attr)
1625 {
1626         struct ath10k_ce_ring *dest_ring;
1627         u32 nentries;
1628         dma_addr_t base_addr;
1629
1630         nentries = roundup_pow_of_two(attr->dest_nentries);
1631
1632         dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1633                                         nentries), GFP_KERNEL);
1634         if (!dest_ring)
1635                 return ERR_PTR(-ENOMEM);
1636
1637         dest_ring->nentries = nentries;
1638         dest_ring->nentries_mask = nentries - 1;
1639
1640         /* Legacy platforms that do not support cache
1641          * coherent DMA are unsupported
1642          */
1643         dest_ring->base_addr_owner_space_unaligned =
1644                 dma_alloc_coherent(ar->dev,
1645                                    (nentries * sizeof(struct ce_desc_64) +
1646                                     CE_DESC_RING_ALIGN),
1647                                    &base_addr, GFP_KERNEL);
1648         if (!dest_ring->base_addr_owner_space_unaligned) {
1649                 kfree(dest_ring);
1650                 return ERR_PTR(-ENOMEM);
1651         }
1652
1653         dest_ring->base_addr_ce_space_unaligned = base_addr;
1654
1655         /* Correctly initialize memory to 0 to prevent garbage
1656          * data crashing system when download firmware
1657          */
1658         memset(dest_ring->base_addr_owner_space_unaligned, 0,
1659                nentries * sizeof(struct ce_desc_64) + CE_DESC_RING_ALIGN);
1660
1661         dest_ring->base_addr_owner_space =
1662                         PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1663                                   CE_DESC_RING_ALIGN);
1664         dest_ring->base_addr_ce_space =
1665                         ALIGN(dest_ring->base_addr_ce_space_unaligned,
1666                               CE_DESC_RING_ALIGN);
1667
1668         return dest_ring;
1669 }
1670
1671 /*
1672  * Initialize a Copy Engine based on caller-supplied attributes.
1673  * This may be called once to initialize both source and destination
1674  * rings or it may be called twice for separate source and destination
1675  * initialization. It may be that only one side or the other is
1676  * initialized by software/firmware.
1677  */
1678 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
1679                         const struct ce_attr *attr)
1680 {
1681         int ret;
1682
1683         if (attr->src_nentries) {
1684                 ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1685                 if (ret) {
1686                         ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",
1687                                    ce_id, ret);
1688                         return ret;
1689                 }
1690         }
1691
1692         if (attr->dest_nentries) {
1693                 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1694                 if (ret) {
1695                         ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",
1696                                    ce_id, ret);
1697                         return ret;
1698                 }
1699         }
1700
1701         return 0;
1702 }
1703 EXPORT_SYMBOL(ath10k_ce_init_pipe);
1704
1705 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1706 {
1707         u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1708
1709         ath10k_ce_src_ring_base_addr_set(ar, ce_id, 0);
1710         ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);
1711         ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);
1712         ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);
1713 }
1714
1715 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1716 {
1717         u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1718
1719         ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 0);
1720         ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);
1721         ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);
1722 }
1723
1724 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1725 {
1726         ath10k_ce_deinit_src_ring(ar, ce_id);
1727         ath10k_ce_deinit_dest_ring(ar, ce_id);
1728 }
1729 EXPORT_SYMBOL(ath10k_ce_deinit_pipe);
1730
1731 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1732 {
1733         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1734         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1735
1736         if (ce_state->src_ring) {
1737                 if (ar->hw_params.shadow_reg_support)
1738                         kfree(ce_state->src_ring->shadow_base_unaligned);
1739                 dma_free_coherent(ar->dev,
1740                                   (ce_state->src_ring->nentries *
1741                                    sizeof(struct ce_desc) +
1742                                    CE_DESC_RING_ALIGN),
1743                                   ce_state->src_ring->base_addr_owner_space,
1744                                   ce_state->src_ring->base_addr_ce_space);
1745                 kfree(ce_state->src_ring);
1746         }
1747
1748         if (ce_state->dest_ring) {
1749                 dma_free_coherent(ar->dev,
1750                                   (ce_state->dest_ring->nentries *
1751                                    sizeof(struct ce_desc) +
1752                                    CE_DESC_RING_ALIGN),
1753                                   ce_state->dest_ring->base_addr_owner_space,
1754                                   ce_state->dest_ring->base_addr_ce_space);
1755                 kfree(ce_state->dest_ring);
1756         }
1757
1758         ce_state->src_ring = NULL;
1759         ce_state->dest_ring = NULL;
1760 }
1761
1762 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
1763 {
1764         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1765         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1766
1767         if (ce_state->src_ring) {
1768                 if (ar->hw_params.shadow_reg_support)
1769                         kfree(ce_state->src_ring->shadow_base_unaligned);
1770                 dma_free_coherent(ar->dev,
1771                                   (ce_state->src_ring->nentries *
1772                                    sizeof(struct ce_desc_64) +
1773                                    CE_DESC_RING_ALIGN),
1774                                   ce_state->src_ring->base_addr_owner_space,
1775                                   ce_state->src_ring->base_addr_ce_space);
1776                 kfree(ce_state->src_ring);
1777         }
1778
1779         if (ce_state->dest_ring) {
1780                 dma_free_coherent(ar->dev,
1781                                   (ce_state->dest_ring->nentries *
1782                                    sizeof(struct ce_desc_64) +
1783                                    CE_DESC_RING_ALIGN),
1784                                   ce_state->dest_ring->base_addr_owner_space,
1785                                   ce_state->dest_ring->base_addr_ce_space);
1786                 kfree(ce_state->dest_ring);
1787         }
1788
1789         ce_state->src_ring = NULL;
1790         ce_state->dest_ring = NULL;
1791 }
1792
1793 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1794 {
1795         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1796         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1797
1798         ce_state->ops->ce_free_pipe(ar, ce_id);
1799 }
1800 EXPORT_SYMBOL(ath10k_ce_free_pipe);
1801
1802 void ath10k_ce_dump_registers(struct ath10k *ar,
1803                               struct ath10k_fw_crash_data *crash_data)
1804 {
1805         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1806         struct ath10k_ce_crash_data ce_data;
1807         u32 addr, id;
1808
1809         lockdep_assert_held(&ar->data_lock);
1810
1811         ath10k_err(ar, "Copy Engine register dump:\n");
1812
1813         spin_lock_bh(&ce->ce_lock);
1814         for (id = 0; id < CE_COUNT; id++) {
1815                 addr = ath10k_ce_base_address(ar, id);
1816                 ce_data.base_addr = cpu_to_le32(addr);
1817
1818                 ce_data.src_wr_idx =
1819                         cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
1820                 ce_data.src_r_idx =
1821                         cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
1822                 ce_data.dst_wr_idx =
1823                         cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
1824                 ce_data.dst_r_idx =
1825                         cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
1826
1827                 if (crash_data)
1828                         crash_data->ce_crash_data[id] = ce_data;
1829
1830                 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,
1831                            le32_to_cpu(ce_data.base_addr),
1832                            le32_to_cpu(ce_data.src_wr_idx),
1833                            le32_to_cpu(ce_data.src_r_idx),
1834                            le32_to_cpu(ce_data.dst_wr_idx),
1835                            le32_to_cpu(ce_data.dst_r_idx));
1836         }
1837
1838         spin_unlock_bh(&ce->ce_lock);
1839 }
1840 EXPORT_SYMBOL(ath10k_ce_dump_registers);
1841
1842 static const struct ath10k_ce_ops ce_ops = {
1843         .ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
1844         .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,
1845         .ce_rx_post_buf = __ath10k_ce_rx_post_buf,
1846         .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,
1847         .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,
1848         .ce_extract_desc_data = ath10k_ce_extract_desc_data,
1849         .ce_free_pipe = _ath10k_ce_free_pipe,
1850         .ce_send_nolock = _ath10k_ce_send_nolock,
1851         .ce_set_src_ring_base_addr_hi = NULL,
1852         .ce_set_dest_ring_base_addr_hi = NULL,
1853 };
1854
1855 static const struct ath10k_ce_ops ce_64_ops = {
1856         .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,
1857         .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,
1858         .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,
1859         .ce_completed_recv_next_nolock =
1860                                 _ath10k_ce_completed_recv_next_nolock_64,
1861         .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,
1862         .ce_extract_desc_data = ath10k_ce_extract_desc_data_64,
1863         .ce_free_pipe = _ath10k_ce_free_pipe_64,
1864         .ce_send_nolock = _ath10k_ce_send_nolock_64,
1865         .ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi,
1866         .ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi,
1867 };
1868
1869 static void ath10k_ce_set_ops(struct ath10k *ar,
1870                               struct ath10k_ce_pipe *ce_state)
1871 {
1872         switch (ar->hw_rev) {
1873         case ATH10K_HW_WCN3990:
1874                 ce_state->ops = &ce_64_ops;
1875                 break;
1876         default:
1877                 ce_state->ops = &ce_ops;
1878                 break;
1879         }
1880 }
1881
1882 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1883                          const struct ce_attr *attr)
1884 {
1885         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1886         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1887         int ret;
1888
1889         ath10k_ce_set_ops(ar, ce_state);
1890         /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1891          * additional TX locking checks.
1892          *
1893          * For the lack of a better place do the check here.
1894          */
1895         BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1896                      (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1897         BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
1898                      (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1899         BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1900                      (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1901
1902         ce_state->ar = ar;
1903         ce_state->id = ce_id;
1904         ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1905         ce_state->attr_flags = attr->flags;
1906         ce_state->src_sz_max = attr->src_sz_max;
1907
1908         if (attr->src_nentries)
1909                 ce_state->send_cb = attr->send_cb;
1910
1911         if (attr->dest_nentries)
1912                 ce_state->recv_cb = attr->recv_cb;
1913
1914         if (attr->src_nentries) {
1915                 ce_state->src_ring =
1916                         ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);
1917                 if (IS_ERR(ce_state->src_ring)) {
1918                         ret = PTR_ERR(ce_state->src_ring);
1919                         ath10k_err(ar, "failed to alloc CE src ring %d: %d\n",
1920                                    ce_id, ret);
1921                         ce_state->src_ring = NULL;
1922                         return ret;
1923                 }
1924         }
1925
1926         if (attr->dest_nentries) {
1927                 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,
1928                                                                         ce_id,
1929                                                                         attr);
1930                 if (IS_ERR(ce_state->dest_ring)) {
1931                         ret = PTR_ERR(ce_state->dest_ring);
1932                         ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n",
1933                                    ce_id, ret);
1934                         ce_state->dest_ring = NULL;
1935                         return ret;
1936                 }
1937         }
1938
1939         return 0;
1940 }
1941 EXPORT_SYMBOL(ath10k_ce_alloc_pipe);
1942
1943 void ath10k_ce_alloc_rri(struct ath10k *ar)
1944 {
1945         int i;
1946         u32 value;
1947         u32 ctrl1_regs;
1948         u32 ce_base_addr;
1949         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1950
1951         ce->vaddr_rri = dma_alloc_coherent(ar->dev,
1952                                            (CE_COUNT * sizeof(u32)),
1953                                            &ce->paddr_rri, GFP_KERNEL);
1954
1955         if (!ce->vaddr_rri)
1956                 return;
1957
1958         ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low,
1959                           lower_32_bits(ce->paddr_rri));
1960         ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high,
1961                           (upper_32_bits(ce->paddr_rri) &
1962                           CE_DESC_ADDR_HI_MASK));
1963
1964         for (i = 0; i < CE_COUNT; i++) {
1965                 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;
1966                 ce_base_addr = ath10k_ce_base_address(ar, i);
1967                 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs);
1968                 value |= ar->hw_ce_regs->upd->mask;
1969                 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value);
1970         }
1971
1972         memset(ce->vaddr_rri, 0, CE_COUNT * sizeof(u32));
1973 }
1974 EXPORT_SYMBOL(ath10k_ce_alloc_rri);
1975
1976 void ath10k_ce_free_rri(struct ath10k *ar)
1977 {
1978         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1979
1980         dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)),
1981                           ce->vaddr_rri,
1982                           ce->paddr_rri);
1983 }
1984 EXPORT_SYMBOL(ath10k_ce_free_rri);