drivers/dma/bcm-sba-raid.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 // Copyright (C) 2017 Broadcom
   3
   4 /*
   5  * Broadcom SBA RAID Driver
   6  *
   7  * The Broadcom stream buffer accelerator (SBA) provides offloading
   8  * capabilities for RAID operations. The SBA offload engine is accessible
   9  * via Broadcom SoC specific ring manager. Two or more offload engines
  10  * can share same Broadcom SoC specific ring manager due to this Broadcom
  11  * SoC specific ring manager driver is implemented as a mailbox controller
  12  * driver and offload engine drivers are implemented as mallbox clients.
  13  *
  14  * Typically, Broadcom SoC specific ring manager will implement larger
  15  * number of hardware rings over one or more SBA hardware devices. By
  16  * design, the internal buffer size of SBA hardware device is limited
  17  * but all offload operations supported by SBA can be broken down into
  18  * multiple small size requests and executed parallely on multiple SBA
  19  * hardware devices for achieving high through-put.
  20  *
  21  * The Broadcom SBA RAID driver does not require any register programming
  22  * except submitting request to SBA hardware device via mailbox channels.
  23  * This driver implements a DMA device with one DMA channel using a single
  24  * mailbox channel provided by Broadcom SoC specific ring manager driver.
  25  * For having more SBA DMA channels, we can create more SBA device nodes
  26  * in Broadcom SoC specific DTS based on number of hardware rings supported
  27  * by Broadcom SoC ring manager.
  28  */
  29
  30 #include <linux/bitops.h>
  31 #include <linux/debugfs.h>
  32 #include <linux/dma-mapping.h>
  33 #include <linux/dmaengine.h>
  34 #include <linux/list.h>
  35 #include <linux/mailbox_client.h>
  36 #include <linux/mailbox/brcm-message.h>
  37 #include <linux/module.h>
  38 #include <linux/of.h>
  39 #include <linux/of_platform.h>
  40 #include <linux/platform_device.h>
  41 #include <linux/slab.h>
  42 #include <linux/raid/pq.h>
  43
  44 #include "dmaengine.h"
  45
  46 /* ====== Driver macros and defines ===== */
  47
  48 #define SBA_TYPE_SHIFT                                  48
  49 #define SBA_TYPE_MASK                                   GENMASK(1, 0)
  50 #define SBA_TYPE_A                                      0x0
  51 #define SBA_TYPE_B                                      0x2
  52 #define SBA_TYPE_C                                      0x3
  53 #define SBA_USER_DEF_SHIFT                              32
  54 #define SBA_USER_DEF_MASK                               GENMASK(15, 0)
  55 #define SBA_R_MDATA_SHIFT                               24
  56 #define SBA_R_MDATA_MASK                                GENMASK(7, 0)
  57 #define SBA_C_MDATA_MS_SHIFT                            18
  58 #define SBA_C_MDATA_MS_MASK                             GENMASK(1, 0)
  59 #define SBA_INT_SHIFT                                   17
  60 #define SBA_INT_MASK                                    BIT(0)
  61 #define SBA_RESP_SHIFT                                  16
  62 #define SBA_RESP_MASK                                   BIT(0)
  63 #define SBA_C_MDATA_SHIFT                               8
  64 #define SBA_C_MDATA_MASK                                GENMASK(7, 0)
  65 #define SBA_C_MDATA_BNUMx_SHIFT(__bnum)                 (2 * (__bnum))
  66 #define SBA_C_MDATA_BNUMx_MASK                          GENMASK(1, 0)
  67 #define SBA_C_MDATA_DNUM_SHIFT                          5
  68 #define SBA_C_MDATA_DNUM_MASK                           GENMASK(4, 0)
  69 #define SBA_C_MDATA_LS(__v)                             ((__v) & 0xff)
  70 #define SBA_C_MDATA_MS(__v)                             (((__v) >> 8) & 0x3)
  71 #define SBA_CMD_SHIFT                                   0
  72 #define SBA_CMD_MASK                                    GENMASK(3, 0)
  73 #define SBA_CMD_ZERO_BUFFER                             0x4
  74 #define SBA_CMD_ZERO_ALL_BUFFERS                        0x8
  75 #define SBA_CMD_LOAD_BUFFER                             0x9
  76 #define SBA_CMD_XOR                                     0xa
  77 #define SBA_CMD_GALOIS_XOR                              0xb
  78 #define SBA_CMD_WRITE_BUFFER                            0xc
  79 #define SBA_CMD_GALOIS                                  0xe
  80
  81 #define SBA_MAX_REQ_PER_MBOX_CHANNEL                    8192
  82 #define SBA_MAX_MSG_SEND_PER_MBOX_CHANNEL               8
  83
  84 /* Driver helper macros */
  85 #define to_sba_request(tx)              \
  86         container_of(tx, struct sba_request, tx)
  87 #define to_sba_device(dchan)            \
  88         container_of(dchan, struct sba_device, dma_chan)
  89
  90 /* ===== Driver data structures ===== */
  91
  92 enum sba_request_flags {
  93         SBA_REQUEST_STATE_FREE          = 0x001,
  94         SBA_REQUEST_STATE_ALLOCED       = 0x002,
  95         SBA_REQUEST_STATE_PENDING       = 0x004,
  96         SBA_REQUEST_STATE_ACTIVE        = 0x008,
  97         SBA_REQUEST_STATE_ABORTED       = 0x010,
  98         SBA_REQUEST_STATE_MASK          = 0x0ff,
  99         SBA_REQUEST_FENCE               = 0x100,
 100 };
 101
 102 struct sba_request {
 103         /* Global state */
 104         struct list_head node;
 105         struct sba_device *sba;
 106         u32 flags;
 107         /* Chained requests management */
 108         struct sba_request *first;
 109         struct list_head next;
 110         atomic_t next_pending_count;
 111         /* BRCM message data */
 112         struct brcm_message msg;
 113         struct dma_async_tx_descriptor tx;
 114         /* SBA commands */
 115         struct brcm_sba_command cmds[];
 116 };
 117
 118 enum sba_version {
 119         SBA_VER_1 = 0,
 120         SBA_VER_2
 121 };
 122
 123 struct sba_device {
 124         /* Underlying device */
 125         struct device *dev;
 126         /* DT configuration parameters */
 127         enum sba_version ver;
 128         /* Derived configuration parameters */
 129         u32 max_req;
 130         u32 hw_buf_size;
 131         u32 hw_resp_size;
 132         u32 max_pq_coefs;
 133         u32 max_pq_srcs;
 134         u32 max_cmd_per_req;
 135         u32 max_xor_srcs;
 136         u32 max_resp_pool_size;
 137         u32 max_cmds_pool_size;
 138         /* Maibox client and Mailbox channels */
 139         struct mbox_client client;
 140         struct mbox_chan *mchan;
 141         struct device *mbox_dev;
 142         /* DMA device and DMA channel */
 143         struct dma_device dma_dev;
 144         struct dma_chan dma_chan;
 145         /* DMA channel resources */
 146         void *resp_base;
 147         dma_addr_t resp_dma_base;
 148         void *cmds_base;
 149         dma_addr_t cmds_dma_base;
 150         spinlock_t reqs_lock;
 151         bool reqs_fence;
 152         struct list_head reqs_alloc_list;
 153         struct list_head reqs_pending_list;
 154         struct list_head reqs_active_list;
 155         struct list_head reqs_aborted_list;
 156         struct list_head reqs_free_list;
 157         /* DebugFS directory entries */
 158         struct dentry *root;
 159 };
 160
 161 /* ====== Command helper routines ===== */
 162
 163 static inline u64 __pure sba_cmd_enc(u64 cmd, u32 val, u32 shift, u32 mask)
 164 {
 165         cmd &= ~((u64)mask << shift);
 166         cmd |= ((u64)(val & mask) << shift);
 167         return cmd;
 168 }
 169
 170 static inline u32 __pure sba_cmd_load_c_mdata(u32 b0)
 171 {
 172         return b0 & SBA_C_MDATA_BNUMx_MASK;
 173 }
 174
 175 static inline u32 __pure sba_cmd_write_c_mdata(u32 b0)
 176 {
 177         return b0 & SBA_C_MDATA_BNUMx_MASK;
 178 }
 179
 180 static inline u32 __pure sba_cmd_xor_c_mdata(u32 b1, u32 b0)
 181 {
 182         return (b0 & SBA_C_MDATA_BNUMx_MASK) |
 183                ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1));
 184 }
 185
 186 static inline u32 __pure sba_cmd_pq_c_mdata(u32 d, u32 b1, u32 b0)
 187 {
 188         return (b0 & SBA_C_MDATA_BNUMx_MASK) |
 189                ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)) |
 190                ((d & SBA_C_MDATA_DNUM_MASK) << SBA_C_MDATA_DNUM_SHIFT);
 191 }
 192
 193 /* ====== General helper routines ===== */
 194
 195 static struct sba_request *sba_alloc_request(struct sba_device *sba)
 196 {
 197         bool found = false;
 198         unsigned long flags;
 199         struct sba_request *req = NULL;
 200
 201         spin_lock_irqsave(&sba->reqs_lock, flags);
 202         list_for_each_entry(req, &sba->reqs_free_list, node) {
 203                 if (async_tx_test_ack(&req->tx)) {
 204                         list_move_tail(&req->node, &sba->reqs_alloc_list);
 205                         found = true;
 206                         break;
 207                 }
 208         }
 209         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 210
 211         if (!found) {
 212                 /*
 213                  * We have no more free requests so, we peek
 214                  * mailbox channels hoping few active requests
 215                  * would have completed which will create more
 216                  * room for new requests.
 217                  */
 218                 mbox_client_peek_data(sba->mchan);
 219                 return NULL;
 220         }
 221
 222         req->flags = SBA_REQUEST_STATE_ALLOCED;
 223         req->first = req;
 224         INIT_LIST_HEAD(&req->next);
 225         atomic_set(&req->next_pending_count, 1);
 226
 227         dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
 228         async_tx_ack(&req->tx);
 229
 230         return req;
 231 }
 232
 233 /* Note: Must be called with sba->reqs_lock held */
 234 static void _sba_pending_request(struct sba_device *sba,
 235                                  struct sba_request *req)
 236 {
 237         lockdep_assert_held(&sba->reqs_lock);
 238         req->flags &= ~SBA_REQUEST_STATE_MASK;
 239         req->flags |= SBA_REQUEST_STATE_PENDING;
 240         list_move_tail(&req->node, &sba->reqs_pending_list);
 241         if (list_empty(&sba->reqs_active_list))
 242                 sba->reqs_fence = false;
 243 }
 244
 245 /* Note: Must be called with sba->reqs_lock held */
 246 static bool _sba_active_request(struct sba_device *sba,
 247                                 struct sba_request *req)
 248 {
 249         lockdep_assert_held(&sba->reqs_lock);
 250         if (list_empty(&sba->reqs_active_list))
 251                 sba->reqs_fence = false;
 252         if (sba->reqs_fence)
 253                 return false;
 254         req->flags &= ~SBA_REQUEST_STATE_MASK;
 255         req->flags |= SBA_REQUEST_STATE_ACTIVE;
 256         list_move_tail(&req->node, &sba->reqs_active_list);
 257         if (req->flags & SBA_REQUEST_FENCE)
 258                 sba->reqs_fence = true;
 259         return true;
 260 }
 261
 262 /* Note: Must be called with sba->reqs_lock held */
 263 static void _sba_abort_request(struct sba_device *sba,
 264                                struct sba_request *req)
 265 {
 266         lockdep_assert_held(&sba->reqs_lock);
 267         req->flags &= ~SBA_REQUEST_STATE_MASK;
 268         req->flags |= SBA_REQUEST_STATE_ABORTED;
 269         list_move_tail(&req->node, &sba->reqs_aborted_list);
 270         if (list_empty(&sba->reqs_active_list))
 271                 sba->reqs_fence = false;
 272 }
 273
 274 /* Note: Must be called with sba->reqs_lock held */
 275 static void _sba_free_request(struct sba_device *sba,
 276                               struct sba_request *req)
 277 {
 278         lockdep_assert_held(&sba->reqs_lock);
 279         req->flags &= ~SBA_REQUEST_STATE_MASK;
 280         req->flags |= SBA_REQUEST_STATE_FREE;
 281         list_move_tail(&req->node, &sba->reqs_free_list);
 282         if (list_empty(&sba->reqs_active_list))
 283                 sba->reqs_fence = false;
 284 }
 285
 286 static void sba_free_chained_requests(struct sba_request *req)
 287 {
 288         unsigned long flags;
 289         struct sba_request *nreq;
 290         struct sba_device *sba = req->sba;
 291
 292         spin_lock_irqsave(&sba->reqs_lock, flags);
 293
 294         _sba_free_request(sba, req);
 295         list_for_each_entry(nreq, &req->next, next)
 296                 _sba_free_request(sba, nreq);
 297
 298         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 299 }
 300
 301 static void sba_chain_request(struct sba_request *first,
 302                               struct sba_request *req)
 303 {
 304         unsigned long flags;
 305         struct sba_device *sba = req->sba;
 306
 307         spin_lock_irqsave(&sba->reqs_lock, flags);
 308
 309         list_add_tail(&req->next, &first->next);
 310         req->first = first;
 311         atomic_inc(&first->next_pending_count);
 312
 313         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 314 }
 315
 316 static void sba_cleanup_nonpending_requests(struct sba_device *sba)
 317 {
 318         unsigned long flags;
 319         struct sba_request *req, *req1;
 320
 321         spin_lock_irqsave(&sba->reqs_lock, flags);
 322
 323         /* Freeup all alloced request */
 324         list_for_each_entry_safe(req, req1, &sba->reqs_alloc_list, node)
 325                 _sba_free_request(sba, req);
 326
 327         /* Set all active requests as aborted */
 328         list_for_each_entry_safe(req, req1, &sba->reqs_active_list, node)
 329                 _sba_abort_request(sba, req);
 330
 331         /*
 332          * Note: We expect that aborted request will be eventually
 333          * freed by sba_receive_message()
 334          */
 335
 336         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 337 }
 338
 339 static void sba_cleanup_pending_requests(struct sba_device *sba)
 340 {
 341         unsigned long flags;
 342         struct sba_request *req, *req1;
 343
 344         spin_lock_irqsave(&sba->reqs_lock, flags);
 345
 346         /* Freeup all pending request */
 347         list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node)
 348                 _sba_free_request(sba, req);
 349
 350         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 351 }
 352
 353 static int sba_send_mbox_request(struct sba_device *sba,
 354                                  struct sba_request *req)
 355 {
 356         int ret = 0;
 357
 358         /* Send message for the request */
 359         req->msg.error = 0;
 360         ret = mbox_send_message(sba->mchan, &req->msg);
 361         if (ret < 0) {
 362                 dev_err(sba->dev, "send message failed with error %d", ret);
 363                 return ret;
 364         }
 365
 366         /* Check error returned by mailbox controller */
 367         ret = req->msg.error;
 368         if (ret < 0) {
 369                 dev_err(sba->dev, "message error %d", ret);
 370         }
 371
 372         /* Signal txdone for mailbox channel */
 373         mbox_client_txdone(sba->mchan, ret);
 374
 375         return ret;
 376 }
 377
 378 /* Note: Must be called with sba->reqs_lock held */
 379 static void _sba_process_pending_requests(struct sba_device *sba)
 380 {
 381         int ret;
 382         u32 count;
 383         struct sba_request *req;
 384
 385         /* Process few pending requests */
 386         count = SBA_MAX_MSG_SEND_PER_MBOX_CHANNEL;
 387         while (!list_empty(&sba->reqs_pending_list) && count) {
 388                 /* Get the first pending request */
 389                 req = list_first_entry(&sba->reqs_pending_list,
 390                                        struct sba_request, node);
 391
 392                 /* Try to make request active */
 393                 if (!_sba_active_request(sba, req))
 394                         break;
 395
 396                 /* Send request to mailbox channel */
 397                 ret = sba_send_mbox_request(sba, req);
 398                 if (ret < 0) {
 399                         _sba_pending_request(sba, req);
 400                         break;
 401                 }
 402
 403                 count--;
 404         }
 405 }
 406
 407 static void sba_process_received_request(struct sba_device *sba,
 408                                          struct sba_request *req)
 409 {
 410         unsigned long flags;
 411         struct dma_async_tx_descriptor *tx;
 412         struct sba_request *nreq, *first = req->first;
 413
 414         /* Process only after all chained requests are received */
 415         if (!atomic_dec_return(&first->next_pending_count)) {
 416                 tx = &first->tx;
 417
 418                 WARN_ON(tx->cookie < 0);
 419                 if (tx->cookie > 0) {
 420                         spin_lock_irqsave(&sba->reqs_lock, flags);
 421                         dma_cookie_complete(tx);
 422                         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 423                         dmaengine_desc_get_callback_invoke(tx, NULL);
 424                         dma_descriptor_unmap(tx);
 425                         tx->callback = NULL;
 426                         tx->callback_result = NULL;
 427                 }
 428
 429                 dma_run_dependencies(tx);
 430
 431                 spin_lock_irqsave(&sba->reqs_lock, flags);
 432
 433                 /* Free all requests chained to first request */
 434                 list_for_each_entry(nreq, &first->next, next)
 435                         _sba_free_request(sba, nreq);
 436                 INIT_LIST_HEAD(&first->next);
 437
 438                 /* Free the first request */
 439                 _sba_free_request(sba, first);
 440
 441                 /* Process pending requests */
 442                 _sba_process_pending_requests(sba);
 443
 444                 spin_unlock_irqrestore(&sba->reqs_lock, flags);
 445         }
 446 }
 447
 448 static void sba_write_stats_in_seqfile(struct sba_device *sba,
 449                                        struct seq_file *file)
 450 {
 451         unsigned long flags;
 452         struct sba_request *req;
 453         u32 free_count = 0, alloced_count = 0;
 454         u32 pending_count = 0, active_count = 0, aborted_count = 0;
 455
 456         spin_lock_irqsave(&sba->reqs_lock, flags);
 457
 458         list_for_each_entry(req, &sba->reqs_free_list, node)
 459                 if (async_tx_test_ack(&req->tx))
 460                         free_count++;
 461
 462         list_for_each_entry(req, &sba->reqs_alloc_list, node)
 463                 alloced_count++;
 464
 465         list_for_each_entry(req, &sba->reqs_pending_list, node)
 466                 pending_count++;
 467
 468         list_for_each_entry(req, &sba->reqs_active_list, node)
 469                 active_count++;
 470
 471         list_for_each_entry(req, &sba->reqs_aborted_list, node)
 472                 aborted_count++;
 473
 474         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 475
 476         seq_printf(file, "maximum requests   = %d\n", sba->max_req);
 477         seq_printf(file, "free requests      = %d\n", free_count);
 478         seq_printf(file, "alloced requests   = %d\n", alloced_count);
 479         seq_printf(file, "pending requests   = %d\n", pending_count);
 480         seq_printf(file, "active requests    = %d\n", active_count);
 481         seq_printf(file, "aborted requests   = %d\n", aborted_count);
 482 }
 483
 484 /* ====== DMAENGINE callbacks ===== */
 485
 486 static void sba_free_chan_resources(struct dma_chan *dchan)
 487 {
 488         /*
 489          * Channel resources are pre-alloced so we just free-up
 490          * whatever we can so that we can re-use pre-alloced
 491          * channel resources next time.
 492          */
 493         sba_cleanup_nonpending_requests(to_sba_device(dchan));
 494 }
 495
 496 static int sba_device_terminate_all(struct dma_chan *dchan)
 497 {
 498         /* Cleanup all pending requests */
 499         sba_cleanup_pending_requests(to_sba_device(dchan));
 500
 501         return 0;
 502 }
 503
 504 static void sba_issue_pending(struct dma_chan *dchan)
 505 {
 506         unsigned long flags;
 507         struct sba_device *sba = to_sba_device(dchan);
 508
 509         /* Process pending requests */
 510         spin_lock_irqsave(&sba->reqs_lock, flags);
 511         _sba_process_pending_requests(sba);
 512         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 513 }
 514
 515 static dma_cookie_t sba_tx_submit(struct dma_async_tx_descriptor *tx)
 516 {
 517         unsigned long flags;
 518         dma_cookie_t cookie;
 519         struct sba_device *sba;
 520         struct sba_request *req, *nreq;
 521
 522         if (unlikely(!tx))
 523                 return -EINVAL;
 524
 525         sba = to_sba_device(tx->chan);
 526         req = to_sba_request(tx);
 527
 528         /* Assign cookie and mark all chained requests pending */
 529         spin_lock_irqsave(&sba->reqs_lock, flags);
 530         cookie = dma_cookie_assign(tx);
 531         _sba_pending_request(sba, req);
 532         list_for_each_entry(nreq, &req->next, next)
 533                 _sba_pending_request(sba, nreq);
 534         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 535
 536         return cookie;
 537 }
 538
 539 static enum dma_status sba_tx_status(struct dma_chan *dchan,
 540                                      dma_cookie_t cookie,
 541                                      struct dma_tx_state *txstate)
 542 {
 543         enum dma_status ret;
 544         struct sba_device *sba = to_sba_device(dchan);
 545
 546         ret = dma_cookie_status(dchan, cookie, txstate);
 547         if (ret == DMA_COMPLETE)
 548                 return ret;
 549
 550         mbox_client_peek_data(sba->mchan);
 551
 552         return dma_cookie_status(dchan, cookie, txstate);
 553 }
 554
 555 static void sba_fillup_interrupt_msg(struct sba_request *req,
 556                                      struct brcm_sba_command *cmds,
 557                                      struct brcm_message *msg)
 558 {
 559         u64 cmd;
 560         u32 c_mdata;
 561         dma_addr_t resp_dma = req->tx.phys;
 562         struct brcm_sba_command *cmdsp = cmds;
 563
 564         /* Type-B command to load dummy data into buf0 */
 565         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 566                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 567         cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size,
 568                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 569         c_mdata = sba_cmd_load_c_mdata(0);
 570         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 571                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 572         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 573                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 574         cmdsp->cmd = cmd;
 575         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 576         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 577         cmdsp->data = resp_dma;
 578         cmdsp->data_len = req->sba->hw_resp_size;
 579         cmdsp++;
 580
 581         /* Type-A command to write buf0 to dummy location */
 582         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 583                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 584         cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size,
 585                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 586         cmd = sba_cmd_enc(cmd, 0x1,
 587                           SBA_RESP_SHIFT, SBA_RESP_MASK);
 588         c_mdata = sba_cmd_write_c_mdata(0);
 589         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 590                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 591         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
 592                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 593         cmdsp->cmd = cmd;
 594         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 595         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 596         if (req->sba->hw_resp_size) {
 597                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
 598                 cmdsp->resp = resp_dma;
 599                 cmdsp->resp_len = req->sba->hw_resp_size;
 600         }
 601         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
 602         cmdsp->data = resp_dma;
 603         cmdsp->data_len = req->sba->hw_resp_size;
 604         cmdsp++;
 605
 606         /* Fillup brcm_message */
 607         msg->type = BRCM_MESSAGE_SBA;
 608         msg->sba.cmds = cmds;
 609         msg->sba.cmds_count = cmdsp - cmds;
 610         msg->ctx = req;
 611         msg->error = 0;
 612 }
 613
 614 static struct dma_async_tx_descriptor *
 615 sba_prep_dma_interrupt(struct dma_chan *dchan, unsigned long flags)
 616 {
 617         struct sba_request *req = NULL;
 618         struct sba_device *sba = to_sba_device(dchan);
 619
 620         /* Alloc new request */
 621         req = sba_alloc_request(sba);
 622         if (!req)
 623                 return NULL;
 624
 625         /*
 626          * Force fence so that no requests are submitted
 627          * until DMA callback for this request is invoked.
 628          */
 629         req->flags |= SBA_REQUEST_FENCE;
 630
 631         /* Fillup request message */
 632         sba_fillup_interrupt_msg(req, req->cmds, &req->msg);
 633
 634         /* Init async_tx descriptor */
 635         req->tx.flags = flags;
 636         req->tx.cookie = -EBUSY;
 637
 638         return &req->tx;
 639 }
 640
 641 static void sba_fillup_memcpy_msg(struct sba_request *req,
 642                                   struct brcm_sba_command *cmds,
 643                                   struct brcm_message *msg,
 644                                   dma_addr_t msg_offset, size_t msg_len,
 645                                   dma_addr_t dst, dma_addr_t src)
 646 {
 647         u64 cmd;
 648         u32 c_mdata;
 649         dma_addr_t resp_dma = req->tx.phys;
 650         struct brcm_sba_command *cmdsp = cmds;
 651
 652         /* Type-B command to load data into buf0 */
 653         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 654                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 655         cmd = sba_cmd_enc(cmd, msg_len,
 656                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 657         c_mdata = sba_cmd_load_c_mdata(0);
 658         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 659                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 660         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 661                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 662         cmdsp->cmd = cmd;
 663         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 664         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 665         cmdsp->data = src + msg_offset;
 666         cmdsp->data_len = msg_len;
 667         cmdsp++;
 668
 669         /* Type-A command to write buf0 */
 670         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 671                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 672         cmd = sba_cmd_enc(cmd, msg_len,
 673                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 674         cmd = sba_cmd_enc(cmd, 0x1,
 675                           SBA_RESP_SHIFT, SBA_RESP_MASK);
 676         c_mdata = sba_cmd_write_c_mdata(0);
 677         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 678                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 679         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
 680                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 681         cmdsp->cmd = cmd;
 682         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 683         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 684         if (req->sba->hw_resp_size) {
 685                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
 686                 cmdsp->resp = resp_dma;
 687                 cmdsp->resp_len = req->sba->hw_resp_size;
 688         }
 689         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
 690         cmdsp->data = dst + msg_offset;
 691         cmdsp->data_len = msg_len;
 692         cmdsp++;
 693
 694         /* Fillup brcm_message */
 695         msg->type = BRCM_MESSAGE_SBA;
 696         msg->sba.cmds = cmds;
 697         msg->sba.cmds_count = cmdsp - cmds;
 698         msg->ctx = req;
 699         msg->error = 0;
 700 }
 701
 702 static struct sba_request *
 703 sba_prep_dma_memcpy_req(struct sba_device *sba,
 704                         dma_addr_t off, dma_addr_t dst, dma_addr_t src,
 705                         size_t len, unsigned long flags)
 706 {
 707         struct sba_request *req = NULL;
 708
 709         /* Alloc new request */
 710         req = sba_alloc_request(sba);
 711         if (!req)
 712                 return NULL;
 713         if (flags & DMA_PREP_FENCE)
 714                 req->flags |= SBA_REQUEST_FENCE;
 715
 716         /* Fillup request message */
 717         sba_fillup_memcpy_msg(req, req->cmds, &req->msg,
 718                               off, len, dst, src);
 719
 720         /* Init async_tx descriptor */
 721         req->tx.flags = flags;
 722         req->tx.cookie = -EBUSY;
 723
 724         return req;
 725 }
 726
 727 static struct dma_async_tx_descriptor *
 728 sba_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src,
 729                     size_t len, unsigned long flags)
 730 {
 731         size_t req_len;
 732         dma_addr_t off = 0;
 733         struct sba_device *sba = to_sba_device(dchan);
 734         struct sba_request *first = NULL, *req;
 735
 736         /* Create chained requests where each request is upto hw_buf_size */
 737         while (len) {
 738                 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
 739
 740                 req = sba_prep_dma_memcpy_req(sba, off, dst, src,
 741                                               req_len, flags);
 742                 if (!req) {
 743                         if (first)
 744                                 sba_free_chained_requests(first);
 745                         return NULL;
 746                 }
 747
 748                 if (first)
 749                         sba_chain_request(first, req);
 750                 else
 751                         first = req;
 752
 753                 off += req_len;
 754                 len -= req_len;
 755         }
 756
 757         return (first) ? &first->tx : NULL;
 758 }
 759
 760 static void sba_fillup_xor_msg(struct sba_request *req,
 761                                 struct brcm_sba_command *cmds,
 762                                 struct brcm_message *msg,
 763                                 dma_addr_t msg_offset, size_t msg_len,
 764                                 dma_addr_t dst, dma_addr_t *src, u32 src_cnt)
 765 {
 766         u64 cmd;
 767         u32 c_mdata;
 768         unsigned int i;
 769         dma_addr_t resp_dma = req->tx.phys;
 770         struct brcm_sba_command *cmdsp = cmds;
 771
 772         /* Type-B command to load data into buf0 */
 773         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 774                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 775         cmd = sba_cmd_enc(cmd, msg_len,
 776                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 777         c_mdata = sba_cmd_load_c_mdata(0);
 778         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 779                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 780         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 781                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 782         cmdsp->cmd = cmd;
 783         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 784         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 785         cmdsp->data = src[0] + msg_offset;
 786         cmdsp->data_len = msg_len;
 787         cmdsp++;
 788
 789         /* Type-B commands to xor data with buf0 and put it back in buf0 */
 790         for (i = 1; i < src_cnt; i++) {
 791                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 792                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 793                 cmd = sba_cmd_enc(cmd, msg_len,
 794                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 795                 c_mdata = sba_cmd_xor_c_mdata(0, 0);
 796                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 797                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 798                 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
 799                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
 800                 cmdsp->cmd = cmd;
 801                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
 802                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 803                 cmdsp->data = src[i] + msg_offset;
 804                 cmdsp->data_len = msg_len;
 805                 cmdsp++;
 806         }
 807
 808         /* Type-A command to write buf0 */
 809         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 810                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 811         cmd = sba_cmd_enc(cmd, msg_len,
 812                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 813         cmd = sba_cmd_enc(cmd, 0x1,
 814                           SBA_RESP_SHIFT, SBA_RESP_MASK);
 815         c_mdata = sba_cmd_write_c_mdata(0);
 816         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 817                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 818         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
 819                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 820         cmdsp->cmd = cmd;
 821         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 822         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 823         if (req->sba->hw_resp_size) {
 824                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
 825                 cmdsp->resp = resp_dma;
 826                 cmdsp->resp_len = req->sba->hw_resp_size;
 827         }
 828         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
 829         cmdsp->data = dst + msg_offset;
 830         cmdsp->data_len = msg_len;
 831         cmdsp++;
 832
 833         /* Fillup brcm_message */
 834         msg->type = BRCM_MESSAGE_SBA;
 835         msg->sba.cmds = cmds;
 836         msg->sba.cmds_count = cmdsp - cmds;
 837         msg->ctx = req;
 838         msg->error = 0;
 839 }
 840
 841 static struct sba_request *
 842 sba_prep_dma_xor_req(struct sba_device *sba,
 843                      dma_addr_t off, dma_addr_t dst, dma_addr_t *src,
 844                      u32 src_cnt, size_t len, unsigned long flags)
 845 {
 846         struct sba_request *req = NULL;
 847
 848         /* Alloc new request */
 849         req = sba_alloc_request(sba);
 850         if (!req)
 851                 return NULL;
 852         if (flags & DMA_PREP_FENCE)
 853                 req->flags |= SBA_REQUEST_FENCE;
 854
 855         /* Fillup request message */
 856         sba_fillup_xor_msg(req, req->cmds, &req->msg,
 857                            off, len, dst, src, src_cnt);
 858
 859         /* Init async_tx descriptor */
 860         req->tx.flags = flags;
 861         req->tx.cookie = -EBUSY;
 862
 863         return req;
 864 }
 865
 866 static struct dma_async_tx_descriptor *
 867 sba_prep_dma_xor(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t *src,
 868                  u32 src_cnt, size_t len, unsigned long flags)
 869 {
 870         size_t req_len;
 871         dma_addr_t off = 0;
 872         struct sba_device *sba = to_sba_device(dchan);
 873         struct sba_request *first = NULL, *req;
 874
 875         /* Sanity checks */
 876         if (unlikely(src_cnt > sba->max_xor_srcs))
 877                 return NULL;
 878
 879         /* Create chained requests where each request is upto hw_buf_size */
 880         while (len) {
 881                 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
 882
 883                 req = sba_prep_dma_xor_req(sba, off, dst, src, src_cnt,
 884                                            req_len, flags);
 885                 if (!req) {
 886                         if (first)
 887                                 sba_free_chained_requests(first);
 888                         return NULL;
 889                 }
 890
 891                 if (first)
 892                         sba_chain_request(first, req);
 893                 else
 894                         first = req;
 895
 896                 off += req_len;
 897                 len -= req_len;
 898         }
 899
 900         return (first) ? &first->tx : NULL;
 901 }
 902
 903 static void sba_fillup_pq_msg(struct sba_request *req,
 904                                 bool pq_continue,
 905                                 struct brcm_sba_command *cmds,
 906                                 struct brcm_message *msg,
 907                                 dma_addr_t msg_offset, size_t msg_len,
 908                                 dma_addr_t *dst_p, dma_addr_t *dst_q,
 909                                 const u8 *scf, dma_addr_t *src, u32 src_cnt)
 910 {
 911         u64 cmd;
 912         u32 c_mdata;
 913         unsigned int i;
 914         dma_addr_t resp_dma = req->tx.phys;
 915         struct brcm_sba_command *cmdsp = cmds;
 916
 917         if (pq_continue) {
 918                 /* Type-B command to load old P into buf0 */
 919                 if (dst_p) {
 920                         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 921                                 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 922                         cmd = sba_cmd_enc(cmd, msg_len,
 923                                 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 924                         c_mdata = sba_cmd_load_c_mdata(0);
 925                         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 926                                 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 927                         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 928                                 SBA_CMD_SHIFT, SBA_CMD_MASK);
 929                         cmdsp->cmd = cmd;
 930                         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 931                         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 932                         cmdsp->data = *dst_p + msg_offset;
 933                         cmdsp->data_len = msg_len;
 934                         cmdsp++;
 935                 }
 936
 937                 /* Type-B command to load old Q into buf1 */
 938                 if (dst_q) {
 939                         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 940                                 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 941                         cmd = sba_cmd_enc(cmd, msg_len,
 942                                 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 943                         c_mdata = sba_cmd_load_c_mdata(1);
 944                         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 945                                 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 946                         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 947                                 SBA_CMD_SHIFT, SBA_CMD_MASK);
 948                         cmdsp->cmd = cmd;
 949                         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 950                         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 951                         cmdsp->data = *dst_q + msg_offset;
 952                         cmdsp->data_len = msg_len;
 953                         cmdsp++;
 954                 }
 955         } else {
 956                 /* Type-A command to zero all buffers */
 957                 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 958                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 959                 cmd = sba_cmd_enc(cmd, msg_len,
 960                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 961                 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS,
 962                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
 963                 cmdsp->cmd = cmd;
 964                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
 965                 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 966                 cmdsp++;
 967         }
 968
 969         /* Type-B commands for generate P onto buf0 and Q onto buf1 */
 970         for (i = 0; i < src_cnt; i++) {
 971                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 972                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 973                 cmd = sba_cmd_enc(cmd, msg_len,
 974                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 975                 c_mdata = sba_cmd_pq_c_mdata(raid6_gflog[scf[i]], 1, 0);
 976                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 977                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 978                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
 979                                   SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
 980                 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS_XOR,
 981                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
 982                 cmdsp->cmd = cmd;
 983                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
 984                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 985                 cmdsp->data = src[i] + msg_offset;
 986                 cmdsp->data_len = msg_len;
 987                 cmdsp++;
 988         }
 989
 990         /* Type-A command to write buf0 */
 991         if (dst_p) {
 992                 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 993                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 994                 cmd = sba_cmd_enc(cmd, msg_len,
 995                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 996                 cmd = sba_cmd_enc(cmd, 0x1,
 997                                   SBA_RESP_SHIFT, SBA_RESP_MASK);
 998                 c_mdata = sba_cmd_write_c_mdata(0);
 999                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1000                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1001                 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1002                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1003                 cmdsp->cmd = cmd;
1004                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1005                 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1006                 if (req->sba->hw_resp_size) {
1007                         cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1008                         cmdsp->resp = resp_dma;
1009                         cmdsp->resp_len = req->sba->hw_resp_size;
1010                 }
1011                 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1012                 cmdsp->data = *dst_p + msg_offset;
1013                 cmdsp->data_len = msg_len;
1014                 cmdsp++;
1015         }
1016
1017         /* Type-A command to write buf1 */
1018         if (dst_q) {
1019                 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1020                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1021                 cmd = sba_cmd_enc(cmd, msg_len,
1022                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1023                 cmd = sba_cmd_enc(cmd, 0x1,
1024                                   SBA_RESP_SHIFT, SBA_RESP_MASK);
1025                 c_mdata = sba_cmd_write_c_mdata(1);
1026                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1027                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1028                 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1029                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1030                 cmdsp->cmd = cmd;
1031                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1032                 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1033                 if (req->sba->hw_resp_size) {
1034                         cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1035                         cmdsp->resp = resp_dma;
1036                         cmdsp->resp_len = req->sba->hw_resp_size;
1037                 }
1038                 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1039                 cmdsp->data = *dst_q + msg_offset;
1040                 cmdsp->data_len = msg_len;
1041                 cmdsp++;
1042         }
1043
1044         /* Fillup brcm_message */
1045         msg->type = BRCM_MESSAGE_SBA;
1046         msg->sba.cmds = cmds;
1047         msg->sba.cmds_count = cmdsp - cmds;
1048         msg->ctx = req;
1049         msg->error = 0;
1050 }
1051
1052 static struct sba_request *
1053 sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off,
1054                     dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t *src,
1055                     u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
1056 {
1057         struct sba_request *req = NULL;
1058
1059         /* Alloc new request */
1060         req = sba_alloc_request(sba);
1061         if (!req)
1062                 return NULL;
1063         if (flags & DMA_PREP_FENCE)
1064                 req->flags |= SBA_REQUEST_FENCE;
1065
1066         /* Fillup request messages */
1067         sba_fillup_pq_msg(req, dmaf_continue(flags),
1068                           req->cmds, &req->msg,
1069                           off, len, dst_p, dst_q, scf, src, src_cnt);
1070
1071         /* Init async_tx descriptor */
1072         req->tx.flags = flags;
1073         req->tx.cookie = -EBUSY;
1074
1075         return req;
1076 }
1077
1078 static void sba_fillup_pq_single_msg(struct sba_request *req,
1079                                 bool pq_continue,
1080                                 struct brcm_sba_command *cmds,
1081                                 struct brcm_message *msg,
1082                                 dma_addr_t msg_offset, size_t msg_len,
1083                                 dma_addr_t *dst_p, dma_addr_t *dst_q,
1084                                 dma_addr_t src, u8 scf)
1085 {
1086         u64 cmd;
1087         u32 c_mdata;
1088         u8 pos, dpos = raid6_gflog[scf];
1089         dma_addr_t resp_dma = req->tx.phys;
1090         struct brcm_sba_command *cmdsp = cmds;
1091
1092         if (!dst_p)
1093                 goto skip_p;
1094
1095         if (pq_continue) {
1096                 /* Type-B command to load old P into buf0 */
1097                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1098                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1099                 cmd = sba_cmd_enc(cmd, msg_len,
1100                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1101                 c_mdata = sba_cmd_load_c_mdata(0);
1102                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1103                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1104                 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
1105                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1106                 cmdsp->cmd = cmd;
1107                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1108                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1109                 cmdsp->data = *dst_p + msg_offset;
1110                 cmdsp->data_len = msg_len;
1111                 cmdsp++;
1112
1113                 /*
1114                  * Type-B commands to xor data with buf0 and put it
1115                  * back in buf0
1116                  */
1117                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1118                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1119                 cmd = sba_cmd_enc(cmd, msg_len,
1120                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1121                 c_mdata = sba_cmd_xor_c_mdata(0, 0);
1122                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1123                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1124                 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
1125                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1126                 cmdsp->cmd = cmd;
1127                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1128                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1129                 cmdsp->data = src + msg_offset;
1130                 cmdsp->data_len = msg_len;
1131                 cmdsp++;
1132         } else {
1133                 /* Type-B command to load old P into buf0 */
1134                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1135                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1136                 cmd = sba_cmd_enc(cmd, msg_len,
1137                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1138                 c_mdata = sba_cmd_load_c_mdata(0);
1139                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1140                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1141                 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
1142                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1143                 cmdsp->cmd = cmd;
1144                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1145                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1146                 cmdsp->data = src + msg_offset;
1147                 cmdsp->data_len = msg_len;
1148                 cmdsp++;
1149         }
1150
1151         /* Type-A command to write buf0 */
1152         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1153                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1154         cmd = sba_cmd_enc(cmd, msg_len,
1155                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1156         cmd = sba_cmd_enc(cmd, 0x1,
1157                           SBA_RESP_SHIFT, SBA_RESP_MASK);
1158         c_mdata = sba_cmd_write_c_mdata(0);
1159         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1160                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1161         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1162                           SBA_CMD_SHIFT, SBA_CMD_MASK);
1163         cmdsp->cmd = cmd;
1164         *cmdsp->cmd_dma = cpu_to_le64(cmd);
1165         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1166         if (req->sba->hw_resp_size) {
1167                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1168                 cmdsp->resp = resp_dma;
1169                 cmdsp->resp_len = req->sba->hw_resp_size;
1170         }
1171         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1172         cmdsp->data = *dst_p + msg_offset;
1173         cmdsp->data_len = msg_len;
1174         cmdsp++;
1175
1176 skip_p:
1177         if (!dst_q)
1178                 goto skip_q;
1179
1180         /* Type-A command to zero all buffers */
1181         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1182                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1183         cmd = sba_cmd_enc(cmd, msg_len,
1184                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1185         cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS,
1186                           SBA_CMD_SHIFT, SBA_CMD_MASK);
1187         cmdsp->cmd = cmd;
1188         *cmdsp->cmd_dma = cpu_to_le64(cmd);
1189         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1190         cmdsp++;
1191
1192         if (dpos == 255)
1193                 goto skip_q_computation;
1194         pos = (dpos < req->sba->max_pq_coefs) ?
1195                 dpos : (req->sba->max_pq_coefs - 1);
1196
1197         /*
1198          * Type-B command to generate initial Q from data
1199          * and store output into buf0
1200          */
1201         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1202                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1203         cmd = sba_cmd_enc(cmd, msg_len,
1204                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1205         c_mdata = sba_cmd_pq_c_mdata(pos, 0, 0);
1206         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1207                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1208         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
1209                           SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
1210         cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS,
1211                           SBA_CMD_SHIFT, SBA_CMD_MASK);
1212         cmdsp->cmd = cmd;
1213         *cmdsp->cmd_dma = cpu_to_le64(cmd);
1214         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1215         cmdsp->data = src + msg_offset;
1216         cmdsp->data_len = msg_len;
1217         cmdsp++;
1218
1219         dpos -= pos;
1220
1221         /* Multiple Type-A command to generate final Q */
1222         while (dpos) {
1223                 pos = (dpos < req->sba->max_pq_coefs) ?
1224                         dpos : (req->sba->max_pq_coefs - 1);
1225
1226                 /*
1227                  * Type-A command to generate Q with buf0 and
1228                  * buf1 store result in buf0
1229                  */
1230                 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1231                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1232                 cmd = sba_cmd_enc(cmd, msg_len,
1233                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1234                 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 1);
1235                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1236                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1237                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
1238                                   SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
1239                 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS,
1240                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1241                 cmdsp->cmd = cmd;
1242                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1243                 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1244                 cmdsp++;
1245
1246                 dpos -= pos;
1247         }
1248
1249 skip_q_computation:
1250         if (pq_continue) {
1251                 /*
1252                  * Type-B command to XOR previous output with
1253                  * buf0 and write it into buf0
1254                  */
1255                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1256                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1257                 cmd = sba_cmd_enc(cmd, msg_len,
1258                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1259                 c_mdata = sba_cmd_xor_c_mdata(0, 0);
1260                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1261                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1262                 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
1263                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1264                 cmdsp->cmd = cmd;
1265                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1266                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1267                 cmdsp->data = *dst_q + msg_offset;
1268                 cmdsp->data_len = msg_len;
1269                 cmdsp++;
1270         }
1271
1272         /* Type-A command to write buf0 */
1273         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1274                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1275         cmd = sba_cmd_enc(cmd, msg_len,
1276                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1277         cmd = sba_cmd_enc(cmd, 0x1,
1278                           SBA_RESP_SHIFT, SBA_RESP_MASK);
1279         c_mdata = sba_cmd_write_c_mdata(0);
1280         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1281                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1282         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1283                           SBA_CMD_SHIFT, SBA_CMD_MASK);
1284         cmdsp->cmd = cmd;
1285         *cmdsp->cmd_dma = cpu_to_le64(cmd);
1286         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1287         if (req->sba->hw_resp_size) {
1288                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1289                 cmdsp->resp = resp_dma;
1290                 cmdsp->resp_len = req->sba->hw_resp_size;
1291         }
1292         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1293         cmdsp->data = *dst_q + msg_offset;
1294         cmdsp->data_len = msg_len;
1295         cmdsp++;
1296
1297 skip_q:
1298         /* Fillup brcm_message */
1299         msg->type = BRCM_MESSAGE_SBA;
1300         msg->sba.cmds = cmds;
1301         msg->sba.cmds_count = cmdsp - cmds;
1302         msg->ctx = req;
1303         msg->error = 0;
1304 }
1305
1306 static struct sba_request *
1307 sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off,
1308                            dma_addr_t *dst_p, dma_addr_t *dst_q,
1309                            dma_addr_t src, u8 scf, size_t len,
1310                            unsigned long flags)
1311 {
1312         struct sba_request *req = NULL;
1313
1314         /* Alloc new request */
1315         req = sba_alloc_request(sba);
1316         if (!req)
1317                 return NULL;
1318         if (flags & DMA_PREP_FENCE)
1319                 req->flags |= SBA_REQUEST_FENCE;
1320
1321         /* Fillup request messages */
1322         sba_fillup_pq_single_msg(req,  dmaf_continue(flags),
1323                                  req->cmds, &req->msg, off, len,
1324                                  dst_p, dst_q, src, scf);
1325
1326         /* Init async_tx descriptor */
1327         req->tx.flags = flags;
1328         req->tx.cookie = -EBUSY;
1329
1330         return req;
1331 }
1332
1333 static struct dma_async_tx_descriptor *
1334 sba_prep_dma_pq(struct dma_chan *dchan, dma_addr_t *dst, dma_addr_t *src,
1335                 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
1336 {
1337         u32 i, dst_q_index;
1338         size_t req_len;
1339         bool slow = false;
1340         dma_addr_t off = 0;
1341         dma_addr_t *dst_p = NULL, *dst_q = NULL;
1342         struct sba_device *sba = to_sba_device(dchan);
1343         struct sba_request *first = NULL, *req;
1344
1345         /* Sanity checks */
1346         if (unlikely(src_cnt > sba->max_pq_srcs))
1347                 return NULL;
1348         for (i = 0; i < src_cnt; i++)
1349                 if (sba->max_pq_coefs <= raid6_gflog[scf[i]])
1350                         slow = true;
1351
1352         /* Figure-out P and Q destination addresses */
1353         if (!(flags & DMA_PREP_PQ_DISABLE_P))
1354                 dst_p = &dst[0];
1355         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
1356                 dst_q = &dst[1];
1357
1358         /* Create chained requests where each request is upto hw_buf_size */
1359         while (len) {
1360                 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
1361
1362                 if (slow) {
1363                         dst_q_index = src_cnt;
1364
1365                         if (dst_q) {
1366                                 for (i = 0; i < src_cnt; i++) {
1367                                         if (*dst_q == src[i]) {
1368                                                 dst_q_index = i;
1369                                                 break;
1370                                         }
1371                                 }
1372                         }
1373
1374                         if (dst_q_index < src_cnt) {
1375                                 i = dst_q_index;
1376                                 req = sba_prep_dma_pq_single_req(sba,
1377                                         off, dst_p, dst_q, src[i], scf[i],
1378                                         req_len, flags | DMA_PREP_FENCE);
1379                                 if (!req)
1380                                         goto fail;
1381
1382                                 if (first)
1383                                         sba_chain_request(first, req);
1384                                 else
1385                                         first = req;
1386
1387                                 flags |= DMA_PREP_CONTINUE;
1388                         }
1389
1390                         for (i = 0; i < src_cnt; i++) {
1391                                 if (dst_q_index == i)
1392                                         continue;
1393
1394                                 req = sba_prep_dma_pq_single_req(sba,
1395                                         off, dst_p, dst_q, src[i], scf[i],
1396                                         req_len, flags | DMA_PREP_FENCE);
1397                                 if (!req)
1398                                         goto fail;
1399
1400                                 if (first)
1401                                         sba_chain_request(first, req);
1402                                 else
1403                                         first = req;
1404
1405                                 flags |= DMA_PREP_CONTINUE;
1406                         }
1407                 } else {
1408                         req = sba_prep_dma_pq_req(sba, off,
1409                                                   dst_p, dst_q, src, src_cnt,
1410                                                   scf, req_len, flags);
1411                         if (!req)
1412                                 goto fail;
1413
1414                         if (first)
1415                                 sba_chain_request(first, req);
1416                         else
1417                                 first = req;
1418                 }
1419
1420                 off += req_len;
1421                 len -= req_len;
1422         }
1423
1424         return (first) ? &first->tx : NULL;
1425
1426 fail:
1427         if (first)
1428                 sba_free_chained_requests(first);
1429         return NULL;
1430 }
1431
1432 /* ====== Mailbox callbacks ===== */
1433
1434 static void sba_receive_message(struct mbox_client *cl, void *msg)
1435 {
1436         struct brcm_message *m = msg;
1437         struct sba_request *req = m->ctx;
1438         struct sba_device *sba = req->sba;
1439
1440         /* Error count if message has error */
1441         if (m->error < 0)
1442                 dev_err(sba->dev, "%s got message with error %d",
1443                         dma_chan_name(&sba->dma_chan), m->error);
1444
1445         /* Process received request */
1446         sba_process_received_request(sba, req);
1447 }
1448
1449 /* ====== Debugfs callbacks ====== */
1450
1451 static int sba_debugfs_stats_show(struct seq_file *file, void *offset)
1452 {
1453         struct sba_device *sba = dev_get_drvdata(file->private);
1454
1455         /* Write stats in file */
1456         sba_write_stats_in_seqfile(sba, file);
1457
1458         return 0;
1459 }
1460
1461 /* ====== Platform driver routines ===== */
1462
1463 static int sba_prealloc_channel_resources(struct sba_device *sba)
1464 {
1465         int i, j, ret = 0;
1466         struct sba_request *req = NULL;
1467
1468         sba->resp_base = dma_alloc_coherent(sba->mbox_dev,
1469                                             sba->max_resp_pool_size,
1470                                             &sba->resp_dma_base, GFP_KERNEL);
1471         if (!sba->resp_base)
1472                 return -ENOMEM;
1473
1474         sba->cmds_base = dma_alloc_coherent(sba->mbox_dev,
1475                                             sba->max_cmds_pool_size,
1476                                             &sba->cmds_dma_base, GFP_KERNEL);
1477         if (!sba->cmds_base) {
1478                 ret = -ENOMEM;
1479                 goto fail_free_resp_pool;
1480         }
1481
1482         spin_lock_init(&sba->reqs_lock);
1483         sba->reqs_fence = false;
1484         INIT_LIST_HEAD(&sba->reqs_alloc_list);
1485         INIT_LIST_HEAD(&sba->reqs_pending_list);
1486         INIT_LIST_HEAD(&sba->reqs_active_list);
1487         INIT_LIST_HEAD(&sba->reqs_aborted_list);
1488         INIT_LIST_HEAD(&sba->reqs_free_list);
1489
1490         for (i = 0; i < sba->max_req; i++) {
1491                 req = devm_kzalloc(sba->dev,
1492                                    struct_size(req, cmds, sba->max_cmd_per_req),
1493                                    GFP_KERNEL);
1494                 if (!req) {
1495                         ret = -ENOMEM;
1496                         goto fail_free_cmds_pool;
1497                 }
1498                 INIT_LIST_HEAD(&req->node);
1499                 req->sba = sba;
1500                 req->flags = SBA_REQUEST_STATE_FREE;
1501                 INIT_LIST_HEAD(&req->next);
1502                 atomic_set(&req->next_pending_count, 0);
1503                 for (j = 0; j < sba->max_cmd_per_req; j++) {
1504                         req->cmds[j].cmd = 0;
1505                         req->cmds[j].cmd_dma = sba->cmds_base +
1506                                 (i * sba->max_cmd_per_req + j) * sizeof(u64);
1507                         req->cmds[j].cmd_dma_addr = sba->cmds_dma_base +
1508                                 (i * sba->max_cmd_per_req + j) * sizeof(u64);
1509                         req->cmds[j].flags = 0;
1510                 }
1511                 memset(&req->msg, 0, sizeof(req->msg));
1512                 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
1513                 async_tx_ack(&req->tx);
1514                 req->tx.tx_submit = sba_tx_submit;
1515                 req->tx.phys = sba->resp_dma_base + i * sba->hw_resp_size;
1516                 list_add_tail(&req->node, &sba->reqs_free_list);
1517         }
1518
1519         return 0;
1520
1521 fail_free_cmds_pool:
1522         dma_free_coherent(sba->mbox_dev,
1523                           sba->max_cmds_pool_size,
1524                           sba->cmds_base, sba->cmds_dma_base);
1525 fail_free_resp_pool:
1526         dma_free_coherent(sba->mbox_dev,
1527                           sba->max_resp_pool_size,
1528                           sba->resp_base, sba->resp_dma_base);
1529         return ret;
1530 }
1531
1532 static void sba_freeup_channel_resources(struct sba_device *sba)
1533 {
1534         dmaengine_terminate_all(&sba->dma_chan);
1535         dma_free_coherent(sba->mbox_dev, sba->max_cmds_pool_size,
1536                           sba->cmds_base, sba->cmds_dma_base);
1537         dma_free_coherent(sba->mbox_dev, sba->max_resp_pool_size,
1538                           sba->resp_base, sba->resp_dma_base);
1539         sba->resp_base = NULL;
1540         sba->resp_dma_base = 0;
1541 }
1542
1543 static int sba_async_register(struct sba_device *sba)
1544 {
1545         int ret;
1546         struct dma_device *dma_dev = &sba->dma_dev;
1547
1548         /* Initialize DMA channel cookie */
1549         sba->dma_chan.device = dma_dev;
1550         dma_cookie_init(&sba->dma_chan);
1551
1552         /* Initialize DMA device capability mask */
1553         dma_cap_zero(dma_dev->cap_mask);
1554         dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
1555         dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1556         dma_cap_set(DMA_XOR, dma_dev->cap_mask);
1557         dma_cap_set(DMA_PQ, dma_dev->cap_mask);
1558
1559         /*
1560          * Set mailbox channel device as the base device of
1561          * our dma_device because the actual memory accesses
1562          * will be done by mailbox controller
1563          */
1564         dma_dev->dev = sba->mbox_dev;
1565
1566         /* Set base prep routines */
1567         dma_dev->device_free_chan_resources = sba_free_chan_resources;
1568         dma_dev->device_terminate_all = sba_device_terminate_all;
1569         dma_dev->device_issue_pending = sba_issue_pending;
1570         dma_dev->device_tx_status = sba_tx_status;
1571
1572         /* Set interrupt routine */
1573         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1574                 dma_dev->device_prep_dma_interrupt = sba_prep_dma_interrupt;
1575
1576         /* Set memcpy routine */
1577         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1578                 dma_dev->device_prep_dma_memcpy = sba_prep_dma_memcpy;
1579
1580         /* Set xor routine and capability */
1581         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1582                 dma_dev->device_prep_dma_xor = sba_prep_dma_xor;
1583                 dma_dev->max_xor = sba->max_xor_srcs;
1584         }
1585
1586         /* Set pq routine and capability */
1587         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1588                 dma_dev->device_prep_dma_pq = sba_prep_dma_pq;
1589                 dma_set_maxpq(dma_dev, sba->max_pq_srcs, 0);
1590         }
1591
1592         /* Initialize DMA device channel list */
1593         INIT_LIST_HEAD(&dma_dev->channels);
1594         list_add_tail(&sba->dma_chan.device_node, &dma_dev->channels);
1595
1596         /* Register with Linux async DMA framework*/
1597         ret = dma_async_device_register(dma_dev);
1598         if (ret) {
1599                 dev_err(sba->dev, "async device register error %d", ret);
1600                 return ret;
1601         }
1602
1603         dev_info(sba->dev, "%s capabilities: %s%s%s%s\n",
1604         dma_chan_name(&sba->dma_chan),
1605         dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "interrupt " : "",
1606         dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "memcpy " : "",
1607         dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1608         dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "");
1609
1610         return 0;
1611 }
1612
1613 static int sba_probe(struct platform_device *pdev)
1614 {
1615         int ret = 0;
1616         struct sba_device *sba;
1617         struct platform_device *mbox_pdev;
1618         struct of_phandle_args args;
1619
1620         /* Allocate main SBA struct */
1621         sba = devm_kzalloc(&pdev->dev, sizeof(*sba), GFP_KERNEL);
1622         if (!sba)
1623                 return -ENOMEM;
1624
1625         sba->dev = &pdev->dev;
1626         platform_set_drvdata(pdev, sba);
1627
1628         /* Number of mailbox channels should be atleast 1 */
1629         ret = of_count_phandle_with_args(pdev->dev.of_node,
1630                                          "mboxes", "#mbox-cells");
1631         if (ret <= 0)
1632                 return -ENODEV;
1633
1634         /* Determine SBA version from DT compatible string */
1635         if (of_device_is_compatible(sba->dev->of_node, "brcm,iproc-sba"))
1636                 sba->ver = SBA_VER_1;
1637         else if (of_device_is_compatible(sba->dev->of_node,
1638                                          "brcm,iproc-sba-v2"))
1639                 sba->ver = SBA_VER_2;
1640         else
1641                 return -ENODEV;
1642
1643         /* Derived Configuration parameters */
1644         switch (sba->ver) {
1645         case SBA_VER_1:
1646                 sba->hw_buf_size = 4096;
1647                 sba->hw_resp_size = 8;
1648                 sba->max_pq_coefs = 6;
1649                 sba->max_pq_srcs = 6;
1650                 break;
1651         case SBA_VER_2:
1652                 sba->hw_buf_size = 4096;
1653                 sba->hw_resp_size = 8;
1654                 sba->max_pq_coefs = 30;
1655                 /*
1656                  * We can support max_pq_srcs == max_pq_coefs because
1657                  * we are limited by number of SBA commands that we can
1658                  * fit in one message for underlying ring manager HW.
1659                  */
1660                 sba->max_pq_srcs = 12;
1661                 break;
1662         default:
1663                 return -EINVAL;
1664         }
1665         sba->max_req = SBA_MAX_REQ_PER_MBOX_CHANNEL;
1666         sba->max_cmd_per_req = sba->max_pq_srcs + 3;
1667         sba->max_xor_srcs = sba->max_cmd_per_req - 1;
1668         sba->max_resp_pool_size = sba->max_req * sba->hw_resp_size;
1669         sba->max_cmds_pool_size = sba->max_req *
1670                                   sba->max_cmd_per_req * sizeof(u64);
1671
1672         /* Setup mailbox client */
1673         sba->client.dev                 = &pdev->dev;
1674         sba->client.rx_callback         = sba_receive_message;
1675         sba->client.tx_block            = false;
1676         sba->client.knows_txdone        = true;
1677         sba->client.tx_tout             = 0;
1678
1679         /* Request mailbox channel */
1680         sba->mchan = mbox_request_channel(&sba->client, 0);
1681         if (IS_ERR(sba->mchan)) {
1682                 ret = PTR_ERR(sba->mchan);
1683                 goto fail_free_mchan;
1684         }
1685
1686         /* Find-out underlying mailbox device */
1687         ret = of_parse_phandle_with_args(pdev->dev.of_node,
1688                                          "mboxes", "#mbox-cells", 0, &args);
1689         if (ret)
1690                 goto fail_free_mchan;
1691         mbox_pdev = of_find_device_by_node(args.np);
1692         of_node_put(args.np);
1693         if (!mbox_pdev) {
1694                 ret = -ENODEV;
1695                 goto fail_free_mchan;
1696         }
1697         sba->mbox_dev = &mbox_pdev->dev;
1698
1699         /* Prealloc channel resource */
1700         ret = sba_prealloc_channel_resources(sba);
1701         if (ret)
1702                 goto fail_free_mchan;
1703
1704         /* Check availability of debugfs */
1705         if (!debugfs_initialized())
1706                 goto skip_debugfs;
1707
1708         /* Create debugfs root entry */
1709         sba->root = debugfs_create_dir(dev_name(sba->dev), NULL);
1710
1711         /* Create debugfs stats entry */
1712         debugfs_create_devm_seqfile(sba->dev, "stats", sba->root,
1713                                     sba_debugfs_stats_show);
1714
1715 skip_debugfs:
1716
1717         /* Register DMA device with Linux async framework */
1718         ret = sba_async_register(sba);
1719         if (ret)
1720                 goto fail_free_resources;
1721
1722         /* Print device info */
1723         dev_info(sba->dev, "%s using SBAv%d mailbox channel from %s",
1724                  dma_chan_name(&sba->dma_chan), sba->ver+1,
1725                  dev_name(sba->mbox_dev));
1726
1727         return 0;
1728
1729 fail_free_resources:
1730         debugfs_remove_recursive(sba->root);
1731         sba_freeup_channel_resources(sba);
1732 fail_free_mchan:
1733         mbox_free_channel(sba->mchan);
1734         return ret;
1735 }
1736
1737 static int sba_remove(struct platform_device *pdev)
1738 {
1739         struct sba_device *sba = platform_get_drvdata(pdev);
1740
1741         dma_async_device_unregister(&sba->dma_dev);
1742
1743         debugfs_remove_recursive(sba->root);
1744
1745         sba_freeup_channel_resources(sba);
1746
1747         mbox_free_channel(sba->mchan);
1748
1749         return 0;
1750 }
1751
1752 static const struct of_device_id sba_of_match[] = {
1753         { .compatible = "brcm,iproc-sba", },
1754         { .compatible = "brcm,iproc-sba-v2", },
1755         {},
1756 };
1757 MODULE_DEVICE_TABLE(of, sba_of_match);
1758
1759 static struct platform_driver sba_driver = {
1760         .probe = sba_probe,
1761         .remove = sba_remove,
1762         .driver = {
1763                 .name = "bcm-sba-raid",
1764                 .of_match_table = sba_of_match,
1765         },
1766 };
1767 module_platform_driver(sba_driver);
1768
1769 MODULE_DESCRIPTION("Broadcom SBA RAID driver");
1770 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
1771 MODULE_LICENSE("GPL v2");