drivers/dma/bcm-sba-raid.c

   1 /*
   2  * Copyright (C) 2017 Broadcom
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 /*
  10  * Broadcom SBA RAID Driver
  11  *
  12  * The Broadcom stream buffer accelerator (SBA) provides offloading
  13  * capabilities for RAID operations. The SBA offload engine is accessible
  14  * via Broadcom SoC specific ring manager. Two or more offload engines
  15  * can share same Broadcom SoC specific ring manager due to this Broadcom
  16  * SoC specific ring manager driver is implemented as a mailbox controller
  17  * driver and offload engine drivers are implemented as mallbox clients.
  18  *
  19  * Typically, Broadcom SoC specific ring manager will implement larger
  20  * number of hardware rings over one or more SBA hardware devices. By
  21  * design, the internal buffer size of SBA hardware device is limited
  22  * but all offload operations supported by SBA can be broken down into
  23  * multiple small size requests and executed parallely on multiple SBA
  24  * hardware devices for achieving high through-put.
  25  *
  26  * The Broadcom SBA RAID driver does not require any register programming
  27  * except submitting request to SBA hardware device via mailbox channels.
  28  * This driver implements a DMA device with one DMA channel using a set
  29  * of mailbox channels provided by Broadcom SoC specific ring manager
  30  * driver. To exploit parallelism (as described above), all DMA request
  31  * coming to SBA RAID DMA channel are broken down to smaller requests
  32  * and submitted to multiple mailbox channels in round-robin fashion.
  33  * For having more SBA DMA channels, we can create more SBA device nodes
  34  * in Broadcom SoC specific DTS based on number of hardware rings supported
  35  * by Broadcom SoC ring manager.
  36  */
  37
  38 #include <linux/bitops.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/dmaengine.h>
  41 #include <linux/list.h>
  42 #include <linux/mailbox_client.h>
  43 #include <linux/mailbox/brcm-message.h>
  44 #include <linux/module.h>
  45 #include <linux/of_device.h>
  46 #include <linux/slab.h>
  47 #include <linux/raid/pq.h>
  48
  49 #include "dmaengine.h"
  50
  51 /* ====== Driver macros and defines ===== */
  52
  53 #define SBA_TYPE_SHIFT                                  48
  54 #define SBA_TYPE_MASK                                   GENMASK(1, 0)
  55 #define SBA_TYPE_A                                      0x0
  56 #define SBA_TYPE_B                                      0x2
  57 #define SBA_TYPE_C                                      0x3
  58 #define SBA_USER_DEF_SHIFT                              32
  59 #define SBA_USER_DEF_MASK                               GENMASK(15, 0)
  60 #define SBA_R_MDATA_SHIFT                               24
  61 #define SBA_R_MDATA_MASK                                GENMASK(7, 0)
  62 #define SBA_C_MDATA_MS_SHIFT                            18
  63 #define SBA_C_MDATA_MS_MASK                             GENMASK(1, 0)
  64 #define SBA_INT_SHIFT                                   17
  65 #define SBA_INT_MASK                                    BIT(0)
  66 #define SBA_RESP_SHIFT                                  16
  67 #define SBA_RESP_MASK                                   BIT(0)
  68 #define SBA_C_MDATA_SHIFT                               8
  69 #define SBA_C_MDATA_MASK                                GENMASK(7, 0)
  70 #define SBA_C_MDATA_BNUMx_SHIFT(__bnum)                 (2 * (__bnum))
  71 #define SBA_C_MDATA_BNUMx_MASK                          GENMASK(1, 0)
  72 #define SBA_C_MDATA_DNUM_SHIFT                          5
  73 #define SBA_C_MDATA_DNUM_MASK                           GENMASK(4, 0)
  74 #define SBA_C_MDATA_LS(__v)                             ((__v) & 0xff)
  75 #define SBA_C_MDATA_MS(__v)                             (((__v) >> 8) & 0x3)
  76 #define SBA_CMD_SHIFT                                   0
  77 #define SBA_CMD_MASK                                    GENMASK(3, 0)
  78 #define SBA_CMD_ZERO_BUFFER                             0x4
  79 #define SBA_CMD_ZERO_ALL_BUFFERS                        0x8
  80 #define SBA_CMD_LOAD_BUFFER                             0x9
  81 #define SBA_CMD_XOR                                     0xa
  82 #define SBA_CMD_GALOIS_XOR                              0xb
  83 #define SBA_CMD_WRITE_BUFFER                            0xc
  84 #define SBA_CMD_GALOIS                                  0xe
  85
  86 #define SBA_MAX_REQ_PER_MBOX_CHANNEL                    8192
  87
  88 /* Driver helper macros */
  89 #define to_sba_request(tx)              \
  90         container_of(tx, struct sba_request, tx)
  91 #define to_sba_device(dchan)            \
  92         container_of(dchan, struct sba_device, dma_chan)
  93
  94 /* ===== Driver data structures ===== */
  95
  96 enum sba_request_flags {
  97         SBA_REQUEST_STATE_FREE          = 0x001,
  98         SBA_REQUEST_STATE_ALLOCED       = 0x002,
  99         SBA_REQUEST_STATE_PENDING       = 0x004,
 100         SBA_REQUEST_STATE_ACTIVE        = 0x008,
 101         SBA_REQUEST_STATE_COMPLETED     = 0x010,
 102         SBA_REQUEST_STATE_ABORTED       = 0x020,
 103         SBA_REQUEST_STATE_MASK          = 0x0ff,
 104         SBA_REQUEST_FENCE               = 0x100,
 105 };
 106
 107 struct sba_request {
 108         /* Global state */
 109         struct list_head node;
 110         struct sba_device *sba;
 111         u32 flags;
 112         /* Chained requests management */
 113         struct sba_request *first;
 114         struct list_head next;
 115         atomic_t next_pending_count;
 116         /* BRCM message data */
 117         struct brcm_message msg;
 118         struct dma_async_tx_descriptor tx;
 119         /* SBA commands */
 120         struct brcm_sba_command cmds[0];
 121 };
 122
 123 enum sba_version {
 124         SBA_VER_1 = 0,
 125         SBA_VER_2
 126 };
 127
 128 struct sba_device {
 129         /* Underlying device */
 130         struct device *dev;
 131         /* DT configuration parameters */
 132         enum sba_version ver;
 133         /* Derived configuration parameters */
 134         u32 max_req;
 135         u32 hw_buf_size;
 136         u32 hw_resp_size;
 137         u32 max_pq_coefs;
 138         u32 max_pq_srcs;
 139         u32 max_cmd_per_req;
 140         u32 max_xor_srcs;
 141         u32 max_resp_pool_size;
 142         u32 max_cmds_pool_size;
 143         /* Maibox client and Mailbox channels */
 144         struct mbox_client client;
 145         int mchans_count;
 146         atomic_t mchans_current;
 147         struct mbox_chan **mchans;
 148         struct device *mbox_dev;
 149         /* DMA device and DMA channel */
 150         struct dma_device dma_dev;
 151         struct dma_chan dma_chan;
 152         /* DMA channel resources */
 153         void *resp_base;
 154         dma_addr_t resp_dma_base;
 155         void *cmds_base;
 156         dma_addr_t cmds_dma_base;
 157         spinlock_t reqs_lock;
 158         bool reqs_fence;
 159         struct list_head reqs_alloc_list;
 160         struct list_head reqs_pending_list;
 161         struct list_head reqs_active_list;
 162         struct list_head reqs_completed_list;
 163         struct list_head reqs_aborted_list;
 164         struct list_head reqs_free_list;
 165 };
 166
 167 /* ====== Command helper routines ===== */
 168
 169 static inline u64 __pure sba_cmd_enc(u64 cmd, u32 val, u32 shift, u32 mask)
 170 {
 171         cmd &= ~((u64)mask << shift);
 172         cmd |= ((u64)(val & mask) << shift);
 173         return cmd;
 174 }
 175
 176 static inline u32 __pure sba_cmd_load_c_mdata(u32 b0)
 177 {
 178         return b0 & SBA_C_MDATA_BNUMx_MASK;
 179 }
 180
 181 static inline u32 __pure sba_cmd_write_c_mdata(u32 b0)
 182 {
 183         return b0 & SBA_C_MDATA_BNUMx_MASK;
 184 }
 185
 186 static inline u32 __pure sba_cmd_xor_c_mdata(u32 b1, u32 b0)
 187 {
 188         return (b0 & SBA_C_MDATA_BNUMx_MASK) |
 189                ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1));
 190 }
 191
 192 static inline u32 __pure sba_cmd_pq_c_mdata(u32 d, u32 b1, u32 b0)
 193 {
 194         return (b0 & SBA_C_MDATA_BNUMx_MASK) |
 195                ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)) |
 196                ((d & SBA_C_MDATA_DNUM_MASK) << SBA_C_MDATA_DNUM_SHIFT);
 197 }
 198
 199 /* ====== General helper routines ===== */
 200
 201 static void sba_peek_mchans(struct sba_device *sba)
 202 {
 203         int mchan_idx;
 204
 205         for (mchan_idx = 0; mchan_idx < sba->mchans_count; mchan_idx++)
 206                 mbox_client_peek_data(sba->mchans[mchan_idx]);
 207 }
 208
 209 static struct sba_request *sba_alloc_request(struct sba_device *sba)
 210 {
 211         unsigned long flags;
 212         struct sba_request *req = NULL;
 213
 214         spin_lock_irqsave(&sba->reqs_lock, flags);
 215         req = list_first_entry_or_null(&sba->reqs_free_list,
 216                                        struct sba_request, node);
 217         if (req)
 218                 list_move_tail(&req->node, &sba->reqs_alloc_list);
 219         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 220
 221         if (!req) {
 222                 /*
 223                  * We have no more free requests so, we peek
 224                  * mailbox channels hoping few active requests
 225                  * would have completed which will create more
 226                  * room for new requests.
 227                  */
 228                 sba_peek_mchans(sba);
 229                 return NULL;
 230         }
 231
 232         req->flags = SBA_REQUEST_STATE_ALLOCED;
 233         req->first = req;
 234         INIT_LIST_HEAD(&req->next);
 235         atomic_set(&req->next_pending_count, 1);
 236
 237         dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
 238         async_tx_ack(&req->tx);
 239
 240         return req;
 241 }
 242
 243 /* Note: Must be called with sba->reqs_lock held */
 244 static void _sba_pending_request(struct sba_device *sba,
 245                                  struct sba_request *req)
 246 {
 247         lockdep_assert_held(&sba->reqs_lock);
 248         req->flags &= ~SBA_REQUEST_STATE_MASK;
 249         req->flags |= SBA_REQUEST_STATE_PENDING;
 250         list_move_tail(&req->node, &sba->reqs_pending_list);
 251         if (list_empty(&sba->reqs_active_list))
 252                 sba->reqs_fence = false;
 253 }
 254
 255 /* Note: Must be called with sba->reqs_lock held */
 256 static bool _sba_active_request(struct sba_device *sba,
 257                                 struct sba_request *req)
 258 {
 259         lockdep_assert_held(&sba->reqs_lock);
 260         if (list_empty(&sba->reqs_active_list))
 261                 sba->reqs_fence = false;
 262         if (sba->reqs_fence)
 263                 return false;
 264         req->flags &= ~SBA_REQUEST_STATE_MASK;
 265         req->flags |= SBA_REQUEST_STATE_ACTIVE;
 266         list_move_tail(&req->node, &sba->reqs_active_list);
 267         if (req->flags & SBA_REQUEST_FENCE)
 268                 sba->reqs_fence = true;
 269         return true;
 270 }
 271
 272 /* Note: Must be called with sba->reqs_lock held */
 273 static void _sba_abort_request(struct sba_device *sba,
 274                                struct sba_request *req)
 275 {
 276         lockdep_assert_held(&sba->reqs_lock);
 277         req->flags &= ~SBA_REQUEST_STATE_MASK;
 278         req->flags |= SBA_REQUEST_STATE_ABORTED;
 279         list_move_tail(&req->node, &sba->reqs_aborted_list);
 280         if (list_empty(&sba->reqs_active_list))
 281                 sba->reqs_fence = false;
 282 }
 283
 284 /* Note: Must be called with sba->reqs_lock held */
 285 static void _sba_free_request(struct sba_device *sba,
 286                               struct sba_request *req)
 287 {
 288         lockdep_assert_held(&sba->reqs_lock);
 289         req->flags &= ~SBA_REQUEST_STATE_MASK;
 290         req->flags |= SBA_REQUEST_STATE_FREE;
 291         list_move_tail(&req->node, &sba->reqs_free_list);
 292         if (list_empty(&sba->reqs_active_list))
 293                 sba->reqs_fence = false;
 294 }
 295
 296 /* Note: Must be called with sba->reqs_lock held */
 297 static void _sba_complete_request(struct sba_device *sba,
 298                                   struct sba_request *req)
 299 {
 300         lockdep_assert_held(&sba->reqs_lock);
 301         req->flags &= ~SBA_REQUEST_STATE_MASK;
 302         req->flags |= SBA_REQUEST_STATE_COMPLETED;
 303         list_move_tail(&req->node, &sba->reqs_completed_list);
 304         if (list_empty(&sba->reqs_active_list))
 305                 sba->reqs_fence = false;
 306 }
 307
 308 static void sba_free_chained_requests(struct sba_request *req)
 309 {
 310         unsigned long flags;
 311         struct sba_request *nreq;
 312         struct sba_device *sba = req->sba;
 313
 314         spin_lock_irqsave(&sba->reqs_lock, flags);
 315
 316         _sba_free_request(sba, req);
 317         list_for_each_entry(nreq, &req->next, next)
 318                 _sba_free_request(sba, nreq);
 319
 320         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 321 }
 322
 323 static void sba_chain_request(struct sba_request *first,
 324                               struct sba_request *req)
 325 {
 326         unsigned long flags;
 327         struct sba_device *sba = req->sba;
 328
 329         spin_lock_irqsave(&sba->reqs_lock, flags);
 330
 331         list_add_tail(&req->next, &first->next);
 332         req->first = first;
 333         atomic_inc(&first->next_pending_count);
 334
 335         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 336 }
 337
 338 static void sba_cleanup_nonpending_requests(struct sba_device *sba)
 339 {
 340         unsigned long flags;
 341         struct sba_request *req, *req1;
 342
 343         spin_lock_irqsave(&sba->reqs_lock, flags);
 344
 345         /* Freeup all alloced request */
 346         list_for_each_entry_safe(req, req1, &sba->reqs_alloc_list, node)
 347                 _sba_free_request(sba, req);
 348
 349         /* Freeup all completed request */
 350         list_for_each_entry_safe(req, req1, &sba->reqs_completed_list, node)
 351                 _sba_free_request(sba, req);
 352
 353         /* Set all active requests as aborted */
 354         list_for_each_entry_safe(req, req1, &sba->reqs_active_list, node)
 355                 _sba_abort_request(sba, req);
 356
 357         /*
 358          * Note: We expect that aborted request will be eventually
 359          * freed by sba_receive_message()
 360          */
 361
 362         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 363 }
 364
 365 static void sba_cleanup_pending_requests(struct sba_device *sba)
 366 {
 367         unsigned long flags;
 368         struct sba_request *req, *req1;
 369
 370         spin_lock_irqsave(&sba->reqs_lock, flags);
 371
 372         /* Freeup all pending request */
 373         list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node)
 374                 _sba_free_request(sba, req);
 375
 376         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 377 }
 378
 379 static int sba_send_mbox_request(struct sba_device *sba,
 380                                  struct sba_request *req)
 381 {
 382         int mchans_idx, ret = 0;
 383
 384         /* Select mailbox channel in round-robin fashion */
 385         mchans_idx = atomic_inc_return(&sba->mchans_current);
 386         mchans_idx = mchans_idx % sba->mchans_count;
 387
 388         /* Send message for the request */
 389         req->msg.error = 0;
 390         ret = mbox_send_message(sba->mchans[mchans_idx], &req->msg);
 391         if (ret < 0) {
 392                 dev_err(sba->dev, "send message failed with error %d", ret);
 393                 return ret;
 394         }
 395         ret = req->msg.error;
 396         if (ret < 0) {
 397                 dev_err(sba->dev, "message error %d", ret);
 398                 return ret;
 399         }
 400
 401         return 0;
 402 }
 403
 404 /* Note: Must be called with sba->reqs_lock held */
 405 static void _sba_process_pending_requests(struct sba_device *sba)
 406 {
 407         int ret;
 408         u32 count;
 409         struct sba_request *req;
 410
 411         /*
 412          * Process few pending requests
 413          *
 414          * For now, we process (<number_of_mailbox_channels> * 8)
 415          * number of requests at a time.
 416          */
 417         count = sba->mchans_count * 8;
 418         while (!list_empty(&sba->reqs_pending_list) && count) {
 419                 /* Get the first pending request */
 420                 req = list_first_entry(&sba->reqs_pending_list,
 421                                        struct sba_request, node);
 422
 423                 /* Try to make request active */
 424                 if (!_sba_active_request(sba, req))
 425                         break;
 426
 427                 /* Send request to mailbox channel */
 428                 ret = sba_send_mbox_request(sba, req);
 429                 if (ret < 0) {
 430                         _sba_pending_request(sba, req);
 431                         break;
 432                 }
 433
 434                 count--;
 435         }
 436 }
 437
 438 static void sba_process_received_request(struct sba_device *sba,
 439                                          struct sba_request *req)
 440 {
 441         unsigned long flags;
 442         struct dma_async_tx_descriptor *tx;
 443         struct sba_request *nreq, *first = req->first;
 444
 445         /* Process only after all chained requests are received */
 446         if (!atomic_dec_return(&first->next_pending_count)) {
 447                 tx = &first->tx;
 448
 449                 WARN_ON(tx->cookie < 0);
 450                 if (tx->cookie > 0) {
 451                         dma_cookie_complete(tx);
 452                         dmaengine_desc_get_callback_invoke(tx, NULL);
 453                         dma_descriptor_unmap(tx);
 454                         tx->callback = NULL;
 455                         tx->callback_result = NULL;
 456                 }
 457
 458                 dma_run_dependencies(tx);
 459
 460                 spin_lock_irqsave(&sba->reqs_lock, flags);
 461
 462                 /* Free all requests chained to first request */
 463                 list_for_each_entry(nreq, &first->next, next)
 464                         _sba_free_request(sba, nreq);
 465                 INIT_LIST_HEAD(&first->next);
 466
 467                 /* The client is allowed to attach dependent operations
 468                  * until 'ack' is set
 469                  */
 470                 if (!async_tx_test_ack(tx))
 471                         _sba_complete_request(sba, first);
 472                 else
 473                         _sba_free_request(sba, first);
 474
 475                 /* Cleanup completed requests */
 476                 list_for_each_entry_safe(req, nreq,
 477                                          &sba->reqs_completed_list, node) {
 478                         if (async_tx_test_ack(&req->tx))
 479                                 _sba_free_request(sba, req);
 480                 }
 481
 482                 /* Process pending requests */
 483                 _sba_process_pending_requests(sba);
 484
 485                 spin_unlock_irqrestore(&sba->reqs_lock, flags);
 486         }
 487 }
 488
 489 /* ====== DMAENGINE callbacks ===== */
 490
 491 static void sba_free_chan_resources(struct dma_chan *dchan)
 492 {
 493         /*
 494          * Channel resources are pre-alloced so we just free-up
 495          * whatever we can so that we can re-use pre-alloced
 496          * channel resources next time.
 497          */
 498         sba_cleanup_nonpending_requests(to_sba_device(dchan));
 499 }
 500
 501 static int sba_device_terminate_all(struct dma_chan *dchan)
 502 {
 503         /* Cleanup all pending requests */
 504         sba_cleanup_pending_requests(to_sba_device(dchan));
 505
 506         return 0;
 507 }
 508
 509 static void sba_issue_pending(struct dma_chan *dchan)
 510 {
 511         unsigned long flags;
 512         struct sba_device *sba = to_sba_device(dchan);
 513
 514         /* Process pending requests */
 515         spin_lock_irqsave(&sba->reqs_lock, flags);
 516         _sba_process_pending_requests(sba);
 517         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 518 }
 519
 520 static dma_cookie_t sba_tx_submit(struct dma_async_tx_descriptor *tx)
 521 {
 522         unsigned long flags;
 523         dma_cookie_t cookie;
 524         struct sba_device *sba;
 525         struct sba_request *req, *nreq;
 526
 527         if (unlikely(!tx))
 528                 return -EINVAL;
 529
 530         sba = to_sba_device(tx->chan);
 531         req = to_sba_request(tx);
 532
 533         /* Assign cookie and mark all chained requests pending */
 534         spin_lock_irqsave(&sba->reqs_lock, flags);
 535         cookie = dma_cookie_assign(tx);
 536         _sba_pending_request(sba, req);
 537         list_for_each_entry(nreq, &req->next, next)
 538                 _sba_pending_request(sba, nreq);
 539         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 540
 541         return cookie;
 542 }
 543
 544 static enum dma_status sba_tx_status(struct dma_chan *dchan,
 545                                      dma_cookie_t cookie,
 546                                      struct dma_tx_state *txstate)
 547 {
 548         enum dma_status ret;
 549         struct sba_device *sba = to_sba_device(dchan);
 550
 551         ret = dma_cookie_status(dchan, cookie, txstate);
 552         if (ret == DMA_COMPLETE)
 553                 return ret;
 554
 555         sba_peek_mchans(sba);
 556
 557         return dma_cookie_status(dchan, cookie, txstate);
 558 }
 559
 560 static void sba_fillup_interrupt_msg(struct sba_request *req,
 561                                      struct brcm_sba_command *cmds,
 562                                      struct brcm_message *msg)
 563 {
 564         u64 cmd;
 565         u32 c_mdata;
 566         dma_addr_t resp_dma = req->tx.phys;
 567         struct brcm_sba_command *cmdsp = cmds;
 568
 569         /* Type-B command to load dummy data into buf0 */
 570         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 571                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 572         cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size,
 573                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 574         c_mdata = sba_cmd_load_c_mdata(0);
 575         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 576                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 577         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 578                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 579         cmdsp->cmd = cmd;
 580         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 581         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 582         cmdsp->data = resp_dma;
 583         cmdsp->data_len = req->sba->hw_resp_size;
 584         cmdsp++;
 585
 586         /* Type-A command to write buf0 to dummy location */
 587         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 588                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 589         cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size,
 590                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 591         cmd = sba_cmd_enc(cmd, 0x1,
 592                           SBA_RESP_SHIFT, SBA_RESP_MASK);
 593         c_mdata = sba_cmd_write_c_mdata(0);
 594         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 595                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 596         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
 597                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 598         cmdsp->cmd = cmd;
 599         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 600         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 601         if (req->sba->hw_resp_size) {
 602                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
 603                 cmdsp->resp = resp_dma;
 604                 cmdsp->resp_len = req->sba->hw_resp_size;
 605         }
 606         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
 607         cmdsp->data = resp_dma;
 608         cmdsp->data_len = req->sba->hw_resp_size;
 609         cmdsp++;
 610
 611         /* Fillup brcm_message */
 612         msg->type = BRCM_MESSAGE_SBA;
 613         msg->sba.cmds = cmds;
 614         msg->sba.cmds_count = cmdsp - cmds;
 615         msg->ctx = req;
 616         msg->error = 0;
 617 }
 618
 619 static struct dma_async_tx_descriptor *
 620 sba_prep_dma_interrupt(struct dma_chan *dchan, unsigned long flags)
 621 {
 622         struct sba_request *req = NULL;
 623         struct sba_device *sba = to_sba_device(dchan);
 624
 625         /* Alloc new request */
 626         req = sba_alloc_request(sba);
 627         if (!req)
 628                 return NULL;
 629
 630         /*
 631          * Force fence so that no requests are submitted
 632          * until DMA callback for this request is invoked.
 633          */
 634         req->flags |= SBA_REQUEST_FENCE;
 635
 636         /* Fillup request message */
 637         sba_fillup_interrupt_msg(req, req->cmds, &req->msg);
 638
 639         /* Init async_tx descriptor */
 640         req->tx.flags = flags;
 641         req->tx.cookie = -EBUSY;
 642
 643         return &req->tx;
 644 }
 645
 646 static void sba_fillup_memcpy_msg(struct sba_request *req,
 647                                   struct brcm_sba_command *cmds,
 648                                   struct brcm_message *msg,
 649                                   dma_addr_t msg_offset, size_t msg_len,
 650                                   dma_addr_t dst, dma_addr_t src)
 651 {
 652         u64 cmd;
 653         u32 c_mdata;
 654         dma_addr_t resp_dma = req->tx.phys;
 655         struct brcm_sba_command *cmdsp = cmds;
 656
 657         /* Type-B command to load data into buf0 */
 658         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 659                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 660         cmd = sba_cmd_enc(cmd, msg_len,
 661                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 662         c_mdata = sba_cmd_load_c_mdata(0);
 663         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 664                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 665         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 666                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 667         cmdsp->cmd = cmd;
 668         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 669         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 670         cmdsp->data = src + msg_offset;
 671         cmdsp->data_len = msg_len;
 672         cmdsp++;
 673
 674         /* Type-A command to write buf0 */
 675         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 676                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 677         cmd = sba_cmd_enc(cmd, msg_len,
 678                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 679         cmd = sba_cmd_enc(cmd, 0x1,
 680                           SBA_RESP_SHIFT, SBA_RESP_MASK);
 681         c_mdata = sba_cmd_write_c_mdata(0);
 682         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 683                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 684         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
 685                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 686         cmdsp->cmd = cmd;
 687         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 688         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 689         if (req->sba->hw_resp_size) {
 690                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
 691                 cmdsp->resp = resp_dma;
 692                 cmdsp->resp_len = req->sba->hw_resp_size;
 693         }
 694         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
 695         cmdsp->data = dst + msg_offset;
 696         cmdsp->data_len = msg_len;
 697         cmdsp++;
 698
 699         /* Fillup brcm_message */
 700         msg->type = BRCM_MESSAGE_SBA;
 701         msg->sba.cmds = cmds;
 702         msg->sba.cmds_count = cmdsp - cmds;
 703         msg->ctx = req;
 704         msg->error = 0;
 705 }
 706
 707 static struct sba_request *
 708 sba_prep_dma_memcpy_req(struct sba_device *sba,
 709                         dma_addr_t off, dma_addr_t dst, dma_addr_t src,
 710                         size_t len, unsigned long flags)
 711 {
 712         struct sba_request *req = NULL;
 713
 714         /* Alloc new request */
 715         req = sba_alloc_request(sba);
 716         if (!req)
 717                 return NULL;
 718         if (flags & DMA_PREP_FENCE)
 719                 req->flags |= SBA_REQUEST_FENCE;
 720
 721         /* Fillup request message */
 722         sba_fillup_memcpy_msg(req, req->cmds, &req->msg,
 723                               off, len, dst, src);
 724
 725         /* Init async_tx descriptor */
 726         req->tx.flags = flags;
 727         req->tx.cookie = -EBUSY;
 728
 729         return req;
 730 }
 731
 732 static struct dma_async_tx_descriptor *
 733 sba_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src,
 734                     size_t len, unsigned long flags)
 735 {
 736         size_t req_len;
 737         dma_addr_t off = 0;
 738         struct sba_device *sba = to_sba_device(dchan);
 739         struct sba_request *first = NULL, *req;
 740
 741         /* Create chained requests where each request is upto hw_buf_size */
 742         while (len) {
 743                 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
 744
 745                 req = sba_prep_dma_memcpy_req(sba, off, dst, src,
 746                                               req_len, flags);
 747                 if (!req) {
 748                         if (first)
 749                                 sba_free_chained_requests(first);
 750                         return NULL;
 751                 }
 752
 753                 if (first)
 754                         sba_chain_request(first, req);
 755                 else
 756                         first = req;
 757
 758                 off += req_len;
 759                 len -= req_len;
 760         }
 761
 762         return (first) ? &first->tx : NULL;
 763 }
 764
 765 static void sba_fillup_xor_msg(struct sba_request *req,
 766                                 struct brcm_sba_command *cmds,
 767                                 struct brcm_message *msg,
 768                                 dma_addr_t msg_offset, size_t msg_len,
 769                                 dma_addr_t dst, dma_addr_t *src, u32 src_cnt)
 770 {
 771         u64 cmd;
 772         u32 c_mdata;
 773         unsigned int i;
 774         dma_addr_t resp_dma = req->tx.phys;
 775         struct brcm_sba_command *cmdsp = cmds;
 776
 777         /* Type-B command to load data into buf0 */
 778         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 779                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 780         cmd = sba_cmd_enc(cmd, msg_len,
 781                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 782         c_mdata = sba_cmd_load_c_mdata(0);
 783         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 784                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 785         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 786                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 787         cmdsp->cmd = cmd;
 788         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 789         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 790         cmdsp->data = src[0] + msg_offset;
 791         cmdsp->data_len = msg_len;
 792         cmdsp++;
 793
 794         /* Type-B commands to xor data with buf0 and put it back in buf0 */
 795         for (i = 1; i < src_cnt; i++) {
 796                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 797                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 798                 cmd = sba_cmd_enc(cmd, msg_len,
 799                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 800                 c_mdata = sba_cmd_xor_c_mdata(0, 0);
 801                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 802                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 803                 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
 804                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
 805                 cmdsp->cmd = cmd;
 806                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
 807                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 808                 cmdsp->data = src[i] + msg_offset;
 809                 cmdsp->data_len = msg_len;
 810                 cmdsp++;
 811         }
 812
 813         /* Type-A command to write buf0 */
 814         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 815                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 816         cmd = sba_cmd_enc(cmd, msg_len,
 817                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 818         cmd = sba_cmd_enc(cmd, 0x1,
 819                           SBA_RESP_SHIFT, SBA_RESP_MASK);
 820         c_mdata = sba_cmd_write_c_mdata(0);
 821         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 822                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 823         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
 824                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 825         cmdsp->cmd = cmd;
 826         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 827         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 828         if (req->sba->hw_resp_size) {
 829                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
 830                 cmdsp->resp = resp_dma;
 831                 cmdsp->resp_len = req->sba->hw_resp_size;
 832         }
 833         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
 834         cmdsp->data = dst + msg_offset;
 835         cmdsp->data_len = msg_len;
 836         cmdsp++;
 837
 838         /* Fillup brcm_message */
 839         msg->type = BRCM_MESSAGE_SBA;
 840         msg->sba.cmds = cmds;
 841         msg->sba.cmds_count = cmdsp - cmds;
 842         msg->ctx = req;
 843         msg->error = 0;
 844 }
 845
 846 static struct sba_request *
 847 sba_prep_dma_xor_req(struct sba_device *sba,
 848                      dma_addr_t off, dma_addr_t dst, dma_addr_t *src,
 849                      u32 src_cnt, size_t len, unsigned long flags)
 850 {
 851         struct sba_request *req = NULL;
 852
 853         /* Alloc new request */
 854         req = sba_alloc_request(sba);
 855         if (!req)
 856                 return NULL;
 857         if (flags & DMA_PREP_FENCE)
 858                 req->flags |= SBA_REQUEST_FENCE;
 859
 860         /* Fillup request message */
 861         sba_fillup_xor_msg(req, req->cmds, &req->msg,
 862                            off, len, dst, src, src_cnt);
 863
 864         /* Init async_tx descriptor */
 865         req->tx.flags = flags;
 866         req->tx.cookie = -EBUSY;
 867
 868         return req;
 869 }
 870
 871 static struct dma_async_tx_descriptor *
 872 sba_prep_dma_xor(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t *src,
 873                  u32 src_cnt, size_t len, unsigned long flags)
 874 {
 875         size_t req_len;
 876         dma_addr_t off = 0;
 877         struct sba_device *sba = to_sba_device(dchan);
 878         struct sba_request *first = NULL, *req;
 879
 880         /* Sanity checks */
 881         if (unlikely(src_cnt > sba->max_xor_srcs))
 882                 return NULL;
 883
 884         /* Create chained requests where each request is upto hw_buf_size */
 885         while (len) {
 886                 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
 887
 888                 req = sba_prep_dma_xor_req(sba, off, dst, src, src_cnt,
 889                                            req_len, flags);
 890                 if (!req) {
 891                         if (first)
 892                                 sba_free_chained_requests(first);
 893                         return NULL;
 894                 }
 895
 896                 if (first)
 897                         sba_chain_request(first, req);
 898                 else
 899                         first = req;
 900
 901                 off += req_len;
 902                 len -= req_len;
 903         }
 904
 905         return (first) ? &first->tx : NULL;
 906 }
 907
 908 static void sba_fillup_pq_msg(struct sba_request *req,
 909                                 bool pq_continue,
 910                                 struct brcm_sba_command *cmds,
 911                                 struct brcm_message *msg,
 912                                 dma_addr_t msg_offset, size_t msg_len,
 913                                 dma_addr_t *dst_p, dma_addr_t *dst_q,
 914                                 const u8 *scf, dma_addr_t *src, u32 src_cnt)
 915 {
 916         u64 cmd;
 917         u32 c_mdata;
 918         unsigned int i;
 919         dma_addr_t resp_dma = req->tx.phys;
 920         struct brcm_sba_command *cmdsp = cmds;
 921
 922         if (pq_continue) {
 923                 /* Type-B command to load old P into buf0 */
 924                 if (dst_p) {
 925                         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 926                                 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 927                         cmd = sba_cmd_enc(cmd, msg_len,
 928                                 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 929                         c_mdata = sba_cmd_load_c_mdata(0);
 930                         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 931                                 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 932                         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 933                                 SBA_CMD_SHIFT, SBA_CMD_MASK);
 934                         cmdsp->cmd = cmd;
 935                         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 936                         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 937                         cmdsp->data = *dst_p + msg_offset;
 938                         cmdsp->data_len = msg_len;
 939                         cmdsp++;
 940                 }
 941
 942                 /* Type-B command to load old Q into buf1 */
 943                 if (dst_q) {
 944                         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 945                                 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 946                         cmd = sba_cmd_enc(cmd, msg_len,
 947                                 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 948                         c_mdata = sba_cmd_load_c_mdata(1);
 949                         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 950                                 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 951                         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 952                                 SBA_CMD_SHIFT, SBA_CMD_MASK);
 953                         cmdsp->cmd = cmd;
 954                         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 955                         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 956                         cmdsp->data = *dst_q + msg_offset;
 957                         cmdsp->data_len = msg_len;
 958                         cmdsp++;
 959                 }
 960         } else {
 961                 /* Type-A command to zero all buffers */
 962                 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 963                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 964                 cmd = sba_cmd_enc(cmd, msg_len,
 965                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 966                 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS,
 967                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
 968                 cmdsp->cmd = cmd;
 969                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
 970                 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 971                 cmdsp++;
 972         }
 973
 974         /* Type-B commands for generate P onto buf0 and Q onto buf1 */
 975         for (i = 0; i < src_cnt; i++) {
 976                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 977                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 978                 cmd = sba_cmd_enc(cmd, msg_len,
 979                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 980                 c_mdata = sba_cmd_pq_c_mdata(raid6_gflog[scf[i]], 1, 0);
 981                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 982                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 983                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
 984                                   SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
 985                 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS_XOR,
 986                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
 987                 cmdsp->cmd = cmd;
 988                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
 989                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 990                 cmdsp->data = src[i] + msg_offset;
 991                 cmdsp->data_len = msg_len;
 992                 cmdsp++;
 993         }
 994
 995         /* Type-A command to write buf0 */
 996         if (dst_p) {
 997                 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 998                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 999                 cmd = sba_cmd_enc(cmd, msg_len,
1000                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1001                 cmd = sba_cmd_enc(cmd, 0x1,
1002                                   SBA_RESP_SHIFT, SBA_RESP_MASK);
1003                 c_mdata = sba_cmd_write_c_mdata(0);
1004                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1005                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1006                 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1007                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1008                 cmdsp->cmd = cmd;
1009                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1010                 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1011                 if (req->sba->hw_resp_size) {
1012                         cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1013                         cmdsp->resp = resp_dma;
1014                         cmdsp->resp_len = req->sba->hw_resp_size;
1015                 }
1016                 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1017                 cmdsp->data = *dst_p + msg_offset;
1018                 cmdsp->data_len = msg_len;
1019                 cmdsp++;
1020         }
1021
1022         /* Type-A command to write buf1 */
1023         if (dst_q) {
1024                 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1025                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1026                 cmd = sba_cmd_enc(cmd, msg_len,
1027                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1028                 cmd = sba_cmd_enc(cmd, 0x1,
1029                                   SBA_RESP_SHIFT, SBA_RESP_MASK);
1030                 c_mdata = sba_cmd_write_c_mdata(1);
1031                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1032                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1033                 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1034                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1035                 cmdsp->cmd = cmd;
1036                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1037                 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1038                 if (req->sba->hw_resp_size) {
1039                         cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1040                         cmdsp->resp = resp_dma;
1041                         cmdsp->resp_len = req->sba->hw_resp_size;
1042                 }
1043                 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1044                 cmdsp->data = *dst_q + msg_offset;
1045                 cmdsp->data_len = msg_len;
1046                 cmdsp++;
1047         }
1048
1049         /* Fillup brcm_message */
1050         msg->type = BRCM_MESSAGE_SBA;
1051         msg->sba.cmds = cmds;
1052         msg->sba.cmds_count = cmdsp - cmds;
1053         msg->ctx = req;
1054         msg->error = 0;
1055 }
1056
1057 static struct sba_request *
1058 sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off,
1059                     dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t *src,
1060                     u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
1061 {
1062         struct sba_request *req = NULL;
1063
1064         /* Alloc new request */
1065         req = sba_alloc_request(sba);
1066         if (!req)
1067                 return NULL;
1068         if (flags & DMA_PREP_FENCE)
1069                 req->flags |= SBA_REQUEST_FENCE;
1070
1071         /* Fillup request messages */
1072         sba_fillup_pq_msg(req, dmaf_continue(flags),
1073                           req->cmds, &req->msg,
1074                           off, len, dst_p, dst_q, scf, src, src_cnt);
1075
1076         /* Init async_tx descriptor */
1077         req->tx.flags = flags;
1078         req->tx.cookie = -EBUSY;
1079
1080         return req;
1081 }
1082
1083 static void sba_fillup_pq_single_msg(struct sba_request *req,
1084                                 bool pq_continue,
1085                                 struct brcm_sba_command *cmds,
1086                                 struct brcm_message *msg,
1087                                 dma_addr_t msg_offset, size_t msg_len,
1088                                 dma_addr_t *dst_p, dma_addr_t *dst_q,
1089                                 dma_addr_t src, u8 scf)
1090 {
1091         u64 cmd;
1092         u32 c_mdata;
1093         u8 pos, dpos = raid6_gflog[scf];
1094         dma_addr_t resp_dma = req->tx.phys;
1095         struct brcm_sba_command *cmdsp = cmds;
1096
1097         if (!dst_p)
1098                 goto skip_p;
1099
1100         if (pq_continue) {
1101                 /* Type-B command to load old P into buf0 */
1102                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1103                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1104                 cmd = sba_cmd_enc(cmd, msg_len,
1105                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1106                 c_mdata = sba_cmd_load_c_mdata(0);
1107                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1108                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1109                 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
1110                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1111                 cmdsp->cmd = cmd;
1112                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1113                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1114                 cmdsp->data = *dst_p + msg_offset;
1115                 cmdsp->data_len = msg_len;
1116                 cmdsp++;
1117
1118                 /*
1119                  * Type-B commands to xor data with buf0 and put it
1120                  * back in buf0
1121                  */
1122                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1123                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1124                 cmd = sba_cmd_enc(cmd, msg_len,
1125                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1126                 c_mdata = sba_cmd_xor_c_mdata(0, 0);
1127                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1128                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1129                 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
1130                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1131                 cmdsp->cmd = cmd;
1132                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1133                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1134                 cmdsp->data = src + msg_offset;
1135                 cmdsp->data_len = msg_len;
1136                 cmdsp++;
1137         } else {
1138                 /* Type-B command to load old P into buf0 */
1139                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1140                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1141                 cmd = sba_cmd_enc(cmd, msg_len,
1142                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1143                 c_mdata = sba_cmd_load_c_mdata(0);
1144                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1145                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1146                 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
1147                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1148                 cmdsp->cmd = cmd;
1149                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1150                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1151                 cmdsp->data = src + msg_offset;
1152                 cmdsp->data_len = msg_len;
1153                 cmdsp++;
1154         }
1155
1156         /* Type-A command to write buf0 */
1157         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1158                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1159         cmd = sba_cmd_enc(cmd, msg_len,
1160                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1161         cmd = sba_cmd_enc(cmd, 0x1,
1162                           SBA_RESP_SHIFT, SBA_RESP_MASK);
1163         c_mdata = sba_cmd_write_c_mdata(0);
1164         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1165                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1166         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1167                           SBA_CMD_SHIFT, SBA_CMD_MASK);
1168         cmdsp->cmd = cmd;
1169         *cmdsp->cmd_dma = cpu_to_le64(cmd);
1170         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1171         if (req->sba->hw_resp_size) {
1172                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1173                 cmdsp->resp = resp_dma;
1174                 cmdsp->resp_len = req->sba->hw_resp_size;
1175         }
1176         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1177         cmdsp->data = *dst_p + msg_offset;
1178         cmdsp->data_len = msg_len;
1179         cmdsp++;
1180
1181 skip_p:
1182         if (!dst_q)
1183                 goto skip_q;
1184
1185         /* Type-A command to zero all buffers */
1186         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1187                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1188         cmd = sba_cmd_enc(cmd, msg_len,
1189                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1190         cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS,
1191                           SBA_CMD_SHIFT, SBA_CMD_MASK);
1192         cmdsp->cmd = cmd;
1193         *cmdsp->cmd_dma = cpu_to_le64(cmd);
1194         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1195         cmdsp++;
1196
1197         if (dpos == 255)
1198                 goto skip_q_computation;
1199         pos = (dpos < req->sba->max_pq_coefs) ?
1200                 dpos : (req->sba->max_pq_coefs - 1);
1201
1202         /*
1203          * Type-B command to generate initial Q from data
1204          * and store output into buf0
1205          */
1206         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1207                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1208         cmd = sba_cmd_enc(cmd, msg_len,
1209                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1210         c_mdata = sba_cmd_pq_c_mdata(pos, 0, 0);
1211         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1212                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1213         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
1214                           SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
1215         cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS,
1216                           SBA_CMD_SHIFT, SBA_CMD_MASK);
1217         cmdsp->cmd = cmd;
1218         *cmdsp->cmd_dma = cpu_to_le64(cmd);
1219         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1220         cmdsp->data = src + msg_offset;
1221         cmdsp->data_len = msg_len;
1222         cmdsp++;
1223
1224         dpos -= pos;
1225
1226         /* Multiple Type-A command to generate final Q */
1227         while (dpos) {
1228                 pos = (dpos < req->sba->max_pq_coefs) ?
1229                         dpos : (req->sba->max_pq_coefs - 1);
1230
1231                 /*
1232                  * Type-A command to generate Q with buf0 and
1233                  * buf1 store result in buf0
1234                  */
1235                 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1236                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1237                 cmd = sba_cmd_enc(cmd, msg_len,
1238                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1239                 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 1);
1240                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1241                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1242                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
1243                                   SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
1244                 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS,
1245                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1246                 cmdsp->cmd = cmd;
1247                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1248                 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1249                 cmdsp++;
1250
1251                 dpos -= pos;
1252         }
1253
1254 skip_q_computation:
1255         if (pq_continue) {
1256                 /*
1257                  * Type-B command to XOR previous output with
1258                  * buf0 and write it into buf0
1259                  */
1260                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1261                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1262                 cmd = sba_cmd_enc(cmd, msg_len,
1263                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1264                 c_mdata = sba_cmd_xor_c_mdata(0, 0);
1265                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1266                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1267                 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
1268                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1269                 cmdsp->cmd = cmd;
1270                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1271                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1272                 cmdsp->data = *dst_q + msg_offset;
1273                 cmdsp->data_len = msg_len;
1274                 cmdsp++;
1275         }
1276
1277         /* Type-A command to write buf0 */
1278         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1279                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1280         cmd = sba_cmd_enc(cmd, msg_len,
1281                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1282         cmd = sba_cmd_enc(cmd, 0x1,
1283                           SBA_RESP_SHIFT, SBA_RESP_MASK);
1284         c_mdata = sba_cmd_write_c_mdata(0);
1285         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1286                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1287         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1288                           SBA_CMD_SHIFT, SBA_CMD_MASK);
1289         cmdsp->cmd = cmd;
1290         *cmdsp->cmd_dma = cpu_to_le64(cmd);
1291         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1292         if (req->sba->hw_resp_size) {
1293                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1294                 cmdsp->resp = resp_dma;
1295                 cmdsp->resp_len = req->sba->hw_resp_size;
1296         }
1297         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1298         cmdsp->data = *dst_q + msg_offset;
1299         cmdsp->data_len = msg_len;
1300         cmdsp++;
1301
1302 skip_q:
1303         /* Fillup brcm_message */
1304         msg->type = BRCM_MESSAGE_SBA;
1305         msg->sba.cmds = cmds;
1306         msg->sba.cmds_count = cmdsp - cmds;
1307         msg->ctx = req;
1308         msg->error = 0;
1309 }
1310
1311 static struct sba_request *
1312 sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off,
1313                            dma_addr_t *dst_p, dma_addr_t *dst_q,
1314                            dma_addr_t src, u8 scf, size_t len,
1315                            unsigned long flags)
1316 {
1317         struct sba_request *req = NULL;
1318
1319         /* Alloc new request */
1320         req = sba_alloc_request(sba);
1321         if (!req)
1322                 return NULL;
1323         if (flags & DMA_PREP_FENCE)
1324                 req->flags |= SBA_REQUEST_FENCE;
1325
1326         /* Fillup request messages */
1327         sba_fillup_pq_single_msg(req,  dmaf_continue(flags),
1328                                  req->cmds, &req->msg, off, len,
1329                                  dst_p, dst_q, src, scf);
1330
1331         /* Init async_tx descriptor */
1332         req->tx.flags = flags;
1333         req->tx.cookie = -EBUSY;
1334
1335         return req;
1336 }
1337
1338 static struct dma_async_tx_descriptor *
1339 sba_prep_dma_pq(struct dma_chan *dchan, dma_addr_t *dst, dma_addr_t *src,
1340                 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
1341 {
1342         u32 i, dst_q_index;
1343         size_t req_len;
1344         bool slow = false;
1345         dma_addr_t off = 0;
1346         dma_addr_t *dst_p = NULL, *dst_q = NULL;
1347         struct sba_device *sba = to_sba_device(dchan);
1348         struct sba_request *first = NULL, *req;
1349
1350         /* Sanity checks */
1351         if (unlikely(src_cnt > sba->max_pq_srcs))
1352                 return NULL;
1353         for (i = 0; i < src_cnt; i++)
1354                 if (sba->max_pq_coefs <= raid6_gflog[scf[i]])
1355                         slow = true;
1356
1357         /* Figure-out P and Q destination addresses */
1358         if (!(flags & DMA_PREP_PQ_DISABLE_P))
1359                 dst_p = &dst[0];
1360         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
1361                 dst_q = &dst[1];
1362
1363         /* Create chained requests where each request is upto hw_buf_size */
1364         while (len) {
1365                 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
1366
1367                 if (slow) {
1368                         dst_q_index = src_cnt;
1369
1370                         if (dst_q) {
1371                                 for (i = 0; i < src_cnt; i++) {
1372                                         if (*dst_q == src[i]) {
1373                                                 dst_q_index = i;
1374                                                 break;
1375                                         }
1376                                 }
1377                         }
1378
1379                         if (dst_q_index < src_cnt) {
1380                                 i = dst_q_index;
1381                                 req = sba_prep_dma_pq_single_req(sba,
1382                                         off, dst_p, dst_q, src[i], scf[i],
1383                                         req_len, flags | DMA_PREP_FENCE);
1384                                 if (!req)
1385                                         goto fail;
1386
1387                                 if (first)
1388                                         sba_chain_request(first, req);
1389                                 else
1390                                         first = req;
1391
1392                                 flags |= DMA_PREP_CONTINUE;
1393                         }
1394
1395                         for (i = 0; i < src_cnt; i++) {
1396                                 if (dst_q_index == i)
1397                                         continue;
1398
1399                                 req = sba_prep_dma_pq_single_req(sba,
1400                                         off, dst_p, dst_q, src[i], scf[i],
1401                                         req_len, flags | DMA_PREP_FENCE);
1402                                 if (!req)
1403                                         goto fail;
1404
1405                                 if (first)
1406                                         sba_chain_request(first, req);
1407                                 else
1408                                         first = req;
1409
1410                                 flags |= DMA_PREP_CONTINUE;
1411                         }
1412                 } else {
1413                         req = sba_prep_dma_pq_req(sba, off,
1414                                                   dst_p, dst_q, src, src_cnt,
1415                                                   scf, req_len, flags);
1416                         if (!req)
1417                                 goto fail;
1418
1419                         if (first)
1420                                 sba_chain_request(first, req);
1421                         else
1422                                 first = req;
1423                 }
1424
1425                 off += req_len;
1426                 len -= req_len;
1427         }
1428
1429         return (first) ? &first->tx : NULL;
1430
1431 fail:
1432         if (first)
1433                 sba_free_chained_requests(first);
1434         return NULL;
1435 }
1436
1437 /* ====== Mailbox callbacks ===== */
1438
1439 static void sba_receive_message(struct mbox_client *cl, void *msg)
1440 {
1441         struct brcm_message *m = msg;
1442         struct sba_request *req = m->ctx;
1443         struct sba_device *sba = req->sba;
1444
1445         /* Error count if message has error */
1446         if (m->error < 0)
1447                 dev_err(sba->dev, "%s got message with error %d",
1448                         dma_chan_name(&sba->dma_chan), m->error);
1449
1450         /* Process received request */
1451         sba_process_received_request(sba, req);
1452 }
1453
1454 /* ====== Platform driver routines ===== */
1455
1456 static int sba_prealloc_channel_resources(struct sba_device *sba)
1457 {
1458         int i, j, ret = 0;
1459         struct sba_request *req = NULL;
1460
1461         sba->resp_base = dma_alloc_coherent(sba->mbox_dev,
1462                                             sba->max_resp_pool_size,
1463                                             &sba->resp_dma_base, GFP_KERNEL);
1464         if (!sba->resp_base)
1465                 return -ENOMEM;
1466
1467         sba->cmds_base = dma_alloc_coherent(sba->mbox_dev,
1468                                             sba->max_cmds_pool_size,
1469                                             &sba->cmds_dma_base, GFP_KERNEL);
1470         if (!sba->cmds_base) {
1471                 ret = -ENOMEM;
1472                 goto fail_free_resp_pool;
1473         }
1474
1475         spin_lock_init(&sba->reqs_lock);
1476         sba->reqs_fence = false;
1477         INIT_LIST_HEAD(&sba->reqs_alloc_list);
1478         INIT_LIST_HEAD(&sba->reqs_pending_list);
1479         INIT_LIST_HEAD(&sba->reqs_active_list);
1480         INIT_LIST_HEAD(&sba->reqs_completed_list);
1481         INIT_LIST_HEAD(&sba->reqs_aborted_list);
1482         INIT_LIST_HEAD(&sba->reqs_free_list);
1483
1484         for (i = 0; i < sba->max_req; i++) {
1485                 req = devm_kzalloc(sba->dev,
1486                                 sizeof(*req) +
1487                                 sba->max_cmd_per_req * sizeof(req->cmds[0]),
1488                                 GFP_KERNEL);
1489                 if (!req) {
1490                         ret = -ENOMEM;
1491                         goto fail_free_cmds_pool;
1492                 }
1493                 INIT_LIST_HEAD(&req->node);
1494                 req->sba = sba;
1495                 req->flags = SBA_REQUEST_STATE_FREE;
1496                 INIT_LIST_HEAD(&req->next);
1497                 atomic_set(&req->next_pending_count, 0);
1498                 for (j = 0; j < sba->max_cmd_per_req; j++) {
1499                         req->cmds[j].cmd = 0;
1500                         req->cmds[j].cmd_dma = sba->cmds_base +
1501                                 (i * sba->max_cmd_per_req + j) * sizeof(u64);
1502                         req->cmds[j].cmd_dma_addr = sba->cmds_dma_base +
1503                                 (i * sba->max_cmd_per_req + j) * sizeof(u64);
1504                         req->cmds[j].flags = 0;
1505                 }
1506                 memset(&req->msg, 0, sizeof(req->msg));
1507                 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
1508                 req->tx.tx_submit = sba_tx_submit;
1509                 req->tx.phys = sba->resp_dma_base + i * sba->hw_resp_size;
1510                 list_add_tail(&req->node, &sba->reqs_free_list);
1511         }
1512
1513         return 0;
1514
1515 fail_free_cmds_pool:
1516         dma_free_coherent(sba->mbox_dev,
1517                           sba->max_cmds_pool_size,
1518                           sba->cmds_base, sba->cmds_dma_base);
1519 fail_free_resp_pool:
1520         dma_free_coherent(sba->mbox_dev,
1521                           sba->max_resp_pool_size,
1522                           sba->resp_base, sba->resp_dma_base);
1523         return ret;
1524 }
1525
1526 static void sba_freeup_channel_resources(struct sba_device *sba)
1527 {
1528         dmaengine_terminate_all(&sba->dma_chan);
1529         dma_free_coherent(sba->mbox_dev, sba->max_cmds_pool_size,
1530                           sba->cmds_base, sba->cmds_dma_base);
1531         dma_free_coherent(sba->mbox_dev, sba->max_resp_pool_size,
1532                           sba->resp_base, sba->resp_dma_base);
1533         sba->resp_base = NULL;
1534         sba->resp_dma_base = 0;
1535 }
1536
1537 static int sba_async_register(struct sba_device *sba)
1538 {
1539         int ret;
1540         struct dma_device *dma_dev = &sba->dma_dev;
1541
1542         /* Initialize DMA channel cookie */
1543         sba->dma_chan.device = dma_dev;
1544         dma_cookie_init(&sba->dma_chan);
1545
1546         /* Initialize DMA device capability mask */
1547         dma_cap_zero(dma_dev->cap_mask);
1548         dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
1549         dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1550         dma_cap_set(DMA_XOR, dma_dev->cap_mask);
1551         dma_cap_set(DMA_PQ, dma_dev->cap_mask);
1552
1553         /*
1554          * Set mailbox channel device as the base device of
1555          * our dma_device because the actual memory accesses
1556          * will be done by mailbox controller
1557          */
1558         dma_dev->dev = sba->mbox_dev;
1559
1560         /* Set base prep routines */
1561         dma_dev->device_free_chan_resources = sba_free_chan_resources;
1562         dma_dev->device_terminate_all = sba_device_terminate_all;
1563         dma_dev->device_issue_pending = sba_issue_pending;
1564         dma_dev->device_tx_status = sba_tx_status;
1565
1566         /* Set interrupt routine */
1567         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1568                 dma_dev->device_prep_dma_interrupt = sba_prep_dma_interrupt;
1569
1570         /* Set memcpy routine */
1571         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1572                 dma_dev->device_prep_dma_memcpy = sba_prep_dma_memcpy;
1573
1574         /* Set xor routine and capability */
1575         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1576                 dma_dev->device_prep_dma_xor = sba_prep_dma_xor;
1577                 dma_dev->max_xor = sba->max_xor_srcs;
1578         }
1579
1580         /* Set pq routine and capability */
1581         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1582                 dma_dev->device_prep_dma_pq = sba_prep_dma_pq;
1583                 dma_set_maxpq(dma_dev, sba->max_pq_srcs, 0);
1584         }
1585
1586         /* Initialize DMA device channel list */
1587         INIT_LIST_HEAD(&dma_dev->channels);
1588         list_add_tail(&sba->dma_chan.device_node, &dma_dev->channels);
1589
1590         /* Register with Linux async DMA framework*/
1591         ret = dma_async_device_register(dma_dev);
1592         if (ret) {
1593                 dev_err(sba->dev, "async device register error %d", ret);
1594                 return ret;
1595         }
1596
1597         dev_info(sba->dev, "%s capabilities: %s%s%s%s\n",
1598         dma_chan_name(&sba->dma_chan),
1599         dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "interrupt " : "",
1600         dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "memcpy " : "",
1601         dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1602         dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "");
1603
1604         return 0;
1605 }
1606
1607 static int sba_probe(struct platform_device *pdev)
1608 {
1609         int i, ret = 0, mchans_count;
1610         struct sba_device *sba;
1611         struct platform_device *mbox_pdev;
1612         struct of_phandle_args args;
1613
1614         /* Allocate main SBA struct */
1615         sba = devm_kzalloc(&pdev->dev, sizeof(*sba), GFP_KERNEL);
1616         if (!sba)
1617                 return -ENOMEM;
1618
1619         sba->dev = &pdev->dev;
1620         platform_set_drvdata(pdev, sba);
1621
1622         /* Number of channels equals number of mailbox channels */
1623         ret = of_count_phandle_with_args(pdev->dev.of_node,
1624                                          "mboxes", "#mbox-cells");
1625         if (ret <= 0)
1626                 return -ENODEV;
1627         mchans_count = ret;
1628
1629         /* Determine SBA version from DT compatible string */
1630         if (of_device_is_compatible(sba->dev->of_node, "brcm,iproc-sba"))
1631                 sba->ver = SBA_VER_1;
1632         else if (of_device_is_compatible(sba->dev->of_node,
1633                                          "brcm,iproc-sba-v2"))
1634                 sba->ver = SBA_VER_2;
1635         else
1636                 return -ENODEV;
1637
1638         /* Derived Configuration parameters */
1639         switch (sba->ver) {
1640         case SBA_VER_1:
1641                 sba->hw_buf_size = 4096;
1642                 sba->hw_resp_size = 8;
1643                 sba->max_pq_coefs = 6;
1644                 sba->max_pq_srcs = 6;
1645                 break;
1646         case SBA_VER_2:
1647                 sba->hw_buf_size = 4096;
1648                 sba->hw_resp_size = 8;
1649                 sba->max_pq_coefs = 30;
1650                 /*
1651                  * We can support max_pq_srcs == max_pq_coefs because
1652                  * we are limited by number of SBA commands that we can
1653                  * fit in one message for underlying ring manager HW.
1654                  */
1655                 sba->max_pq_srcs = 12;
1656                 break;
1657         default:
1658                 return -EINVAL;
1659         }
1660         sba->max_req = SBA_MAX_REQ_PER_MBOX_CHANNEL * mchans_count;
1661         sba->max_cmd_per_req = sba->max_pq_srcs + 3;
1662         sba->max_xor_srcs = sba->max_cmd_per_req - 1;
1663         sba->max_resp_pool_size = sba->max_req * sba->hw_resp_size;
1664         sba->max_cmds_pool_size = sba->max_req *
1665                                   sba->max_cmd_per_req * sizeof(u64);
1666
1667         /* Setup mailbox client */
1668         sba->client.dev                 = &pdev->dev;
1669         sba->client.rx_callback         = sba_receive_message;
1670         sba->client.tx_block            = false;
1671         sba->client.knows_txdone        = false;
1672         sba->client.tx_tout             = 0;
1673
1674         /* Allocate mailbox channel array */
1675         sba->mchans = devm_kcalloc(&pdev->dev, mchans_count,
1676                                    sizeof(*sba->mchans), GFP_KERNEL);
1677         if (!sba->mchans)
1678                 return -ENOMEM;
1679
1680         /* Request mailbox channels */
1681         sba->mchans_count = 0;
1682         for (i = 0; i < mchans_count; i++) {
1683                 sba->mchans[i] = mbox_request_channel(&sba->client, i);
1684                 if (IS_ERR(sba->mchans[i])) {
1685                         ret = PTR_ERR(sba->mchans[i]);
1686                         goto fail_free_mchans;
1687                 }
1688                 sba->mchans_count++;
1689         }
1690         atomic_set(&sba->mchans_current, 0);
1691
1692         /* Find-out underlying mailbox device */
1693         ret = of_parse_phandle_with_args(pdev->dev.of_node,
1694                                          "mboxes", "#mbox-cells", 0, &args);
1695         if (ret)
1696                 goto fail_free_mchans;
1697         mbox_pdev = of_find_device_by_node(args.np);
1698         of_node_put(args.np);
1699         if (!mbox_pdev) {
1700                 ret = -ENODEV;
1701                 goto fail_free_mchans;
1702         }
1703         sba->mbox_dev = &mbox_pdev->dev;
1704
1705         /* All mailbox channels should be of same ring manager device */
1706         for (i = 1; i < mchans_count; i++) {
1707                 ret = of_parse_phandle_with_args(pdev->dev.of_node,
1708                                          "mboxes", "#mbox-cells", i, &args);
1709                 if (ret)
1710                         goto fail_free_mchans;
1711                 mbox_pdev = of_find_device_by_node(args.np);
1712                 of_node_put(args.np);
1713                 if (sba->mbox_dev != &mbox_pdev->dev) {
1714                         ret = -EINVAL;
1715                         goto fail_free_mchans;
1716                 }
1717         }
1718
1719         /* Prealloc channel resource */
1720         ret = sba_prealloc_channel_resources(sba);
1721         if (ret)
1722                 goto fail_free_mchans;
1723
1724         /* Register DMA device with Linux async framework */
1725         ret = sba_async_register(sba);
1726         if (ret)
1727                 goto fail_free_resources;
1728
1729         /* Print device info */
1730         dev_info(sba->dev, "%s using SBAv%d and %d mailbox channels",
1731                  dma_chan_name(&sba->dma_chan), sba->ver+1,
1732                  sba->mchans_count);
1733
1734         return 0;
1735
1736 fail_free_resources:
1737         sba_freeup_channel_resources(sba);
1738 fail_free_mchans:
1739         for (i = 0; i < sba->mchans_count; i++)
1740                 mbox_free_channel(sba->mchans[i]);
1741         return ret;
1742 }
1743
1744 static int sba_remove(struct platform_device *pdev)
1745 {
1746         int i;
1747         struct sba_device *sba = platform_get_drvdata(pdev);
1748
1749         dma_async_device_unregister(&sba->dma_dev);
1750
1751         sba_freeup_channel_resources(sba);
1752
1753         for (i = 0; i < sba->mchans_count; i++)
1754                 mbox_free_channel(sba->mchans[i]);
1755
1756         return 0;
1757 }
1758
1759 static const struct of_device_id sba_of_match[] = {
1760         { .compatible = "brcm,iproc-sba", },
1761         { .compatible = "brcm,iproc-sba-v2", },
1762         {},
1763 };
1764 MODULE_DEVICE_TABLE(of, sba_of_match);
1765
1766 static struct platform_driver sba_driver = {
1767         .probe = sba_probe,
1768         .remove = sba_remove,
1769         .driver = {
1770                 .name = "bcm-sba-raid",
1771                 .of_match_table = sba_of_match,
1772         },
1773 };
1774 module_platform_driver(sba_driver);
1775
1776 MODULE_DESCRIPTION("Broadcom SBA RAID driver");
1777 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
1778 MODULE_LICENSE("GPL v2");