drivers/dma/ppc4xx/adma.c

   1 /*
   2  * Copyright (C) 2006-2009 DENX Software Engineering.
   3  *
   4  * Author: Yuri Tikhonov <yur@emcraft.com>
   5  *
   6  * Further porting to arch/powerpc by
   7  *      Anatolij Gustschin <agust@denx.de>
   8  *
   9  * This program is free software; you can redistribute it and/or modify it
  10  * under the terms of the GNU General Public License as published by the Free
  11  * Software Foundation; either version 2 of the License, or (at your option)
  12  * any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful, but WITHOUT
  15  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  16  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  17  * more details.
  18  *
  19  * You should have received a copy of the GNU General Public License along with
  20  * this program; if not, write to the Free Software Foundation, Inc., 59
  21  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  22  *
  23  * The full GNU General Public License is included in this distribution in the
  24  * file called COPYING.
  25  */
  26
  27 /*
  28  * This driver supports the asynchrounous DMA copy and RAID engines available
  29  * on the AMCC PPC440SPe Processors.
  30  * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
  31  * ADMA driver written by D.Williams.
  32  */
  33
  34 #include <linux/init.h>
  35 #include <linux/module.h>
  36 #include <linux/async_tx.h>
  37 #include <linux/delay.h>
  38 #include <linux/dma-mapping.h>
  39 #include <linux/spinlock.h>
  40 #include <linux/interrupt.h>
  41 #include <linux/slab.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/proc_fs.h>
  44 #include <linux/of.h>
  45 #include <linux/of_address.h>
  46 #include <linux/of_irq.h>
  47 #include <linux/of_platform.h>
  48 #include <asm/dcr.h>
  49 #include <asm/dcr-regs.h>
  50 #include "adma.h"
  51 #include "../dmaengine.h"
  52
  53 enum ppc_adma_init_code {
  54         PPC_ADMA_INIT_OK = 0,
  55         PPC_ADMA_INIT_MEMRES,
  56         PPC_ADMA_INIT_MEMREG,
  57         PPC_ADMA_INIT_ALLOC,
  58         PPC_ADMA_INIT_COHERENT,
  59         PPC_ADMA_INIT_CHANNEL,
  60         PPC_ADMA_INIT_IRQ1,
  61         PPC_ADMA_INIT_IRQ2,
  62         PPC_ADMA_INIT_REGISTER
  63 };
  64
  65 static char *ppc_adma_errors[] = {
  66         [PPC_ADMA_INIT_OK] = "ok",
  67         [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
  68         [PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
  69         [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
  70                                 "structure",
  71         [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
  72                                    "hardware descriptors",
  73         [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
  74         [PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
  75         [PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
  76         [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
  77 };
  78
  79 static enum ppc_adma_init_code
  80 ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
  81
  82 struct ppc_dma_chan_ref {
  83         struct dma_chan *chan;
  84         struct list_head node;
  85 };
  86
  87 /* The list of channels exported by ppc440spe ADMA */
  88 struct list_head
  89 ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
  90
  91 /* This flag is set when want to refetch the xor chain in the interrupt
  92  * handler
  93  */
  94 static u32 do_xor_refetch;
  95
  96 /* Pointer to DMA0, DMA1 CP/CS FIFO */
  97 static void *ppc440spe_dma_fifo_buf;
  98
  99 /* Pointers to last submitted to DMA0, DMA1 CDBs */
 100 static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
 101 static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
 102
 103 /* Pointer to last linked and submitted xor CB */
 104 static struct ppc440spe_adma_desc_slot *xor_last_linked;
 105 static struct ppc440spe_adma_desc_slot *xor_last_submit;
 106
 107 /* This array is used in data-check operations for storing a pattern */
 108 static char ppc440spe_qword[16];
 109
 110 static atomic_t ppc440spe_adma_err_irq_ref;
 111 static dcr_host_t ppc440spe_mq_dcr_host;
 112 static unsigned int ppc440spe_mq_dcr_len;
 113
 114 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
 115  * the block size in transactions, then we do not allow to activate more than
 116  * only one RXOR transactions simultaneously. So use this var to store
 117  * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
 118  * set) or not (PPC440SPE_RXOR_RUN is clear).
 119  */
 120 static unsigned long ppc440spe_rxor_state;
 121
 122 /* These are used in enable & check routines
 123  */
 124 static u32 ppc440spe_r6_enabled;
 125 static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
 126 static struct completion ppc440spe_r6_test_comp;
 127
 128 static int ppc440spe_adma_dma2rxor_prep_src(
 129                 struct ppc440spe_adma_desc_slot *desc,
 130                 struct ppc440spe_rxor *cursor, int index,
 131                 int src_cnt, u32 addr);
 132 static void ppc440spe_adma_dma2rxor_set_src(
 133                 struct ppc440spe_adma_desc_slot *desc,
 134                 int index, dma_addr_t addr);
 135 static void ppc440spe_adma_dma2rxor_set_mult(
 136                 struct ppc440spe_adma_desc_slot *desc,
 137                 int index, u8 mult);
 138
 139 #ifdef ADMA_LL_DEBUG
 140 #define ADMA_LL_DBG(x) ({ if (1) x; 0; })
 141 #else
 142 #define ADMA_LL_DBG(x) ({ if (0) x; 0; })
 143 #endif
 144
 145 static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
 146 {
 147         struct dma_cdb *cdb;
 148         struct xor_cb *cb;
 149         int i;
 150
 151         switch (chan->device->id) {
 152         case 0:
 153         case 1:
 154                 cdb = block;
 155
 156                 pr_debug("CDB at %p [%d]:\n"
 157                         "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
 158                         "\t sg1u 0x%08x sg1l 0x%08x\n"
 159                         "\t sg2u 0x%08x sg2l 0x%08x\n"
 160                         "\t sg3u 0x%08x sg3l 0x%08x\n",
 161                         cdb, chan->device->id,
 162                         cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
 163                         le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
 164                         le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
 165                         le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
 166                 );
 167                 break;
 168         case 2:
 169                 cb = block;
 170
 171                 pr_debug("CB at %p [%d]:\n"
 172                         "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
 173                         "\t cbtah 0x%08x cbtal 0x%08x\n"
 174                         "\t cblah 0x%08x cblal 0x%08x\n",
 175                         cb, chan->device->id,
 176                         cb->cbc, cb->cbbc, cb->cbs,
 177                         cb->cbtah, cb->cbtal,
 178                         cb->cblah, cb->cblal);
 179                 for (i = 0; i < 16; i++) {
 180                         if (i && !cb->ops[i].h && !cb->ops[i].l)
 181                                 continue;
 182                         pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
 183                                 i, cb->ops[i].h, cb->ops[i].l);
 184                 }
 185                 break;
 186         }
 187 }
 188
 189 static void print_cb_list(struct ppc440spe_adma_chan *chan,
 190                           struct ppc440spe_adma_desc_slot *iter)
 191 {
 192         for (; iter; iter = iter->hw_next)
 193                 print_cb(chan, iter->hw_desc);
 194 }
 195
 196 static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
 197                              unsigned int src_cnt)
 198 {
 199         int i;
 200
 201         pr_debug("\n%s(%d):\nsrc: ", __func__, id);
 202         for (i = 0; i < src_cnt; i++)
 203                 pr_debug("\t0x%016llx ", src[i]);
 204         pr_debug("dst:\n\t0x%016llx\n", dst);
 205 }
 206
 207 static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
 208                             unsigned int src_cnt)
 209 {
 210         int i;
 211
 212         pr_debug("\n%s(%d):\nsrc: ", __func__, id);
 213         for (i = 0; i < src_cnt; i++)
 214                 pr_debug("\t0x%016llx ", src[i]);
 215         pr_debug("dst: ");
 216         for (i = 0; i < 2; i++)
 217                 pr_debug("\t0x%016llx ", dst[i]);
 218 }
 219
 220 static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
 221                                     unsigned int src_cnt,
 222                                     const unsigned char *scf)
 223 {
 224         int i;
 225
 226         pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
 227         if (scf) {
 228                 for (i = 0; i < src_cnt; i++)
 229                         pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
 230         } else {
 231                 for (i = 0; i < src_cnt; i++)
 232                         pr_debug("\t0x%016llx(no) ", src[i]);
 233         }
 234
 235         pr_debug("dst: ");
 236         for (i = 0; i < 2; i++)
 237                 pr_debug("\t0x%016llx ", src[src_cnt + i]);
 238 }
 239
 240 /******************************************************************************
 241  * Command (Descriptor) Blocks low-level routines
 242  ******************************************************************************/
 243 /**
 244  * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
 245  * pseudo operation
 246  */
 247 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
 248                                           struct ppc440spe_adma_chan *chan)
 249 {
 250         struct xor_cb *p;
 251
 252         switch (chan->device->id) {
 253         case PPC440SPE_XOR_ID:
 254                 p = desc->hw_desc;
 255                 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 256                 /* NOP with Command Block Complete Enable */
 257                 p->cbc = XOR_CBCR_CBCE_BIT;
 258                 break;
 259         case PPC440SPE_DMA0_ID:
 260         case PPC440SPE_DMA1_ID:
 261                 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
 262                 /* NOP with interrupt */
 263                 set_bit(PPC440SPE_DESC_INT, &desc->flags);
 264                 break;
 265         default:
 266                 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
 267                                 __func__);
 268                 break;
 269         }
 270 }
 271
 272 /**
 273  * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
 274  * pseudo operation
 275  */
 276 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
 277 {
 278         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 279         desc->hw_next = NULL;
 280         desc->src_cnt = 0;
 281         desc->dst_cnt = 1;
 282 }
 283
 284 /**
 285  * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
 286  */
 287 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
 288                                          int src_cnt, unsigned long flags)
 289 {
 290         struct xor_cb *hw_desc = desc->hw_desc;
 291
 292         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 293         desc->hw_next = NULL;
 294         desc->src_cnt = src_cnt;
 295         desc->dst_cnt = 1;
 296
 297         hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
 298         if (flags & DMA_PREP_INTERRUPT)
 299                 /* Enable interrupt on completion */
 300                 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
 301 }
 302
 303 /**
 304  * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
 305  * operation in DMA2 controller
 306  */
 307 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
 308                 int dst_cnt, int src_cnt, unsigned long flags)
 309 {
 310         struct xor_cb *hw_desc = desc->hw_desc;
 311
 312         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 313         desc->hw_next = NULL;
 314         desc->src_cnt = src_cnt;
 315         desc->dst_cnt = dst_cnt;
 316         memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
 317         desc->descs_per_op = 0;
 318
 319         hw_desc->cbc = XOR_CBCR_TGT_BIT;
 320         if (flags & DMA_PREP_INTERRUPT)
 321                 /* Enable interrupt on completion */
 322                 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
 323 }
 324
 325 #define DMA_CTRL_FLAGS_LAST     DMA_PREP_FENCE
 326 #define DMA_PREP_ZERO_P         (DMA_CTRL_FLAGS_LAST << 1)
 327 #define DMA_PREP_ZERO_Q         (DMA_PREP_ZERO_P << 1)
 328
 329 /**
 330  * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
 331  * with DMA0/1
 332  */
 333 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
 334                                 int dst_cnt, int src_cnt, unsigned long flags,
 335                                 unsigned long op)
 336 {
 337         struct dma_cdb *hw_desc;
 338         struct ppc440spe_adma_desc_slot *iter;
 339         u8 dopc;
 340
 341         /* Common initialization of a PQ descriptors chain */
 342         set_bits(op, &desc->flags);
 343         desc->src_cnt = src_cnt;
 344         desc->dst_cnt = dst_cnt;
 345
 346         /* WXOR MULTICAST if both P and Q are being computed
 347          * MV_SG1_SG2 if Q only
 348          */
 349         dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
 350                 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
 351
 352         list_for_each_entry(iter, &desc->group_list, chain_node) {
 353                 hw_desc = iter->hw_desc;
 354                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
 355
 356                 if (likely(!list_is_last(&iter->chain_node,
 357                                 &desc->group_list))) {
 358                         /* set 'next' pointer */
 359                         iter->hw_next = list_entry(iter->chain_node.next,
 360                                 struct ppc440spe_adma_desc_slot, chain_node);
 361                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
 362                 } else {
 363                         /* this is the last descriptor.
 364                          * this slot will be pasted from ADMA level
 365                          * each time it wants to configure parameters
 366                          * of the transaction (src, dst, ...)
 367                          */
 368                         iter->hw_next = NULL;
 369                         if (flags & DMA_PREP_INTERRUPT)
 370                                 set_bit(PPC440SPE_DESC_INT, &iter->flags);
 371                         else
 372                                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
 373                 }
 374         }
 375
 376         /* Set OPS depending on WXOR/RXOR type of operation */
 377         if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
 378                 /* This is a WXOR only chain:
 379                  * - first descriptors are for zeroing destinations
 380                  *   if PPC440SPE_ZERO_P/Q set;
 381                  * - descriptors remained are for GF-XOR operations.
 382                  */
 383                 iter = list_first_entry(&desc->group_list,
 384                                         struct ppc440spe_adma_desc_slot,
 385                                         chain_node);
 386
 387                 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
 388                         hw_desc = iter->hw_desc;
 389                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 390                         iter = list_first_entry(&iter->chain_node,
 391                                         struct ppc440spe_adma_desc_slot,
 392                                         chain_node);
 393                 }
 394
 395                 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
 396                         hw_desc = iter->hw_desc;
 397                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 398                         iter = list_first_entry(&iter->chain_node,
 399                                         struct ppc440spe_adma_desc_slot,
 400                                         chain_node);
 401                 }
 402
 403                 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
 404                         hw_desc = iter->hw_desc;
 405                         hw_desc->opc = dopc;
 406                 }
 407         } else {
 408                 /* This is either RXOR-only or mixed RXOR/WXOR */
 409
 410                 /* The first 1 or 2 slots in chain are always RXOR,
 411                  * if need to calculate P & Q, then there are two
 412                  * RXOR slots; if only P or only Q, then there is one
 413                  */
 414                 iter = list_first_entry(&desc->group_list,
 415                                         struct ppc440spe_adma_desc_slot,
 416                                         chain_node);
 417                 hw_desc = iter->hw_desc;
 418                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 419
 420                 if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
 421                         iter = list_first_entry(&iter->chain_node,
 422                                                 struct ppc440spe_adma_desc_slot,
 423                                                 chain_node);
 424                         hw_desc = iter->hw_desc;
 425                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 426                 }
 427
 428                 /* The remaining descs (if any) are WXORs */
 429                 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
 430                         iter = list_first_entry(&iter->chain_node,
 431                                                 struct ppc440spe_adma_desc_slot,
 432                                                 chain_node);
 433                         list_for_each_entry_from(iter, &desc->group_list,
 434                                                 chain_node) {
 435                                 hw_desc = iter->hw_desc;
 436                                 hw_desc->opc = dopc;
 437                         }
 438                 }
 439         }
 440 }
 441
 442 /**
 443  * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
 444  * for PQ_ZERO_SUM operation
 445  */
 446 static void ppc440spe_desc_init_dma01pqzero_sum(
 447                                 struct ppc440spe_adma_desc_slot *desc,
 448                                 int dst_cnt, int src_cnt)
 449 {
 450         struct dma_cdb *hw_desc;
 451         struct ppc440spe_adma_desc_slot *iter;
 452         int i = 0;
 453         u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
 454                                    DMA_CDB_OPC_MV_SG1_SG2;
 455         /*
 456          * Initialize starting from 2nd or 3rd descriptor dependent
 457          * on dst_cnt. First one or two slots are for cloning P
 458          * and/or Q to chan->pdest and/or chan->qdest as we have
 459          * to preserve original P/Q.
 460          */
 461         iter = list_first_entry(&desc->group_list,
 462                                 struct ppc440spe_adma_desc_slot, chain_node);
 463         iter = list_entry(iter->chain_node.next,
 464                           struct ppc440spe_adma_desc_slot, chain_node);
 465
 466         if (dst_cnt > 1) {
 467                 iter = list_entry(iter->chain_node.next,
 468                                   struct ppc440spe_adma_desc_slot, chain_node);
 469         }
 470         /* initialize each source descriptor in chain */
 471         list_for_each_entry_from(iter, &desc->group_list, chain_node) {
 472                 hw_desc = iter->hw_desc;
 473                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
 474                 iter->src_cnt = 0;
 475                 iter->dst_cnt = 0;
 476
 477                 /* This is a ZERO_SUM operation:
 478                  * - <src_cnt> descriptors starting from 2nd or 3rd
 479                  *   descriptor are for GF-XOR operations;
 480                  * - remaining <dst_cnt> descriptors are for checking the result
 481                  */
 482                 if (i++ < src_cnt)
 483                         /* MV_SG1_SG2 if only Q is being verified
 484                          * MULTICAST if both P and Q are being verified
 485                          */
 486                         hw_desc->opc = dopc;
 487                 else
 488                         /* DMA_CDB_OPC_DCHECK128 operation */
 489                         hw_desc->opc = DMA_CDB_OPC_DCHECK128;
 490
 491                 if (likely(!list_is_last(&iter->chain_node,
 492                                          &desc->group_list))) {
 493                         /* set 'next' pointer */
 494                         iter->hw_next = list_entry(iter->chain_node.next,
 495                                                 struct ppc440spe_adma_desc_slot,
 496                                                 chain_node);
 497                 } else {
 498                         /* this is the last descriptor.
 499                          * this slot will be pasted from ADMA level
 500                          * each time it wants to configure parameters
 501                          * of the transaction (src, dst, ...)
 502                          */
 503                         iter->hw_next = NULL;
 504                         /* always enable interrupt generation since we get
 505                          * the status of pqzero from the handler
 506                          */
 507                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
 508                 }
 509         }
 510         desc->src_cnt = src_cnt;
 511         desc->dst_cnt = dst_cnt;
 512 }
 513
 514 /**
 515  * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
 516  */
 517 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
 518                                         unsigned long flags)
 519 {
 520         struct dma_cdb *hw_desc = desc->hw_desc;
 521
 522         memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
 523         desc->hw_next = NULL;
 524         desc->src_cnt = 1;
 525         desc->dst_cnt = 1;
 526
 527         if (flags & DMA_PREP_INTERRUPT)
 528                 set_bit(PPC440SPE_DESC_INT, &desc->flags);
 529         else
 530                 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
 531
 532         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 533 }
 534
 535 /**
 536  * ppc440spe_desc_init_memset - initialize the descriptor for MEMSET operation
 537  */
 538 static void ppc440spe_desc_init_memset(struct ppc440spe_adma_desc_slot *desc,
 539                                         int value, unsigned long flags)
 540 {
 541         struct dma_cdb *hw_desc = desc->hw_desc;
 542
 543         memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
 544         desc->hw_next = NULL;
 545         desc->src_cnt = 1;
 546         desc->dst_cnt = 1;
 547
 548         if (flags & DMA_PREP_INTERRUPT)
 549                 set_bit(PPC440SPE_DESC_INT, &desc->flags);
 550         else
 551                 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
 552
 553         hw_desc->sg1u = hw_desc->sg1l = cpu_to_le32((u32)value);
 554         hw_desc->sg3u = hw_desc->sg3l = cpu_to_le32((u32)value);
 555         hw_desc->opc = DMA_CDB_OPC_DFILL128;
 556 }
 557
 558 /**
 559  * ppc440spe_desc_set_src_addr - set source address into the descriptor
 560  */
 561 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
 562                                         struct ppc440spe_adma_chan *chan,
 563                                         int src_idx, dma_addr_t addrh,
 564                                         dma_addr_t addrl)
 565 {
 566         struct dma_cdb *dma_hw_desc;
 567         struct xor_cb *xor_hw_desc;
 568         phys_addr_t addr64, tmplow, tmphi;
 569
 570         switch (chan->device->id) {
 571         case PPC440SPE_DMA0_ID:
 572         case PPC440SPE_DMA1_ID:
 573                 if (!addrh) {
 574                         addr64 = addrl;
 575                         tmphi = (addr64 >> 32);
 576                         tmplow = (addr64 & 0xFFFFFFFF);
 577                 } else {
 578                         tmphi = addrh;
 579                         tmplow = addrl;
 580                 }
 581                 dma_hw_desc = desc->hw_desc;
 582                 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
 583                 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
 584                 break;
 585         case PPC440SPE_XOR_ID:
 586                 xor_hw_desc = desc->hw_desc;
 587                 xor_hw_desc->ops[src_idx].l = addrl;
 588                 xor_hw_desc->ops[src_idx].h |= addrh;
 589                 break;
 590         }
 591 }
 592
 593 /**
 594  * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
 595  */
 596 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
 597                         struct ppc440spe_adma_chan *chan, u32 mult_index,
 598                         int sg_index, unsigned char mult_value)
 599 {
 600         struct dma_cdb *dma_hw_desc;
 601         struct xor_cb *xor_hw_desc;
 602         u32 *psgu;
 603
 604         switch (chan->device->id) {
 605         case PPC440SPE_DMA0_ID:
 606         case PPC440SPE_DMA1_ID:
 607                 dma_hw_desc = desc->hw_desc;
 608
 609                 switch (sg_index) {
 610                 /* for RXOR operations set multiplier
 611                  * into source cued address
 612                  */
 613                 case DMA_CDB_SG_SRC:
 614                         psgu = &dma_hw_desc->sg1u;
 615                         break;
 616                 /* for WXOR operations set multiplier
 617                  * into destination cued address(es)
 618                  */
 619                 case DMA_CDB_SG_DST1:
 620                         psgu = &dma_hw_desc->sg2u;
 621                         break;
 622                 case DMA_CDB_SG_DST2:
 623                         psgu = &dma_hw_desc->sg3u;
 624                         break;
 625                 default:
 626                         BUG();
 627                 }
 628
 629                 *psgu |= cpu_to_le32(mult_value << mult_index);
 630                 break;
 631         case PPC440SPE_XOR_ID:
 632                 xor_hw_desc = desc->hw_desc;
 633                 break;
 634         default:
 635                 BUG();
 636         }
 637 }
 638
 639 /**
 640  * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
 641  */
 642 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
 643                                 struct ppc440spe_adma_chan *chan,
 644                                 dma_addr_t addrh, dma_addr_t addrl,
 645                                 u32 dst_idx)
 646 {
 647         struct dma_cdb *dma_hw_desc;
 648         struct xor_cb *xor_hw_desc;
 649         phys_addr_t addr64, tmphi, tmplow;
 650         u32 *psgu, *psgl;
 651
 652         switch (chan->device->id) {
 653         case PPC440SPE_DMA0_ID:
 654         case PPC440SPE_DMA1_ID:
 655                 if (!addrh) {
 656                         addr64 = addrl;
 657                         tmphi = (addr64 >> 32);
 658                         tmplow = (addr64 & 0xFFFFFFFF);
 659                 } else {
 660                         tmphi = addrh;
 661                         tmplow = addrl;
 662                 }
 663                 dma_hw_desc = desc->hw_desc;
 664
 665                 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
 666                 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
 667
 668                 *psgl = cpu_to_le32((u32)tmplow);
 669                 *psgu |= cpu_to_le32((u32)tmphi);
 670                 break;
 671         case PPC440SPE_XOR_ID:
 672                 xor_hw_desc = desc->hw_desc;
 673                 xor_hw_desc->cbtal = addrl;
 674                 xor_hw_desc->cbtah |= addrh;
 675                 break;
 676         }
 677 }
 678
 679 /**
 680  * ppc440spe_desc_set_byte_count - set number of data bytes involved
 681  * into the operation
 682  */
 683 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
 684                                 struct ppc440spe_adma_chan *chan,
 685                                 u32 byte_count)
 686 {
 687         struct dma_cdb *dma_hw_desc;
 688         struct xor_cb *xor_hw_desc;
 689
 690         switch (chan->device->id) {
 691         case PPC440SPE_DMA0_ID:
 692         case PPC440SPE_DMA1_ID:
 693                 dma_hw_desc = desc->hw_desc;
 694                 dma_hw_desc->cnt = cpu_to_le32(byte_count);
 695                 break;
 696         case PPC440SPE_XOR_ID:
 697                 xor_hw_desc = desc->hw_desc;
 698                 xor_hw_desc->cbbc = byte_count;
 699                 break;
 700         }
 701 }
 702
 703 /**
 704  * ppc440spe_desc_set_rxor_block_size - set RXOR block size
 705  */
 706 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
 707 {
 708         /* assume that byte_count is aligned on the 512-boundary;
 709          * thus write it directly to the register (bits 23:31 are
 710          * reserved there).
 711          */
 712         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
 713 }
 714
 715 /**
 716  * ppc440spe_desc_set_dcheck - set CHECK pattern
 717  */
 718 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
 719                                 struct ppc440spe_adma_chan *chan, u8 *qword)
 720 {
 721         struct dma_cdb *dma_hw_desc;
 722
 723         switch (chan->device->id) {
 724         case PPC440SPE_DMA0_ID:
 725         case PPC440SPE_DMA1_ID:
 726                 dma_hw_desc = desc->hw_desc;
 727                 iowrite32(qword[0], &dma_hw_desc->sg3l);
 728                 iowrite32(qword[4], &dma_hw_desc->sg3u);
 729                 iowrite32(qword[8], &dma_hw_desc->sg2l);
 730                 iowrite32(qword[12], &dma_hw_desc->sg2u);
 731                 break;
 732         default:
 733                 BUG();
 734         }
 735 }
 736
 737 /**
 738  * ppc440spe_xor_set_link - set link address in xor CB
 739  */
 740 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
 741                                 struct ppc440spe_adma_desc_slot *next_desc)
 742 {
 743         struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
 744
 745         if (unlikely(!next_desc || !(next_desc->phys))) {
 746                 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
 747                         __func__, next_desc,
 748                         next_desc ? next_desc->phys : 0);
 749                 BUG();
 750         }
 751
 752         xor_hw_desc->cbs = 0;
 753         xor_hw_desc->cblal = next_desc->phys;
 754         xor_hw_desc->cblah = 0;
 755         xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
 756 }
 757
 758 /**
 759  * ppc440spe_desc_set_link - set the address of descriptor following this
 760  * descriptor in chain
 761  */
 762 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
 763                                 struct ppc440spe_adma_desc_slot *prev_desc,
 764                                 struct ppc440spe_adma_desc_slot *next_desc)
 765 {
 766         unsigned long flags;
 767         struct ppc440spe_adma_desc_slot *tail = next_desc;
 768
 769         if (unlikely(!prev_desc || !next_desc ||
 770                 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
 771                 /* If previous next is overwritten something is wrong.
 772                  * though we may refetch from append to initiate list
 773                  * processing; in this case - it's ok.
 774                  */
 775                 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
 776                         "prev->hw_next=0x%p\n", __func__, prev_desc,
 777                         next_desc, prev_desc ? prev_desc->hw_next : 0);
 778                 BUG();
 779         }
 780
 781         local_irq_save(flags);
 782
 783         /* do s/w chaining both for DMA and XOR descriptors */
 784         prev_desc->hw_next = next_desc;
 785
 786         switch (chan->device->id) {
 787         case PPC440SPE_DMA0_ID:
 788         case PPC440SPE_DMA1_ID:
 789                 break;
 790         case PPC440SPE_XOR_ID:
 791                 /* bind descriptor to the chain */
 792                 while (tail->hw_next)
 793                         tail = tail->hw_next;
 794                 xor_last_linked = tail;
 795
 796                 if (prev_desc == xor_last_submit)
 797                         /* do not link to the last submitted CB */
 798                         break;
 799                 ppc440spe_xor_set_link(prev_desc, next_desc);
 800                 break;
 801         }
 802
 803         local_irq_restore(flags);
 804 }
 805
 806 /**
 807  * ppc440spe_desc_get_link - get the address of the descriptor that
 808  * follows this one
 809  */
 810 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
 811                                         struct ppc440spe_adma_chan *chan)
 812 {
 813         if (!desc->hw_next)
 814                 return 0;
 815
 816         return desc->hw_next->phys;
 817 }
 818
 819 /**
 820  * ppc440spe_desc_is_aligned - check alignment
 821  */
 822 static inline int ppc440spe_desc_is_aligned(
 823         struct ppc440spe_adma_desc_slot *desc, int num_slots)
 824 {
 825         return (desc->idx & (num_slots - 1)) ? 0 : 1;
 826 }
 827
 828 /**
 829  * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
 830  * XOR operation
 831  */
 832 static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
 833                         int *slots_per_op)
 834 {
 835         int slot_cnt;
 836
 837         /* each XOR descriptor provides up to 16 source operands */
 838         slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
 839
 840         if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
 841                 return slot_cnt;
 842
 843         printk(KERN_ERR "%s: len %d > max %d !!\n",
 844                 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
 845         BUG();
 846         return slot_cnt;
 847 }
 848
 849 /**
 850  * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
 851  * DMA2 PQ operation
 852  */
 853 static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
 854                 int src_cnt, size_t len)
 855 {
 856         signed long long order = 0;
 857         int state = 0;
 858         int addr_count = 0;
 859         int i;
 860         for (i = 1; i < src_cnt; i++) {
 861                 dma_addr_t cur_addr = srcs[i];
 862                 dma_addr_t old_addr = srcs[i-1];
 863                 switch (state) {
 864                 case 0:
 865                         if (cur_addr == old_addr + len) {
 866                                 /* direct RXOR */
 867                                 order = 1;
 868                                 state = 1;
 869                                 if (i == src_cnt-1)
 870                                         addr_count++;
 871                         } else if (old_addr == cur_addr + len) {
 872                                 /* reverse RXOR */
 873                                 order = -1;
 874                                 state = 1;
 875                                 if (i == src_cnt-1)
 876                                         addr_count++;
 877                         } else {
 878                                 state = 3;
 879                         }
 880                         break;
 881                 case 1:
 882                         if (i == src_cnt-2 || (order == -1
 883                                 && cur_addr != old_addr - len)) {
 884                                 order = 0;
 885                                 state = 0;
 886                                 addr_count++;
 887                         } else if (cur_addr == old_addr + len*order) {
 888                                 state = 2;
 889                                 if (i == src_cnt-1)
 890                                         addr_count++;
 891                         } else if (cur_addr == old_addr + 2*len) {
 892                                 state = 2;
 893                                 if (i == src_cnt-1)
 894                                         addr_count++;
 895                         } else if (cur_addr == old_addr + 3*len) {
 896                                 state = 2;
 897                                 if (i == src_cnt-1)
 898                                         addr_count++;
 899                         } else {
 900                                 order = 0;
 901                                 state = 0;
 902                                 addr_count++;
 903                         }
 904                         break;
 905                 case 2:
 906                         order = 0;
 907                         state = 0;
 908                         addr_count++;
 909                                 break;
 910                 }
 911                 if (state == 3)
 912                         break;
 913         }
 914         if (src_cnt <= 1 || (state != 1 && state != 2)) {
 915                 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
 916                         __func__, src_cnt, state, addr_count, order);
 917                 for (i = 0; i < src_cnt; i++)
 918                         pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
 919                 BUG();
 920         }
 921
 922         return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
 923 }
 924
 925
 926 /******************************************************************************
 927  * ADMA channel low-level routines
 928  ******************************************************************************/
 929
 930 static u32
 931 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
 932 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
 933
 934 /**
 935  * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
 936  */
 937 static void ppc440spe_adma_device_clear_eot_status(
 938                                         struct ppc440spe_adma_chan *chan)
 939 {
 940         struct dma_regs *dma_reg;
 941         struct xor_regs *xor_reg;
 942         u8 *p = chan->device->dma_desc_pool_virt;
 943         struct dma_cdb *cdb;
 944         u32 rv, i;
 945
 946         switch (chan->device->id) {
 947         case PPC440SPE_DMA0_ID:
 948         case PPC440SPE_DMA1_ID:
 949                 /* read FIFO to ack */
 950                 dma_reg = chan->device->dma_reg;
 951                 while ((rv = ioread32(&dma_reg->csfpl))) {
 952                         i = rv & DMA_CDB_ADDR_MSK;
 953                         cdb = (struct dma_cdb *)&p[i -
 954                             (u32)chan->device->dma_desc_pool];
 955
 956                         /* Clear opcode to ack. This is necessary for
 957                          * ZeroSum operations only
 958                          */
 959                         cdb->opc = 0;
 960
 961                         if (test_bit(PPC440SPE_RXOR_RUN,
 962                             &ppc440spe_rxor_state)) {
 963                                 /* probably this is a completed RXOR op,
 964                                  * get pointer to CDB using the fact that
 965                                  * physical and virtual addresses of CDB
 966                                  * in pools have the same offsets
 967                                  */
 968                                 if (le32_to_cpu(cdb->sg1u) &
 969                                     DMA_CUED_XOR_BASE) {
 970                                         /* this is a RXOR */
 971                                         clear_bit(PPC440SPE_RXOR_RUN,
 972                                                   &ppc440spe_rxor_state);
 973                                 }
 974                         }
 975
 976                         if (rv & DMA_CDB_STATUS_MSK) {
 977                                 /* ZeroSum check failed
 978                                  */
 979                                 struct ppc440spe_adma_desc_slot *iter;
 980                                 dma_addr_t phys = rv & ~DMA_CDB_MSK;
 981
 982                                 /*
 983                                  * Update the status of corresponding
 984                                  * descriptor.
 985                                  */
 986                                 list_for_each_entry(iter, &chan->chain,
 987                                     chain_node) {
 988                                         if (iter->phys == phys)
 989                                                 break;
 990                                 }
 991                                 /*
 992                                  * if cannot find the corresponding
 993                                  * slot it's a bug
 994                                  */
 995                                 BUG_ON(&iter->chain_node == &chan->chain);
 996
 997                                 if (iter->xor_check_result) {
 998                                         if (test_bit(PPC440SPE_DESC_PCHECK,
 999                                                      &iter->flags)) {
1000                                                 *iter->xor_check_result |=
1001                                                         SUM_CHECK_P_RESULT;
1002                                         } else
1003                                         if (test_bit(PPC440SPE_DESC_QCHECK,
1004                                                      &iter->flags)) {
1005                                                 *iter->xor_check_result |=
1006                                                         SUM_CHECK_Q_RESULT;
1007                                         } else
1008                                                 BUG();
1009                                 }
1010                         }
1011                 }
1012
1013                 rv = ioread32(&dma_reg->dsts);
1014                 if (rv) {
1015                         pr_err("DMA%d err status: 0x%x\n",
1016                                chan->device->id, rv);
1017                         /* write back to clear */
1018                         iowrite32(rv, &dma_reg->dsts);
1019                 }
1020                 break;
1021         case PPC440SPE_XOR_ID:
1022                 /* reset status bits to ack */
1023                 xor_reg = chan->device->xor_reg;
1024                 rv = ioread32be(&xor_reg->sr);
1025                 iowrite32be(rv, &xor_reg->sr);
1026
1027                 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
1028                         if (rv & XOR_IE_RPTIE_BIT) {
1029                                 /* Read PLB Timeout Error.
1030                                  * Try to resubmit the CB
1031                                  */
1032                                 u32 val = ioread32be(&xor_reg->ccbalr);
1033
1034                                 iowrite32be(val, &xor_reg->cblalr);
1035
1036                                 val = ioread32be(&xor_reg->crsr);
1037                                 iowrite32be(val | XOR_CRSR_XAE_BIT,
1038                                             &xor_reg->crsr);
1039                         } else
1040                                 pr_err("XOR ERR 0x%x status\n", rv);
1041                         break;
1042                 }
1043
1044                 /*  if the XORcore is idle, but there are unprocessed CBs
1045                  * then refetch the s/w chain here
1046                  */
1047                 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1048                     do_xor_refetch)
1049                         ppc440spe_chan_append(chan);
1050                 break;
1051         }
1052 }
1053
1054 /**
1055  * ppc440spe_chan_is_busy - get the channel status
1056  */
1057 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1058 {
1059         struct dma_regs *dma_reg;
1060         struct xor_regs *xor_reg;
1061         int busy = 0;
1062
1063         switch (chan->device->id) {
1064         case PPC440SPE_DMA0_ID:
1065         case PPC440SPE_DMA1_ID:
1066                 dma_reg = chan->device->dma_reg;
1067                 /*  if command FIFO's head and tail pointers are equal and
1068                  * status tail is the same as command, then channel is free
1069                  */
1070                 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1071                     ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1072                         busy = 1;
1073                 break;
1074         case PPC440SPE_XOR_ID:
1075                 /* use the special status bit for the XORcore
1076                  */
1077                 xor_reg = chan->device->xor_reg;
1078                 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1079                 break;
1080         }
1081
1082         return busy;
1083 }
1084
1085 /**
1086  * ppc440spe_chan_set_first_xor_descriptor -  init XORcore chain
1087  */
1088 static void ppc440spe_chan_set_first_xor_descriptor(
1089                                 struct ppc440spe_adma_chan *chan,
1090                                 struct ppc440spe_adma_desc_slot *next_desc)
1091 {
1092         struct xor_regs *xor_reg = chan->device->xor_reg;
1093
1094         if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1095                 printk(KERN_INFO "%s: Warn: XORcore is running "
1096                         "when try to set the first CDB!\n",
1097                         __func__);
1098
1099         xor_last_submit = xor_last_linked = next_desc;
1100
1101         iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1102
1103         iowrite32be(next_desc->phys, &xor_reg->cblalr);
1104         iowrite32be(0, &xor_reg->cblahr);
1105         iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1106                     &xor_reg->cbcr);
1107
1108         chan->hw_chain_inited = 1;
1109 }
1110
1111 /**
1112  * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1113  * called with irqs disabled
1114  */
1115 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1116                 struct ppc440spe_adma_desc_slot *desc)
1117 {
1118         u32 pcdb;
1119         struct dma_regs *dma_reg = chan->device->dma_reg;
1120
1121         pcdb = desc->phys;
1122         if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1123                 pcdb |= DMA_CDB_NO_INT;
1124
1125         chan_last_sub[chan->device->id] = desc;
1126
1127         ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1128
1129         iowrite32(pcdb, &dma_reg->cpfpl);
1130 }
1131
1132 /**
1133  * ppc440spe_chan_append - update the h/w chain in the channel
1134  */
1135 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1136 {
1137         struct xor_regs *xor_reg;
1138         struct ppc440spe_adma_desc_slot *iter;
1139         struct xor_cb *xcb;
1140         u32 cur_desc;
1141         unsigned long flags;
1142
1143         local_irq_save(flags);
1144
1145         switch (chan->device->id) {
1146         case PPC440SPE_DMA0_ID:
1147         case PPC440SPE_DMA1_ID:
1148                 cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1149
1150                 if (likely(cur_desc)) {
1151                         iter = chan_last_sub[chan->device->id];
1152                         BUG_ON(!iter);
1153                 } else {
1154                         /* first peer */
1155                         iter = chan_first_cdb[chan->device->id];
1156                         BUG_ON(!iter);
1157                         ppc440spe_dma_put_desc(chan, iter);
1158                         chan->hw_chain_inited = 1;
1159                 }
1160
1161                 /* is there something new to append */
1162                 if (!iter->hw_next)
1163                         break;
1164
1165                 /* flush descriptors from the s/w queue to fifo */
1166                 list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1167                         ppc440spe_dma_put_desc(chan, iter);
1168                         if (!iter->hw_next)
1169                                 break;
1170                 }
1171                 break;
1172         case PPC440SPE_XOR_ID:
1173                 /* update h/w links and refetch */
1174                 if (!xor_last_submit->hw_next)
1175                         break;
1176
1177                 xor_reg = chan->device->xor_reg;
1178                 /* the last linked CDB has to generate an interrupt
1179                  * that we'd be able to append the next lists to h/w
1180                  * regardless of the XOR engine state at the moment of
1181                  * appending of these next lists
1182                  */
1183                 xcb = xor_last_linked->hw_desc;
1184                 xcb->cbc |= XOR_CBCR_CBCE_BIT;
1185
1186                 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1187                         /* XORcore is idle. Refetch now */
1188                         do_xor_refetch = 0;
1189                         ppc440spe_xor_set_link(xor_last_submit,
1190                                 xor_last_submit->hw_next);
1191
1192                         ADMA_LL_DBG(print_cb_list(chan,
1193                                 xor_last_submit->hw_next));
1194
1195                         xor_last_submit = xor_last_linked;
1196                         iowrite32be(ioread32be(&xor_reg->crsr) |
1197                                     XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1198                                     &xor_reg->crsr);
1199                 } else {
1200                         /* XORcore is running. Refetch later in the handler */
1201                         do_xor_refetch = 1;
1202                 }
1203
1204                 break;
1205         }
1206
1207         local_irq_restore(flags);
1208 }
1209
1210 /**
1211  * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1212  */
1213 static u32
1214 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1215 {
1216         struct dma_regs *dma_reg;
1217         struct xor_regs *xor_reg;
1218
1219         if (unlikely(!chan->hw_chain_inited))
1220                 /* h/w descriptor chain is not initialized yet */
1221                 return 0;
1222
1223         switch (chan->device->id) {
1224         case PPC440SPE_DMA0_ID:
1225         case PPC440SPE_DMA1_ID:
1226                 dma_reg = chan->device->dma_reg;
1227                 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1228         case PPC440SPE_XOR_ID:
1229                 xor_reg = chan->device->xor_reg;
1230                 return ioread32be(&xor_reg->ccbalr);
1231         }
1232         return 0;
1233 }
1234
1235 /**
1236  * ppc440spe_chan_run - enable the channel
1237  */
1238 static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1239 {
1240         struct xor_regs *xor_reg;
1241
1242         switch (chan->device->id) {
1243         case PPC440SPE_DMA0_ID:
1244         case PPC440SPE_DMA1_ID:
1245                 /* DMAs are always enabled, do nothing */
1246                 break;
1247         case PPC440SPE_XOR_ID:
1248                 /* drain write buffer */
1249                 xor_reg = chan->device->xor_reg;
1250
1251                 /* fetch descriptor pointed to in <link> */
1252                 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1253                             &xor_reg->crsr);
1254                 break;
1255         }
1256 }
1257
1258 /******************************************************************************
1259  * ADMA device level
1260  ******************************************************************************/
1261
1262 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1263 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1264
1265 static dma_cookie_t
1266 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1267
1268 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1269                                     dma_addr_t addr, int index);
1270 static void
1271 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1272                                   dma_addr_t addr, int index);
1273
1274 static void
1275 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1276                            dma_addr_t *paddr, unsigned long flags);
1277 static void
1278 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1279                           dma_addr_t addr, int index);
1280 static void
1281 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1282                                unsigned char mult, int index, int dst_pos);
1283 static void
1284 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1285                                    dma_addr_t paddr, dma_addr_t qaddr);
1286
1287 static struct page *ppc440spe_rxor_srcs[32];
1288
1289 /**
1290  * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1291  */
1292 static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1293 {
1294         int i, order = 0, state = 0;
1295         int idx = 0;
1296
1297         if (unlikely(!(src_cnt > 1)))
1298                 return 0;
1299
1300         BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1301
1302         /* Skip holes in the source list before checking */
1303         for (i = 0; i < src_cnt; i++) {
1304                 if (!srcs[i])
1305                         continue;
1306                 ppc440spe_rxor_srcs[idx++] = srcs[i];
1307         }
1308         src_cnt = idx;
1309
1310         for (i = 1; i < src_cnt; i++) {
1311                 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1312                 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1313
1314                 switch (state) {
1315                 case 0:
1316                         if (cur_addr == old_addr + len) {
1317                                 /* direct RXOR */
1318                                 order = 1;
1319                                 state = 1;
1320                         } else if (old_addr == cur_addr + len) {
1321                                 /* reverse RXOR */
1322                                 order = -1;
1323                                 state = 1;
1324                         } else
1325                                 goto out;
1326                         break;
1327                 case 1:
1328                         if ((i == src_cnt - 2) ||
1329                             (order == -1 && cur_addr != old_addr - len)) {
1330                                 order = 0;
1331                                 state = 0;
1332                         } else if ((cur_addr == old_addr + len * order) ||
1333                                    (cur_addr == old_addr + 2 * len) ||
1334                                    (cur_addr == old_addr + 3 * len)) {
1335                                 state = 2;
1336                         } else {
1337                                 order = 0;
1338                                 state = 0;
1339                         }
1340                         break;
1341                 case 2:
1342                         order = 0;
1343                         state = 0;
1344                         break;
1345                 }
1346         }
1347
1348 out:
1349         if (state == 1 || state == 2)
1350                 return 1;
1351
1352         return 0;
1353 }
1354
1355 /**
1356  * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1357  *      the operation given on this channel. It's assumed that 'chan' is
1358  *      capable to process 'cap' type of operation.
1359  * @chan: channel to use
1360  * @cap: type of transaction
1361  * @dst_lst: array of destination pointers
1362  * @dst_cnt: number of destination operands
1363  * @src_lst: array of source pointers
1364  * @src_cnt: number of source operands
1365  * @src_sz: size of each source operand
1366  */
1367 static int ppc440spe_adma_estimate(struct dma_chan *chan,
1368         enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1369         struct page **src_lst, int src_cnt, size_t src_sz)
1370 {
1371         int ef = 1;
1372
1373         if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1374                 /* If RAID-6 capabilities were not activated don't try
1375                  * to use them
1376                  */
1377                 if (unlikely(!ppc440spe_r6_enabled))
1378                         return -1;
1379         }
1380         /*  In the current implementation of ppc440spe ADMA driver it
1381          * makes sense to pick out only pq case, because it may be
1382          * processed:
1383          * (1) either using Biskup method on DMA2;
1384          * (2) or on DMA0/1.
1385          *  Thus we give a favour to (1) if the sources are suitable;
1386          * else let it be processed on one of the DMA0/1 engines.
1387          *  In the sum_product case where destination is also the
1388          * source process it on DMA0/1 only.
1389          */
1390         if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1391
1392                 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1393                         ef = 0; /* sum_product case, process on DMA0/1 */
1394                 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
1395                         ef = 3; /* override (DMA0/1 + idle) */
1396                 else
1397                         ef = 0; /* can't process on DMA2 if !rxor */
1398         }
1399
1400         /* channel idleness increases the priority */
1401         if (likely(ef) &&
1402             !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1403                 ef++;
1404
1405         return ef;
1406 }
1407
1408 struct dma_chan *
1409 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1410         struct page **dst_lst, int dst_cnt, struct page **src_lst,
1411         int src_cnt, size_t src_sz)
1412 {
1413         struct dma_chan *best_chan = NULL;
1414         struct ppc_dma_chan_ref *ref;
1415         int best_rank = -1;
1416
1417         if (unlikely(!src_sz))
1418                 return NULL;
1419         if (src_sz > PAGE_SIZE) {
1420                 /*
1421                  * should a user of the api ever pass > PAGE_SIZE requests
1422                  * we sort out cases where temporary page-sized buffers
1423                  * are used.
1424                  */
1425                 switch (cap) {
1426                 case DMA_PQ:
1427                         if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1428                                 return NULL;
1429                         if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1430                                 return NULL;
1431                         break;
1432                 case DMA_PQ_VAL:
1433                 case DMA_XOR_VAL:
1434                         return NULL;
1435                 default:
1436                         break;
1437                 }
1438         }
1439
1440         list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1441                 if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1442                         int rank;
1443
1444                         rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
1445                                         dst_cnt, src_lst, src_cnt, src_sz);
1446                         if (rank > best_rank) {
1447                                 best_rank = rank;
1448                                 best_chan = ref->chan;
1449                         }
1450                 }
1451         }
1452
1453         return best_chan;
1454 }
1455 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1456
1457 /**
1458  * ppc440spe_get_group_entry - get group entry with index idx
1459  * @tdesc: is the last allocated slot in the group.
1460  */
1461 static struct ppc440spe_adma_desc_slot *
1462 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1463 {
1464         struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1465         int i = 0;
1466
1467         if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1468                 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1469                         __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1470                 BUG();
1471         }
1472
1473         list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1474                 if (i++ == entry_idx)
1475                         break;
1476         }
1477         return iter;
1478 }
1479
1480 /**
1481  * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1482  * @slot: Slot to free
1483  * Caller must hold &ppc440spe_chan->lock while calling this function
1484  */
1485 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1486                                       struct ppc440spe_adma_chan *chan)
1487 {
1488         int stride = slot->slots_per_op;
1489
1490         while (stride--) {
1491                 slot->slots_per_op = 0;
1492                 slot = list_entry(slot->slot_node.next,
1493                                 struct ppc440spe_adma_desc_slot,
1494                                 slot_node);
1495         }
1496 }
1497
1498 /**
1499  * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1500  * upon completion
1501  */
1502 static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1503                 struct ppc440spe_adma_desc_slot *desc,
1504                 struct ppc440spe_adma_chan *chan,
1505                 dma_cookie_t cookie)
1506 {
1507         int i;
1508
1509         BUG_ON(desc->async_tx.cookie < 0);
1510         if (desc->async_tx.cookie > 0) {
1511                 cookie = desc->async_tx.cookie;
1512                 desc->async_tx.cookie = 0;
1513
1514                 /* call the callback (must not sleep or submit new
1515                  * operations to this channel)
1516                  */
1517                 if (desc->async_tx.callback)
1518                         desc->async_tx.callback(
1519                                 desc->async_tx.callback_param);
1520
1521                 dma_descriptor_unmap(&desc->async_tx);
1522         }
1523
1524         /* run dependent operations */
1525         dma_run_dependencies(&desc->async_tx);
1526
1527         return cookie;
1528 }
1529
1530 /**
1531  * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1532  */
1533 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1534                 struct ppc440spe_adma_chan *chan)
1535 {
1536         /* the client is allowed to attach dependent operations
1537          * until 'ack' is set
1538          */
1539         if (!async_tx_test_ack(&desc->async_tx))
1540                 return 0;
1541
1542         /* leave the last descriptor in the chain
1543          * so we can append to it
1544          */
1545         if (list_is_last(&desc->chain_node, &chan->chain) ||
1546             desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1547                 return 1;
1548
1549         if (chan->device->id != PPC440SPE_XOR_ID) {
1550                 /* our DMA interrupt handler clears opc field of
1551                  * each processed descriptor. For all types of
1552                  * operations except for ZeroSum we do not actually
1553                  * need ack from the interrupt handler. ZeroSum is a
1554                  * special case since the result of this operation
1555                  * is available from the handler only, so if we see
1556                  * such type of descriptor (which is unprocessed yet)
1557                  * then leave it in chain.
1558                  */
1559                 struct dma_cdb *cdb = desc->hw_desc;
1560                 if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1561                         return 1;
1562         }
1563
1564         dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1565                 desc->phys, desc->idx, desc->slots_per_op);
1566
1567         list_del(&desc->chain_node);
1568         ppc440spe_adma_free_slots(desc, chan);
1569         return 0;
1570 }
1571
1572 /**
1573  * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1574  *      which runs through the channel CDBs list until reach the descriptor
1575  *      currently processed. When routine determines that all CDBs of group
1576  *      are completed then corresponding callbacks (if any) are called and slots
1577  *      are freed.
1578  */
1579 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1580 {
1581         struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1582         dma_cookie_t cookie = 0;
1583         u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1584         int busy = ppc440spe_chan_is_busy(chan);
1585         int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1586
1587         dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1588                 chan->device->id, __func__);
1589
1590         if (!current_desc) {
1591                 /*  There were no transactions yet, so
1592                  * nothing to clean
1593                  */
1594                 return;
1595         }
1596
1597         /* free completed slots from the chain starting with
1598          * the oldest descriptor
1599          */
1600         list_for_each_entry_safe(iter, _iter, &chan->chain,
1601                                         chain_node) {
1602                 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1603                     "busy: %d this_desc: %#llx next_desc: %#x "
1604                     "cur: %#x ack: %d\n",
1605                     iter->async_tx.cookie, iter->idx, busy, iter->phys,
1606                     ppc440spe_desc_get_link(iter, chan), current_desc,
1607                     async_tx_test_ack(&iter->async_tx));
1608                 prefetch(_iter);
1609                 prefetch(&_iter->async_tx);
1610
1611                 /* do not advance past the current descriptor loaded into the
1612                  * hardware channel,subsequent descriptors are either in process
1613                  * or have not been submitted
1614                  */
1615                 if (seen_current)
1616                         break;
1617
1618                 /* stop the search if we reach the current descriptor and the
1619                  * channel is busy, or if it appears that the current descriptor
1620                  * needs to be re-read (i.e. has been appended to)
1621                  */
1622                 if (iter->phys == current_desc) {
1623                         BUG_ON(seen_current++);
1624                         if (busy || ppc440spe_desc_get_link(iter, chan)) {
1625                                 /* not all descriptors of the group have
1626                                  * been completed; exit.
1627                                  */
1628                                 break;
1629                         }
1630                 }
1631
1632                 /* detect the start of a group transaction */
1633                 if (!slot_cnt && !slots_per_op) {
1634                         slot_cnt = iter->slot_cnt;
1635                         slots_per_op = iter->slots_per_op;
1636                         if (slot_cnt <= slots_per_op) {
1637                                 slot_cnt = 0;
1638                                 slots_per_op = 0;
1639                         }
1640                 }
1641
1642                 if (slot_cnt) {
1643                         if (!group_start)
1644                                 group_start = iter;
1645                         slot_cnt -= slots_per_op;
1646                 }
1647
1648                 /* all the members of a group are complete */
1649                 if (slots_per_op != 0 && slot_cnt == 0) {
1650                         struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1651                         int end_of_chain = 0;
1652
1653                         /* clean up the group */
1654                         slot_cnt = group_start->slot_cnt;
1655                         grp_iter = group_start;
1656                         list_for_each_entry_safe_from(grp_iter, _grp_iter,
1657                                 &chan->chain, chain_node) {
1658
1659                                 cookie = ppc440spe_adma_run_tx_complete_actions(
1660                                         grp_iter, chan, cookie);
1661
1662                                 slot_cnt -= slots_per_op;
1663                                 end_of_chain = ppc440spe_adma_clean_slot(
1664                                     grp_iter, chan);
1665                                 if (end_of_chain && slot_cnt) {
1666                                         /* Should wait for ZeroSum completion */
1667                                         if (cookie > 0)
1668                                                 chan->common.completed_cookie = cookie;
1669                                         return;
1670                                 }
1671
1672                                 if (slot_cnt == 0 || end_of_chain)
1673                                         break;
1674                         }
1675
1676                         /* the group should be complete at this point */
1677                         BUG_ON(slot_cnt);
1678
1679                         slots_per_op = 0;
1680                         group_start = NULL;
1681                         if (end_of_chain)
1682                                 break;
1683                         else
1684                                 continue;
1685                 } else if (slots_per_op) /* wait for group completion */
1686                         continue;
1687
1688                 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
1689                     cookie);
1690
1691                 if (ppc440spe_adma_clean_slot(iter, chan))
1692                         break;
1693         }
1694
1695         BUG_ON(!seen_current);
1696
1697         if (cookie > 0) {
1698                 chan->common.completed_cookie = cookie;
1699                 pr_debug("\tcompleted cookie %d\n", cookie);
1700         }
1701
1702 }
1703
1704 /**
1705  * ppc440spe_adma_tasklet - clean up watch-dog initiator
1706  */
1707 static void ppc440spe_adma_tasklet(unsigned long data)
1708 {
1709         struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data;
1710
1711         spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1712         __ppc440spe_adma_slot_cleanup(chan);
1713         spin_unlock(&chan->lock);
1714 }
1715
1716 /**
1717  * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1718  */
1719 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1720 {
1721         spin_lock_bh(&chan->lock);
1722         __ppc440spe_adma_slot_cleanup(chan);
1723         spin_unlock_bh(&chan->lock);
1724 }
1725
1726 /**
1727  * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1728  */
1729 static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1730                 struct ppc440spe_adma_chan *chan, int num_slots,
1731                 int slots_per_op)
1732 {
1733         struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
1734         struct ppc440spe_adma_desc_slot *alloc_start = NULL;
1735         struct list_head chain = LIST_HEAD_INIT(chain);
1736         int slots_found, retry = 0;
1737
1738
1739         BUG_ON(!num_slots || !slots_per_op);
1740         /* start search from the last allocated descrtiptor
1741          * if a contiguous allocation can not be found start searching
1742          * from the beginning of the list
1743          */
1744 retry:
1745         slots_found = 0;
1746         if (retry == 0)
1747                 iter = chan->last_used;
1748         else
1749                 iter = list_entry(&chan->all_slots,
1750                                   struct ppc440spe_adma_desc_slot,
1751                                   slot_node);
1752         list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
1753             slot_node) {
1754                 prefetch(_iter);
1755                 prefetch(&_iter->async_tx);
1756                 if (iter->slots_per_op) {
1757                         slots_found = 0;
1758                         continue;
1759                 }
1760
1761                 /* start the allocation if the slot is correctly aligned */
1762                 if (!slots_found++)
1763                         alloc_start = iter;
1764
1765                 if (slots_found == num_slots) {
1766                         struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
1767                         struct ppc440spe_adma_desc_slot *last_used = NULL;
1768
1769                         iter = alloc_start;
1770                         while (num_slots) {
1771                                 int i;
1772                                 /* pre-ack all but the last descriptor */
1773                                 if (num_slots != slots_per_op)
1774                                         async_tx_ack(&iter->async_tx);
1775
1776                                 list_add_tail(&iter->chain_node, &chain);
1777                                 alloc_tail = iter;
1778                                 iter->async_tx.cookie = 0;
1779                                 iter->hw_next = NULL;
1780                                 iter->flags = 0;
1781                                 iter->slot_cnt = num_slots;
1782                                 iter->xor_check_result = NULL;
1783                                 for (i = 0; i < slots_per_op; i++) {
1784                                         iter->slots_per_op = slots_per_op - i;
1785                                         last_used = iter;
1786                                         iter = list_entry(iter->slot_node.next,
1787                                                 struct ppc440spe_adma_desc_slot,
1788                                                 slot_node);
1789                                 }
1790                                 num_slots -= slots_per_op;
1791                         }
1792                         alloc_tail->group_head = alloc_start;
1793                         alloc_tail->async_tx.cookie = -EBUSY;
1794                         list_splice(&chain, &alloc_tail->group_list);
1795                         chan->last_used = last_used;
1796                         return alloc_tail;
1797                 }
1798         }
1799         if (!retry++)
1800                 goto retry;
1801
1802         /* try to free some slots if the allocation fails */
1803         tasklet_schedule(&chan->irq_tasklet);
1804         return NULL;
1805 }
1806
1807 /**
1808  * ppc440spe_adma_alloc_chan_resources -  allocate pools for CDB slots
1809  */
1810 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
1811 {
1812         struct ppc440spe_adma_chan *ppc440spe_chan;
1813         struct ppc440spe_adma_desc_slot *slot = NULL;
1814         char *hw_desc;
1815         int i, db_sz;
1816         int init;
1817
1818         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1819         init = ppc440spe_chan->slots_allocated ? 0 : 1;
1820         chan->chan_id = ppc440spe_chan->device->id;
1821
1822         /* Allocate descriptor slots */
1823         i = ppc440spe_chan->slots_allocated;
1824         if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
1825                 db_sz = sizeof(struct dma_cdb);
1826         else
1827                 db_sz = sizeof(struct xor_cb);
1828
1829         for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
1830                 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
1831                                GFP_KERNEL);
1832                 if (!slot) {
1833                         printk(KERN_INFO "SPE ADMA Channel only initialized"
1834                                 " %d descriptor slots", i--);
1835                         break;
1836                 }
1837
1838                 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
1839                 slot->hw_desc = (void *) &hw_desc[i * db_sz];
1840                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
1841                 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
1842                 INIT_LIST_HEAD(&slot->chain_node);
1843                 INIT_LIST_HEAD(&slot->slot_node);
1844                 INIT_LIST_HEAD(&slot->group_list);
1845                 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
1846                 slot->idx = i;
1847
1848                 spin_lock_bh(&ppc440spe_chan->lock);
1849                 ppc440spe_chan->slots_allocated++;
1850                 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
1851                 spin_unlock_bh(&ppc440spe_chan->lock);
1852         }
1853
1854         if (i && !ppc440spe_chan->last_used) {
1855                 ppc440spe_chan->last_used =
1856                         list_entry(ppc440spe_chan->all_slots.next,
1857                                 struct ppc440spe_adma_desc_slot,
1858                                 slot_node);
1859         }
1860
1861         dev_dbg(ppc440spe_chan->device->common.dev,
1862                 "ppc440spe adma%d: allocated %d descriptor slots\n",
1863                 ppc440spe_chan->device->id, i);
1864
1865         /* initialize the channel and the chain with a null operation */
1866         if (init) {
1867                 switch (ppc440spe_chan->device->id) {
1868                 case PPC440SPE_DMA0_ID:
1869                 case PPC440SPE_DMA1_ID:
1870                         ppc440spe_chan->hw_chain_inited = 0;
1871                         /* Use WXOR for self-testing */
1872                         if (!ppc440spe_r6_tchan)
1873                                 ppc440spe_r6_tchan = ppc440spe_chan;
1874                         break;
1875                 case PPC440SPE_XOR_ID:
1876                         ppc440spe_chan_start_null_xor(ppc440spe_chan);
1877                         break;
1878                 default:
1879                         BUG();
1880                 }
1881                 ppc440spe_chan->needs_unmap = 1;
1882         }
1883
1884         return (i > 0) ? i : -ENOMEM;
1885 }
1886
1887 /**
1888  * ppc440spe_rxor_set_region_data -
1889  */
1890 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
1891         u8 xor_arg_no, u32 mask)
1892 {
1893         struct xor_cb *xcb = desc->hw_desc;
1894
1895         xcb->ops[xor_arg_no].h |= mask;
1896 }
1897
1898 /**
1899  * ppc440spe_rxor_set_src -
1900  */
1901 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
1902         u8 xor_arg_no, dma_addr_t addr)
1903 {
1904         struct xor_cb *xcb = desc->hw_desc;
1905
1906         xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
1907         xcb->ops[xor_arg_no].l = addr;
1908 }
1909
1910 /**
1911  * ppc440spe_rxor_set_mult -
1912  */
1913 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
1914         u8 xor_arg_no, u8 idx, u8 mult)
1915 {
1916         struct xor_cb *xcb = desc->hw_desc;
1917
1918         xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
1919 }
1920
1921 /**
1922  * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
1923  *      has been achieved
1924  */
1925 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
1926 {
1927         dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
1928                 chan->device->id, chan->pending);
1929
1930         if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
1931                 chan->pending = 0;
1932                 ppc440spe_chan_append(chan);
1933         }
1934 }
1935
1936 /**
1937  * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
1938  *      (it's not necessary that descriptors will be submitted to the h/w
1939  *      chains too right now)
1940  */
1941 static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
1942 {
1943         struct ppc440spe_adma_desc_slot *sw_desc;
1944         struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
1945         struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
1946         int slot_cnt;
1947         int slots_per_op;
1948         dma_cookie_t cookie;
1949
1950         sw_desc = tx_to_ppc440spe_adma_slot(tx);
1951
1952         group_start = sw_desc->group_head;
1953         slot_cnt = group_start->slot_cnt;
1954         slots_per_op = group_start->slots_per_op;
1955
1956         spin_lock_bh(&chan->lock);
1957         cookie = dma_cookie_assign(tx);
1958
1959         if (unlikely(list_empty(&chan->chain))) {
1960                 /* first peer */
1961                 list_splice_init(&sw_desc->group_list, &chan->chain);
1962                 chan_first_cdb[chan->device->id] = group_start;
1963         } else {
1964                 /* isn't first peer, bind CDBs to chain */
1965                 old_chain_tail = list_entry(chan->chain.prev,
1966                                         struct ppc440spe_adma_desc_slot,
1967                                         chain_node);
1968                 list_splice_init(&sw_desc->group_list,
1969                     &old_chain_tail->chain_node);
1970                 /* fix up the hardware chain */
1971                 ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
1972         }
1973
1974         /* increment the pending count by the number of operations */
1975         chan->pending += slot_cnt / slots_per_op;
1976         ppc440spe_adma_check_threshold(chan);
1977         spin_unlock_bh(&chan->lock);
1978
1979         dev_dbg(chan->device->common.dev,
1980                 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
1981                 chan->device->id, __func__,
1982                 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
1983
1984         return cookie;
1985 }
1986
1987 /**
1988  * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
1989  */
1990 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
1991                 struct dma_chan *chan, unsigned long flags)
1992 {
1993         struct ppc440spe_adma_chan *ppc440spe_chan;
1994         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1995         int slot_cnt, slots_per_op;
1996
1997         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1998
1999         dev_dbg(ppc440spe_chan->device->common.dev,
2000                 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
2001                 __func__);
2002
2003         spin_lock_bh(&ppc440spe_chan->lock);
2004         slot_cnt = slots_per_op = 1;
2005         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2006                         slots_per_op);
2007         if (sw_desc) {
2008                 group_start = sw_desc->group_head;
2009                 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
2010                 group_start->unmap_len = 0;
2011                 sw_desc->async_tx.flags = flags;
2012         }
2013         spin_unlock_bh(&ppc440spe_chan->lock);
2014
2015         return sw_desc ? &sw_desc->async_tx : NULL;
2016 }
2017
2018 /**
2019  * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
2020  */
2021 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
2022                 struct dma_chan *chan, dma_addr_t dma_dest,
2023                 dma_addr_t dma_src, size_t len, unsigned long flags)
2024 {
2025         struct ppc440spe_adma_chan *ppc440spe_chan;
2026         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2027         int slot_cnt, slots_per_op;
2028
2029         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2030
2031         if (unlikely(!len))
2032                 return NULL;
2033
2034         BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT);
2035
2036         spin_lock_bh(&ppc440spe_chan->lock);
2037
2038         dev_dbg(ppc440spe_chan->device->common.dev,
2039                 "ppc440spe adma%d: %s len: %u int_en %d\n",
2040                 ppc440spe_chan->device->id, __func__, len,
2041                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2042         slot_cnt = slots_per_op = 1;
2043         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2044                 slots_per_op);
2045         if (sw_desc) {
2046                 group_start = sw_desc->group_head;
2047                 ppc440spe_desc_init_memcpy(group_start, flags);
2048                 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2049                 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
2050                 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2051                 sw_desc->unmap_len = len;
2052                 sw_desc->async_tx.flags = flags;
2053         }
2054         spin_unlock_bh(&ppc440spe_chan->lock);
2055
2056         return sw_desc ? &sw_desc->async_tx : NULL;
2057 }
2058
2059 /**
2060  * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2061  */
2062 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2063                 struct dma_chan *chan, dma_addr_t dma_dest,
2064                 dma_addr_t *dma_src, u32 src_cnt, size_t len,
2065                 unsigned long flags)
2066 {
2067         struct ppc440spe_adma_chan *ppc440spe_chan;
2068         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2069         int slot_cnt, slots_per_op;
2070
2071         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2072
2073         ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2074                                      dma_dest, dma_src, src_cnt));
2075         if (unlikely(!len))
2076                 return NULL;
2077         BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2078
2079         dev_dbg(ppc440spe_chan->device->common.dev,
2080                 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2081                 ppc440spe_chan->device->id, __func__, src_cnt, len,
2082                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2083
2084         spin_lock_bh(&ppc440spe_chan->lock);
2085         slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
2086         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2087                         slots_per_op);
2088         if (sw_desc) {
2089                 group_start = sw_desc->group_head;
2090                 ppc440spe_desc_init_xor(group_start, src_cnt, flags);
2091                 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2092                 while (src_cnt--)
2093                         ppc440spe_adma_memcpy_xor_set_src(group_start,
2094                                 dma_src[src_cnt], src_cnt);
2095                 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2096                 sw_desc->unmap_len = len;
2097                 sw_desc->async_tx.flags = flags;
2098         }
2099         spin_unlock_bh(&ppc440spe_chan->lock);
2100
2101         return sw_desc ? &sw_desc->async_tx : NULL;
2102 }
2103
2104 static inline void
2105 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2106                                 int src_cnt);
2107 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2108
2109 /**
2110  * ppc440spe_adma_init_dma2rxor_slot -
2111  */
2112 static void ppc440spe_adma_init_dma2rxor_slot(
2113                 struct ppc440spe_adma_desc_slot *desc,
2114                 dma_addr_t *src, int src_cnt)
2115 {
2116         int i;
2117
2118         /* initialize CDB */
2119         for (i = 0; i < src_cnt; i++) {
2120                 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
2121                                                  desc->src_cnt, (u32)src[i]);
2122         }
2123 }
2124
2125 /**
2126  * ppc440spe_dma01_prep_mult -
2127  * for Q operation where destination is also the source
2128  */
2129 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2130                 struct ppc440spe_adma_chan *ppc440spe_chan,
2131                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2132                 const unsigned char *scf, size_t len, unsigned long flags)
2133 {
2134         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2135         unsigned long op = 0;
2136         int slot_cnt;
2137
2138         set_bit(PPC440SPE_DESC_WXOR, &op);
2139         slot_cnt = 2;
2140
2141         spin_lock_bh(&ppc440spe_chan->lock);
2142
2143         /* use WXOR, each descriptor occupies one slot */
2144         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2145         if (sw_desc) {
2146                 struct ppc440spe_adma_chan *chan;
2147                 struct ppc440spe_adma_desc_slot *iter;
2148                 struct dma_cdb *hw_desc;
2149
2150                 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2151                 set_bits(op, &sw_desc->flags);
2152                 sw_desc->src_cnt = src_cnt;
2153                 sw_desc->dst_cnt = dst_cnt;
2154                 /* First descriptor, zero data in the destination and copy it
2155                  * to q page using MULTICAST transfer.
2156                  */
2157                 iter = list_first_entry(&sw_desc->group_list,
2158                                         struct ppc440spe_adma_desc_slot,
2159                                         chain_node);
2160                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2161                 /* set 'next' pointer */
2162                 iter->hw_next = list_entry(iter->chain_node.next,
2163                                            struct ppc440spe_adma_desc_slot,
2164                                            chain_node);
2165                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2166                 hw_desc = iter->hw_desc;
2167                 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2168
2169                 ppc440spe_desc_set_dest_addr(iter, chan,
2170                                              DMA_CUED_XOR_BASE, dst[0], 0);
2171                 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
2172                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2173                                             src[0]);
2174                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2175                 iter->unmap_len = len;
2176
2177                 /*
2178                  * Second descriptor, multiply data from the q page
2179                  * and store the result in real destination.
2180                  */
2181                 iter = list_first_entry(&iter->chain_node,
2182                                         struct ppc440spe_adma_desc_slot,
2183                                         chain_node);
2184                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2185                 iter->hw_next = NULL;
2186                 if (flags & DMA_PREP_INTERRUPT)
2187                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2188                 else
2189                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2190
2191                 hw_desc = iter->hw_desc;
2192                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2193                 ppc440spe_desc_set_src_addr(iter, chan, 0,
2194                                             DMA_CUED_XOR_HB, dst[1]);
2195                 ppc440spe_desc_set_dest_addr(iter, chan,
2196                                              DMA_CUED_XOR_BASE, dst[0], 0);
2197
2198                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2199                                             DMA_CDB_SG_DST1, scf[0]);
2200                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2201                 iter->unmap_len = len;
2202                 sw_desc->async_tx.flags = flags;
2203         }
2204
2205         spin_unlock_bh(&ppc440spe_chan->lock);
2206
2207         return sw_desc;
2208 }
2209
2210 /**
2211  * ppc440spe_dma01_prep_sum_product -
2212  * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2213  * the source.
2214  */
2215 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2216                 struct ppc440spe_adma_chan *ppc440spe_chan,
2217                 dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2218                 const unsigned char *scf, size_t len, unsigned long flags)
2219 {
2220         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2221         unsigned long op = 0;
2222         int slot_cnt;
2223
2224         set_bit(PPC440SPE_DESC_WXOR, &op);
2225         slot_cnt = 3;
2226
2227         spin_lock_bh(&ppc440spe_chan->lock);
2228
2229         /* WXOR, each descriptor occupies one slot */
2230         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2231         if (sw_desc) {
2232                 struct ppc440spe_adma_chan *chan;
2233                 struct ppc440spe_adma_desc_slot *iter;
2234                 struct dma_cdb *hw_desc;
2235
2236                 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2237                 set_bits(op, &sw_desc->flags);
2238                 sw_desc->src_cnt = src_cnt;
2239                 sw_desc->dst_cnt = 1;
2240                 /* 1st descriptor, src[1] data to q page and zero destination */
2241                 iter = list_first_entry(&sw_desc->group_list,
2242                                         struct ppc440spe_adma_desc_slot,
2243                                         chain_node);
2244                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2245                 iter->hw_next = list_entry(iter->chain_node.next,
2246                                            struct ppc440spe_adma_desc_slot,
2247                                            chain_node);
2248                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2249                 hw_desc = iter->hw_desc;
2250                 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2251
2252                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2253                                              *dst, 0);
2254                 ppc440spe_desc_set_dest_addr(iter, chan, 0,
2255                                              ppc440spe_chan->qdest, 1);
2256                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2257                                             src[1]);
2258                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2259                 iter->unmap_len = len;
2260
2261                 /* 2nd descriptor, multiply src[1] data and store the
2262                  * result in destination */
2263                 iter = list_first_entry(&iter->chain_node,
2264                                         struct ppc440spe_adma_desc_slot,
2265                                         chain_node);
2266                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2267                 /* set 'next' pointer */
2268                 iter->hw_next = list_entry(iter->chain_node.next,
2269                                            struct ppc440spe_adma_desc_slot,
2270                                            chain_node);
2271                 if (flags & DMA_PREP_INTERRUPT)
2272                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2273                 else
2274                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2275
2276                 hw_desc = iter->hw_desc;
2277                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2278                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2279                                             ppc440spe_chan->qdest);
2280                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2281                                              *dst, 0);
2282                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2283                                             DMA_CDB_SG_DST1, scf[1]);
2284                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2285                 iter->unmap_len = len;
2286
2287                 /*
2288                  * 3rd descriptor, multiply src[0] data and xor it
2289                  * with destination
2290                  */
2291                 iter = list_first_entry(&iter->chain_node,
2292                                         struct ppc440spe_adma_desc_slot,
2293                                         chain_node);
2294                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2295                 iter->hw_next = NULL;
2296                 if (flags & DMA_PREP_INTERRUPT)
2297                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2298                 else
2299                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2300
2301                 hw_desc = iter->hw_desc;
2302                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2303                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2304                                             src[0]);
2305                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2306                                              *dst, 0);
2307                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2308                                             DMA_CDB_SG_DST1, scf[0]);
2309                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2310                 iter->unmap_len = len;
2311                 sw_desc->async_tx.flags = flags;
2312         }
2313
2314         spin_unlock_bh(&ppc440spe_chan->lock);
2315
2316         return sw_desc;
2317 }
2318
2319 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2320                 struct ppc440spe_adma_chan *ppc440spe_chan,
2321                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2322                 const unsigned char *scf, size_t len, unsigned long flags)
2323 {
2324         int slot_cnt;
2325         struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2326         unsigned long op = 0;
2327         unsigned char mult = 1;
2328
2329         pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2330                  __func__, dst_cnt, src_cnt, len);
2331         /*  select operations WXOR/RXOR depending on the
2332          * source addresses of operators and the number
2333          * of destinations (RXOR support only Q-parity calculations)
2334          */
2335         set_bit(PPC440SPE_DESC_WXOR, &op);
2336         if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
2337                 /* no active RXOR;
2338                  * do RXOR if:
2339                  * - there are more than 1 source,
2340                  * - len is aligned on 512-byte boundary,
2341                  * - source addresses fit to one of 4 possible regions.
2342                  */
2343                 if (src_cnt > 1 &&
2344                     !(len & MQ0_CF2H_RXOR_BS_MASK) &&
2345                     (src[0] + len) == src[1]) {
2346                         /* may do RXOR R1 R2 */
2347                         set_bit(PPC440SPE_DESC_RXOR, &op);
2348                         if (src_cnt != 2) {
2349                                 /* may try to enhance region of RXOR */
2350                                 if ((src[1] + len) == src[2]) {
2351                                         /* do RXOR R1 R2 R3 */
2352                                         set_bit(PPC440SPE_DESC_RXOR123,
2353                                                 &op);
2354                                 } else if ((src[1] + len * 2) == src[2]) {
2355                                         /* do RXOR R1 R2 R4 */
2356                                         set_bit(PPC440SPE_DESC_RXOR124, &op);
2357                                 } else if ((src[1] + len * 3) == src[2]) {
2358                                         /* do RXOR R1 R2 R5 */
2359                                         set_bit(PPC440SPE_DESC_RXOR125,
2360                                                 &op);
2361                                 } else {
2362                                         /* do RXOR R1 R2 */
2363                                         set_bit(PPC440SPE_DESC_RXOR12,
2364                                                 &op);
2365                                 }
2366                         } else {
2367                                 /* do RXOR R1 R2 */
2368                                 set_bit(PPC440SPE_DESC_RXOR12, &op);
2369                         }
2370                 }
2371
2372                 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2373                         /* can not do this operation with RXOR */
2374                         clear_bit(PPC440SPE_RXOR_RUN,
2375                                 &ppc440spe_rxor_state);
2376                 } else {
2377                         /* can do; set block size right now */
2378                         ppc440spe_desc_set_rxor_block_size(len);
2379                 }
2380         }
2381
2382         /* Number of necessary slots depends on operation type selected */
2383         if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2384                 /*  This is a WXOR only chain. Need descriptors for each
2385                  * source to GF-XOR them with WXOR, and need descriptors
2386                  * for each destination to zero them with WXOR
2387                  */
2388                 slot_cnt = src_cnt;
2389
2390                 if (flags & DMA_PREP_ZERO_P) {
2391                         slot_cnt++;
2392                         set_bit(PPC440SPE_ZERO_P, &op);
2393                 }
2394                 if (flags & DMA_PREP_ZERO_Q) {
2395                         slot_cnt++;
2396                         set_bit(PPC440SPE_ZERO_Q, &op);
2397                 }
2398         } else {
2399                 /*  Need 1/2 descriptor for RXOR operation, and
2400                  * need (src_cnt - (2 or 3)) for WXOR of sources
2401                  * remained (if any)
2402                  */
2403                 slot_cnt = dst_cnt;
2404
2405                 if (flags & DMA_PREP_ZERO_P)
2406                         set_bit(PPC440SPE_ZERO_P, &op);
2407                 if (flags & DMA_PREP_ZERO_Q)
2408                         set_bit(PPC440SPE_ZERO_Q, &op);
2409
2410                 if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2411                         slot_cnt += src_cnt - 2;
2412                 else
2413                         slot_cnt += src_cnt - 3;
2414
2415                 /*  Thus we have either RXOR only chain or
2416                  * mixed RXOR/WXOR
2417                  */
2418                 if (slot_cnt == dst_cnt)
2419                         /* RXOR only chain */
2420                         clear_bit(PPC440SPE_DESC_WXOR, &op);
2421         }
2422
2423         spin_lock_bh(&ppc440spe_chan->lock);
2424         /* for both RXOR/WXOR each descriptor occupies one slot */
2425         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2426         if (sw_desc) {
2427                 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2428                                 flags, op);
2429
2430                 /* setup dst/src/mult */
2431                 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2432                          __func__, dst[0], dst[1]);
2433                 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2434                 while (src_cnt--) {
2435                         ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2436                                                   src_cnt);
2437
2438                         /* NOTE: "Multi = 0 is equivalent to = 1" as it
2439                          * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2440                          * doesn't work for RXOR with DMA0/1! Instead, multi=0
2441                          * leads to zeroing source data after RXOR.
2442                          * So, for P case set-up mult=1 explicitly.
2443                          */
2444                         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2445                                 mult = scf[src_cnt];
2446                         ppc440spe_adma_pq_set_src_mult(sw_desc,
2447                                 mult, src_cnt,  dst_cnt - 1);
2448                 }
2449
2450                 /* Setup byte count foreach slot just allocated */
2451                 sw_desc->async_tx.flags = flags;
2452                 list_for_each_entry(iter, &sw_desc->group_list,
2453                                 chain_node) {
2454                         ppc440spe_desc_set_byte_count(iter,
2455                                 ppc440spe_chan, len);
2456                         iter->unmap_len = len;
2457                 }
2458         }
2459         spin_unlock_bh(&ppc440spe_chan->lock);
2460
2461         return sw_desc;
2462 }
2463
2464 static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2465                 struct ppc440spe_adma_chan *ppc440spe_chan,
2466                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2467                 const unsigned char *scf, size_t len, unsigned long flags)
2468 {
2469         int slot_cnt, descs_per_op;
2470         struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2471         unsigned long op = 0;
2472         unsigned char mult = 1;
2473
2474         BUG_ON(!dst_cnt);
2475         /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2476                  __func__, dst_cnt, src_cnt, len);*/
2477
2478         spin_lock_bh(&ppc440spe_chan->lock);
2479         descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
2480         if (descs_per_op < 0) {
2481                 spin_unlock_bh(&ppc440spe_chan->lock);
2482                 return NULL;
2483         }
2484
2485         /* depending on number of sources we have 1 or 2 RXOR chains */
2486         slot_cnt = descs_per_op * dst_cnt;
2487
2488         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2489         if (sw_desc) {
2490                 op = slot_cnt;
2491                 sw_desc->async_tx.flags = flags;
2492                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2493                         ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
2494                                 --op ? 0 : flags);
2495                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2496                                 len);
2497                         iter->unmap_len = len;
2498
2499                         ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
2500                         iter->rxor_cursor.len = len;
2501                         iter->descs_per_op = descs_per_op;
2502                 }
2503                 op = 0;
2504                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2505                         op++;
2506                         if (op % descs_per_op == 0)
2507                                 ppc440spe_adma_init_dma2rxor_slot(iter, src,
2508                                                                   src_cnt);
2509                         if (likely(!list_is_last(&iter->chain_node,
2510                                                  &sw_desc->group_list))) {
2511                                 /* set 'next' pointer */
2512                                 iter->hw_next =
2513                                         list_entry(iter->chain_node.next,
2514                                                 struct ppc440spe_adma_desc_slot,
2515                                                 chain_node);
2516                                 ppc440spe_xor_set_link(iter, iter->hw_next);
2517                         } else {
2518                                 /* this is the last descriptor. */
2519                                 iter->hw_next = NULL;
2520                         }
2521                 }
2522
2523                 /* fixup head descriptor */
2524                 sw_desc->dst_cnt = dst_cnt;
2525                 if (flags & DMA_PREP_ZERO_P)
2526                         set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2527                 if (flags & DMA_PREP_ZERO_Q)
2528                         set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2529
2530                 /* setup dst/src/mult */
2531                 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2532
2533                 while (src_cnt--) {
2534                         /* handle descriptors (if dst_cnt == 2) inside
2535                          * the ppc440spe_adma_pq_set_srcxxx() functions
2536                          */
2537                         ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2538                                                   src_cnt);
2539                         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2540                                 mult = scf[src_cnt];
2541                         ppc440spe_adma_pq_set_src_mult(sw_desc,
2542                                         mult, src_cnt, dst_cnt - 1);
2543                 }
2544         }
2545         spin_unlock_bh(&ppc440spe_chan->lock);
2546         ppc440spe_desc_set_rxor_block_size(len);
2547         return sw_desc;
2548 }
2549
2550 /**
2551  * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2552  */
2553 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2554                 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2555                 unsigned int src_cnt, const unsigned char *scf,
2556                 size_t len, unsigned long flags)
2557 {
2558         struct ppc440spe_adma_chan *ppc440spe_chan;
2559         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2560         int dst_cnt = 0;
2561
2562         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2563
2564         ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2565                                     dst, src, src_cnt));
2566         BUG_ON(!len);
2567         BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2568         BUG_ON(!src_cnt);
2569
2570         if (src_cnt == 1 && dst[1] == src[0]) {
2571                 dma_addr_t dest[2];
2572
2573                 /* dst[1] is real destination (Q) */
2574                 dest[0] = dst[1];
2575                 /* this is the page to multicast source data to */
2576                 dest[1] = ppc440spe_chan->qdest;
2577                 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2578                                 dest, 2, src, src_cnt, scf, len, flags);
2579                 return sw_desc ? &sw_desc->async_tx : NULL;
2580         }
2581
2582         if (src_cnt == 2 && dst[1] == src[1]) {
2583                 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2584                                         &dst[1], src, 2, scf, len, flags);
2585                 return sw_desc ? &sw_desc->async_tx : NULL;
2586         }
2587
2588         if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2589                 BUG_ON(!dst[0]);
2590                 dst_cnt++;
2591                 flags |= DMA_PREP_ZERO_P;
2592         }
2593
2594         if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2595                 BUG_ON(!dst[1]);
2596                 dst_cnt++;
2597                 flags |= DMA_PREP_ZERO_Q;
2598         }
2599
2600         BUG_ON(!dst_cnt);
2601
2602         dev_dbg(ppc440spe_chan->device->common.dev,
2603                 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2604                 ppc440spe_chan->device->id, __func__, src_cnt, len,
2605                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2606
2607         switch (ppc440spe_chan->device->id) {
2608         case PPC440SPE_DMA0_ID:
2609         case PPC440SPE_DMA1_ID:
2610                 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2611                                 dst, dst_cnt, src, src_cnt, scf,
2612                                 len, flags);
2613                 break;
2614
2615         case PPC440SPE_XOR_ID:
2616                 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2617                                 dst, dst_cnt, src, src_cnt, scf,
2618                                 len, flags);
2619                 break;
2620         }
2621
2622         return sw_desc ? &sw_desc->async_tx : NULL;
2623 }
2624
2625 /**
2626  * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2627  * a PQ_ZERO_SUM operation
2628  */
2629 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2630                 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2631                 unsigned int src_cnt, const unsigned char *scf, size_t len,
2632                 enum sum_check_flags *pqres, unsigned long flags)
2633 {
2634         struct ppc440spe_adma_chan *ppc440spe_chan;
2635         struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2636         dma_addr_t pdest, qdest;
2637         int slot_cnt, slots_per_op, idst, dst_cnt;
2638
2639         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2640
2641         if (flags & DMA_PREP_PQ_DISABLE_P)
2642                 pdest = 0;
2643         else
2644                 pdest = pq[0];
2645
2646         if (flags & DMA_PREP_PQ_DISABLE_Q)
2647                 qdest = 0;
2648         else
2649                 qdest = pq[1];
2650
2651         ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2652                                             src, src_cnt, scf));
2653
2654         /* Always use WXOR for P/Q calculations (two destinations).
2655          * Need 1 or 2 extra slots to verify results are zero.
2656          */
2657         idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2658
2659         /* One additional slot per destination to clone P/Q
2660          * before calculation (we have to preserve destinations).
2661          */
2662         slot_cnt = src_cnt + dst_cnt * 2;
2663         slots_per_op = 1;
2664
2665         spin_lock_bh(&ppc440spe_chan->lock);
2666         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2667                                              slots_per_op);
2668         if (sw_desc) {
2669                 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2670
2671                 /* Setup byte count for each slot just allocated */
2672                 sw_desc->async_tx.flags = flags;
2673                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2674                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2675                                                       len);
2676                         iter->unmap_len = len;
2677                 }
2678
2679                 if (pdest) {
2680                         struct dma_cdb *hw_desc;
2681                         struct ppc440spe_adma_chan *chan;
2682
2683                         iter = sw_desc->group_head;
2684                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2685                         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2686                         iter->hw_next = list_entry(iter->chain_node.next,
2687                                                 struct ppc440spe_adma_desc_slot,
2688                                                 chain_node);
2689                         hw_desc = iter->hw_desc;
2690                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2691                         iter->src_cnt = 0;
2692                         iter->dst_cnt = 0;
2693                         ppc440spe_desc_set_dest_addr(iter, chan, 0,
2694                                                      ppc440spe_chan->pdest, 0);
2695                         ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
2696                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2697                                                       len);
2698                         iter->unmap_len = 0;
2699                         /* override pdest to preserve original P */
2700                         pdest = ppc440spe_chan->pdest;
2701                 }
2702                 if (qdest) {
2703                         struct dma_cdb *hw_desc;
2704                         struct ppc440spe_adma_chan *chan;
2705
2706                         iter = list_first_entry(&sw_desc->group_list,
2707                                                 struct ppc440spe_adma_desc_slot,
2708                                                 chain_node);
2709                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2710
2711                         if (pdest) {
2712                                 iter = list_entry(iter->chain_node.next,
2713                                                 struct ppc440spe_adma_desc_slot,
2714                                                 chain_node);
2715                         }
2716
2717                         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2718                         iter->hw_next = list_entry(iter->chain_node.next,
2719                                                 struct ppc440spe_adma_desc_slot,
2720                                                 chain_node);
2721                         hw_desc = iter->hw_desc;
2722                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2723                         iter->src_cnt = 0;
2724                         iter->dst_cnt = 0;
2725                         ppc440spe_desc_set_dest_addr(iter, chan, 0,
2726                                                      ppc440spe_chan->qdest, 0);
2727                         ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
2728                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2729                                                       len);
2730                         iter->unmap_len = 0;
2731                         /* override qdest to preserve original Q */
2732                         qdest = ppc440spe_chan->qdest;
2733                 }
2734
2735                 /* Setup destinations for P/Q ops */
2736                 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
2737
2738                 /* Setup zero QWORDs into DCHECK CDBs */
2739                 idst = dst_cnt;
2740                 list_for_each_entry_reverse(iter, &sw_desc->group_list,
2741                                             chain_node) {
2742                         /*
2743                          * The last CDB corresponds to Q-parity check,
2744                          * the one before last CDB corresponds
2745                          * P-parity check
2746                          */
2747                         if (idst == DMA_DEST_MAX_NUM) {
2748                                 if (idst == dst_cnt) {
2749                                         set_bit(PPC440SPE_DESC_QCHECK,
2750                                                 &iter->flags);
2751                                 } else {
2752                                         set_bit(PPC440SPE_DESC_PCHECK,
2753                                                 &iter->flags);
2754                                 }
2755                         } else {
2756                                 if (qdest) {
2757                                         set_bit(PPC440SPE_DESC_QCHECK,
2758                                                 &iter->flags);
2759                                 } else {
2760                                         set_bit(PPC440SPE_DESC_PCHECK,
2761                                                 &iter->flags);
2762                                 }
2763                         }
2764                         iter->xor_check_result = pqres;
2765
2766                         /*
2767                          * set it to zero, if check fail then result will
2768                          * be updated
2769                          */
2770                         *iter->xor_check_result = 0;
2771                         ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
2772                                 ppc440spe_qword);
2773
2774                         if (!(--dst_cnt))
2775                                 break;
2776                 }
2777
2778                 /* Setup sources and mults for P/Q ops */
2779                 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
2780                                                      chain_node) {
2781                         struct ppc440spe_adma_chan *chan;
2782                         u32 mult_dst;
2783
2784                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2785                         ppc440spe_desc_set_src_addr(iter, chan, 0,
2786                                                     DMA_CUED_XOR_HB,
2787                                                     src[src_cnt - 1]);
2788                         if (qdest) {
2789                                 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
2790                                                            DMA_CDB_SG_DST1;
2791                                 ppc440spe_desc_set_src_mult(iter, chan,
2792                                                             DMA_CUED_MULT1_OFF,
2793                                                             mult_dst,
2794                                                             scf[src_cnt - 1]);
2795                         }
2796                         if (!(--src_cnt))
2797                                 break;
2798                 }
2799         }
2800         spin_unlock_bh(&ppc440spe_chan->lock);
2801         return sw_desc ? &sw_desc->async_tx : NULL;
2802 }
2803
2804 /**
2805  * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
2806  * XOR ZERO_SUM operation
2807  */
2808 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
2809                 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
2810                 size_t len, enum sum_check_flags *result, unsigned long flags)
2811 {
2812         struct dma_async_tx_descriptor *tx;
2813         dma_addr_t pq[2];
2814
2815         /* validate P, disable Q */
2816         pq[0] = src[0];
2817         pq[1] = 0;
2818         flags |= DMA_PREP_PQ_DISABLE_Q;
2819
2820         tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
2821                                                 src_cnt - 1, 0, len,
2822                                                 result, flags);
2823         return tx;
2824 }
2825
2826 /**
2827  * ppc440spe_adma_set_dest - set destination address into descriptor
2828  */
2829 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2830                 dma_addr_t addr, int index)
2831 {
2832         struct ppc440spe_adma_chan *chan;
2833
2834         BUG_ON(index >= sw_desc->dst_cnt);
2835
2836         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2837
2838         switch (chan->device->id) {
2839         case PPC440SPE_DMA0_ID:
2840         case PPC440SPE_DMA1_ID:
2841                 /* to do: support transfers lengths >
2842                  * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
2843                  */
2844                 ppc440spe_desc_set_dest_addr(sw_desc->group_head,
2845                         chan, 0, addr, index);
2846                 break;
2847         case PPC440SPE_XOR_ID:
2848                 sw_desc = ppc440spe_get_group_entry(sw_desc, index);
2849                 ppc440spe_desc_set_dest_addr(sw_desc,
2850                         chan, 0, addr, index);
2851                 break;
2852         }
2853 }
2854
2855 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
2856                 struct ppc440spe_adma_chan *chan, dma_addr_t addr)
2857 {
2858         /*  To clear destinations update the descriptor
2859          * (P or Q depending on index) as follows:
2860          * addr is destination (0 corresponds to SG2):
2861          */
2862         ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
2863
2864         /* ... and the addr is source: */
2865         ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
2866
2867         /* addr is always SG2 then the mult is always DST1 */
2868         ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2869                                     DMA_CDB_SG_DST1, 1);
2870 }
2871
2872 /**
2873  * ppc440spe_adma_pq_set_dest - set destination address into descriptor
2874  * for the PQXOR operation
2875  */
2876 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2877                 dma_addr_t *addrs, unsigned long flags)
2878 {
2879         struct ppc440spe_adma_desc_slot *iter;
2880         struct ppc440spe_adma_chan *chan;
2881         dma_addr_t paddr, qaddr;
2882         dma_addr_t addr = 0, ppath, qpath;
2883         int index = 0, i;
2884
2885         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2886
2887         if (flags & DMA_PREP_PQ_DISABLE_P)
2888                 paddr = 0;
2889         else
2890                 paddr = addrs[0];
2891
2892         if (flags & DMA_PREP_PQ_DISABLE_Q)
2893                 qaddr = 0;
2894         else
2895                 qaddr = addrs[1];
2896
2897         if (!paddr || !qaddr)
2898                 addr = paddr ? paddr : qaddr;
2899
2900         switch (chan->device->id) {
2901         case PPC440SPE_DMA0_ID:
2902         case PPC440SPE_DMA1_ID:
2903                 /* walk through the WXOR source list and set P/Q-destinations
2904                  * for each slot:
2905                  */
2906                 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
2907                         /* This is WXOR-only chain; may have 1/2 zero descs */
2908                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
2909                                 index++;
2910                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
2911                                 index++;
2912
2913                         iter = ppc440spe_get_group_entry(sw_desc, index);
2914                         if (addr) {
2915                                 /* one destination */
2916                                 list_for_each_entry_from(iter,
2917                                         &sw_desc->group_list, chain_node)
2918                                         ppc440spe_desc_set_dest_addr(iter, chan,
2919                                                 DMA_CUED_XOR_BASE, addr, 0);
2920                         } else {
2921                                 /* two destinations */
2922                                 list_for_each_entry_from(iter,
2923                                         &sw_desc->group_list, chain_node) {
2924                                         ppc440spe_desc_set_dest_addr(iter, chan,
2925                                                 DMA_CUED_XOR_BASE, paddr, 0);
2926                                         ppc440spe_desc_set_dest_addr(iter, chan,
2927                                                 DMA_CUED_XOR_BASE, qaddr, 1);
2928                                 }
2929                         }
2930
2931                         if (index) {
2932                                 /*  To clear destinations update the descriptor
2933                                  * (1st,2nd, or both depending on flags)
2934                                  */
2935                                 index = 0;
2936                                 if (test_bit(PPC440SPE_ZERO_P,
2937                                                 &sw_desc->flags)) {
2938                                         iter = ppc440spe_get_group_entry(
2939                                                         sw_desc, index++);
2940                                         ppc440spe_adma_pq_zero_op(iter, chan,
2941                                                         paddr);
2942                                 }
2943
2944                                 if (test_bit(PPC440SPE_ZERO_Q,
2945                                                 &sw_desc->flags)) {
2946                                         iter = ppc440spe_get_group_entry(
2947                                                         sw_desc, index++);
2948                                         ppc440spe_adma_pq_zero_op(iter, chan,
2949                                                         qaddr);
2950                                 }
2951
2952                                 return;
2953                         }
2954                 } else {
2955                         /* This is RXOR-only or RXOR/WXOR mixed chain */
2956
2957                         /* If we want to include destination into calculations,
2958                          * then make dest addresses cued with mult=1 (XOR).
2959                          */
2960                         ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2961                                         DMA_CUED_XOR_HB :
2962                                         DMA_CUED_XOR_BASE |
2963                                                 (1 << DMA_CUED_MULT1_OFF);
2964                         qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2965                                         DMA_CUED_XOR_HB :
2966                                         DMA_CUED_XOR_BASE |
2967                                                 (1 << DMA_CUED_MULT1_OFF);
2968
2969                         /* Setup destination(s) in RXOR slot(s) */
2970                         iter = ppc440spe_get_group_entry(sw_desc, index++);
2971                         ppc440spe_desc_set_dest_addr(iter, chan,
2972                                                 paddr ? ppath : qpath,
2973                                                 paddr ? paddr : qaddr, 0);
2974                         if (!addr) {
2975                                 /* two destinations */
2976                                 iter = ppc440spe_get_group_entry(sw_desc,
2977                                                                  index++);
2978                                 ppc440spe_desc_set_dest_addr(iter, chan,
2979                                                 qpath, qaddr, 0);
2980                         }
2981
2982                         if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
2983                                 /* Setup destination(s) in remaining WXOR
2984                                  * slots
2985                                  */
2986                                 iter = ppc440spe_get_group_entry(sw_desc,
2987                                                                  index);
2988                                 if (addr) {
2989                                         /* one destination */
2990                                         list_for_each_entry_from(iter,
2991                                             &sw_desc->group_list,
2992                                             chain_node)
2993                                                 ppc440spe_desc_set_dest_addr(
2994                                                         iter, chan,
2995                                                         DMA_CUED_XOR_BASE,
2996                                                         addr, 0);
2997
2998                                 } else {
2999                                         /* two destinations */
3000                                         list_for_each_entry_from(iter,
3001                                             &sw_desc->group_list,
3002                                             chain_node) {
3003                                                 ppc440spe_desc_set_dest_addr(
3004                                                         iter, chan,
3005                                                         DMA_CUED_XOR_BASE,
3006                                                         paddr, 0);
3007                                                 ppc440spe_desc_set_dest_addr(
3008                                                         iter, chan,
3009                                                         DMA_CUED_XOR_BASE,
3010                                                         qaddr, 1);
3011                                         }
3012                                 }
3013                         }
3014
3015                 }
3016                 break;
3017
3018         case PPC440SPE_XOR_ID:
3019                 /* DMA2 descriptors have only 1 destination, so there are
3020                  * two chains - one for each dest.
3021                  * If we want to include destination into calculations,
3022                  * then make dest addresses cued with mult=1 (XOR).
3023                  */
3024                 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
3025                                 DMA_CUED_XOR_HB :
3026                                 DMA_CUED_XOR_BASE |
3027                                         (1 << DMA_CUED_MULT1_OFF);
3028
3029                 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
3030                                 DMA_CUED_XOR_HB :
3031                                 DMA_CUED_XOR_BASE |
3032                                         (1 << DMA_CUED_MULT1_OFF);
3033
3034                 iter = ppc440spe_get_group_entry(sw_desc, 0);
3035                 for (i = 0; i < sw_desc->descs_per_op; i++) {
3036                         ppc440spe_desc_set_dest_addr(iter, chan,
3037                                 paddr ? ppath : qpath,
3038                                 paddr ? paddr : qaddr, 0);
3039                         iter = list_entry(iter->chain_node.next,
3040                                           struct ppc440spe_adma_desc_slot,
3041                                           chain_node);
3042                 }
3043
3044                 if (!addr) {
3045                         /* Two destinations; setup Q here */
3046                         iter = ppc440spe_get_group_entry(sw_desc,
3047                                 sw_desc->descs_per_op);
3048                         for (i = 0; i < sw_desc->descs_per_op; i++) {
3049                                 ppc440spe_desc_set_dest_addr(iter,
3050                                         chan, qpath, qaddr, 0);
3051                                 iter = list_entry(iter->chain_node.next,
3052                                                 struct ppc440spe_adma_desc_slot,
3053                                                 chain_node);
3054                         }
3055                 }
3056
3057                 break;
3058         }
3059 }
3060
3061 /**
3062  * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3063  * for the PQ_ZERO_SUM operation
3064  */
3065 static void ppc440spe_adma_pqzero_sum_set_dest(
3066                 struct ppc440spe_adma_desc_slot *sw_desc,
3067                 dma_addr_t paddr, dma_addr_t qaddr)
3068 {
3069         struct ppc440spe_adma_desc_slot *iter, *end;
3070         struct ppc440spe_adma_chan *chan;
3071         dma_addr_t addr = 0;
3072         int idx;
3073
3074         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3075
3076         /* walk through the WXOR source list and set P/Q-destinations
3077          * for each slot
3078          */
3079         idx = (paddr && qaddr) ? 2 : 1;
3080         /* set end */
3081         list_for_each_entry_reverse(end, &sw_desc->group_list,
3082                                     chain_node) {
3083                 if (!(--idx))
3084                         break;
3085         }
3086         /* set start */
3087         idx = (paddr && qaddr) ? 2 : 1;
3088         iter = ppc440spe_get_group_entry(sw_desc, idx);
3089
3090         if (paddr && qaddr) {
3091                 /* two destinations */
3092                 list_for_each_entry_from(iter, &sw_desc->group_list,
3093                                          chain_node) {
3094                         if (unlikely(iter == end))
3095                                 break;
3096                         ppc440spe_desc_set_dest_addr(iter, chan,
3097                                                 DMA_CUED_XOR_BASE, paddr, 0);
3098                         ppc440spe_desc_set_dest_addr(iter, chan,
3099                                                 DMA_CUED_XOR_BASE, qaddr, 1);
3100                 }
3101         } else {
3102                 /* one destination */
3103                 addr = paddr ? paddr : qaddr;
3104                 list_for_each_entry_from(iter, &sw_desc->group_list,
3105                                          chain_node) {
3106                         if (unlikely(iter == end))
3107                                 break;
3108                         ppc440spe_desc_set_dest_addr(iter, chan,
3109                                                 DMA_CUED_XOR_BASE, addr, 0);
3110                 }
3111         }
3112
3113         /*  The remaining descriptors are DATACHECK. These have no need in
3114          * destination. Actually, these destinations are used there
3115          * as sources for check operation. So, set addr as source.
3116          */
3117         ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
3118
3119         if (!addr) {
3120                 end = list_entry(end->chain_node.next,
3121                                  struct ppc440spe_adma_desc_slot, chain_node);
3122                 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
3123         }
3124 }
3125
3126 /**
3127  * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3128  */
3129 static inline void ppc440spe_desc_set_xor_src_cnt(
3130                         struct ppc440spe_adma_desc_slot *desc,
3131                         int src_cnt)
3132 {
3133         struct xor_cb *hw_desc = desc->hw_desc;
3134
3135         hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3136         hw_desc->cbc |= src_cnt;
3137 }
3138
3139 /**
3140  * ppc440spe_adma_pq_set_src - set source address into descriptor
3141  */
3142 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3143                 dma_addr_t addr, int index)
3144 {
3145         struct ppc440spe_adma_chan *chan;
3146         dma_addr_t haddr = 0;
3147         struct ppc440spe_adma_desc_slot *iter = NULL;
3148
3149         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3150
3151         switch (chan->device->id) {
3152         case PPC440SPE_DMA0_ID:
3153         case PPC440SPE_DMA1_ID:
3154                 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3155                  */
3156                 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3157                         /* RXOR-only or RXOR/WXOR operation */
3158                         int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3159                                 &sw_desc->flags) ?  2 : 3;
3160
3161                         if (index == 0) {
3162                                 /* 1st slot (RXOR) */
3163                                 /* setup sources region (R1-2-3, R1-2-4,
3164                                  * or R1-2-5)
3165                                  */
3166                                 if (test_bit(PPC440SPE_DESC_RXOR12,
3167                                                 &sw_desc->flags))
3168                                         haddr = DMA_RXOR12 <<
3169                                                 DMA_CUED_REGION_OFF;
3170                                 else if (test_bit(PPC440SPE_DESC_RXOR123,
3171                                     &sw_desc->flags))
3172                                         haddr = DMA_RXOR123 <<
3173                                                 DMA_CUED_REGION_OFF;
3174                                 else if (test_bit(PPC440SPE_DESC_RXOR124,
3175                                     &sw_desc->flags))
3176                                         haddr = DMA_RXOR124 <<
3177                                                 DMA_CUED_REGION_OFF;
3178                                 else if (test_bit(PPC440SPE_DESC_RXOR125,
3179                                     &sw_desc->flags))
3180                                         haddr = DMA_RXOR125 <<
3181                                                 DMA_CUED_REGION_OFF;
3182                                 else
3183                                         BUG();
3184                                 haddr |= DMA_CUED_XOR_BASE;
3185                                 iter = ppc440spe_get_group_entry(sw_desc, 0);
3186                         } else if (index < iskip) {
3187                                 /* 1st slot (RXOR)
3188                                  * shall actually set source address only once
3189                                  * instead of first <iskip>
3190                                  */
3191                                 iter = NULL;
3192                         } else {
3193                                 /* 2nd/3d and next slots (WXOR);
3194                                  * skip first slot with RXOR
3195                                  */
3196                                 haddr = DMA_CUED_XOR_HB;
3197                                 iter = ppc440spe_get_group_entry(sw_desc,
3198                                     index - iskip + sw_desc->dst_cnt);
3199                         }
3200                 } else {
3201                         int znum = 0;
3202
3203                         /* WXOR-only operation; skip first slots with
3204                          * zeroing destinations
3205                          */
3206                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3207                                 znum++;
3208                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3209                                 znum++;
3210
3211                         haddr = DMA_CUED_XOR_HB;
3212                         iter = ppc440spe_get_group_entry(sw_desc,
3213                                         index + znum);
3214                 }
3215
3216                 if (likely(iter)) {
3217                         ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
3218
3219                         if (!index &&
3220                             test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3221                             sw_desc->dst_cnt == 2) {
3222                                 /* if we have two destinations for RXOR, then
3223                                  * setup source in the second descr too
3224                                  */
3225                                 iter = ppc440spe_get_group_entry(sw_desc, 1);
3226                                 ppc440spe_desc_set_src_addr(iter, chan, 0,
3227                                         haddr, addr);
3228                         }
3229                 }
3230                 break;
3231
3232         case PPC440SPE_XOR_ID:
3233                 /* DMA2 may do Biskup */
3234                 iter = sw_desc->group_head;
3235                 if (iter->dst_cnt == 2) {
3236                         /* both P & Q calculations required; set P src here */
3237                         ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3238
3239                         /* this is for Q */
3240                         iter = ppc440spe_get_group_entry(sw_desc,
3241                                 sw_desc->descs_per_op);
3242                 }
3243                 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3244                 break;
3245         }
3246 }
3247
3248 /**
3249  * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3250  */
3251 static void ppc440spe_adma_memcpy_xor_set_src(
3252                 struct ppc440spe_adma_desc_slot *sw_desc,
3253                 dma_addr_t addr, int index)
3254 {
3255         struct ppc440spe_adma_chan *chan;
3256
3257         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3258         sw_desc = sw_desc->group_head;
3259
3260         if (likely(sw_desc))
3261                 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3262 }
3263
3264 /**
3265  * ppc440spe_adma_dma2rxor_inc_addr  -
3266  */
3267 static void ppc440spe_adma_dma2rxor_inc_addr(
3268                 struct ppc440spe_adma_desc_slot *desc,
3269                 struct ppc440spe_rxor *cursor, int index, int src_cnt)
3270 {
3271         cursor->addr_count++;
3272         if (index == src_cnt - 1) {
3273                 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3274         } else if (cursor->addr_count == XOR_MAX_OPS) {
3275                 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3276                 cursor->addr_count = 0;
3277                 cursor->desc_count++;
3278         }
3279 }
3280
3281 /**
3282  * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3283  */
3284 static int ppc440spe_adma_dma2rxor_prep_src(
3285                 struct ppc440spe_adma_desc_slot *hdesc,
3286                 struct ppc440spe_rxor *cursor, int index,
3287                 int src_cnt, u32 addr)
3288 {
3289         int rval = 0;
3290         u32 sign;
3291         struct ppc440spe_adma_desc_slot *desc = hdesc;
3292         int i;
3293
3294         for (i = 0; i < cursor->desc_count; i++) {
3295                 desc = list_entry(hdesc->chain_node.next,
3296                                   struct ppc440spe_adma_desc_slot,
3297                                   chain_node);
3298         }
3299
3300         switch (cursor->state) {
3301         case 0:
3302                 if (addr == cursor->addrl + cursor->len) {
3303                         /* direct RXOR */
3304                         cursor->state = 1;
3305                         cursor->xor_count++;
3306                         if (index == src_cnt-1) {
3307                                 ppc440spe_rxor_set_region(desc,
3308                                         cursor->addr_count,
3309                                         DMA_RXOR12 << DMA_CUED_REGION_OFF);
3310                                 ppc440spe_adma_dma2rxor_inc_addr(
3311                                         desc, cursor, index, src_cnt);
3312                         }
3313                 } else if (cursor->addrl == addr + cursor->len) {
3314                         /* reverse RXOR */
3315                         cursor->state = 1;
3316                         cursor->xor_count++;
3317                         set_bit(cursor->addr_count, &desc->reverse_flags[0]);
3318                         if (index == src_cnt-1) {
3319                                 ppc440spe_rxor_set_region(desc,
3320                                         cursor->addr_count,
3321                                         DMA_RXOR12 << DMA_CUED_REGION_OFF);
3322                                 ppc440spe_adma_dma2rxor_inc_addr(
3323                                         desc, cursor, index, src_cnt);
3324                         }
3325                 } else {
3326                         printk(KERN_ERR "Cannot build "
3327                                 "DMA2 RXOR command block.\n");
3328                         BUG();
3329                 }
3330                 break;
3331         case 1:
3332                 sign = test_bit(cursor->addr_count,
3333                                 desc->reverse_flags)
3334                         ? -1 : 1;
3335                 if (index == src_cnt-2 || (sign == -1
3336                         && addr != cursor->addrl - 2*cursor->len)) {
3337                         cursor->state = 0;
3338                         cursor->xor_count = 1;
3339                         cursor->addrl = addr;
3340                         ppc440spe_rxor_set_region(desc,
3341                                 cursor->addr_count,
3342                                 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3343                         ppc440spe_adma_dma2rxor_inc_addr(
3344                                 desc, cursor, index, src_cnt);
3345                 } else if (addr == cursor->addrl + 2*sign*cursor->len) {
3346                         cursor->state = 2;
3347                         cursor->xor_count = 0;
3348                         ppc440spe_rxor_set_region(desc,
3349                                 cursor->addr_count,
3350                                 DMA_RXOR123 << DMA_CUED_REGION_OFF);
3351                         if (index == src_cnt-1) {
3352                                 ppc440spe_adma_dma2rxor_inc_addr(
3353                                         desc, cursor, index, src_cnt);
3354                         }
3355                 } else if (addr == cursor->addrl + 3*cursor->len) {
3356                         cursor->state = 2;
3357                         cursor->xor_count = 0;
3358                         ppc440spe_rxor_set_region(desc,
3359                                 cursor->addr_count,
3360                                 DMA_RXOR124 << DMA_CUED_REGION_OFF);
3361                         if (index == src_cnt-1) {
3362                                 ppc440spe_adma_dma2rxor_inc_addr(
3363                                         desc, cursor, index, src_cnt);
3364                         }
3365                 } else if (addr == cursor->addrl + 4*cursor->len) {
3366                         cursor->state = 2;
3367                         cursor->xor_count = 0;
3368                         ppc440spe_rxor_set_region(desc,
3369                                 cursor->addr_count,
3370                                 DMA_RXOR125 << DMA_CUED_REGION_OFF);
3371                         if (index == src_cnt-1) {
3372                                 ppc440spe_adma_dma2rxor_inc_addr(
3373                                         desc, cursor, index, src_cnt);
3374                         }
3375                 } else {
3376                         cursor->state = 0;
3377                         cursor->xor_count = 1;
3378                         cursor->addrl = addr;
3379                         ppc440spe_rxor_set_region(desc,
3380                                 cursor->addr_count,
3381                                 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3382                         ppc440spe_adma_dma2rxor_inc_addr(
3383                                 desc, cursor, index, src_cnt);
3384                 }
3385                 break;
3386         case 2:
3387                 cursor->state = 0;
3388                 cursor->addrl = addr;
3389                 cursor->xor_count++;
3390                 if (index) {
3391                         ppc440spe_adma_dma2rxor_inc_addr(
3392                                 desc, cursor, index, src_cnt);
3393                 }
3394                 break;
3395         }
3396
3397         return rval;
3398 }
3399
3400 /**
3401  * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3402  *      ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3403  */
3404 static void ppc440spe_adma_dma2rxor_set_src(
3405                 struct ppc440spe_adma_desc_slot *desc,
3406                 int index, dma_addr_t addr)
3407 {
3408         struct xor_cb *xcb = desc->hw_desc;
3409         int k = 0, op = 0, lop = 0;
3410
3411         /* get the RXOR operand which corresponds to index addr */
3412         while (op <= index) {
3413                 lop = op;
3414                 if (k == XOR_MAX_OPS) {
3415                         k = 0;
3416                         desc = list_entry(desc->chain_node.next,
3417                                 struct ppc440spe_adma_desc_slot, chain_node);
3418                         xcb = desc->hw_desc;
3419
3420                 }
3421                 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3422                     (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3423                         op += 2;
3424                 else
3425                         op += 3;
3426         }
3427
3428         BUG_ON(k < 1);
3429
3430         if (test_bit(k-1, desc->reverse_flags)) {
3431                 /* reverse operand order; put last op in RXOR group */
3432                 if (index == op - 1)
3433                         ppc440spe_rxor_set_src(desc, k - 1, addr);
3434         } else {
3435                 /* direct operand order; put first op in RXOR group */
3436                 if (index == lop)
3437                         ppc440spe_rxor_set_src(desc, k - 1, addr);
3438         }
3439 }
3440
3441 /**
3442  * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3443  *      ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3444  */
3445 static void ppc440spe_adma_dma2rxor_set_mult(
3446                 struct ppc440spe_adma_desc_slot *desc,
3447                 int index, u8 mult)
3448 {
3449         struct xor_cb *xcb = desc->hw_desc;
3450         int k = 0, op = 0, lop = 0;
3451
3452         /* get the RXOR operand which corresponds to index mult */
3453         while (op <= index) {
3454                 lop = op;
3455                 if (k == XOR_MAX_OPS) {
3456                         k = 0;
3457                         desc = list_entry(desc->chain_node.next,
3458                                           struct ppc440spe_adma_desc_slot,
3459                                           chain_node);
3460                         xcb = desc->hw_desc;
3461
3462                 }
3463                 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3464                     (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3465                         op += 2;
3466                 else
3467                         op += 3;
3468         }
3469
3470         BUG_ON(k < 1);
3471         if (test_bit(k-1, desc->reverse_flags)) {
3472                 /* reverse order */
3473                 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
3474         } else {
3475                 /* direct order */
3476                 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
3477         }
3478 }
3479
3480 /**
3481  * ppc440spe_init_rxor_cursor -
3482  */
3483 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3484 {
3485         memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3486         cursor->state = 2;
3487 }
3488
3489 /**
3490  * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3491  * descriptor for the PQXOR operation
3492  */
3493 static void ppc440spe_adma_pq_set_src_mult(
3494                 struct ppc440spe_adma_desc_slot *sw_desc,
3495                 unsigned char mult, int index, int dst_pos)
3496 {
3497         struct ppc440spe_adma_chan *chan;
3498         u32 mult_idx, mult_dst;
3499         struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3500
3501         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3502
3503         switch (chan->device->id) {
3504         case PPC440SPE_DMA0_ID:
3505         case PPC440SPE_DMA1_ID:
3506                 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3507                         int region = test_bit(PPC440SPE_DESC_RXOR12,
3508                                         &sw_desc->flags) ? 2 : 3;
3509
3510                         if (index < region) {
3511                                 /* RXOR multipliers */
3512                                 iter = ppc440spe_get_group_entry(sw_desc,
3513                                         sw_desc->dst_cnt - 1);
3514                                 if (sw_desc->dst_cnt == 2)
3515                                         iter1 = ppc440spe_get_group_entry(
3516                                                         sw_desc, 0);
3517
3518                                 mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3519                                 mult_dst = DMA_CDB_SG_SRC;
3520                         } else {
3521                                 /* WXOR multiplier */
3522                                 iter = ppc440spe_get_group_entry(sw_desc,
3523                                                         index - region +
3524                                                         sw_desc->dst_cnt);
3525                                 mult_idx = DMA_CUED_MULT1_OFF;
3526                                 mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3527                                                      DMA_CDB_SG_DST1;
3528                         }
3529                 } else {
3530                         int znum = 0;
3531
3532                         /* WXOR-only;
3533                          * skip first slots with destinations (if ZERO_DST has
3534                          * place)
3535                          */
3536                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3537                                 znum++;
3538                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3539                                 znum++;
3540
3541                         iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3542                         mult_idx = DMA_CUED_MULT1_OFF;
3543                         mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3544                 }
3545
3546                 if (likely(iter)) {
3547                         ppc440spe_desc_set_src_mult(iter, chan,
3548                                 mult_idx, mult_dst, mult);
3549
3550                         if (unlikely(iter1)) {
3551                                 /* if we have two destinations for RXOR, then
3552                                  * we've just set Q mult. Set-up P now.
3553                                  */
3554                                 ppc440spe_desc_set_src_mult(iter1, chan,
3555                                         mult_idx, mult_dst, 1);
3556                         }
3557
3558                 }
3559                 break;
3560
3561         case PPC440SPE_XOR_ID:
3562                 iter = sw_desc->group_head;
3563                 if (sw_desc->dst_cnt == 2) {
3564                         /* both P & Q calculations required; set P mult here */
3565                         ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
3566
3567                         /* and then set Q mult */
3568                         iter = ppc440spe_get_group_entry(sw_desc,
3569                                sw_desc->descs_per_op);
3570                 }
3571                 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
3572                 break;
3573         }
3574 }
3575
3576 /**
3577  * ppc440spe_adma_free_chan_resources - free the resources allocated
3578  */
3579 static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3580 {
3581         struct ppc440spe_adma_chan *ppc440spe_chan;
3582         struct ppc440spe_adma_desc_slot *iter, *_iter;
3583         int in_use_descs = 0;
3584
3585         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3586         ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3587
3588         spin_lock_bh(&ppc440spe_chan->lock);
3589         list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3590                                         chain_node) {
3591                 in_use_descs++;
3592                 list_del(&iter->chain_node);
3593         }
3594         list_for_each_entry_safe_reverse(iter, _iter,
3595                         &ppc440spe_chan->all_slots, slot_node) {
3596                 list_del(&iter->slot_node);
3597                 kfree(iter);
3598                 ppc440spe_chan->slots_allocated--;
3599         }
3600         ppc440spe_chan->last_used = NULL;
3601
3602         dev_dbg(ppc440spe_chan->device->common.dev,
3603                 "ppc440spe adma%d %s slots_allocated %d\n",
3604                 ppc440spe_chan->device->id,
3605                 __func__, ppc440spe_chan->slots_allocated);
3606         spin_unlock_bh(&ppc440spe_chan->lock);
3607
3608         /* one is ok since we left it on there on purpose */
3609         if (in_use_descs > 1)
3610                 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3611                         in_use_descs - 1);
3612 }
3613
3614 /**
3615  * ppc440spe_adma_tx_status - poll the status of an ADMA transaction
3616  * @chan: ADMA channel handle
3617  * @cookie: ADMA transaction identifier
3618  * @txstate: a holder for the current state of the channel
3619  */
3620 static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan,
3621                         dma_cookie_t cookie, struct dma_tx_state *txstate)
3622 {
3623         struct ppc440spe_adma_chan *ppc440spe_chan;
3624         enum dma_status ret;
3625
3626         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3627         ret = dma_cookie_status(chan, cookie, txstate);
3628         if (ret == DMA_COMPLETE)
3629                 return ret;
3630
3631         ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3632
3633         return dma_cookie_status(chan, cookie, txstate);
3634 }
3635
3636 /**
3637  * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3638  */
3639 static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3640 {
3641         struct ppc440spe_adma_chan *chan = data;
3642
3643         dev_dbg(chan->device->common.dev,
3644                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3645
3646         tasklet_schedule(&chan->irq_tasklet);
3647         ppc440spe_adma_device_clear_eot_status(chan);
3648
3649         return IRQ_HANDLED;
3650 }
3651
3652 /**
3653  * ppc440spe_adma_err_handler - DMA error interrupt handler;
3654  *      do the same things as a eot handler
3655  */
3656 static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3657 {
3658         struct ppc440spe_adma_chan *chan = data;
3659
3660         dev_dbg(chan->device->common.dev,
3661                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3662
3663         tasklet_schedule(&chan->irq_tasklet);
3664         ppc440spe_adma_device_clear_eot_status(chan);
3665
3666         return IRQ_HANDLED;
3667 }
3668
3669 /**
3670  * ppc440spe_test_callback - called when test operation has been done
3671  */
3672 static void ppc440spe_test_callback(void *unused)
3673 {
3674         complete(&ppc440spe_r6_test_comp);
3675 }
3676
3677 /**
3678  * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
3679  */
3680 static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
3681 {
3682         struct ppc440spe_adma_chan *ppc440spe_chan;
3683
3684         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3685         dev_dbg(ppc440spe_chan->device->common.dev,
3686                 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
3687                 __func__, ppc440spe_chan->pending);
3688
3689         if (ppc440spe_chan->pending) {
3690                 ppc440spe_chan->pending = 0;
3691                 ppc440spe_chan_append(ppc440spe_chan);
3692         }
3693 }
3694
3695 /**
3696  * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
3697  *      use FIFOs (as opposite to chains used in XOR) so this is a XOR
3698  *      specific operation)
3699  */
3700 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
3701 {
3702         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
3703         dma_cookie_t cookie;
3704         int slot_cnt, slots_per_op;
3705
3706         dev_dbg(chan->device->common.dev,
3707                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3708
3709         spin_lock_bh(&chan->lock);
3710         slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
3711         sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
3712         if (sw_desc) {
3713                 group_start = sw_desc->group_head;
3714                 list_splice_init(&sw_desc->group_list, &chan->chain);
3715                 async_tx_ack(&sw_desc->async_tx);
3716                 ppc440spe_desc_init_null_xor(group_start);
3717
3718                 cookie = dma_cookie_assign(&sw_desc->async_tx);
3719
3720                 /* initialize the completed cookie to be less than
3721                  * the most recently used cookie
3722                  */
3723                 chan->common.completed_cookie = cookie - 1;
3724
3725                 /* channel should not be busy */
3726                 BUG_ON(ppc440spe_chan_is_busy(chan));
3727
3728                 /* set the descriptor address */
3729                 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
3730
3731                 /* run the descriptor */
3732                 ppc440spe_chan_run(chan);
3733         } else
3734                 printk(KERN_ERR "ppc440spe adma%d"
3735                         " failed to allocate null descriptor\n",
3736                         chan->device->id);
3737         spin_unlock_bh(&chan->lock);
3738 }
3739
3740 /**
3741  * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
3742  *      For this we just perform one WXOR operation with the same source
3743  *      and destination addresses, the GF-multiplier is 1; so if RAID-6
3744  *      capabilities are enabled then we'll get src/dst filled with zero.
3745  */
3746 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
3747 {
3748         struct ppc440spe_adma_desc_slot *sw_desc, *iter;
3749         struct page *pg;
3750         char *a;
3751         dma_addr_t dma_addr, addrs[2];
3752         unsigned long op = 0;
3753         int rval = 0;
3754
3755         set_bit(PPC440SPE_DESC_WXOR, &op);
3756
3757         pg = alloc_page(GFP_KERNEL);
3758         if (!pg)
3759                 return -ENOMEM;
3760
3761         spin_lock_bh(&chan->lock);
3762         sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
3763         if (sw_desc) {
3764                 /* 1 src, 1 dsr, int_ena, WXOR */
3765                 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
3766                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
3767                         ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
3768                         iter->unmap_len = PAGE_SIZE;
3769                 }
3770         } else {
3771                 rval = -EFAULT;
3772                 spin_unlock_bh(&chan->lock);
3773                 goto exit;
3774         }
3775         spin_unlock_bh(&chan->lock);
3776
3777         /* Fill the test page with ones */
3778         memset(page_address(pg), 0xFF, PAGE_SIZE);
3779         dma_addr = dma_map_page(chan->device->dev, pg, 0,
3780                                 PAGE_SIZE, DMA_BIDIRECTIONAL);
3781
3782         /* Setup addresses */
3783         ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
3784         ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
3785         addrs[0] = dma_addr;
3786         addrs[1] = 0;
3787         ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
3788
3789         async_tx_ack(&sw_desc->async_tx);
3790         sw_desc->async_tx.callback = ppc440spe_test_callback;
3791         sw_desc->async_tx.callback_param = NULL;
3792
3793         init_completion(&ppc440spe_r6_test_comp);
3794
3795         ppc440spe_adma_tx_submit(&sw_desc->async_tx);
3796         ppc440spe_adma_issue_pending(&chan->common);
3797
3798         wait_for_completion(&ppc440spe_r6_test_comp);
3799
3800         /* Now check if the test page is zeroed */
3801         a = page_address(pg);
3802         if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
3803                 /* page is zero - RAID-6 enabled */
3804                 rval = 0;
3805         } else {
3806                 /* RAID-6 was not enabled */
3807                 rval = -EINVAL;
3808         }
3809 exit:
3810         __free_page(pg);
3811         return rval;
3812 }
3813
3814 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
3815 {
3816         switch (adev->id) {
3817         case PPC440SPE_DMA0_ID:
3818         case PPC440SPE_DMA1_ID:
3819                 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
3820                 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3821                 dma_cap_set(DMA_PQ, adev->common.cap_mask);
3822                 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
3823                 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
3824                 break;
3825         case PPC440SPE_XOR_ID:
3826                 dma_cap_set(DMA_XOR, adev->common.cap_mask);
3827                 dma_cap_set(DMA_PQ, adev->common.cap_mask);
3828                 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3829                 adev->common.cap_mask = adev->common.cap_mask;
3830                 break;
3831         }
3832
3833         /* Set base routines */
3834         adev->common.device_alloc_chan_resources =
3835                                 ppc440spe_adma_alloc_chan_resources;
3836         adev->common.device_free_chan_resources =
3837                                 ppc440spe_adma_free_chan_resources;
3838         adev->common.device_tx_status = ppc440spe_adma_tx_status;
3839         adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
3840
3841         /* Set prep routines based on capability */
3842         if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
3843                 adev->common.device_prep_dma_memcpy =
3844                         ppc440spe_adma_prep_dma_memcpy;
3845         }
3846         if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
3847                 adev->common.max_xor = XOR_MAX_OPS;
3848                 adev->common.device_prep_dma_xor =
3849                         ppc440spe_adma_prep_dma_xor;
3850         }
3851         if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
3852                 switch (adev->id) {
3853                 case PPC440SPE_DMA0_ID:
3854                         dma_set_maxpq(&adev->common,
3855                                 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3856                         break;
3857                 case PPC440SPE_DMA1_ID:
3858                         dma_set_maxpq(&adev->common,
3859                                 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3860                         break;
3861                 case PPC440SPE_XOR_ID:
3862                         adev->common.max_pq = XOR_MAX_OPS * 3;
3863                         break;
3864                 }
3865                 adev->common.device_prep_dma_pq =
3866                         ppc440spe_adma_prep_dma_pq;
3867         }
3868         if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
3869                 switch (adev->id) {
3870                 case PPC440SPE_DMA0_ID:
3871                         adev->common.max_pq = DMA0_FIFO_SIZE /
3872                                                 sizeof(struct dma_cdb);
3873                         break;
3874                 case PPC440SPE_DMA1_ID:
3875                         adev->common.max_pq = DMA1_FIFO_SIZE /
3876                                                 sizeof(struct dma_cdb);
3877                         break;
3878                 }
3879                 adev->common.device_prep_dma_pq_val =
3880                         ppc440spe_adma_prep_dma_pqzero_sum;
3881         }
3882         if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
3883                 switch (adev->id) {
3884                 case PPC440SPE_DMA0_ID:
3885                         adev->common.max_xor = DMA0_FIFO_SIZE /
3886                                                 sizeof(struct dma_cdb);
3887                         break;
3888                 case PPC440SPE_DMA1_ID:
3889                         adev->common.max_xor = DMA1_FIFO_SIZE /
3890                                                 sizeof(struct dma_cdb);
3891                         break;
3892                 }
3893                 adev->common.device_prep_dma_xor_val =
3894                         ppc440spe_adma_prep_dma_xor_zero_sum;
3895         }
3896         if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
3897                 adev->common.device_prep_dma_interrupt =
3898                         ppc440spe_adma_prep_dma_interrupt;
3899         }
3900         pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
3901           "( %s%s%s%s%s%s%s)\n",
3902           dev_name(adev->dev),
3903           dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
3904           dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
3905           dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
3906           dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
3907           dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
3908           dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
3909 }
3910
3911 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
3912                                      struct ppc440spe_adma_chan *chan,
3913                                      int *initcode)
3914 {
3915         struct platform_device *ofdev;
3916         struct device_node *np;
3917         int ret;
3918
3919         ofdev = container_of(adev->dev, struct platform_device, dev);
3920         np = ofdev->dev.of_node;
3921         if (adev->id != PPC440SPE_XOR_ID) {
3922                 adev->err_irq = irq_of_parse_and_map(np, 1);
3923                 if (adev->err_irq == NO_IRQ) {
3924                         dev_warn(adev->dev, "no err irq resource?\n");
3925                         *initcode = PPC_ADMA_INIT_IRQ2;
3926                         adev->err_irq = -ENXIO;
3927                 } else
3928                         atomic_inc(&ppc440spe_adma_err_irq_ref);
3929         } else {
3930                 adev->err_irq = -ENXIO;
3931         }
3932
3933         adev->irq = irq_of_parse_and_map(np, 0);
3934         if (adev->irq == NO_IRQ) {
3935                 dev_err(adev->dev, "no irq resource\n");
3936                 *initcode = PPC_ADMA_INIT_IRQ1;
3937                 ret = -ENXIO;
3938                 goto err_irq_map;
3939         }
3940         dev_dbg(adev->dev, "irq %d, err irq %d\n",
3941                 adev->irq, adev->err_irq);
3942
3943         ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
3944                           0, dev_driver_string(adev->dev), chan);
3945         if (ret) {
3946                 dev_err(adev->dev, "can't request irq %d\n",
3947                         adev->irq);
3948                 *initcode = PPC_ADMA_INIT_IRQ1;
3949                 ret = -EIO;
3950                 goto err_req1;
3951         }
3952
3953         /* only DMA engines have a separate error IRQ
3954          * so it's Ok if err_irq < 0 in XOR engine case.
3955          */
3956         if (adev->err_irq > 0) {
3957                 /* both DMA engines share common error IRQ */
3958                 ret = request_irq(adev->err_irq,
3959                                   ppc440spe_adma_err_handler,
3960                                   IRQF_SHARED,
3961                                   dev_driver_string(adev->dev),
3962                                   chan);
3963                 if (ret) {
3964                         dev_err(adev->dev, "can't request irq %d\n",
3965                                 adev->err_irq);
3966                         *initcode = PPC_ADMA_INIT_IRQ2;
3967                         ret = -EIO;
3968                         goto err_req2;
3969                 }
3970         }
3971
3972         if (adev->id == PPC440SPE_XOR_ID) {
3973                 /* enable XOR engine interrupts */
3974                 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3975                             XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
3976                             &adev->xor_reg->ier);
3977         } else {
3978                 u32 mask, enable;
3979
3980                 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
3981                 if (!np) {
3982                         pr_err("%s: can't find I2O device tree node\n",
3983                                 __func__);
3984                         ret = -ENODEV;
3985                         goto err_req2;
3986                 }
3987                 adev->i2o_reg = of_iomap(np, 0);
3988                 if (!adev->i2o_reg) {
3989                         pr_err("%s: failed to map I2O registers\n", __func__);
3990                         of_node_put(np);
3991                         ret = -EINVAL;
3992                         goto err_req2;
3993                 }
3994                 of_node_put(np);
3995                 /* Unmask 'CS FIFO Attention' interrupts and
3996                  * enable generating interrupts on errors
3997                  */
3998                 enable = (adev->id == PPC440SPE_DMA0_ID) ?
3999                          ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
4000                          ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
4001                 mask = ioread32(&adev->i2o_reg->iopim) & enable;
4002                 iowrite32(mask, &adev->i2o_reg->iopim);
4003         }
4004         return 0;
4005
4006 err_req2:
4007         free_irq(adev->irq, chan);
4008 err_req1:
4009         irq_dispose_mapping(adev->irq);
4010 err_irq_map:
4011         if (adev->err_irq > 0) {
4012                 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
4013                         irq_dispose_mapping(adev->err_irq);
4014         }
4015         return ret;
4016 }
4017
4018 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
4019                                         struct ppc440spe_adma_chan *chan)
4020 {
4021         u32 mask, disable;
4022
4023         if (adev->id == PPC440SPE_XOR_ID) {
4024                 /* disable XOR engine interrupts */
4025                 mask = ioread32be(&adev->xor_reg->ier);
4026                 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
4027                           XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
4028                 iowrite32be(mask, &adev->xor_reg->ier);
4029         } else {
4030                 /* disable DMAx engine interrupts */
4031                 disable = (adev->id == PPC440SPE_DMA0_ID) ?
4032                           (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
4033                           (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
4034                 mask = ioread32(&adev->i2o_reg->iopim) | disable;
4035                 iowrite32(mask, &adev->i2o_reg->iopim);
4036         }
4037         free_irq(adev->irq, chan);
4038         irq_dispose_mapping(adev->irq);
4039         if (adev->err_irq > 0) {
4040                 free_irq(adev->err_irq, chan);
4041                 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
4042                         irq_dispose_mapping(adev->err_irq);
4043                         iounmap(adev->i2o_reg);
4044                 }
4045         }
4046 }
4047
4048 /**
4049  * ppc440spe_adma_probe - probe the asynch device
4050  */
4051 static int ppc440spe_adma_probe(struct platform_device *ofdev)
4052 {
4053         struct device_node *np = ofdev->dev.of_node;
4054         struct resource res;
4055         struct ppc440spe_adma_device *adev;
4056         struct ppc440spe_adma_chan *chan;
4057         struct ppc_dma_chan_ref *ref, *_ref;
4058         int ret = 0, initcode = PPC_ADMA_INIT_OK;
4059         const u32 *idx;
4060         int len;
4061         void *regs;
4062         u32 id, pool_size;
4063
4064         if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
4065                 id = PPC440SPE_XOR_ID;
4066                 /* As far as the XOR engine is concerned, it does not
4067                  * use FIFOs but uses linked list. So there is no dependency
4068                  * between pool size to allocate and the engine configuration.
4069                  */
4070                 pool_size = PAGE_SIZE << 1;
4071         } else {
4072                 /* it is DMA0 or DMA1 */
4073                 idx = of_get_property(np, "cell-index", &len);
4074                 if (!idx || (len != sizeof(u32))) {
4075                         dev_err(&ofdev->dev, "Device node %s has missing "
4076                                 "or invalid cell-index property\n",
4077                                 np->full_name);
4078                         return -EINVAL;
4079                 }
4080                 id = *idx;
4081                 /* DMA0,1 engines use FIFO to maintain CDBs, so we
4082                  * should allocate the pool accordingly to size of this
4083                  * FIFO. Thus, the pool size depends on the FIFO depth:
4084                  * how much CDBs pointers the FIFO may contain then so
4085                  * much CDBs we should provide in the pool.
4086                  * That is
4087                  *   CDB size = 32B;
4088                  *   CDBs number = (DMA0_FIFO_SIZE >> 3);
4089                  *   Pool size = CDBs number * CDB size =
4090                  *      = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4091                  */
4092                 pool_size = (id == PPC440SPE_DMA0_ID) ?
4093                             DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4094                 pool_size <<= 2;
4095         }
4096
4097         if (of_address_to_resource(np, 0, &res)) {
4098                 dev_err(&ofdev->dev, "failed to get memory resource\n");
4099                 initcode = PPC_ADMA_INIT_MEMRES;
4100                 ret = -ENODEV;
4101                 goto out;
4102         }
4103
4104         if (!request_mem_region(res.start, resource_size(&res),
4105                                 dev_driver_string(&ofdev->dev))) {
4106                 dev_err(&ofdev->dev, "failed to request memory region %pR\n",
4107                         &res);
4108                 initcode = PPC_ADMA_INIT_MEMREG;
4109                 ret = -EBUSY;
4110                 goto out;
4111         }
4112
4113         /* create a device */
4114         adev = kzalloc(sizeof(*adev), GFP_KERNEL);
4115         if (!adev) {
4116                 dev_err(&ofdev->dev, "failed to allocate device\n");
4117                 initcode = PPC_ADMA_INIT_ALLOC;
4118                 ret = -ENOMEM;
4119                 goto err_adev_alloc;
4120         }
4121
4122         adev->id = id;
4123         adev->pool_size = pool_size;
4124         /* allocate coherent memory for hardware descriptors */
4125         adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
4126                                         adev->pool_size, &adev->dma_desc_pool,
4127                                         GFP_KERNEL);
4128         if (adev->dma_desc_pool_virt == NULL) {
4129                 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4130                         "memory for hardware descriptors\n",
4131                         adev->pool_size);
4132                 initcode = PPC_ADMA_INIT_COHERENT;
4133                 ret = -ENOMEM;
4134                 goto err_dma_alloc;
4135         }
4136         dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n",
4137                 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4138
4139         regs = ioremap(res.start, resource_size(&res));
4140         if (!regs) {
4141                 dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4142                 goto err_regs_alloc;
4143         }
4144
4145         if (adev->id == PPC440SPE_XOR_ID) {
4146                 adev->xor_reg = regs;
4147                 /* Reset XOR */
4148                 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4149                 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4150         } else {
4151                 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4152                                    DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4153                 adev->dma_reg = regs;
4154                 /* DMAx_FIFO_SIZE is defined in bytes,
4155                  * <fsiz> - is defined in number of CDB pointers (8byte).
4156                  * DMA FIFO Length = CSlength + CPlength, where
4157                  * CSlength = CPlength = (fsiz + 1) * 8.
4158                  */
4159                 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4160                           &adev->dma_reg->fsiz);
4161                 /* Configure DMA engine */
4162                 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4163                           &adev->dma_reg->cfg);
4164                 /* Clear Status */
4165                 iowrite32(~0, &adev->dma_reg->dsts);
4166         }
4167
4168         adev->dev = &ofdev->dev;
4169         adev->common.dev = &ofdev->dev;
4170         INIT_LIST_HEAD(&adev->common.channels);
4171         platform_set_drvdata(ofdev, adev);
4172
4173         /* create a channel */
4174         chan = kzalloc(sizeof(*chan), GFP_KERNEL);
4175         if (!chan) {
4176                 dev_err(&ofdev->dev, "can't allocate channel structure\n");
4177                 initcode = PPC_ADMA_INIT_CHANNEL;
4178                 ret = -ENOMEM;
4179                 goto err_chan_alloc;
4180         }
4181
4182         spin_lock_init(&chan->lock);
4183         INIT_LIST_HEAD(&chan->chain);
4184         INIT_LIST_HEAD(&chan->all_slots);
4185         chan->device = adev;
4186         chan->common.device = &adev->common;
4187         dma_cookie_init(&chan->common);
4188         list_add_tail(&chan->common.device_node, &adev->common.channels);
4189         tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet,
4190                      (unsigned long)chan);
4191
4192         /* allocate and map helper pages for async validation or
4193          * async_mult/async_sum_product operations on DMA0/1.
4194          */
4195         if (adev->id != PPC440SPE_XOR_ID) {
4196                 chan->pdest_page = alloc_page(GFP_KERNEL);
4197                 chan->qdest_page = alloc_page(GFP_KERNEL);
4198                 if (!chan->pdest_page ||
4199                     !chan->qdest_page) {
4200                         if (chan->pdest_page)
4201                                 __free_page(chan->pdest_page);
4202                         if (chan->qdest_page)
4203                                 __free_page(chan->qdest_page);
4204                         ret = -ENOMEM;
4205                         goto err_page_alloc;
4206                 }
4207                 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4208                                            PAGE_SIZE, DMA_BIDIRECTIONAL);
4209                 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4210                                            PAGE_SIZE, DMA_BIDIRECTIONAL);
4211         }
4212
4213         ref = kmalloc(sizeof(*ref), GFP_KERNEL);
4214         if (ref) {
4215                 ref->chan = &chan->common;
4216                 INIT_LIST_HEAD(&ref->node);
4217                 list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
4218         } else {
4219                 dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4220                 ret = -ENOMEM;
4221                 goto err_ref_alloc;
4222         }
4223
4224         ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
4225         if (ret)
4226                 goto err_irq;
4227
4228         ppc440spe_adma_init_capabilities(adev);
4229
4230         ret = dma_async_device_register(&adev->common);
4231         if (ret) {
4232                 initcode = PPC_ADMA_INIT_REGISTER;
4233                 dev_err(&ofdev->dev, "failed to register dma device\n");
4234                 goto err_dev_reg;
4235         }
4236
4237         goto out;
4238
4239 err_dev_reg:
4240         ppc440spe_adma_release_irqs(adev, chan);
4241 err_irq:
4242         list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4243                 if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4244                         list_del(&ref->node);
4245                         kfree(ref);
4246                 }
4247         }
4248 err_ref_alloc:
4249         if (adev->id != PPC440SPE_XOR_ID) {
4250                 dma_unmap_page(&ofdev->dev, chan->pdest,
4251                                PAGE_SIZE, DMA_BIDIRECTIONAL);
4252                 dma_unmap_page(&ofdev->dev, chan->qdest,
4253                                PAGE_SIZE, DMA_BIDIRECTIONAL);
4254                 __free_page(chan->pdest_page);
4255                 __free_page(chan->qdest_page);
4256         }
4257 err_page_alloc:
4258         kfree(chan);
4259 err_chan_alloc:
4260         if (adev->id == PPC440SPE_XOR_ID)
4261                 iounmap(adev->xor_reg);
4262         else
4263                 iounmap(adev->dma_reg);
4264 err_regs_alloc:
4265         dma_free_coherent(adev->dev, adev->pool_size,
4266                           adev->dma_desc_pool_virt,
4267                           adev->dma_desc_pool);
4268 err_dma_alloc:
4269         kfree(adev);
4270 err_adev_alloc:
4271         release_mem_region(res.start, resource_size(&res));
4272 out:
4273         if (id < PPC440SPE_ADMA_ENGINES_NUM)
4274                 ppc440spe_adma_devices[id] = initcode;
4275
4276         return ret;
4277 }
4278
4279 /**
4280  * ppc440spe_adma_remove - remove the asynch device
4281  */
4282 static int ppc440spe_adma_remove(struct platform_device *ofdev)
4283 {
4284         struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev);
4285         struct device_node *np = ofdev->dev.of_node;
4286         struct resource res;
4287         struct dma_chan *chan, *_chan;
4288         struct ppc_dma_chan_ref *ref, *_ref;
4289         struct ppc440spe_adma_chan *ppc440spe_chan;
4290
4291         if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4292                 ppc440spe_adma_devices[adev->id] = -1;
4293
4294         dma_async_device_unregister(&adev->common);
4295
4296         list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4297                                  device_node) {
4298                 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4299                 ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
4300                 tasklet_kill(&ppc440spe_chan->irq_tasklet);
4301                 if (adev->id != PPC440SPE_XOR_ID) {
4302                         dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4303                                         PAGE_SIZE, DMA_BIDIRECTIONAL);
4304                         dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4305                                         PAGE_SIZE, DMA_BIDIRECTIONAL);
4306                         __free_page(ppc440spe_chan->pdest_page);
4307                         __free_page(ppc440spe_chan->qdest_page);
4308                 }
4309                 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4310                                          node) {
4311                         if (ppc440spe_chan ==
4312                             to_ppc440spe_adma_chan(ref->chan)) {
4313                                 list_del(&ref->node);
4314                                 kfree(ref);
4315                         }
4316                 }
4317                 list_del(&chan->device_node);
4318                 kfree(ppc440spe_chan);
4319         }
4320
4321         dma_free_coherent(adev->dev, adev->pool_size,
4322                           adev->dma_desc_pool_virt, adev->dma_desc_pool);
4323         if (adev->id == PPC440SPE_XOR_ID)
4324                 iounmap(adev->xor_reg);
4325         else
4326                 iounmap(adev->dma_reg);
4327         of_address_to_resource(np, 0, &res);
4328         release_mem_region(res.start, resource_size(&res));
4329         kfree(adev);
4330         return 0;
4331 }
4332
4333 /*
4334  * /sys driver interface to enable h/w RAID-6 capabilities
4335  * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4336  * directory are "devices", "enable" and "poly".
4337  * "devices" shows available engines.
4338  * "enable" is used to enable RAID-6 capabilities or to check
4339  * whether these has been activated.
4340  * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4341  */
4342
4343 static ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf)
4344 {
4345         ssize_t size = 0;
4346         int i;
4347
4348         for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4349                 if (ppc440spe_adma_devices[i] == -1)
4350                         continue;
4351                 size += snprintf(buf + size, PAGE_SIZE - size,
4352                                  "PPC440SP(E)-ADMA.%d: %s\n", i,
4353                                  ppc_adma_errors[ppc440spe_adma_devices[i]]);
4354         }
4355         return size;
4356 }
4357
4358 static ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf)
4359 {
4360         return snprintf(buf, PAGE_SIZE,
4361                         "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4362                         ppc440spe_r6_enabled ? "EN" : "DIS");
4363 }
4364
4365 static ssize_t store_ppc440spe_r6enable(struct device_driver *dev,
4366                                         const char *buf, size_t count)
4367 {
4368         unsigned long val;
4369
4370         if (!count || count > 11)
4371                 return -EINVAL;
4372
4373         if (!ppc440spe_r6_tchan)
4374                 return -EFAULT;
4375
4376         /* Write a key */
4377         sscanf(buf, "%lx", &val);
4378         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4379         isync();
4380
4381         /* Verify whether it really works now */
4382         if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
4383                 pr_info("PPC440SP(e) RAID-6 has been activated "
4384                         "successfully\n");
4385                 ppc440spe_r6_enabled = 1;
4386         } else {
4387                 pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4388                         " Error key ?\n");
4389                 ppc440spe_r6_enabled = 0;
4390         }
4391         return count;
4392 }
4393
4394 static ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf)
4395 {
4396         ssize_t size = 0;
4397         u32 reg;
4398
4399 #ifdef CONFIG_440SP
4400         /* 440SP has fixed polynomial */
4401         reg = 0x4d;
4402 #else
4403         reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4404         reg >>= MQ0_CFBHL_POLY;
4405         reg &= 0xFF;
4406 #endif
4407
4408         size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver "
4409                         "uses 0x1%02x polynomial.\n", reg);
4410         return size;
4411 }
4412
4413 static ssize_t store_ppc440spe_r6poly(struct device_driver *dev,
4414                                       const char *buf, size_t count)
4415 {
4416         unsigned long reg, val;
4417
4418 #ifdef CONFIG_440SP
4419         /* 440SP uses default 0x14D polynomial only */
4420         return -EINVAL;
4421 #endif
4422
4423         if (!count || count > 6)
4424                 return -EINVAL;
4425
4426         /* e.g., 0x14D or 0x11D */
4427         sscanf(buf, "%lx", &val);
4428
4429         if (val & ~0x1FF)
4430                 return -EINVAL;
4431
4432         val &= 0xFF;
4433         reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4434         reg &= ~(0xFF << MQ0_CFBHL_POLY);
4435         reg |= val << MQ0_CFBHL_POLY;
4436         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4437
4438         return count;
4439 }
4440
4441 static DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL);
4442 static DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable,
4443                    store_ppc440spe_r6enable);
4444 static DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly,
4445                    store_ppc440spe_r6poly);
4446
4447 /*
4448  * Common initialisation for RAID engines; allocate memory for
4449  * DMAx FIFOs, perform configuration common for all DMA engines.
4450  * Further DMA engine specific configuration is done at probe time.
4451  */
4452 static int ppc440spe_configure_raid_devices(void)
4453 {
4454         struct device_node *np;
4455         struct resource i2o_res;
4456         struct i2o_regs __iomem *i2o_reg;
4457         dcr_host_t i2o_dcr_host;
4458         unsigned int dcr_base, dcr_len;
4459         int i, ret;
4460
4461         np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4462         if (!np) {
4463                 pr_err("%s: can't find I2O device tree node\n",
4464                         __func__);
4465                 return -ENODEV;
4466         }
4467
4468         if (of_address_to_resource(np, 0, &i2o_res)) {
4469                 of_node_put(np);
4470                 return -EINVAL;
4471         }
4472
4473         i2o_reg = of_iomap(np, 0);
4474         if (!i2o_reg) {
4475                 pr_err("%s: failed to map I2O registers\n", __func__);
4476                 of_node_put(np);
4477                 return -EINVAL;
4478         }
4479
4480         /* Get I2O DCRs base */
4481         dcr_base = dcr_resource_start(np, 0);
4482         dcr_len = dcr_resource_len(np, 0);
4483         if (!dcr_base && !dcr_len) {
4484                 pr_err("%s: can't get DCR registers base/len!\n",
4485                         np->full_name);
4486                 of_node_put(np);
4487                 iounmap(i2o_reg);
4488                 return -ENODEV;
4489         }
4490
4491         i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4492         if (!DCR_MAP_OK(i2o_dcr_host)) {
4493                 pr_err("%s: failed to map DCRs!\n", np->full_name);
4494                 of_node_put(np);
4495                 iounmap(i2o_reg);
4496                 return -ENODEV;
4497         }
4498         of_node_put(np);
4499
4500         /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4501          * the base address of FIFO memory space.
4502          * Actually we need twice more physical memory than programmed in the
4503          * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4504          */
4505         ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4506                                          GFP_KERNEL);
4507         if (!ppc440spe_dma_fifo_buf) {
4508                 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4509                 iounmap(i2o_reg);
4510                 dcr_unmap(i2o_dcr_host, dcr_len);
4511                 return -ENOMEM;
4512         }
4513
4514         /*
4515          * Configure h/w
4516          */
4517         /* Reset I2O/DMA */
4518         mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4519         mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4520
4521         /* Setup the base address of mmaped registers */
4522         dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4523         dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4524                                                 I2O_REG_ENABLE);
4525         dcr_unmap(i2o_dcr_host, dcr_len);
4526
4527         /* Setup FIFO memory space base address */
4528         iowrite32(0, &i2o_reg->ifbah);
4529         iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4530
4531         /* set zero FIFO size for I2O, so the whole
4532          * ppc440spe_dma_fifo_buf is used by DMAs.
4533          * DMAx_FIFOs will be configured while probe.
4534          */
4535         iowrite32(0, &i2o_reg->ifsiz);
4536         iounmap(i2o_reg);
4537
4538         /* To prepare WXOR/RXOR functionality we need access to
4539          * Memory Queue Module DCRs (finally it will be enabled
4540          * via /sys interface of the ppc440spe ADMA driver).
4541          */
4542         np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
4543         if (!np) {
4544                 pr_err("%s: can't find MQ device tree node\n",
4545                         __func__);
4546                 ret = -ENODEV;
4547                 goto out_free;
4548         }
4549
4550         /* Get MQ DCRs base */
4551         dcr_base = dcr_resource_start(np, 0);
4552         dcr_len = dcr_resource_len(np, 0);
4553         if (!dcr_base && !dcr_len) {
4554                 pr_err("%s: can't get DCR registers base/len!\n",
4555                         np->full_name);
4556                 ret = -ENODEV;
4557                 goto out_mq;
4558         }
4559
4560         ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4561         if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4562                 pr_err("%s: failed to map DCRs!\n", np->full_name);
4563                 ret = -ENODEV;
4564                 goto out_mq;
4565         }
4566         of_node_put(np);
4567         ppc440spe_mq_dcr_len = dcr_len;
4568
4569         /* Set HB alias */
4570         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4571
4572         /* Set:
4573          * - LL transaction passing limit to 1;
4574          * - Memory controller cycle limit to 1;
4575          * - Galois Polynomial to 0x14d (default)
4576          */
4577         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4578                   (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4579                   (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4580
4581         atomic_set(&ppc440spe_adma_err_irq_ref, 0);
4582         for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4583                 ppc440spe_adma_devices[i] = -1;
4584
4585         return 0;
4586
4587 out_mq:
4588         of_node_put(np);
4589 out_free:
4590         kfree(ppc440spe_dma_fifo_buf);
4591         return ret;
4592 }
4593
4594 static const struct of_device_id ppc440spe_adma_of_match[] = {
4595         { .compatible   = "ibm,dma-440spe", },
4596         { .compatible   = "amcc,xor-accelerator", },
4597         {},
4598 };
4599 MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4600
4601 static struct platform_driver ppc440spe_adma_driver = {
4602         .probe = ppc440spe_adma_probe,
4603         .remove = ppc440spe_adma_remove,
4604         .driver = {
4605                 .name = "PPC440SP(E)-ADMA",
4606                 .owner = THIS_MODULE,
4607                 .of_match_table = ppc440spe_adma_of_match,
4608         },
4609 };
4610
4611 static __init int ppc440spe_adma_init(void)
4612 {
4613         int ret;
4614
4615         ret = ppc440spe_configure_raid_devices();
4616         if (ret)
4617                 return ret;
4618
4619         ret = platform_driver_register(&ppc440spe_adma_driver);
4620         if (ret) {
4621                 pr_err("%s: failed to register platform driver\n",
4622                         __func__);
4623                 goto out_reg;
4624         }
4625
4626         /* Initialization status */
4627         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4628                                  &driver_attr_devices);
4629         if (ret)
4630                 goto out_dev;
4631
4632         /* RAID-6 h/w enable entry */
4633         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4634                                  &driver_attr_enable);
4635         if (ret)
4636                 goto out_en;
4637
4638         /* GF polynomial to use */
4639         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4640                                  &driver_attr_poly);
4641         if (!ret)
4642                 return ret;
4643
4644         driver_remove_file(&ppc440spe_adma_driver.driver,
4645                            &driver_attr_enable);
4646 out_en:
4647         driver_remove_file(&ppc440spe_adma_driver.driver,
4648                            &driver_attr_devices);
4649 out_dev:
4650         /* User will not be able to enable h/w RAID-6 */
4651         pr_err("%s: failed to create RAID-6 driver interface\n",
4652                 __func__);
4653         platform_driver_unregister(&ppc440spe_adma_driver);
4654 out_reg:
4655         dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4656         kfree(ppc440spe_dma_fifo_buf);
4657         return ret;
4658 }
4659
4660 static void __exit ppc440spe_adma_exit(void)
4661 {
4662         driver_remove_file(&ppc440spe_adma_driver.driver,
4663                            &driver_attr_poly);
4664         driver_remove_file(&ppc440spe_adma_driver.driver,
4665                            &driver_attr_enable);
4666         driver_remove_file(&ppc440spe_adma_driver.driver,
4667                            &driver_attr_devices);
4668         platform_driver_unregister(&ppc440spe_adma_driver);
4669         dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4670         kfree(ppc440spe_dma_fifo_buf);
4671 }
4672
4673 arch_initcall(ppc440spe_adma_init);
4674 module_exit(ppc440spe_adma_exit);
4675
4676 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
4677 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
4678 MODULE_LICENSE("GPL");