drivers/dma/iop-adma.c

   1 /*
   2  * offload engine driver for the Intel Xscale series of i/o processors
   3  * Copyright © 2006, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  */
  19
  20 /*
  21  * This driver supports the asynchrounous DMA copy and RAID engines available
  22  * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
  23  */
  24
  25 #include <linux/init.h>
  26 #include <linux/module.h>
  27 #include <linux/delay.h>
  28 #include <linux/dma-mapping.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/interrupt.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/memory.h>
  33 #include <linux/ioport.h>
  34 #include <linux/raid/pq.h>
  35 #include <linux/slab.h>
  36
  37 #include <mach/adma.h>
  38
  39 #include "dmaengine.h"
  40
  41 #define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common)
  42 #define to_iop_adma_device(dev) \
  43         container_of(dev, struct iop_adma_device, common)
  44 #define tx_to_iop_adma_slot(tx) \
  45         container_of(tx, struct iop_adma_desc_slot, async_tx)
  46
  47 /**
  48  * iop_adma_free_slots - flags descriptor slots for reuse
  49  * @slot: Slot to free
  50  * Caller must hold &iop_chan->lock while calling this function
  51  */
  52 static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
  53 {
  54         int stride = slot->slots_per_op;
  55
  56         while (stride--) {
  57                 slot->slots_per_op = 0;
  58                 slot = list_entry(slot->slot_node.next,
  59                                 struct iop_adma_desc_slot,
  60                                 slot_node);
  61         }
  62 }
  63
  64 static void
  65 iop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
  66 {
  67         struct dma_async_tx_descriptor *tx = &desc->async_tx;
  68         struct iop_adma_desc_slot *unmap = desc->group_head;
  69         struct device *dev = &iop_chan->device->pdev->dev;
  70         u32 len = unmap->unmap_len;
  71         enum dma_ctrl_flags flags = tx->flags;
  72         u32 src_cnt;
  73         dma_addr_t addr;
  74         dma_addr_t dest;
  75
  76         src_cnt = unmap->unmap_src_cnt;
  77         dest = iop_desc_get_dest_addr(unmap, iop_chan);
  78         if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
  79                 enum dma_data_direction dir;
  80
  81                 if (src_cnt > 1) /* is xor? */
  82                         dir = DMA_BIDIRECTIONAL;
  83                 else
  84                         dir = DMA_FROM_DEVICE;
  85
  86                 dma_unmap_page(dev, dest, len, dir);
  87         }
  88
  89         if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
  90                 while (src_cnt--) {
  91                         addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt);
  92                         if (addr == dest)
  93                                 continue;
  94                         dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
  95                 }
  96         }
  97         desc->group_head = NULL;
  98 }
  99
 100 static void
 101 iop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
 102 {
 103         struct dma_async_tx_descriptor *tx = &desc->async_tx;
 104         struct iop_adma_desc_slot *unmap = desc->group_head;
 105         struct device *dev = &iop_chan->device->pdev->dev;
 106         u32 len = unmap->unmap_len;
 107         enum dma_ctrl_flags flags = tx->flags;
 108         u32 src_cnt = unmap->unmap_src_cnt;
 109         dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan);
 110         dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan);
 111         int i;
 112
 113         if (tx->flags & DMA_PREP_CONTINUE)
 114                 src_cnt -= 3;
 115
 116         if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) {
 117                 dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL);
 118                 dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL);
 119         }
 120
 121         if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
 122                 dma_addr_t addr;
 123
 124                 for (i = 0; i < src_cnt; i++) {
 125                         addr = iop_desc_get_src_addr(unmap, iop_chan, i);
 126                         dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
 127                 }
 128                 if (desc->pq_check_result) {
 129                         dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE);
 130                         dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE);
 131                 }
 132         }
 133
 134         desc->group_head = NULL;
 135 }
 136
 137
 138 static dma_cookie_t
 139 iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
 140         struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
 141 {
 142         struct dma_async_tx_descriptor *tx = &desc->async_tx;
 143
 144         BUG_ON(tx->cookie < 0);
 145         if (tx->cookie > 0) {
 146                 cookie = tx->cookie;
 147                 tx->cookie = 0;
 148
 149                 /* call the callback (must not sleep or submit new
 150                  * operations to this channel)
 151                  */
 152                 if (tx->callback)
 153                         tx->callback(tx->callback_param);
 154
 155                 /* unmap dma addresses
 156                  * (unmap_single vs unmap_page?)
 157                  */
 158                 if (desc->group_head && desc->unmap_len) {
 159                         if (iop_desc_is_pq(desc))
 160                                 iop_desc_unmap_pq(iop_chan, desc);
 161                         else
 162                                 iop_desc_unmap(iop_chan, desc);
 163                 }
 164         }
 165
 166         /* run dependent operations */
 167         dma_run_dependencies(tx);
 168
 169         return cookie;
 170 }
 171
 172 static int
 173 iop_adma_clean_slot(struct iop_adma_desc_slot *desc,
 174         struct iop_adma_chan *iop_chan)
 175 {
 176         /* the client is allowed to attach dependent operations
 177          * until 'ack' is set
 178          */
 179         if (!async_tx_test_ack(&desc->async_tx))
 180                 return 0;
 181
 182         /* leave the last descriptor in the chain
 183          * so we can append to it
 184          */
 185         if (desc->chain_node.next == &iop_chan->chain)
 186                 return 1;
 187
 188         dev_dbg(iop_chan->device->common.dev,
 189                 "\tfree slot: %d slots_per_op: %d\n",
 190                 desc->idx, desc->slots_per_op);
 191
 192         list_del(&desc->chain_node);
 193         iop_adma_free_slots(desc);
 194
 195         return 0;
 196 }
 197
 198 static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
 199 {
 200         struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL;
 201         dma_cookie_t cookie = 0;
 202         u32 current_desc = iop_chan_get_current_descriptor(iop_chan);
 203         int busy = iop_chan_is_busy(iop_chan);
 204         int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
 205
 206         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
 207         /* free completed slots from the chain starting with
 208          * the oldest descriptor
 209          */
 210         list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
 211                                         chain_node) {
 212                 pr_debug("\tcookie: %d slot: %d busy: %d "
 213                         "this_desc: %#x next_desc: %#x ack: %d\n",
 214                         iter->async_tx.cookie, iter->idx, busy,
 215                         iter->async_tx.phys, iop_desc_get_next_desc(iter),
 216                         async_tx_test_ack(&iter->async_tx));
 217                 prefetch(_iter);
 218                 prefetch(&_iter->async_tx);
 219
 220                 /* do not advance past the current descriptor loaded into the
 221                  * hardware channel, subsequent descriptors are either in
 222                  * process or have not been submitted
 223                  */
 224                 if (seen_current)
 225                         break;
 226
 227                 /* stop the search if we reach the current descriptor and the
 228                  * channel is busy, or if it appears that the current descriptor
 229                  * needs to be re-read (i.e. has been appended to)
 230                  */
 231                 if (iter->async_tx.phys == current_desc) {
 232                         BUG_ON(seen_current++);
 233                         if (busy || iop_desc_get_next_desc(iter))
 234                                 break;
 235                 }
 236
 237                 /* detect the start of a group transaction */
 238                 if (!slot_cnt && !slots_per_op) {
 239                         slot_cnt = iter->slot_cnt;
 240                         slots_per_op = iter->slots_per_op;
 241                         if (slot_cnt <= slots_per_op) {
 242                                 slot_cnt = 0;
 243                                 slots_per_op = 0;
 244                         }
 245                 }
 246
 247                 if (slot_cnt) {
 248                         pr_debug("\tgroup++\n");
 249                         if (!grp_start)
 250                                 grp_start = iter;
 251                         slot_cnt -= slots_per_op;
 252                 }
 253
 254                 /* all the members of a group are complete */
 255                 if (slots_per_op != 0 && slot_cnt == 0) {
 256                         struct iop_adma_desc_slot *grp_iter, *_grp_iter;
 257                         int end_of_chain = 0;
 258                         pr_debug("\tgroup end\n");
 259
 260                         /* collect the total results */
 261                         if (grp_start->xor_check_result) {
 262                                 u32 zero_sum_result = 0;
 263                                 slot_cnt = grp_start->slot_cnt;
 264                                 grp_iter = grp_start;
 265
 266                                 list_for_each_entry_from(grp_iter,
 267                                         &iop_chan->chain, chain_node) {
 268                                         zero_sum_result |=
 269                                             iop_desc_get_zero_result(grp_iter);
 270                                             pr_debug("\titer%d result: %d\n",
 271                                             grp_iter->idx, zero_sum_result);
 272                                         slot_cnt -= slots_per_op;
 273                                         if (slot_cnt == 0)
 274                                                 break;
 275                                 }
 276                                 pr_debug("\tgrp_start->xor_check_result: %p\n",
 277                                         grp_start->xor_check_result);
 278                                 *grp_start->xor_check_result = zero_sum_result;
 279                         }
 280
 281                         /* clean up the group */
 282                         slot_cnt = grp_start->slot_cnt;
 283                         grp_iter = grp_start;
 284                         list_for_each_entry_safe_from(grp_iter, _grp_iter,
 285                                 &iop_chan->chain, chain_node) {
 286                                 cookie = iop_adma_run_tx_complete_actions(
 287                                         grp_iter, iop_chan, cookie);
 288
 289                                 slot_cnt -= slots_per_op;
 290                                 end_of_chain = iop_adma_clean_slot(grp_iter,
 291                                         iop_chan);
 292
 293                                 if (slot_cnt == 0 || end_of_chain)
 294                                         break;
 295                         }
 296
 297                         /* the group should be complete at this point */
 298                         BUG_ON(slot_cnt);
 299
 300                         slots_per_op = 0;
 301                         grp_start = NULL;
 302                         if (end_of_chain)
 303                                 break;
 304                         else
 305                                 continue;
 306                 } else if (slots_per_op) /* wait for group completion */
 307                         continue;
 308
 309                 /* write back zero sum results (single descriptor case) */
 310                 if (iter->xor_check_result && iter->async_tx.cookie)
 311                         *iter->xor_check_result =
 312                                 iop_desc_get_zero_result(iter);
 313
 314                 cookie = iop_adma_run_tx_complete_actions(
 315                                         iter, iop_chan, cookie);
 316
 317                 if (iop_adma_clean_slot(iter, iop_chan))
 318                         break;
 319         }
 320
 321         if (cookie > 0) {
 322                 iop_chan->common.completed_cookie = cookie;
 323                 pr_debug("\tcompleted cookie %d\n", cookie);
 324         }
 325 }
 326
 327 static void
 328 iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
 329 {
 330         spin_lock_bh(&iop_chan->lock);
 331         __iop_adma_slot_cleanup(iop_chan);
 332         spin_unlock_bh(&iop_chan->lock);
 333 }
 334
 335 static void iop_adma_tasklet(unsigned long data)
 336 {
 337         struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data;
 338
 339         /* lockdep will flag depedency submissions as potentially
 340          * recursive locking, this is not the case as a dependency
 341          * submission will never recurse a channels submit routine.
 342          * There are checks in async_tx.c to prevent this.
 343          */
 344         spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING);
 345         __iop_adma_slot_cleanup(iop_chan);
 346         spin_unlock(&iop_chan->lock);
 347 }
 348
 349 static struct iop_adma_desc_slot *
 350 iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots,
 351                         int slots_per_op)
 352 {
 353         struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL;
 354         LIST_HEAD(chain);
 355         int slots_found, retry = 0;
 356
 357         /* start search from the last allocated descrtiptor
 358          * if a contiguous allocation can not be found start searching
 359          * from the beginning of the list
 360          */
 361 retry:
 362         slots_found = 0;
 363         if (retry == 0)
 364                 iter = iop_chan->last_used;
 365         else
 366                 iter = list_entry(&iop_chan->all_slots,
 367                         struct iop_adma_desc_slot,
 368                         slot_node);
 369
 370         list_for_each_entry_safe_continue(
 371                 iter, _iter, &iop_chan->all_slots, slot_node) {
 372                 prefetch(_iter);
 373                 prefetch(&_iter->async_tx);
 374                 if (iter->slots_per_op) {
 375                         /* give up after finding the first busy slot
 376                          * on the second pass through the list
 377                          */
 378                         if (retry)
 379                                 break;
 380
 381                         slots_found = 0;
 382                         continue;
 383                 }
 384
 385                 /* start the allocation if the slot is correctly aligned */
 386                 if (!slots_found++) {
 387                         if (iop_desc_is_aligned(iter, slots_per_op))
 388                                 alloc_start = iter;
 389                         else {
 390                                 slots_found = 0;
 391                                 continue;
 392                         }
 393                 }
 394
 395                 if (slots_found == num_slots) {
 396                         struct iop_adma_desc_slot *alloc_tail = NULL;
 397                         struct iop_adma_desc_slot *last_used = NULL;
 398                         iter = alloc_start;
 399                         while (num_slots) {
 400                                 int i;
 401                                 dev_dbg(iop_chan->device->common.dev,
 402                                         "allocated slot: %d "
 403                                         "(desc %p phys: %#x) slots_per_op %d\n",
 404                                         iter->idx, iter->hw_desc,
 405                                         iter->async_tx.phys, slots_per_op);
 406
 407                                 /* pre-ack all but the last descriptor */
 408                                 if (num_slots != slots_per_op)
 409                                         async_tx_ack(&iter->async_tx);
 410
 411                                 list_add_tail(&iter->chain_node, &chain);
 412                                 alloc_tail = iter;
 413                                 iter->async_tx.cookie = 0;
 414                                 iter->slot_cnt = num_slots;
 415                                 iter->xor_check_result = NULL;
 416                                 for (i = 0; i < slots_per_op; i++) {
 417                                         iter->slots_per_op = slots_per_op - i;
 418                                         last_used = iter;
 419                                         iter = list_entry(iter->slot_node.next,
 420                                                 struct iop_adma_desc_slot,
 421                                                 slot_node);
 422                                 }
 423                                 num_slots -= slots_per_op;
 424                         }
 425                         alloc_tail->group_head = alloc_start;
 426                         alloc_tail->async_tx.cookie = -EBUSY;
 427                         list_splice(&chain, &alloc_tail->tx_list);
 428                         iop_chan->last_used = last_used;
 429                         iop_desc_clear_next_desc(alloc_start);
 430                         iop_desc_clear_next_desc(alloc_tail);
 431                         return alloc_tail;
 432                 }
 433         }
 434         if (!retry++)
 435                 goto retry;
 436
 437         /* perform direct reclaim if the allocation fails */
 438         __iop_adma_slot_cleanup(iop_chan);
 439
 440         return NULL;
 441 }
 442
 443 static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan)
 444 {
 445         dev_dbg(iop_chan->device->common.dev, "pending: %d\n",
 446                 iop_chan->pending);
 447
 448         if (iop_chan->pending >= IOP_ADMA_THRESHOLD) {
 449                 iop_chan->pending = 0;
 450                 iop_chan_append(iop_chan);
 451         }
 452 }
 453
 454 static dma_cookie_t
 455 iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
 456 {
 457         struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
 458         struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan);
 459         struct iop_adma_desc_slot *grp_start, *old_chain_tail;
 460         int slot_cnt;
 461         int slots_per_op;
 462         dma_cookie_t cookie;
 463         dma_addr_t next_dma;
 464
 465         grp_start = sw_desc->group_head;
 466         slot_cnt = grp_start->slot_cnt;
 467         slots_per_op = grp_start->slots_per_op;
 468
 469         spin_lock_bh(&iop_chan->lock);
 470         cookie = dma_cookie_assign(tx);
 471
 472         old_chain_tail = list_entry(iop_chan->chain.prev,
 473                 struct iop_adma_desc_slot, chain_node);
 474         list_splice_init(&sw_desc->tx_list,
 475                          &old_chain_tail->chain_node);
 476
 477         /* fix up the hardware chain */
 478         next_dma = grp_start->async_tx.phys;
 479         iop_desc_set_next_desc(old_chain_tail, next_dma);
 480         BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */
 481
 482         /* check for pre-chained descriptors */
 483         iop_paranoia(iop_desc_get_next_desc(sw_desc));
 484
 485         /* increment the pending count by the number of slots
 486          * memcpy operations have a 1:1 (slot:operation) relation
 487          * other operations are heavier and will pop the threshold
 488          * more often.
 489          */
 490         iop_chan->pending += slot_cnt;
 491         iop_adma_check_threshold(iop_chan);
 492         spin_unlock_bh(&iop_chan->lock);
 493
 494         dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n",
 495                 __func__, sw_desc->async_tx.cookie, sw_desc->idx);
 496
 497         return cookie;
 498 }
 499
 500 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan);
 501 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan);
 502
 503 /**
 504  * iop_adma_alloc_chan_resources -  returns the number of allocated descriptors
 505  * @chan - allocate descriptor resources for this channel
 506  * @client - current client requesting the channel be ready for requests
 507  *
 508  * Note: We keep the slots for 1 operation on iop_chan->chain at all times.  To
 509  * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be
 510  * greater than 2x the number slots needed to satisfy a device->max_xor
 511  * request.
 512  * */
 513 static int iop_adma_alloc_chan_resources(struct dma_chan *chan)
 514 {
 515         char *hw_desc;
 516         int idx;
 517         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 518         struct iop_adma_desc_slot *slot = NULL;
 519         int init = iop_chan->slots_allocated ? 0 : 1;
 520         struct iop_adma_platform_data *plat_data =
 521                 iop_chan->device->pdev->dev.platform_data;
 522         int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE;
 523
 524         /* Allocate descriptor slots */
 525         do {
 526                 idx = iop_chan->slots_allocated;
 527                 if (idx == num_descs_in_pool)
 528                         break;
 529
 530                 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
 531                 if (!slot) {
 532                         printk(KERN_INFO "IOP ADMA Channel only initialized"
 533                                 " %d descriptor slots", idx);
 534                         break;
 535                 }
 536                 hw_desc = (char *) iop_chan->device->dma_desc_pool_virt;
 537                 slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
 538
 539                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
 540                 slot->async_tx.tx_submit = iop_adma_tx_submit;
 541                 INIT_LIST_HEAD(&slot->tx_list);
 542                 INIT_LIST_HEAD(&slot->chain_node);
 543                 INIT_LIST_HEAD(&slot->slot_node);
 544                 hw_desc = (char *) iop_chan->device->dma_desc_pool;
 545                 slot->async_tx.phys =
 546                         (dma_addr_t) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
 547                 slot->idx = idx;
 548
 549                 spin_lock_bh(&iop_chan->lock);
 550                 iop_chan->slots_allocated++;
 551                 list_add_tail(&slot->slot_node, &iop_chan->all_slots);
 552                 spin_unlock_bh(&iop_chan->lock);
 553         } while (iop_chan->slots_allocated < num_descs_in_pool);
 554
 555         if (idx && !iop_chan->last_used)
 556                 iop_chan->last_used = list_entry(iop_chan->all_slots.next,
 557                                         struct iop_adma_desc_slot,
 558                                         slot_node);
 559
 560         dev_dbg(iop_chan->device->common.dev,
 561                 "allocated %d descriptor slots last_used: %p\n",
 562                 iop_chan->slots_allocated, iop_chan->last_used);
 563
 564         /* initialize the channel and the chain with a null operation */
 565         if (init) {
 566                 if (dma_has_cap(DMA_MEMCPY,
 567                         iop_chan->device->common.cap_mask))
 568                         iop_chan_start_null_memcpy(iop_chan);
 569                 else if (dma_has_cap(DMA_XOR,
 570                         iop_chan->device->common.cap_mask))
 571                         iop_chan_start_null_xor(iop_chan);
 572                 else
 573                         BUG();
 574         }
 575
 576         return (idx > 0) ? idx : -ENOMEM;
 577 }
 578
 579 static struct dma_async_tx_descriptor *
 580 iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
 581 {
 582         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 583         struct iop_adma_desc_slot *sw_desc, *grp_start;
 584         int slot_cnt, slots_per_op;
 585
 586         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
 587
 588         spin_lock_bh(&iop_chan->lock);
 589         slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan);
 590         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 591         if (sw_desc) {
 592                 grp_start = sw_desc->group_head;
 593                 iop_desc_init_interrupt(grp_start, iop_chan);
 594                 grp_start->unmap_len = 0;
 595                 sw_desc->async_tx.flags = flags;
 596         }
 597         spin_unlock_bh(&iop_chan->lock);
 598
 599         return sw_desc ? &sw_desc->async_tx : NULL;
 600 }
 601
 602 static struct dma_async_tx_descriptor *
 603 iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
 604                          dma_addr_t dma_src, size_t len, unsigned long flags)
 605 {
 606         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 607         struct iop_adma_desc_slot *sw_desc, *grp_start;
 608         int slot_cnt, slots_per_op;
 609
 610         if (unlikely(!len))
 611                 return NULL;
 612         BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT);
 613
 614         dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
 615                 __func__, len);
 616
 617         spin_lock_bh(&iop_chan->lock);
 618         slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op);
 619         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 620         if (sw_desc) {
 621                 grp_start = sw_desc->group_head;
 622                 iop_desc_init_memcpy(grp_start, flags);
 623                 iop_desc_set_byte_count(grp_start, iop_chan, len);
 624                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 625                 iop_desc_set_memcpy_src_addr(grp_start, dma_src);
 626                 sw_desc->unmap_src_cnt = 1;
 627                 sw_desc->unmap_len = len;
 628                 sw_desc->async_tx.flags = flags;
 629         }
 630         spin_unlock_bh(&iop_chan->lock);
 631
 632         return sw_desc ? &sw_desc->async_tx : NULL;
 633 }
 634
 635 static struct dma_async_tx_descriptor *
 636 iop_adma_prep_dma_memset(struct dma_chan *chan, dma_addr_t dma_dest,
 637                          int value, size_t len, unsigned long flags)
 638 {
 639         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 640         struct iop_adma_desc_slot *sw_desc, *grp_start;
 641         int slot_cnt, slots_per_op;
 642
 643         if (unlikely(!len))
 644                 return NULL;
 645         BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT);
 646
 647         dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
 648                 __func__, len);
 649
 650         spin_lock_bh(&iop_chan->lock);
 651         slot_cnt = iop_chan_memset_slot_count(len, &slots_per_op);
 652         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 653         if (sw_desc) {
 654                 grp_start = sw_desc->group_head;
 655                 iop_desc_init_memset(grp_start, flags);
 656                 iop_desc_set_byte_count(grp_start, iop_chan, len);
 657                 iop_desc_set_block_fill_val(grp_start, value);
 658                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 659                 sw_desc->unmap_src_cnt = 1;
 660                 sw_desc->unmap_len = len;
 661                 sw_desc->async_tx.flags = flags;
 662         }
 663         spin_unlock_bh(&iop_chan->lock);
 664
 665         return sw_desc ? &sw_desc->async_tx : NULL;
 666 }
 667
 668 static struct dma_async_tx_descriptor *
 669 iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
 670                       dma_addr_t *dma_src, unsigned int src_cnt, size_t len,
 671                       unsigned long flags)
 672 {
 673         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 674         struct iop_adma_desc_slot *sw_desc, *grp_start;
 675         int slot_cnt, slots_per_op;
 676
 677         if (unlikely(!len))
 678                 return NULL;
 679         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 680
 681         dev_dbg(iop_chan->device->common.dev,
 682                 "%s src_cnt: %d len: %u flags: %lx\n",
 683                 __func__, src_cnt, len, flags);
 684
 685         spin_lock_bh(&iop_chan->lock);
 686         slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op);
 687         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 688         if (sw_desc) {
 689                 grp_start = sw_desc->group_head;
 690                 iop_desc_init_xor(grp_start, src_cnt, flags);
 691                 iop_desc_set_byte_count(grp_start, iop_chan, len);
 692                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 693                 sw_desc->unmap_src_cnt = src_cnt;
 694                 sw_desc->unmap_len = len;
 695                 sw_desc->async_tx.flags = flags;
 696                 while (src_cnt--)
 697                         iop_desc_set_xor_src_addr(grp_start, src_cnt,
 698                                                   dma_src[src_cnt]);
 699         }
 700         spin_unlock_bh(&iop_chan->lock);
 701
 702         return sw_desc ? &sw_desc->async_tx : NULL;
 703 }
 704
 705 static struct dma_async_tx_descriptor *
 706 iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
 707                           unsigned int src_cnt, size_t len, u32 *result,
 708                           unsigned long flags)
 709 {
 710         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 711         struct iop_adma_desc_slot *sw_desc, *grp_start;
 712         int slot_cnt, slots_per_op;
 713
 714         if (unlikely(!len))
 715                 return NULL;
 716
 717         dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
 718                 __func__, src_cnt, len);
 719
 720         spin_lock_bh(&iop_chan->lock);
 721         slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op);
 722         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 723         if (sw_desc) {
 724                 grp_start = sw_desc->group_head;
 725                 iop_desc_init_zero_sum(grp_start, src_cnt, flags);
 726                 iop_desc_set_zero_sum_byte_count(grp_start, len);
 727                 grp_start->xor_check_result = result;
 728                 pr_debug("\t%s: grp_start->xor_check_result: %p\n",
 729                         __func__, grp_start->xor_check_result);
 730                 sw_desc->unmap_src_cnt = src_cnt;
 731                 sw_desc->unmap_len = len;
 732                 sw_desc->async_tx.flags = flags;
 733                 while (src_cnt--)
 734                         iop_desc_set_zero_sum_src_addr(grp_start, src_cnt,
 735                                                        dma_src[src_cnt]);
 736         }
 737         spin_unlock_bh(&iop_chan->lock);
 738
 739         return sw_desc ? &sw_desc->async_tx : NULL;
 740 }
 741
 742 static struct dma_async_tx_descriptor *
 743 iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
 744                      unsigned int src_cnt, const unsigned char *scf, size_t len,
 745                      unsigned long flags)
 746 {
 747         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 748         struct iop_adma_desc_slot *sw_desc, *g;
 749         int slot_cnt, slots_per_op;
 750         int continue_srcs;
 751
 752         if (unlikely(!len))
 753                 return NULL;
 754         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 755
 756         dev_dbg(iop_chan->device->common.dev,
 757                 "%s src_cnt: %d len: %u flags: %lx\n",
 758                 __func__, src_cnt, len, flags);
 759
 760         if (dmaf_p_disabled_continue(flags))
 761                 continue_srcs = 1+src_cnt;
 762         else if (dmaf_continue(flags))
 763                 continue_srcs = 3+src_cnt;
 764         else
 765                 continue_srcs = 0+src_cnt;
 766
 767         spin_lock_bh(&iop_chan->lock);
 768         slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op);
 769         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 770         if (sw_desc) {
 771                 int i;
 772
 773                 g = sw_desc->group_head;
 774                 iop_desc_set_byte_count(g, iop_chan, len);
 775
 776                 /* even if P is disabled its destination address (bits
 777                  * [3:0]) must match Q.  It is ok if P points to an
 778                  * invalid address, it won't be written.
 779                  */
 780                 if (flags & DMA_PREP_PQ_DISABLE_P)
 781                         dst[0] = dst[1] & 0x7;
 782
 783                 iop_desc_set_pq_addr(g, dst);
 784                 sw_desc->unmap_src_cnt = src_cnt;
 785                 sw_desc->unmap_len = len;
 786                 sw_desc->async_tx.flags = flags;
 787                 for (i = 0; i < src_cnt; i++)
 788                         iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);
 789
 790                 /* if we are continuing a previous operation factor in
 791                  * the old p and q values, see the comment for dma_maxpq
 792                  * in include/linux/dmaengine.h
 793                  */
 794                 if (dmaf_p_disabled_continue(flags))
 795                         iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
 796                 else if (dmaf_continue(flags)) {
 797                         iop_desc_set_pq_src_addr(g, i++, dst[0], 0);
 798                         iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
 799                         iop_desc_set_pq_src_addr(g, i++, dst[1], 0);
 800                 }
 801                 iop_desc_init_pq(g, i, flags);
 802         }
 803         spin_unlock_bh(&iop_chan->lock);
 804
 805         return sw_desc ? &sw_desc->async_tx : NULL;
 806 }
 807
 808 static struct dma_async_tx_descriptor *
 809 iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
 810                          unsigned int src_cnt, const unsigned char *scf,
 811                          size_t len, enum sum_check_flags *pqres,
 812                          unsigned long flags)
 813 {
 814         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 815         struct iop_adma_desc_slot *sw_desc, *g;
 816         int slot_cnt, slots_per_op;
 817
 818         if (unlikely(!len))
 819                 return NULL;
 820         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 821
 822         dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
 823                 __func__, src_cnt, len);
 824
 825         spin_lock_bh(&iop_chan->lock);
 826         slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op);
 827         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 828         if (sw_desc) {
 829                 /* for validate operations p and q are tagged onto the
 830                  * end of the source list
 831                  */
 832                 int pq_idx = src_cnt;
 833
 834                 g = sw_desc->group_head;
 835                 iop_desc_init_pq_zero_sum(g, src_cnt+2, flags);
 836                 iop_desc_set_pq_zero_sum_byte_count(g, len);
 837                 g->pq_check_result = pqres;
 838                 pr_debug("\t%s: g->pq_check_result: %p\n",
 839                         __func__, g->pq_check_result);
 840                 sw_desc->unmap_src_cnt = src_cnt+2;
 841                 sw_desc->unmap_len = len;
 842                 sw_desc->async_tx.flags = flags;
 843                 while (src_cnt--)
 844                         iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
 845                                                           src[src_cnt],
 846                                                           scf[src_cnt]);
 847                 iop_desc_set_pq_zero_sum_addr(g, pq_idx, src);
 848         }
 849         spin_unlock_bh(&iop_chan->lock);
 850
 851         return sw_desc ? &sw_desc->async_tx : NULL;
 852 }
 853
 854 static void iop_adma_free_chan_resources(struct dma_chan *chan)
 855 {
 856         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 857         struct iop_adma_desc_slot *iter, *_iter;
 858         int in_use_descs = 0;
 859
 860         iop_adma_slot_cleanup(iop_chan);
 861
 862         spin_lock_bh(&iop_chan->lock);
 863         list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
 864                                         chain_node) {
 865                 in_use_descs++;
 866                 list_del(&iter->chain_node);
 867         }
 868         list_for_each_entry_safe_reverse(
 869                 iter, _iter, &iop_chan->all_slots, slot_node) {
 870                 list_del(&iter->slot_node);
 871                 kfree(iter);
 872                 iop_chan->slots_allocated--;
 873         }
 874         iop_chan->last_used = NULL;
 875
 876         dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n",
 877                 __func__, iop_chan->slots_allocated);
 878         spin_unlock_bh(&iop_chan->lock);
 879
 880         /* one is ok since we left it on there on purpose */
 881         if (in_use_descs > 1)
 882                 printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n",
 883                         in_use_descs - 1);
 884 }
 885
 886 /**
 887  * iop_adma_status - poll the status of an ADMA transaction
 888  * @chan: ADMA channel handle
 889  * @cookie: ADMA transaction identifier
 890  * @txstate: a holder for the current state of the channel or NULL
 891  */
 892 static enum dma_status iop_adma_status(struct dma_chan *chan,
 893                                         dma_cookie_t cookie,
 894                                         struct dma_tx_state *txstate)
 895 {
 896         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 897         int ret;
 898
 899         ret = dma_cookie_status(chan, cookie, txstate);
 900         if (ret == DMA_SUCCESS)
 901                 return ret;
 902
 903         iop_adma_slot_cleanup(iop_chan);
 904
 905         return dma_cookie_status(chan, cookie, txstate);
 906 }
 907
 908 static irqreturn_t iop_adma_eot_handler(int irq, void *data)
 909 {
 910         struct iop_adma_chan *chan = data;
 911
 912         dev_dbg(chan->device->common.dev, "%s\n", __func__);
 913
 914         tasklet_schedule(&chan->irq_tasklet);
 915
 916         iop_adma_device_clear_eot_status(chan);
 917
 918         return IRQ_HANDLED;
 919 }
 920
 921 static irqreturn_t iop_adma_eoc_handler(int irq, void *data)
 922 {
 923         struct iop_adma_chan *chan = data;
 924
 925         dev_dbg(chan->device->common.dev, "%s\n", __func__);
 926
 927         tasklet_schedule(&chan->irq_tasklet);
 928
 929         iop_adma_device_clear_eoc_status(chan);
 930
 931         return IRQ_HANDLED;
 932 }
 933
 934 static irqreturn_t iop_adma_err_handler(int irq, void *data)
 935 {
 936         struct iop_adma_chan *chan = data;
 937         unsigned long status = iop_chan_get_status(chan);
 938
 939         dev_err(chan->device->common.dev,
 940                 "error ( %s%s%s%s%s%s%s)\n",
 941                 iop_is_err_int_parity(status, chan) ? "int_parity " : "",
 942                 iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "",
 943                 iop_is_err_int_tabort(status, chan) ? "int_tabort " : "",
 944                 iop_is_err_int_mabort(status, chan) ? "int_mabort " : "",
 945                 iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "",
 946                 iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "",
 947                 iop_is_err_split_tx(status, chan) ? "split_tx " : "");
 948
 949         iop_adma_device_clear_err_status(chan);
 950
 951         BUG();
 952
 953         return IRQ_HANDLED;
 954 }
 955
 956 static void iop_adma_issue_pending(struct dma_chan *chan)
 957 {
 958         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 959
 960         if (iop_chan->pending) {
 961                 iop_chan->pending = 0;
 962                 iop_chan_append(iop_chan);
 963         }
 964 }
 965
 966 /*
 967  * Perform a transaction to verify the HW works.
 968  */
 969 #define IOP_ADMA_TEST_SIZE 2000
 970
 971 static int iop_adma_memcpy_self_test(struct iop_adma_device *device)
 972 {
 973         int i;
 974         void *src, *dest;
 975         dma_addr_t src_dma, dest_dma;
 976         struct dma_chan *dma_chan;
 977         dma_cookie_t cookie;
 978         struct dma_async_tx_descriptor *tx;
 979         int err = 0;
 980         struct iop_adma_chan *iop_chan;
 981
 982         dev_dbg(device->common.dev, "%s\n", __func__);
 983
 984         src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
 985         if (!src)
 986                 return -ENOMEM;
 987         dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
 988         if (!dest) {
 989                 kfree(src);
 990                 return -ENOMEM;
 991         }
 992
 993         /* Fill in src buffer */
 994         for (i = 0; i < IOP_ADMA_TEST_SIZE; i++)
 995                 ((u8 *) src)[i] = (u8)i;
 996
 997         /* Start copy, using first DMA channel */
 998         dma_chan = container_of(device->common.channels.next,
 999                                 struct dma_chan,
1000                                 device_node);
1001         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1002                 err = -ENODEV;
1003                 goto out;
1004         }
1005
1006         dest_dma = dma_map_single(dma_chan->device->dev, dest,
1007                                 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
1008         src_dma = dma_map_single(dma_chan->device->dev, src,
1009                                 IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE);
1010         tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
1011                                       IOP_ADMA_TEST_SIZE,
1012                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1013
1014         cookie = iop_adma_tx_submit(tx);
1015         iop_adma_issue_pending(dma_chan);
1016         msleep(1);
1017
1018         if (iop_adma_status(dma_chan, cookie, NULL) !=
1019                         DMA_SUCCESS) {
1020                 dev_err(dma_chan->device->dev,
1021                         "Self-test copy timed out, disabling\n");
1022                 err = -ENODEV;
1023                 goto free_resources;
1024         }
1025
1026         iop_chan = to_iop_adma_chan(dma_chan);
1027         dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
1028                 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
1029         if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) {
1030                 dev_err(dma_chan->device->dev,
1031                         "Self-test copy failed compare, disabling\n");
1032                 err = -ENODEV;
1033                 goto free_resources;
1034         }
1035
1036 free_resources:
1037         iop_adma_free_chan_resources(dma_chan);
1038 out:
1039         kfree(src);
1040         kfree(dest);
1041         return err;
1042 }
1043
1044 #define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */
1045 static int
1046 iop_adma_xor_val_self_test(struct iop_adma_device *device)
1047 {
1048         int i, src_idx;
1049         struct page *dest;
1050         struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST];
1051         struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
1052         dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
1053         dma_addr_t dma_addr, dest_dma;
1054         struct dma_async_tx_descriptor *tx;
1055         struct dma_chan *dma_chan;
1056         dma_cookie_t cookie;
1057         u8 cmp_byte = 0;
1058         u32 cmp_word;
1059         u32 zero_sum_result;
1060         int err = 0;
1061         struct iop_adma_chan *iop_chan;
1062
1063         dev_dbg(device->common.dev, "%s\n", __func__);
1064
1065         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
1066                 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
1067                 if (!xor_srcs[src_idx]) {
1068                         while (src_idx--)
1069                                 __free_page(xor_srcs[src_idx]);
1070                         return -ENOMEM;
1071                 }
1072         }
1073
1074         dest = alloc_page(GFP_KERNEL);
1075         if (!dest) {
1076                 while (src_idx--)
1077                         __free_page(xor_srcs[src_idx]);
1078                 return -ENOMEM;
1079         }
1080
1081         /* Fill in src buffers */
1082         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
1083                 u8 *ptr = page_address(xor_srcs[src_idx]);
1084                 for (i = 0; i < PAGE_SIZE; i++)
1085                         ptr[i] = (1 << src_idx);
1086         }
1087
1088         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++)
1089                 cmp_byte ^= (u8) (1 << src_idx);
1090
1091         cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
1092                         (cmp_byte << 8) | cmp_byte;
1093
1094         memset(page_address(dest), 0, PAGE_SIZE);
1095
1096         dma_chan = container_of(device->common.channels.next,
1097                                 struct dma_chan,
1098                                 device_node);
1099         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1100                 err = -ENODEV;
1101                 goto out;
1102         }
1103
1104         /* test xor */
1105         dest_dma = dma_map_page(dma_chan->device->dev, dest, 0,
1106                                 PAGE_SIZE, DMA_FROM_DEVICE);
1107         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1108                 dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
1109                                            0, PAGE_SIZE, DMA_TO_DEVICE);
1110         tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
1111                                    IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE,
1112                                    DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1113
1114         cookie = iop_adma_tx_submit(tx);
1115         iop_adma_issue_pending(dma_chan);
1116         msleep(8);
1117
1118         if (iop_adma_status(dma_chan, cookie, NULL) !=
1119                 DMA_SUCCESS) {
1120                 dev_err(dma_chan->device->dev,
1121                         "Self-test xor timed out, disabling\n");
1122                 err = -ENODEV;
1123                 goto free_resources;
1124         }
1125
1126         iop_chan = to_iop_adma_chan(dma_chan);
1127         dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
1128                 PAGE_SIZE, DMA_FROM_DEVICE);
1129         for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
1130                 u32 *ptr = page_address(dest);
1131                 if (ptr[i] != cmp_word) {
1132                         dev_err(dma_chan->device->dev,
1133                                 "Self-test xor failed compare, disabling\n");
1134                         err = -ENODEV;
1135                         goto free_resources;
1136                 }
1137         }
1138         dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma,
1139                 PAGE_SIZE, DMA_TO_DEVICE);
1140
1141         /* skip zero sum if the capability is not present */
1142         if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
1143                 goto free_resources;
1144
1145         /* zero sum the sources with the destintation page */
1146         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1147                 zero_sum_srcs[i] = xor_srcs[i];
1148         zero_sum_srcs[i] = dest;
1149
1150         zero_sum_result = 1;
1151
1152         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1153                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1154                                            zero_sum_srcs[i], 0, PAGE_SIZE,
1155                                            DMA_TO_DEVICE);
1156         tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1157                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1158                                        &zero_sum_result,
1159                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1160
1161         cookie = iop_adma_tx_submit(tx);
1162         iop_adma_issue_pending(dma_chan);
1163         msleep(8);
1164
1165         if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
1166                 dev_err(dma_chan->device->dev,
1167                         "Self-test zero sum timed out, disabling\n");
1168                 err = -ENODEV;
1169                 goto free_resources;
1170         }
1171
1172         if (zero_sum_result != 0) {
1173                 dev_err(dma_chan->device->dev,
1174                         "Self-test zero sum failed compare, disabling\n");
1175                 err = -ENODEV;
1176                 goto free_resources;
1177         }
1178
1179         /* test memset */
1180         dma_addr = dma_map_page(dma_chan->device->dev, dest, 0,
1181                         PAGE_SIZE, DMA_FROM_DEVICE);
1182         tx = iop_adma_prep_dma_memset(dma_chan, dma_addr, 0, PAGE_SIZE,
1183                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1184
1185         cookie = iop_adma_tx_submit(tx);
1186         iop_adma_issue_pending(dma_chan);
1187         msleep(8);
1188
1189         if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
1190                 dev_err(dma_chan->device->dev,
1191                         "Self-test memset timed out, disabling\n");
1192                 err = -ENODEV;
1193                 goto free_resources;
1194         }
1195
1196         for (i = 0; i < PAGE_SIZE/sizeof(u32); i++) {
1197                 u32 *ptr = page_address(dest);
1198                 if (ptr[i]) {
1199                         dev_err(dma_chan->device->dev,
1200                                 "Self-test memset failed compare, disabling\n");
1201                         err = -ENODEV;
1202                         goto free_resources;
1203                 }
1204         }
1205
1206         /* test for non-zero parity sum */
1207         zero_sum_result = 0;
1208         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1209                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1210                                            zero_sum_srcs[i], 0, PAGE_SIZE,
1211                                            DMA_TO_DEVICE);
1212         tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1213                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1214                                        &zero_sum_result,
1215                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1216
1217         cookie = iop_adma_tx_submit(tx);
1218         iop_adma_issue_pending(dma_chan);
1219         msleep(8);
1220
1221         if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) {
1222                 dev_err(dma_chan->device->dev,
1223                         "Self-test non-zero sum timed out, disabling\n");
1224                 err = -ENODEV;
1225                 goto free_resources;
1226         }
1227
1228         if (zero_sum_result != 1) {
1229                 dev_err(dma_chan->device->dev,
1230                         "Self-test non-zero sum failed compare, disabling\n");
1231                 err = -ENODEV;
1232                 goto free_resources;
1233         }
1234
1235 free_resources:
1236         iop_adma_free_chan_resources(dma_chan);
1237 out:
1238         src_idx = IOP_ADMA_NUM_SRC_TEST;
1239         while (src_idx--)
1240                 __free_page(xor_srcs[src_idx]);
1241         __free_page(dest);
1242         return err;
1243 }
1244
1245 #ifdef CONFIG_RAID6_PQ
1246 static int
1247 iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device)
1248 {
1249         /* combined sources, software pq results, and extra hw pq results */
1250         struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2];
1251         /* ptr to the extra hw pq buffers defined above */
1252         struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2];
1253         /* address conversion buffers (dma_map / page_address) */
1254         void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2];
1255         dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST+2];
1256         dma_addr_t *pq_dest = &pq_src[IOP_ADMA_NUM_SRC_TEST];
1257
1258         int i;
1259         struct dma_async_tx_descriptor *tx;
1260         struct dma_chan *dma_chan;
1261         dma_cookie_t cookie;
1262         u32 zero_sum_result;
1263         int err = 0;
1264         struct device *dev;
1265
1266         dev_dbg(device->common.dev, "%s\n", __func__);
1267
1268         for (i = 0; i < ARRAY_SIZE(pq); i++) {
1269                 pq[i] = alloc_page(GFP_KERNEL);
1270                 if (!pq[i]) {
1271                         while (i--)
1272                                 __free_page(pq[i]);
1273                         return -ENOMEM;
1274                 }
1275         }
1276
1277         /* Fill in src buffers */
1278         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) {
1279                 pq_sw[i] = page_address(pq[i]);
1280                 memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE);
1281         }
1282         pq_sw[i] = page_address(pq[i]);
1283         pq_sw[i+1] = page_address(pq[i+1]);
1284
1285         dma_chan = container_of(device->common.channels.next,
1286                                 struct dma_chan,
1287                                 device_node);
1288         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1289                 err = -ENODEV;
1290                 goto out;
1291         }
1292
1293         dev = dma_chan->device->dev;
1294
1295         /* initialize the dests */
1296         memset(page_address(pq_hw[0]), 0 , PAGE_SIZE);
1297         memset(page_address(pq_hw[1]), 0 , PAGE_SIZE);
1298
1299         /* test pq */
1300         pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1301         pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1302         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1303                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1304                                          DMA_TO_DEVICE);
1305
1306         tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src,
1307                                   IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp,
1308                                   PAGE_SIZE,
1309                                   DMA_PREP_INTERRUPT |
1310                                   DMA_CTRL_ACK);
1311
1312         cookie = iop_adma_tx_submit(tx);
1313         iop_adma_issue_pending(dma_chan);
1314         msleep(8);
1315
1316         if (iop_adma_status(dma_chan, cookie, NULL) !=
1317                 DMA_SUCCESS) {
1318                 dev_err(dev, "Self-test pq timed out, disabling\n");
1319                 err = -ENODEV;
1320                 goto free_resources;
1321         }
1322
1323         raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw);
1324
1325         if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST],
1326                    page_address(pq_hw[0]), PAGE_SIZE) != 0) {
1327                 dev_err(dev, "Self-test p failed compare, disabling\n");
1328                 err = -ENODEV;
1329                 goto free_resources;
1330         }
1331         if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1],
1332                    page_address(pq_hw[1]), PAGE_SIZE) != 0) {
1333                 dev_err(dev, "Self-test q failed compare, disabling\n");
1334                 err = -ENODEV;
1335                 goto free_resources;
1336         }
1337
1338         /* test correct zero sum using the software generated pq values */
1339         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1340                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1341                                          DMA_TO_DEVICE);
1342
1343         zero_sum_result = ~0;
1344         tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1345                                       pq_src, IOP_ADMA_NUM_SRC_TEST,
1346                                       raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1347                                       DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1348
1349         cookie = iop_adma_tx_submit(tx);
1350         iop_adma_issue_pending(dma_chan);
1351         msleep(8);
1352
1353         if (iop_adma_status(dma_chan, cookie, NULL) !=
1354                 DMA_SUCCESS) {
1355                 dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n");
1356                 err = -ENODEV;
1357                 goto free_resources;
1358         }
1359
1360         if (zero_sum_result != 0) {
1361                 dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n",
1362                         zero_sum_result);
1363                 err = -ENODEV;
1364                 goto free_resources;
1365         }
1366
1367         /* test incorrect zero sum */
1368         i = IOP_ADMA_NUM_SRC_TEST;
1369         memset(pq_sw[i] + 100, 0, 100);
1370         memset(pq_sw[i+1] + 200, 0, 200);
1371         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1372                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1373                                          DMA_TO_DEVICE);
1374
1375         zero_sum_result = 0;
1376         tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1377                                       pq_src, IOP_ADMA_NUM_SRC_TEST,
1378                                       raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1379                                       DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1380
1381         cookie = iop_adma_tx_submit(tx);
1382         iop_adma_issue_pending(dma_chan);
1383         msleep(8);
1384
1385         if (iop_adma_status(dma_chan, cookie, NULL) !=
1386                 DMA_SUCCESS) {
1387                 dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n");
1388                 err = -ENODEV;
1389                 goto free_resources;
1390         }
1391
1392         if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) {
1393                 dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n",
1394                         zero_sum_result);
1395                 err = -ENODEV;
1396                 goto free_resources;
1397         }
1398
1399 free_resources:
1400         iop_adma_free_chan_resources(dma_chan);
1401 out:
1402         i = ARRAY_SIZE(pq);
1403         while (i--)
1404                 __free_page(pq[i]);
1405         return err;
1406 }
1407 #endif
1408
1409 static int iop_adma_remove(struct platform_device *dev)
1410 {
1411         struct iop_adma_device *device = platform_get_drvdata(dev);
1412         struct dma_chan *chan, *_chan;
1413         struct iop_adma_chan *iop_chan;
1414         struct iop_adma_platform_data *plat_data = dev->dev.platform_data;
1415
1416         dma_async_device_unregister(&device->common);
1417
1418         dma_free_coherent(&dev->dev, plat_data->pool_size,
1419                         device->dma_desc_pool_virt, device->dma_desc_pool);
1420
1421         list_for_each_entry_safe(chan, _chan, &device->common.channels,
1422                                 device_node) {
1423                 iop_chan = to_iop_adma_chan(chan);
1424                 list_del(&chan->device_node);
1425                 kfree(iop_chan);
1426         }
1427         kfree(device);
1428
1429         return 0;
1430 }
1431
1432 static int iop_adma_probe(struct platform_device *pdev)
1433 {
1434         struct resource *res;
1435         int ret = 0, i;
1436         struct iop_adma_device *adev;
1437         struct iop_adma_chan *iop_chan;
1438         struct dma_device *dma_dev;
1439         struct iop_adma_platform_data *plat_data = pdev->dev.platform_data;
1440
1441         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1442         if (!res)
1443                 return -ENODEV;
1444
1445         if (!devm_request_mem_region(&pdev->dev, res->start,
1446                                 resource_size(res), pdev->name))
1447                 return -EBUSY;
1448
1449         adev = kzalloc(sizeof(*adev), GFP_KERNEL);
1450         if (!adev)
1451                 return -ENOMEM;
1452         dma_dev = &adev->common;
1453
1454         /* allocate coherent memory for hardware descriptors
1455          * note: writecombine gives slightly better performance, but
1456          * requires that we explicitly flush the writes
1457          */
1458         if ((adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev,
1459                                         plat_data->pool_size,
1460                                         &adev->dma_desc_pool,
1461                                         GFP_KERNEL)) == NULL) {
1462                 ret = -ENOMEM;
1463                 goto err_free_adev;
1464         }
1465
1466         dev_dbg(&pdev->dev, "%s: allocated descriptor pool virt %p phys %p\n",
1467                 __func__, adev->dma_desc_pool_virt,
1468                 (void *) adev->dma_desc_pool);
1469
1470         adev->id = plat_data->hw_id;
1471
1472         /* discover transaction capabilites from the platform data */
1473         dma_dev->cap_mask = plat_data->cap_mask;
1474
1475         adev->pdev = pdev;
1476         platform_set_drvdata(pdev, adev);
1477
1478         INIT_LIST_HEAD(&dma_dev->channels);
1479
1480         /* set base routines */
1481         dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources;
1482         dma_dev->device_free_chan_resources = iop_adma_free_chan_resources;
1483         dma_dev->device_tx_status = iop_adma_status;
1484         dma_dev->device_issue_pending = iop_adma_issue_pending;
1485         dma_dev->dev = &pdev->dev;
1486
1487         /* set prep routines based on capability */
1488         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1489                 dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy;
1490         if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask))
1491                 dma_dev->device_prep_dma_memset = iop_adma_prep_dma_memset;
1492         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1493                 dma_dev->max_xor = iop_adma_get_max_xor();
1494                 dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor;
1495         }
1496         if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask))
1497                 dma_dev->device_prep_dma_xor_val =
1498                         iop_adma_prep_dma_xor_val;
1499         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1500                 dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0);
1501                 dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq;
1502         }
1503         if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask))
1504                 dma_dev->device_prep_dma_pq_val =
1505                         iop_adma_prep_dma_pq_val;
1506         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1507                 dma_dev->device_prep_dma_interrupt =
1508                         iop_adma_prep_dma_interrupt;
1509
1510         iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL);
1511         if (!iop_chan) {
1512                 ret = -ENOMEM;
1513                 goto err_free_dma;
1514         }
1515         iop_chan->device = adev;
1516
1517         iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start,
1518                                         resource_size(res));
1519         if (!iop_chan->mmr_base) {
1520                 ret = -ENOMEM;
1521                 goto err_free_iop_chan;
1522         }
1523         tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long)
1524                 iop_chan);
1525
1526         /* clear errors before enabling interrupts */
1527         iop_adma_device_clear_err_status(iop_chan);
1528
1529         for (i = 0; i < 3; i++) {
1530                 irq_handler_t handler[] = { iop_adma_eot_handler,
1531                                         iop_adma_eoc_handler,
1532                                         iop_adma_err_handler };
1533                 int irq = platform_get_irq(pdev, i);
1534                 if (irq < 0) {
1535                         ret = -ENXIO;
1536                         goto err_free_iop_chan;
1537                 } else {
1538                         ret = devm_request_irq(&pdev->dev, irq,
1539                                         handler[i], 0, pdev->name, iop_chan);
1540                         if (ret)
1541                                 goto err_free_iop_chan;
1542                 }
1543         }
1544
1545         spin_lock_init(&iop_chan->lock);
1546         INIT_LIST_HEAD(&iop_chan->chain);
1547         INIT_LIST_HEAD(&iop_chan->all_slots);
1548         iop_chan->common.device = dma_dev;
1549         dma_cookie_init(&iop_chan->common);
1550         list_add_tail(&iop_chan->common.device_node, &dma_dev->channels);
1551
1552         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1553                 ret = iop_adma_memcpy_self_test(adev);
1554                 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1555                 if (ret)
1556                         goto err_free_iop_chan;
1557         }
1558
1559         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask) ||
1560             dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
1561                 ret = iop_adma_xor_val_self_test(adev);
1562                 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1563                 if (ret)
1564                         goto err_free_iop_chan;
1565         }
1566
1567         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) &&
1568             dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) {
1569                 #ifdef CONFIG_RAID6_PQ
1570                 ret = iop_adma_pq_zero_sum_self_test(adev);
1571                 dev_dbg(&pdev->dev, "pq self test returned %d\n", ret);
1572                 #else
1573                 /* can not test raid6, so do not publish capability */
1574                 dma_cap_clear(DMA_PQ, dma_dev->cap_mask);
1575                 dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask);
1576                 ret = 0;
1577                 #endif
1578                 if (ret)
1579                         goto err_free_iop_chan;
1580         }
1581
1582         dev_info(&pdev->dev, "Intel(R) IOP: ( %s%s%s%s%s%s%s)\n",
1583                  dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "",
1584                  dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "",
1585                  dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1586                  dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "",
1587                  dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)  ? "fill " : "",
1588                  dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1589                  dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1590
1591         dma_async_device_register(dma_dev);
1592         goto out;
1593
1594  err_free_iop_chan:
1595         kfree(iop_chan);
1596  err_free_dma:
1597         dma_free_coherent(&adev->pdev->dev, plat_data->pool_size,
1598                         adev->dma_desc_pool_virt, adev->dma_desc_pool);
1599  err_free_adev:
1600         kfree(adev);
1601  out:
1602         return ret;
1603 }
1604
1605 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
1606 {
1607         struct iop_adma_desc_slot *sw_desc, *grp_start;
1608         dma_cookie_t cookie;
1609         int slot_cnt, slots_per_op;
1610
1611         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1612
1613         spin_lock_bh(&iop_chan->lock);
1614         slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op);
1615         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1616         if (sw_desc) {
1617                 grp_start = sw_desc->group_head;
1618
1619                 list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1620                 async_tx_ack(&sw_desc->async_tx);
1621                 iop_desc_init_memcpy(grp_start, 0);
1622                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1623                 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1624                 iop_desc_set_memcpy_src_addr(grp_start, 0);
1625
1626                 cookie = dma_cookie_assign(&sw_desc->async_tx);
1627
1628                 /* initialize the completed cookie to be less than
1629                  * the most recently used cookie
1630                  */
1631                 iop_chan->common.completed_cookie = cookie - 1;
1632
1633                 /* channel should not be busy */
1634                 BUG_ON(iop_chan_is_busy(iop_chan));
1635
1636                 /* clear any prior error-status bits */
1637                 iop_adma_device_clear_err_status(iop_chan);
1638
1639                 /* disable operation */
1640                 iop_chan_disable(iop_chan);
1641
1642                 /* set the descriptor address */
1643                 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1644
1645                 /* 1/ don't add pre-chained descriptors
1646                  * 2/ dummy read to flush next_desc write
1647                  */
1648                 BUG_ON(iop_desc_get_next_desc(sw_desc));
1649
1650                 /* run the descriptor */
1651                 iop_chan_enable(iop_chan);
1652         } else
1653                 dev_err(iop_chan->device->common.dev,
1654                         "failed to allocate null descriptor\n");
1655         spin_unlock_bh(&iop_chan->lock);
1656 }
1657
1658 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
1659 {
1660         struct iop_adma_desc_slot *sw_desc, *grp_start;
1661         dma_cookie_t cookie;
1662         int slot_cnt, slots_per_op;
1663
1664         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1665
1666         spin_lock_bh(&iop_chan->lock);
1667         slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op);
1668         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1669         if (sw_desc) {
1670                 grp_start = sw_desc->group_head;
1671                 list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1672                 async_tx_ack(&sw_desc->async_tx);
1673                 iop_desc_init_null_xor(grp_start, 2, 0);
1674                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1675                 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1676                 iop_desc_set_xor_src_addr(grp_start, 0, 0);
1677                 iop_desc_set_xor_src_addr(grp_start, 1, 0);
1678
1679                 cookie = dma_cookie_assign(&sw_desc->async_tx);
1680
1681                 /* initialize the completed cookie to be less than
1682                  * the most recently used cookie
1683                  */
1684                 iop_chan->common.completed_cookie = cookie - 1;
1685
1686                 /* channel should not be busy */
1687                 BUG_ON(iop_chan_is_busy(iop_chan));
1688
1689                 /* clear any prior error-status bits */
1690                 iop_adma_device_clear_err_status(iop_chan);
1691
1692                 /* disable operation */
1693                 iop_chan_disable(iop_chan);
1694
1695                 /* set the descriptor address */
1696                 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1697
1698                 /* 1/ don't add pre-chained descriptors
1699                  * 2/ dummy read to flush next_desc write
1700                  */
1701                 BUG_ON(iop_desc_get_next_desc(sw_desc));
1702
1703                 /* run the descriptor */
1704                 iop_chan_enable(iop_chan);
1705         } else
1706                 dev_err(iop_chan->device->common.dev,
1707                         "failed to allocate null descriptor\n");
1708         spin_unlock_bh(&iop_chan->lock);
1709 }
1710
1711 static struct platform_driver iop_adma_driver = {
1712         .probe          = iop_adma_probe,
1713         .remove         = iop_adma_remove,
1714         .driver         = {
1715                 .owner  = THIS_MODULE,
1716                 .name   = "iop-adma",
1717         },
1718 };
1719
1720 module_platform_driver(iop_adma_driver);
1721
1722 MODULE_AUTHOR("Intel Corporation");
1723 MODULE_DESCRIPTION("IOP ADMA Engine Driver");
1724 MODULE_LICENSE("GPL");
1725 MODULE_ALIAS("platform:iop-adma");