drivers/dma/dmatest.c

   1 /*
   2  * DMA Engine test module
   3  *
   4  * Copyright (C) 2007 Atmel Corporation
   5  * Copyright (C) 2013 Intel Corporation
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/delay.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/freezer.h>
  15 #include <linux/init.h>
  16 #include <linux/kthread.h>
  17 #include <linux/module.h>
  18 #include <linux/moduleparam.h>
  19 #include <linux/random.h>
  20 #include <linux/slab.h>
  21 #include <linux/wait.h>
  22 #include <linux/ctype.h>
  23 #include <linux/debugfs.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/seq_file.h>
  26
  27 static unsigned int test_buf_size = 16384;
  28 module_param(test_buf_size, uint, S_IRUGO);
  29 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
  30
  31 static char test_channel[20];
  32 module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO);
  33 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
  34
  35 static char test_device[20];
  36 module_param_string(device, test_device, sizeof(test_device), S_IRUGO);
  37 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
  38
  39 static unsigned int threads_per_chan = 1;
  40 module_param(threads_per_chan, uint, S_IRUGO);
  41 MODULE_PARM_DESC(threads_per_chan,
  42                 "Number of threads to start per channel (default: 1)");
  43
  44 static unsigned int max_channels;
  45 module_param(max_channels, uint, S_IRUGO);
  46 MODULE_PARM_DESC(max_channels,
  47                 "Maximum number of channels to use (default: all)");
  48
  49 static unsigned int iterations;
  50 module_param(iterations, uint, S_IRUGO);
  51 MODULE_PARM_DESC(iterations,
  52                 "Iterations before stopping test (default: infinite)");
  53
  54 static unsigned int xor_sources = 3;
  55 module_param(xor_sources, uint, S_IRUGO);
  56 MODULE_PARM_DESC(xor_sources,
  57                 "Number of xor source buffers (default: 3)");
  58
  59 static unsigned int pq_sources = 3;
  60 module_param(pq_sources, uint, S_IRUGO);
  61 MODULE_PARM_DESC(pq_sources,
  62                 "Number of p+q source buffers (default: 3)");
  63
  64 static int timeout = 3000;
  65 module_param(timeout, uint, S_IRUGO);
  66 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
  67                  "Pass -1 for infinite timeout");
  68
  69 /* Maximum amount of mismatched bytes in buffer to print */
  70 #define MAX_ERROR_COUNT         32
  71
  72 /*
  73  * Initialization patterns. All bytes in the source buffer has bit 7
  74  * set, all bytes in the destination buffer has bit 7 cleared.
  75  *
  76  * Bit 6 is set for all bytes which are to be copied by the DMA
  77  * engine. Bit 5 is set for all bytes which are to be overwritten by
  78  * the DMA engine.
  79  *
  80  * The remaining bits are the inverse of a counter which increments by
  81  * one for each byte address.
  82  */
  83 #define PATTERN_SRC             0x80
  84 #define PATTERN_DST             0x00
  85 #define PATTERN_COPY            0x40
  86 #define PATTERN_OVERWRITE       0x20
  87 #define PATTERN_COUNT_MASK      0x1f
  88
  89 enum dmatest_error_type {
  90         DMATEST_ET_OK,
  91         DMATEST_ET_MAP_SRC,
  92         DMATEST_ET_MAP_DST,
  93         DMATEST_ET_PREP,
  94         DMATEST_ET_SUBMIT,
  95         DMATEST_ET_TIMEOUT,
  96         DMATEST_ET_DMA_ERROR,
  97         DMATEST_ET_DMA_IN_PROGRESS,
  98         DMATEST_ET_VERIFY,
  99         DMATEST_ET_VERIFY_BUF,
 100 };
 101
 102 struct dmatest_verify_buffer {
 103         unsigned int    index;
 104         u8              expected;
 105         u8              actual;
 106 };
 107
 108 struct dmatest_verify_result {
 109         unsigned int                    error_count;
 110         struct dmatest_verify_buffer    data[MAX_ERROR_COUNT];
 111         u8                              pattern;
 112         bool                            is_srcbuf;
 113 };
 114
 115 struct dmatest_thread_result {
 116         struct list_head        node;
 117         unsigned int            n;
 118         unsigned int            src_off;
 119         unsigned int            dst_off;
 120         unsigned int            len;
 121         enum dmatest_error_type type;
 122         union {
 123                 unsigned long                   data;
 124                 dma_cookie_t                    cookie;
 125                 enum dma_status                 status;
 126                 int                             error;
 127                 struct dmatest_verify_result    *vr;
 128         };
 129 };
 130
 131 struct dmatest_result {
 132         struct list_head        node;
 133         char                    *name;
 134         struct list_head        results;
 135 };
 136
 137 struct dmatest_info;
 138
 139 struct dmatest_thread {
 140         struct list_head        node;
 141         struct dmatest_info     *info;
 142         struct task_struct      *task;
 143         struct dma_chan         *chan;
 144         u8                      **srcs;
 145         u8                      **dsts;
 146         enum dma_transaction_type type;
 147         bool                    done;
 148 };
 149
 150 struct dmatest_chan {
 151         struct list_head        node;
 152         struct dma_chan         *chan;
 153         struct list_head        threads;
 154 };
 155
 156 /**
 157  * struct dmatest_params - test parameters.
 158  * @buf_size:           size of the memcpy test buffer
 159  * @channel:            bus ID of the channel to test
 160  * @device:             bus ID of the DMA Engine to test
 161  * @threads_per_chan:   number of threads to start per channel
 162  * @max_channels:       maximum number of channels to use
 163  * @iterations:         iterations before stopping test
 164  * @xor_sources:        number of xor source buffers
 165  * @pq_sources:         number of p+q source buffers
 166  * @timeout:            transfer timeout in msec, -1 for infinite timeout
 167  */
 168 struct dmatest_params {
 169         unsigned int    buf_size;
 170         char            channel[20];
 171         char            device[20];
 172         unsigned int    threads_per_chan;
 173         unsigned int    max_channels;
 174         unsigned int    iterations;
 175         unsigned int    xor_sources;
 176         unsigned int    pq_sources;
 177         int             timeout;
 178 };
 179
 180 /**
 181  * struct dmatest_info - test information.
 182  * @params:             test parameters
 183  * @lock:               access protection to the fields of this structure
 184  */
 185 struct dmatest_info {
 186         /* Test parameters */
 187         struct dmatest_params   params;
 188
 189         /* Internal state */
 190         struct list_head        channels;
 191         unsigned int            nr_channels;
 192         struct mutex            lock;
 193
 194         /* debugfs related stuff */
 195         struct dentry           *root;
 196         struct dmatest_params   dbgfs_params;
 197
 198         /* Test results */
 199         struct list_head        results;
 200         struct mutex            results_lock;
 201 };
 202
 203 static struct dmatest_info test_info;
 204
 205 static bool dmatest_match_channel(struct dmatest_params *params,
 206                 struct dma_chan *chan)
 207 {
 208         if (params->channel[0] == '\0')
 209                 return true;
 210         return strcmp(dma_chan_name(chan), params->channel) == 0;
 211 }
 212
 213 static bool dmatest_match_device(struct dmatest_params *params,
 214                 struct dma_device *device)
 215 {
 216         if (params->device[0] == '\0')
 217                 return true;
 218         return strcmp(dev_name(device->dev), params->device) == 0;
 219 }
 220
 221 static unsigned long dmatest_random(void)
 222 {
 223         unsigned long buf;
 224
 225         get_random_bytes(&buf, sizeof(buf));
 226         return buf;
 227 }
 228
 229 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
 230                 unsigned int buf_size)
 231 {
 232         unsigned int i;
 233         u8 *buf;
 234
 235         for (; (buf = *bufs); bufs++) {
 236                 for (i = 0; i < start; i++)
 237                         buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 238                 for ( ; i < start + len; i++)
 239                         buf[i] = PATTERN_SRC | PATTERN_COPY
 240                                 | (~i & PATTERN_COUNT_MASK);
 241                 for ( ; i < buf_size; i++)
 242                         buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 243                 buf++;
 244         }
 245 }
 246
 247 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
 248                 unsigned int buf_size)
 249 {
 250         unsigned int i;
 251         u8 *buf;
 252
 253         for (; (buf = *bufs); bufs++) {
 254                 for (i = 0; i < start; i++)
 255                         buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 256                 for ( ; i < start + len; i++)
 257                         buf[i] = PATTERN_DST | PATTERN_OVERWRITE
 258                                 | (~i & PATTERN_COUNT_MASK);
 259                 for ( ; i < buf_size; i++)
 260                         buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 261         }
 262 }
 263
 264 static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs,
 265                 unsigned int start, unsigned int end, unsigned int counter,
 266                 u8 pattern, bool is_srcbuf)
 267 {
 268         unsigned int i;
 269         unsigned int error_count = 0;
 270         u8 actual;
 271         u8 expected;
 272         u8 *buf;
 273         unsigned int counter_orig = counter;
 274         struct dmatest_verify_buffer *vb;
 275
 276         for (; (buf = *bufs); bufs++) {
 277                 counter = counter_orig;
 278                 for (i = start; i < end; i++) {
 279                         actual = buf[i];
 280                         expected = pattern | (~counter & PATTERN_COUNT_MASK);
 281                         if (actual != expected) {
 282                                 if (error_count < MAX_ERROR_COUNT && vr) {
 283                                         vb = &vr->data[error_count];
 284                                         vb->index = i;
 285                                         vb->expected = expected;
 286                                         vb->actual = actual;
 287                                 }
 288                                 error_count++;
 289                         }
 290                         counter++;
 291                 }
 292         }
 293
 294         if (error_count > MAX_ERROR_COUNT)
 295                 pr_warning("%s: %u errors suppressed\n",
 296                         current->comm, error_count - MAX_ERROR_COUNT);
 297
 298         return error_count;
 299 }
 300
 301 /* poor man's completion - we want to use wait_event_freezable() on it */
 302 struct dmatest_done {
 303         bool                    done;
 304         wait_queue_head_t       *wait;
 305 };
 306
 307 static void dmatest_callback(void *arg)
 308 {
 309         struct dmatest_done *done = arg;
 310
 311         done->done = true;
 312         wake_up_all(done->wait);
 313 }
 314
 315 static inline void unmap_src(struct device *dev, dma_addr_t *addr, size_t len,
 316                              unsigned int count)
 317 {
 318         while (count--)
 319                 dma_unmap_single(dev, addr[count], len, DMA_TO_DEVICE);
 320 }
 321
 322 static inline void unmap_dst(struct device *dev, dma_addr_t *addr, size_t len,
 323                              unsigned int count)
 324 {
 325         while (count--)
 326                 dma_unmap_single(dev, addr[count], len, DMA_BIDIRECTIONAL);
 327 }
 328
 329 static unsigned int min_odd(unsigned int x, unsigned int y)
 330 {
 331         unsigned int val = min(x, y);
 332
 333         return val % 2 ? val : val - 1;
 334 }
 335
 336 static char *verify_result_get_one(struct dmatest_verify_result *vr,
 337                 unsigned int i)
 338 {
 339         struct dmatest_verify_buffer *vb = &vr->data[i];
 340         u8 diff = vb->actual ^ vr->pattern;
 341         static char buf[512];
 342         char *msg;
 343
 344         if (vr->is_srcbuf)
 345                 msg = "srcbuf overwritten!";
 346         else if ((vr->pattern & PATTERN_COPY)
 347                         && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
 348                 msg = "dstbuf not copied!";
 349         else if (diff & PATTERN_SRC)
 350                 msg = "dstbuf was copied!";
 351         else
 352                 msg = "dstbuf mismatch!";
 353
 354         snprintf(buf, sizeof(buf) - 1, "%s [0x%x] Expected %02x, got %02x", msg,
 355                  vb->index, vb->expected, vb->actual);
 356
 357         return buf;
 358 }
 359
 360 static char *thread_result_get(const char *name,
 361                 struct dmatest_thread_result *tr)
 362 {
 363         static const char * const messages[] = {
 364                 [DMATEST_ET_OK]                 = "No errors",
 365                 [DMATEST_ET_MAP_SRC]            = "src mapping error",
 366                 [DMATEST_ET_MAP_DST]            = "dst mapping error",
 367                 [DMATEST_ET_PREP]               = "prep error",
 368                 [DMATEST_ET_SUBMIT]             = "submit error",
 369                 [DMATEST_ET_TIMEOUT]            = "test timed out",
 370                 [DMATEST_ET_DMA_ERROR]          =
 371                         "got completion callback (DMA_ERROR)",
 372                 [DMATEST_ET_DMA_IN_PROGRESS]    =
 373                         "got completion callback (DMA_IN_PROGRESS)",
 374                 [DMATEST_ET_VERIFY]             = "errors",
 375                 [DMATEST_ET_VERIFY_BUF]         = "verify errors",
 376         };
 377         static char buf[512];
 378
 379         snprintf(buf, sizeof(buf) - 1,
 380                  "%s: #%u: %s with src_off=0x%x ""dst_off=0x%x len=0x%x (%lu)",
 381                  name, tr->n, messages[tr->type], tr->src_off, tr->dst_off,
 382                  tr->len, tr->data);
 383
 384         return buf;
 385 }
 386
 387 static int thread_result_add(struct dmatest_info *info,
 388                 struct dmatest_result *r, enum dmatest_error_type type,
 389                 unsigned int n, unsigned int src_off, unsigned int dst_off,
 390                 unsigned int len, unsigned long data)
 391 {
 392         struct dmatest_thread_result *tr;
 393
 394         tr = kzalloc(sizeof(*tr), GFP_KERNEL);
 395         if (!tr)
 396                 return -ENOMEM;
 397
 398         tr->type = type;
 399         tr->n = n;
 400         tr->src_off = src_off;
 401         tr->dst_off = dst_off;
 402         tr->len = len;
 403         tr->data = data;
 404
 405         mutex_lock(&info->results_lock);
 406         list_add_tail(&tr->node, &r->results);
 407         mutex_unlock(&info->results_lock);
 408
 409         pr_warn("%s\n", thread_result_get(r->name, tr));
 410         return 0;
 411 }
 412
 413 static unsigned int verify_result_add(struct dmatest_info *info,
 414                 struct dmatest_result *r, unsigned int n,
 415                 unsigned int src_off, unsigned int dst_off, unsigned int len,
 416                 u8 **bufs, int whence, unsigned int counter, u8 pattern,
 417                 bool is_srcbuf)
 418 {
 419         struct dmatest_verify_result *vr;
 420         unsigned int error_count;
 421         unsigned int buf_off = is_srcbuf ? src_off : dst_off;
 422         unsigned int start, end;
 423
 424         if (whence < 0) {
 425                 start = 0;
 426                 end = buf_off;
 427         } else if (whence > 0) {
 428                 start = buf_off + len;
 429                 end = info->params.buf_size;
 430         } else {
 431                 start = buf_off;
 432                 end = buf_off + len;
 433         }
 434
 435         vr = kmalloc(sizeof(*vr), GFP_KERNEL);
 436         if (!vr) {
 437                 pr_warn("dmatest: No memory to store verify result\n");
 438                 return dmatest_verify(NULL, bufs, start, end, counter, pattern,
 439                                       is_srcbuf);
 440         }
 441
 442         vr->pattern = pattern;
 443         vr->is_srcbuf = is_srcbuf;
 444
 445         error_count = dmatest_verify(vr, bufs, start, end, counter, pattern,
 446                                      is_srcbuf);
 447         if (error_count) {
 448                 vr->error_count = error_count;
 449                 thread_result_add(info, r, DMATEST_ET_VERIFY_BUF, n, src_off,
 450                                   dst_off, len, (unsigned long)vr);
 451                 return error_count;
 452         }
 453
 454         kfree(vr);
 455         return 0;
 456 }
 457
 458 static void result_free(struct dmatest_info *info, const char *name)
 459 {
 460         struct dmatest_result *r, *_r;
 461
 462         mutex_lock(&info->results_lock);
 463         list_for_each_entry_safe(r, _r, &info->results, node) {
 464                 struct dmatest_thread_result *tr, *_tr;
 465
 466                 if (name && strcmp(r->name, name))
 467                         continue;
 468
 469                 list_for_each_entry_safe(tr, _tr, &r->results, node) {
 470                         if (tr->type == DMATEST_ET_VERIFY_BUF)
 471                                 kfree(tr->vr);
 472                         list_del(&tr->node);
 473                         kfree(tr);
 474                 }
 475
 476                 kfree(r->name);
 477                 list_del(&r->node);
 478                 kfree(r);
 479         }
 480
 481         mutex_unlock(&info->results_lock);
 482 }
 483
 484 static struct dmatest_result *result_init(struct dmatest_info *info,
 485                 const char *name)
 486 {
 487         struct dmatest_result *r;
 488
 489         r = kzalloc(sizeof(*r), GFP_KERNEL);
 490         if (r) {
 491                 r->name = kstrdup(name, GFP_KERNEL);
 492                 INIT_LIST_HEAD(&r->results);
 493                 mutex_lock(&info->results_lock);
 494                 list_add_tail(&r->node, &info->results);
 495                 mutex_unlock(&info->results_lock);
 496         }
 497         return r;
 498 }
 499
 500 /*
 501  * This function repeatedly tests DMA transfers of various lengths and
 502  * offsets for a given operation type until it is told to exit by
 503  * kthread_stop(). There may be multiple threads running this function
 504  * in parallel for a single channel, and there may be multiple channels
 505  * being tested in parallel.
 506  *
 507  * Before each test, the source and destination buffer is initialized
 508  * with a known pattern. This pattern is different depending on
 509  * whether it's in an area which is supposed to be copied or
 510  * overwritten, and different in the source and destination buffers.
 511  * So if the DMA engine doesn't copy exactly what we tell it to copy,
 512  * we'll notice.
 513  */
 514 static int dmatest_func(void *data)
 515 {
 516         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait);
 517         struct dmatest_thread   *thread = data;
 518         struct dmatest_done     done = { .wait = &done_wait };
 519         struct dmatest_info     *info;
 520         struct dmatest_params   *params;
 521         struct dma_chan         *chan;
 522         struct dma_device       *dev;
 523         const char              *thread_name;
 524         unsigned int            src_off, dst_off, len;
 525         unsigned int            error_count;
 526         unsigned int            failed_tests = 0;
 527         unsigned int            total_tests = 0;
 528         dma_cookie_t            cookie;
 529         enum dma_status         status;
 530         enum dma_ctrl_flags     flags;
 531         u8                      *pq_coefs = NULL;
 532         int                     ret;
 533         int                     src_cnt;
 534         int                     dst_cnt;
 535         int                     i;
 536         struct dmatest_result   *result;
 537
 538         thread_name = current->comm;
 539         set_freezable();
 540
 541         ret = -ENOMEM;
 542
 543         smp_rmb();
 544         info = thread->info;
 545         params = &info->params;
 546         chan = thread->chan;
 547         dev = chan->device;
 548         if (thread->type == DMA_MEMCPY)
 549                 src_cnt = dst_cnt = 1;
 550         else if (thread->type == DMA_XOR) {
 551                 /* force odd to ensure dst = src */
 552                 src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
 553                 dst_cnt = 1;
 554         } else if (thread->type == DMA_PQ) {
 555                 /* force odd to ensure dst = src */
 556                 src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
 557                 dst_cnt = 2;
 558
 559                 pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL);
 560                 if (!pq_coefs)
 561                         goto err_thread_type;
 562
 563                 for (i = 0; i < src_cnt; i++)
 564                         pq_coefs[i] = 1;
 565         } else
 566                 goto err_thread_type;
 567
 568         result = result_init(info, thread_name);
 569         if (!result)
 570                 goto err_srcs;
 571
 572         thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
 573         if (!thread->srcs)
 574                 goto err_srcs;
 575         for (i = 0; i < src_cnt; i++) {
 576                 thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL);
 577                 if (!thread->srcs[i])
 578                         goto err_srcbuf;
 579         }
 580         thread->srcs[i] = NULL;
 581
 582         thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
 583         if (!thread->dsts)
 584                 goto err_dsts;
 585         for (i = 0; i < dst_cnt; i++) {
 586                 thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL);
 587                 if (!thread->dsts[i])
 588                         goto err_dstbuf;
 589         }
 590         thread->dsts[i] = NULL;
 591
 592         set_user_nice(current, 10);
 593
 594         /*
 595          * src buffers are freed by the DMAEngine code with dma_unmap_single()
 596          * dst buffers are freed by ourselves below
 597          */
 598         flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT
 599               | DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SRC_UNMAP_SINGLE;
 600
 601         while (!kthread_should_stop()
 602                && !(params->iterations && total_tests >= params->iterations)) {
 603                 struct dma_async_tx_descriptor *tx = NULL;
 604                 dma_addr_t dma_srcs[src_cnt];
 605                 dma_addr_t dma_dsts[dst_cnt];
 606                 u8 align = 0;
 607
 608                 total_tests++;
 609
 610                 /* honor alignment restrictions */
 611                 if (thread->type == DMA_MEMCPY)
 612                         align = dev->copy_align;
 613                 else if (thread->type == DMA_XOR)
 614                         align = dev->xor_align;
 615                 else if (thread->type == DMA_PQ)
 616                         align = dev->pq_align;
 617
 618                 if (1 << align > params->buf_size) {
 619                         pr_err("%u-byte buffer too small for %d-byte alignment\n",
 620                                params->buf_size, 1 << align);
 621                         break;
 622                 }
 623
 624                 len = dmatest_random() % params->buf_size + 1;
 625                 len = (len >> align) << align;
 626                 if (!len)
 627                         len = 1 << align;
 628                 src_off = dmatest_random() % (params->buf_size - len + 1);
 629                 dst_off = dmatest_random() % (params->buf_size - len + 1);
 630
 631                 src_off = (src_off >> align) << align;
 632                 dst_off = (dst_off >> align) << align;
 633
 634                 dmatest_init_srcs(thread->srcs, src_off, len, params->buf_size);
 635                 dmatest_init_dsts(thread->dsts, dst_off, len, params->buf_size);
 636
 637                 for (i = 0; i < src_cnt; i++) {
 638                         u8 *buf = thread->srcs[i] + src_off;
 639
 640                         dma_srcs[i] = dma_map_single(dev->dev, buf, len,
 641                                                      DMA_TO_DEVICE);
 642                         ret = dma_mapping_error(dev->dev, dma_srcs[i]);
 643                         if (ret) {
 644                                 unmap_src(dev->dev, dma_srcs, len, i);
 645                                 thread_result_add(info, result,
 646                                                   DMATEST_ET_MAP_SRC,
 647                                                   total_tests, src_off, dst_off,
 648                                                   len, ret);
 649                                 failed_tests++;
 650                                 continue;
 651                         }
 652                 }
 653                 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
 654                 for (i = 0; i < dst_cnt; i++) {
 655                         dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i],
 656                                                      params->buf_size,
 657                                                      DMA_BIDIRECTIONAL);
 658                         ret = dma_mapping_error(dev->dev, dma_dsts[i]);
 659                         if (ret) {
 660                                 unmap_src(dev->dev, dma_srcs, len, src_cnt);
 661                                 unmap_dst(dev->dev, dma_dsts, params->buf_size,
 662                                           i);
 663                                 thread_result_add(info, result,
 664                                                   DMATEST_ET_MAP_DST,
 665                                                   total_tests, src_off, dst_off,
 666                                                   len, ret);
 667                                 failed_tests++;
 668                                 continue;
 669                         }
 670                 }
 671
 672                 if (thread->type == DMA_MEMCPY)
 673                         tx = dev->device_prep_dma_memcpy(chan,
 674                                                          dma_dsts[0] + dst_off,
 675                                                          dma_srcs[0], len,
 676                                                          flags);
 677                 else if (thread->type == DMA_XOR)
 678                         tx = dev->device_prep_dma_xor(chan,
 679                                                       dma_dsts[0] + dst_off,
 680                                                       dma_srcs, src_cnt,
 681                                                       len, flags);
 682                 else if (thread->type == DMA_PQ) {
 683                         dma_addr_t dma_pq[dst_cnt];
 684
 685                         for (i = 0; i < dst_cnt; i++)
 686                                 dma_pq[i] = dma_dsts[i] + dst_off;
 687                         tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs,
 688                                                      src_cnt, pq_coefs,
 689                                                      len, flags);
 690                 }
 691
 692                 if (!tx) {
 693                         unmap_src(dev->dev, dma_srcs, len, src_cnt);
 694                         unmap_dst(dev->dev, dma_dsts, params->buf_size,
 695                                   dst_cnt);
 696                         thread_result_add(info, result, DMATEST_ET_PREP,
 697                                           total_tests, src_off, dst_off,
 698                                           len, 0);
 699                         msleep(100);
 700                         failed_tests++;
 701                         continue;
 702                 }
 703
 704                 done.done = false;
 705                 tx->callback = dmatest_callback;
 706                 tx->callback_param = &done;
 707                 cookie = tx->tx_submit(tx);
 708
 709                 if (dma_submit_error(cookie)) {
 710                         thread_result_add(info, result, DMATEST_ET_SUBMIT,
 711                                           total_tests, src_off, dst_off,
 712                                           len, cookie);
 713                         msleep(100);
 714                         failed_tests++;
 715                         continue;
 716                 }
 717                 dma_async_issue_pending(chan);
 718
 719                 wait_event_freezable_timeout(done_wait, done.done,
 720                                              msecs_to_jiffies(params->timeout));
 721
 722                 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
 723
 724                 if (!done.done) {
 725                         /*
 726                          * We're leaving the timed out dma operation with
 727                          * dangling pointer to done_wait.  To make this
 728                          * correct, we'll need to allocate wait_done for
 729                          * each test iteration and perform "who's gonna
 730                          * free it this time?" dancing.  For now, just
 731                          * leave it dangling.
 732                          */
 733                         thread_result_add(info, result, DMATEST_ET_TIMEOUT,
 734                                           total_tests, src_off, dst_off,
 735                                           len, 0);
 736                         failed_tests++;
 737                         continue;
 738                 } else if (status != DMA_SUCCESS) {
 739                         enum dmatest_error_type type = (status == DMA_ERROR) ?
 740                                 DMATEST_ET_DMA_ERROR : DMATEST_ET_DMA_IN_PROGRESS;
 741                         thread_result_add(info, result, type,
 742                                           total_tests, src_off, dst_off,
 743                                           len, status);
 744                         failed_tests++;
 745                         continue;
 746                 }
 747
 748                 /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */
 749                 unmap_dst(dev->dev, dma_dsts, params->buf_size, dst_cnt);
 750
 751                 error_count = 0;
 752
 753                 pr_debug("%s: verifying source buffer...\n", thread_name);
 754                 error_count += verify_result_add(info, result, total_tests,
 755                                 src_off, dst_off, len, thread->srcs, -1,
 756                                 0, PATTERN_SRC, true);
 757                 error_count += verify_result_add(info, result, total_tests,
 758                                 src_off, dst_off, len, thread->srcs, 0,
 759                                 src_off, PATTERN_SRC | PATTERN_COPY, true);
 760                 error_count += verify_result_add(info, result, total_tests,
 761                                 src_off, dst_off, len, thread->srcs, 1,
 762                                 src_off + len, PATTERN_SRC, true);
 763
 764                 pr_debug("%s: verifying dest buffer...\n", thread_name);
 765                 error_count += verify_result_add(info, result, total_tests,
 766                                 src_off, dst_off, len, thread->dsts, -1,
 767                                 0, PATTERN_DST, false);
 768                 error_count += verify_result_add(info, result, total_tests,
 769                                 src_off, dst_off, len, thread->dsts, 0,
 770                                 src_off, PATTERN_SRC | PATTERN_COPY, false);
 771                 error_count += verify_result_add(info, result, total_tests,
 772                                 src_off, dst_off, len, thread->dsts, 1,
 773                                 dst_off + len, PATTERN_DST, false);
 774
 775                 if (error_count) {
 776                         thread_result_add(info, result, DMATEST_ET_VERIFY,
 777                                           total_tests, src_off, dst_off,
 778                                           len, error_count);
 779                         failed_tests++;
 780                 } else {
 781                         thread_result_add(info, result, DMATEST_ET_OK,
 782                                           total_tests, src_off, dst_off,
 783                                           len, 0);
 784                 }
 785         }
 786
 787         ret = 0;
 788         for (i = 0; thread->dsts[i]; i++)
 789                 kfree(thread->dsts[i]);
 790 err_dstbuf:
 791         kfree(thread->dsts);
 792 err_dsts:
 793         for (i = 0; thread->srcs[i]; i++)
 794                 kfree(thread->srcs[i]);
 795 err_srcbuf:
 796         kfree(thread->srcs);
 797 err_srcs:
 798         kfree(pq_coefs);
 799 err_thread_type:
 800         pr_notice("%s: terminating after %u tests, %u failures (status %d)\n",
 801                         thread_name, total_tests, failed_tests, ret);
 802
 803         /* terminate all transfers on specified channels */
 804         if (ret)
 805                 dmaengine_terminate_all(chan);
 806
 807         thread->done = true;
 808
 809         if (params->iterations > 0)
 810                 while (!kthread_should_stop()) {
 811                         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit);
 812                         interruptible_sleep_on(&wait_dmatest_exit);
 813                 }
 814
 815         return ret;
 816 }
 817
 818 static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
 819 {
 820         struct dmatest_thread   *thread;
 821         struct dmatest_thread   *_thread;
 822         int                     ret;
 823
 824         list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
 825                 ret = kthread_stop(thread->task);
 826                 pr_debug("dmatest: thread %s exited with status %d\n",
 827                                 thread->task->comm, ret);
 828                 list_del(&thread->node);
 829                 kfree(thread);
 830         }
 831
 832         /* terminate all transfers on specified channels */
 833         dmaengine_terminate_all(dtc->chan);
 834
 835         kfree(dtc);
 836 }
 837
 838 static int dmatest_add_threads(struct dmatest_info *info,
 839                 struct dmatest_chan *dtc, enum dma_transaction_type type)
 840 {
 841         struct dmatest_params *params = &info->params;
 842         struct dmatest_thread *thread;
 843         struct dma_chan *chan = dtc->chan;
 844         char *op;
 845         unsigned int i;
 846
 847         if (type == DMA_MEMCPY)
 848                 op = "copy";
 849         else if (type == DMA_XOR)
 850                 op = "xor";
 851         else if (type == DMA_PQ)
 852                 op = "pq";
 853         else
 854                 return -EINVAL;
 855
 856         for (i = 0; i < params->threads_per_chan; i++) {
 857                 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
 858                 if (!thread) {
 859                         pr_warning("dmatest: No memory for %s-%s%u\n",
 860                                    dma_chan_name(chan), op, i);
 861
 862                         break;
 863                 }
 864                 thread->info = info;
 865                 thread->chan = dtc->chan;
 866                 thread->type = type;
 867                 smp_wmb();
 868                 thread->task = kthread_run(dmatest_func, thread, "%s-%s%u",
 869                                 dma_chan_name(chan), op, i);
 870                 if (IS_ERR(thread->task)) {
 871                         pr_warning("dmatest: Failed to run thread %s-%s%u\n",
 872                                         dma_chan_name(chan), op, i);
 873                         kfree(thread);
 874                         break;
 875                 }
 876
 877                 /* srcbuf and dstbuf are allocated by the thread itself */
 878
 879                 list_add_tail(&thread->node, &dtc->threads);
 880         }
 881
 882         return i;
 883 }
 884
 885 static int dmatest_add_channel(struct dmatest_info *info,
 886                 struct dma_chan *chan)
 887 {
 888         struct dmatest_chan     *dtc;
 889         struct dma_device       *dma_dev = chan->device;
 890         unsigned int            thread_count = 0;
 891         int cnt;
 892
 893         dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
 894         if (!dtc) {
 895                 pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan));
 896                 return -ENOMEM;
 897         }
 898
 899         dtc->chan = chan;
 900         INIT_LIST_HEAD(&dtc->threads);
 901
 902         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
 903                 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
 904                 thread_count += cnt > 0 ? cnt : 0;
 905         }
 906         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
 907                 cnt = dmatest_add_threads(info, dtc, DMA_XOR);
 908                 thread_count += cnt > 0 ? cnt : 0;
 909         }
 910         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
 911                 cnt = dmatest_add_threads(info, dtc, DMA_PQ);
 912                 thread_count += cnt > 0 ? cnt : 0;
 913         }
 914
 915         pr_info("dmatest: Started %u threads using %s\n",
 916                 thread_count, dma_chan_name(chan));
 917
 918         list_add_tail(&dtc->node, &info->channels);
 919         info->nr_channels++;
 920
 921         return 0;
 922 }
 923
 924 static bool filter(struct dma_chan *chan, void *param)
 925 {
 926         struct dmatest_params *params = param;
 927
 928         if (!dmatest_match_channel(params, chan) ||
 929             !dmatest_match_device(params, chan->device))
 930                 return false;
 931         else
 932                 return true;
 933 }
 934
 935 static int __run_threaded_test(struct dmatest_info *info)
 936 {
 937         dma_cap_mask_t mask;
 938         struct dma_chan *chan;
 939         struct dmatest_params *params = &info->params;
 940         int err = 0;
 941
 942         dma_cap_zero(mask);
 943         dma_cap_set(DMA_MEMCPY, mask);
 944         for (;;) {
 945                 chan = dma_request_channel(mask, filter, params);
 946                 if (chan) {
 947                         err = dmatest_add_channel(info, chan);
 948                         if (err) {
 949                                 dma_release_channel(chan);
 950                                 break; /* add_channel failed, punt */
 951                         }
 952                 } else
 953                         break; /* no more channels available */
 954                 if (params->max_channels &&
 955                     info->nr_channels >= params->max_channels)
 956                         break; /* we have all we need */
 957         }
 958         return err;
 959 }
 960
 961 #ifndef MODULE
 962 static int run_threaded_test(struct dmatest_info *info)
 963 {
 964         int ret;
 965
 966         mutex_lock(&info->lock);
 967         ret = __run_threaded_test(info);
 968         mutex_unlock(&info->lock);
 969         return ret;
 970 }
 971 #endif
 972
 973 static void __stop_threaded_test(struct dmatest_info *info)
 974 {
 975         struct dmatest_chan *dtc, *_dtc;
 976         struct dma_chan *chan;
 977
 978         list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
 979                 list_del(&dtc->node);
 980                 chan = dtc->chan;
 981                 dmatest_cleanup_channel(dtc);
 982                 pr_debug("dmatest: dropped channel %s\n", dma_chan_name(chan));
 983                 dma_release_channel(chan);
 984         }
 985
 986         info->nr_channels = 0;
 987 }
 988
 989 static void stop_threaded_test(struct dmatest_info *info)
 990 {
 991         mutex_lock(&info->lock);
 992         __stop_threaded_test(info);
 993         mutex_unlock(&info->lock);
 994 }
 995
 996 static int __restart_threaded_test(struct dmatest_info *info, bool run)
 997 {
 998         struct dmatest_params *params = &info->params;
 999
1000         /* Stop any running test first */
1001         __stop_threaded_test(info);
1002
1003         if (run == false)
1004                 return 0;
1005
1006         /* Clear results from previous run */
1007         result_free(info, NULL);
1008
1009         /* Copy test parameters */
1010         memcpy(params, &info->dbgfs_params, sizeof(*params));
1011
1012         /* Run test with new parameters */
1013         return __run_threaded_test(info);
1014 }
1015
1016 static bool __is_threaded_test_run(struct dmatest_info *info)
1017 {
1018         struct dmatest_chan *dtc;
1019
1020         list_for_each_entry(dtc, &info->channels, node) {
1021                 struct dmatest_thread *thread;
1022
1023                 list_for_each_entry(thread, &dtc->threads, node) {
1024                         if (!thread->done)
1025                                 return true;
1026                 }
1027         }
1028
1029         return false;
1030 }
1031
1032 static ssize_t dtf_write_string(void *to, size_t available, loff_t *ppos,
1033                 const void __user *from, size_t count)
1034 {
1035         char tmp[20];
1036         ssize_t len;
1037
1038         len = simple_write_to_buffer(tmp, sizeof(tmp) - 1, ppos, from, count);
1039         if (len >= 0) {
1040                 tmp[len] = '\0';
1041                 strlcpy(to, strim(tmp), available);
1042         }
1043
1044         return len;
1045 }
1046
1047 static ssize_t dtf_read_channel(struct file *file, char __user *buf,
1048                 size_t count, loff_t *ppos)
1049 {
1050         struct dmatest_info *info = file->private_data;
1051         return simple_read_from_buffer(buf, count, ppos,
1052                         info->dbgfs_params.channel,
1053                         strlen(info->dbgfs_params.channel));
1054 }
1055
1056 static ssize_t dtf_write_channel(struct file *file, const char __user *buf,
1057                 size_t size, loff_t *ppos)
1058 {
1059         struct dmatest_info *info = file->private_data;
1060         return dtf_write_string(info->dbgfs_params.channel,
1061                                 sizeof(info->dbgfs_params.channel),
1062                                 ppos, buf, size);
1063 }
1064
1065 static const struct file_operations dtf_channel_fops = {
1066         .read   = dtf_read_channel,
1067         .write  = dtf_write_channel,
1068         .open   = simple_open,
1069         .llseek = default_llseek,
1070 };
1071
1072 static ssize_t dtf_read_device(struct file *file, char __user *buf,
1073                 size_t count, loff_t *ppos)
1074 {
1075         struct dmatest_info *info = file->private_data;
1076         return simple_read_from_buffer(buf, count, ppos,
1077                         info->dbgfs_params.device,
1078                         strlen(info->dbgfs_params.device));
1079 }
1080
1081 static ssize_t dtf_write_device(struct file *file, const char __user *buf,
1082                 size_t size, loff_t *ppos)
1083 {
1084         struct dmatest_info *info = file->private_data;
1085         return dtf_write_string(info->dbgfs_params.device,
1086                                 sizeof(info->dbgfs_params.device),
1087                                 ppos, buf, size);
1088 }
1089
1090 static const struct file_operations dtf_device_fops = {
1091         .read   = dtf_read_device,
1092         .write  = dtf_write_device,
1093         .open   = simple_open,
1094         .llseek = default_llseek,
1095 };
1096
1097 static ssize_t dtf_read_run(struct file *file, char __user *user_buf,
1098                 size_t count, loff_t *ppos)
1099 {
1100         struct dmatest_info *info = file->private_data;
1101         char buf[3];
1102
1103         mutex_lock(&info->lock);
1104
1105         if (__is_threaded_test_run(info)) {
1106                 buf[0] = 'Y';
1107         } else {
1108                 __stop_threaded_test(info);
1109                 buf[0] = 'N';
1110         }
1111
1112         mutex_unlock(&info->lock);
1113         buf[1] = '\n';
1114         buf[2] = 0x00;
1115         return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1116 }
1117
1118 static ssize_t dtf_write_run(struct file *file, const char __user *user_buf,
1119                 size_t count, loff_t *ppos)
1120 {
1121         struct dmatest_info *info = file->private_data;
1122         char buf[16];
1123         bool bv;
1124         int ret = 0;
1125
1126         if (copy_from_user(buf, user_buf, min(count, (sizeof(buf) - 1))))
1127                 return -EFAULT;
1128
1129         if (strtobool(buf, &bv) == 0) {
1130                 mutex_lock(&info->lock);
1131
1132                 if (__is_threaded_test_run(info))
1133                         ret = -EBUSY;
1134                 else
1135                         ret = __restart_threaded_test(info, bv);
1136
1137                 mutex_unlock(&info->lock);
1138         }
1139
1140         return ret ? ret : count;
1141 }
1142
1143 static const struct file_operations dtf_run_fops = {
1144         .read   = dtf_read_run,
1145         .write  = dtf_write_run,
1146         .open   = simple_open,
1147         .llseek = default_llseek,
1148 };
1149
1150 static int dtf_results_show(struct seq_file *sf, void *data)
1151 {
1152         struct dmatest_info *info = sf->private;
1153         struct dmatest_result *result;
1154         struct dmatest_thread_result *tr;
1155         unsigned int i;
1156
1157         mutex_lock(&info->results_lock);
1158         list_for_each_entry(result, &info->results, node) {
1159                 list_for_each_entry(tr, &result->results, node) {
1160                         seq_printf(sf, "%s\n",
1161                                 thread_result_get(result->name, tr));
1162                         if (tr->type == DMATEST_ET_VERIFY_BUF) {
1163                                 for (i = 0; i < tr->vr->error_count; i++) {
1164                                         seq_printf(sf, "\t%s\n",
1165                                                 verify_result_get_one(tr->vr, i));
1166                                 }
1167                         }
1168                 }
1169         }
1170
1171         mutex_unlock(&info->results_lock);
1172         return 0;
1173 }
1174
1175 static int dtf_results_open(struct inode *inode, struct file *file)
1176 {
1177         return single_open(file, dtf_results_show, inode->i_private);
1178 }
1179
1180 static const struct file_operations dtf_results_fops = {
1181         .open           = dtf_results_open,
1182         .read           = seq_read,
1183         .llseek         = seq_lseek,
1184         .release        = single_release,
1185 };
1186
1187 static int dmatest_register_dbgfs(struct dmatest_info *info)
1188 {
1189         struct dentry *d;
1190         struct dmatest_params *params = &info->dbgfs_params;
1191         int ret = -ENOMEM;
1192
1193         d = debugfs_create_dir("dmatest", NULL);
1194         if (IS_ERR(d))
1195                 return PTR_ERR(d);
1196         if (!d)
1197                 goto err_root;
1198
1199         info->root = d;
1200
1201         /* Copy initial values */
1202         memcpy(params, &info->params, sizeof(*params));
1203
1204         /* Test parameters */
1205
1206         d = debugfs_create_u32("test_buf_size", S_IWUSR | S_IRUGO, info->root,
1207                                (u32 *)&params->buf_size);
1208         if (IS_ERR_OR_NULL(d))
1209                 goto err_node;
1210
1211         d = debugfs_create_file("channel", S_IRUGO | S_IWUSR, info->root,
1212                                 info, &dtf_channel_fops);
1213         if (IS_ERR_OR_NULL(d))
1214                 goto err_node;
1215
1216         d = debugfs_create_file("device", S_IRUGO | S_IWUSR, info->root,
1217                                 info, &dtf_device_fops);
1218         if (IS_ERR_OR_NULL(d))
1219                 goto err_node;
1220
1221         d = debugfs_create_u32("threads_per_chan", S_IWUSR | S_IRUGO, info->root,
1222                                (u32 *)&params->threads_per_chan);
1223         if (IS_ERR_OR_NULL(d))
1224                 goto err_node;
1225
1226         d = debugfs_create_u32("max_channels", S_IWUSR | S_IRUGO, info->root,
1227                                (u32 *)&params->max_channels);
1228         if (IS_ERR_OR_NULL(d))
1229                 goto err_node;
1230
1231         d = debugfs_create_u32("iterations", S_IWUSR | S_IRUGO, info->root,
1232                                (u32 *)&params->iterations);
1233         if (IS_ERR_OR_NULL(d))
1234                 goto err_node;
1235
1236         d = debugfs_create_u32("xor_sources", S_IWUSR | S_IRUGO, info->root,
1237                                (u32 *)&params->xor_sources);
1238         if (IS_ERR_OR_NULL(d))
1239                 goto err_node;
1240
1241         d = debugfs_create_u32("pq_sources", S_IWUSR | S_IRUGO, info->root,
1242                                (u32 *)&params->pq_sources);
1243         if (IS_ERR_OR_NULL(d))
1244                 goto err_node;
1245
1246         d = debugfs_create_u32("timeout", S_IWUSR | S_IRUGO, info->root,
1247                                (u32 *)&params->timeout);
1248         if (IS_ERR_OR_NULL(d))
1249                 goto err_node;
1250
1251         /* Run or stop threaded test */
1252         d = debugfs_create_file("run", S_IWUSR | S_IRUGO, info->root,
1253                                 info, &dtf_run_fops);
1254         if (IS_ERR_OR_NULL(d))
1255                 goto err_node;
1256
1257         /* Results of test in progress */
1258         d = debugfs_create_file("results", S_IRUGO, info->root, info,
1259                                 &dtf_results_fops);
1260         if (IS_ERR_OR_NULL(d))
1261                 goto err_node;
1262
1263         return 0;
1264
1265 err_node:
1266         debugfs_remove_recursive(info->root);
1267 err_root:
1268         pr_err("dmatest: Failed to initialize debugfs\n");
1269         return ret;
1270 }
1271
1272 static int __init dmatest_init(void)
1273 {
1274         struct dmatest_info *info = &test_info;
1275         struct dmatest_params *params = &info->params;
1276         int ret;
1277
1278         memset(info, 0, sizeof(*info));
1279
1280         mutex_init(&info->lock);
1281         INIT_LIST_HEAD(&info->channels);
1282
1283         mutex_init(&info->results_lock);
1284         INIT_LIST_HEAD(&info->results);
1285
1286         /* Set default parameters */
1287         params->buf_size = test_buf_size;
1288         strlcpy(params->channel, test_channel, sizeof(params->channel));
1289         strlcpy(params->device, test_device, sizeof(params->device));
1290         params->threads_per_chan = threads_per_chan;
1291         params->max_channels = max_channels;
1292         params->iterations = iterations;
1293         params->xor_sources = xor_sources;
1294         params->pq_sources = pq_sources;
1295         params->timeout = timeout;
1296
1297         ret = dmatest_register_dbgfs(info);
1298         if (ret)
1299                 return ret;
1300
1301 #ifdef MODULE
1302         return 0;
1303 #else
1304         return run_threaded_test(info);
1305 #endif
1306 }
1307 /* when compiled-in wait for drivers to load first */
1308 late_initcall(dmatest_init);
1309
1310 static void __exit dmatest_exit(void)
1311 {
1312         struct dmatest_info *info = &test_info;
1313
1314         debugfs_remove_recursive(info->root);
1315         stop_threaded_test(info);
1316         result_free(info, NULL);
1317 }
1318 module_exit(dmatest_exit);
1319
1320 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1321 MODULE_LICENSE("GPL v2");