drivers/dma/dmatest.c

   1 /*
   2  * DMA Engine test module
   3  *
   4  * Copyright (C) 2007 Atmel Corporation
   5  * Copyright (C) 2013 Intel Corporation
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/delay.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/freezer.h>
  15 #include <linux/init.h>
  16 #include <linux/kthread.h>
  17 #include <linux/module.h>
  18 #include <linux/moduleparam.h>
  19 #include <linux/random.h>
  20 #include <linux/slab.h>
  21 #include <linux/wait.h>
  22 #include <linux/ctype.h>
  23 #include <linux/debugfs.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/seq_file.h>
  26
  27 static unsigned int test_buf_size = 16384;
  28 module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
  29 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
  30
  31 static char test_channel[20];
  32 module_param_string(channel, test_channel, sizeof(test_channel),
  33                 S_IRUGO | S_IWUSR);
  34 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
  35
  36 static char test_device[20];
  37 module_param_string(device, test_device, sizeof(test_device),
  38                 S_IRUGO | S_IWUSR);
  39 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
  40
  41 static unsigned int threads_per_chan = 1;
  42 module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
  43 MODULE_PARM_DESC(threads_per_chan,
  44                 "Number of threads to start per channel (default: 1)");
  45
  46 static unsigned int max_channels;
  47 module_param(max_channels, uint, S_IRUGO | S_IWUSR);
  48 MODULE_PARM_DESC(max_channels,
  49                 "Maximum number of channels to use (default: all)");
  50
  51 static unsigned int iterations;
  52 module_param(iterations, uint, S_IRUGO | S_IWUSR);
  53 MODULE_PARM_DESC(iterations,
  54                 "Iterations before stopping test (default: infinite)");
  55
  56 static unsigned int xor_sources = 3;
  57 module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
  58 MODULE_PARM_DESC(xor_sources,
  59                 "Number of xor source buffers (default: 3)");
  60
  61 static unsigned int pq_sources = 3;
  62 module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
  63 MODULE_PARM_DESC(pq_sources,
  64                 "Number of p+q source buffers (default: 3)");
  65
  66 static int timeout = 3000;
  67 module_param(timeout, uint, S_IRUGO | S_IWUSR);
  68 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
  69                  "Pass -1 for infinite timeout");
  70
  71 /* Maximum amount of mismatched bytes in buffer to print */
  72 #define MAX_ERROR_COUNT         32
  73
  74 /*
  75  * Initialization patterns. All bytes in the source buffer has bit 7
  76  * set, all bytes in the destination buffer has bit 7 cleared.
  77  *
  78  * Bit 6 is set for all bytes which are to be copied by the DMA
  79  * engine. Bit 5 is set for all bytes which are to be overwritten by
  80  * the DMA engine.
  81  *
  82  * The remaining bits are the inverse of a counter which increments by
  83  * one for each byte address.
  84  */
  85 #define PATTERN_SRC             0x80
  86 #define PATTERN_DST             0x00
  87 #define PATTERN_COPY            0x40
  88 #define PATTERN_OVERWRITE       0x20
  89 #define PATTERN_COUNT_MASK      0x1f
  90
  91 enum dmatest_error_type {
  92         DMATEST_ET_OK,
  93         DMATEST_ET_MAP_SRC,
  94         DMATEST_ET_MAP_DST,
  95         DMATEST_ET_PREP,
  96         DMATEST_ET_SUBMIT,
  97         DMATEST_ET_TIMEOUT,
  98         DMATEST_ET_DMA_ERROR,
  99         DMATEST_ET_DMA_IN_PROGRESS,
 100         DMATEST_ET_VERIFY,
 101 };
 102
 103 struct dmatest_thread_result {
 104         struct list_head        node;
 105         unsigned int            n;
 106         unsigned int            src_off;
 107         unsigned int            dst_off;
 108         unsigned int            len;
 109         enum dmatest_error_type type;
 110         union {
 111                 unsigned long           data;
 112                 dma_cookie_t            cookie;
 113                 enum dma_status         status;
 114                 int                     error;
 115         };
 116 };
 117
 118 struct dmatest_result {
 119         struct list_head        node;
 120         char                    *name;
 121         struct list_head        results;
 122 };
 123
 124 struct dmatest_info;
 125
 126 struct dmatest_thread {
 127         struct list_head        node;
 128         struct dmatest_info     *info;
 129         struct task_struct      *task;
 130         struct dma_chan         *chan;
 131         u8                      **srcs;
 132         u8                      **dsts;
 133         enum dma_transaction_type type;
 134         bool                    done;
 135 };
 136
 137 struct dmatest_chan {
 138         struct list_head        node;
 139         struct dma_chan         *chan;
 140         struct list_head        threads;
 141 };
 142
 143 /**
 144  * struct dmatest_params - test parameters.
 145  * @buf_size:           size of the memcpy test buffer
 146  * @channel:            bus ID of the channel to test
 147  * @device:             bus ID of the DMA Engine to test
 148  * @threads_per_chan:   number of threads to start per channel
 149  * @max_channels:       maximum number of channels to use
 150  * @iterations:         iterations before stopping test
 151  * @xor_sources:        number of xor source buffers
 152  * @pq_sources:         number of p+q source buffers
 153  * @timeout:            transfer timeout in msec, -1 for infinite timeout
 154  */
 155 struct dmatest_params {
 156         unsigned int    buf_size;
 157         char            channel[20];
 158         char            device[20];
 159         unsigned int    threads_per_chan;
 160         unsigned int    max_channels;
 161         unsigned int    iterations;
 162         unsigned int    xor_sources;
 163         unsigned int    pq_sources;
 164         int             timeout;
 165 };
 166
 167 /**
 168  * struct dmatest_info - test information.
 169  * @params:             test parameters
 170  * @lock:               access protection to the fields of this structure
 171  */
 172 struct dmatest_info {
 173         /* Test parameters */
 174         struct dmatest_params   params;
 175
 176         /* Internal state */
 177         struct list_head        channels;
 178         unsigned int            nr_channels;
 179         struct mutex            lock;
 180
 181         /* debugfs related stuff */
 182         struct dentry           *root;
 183
 184         /* Test results */
 185         struct list_head        results;
 186         struct mutex            results_lock;
 187 };
 188
 189 static struct dmatest_info test_info;
 190
 191 static bool dmatest_match_channel(struct dmatest_params *params,
 192                 struct dma_chan *chan)
 193 {
 194         if (params->channel[0] == '\0')
 195                 return true;
 196         return strcmp(dma_chan_name(chan), params->channel) == 0;
 197 }
 198
 199 static bool dmatest_match_device(struct dmatest_params *params,
 200                 struct dma_device *device)
 201 {
 202         if (params->device[0] == '\0')
 203                 return true;
 204         return strcmp(dev_name(device->dev), params->device) == 0;
 205 }
 206
 207 static unsigned long dmatest_random(void)
 208 {
 209         unsigned long buf;
 210
 211         get_random_bytes(&buf, sizeof(buf));
 212         return buf;
 213 }
 214
 215 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
 216                 unsigned int buf_size)
 217 {
 218         unsigned int i;
 219         u8 *buf;
 220
 221         for (; (buf = *bufs); bufs++) {
 222                 for (i = 0; i < start; i++)
 223                         buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 224                 for ( ; i < start + len; i++)
 225                         buf[i] = PATTERN_SRC | PATTERN_COPY
 226                                 | (~i & PATTERN_COUNT_MASK);
 227                 for ( ; i < buf_size; i++)
 228                         buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 229                 buf++;
 230         }
 231 }
 232
 233 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
 234                 unsigned int buf_size)
 235 {
 236         unsigned int i;
 237         u8 *buf;
 238
 239         for (; (buf = *bufs); bufs++) {
 240                 for (i = 0; i < start; i++)
 241                         buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 242                 for ( ; i < start + len; i++)
 243                         buf[i] = PATTERN_DST | PATTERN_OVERWRITE
 244                                 | (~i & PATTERN_COUNT_MASK);
 245                 for ( ; i < buf_size; i++)
 246                         buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 247         }
 248 }
 249
 250 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
 251                 unsigned int counter, bool is_srcbuf)
 252 {
 253         u8              diff = actual ^ pattern;
 254         u8              expected = pattern | (~counter & PATTERN_COUNT_MASK);
 255         const char      *thread_name = current->comm;
 256
 257         if (is_srcbuf)
 258                 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
 259                         thread_name, index, expected, actual);
 260         else if ((pattern & PATTERN_COPY)
 261                         && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
 262                 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
 263                         thread_name, index, expected, actual);
 264         else if (diff & PATTERN_SRC)
 265                 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
 266                         thread_name, index, expected, actual);
 267         else
 268                 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
 269                         thread_name, index, expected, actual);
 270 }
 271
 272 static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
 273                 unsigned int end, unsigned int counter, u8 pattern,
 274                 bool is_srcbuf)
 275 {
 276         unsigned int i;
 277         unsigned int error_count = 0;
 278         u8 actual;
 279         u8 expected;
 280         u8 *buf;
 281         unsigned int counter_orig = counter;
 282
 283         for (; (buf = *bufs); bufs++) {
 284                 counter = counter_orig;
 285                 for (i = start; i < end; i++) {
 286                         actual = buf[i];
 287                         expected = pattern | (~counter & PATTERN_COUNT_MASK);
 288                         if (actual != expected) {
 289                                 if (error_count < MAX_ERROR_COUNT)
 290                                         dmatest_mismatch(actual, pattern, i,
 291                                                          counter, is_srcbuf);
 292                                 error_count++;
 293                         }
 294                         counter++;
 295                 }
 296         }
 297
 298         if (error_count > MAX_ERROR_COUNT)
 299                 pr_warn("%s: %u errors suppressed\n",
 300                         current->comm, error_count - MAX_ERROR_COUNT);
 301
 302         return error_count;
 303 }
 304
 305 /* poor man's completion - we want to use wait_event_freezable() on it */
 306 struct dmatest_done {
 307         bool                    done;
 308         wait_queue_head_t       *wait;
 309 };
 310
 311 static void dmatest_callback(void *arg)
 312 {
 313         struct dmatest_done *done = arg;
 314
 315         done->done = true;
 316         wake_up_all(done->wait);
 317 }
 318
 319 static inline void unmap_src(struct device *dev, dma_addr_t *addr, size_t len,
 320                              unsigned int count)
 321 {
 322         while (count--)
 323                 dma_unmap_single(dev, addr[count], len, DMA_TO_DEVICE);
 324 }
 325
 326 static inline void unmap_dst(struct device *dev, dma_addr_t *addr, size_t len,
 327                              unsigned int count)
 328 {
 329         while (count--)
 330                 dma_unmap_single(dev, addr[count], len, DMA_BIDIRECTIONAL);
 331 }
 332
 333 static unsigned int min_odd(unsigned int x, unsigned int y)
 334 {
 335         unsigned int val = min(x, y);
 336
 337         return val % 2 ? val : val - 1;
 338 }
 339
 340 static char *thread_result_get(const char *name,
 341                 struct dmatest_thread_result *tr)
 342 {
 343         static const char * const messages[] = {
 344                 [DMATEST_ET_OK]                 = "No errors",
 345                 [DMATEST_ET_MAP_SRC]            = "src mapping error",
 346                 [DMATEST_ET_MAP_DST]            = "dst mapping error",
 347                 [DMATEST_ET_PREP]               = "prep error",
 348                 [DMATEST_ET_SUBMIT]             = "submit error",
 349                 [DMATEST_ET_TIMEOUT]            = "test timed out",
 350                 [DMATEST_ET_DMA_ERROR]          =
 351                         "got completion callback (DMA_ERROR)",
 352                 [DMATEST_ET_DMA_IN_PROGRESS]    =
 353                         "got completion callback (DMA_IN_PROGRESS)",
 354                 [DMATEST_ET_VERIFY]             = "errors",
 355         };
 356         static char buf[512];
 357
 358         snprintf(buf, sizeof(buf) - 1,
 359                  "%s: #%u: %s with src_off=0x%x ""dst_off=0x%x len=0x%x (%lu)",
 360                  name, tr->n, messages[tr->type], tr->src_off, tr->dst_off,
 361                  tr->len, tr->data);
 362
 363         return buf;
 364 }
 365
 366 static int thread_result_add(struct dmatest_info *info,
 367                 struct dmatest_result *r, enum dmatest_error_type type,
 368                 unsigned int n, unsigned int src_off, unsigned int dst_off,
 369                 unsigned int len, unsigned long data)
 370 {
 371         struct dmatest_thread_result *tr;
 372
 373         tr = kzalloc(sizeof(*tr), GFP_KERNEL);
 374         if (!tr)
 375                 return -ENOMEM;
 376
 377         tr->type = type;
 378         tr->n = n;
 379         tr->src_off = src_off;
 380         tr->dst_off = dst_off;
 381         tr->len = len;
 382         tr->data = data;
 383
 384         mutex_lock(&info->results_lock);
 385         list_add_tail(&tr->node, &r->results);
 386         mutex_unlock(&info->results_lock);
 387
 388         if (tr->type == DMATEST_ET_OK)
 389                 pr_debug("%s\n", thread_result_get(r->name, tr));
 390         else
 391                 pr_warn("%s\n", thread_result_get(r->name, tr));
 392
 393         return 0;
 394 }
 395
 396 static void result_free(struct dmatest_info *info, const char *name)
 397 {
 398         struct dmatest_result *r, *_r;
 399
 400         mutex_lock(&info->results_lock);
 401         list_for_each_entry_safe(r, _r, &info->results, node) {
 402                 struct dmatest_thread_result *tr, *_tr;
 403
 404                 if (name && strcmp(r->name, name))
 405                         continue;
 406
 407                 list_for_each_entry_safe(tr, _tr, &r->results, node) {
 408                         list_del(&tr->node);
 409                         kfree(tr);
 410                 }
 411
 412                 kfree(r->name);
 413                 list_del(&r->node);
 414                 kfree(r);
 415         }
 416
 417         mutex_unlock(&info->results_lock);
 418 }
 419
 420 static struct dmatest_result *result_init(struct dmatest_info *info,
 421                 const char *name)
 422 {
 423         struct dmatest_result *r;
 424
 425         r = kzalloc(sizeof(*r), GFP_KERNEL);
 426         if (r) {
 427                 r->name = kstrdup(name, GFP_KERNEL);
 428                 INIT_LIST_HEAD(&r->results);
 429                 mutex_lock(&info->results_lock);
 430                 list_add_tail(&r->node, &info->results);
 431                 mutex_unlock(&info->results_lock);
 432         }
 433         return r;
 434 }
 435
 436 /*
 437  * This function repeatedly tests DMA transfers of various lengths and
 438  * offsets for a given operation type until it is told to exit by
 439  * kthread_stop(). There may be multiple threads running this function
 440  * in parallel for a single channel, and there may be multiple channels
 441  * being tested in parallel.
 442  *
 443  * Before each test, the source and destination buffer is initialized
 444  * with a known pattern. This pattern is different depending on
 445  * whether it's in an area which is supposed to be copied or
 446  * overwritten, and different in the source and destination buffers.
 447  * So if the DMA engine doesn't copy exactly what we tell it to copy,
 448  * we'll notice.
 449  */
 450 static int dmatest_func(void *data)
 451 {
 452         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait);
 453         struct dmatest_thread   *thread = data;
 454         struct dmatest_done     done = { .wait = &done_wait };
 455         struct dmatest_info     *info;
 456         struct dmatest_params   *params;
 457         struct dma_chan         *chan;
 458         struct dma_device       *dev;
 459         const char              *thread_name;
 460         unsigned int            src_off, dst_off, len;
 461         unsigned int            error_count;
 462         unsigned int            failed_tests = 0;
 463         unsigned int            total_tests = 0;
 464         dma_cookie_t            cookie;
 465         enum dma_status         status;
 466         enum dma_ctrl_flags     flags;
 467         u8                      *pq_coefs = NULL;
 468         int                     ret;
 469         int                     src_cnt;
 470         int                     dst_cnt;
 471         int                     i;
 472         struct dmatest_result   *result;
 473
 474         thread_name = current->comm;
 475         set_freezable();
 476
 477         ret = -ENOMEM;
 478
 479         smp_rmb();
 480         info = thread->info;
 481         params = &info->params;
 482         chan = thread->chan;
 483         dev = chan->device;
 484         if (thread->type == DMA_MEMCPY)
 485                 src_cnt = dst_cnt = 1;
 486         else if (thread->type == DMA_XOR) {
 487                 /* force odd to ensure dst = src */
 488                 src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
 489                 dst_cnt = 1;
 490         } else if (thread->type == DMA_PQ) {
 491                 /* force odd to ensure dst = src */
 492                 src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
 493                 dst_cnt = 2;
 494
 495                 pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL);
 496                 if (!pq_coefs)
 497                         goto err_thread_type;
 498
 499                 for (i = 0; i < src_cnt; i++)
 500                         pq_coefs[i] = 1;
 501         } else
 502                 goto err_thread_type;
 503
 504         result = result_init(info, thread_name);
 505         if (!result)
 506                 goto err_srcs;
 507
 508         thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
 509         if (!thread->srcs)
 510                 goto err_srcs;
 511         for (i = 0; i < src_cnt; i++) {
 512                 thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL);
 513                 if (!thread->srcs[i])
 514                         goto err_srcbuf;
 515         }
 516         thread->srcs[i] = NULL;
 517
 518         thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
 519         if (!thread->dsts)
 520                 goto err_dsts;
 521         for (i = 0; i < dst_cnt; i++) {
 522                 thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL);
 523                 if (!thread->dsts[i])
 524                         goto err_dstbuf;
 525         }
 526         thread->dsts[i] = NULL;
 527
 528         set_user_nice(current, 10);
 529
 530         /*
 531          * src and dst buffers are freed by ourselves below
 532          */
 533         flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
 534
 535         while (!kthread_should_stop()
 536                && !(params->iterations && total_tests >= params->iterations)) {
 537                 struct dma_async_tx_descriptor *tx = NULL;
 538                 dma_addr_t dma_srcs[src_cnt];
 539                 dma_addr_t dma_dsts[dst_cnt];
 540                 u8 align = 0;
 541
 542                 total_tests++;
 543
 544                 /* honor alignment restrictions */
 545                 if (thread->type == DMA_MEMCPY)
 546                         align = dev->copy_align;
 547                 else if (thread->type == DMA_XOR)
 548                         align = dev->xor_align;
 549                 else if (thread->type == DMA_PQ)
 550                         align = dev->pq_align;
 551
 552                 if (1 << align > params->buf_size) {
 553                         pr_err("%u-byte buffer too small for %d-byte alignment\n",
 554                                params->buf_size, 1 << align);
 555                         break;
 556                 }
 557
 558                 len = dmatest_random() % params->buf_size + 1;
 559                 len = (len >> align) << align;
 560                 if (!len)
 561                         len = 1 << align;
 562                 src_off = dmatest_random() % (params->buf_size - len + 1);
 563                 dst_off = dmatest_random() % (params->buf_size - len + 1);
 564
 565                 src_off = (src_off >> align) << align;
 566                 dst_off = (dst_off >> align) << align;
 567
 568                 dmatest_init_srcs(thread->srcs, src_off, len, params->buf_size);
 569                 dmatest_init_dsts(thread->dsts, dst_off, len, params->buf_size);
 570
 571                 for (i = 0; i < src_cnt; i++) {
 572                         u8 *buf = thread->srcs[i] + src_off;
 573
 574                         dma_srcs[i] = dma_map_single(dev->dev, buf, len,
 575                                                      DMA_TO_DEVICE);
 576                         ret = dma_mapping_error(dev->dev, dma_srcs[i]);
 577                         if (ret) {
 578                                 unmap_src(dev->dev, dma_srcs, len, i);
 579                                 thread_result_add(info, result,
 580                                                   DMATEST_ET_MAP_SRC,
 581                                                   total_tests, src_off, dst_off,
 582                                                   len, ret);
 583                                 failed_tests++;
 584                                 continue;
 585                         }
 586                 }
 587                 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
 588                 for (i = 0; i < dst_cnt; i++) {
 589                         dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i],
 590                                                      params->buf_size,
 591                                                      DMA_BIDIRECTIONAL);
 592                         ret = dma_mapping_error(dev->dev, dma_dsts[i]);
 593                         if (ret) {
 594                                 unmap_src(dev->dev, dma_srcs, len, src_cnt);
 595                                 unmap_dst(dev->dev, dma_dsts, params->buf_size,
 596                                           i);
 597                                 thread_result_add(info, result,
 598                                                   DMATEST_ET_MAP_DST,
 599                                                   total_tests, src_off, dst_off,
 600                                                   len, ret);
 601                                 failed_tests++;
 602                                 continue;
 603                         }
 604                 }
 605
 606                 if (thread->type == DMA_MEMCPY)
 607                         tx = dev->device_prep_dma_memcpy(chan,
 608                                                          dma_dsts[0] + dst_off,
 609                                                          dma_srcs[0], len,
 610                                                          flags);
 611                 else if (thread->type == DMA_XOR)
 612                         tx = dev->device_prep_dma_xor(chan,
 613                                                       dma_dsts[0] + dst_off,
 614                                                       dma_srcs, src_cnt,
 615                                                       len, flags);
 616                 else if (thread->type == DMA_PQ) {
 617                         dma_addr_t dma_pq[dst_cnt];
 618
 619                         for (i = 0; i < dst_cnt; i++)
 620                                 dma_pq[i] = dma_dsts[i] + dst_off;
 621                         tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs,
 622                                                      src_cnt, pq_coefs,
 623                                                      len, flags);
 624                 }
 625
 626                 if (!tx) {
 627                         unmap_src(dev->dev, dma_srcs, len, src_cnt);
 628                         unmap_dst(dev->dev, dma_dsts, params->buf_size,
 629                                   dst_cnt);
 630                         thread_result_add(info, result, DMATEST_ET_PREP,
 631                                           total_tests, src_off, dst_off,
 632                                           len, 0);
 633                         msleep(100);
 634                         failed_tests++;
 635                         continue;
 636                 }
 637
 638                 done.done = false;
 639                 tx->callback = dmatest_callback;
 640                 tx->callback_param = &done;
 641                 cookie = tx->tx_submit(tx);
 642
 643                 if (dma_submit_error(cookie)) {
 644                         thread_result_add(info, result, DMATEST_ET_SUBMIT,
 645                                           total_tests, src_off, dst_off,
 646                                           len, cookie);
 647                         msleep(100);
 648                         failed_tests++;
 649                         continue;
 650                 }
 651                 dma_async_issue_pending(chan);
 652
 653                 wait_event_freezable_timeout(done_wait, done.done,
 654                                              msecs_to_jiffies(params->timeout));
 655
 656                 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
 657
 658                 if (!done.done) {
 659                         /*
 660                          * We're leaving the timed out dma operation with
 661                          * dangling pointer to done_wait.  To make this
 662                          * correct, we'll need to allocate wait_done for
 663                          * each test iteration and perform "who's gonna
 664                          * free it this time?" dancing.  For now, just
 665                          * leave it dangling.
 666                          */
 667                         thread_result_add(info, result, DMATEST_ET_TIMEOUT,
 668                                           total_tests, src_off, dst_off,
 669                                           len, 0);
 670                         failed_tests++;
 671                         continue;
 672                 } else if (status != DMA_SUCCESS) {
 673                         enum dmatest_error_type type = (status == DMA_ERROR) ?
 674                                 DMATEST_ET_DMA_ERROR : DMATEST_ET_DMA_IN_PROGRESS;
 675                         thread_result_add(info, result, type,
 676                                           total_tests, src_off, dst_off,
 677                                           len, status);
 678                         failed_tests++;
 679                         continue;
 680                 }
 681
 682                 /* Unmap by myself */
 683                 unmap_src(dev->dev, dma_srcs, len, src_cnt);
 684                 unmap_dst(dev->dev, dma_dsts, params->buf_size, dst_cnt);
 685
 686                 error_count = 0;
 687
 688                 pr_debug("%s: verifying source buffer...\n", thread_name);
 689                 error_count += dmatest_verify(thread->srcs, 0, src_off,
 690                                 0, PATTERN_SRC, true);
 691                 error_count += dmatest_verify(thread->srcs, src_off,
 692                                 src_off + len, src_off,
 693                                 PATTERN_SRC | PATTERN_COPY, true);
 694                 error_count += dmatest_verify(thread->srcs, src_off + len,
 695                                 params->buf_size, src_off + len,
 696                                 PATTERN_SRC, true);
 697
 698                 pr_debug("%s: verifying dest buffer...\n",
 699                                 thread->task->comm);
 700                 error_count += dmatest_verify(thread->dsts, 0, dst_off,
 701                                 0, PATTERN_DST, false);
 702                 error_count += dmatest_verify(thread->dsts, dst_off,
 703                                 dst_off + len, src_off,
 704                                 PATTERN_SRC | PATTERN_COPY, false);
 705                 error_count += dmatest_verify(thread->dsts, dst_off + len,
 706                                 params->buf_size, dst_off + len,
 707                                 PATTERN_DST, false);
 708
 709                 if (error_count) {
 710                         thread_result_add(info, result, DMATEST_ET_VERIFY,
 711                                           total_tests, src_off, dst_off,
 712                                           len, error_count);
 713                         failed_tests++;
 714                 } else {
 715                         thread_result_add(info, result, DMATEST_ET_OK,
 716                                           total_tests, src_off, dst_off,
 717                                           len, 0);
 718                 }
 719         }
 720
 721         ret = 0;
 722         for (i = 0; thread->dsts[i]; i++)
 723                 kfree(thread->dsts[i]);
 724 err_dstbuf:
 725         kfree(thread->dsts);
 726 err_dsts:
 727         for (i = 0; thread->srcs[i]; i++)
 728                 kfree(thread->srcs[i]);
 729 err_srcbuf:
 730         kfree(thread->srcs);
 731 err_srcs:
 732         kfree(pq_coefs);
 733 err_thread_type:
 734         pr_notice("%s: terminating after %u tests, %u failures (status %d)\n",
 735                         thread_name, total_tests, failed_tests, ret);
 736
 737         /* terminate all transfers on specified channels */
 738         if (ret)
 739                 dmaengine_terminate_all(chan);
 740
 741         thread->done = true;
 742
 743         if (params->iterations > 0)
 744                 while (!kthread_should_stop()) {
 745                         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit);
 746                         interruptible_sleep_on(&wait_dmatest_exit);
 747                 }
 748
 749         return ret;
 750 }
 751
 752 static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
 753 {
 754         struct dmatest_thread   *thread;
 755         struct dmatest_thread   *_thread;
 756         int                     ret;
 757
 758         list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
 759                 ret = kthread_stop(thread->task);
 760                 pr_debug("dmatest: thread %s exited with status %d\n",
 761                                 thread->task->comm, ret);
 762                 list_del(&thread->node);
 763                 kfree(thread);
 764         }
 765
 766         /* terminate all transfers on specified channels */
 767         dmaengine_terminate_all(dtc->chan);
 768
 769         kfree(dtc);
 770 }
 771
 772 static int dmatest_add_threads(struct dmatest_info *info,
 773                 struct dmatest_chan *dtc, enum dma_transaction_type type)
 774 {
 775         struct dmatest_params *params = &info->params;
 776         struct dmatest_thread *thread;
 777         struct dma_chan *chan = dtc->chan;
 778         char *op;
 779         unsigned int i;
 780
 781         if (type == DMA_MEMCPY)
 782                 op = "copy";
 783         else if (type == DMA_XOR)
 784                 op = "xor";
 785         else if (type == DMA_PQ)
 786                 op = "pq";
 787         else
 788                 return -EINVAL;
 789
 790         for (i = 0; i < params->threads_per_chan; i++) {
 791                 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
 792                 if (!thread) {
 793                         pr_warning("dmatest: No memory for %s-%s%u\n",
 794                                    dma_chan_name(chan), op, i);
 795
 796                         break;
 797                 }
 798                 thread->info = info;
 799                 thread->chan = dtc->chan;
 800                 thread->type = type;
 801                 smp_wmb();
 802                 thread->task = kthread_run(dmatest_func, thread, "%s-%s%u",
 803                                 dma_chan_name(chan), op, i);
 804                 if (IS_ERR(thread->task)) {
 805                         pr_warning("dmatest: Failed to run thread %s-%s%u\n",
 806                                         dma_chan_name(chan), op, i);
 807                         kfree(thread);
 808                         break;
 809                 }
 810
 811                 /* srcbuf and dstbuf are allocated by the thread itself */
 812
 813                 list_add_tail(&thread->node, &dtc->threads);
 814         }
 815
 816         return i;
 817 }
 818
 819 static int dmatest_add_channel(struct dmatest_info *info,
 820                 struct dma_chan *chan)
 821 {
 822         struct dmatest_chan     *dtc;
 823         struct dma_device       *dma_dev = chan->device;
 824         unsigned int            thread_count = 0;
 825         int cnt;
 826
 827         dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
 828         if (!dtc) {
 829                 pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan));
 830                 return -ENOMEM;
 831         }
 832
 833         dtc->chan = chan;
 834         INIT_LIST_HEAD(&dtc->threads);
 835
 836         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
 837                 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
 838                 thread_count += cnt > 0 ? cnt : 0;
 839         }
 840         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
 841                 cnt = dmatest_add_threads(info, dtc, DMA_XOR);
 842                 thread_count += cnt > 0 ? cnt : 0;
 843         }
 844         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
 845                 cnt = dmatest_add_threads(info, dtc, DMA_PQ);
 846                 thread_count += cnt > 0 ? cnt : 0;
 847         }
 848
 849         pr_info("dmatest: Started %u threads using %s\n",
 850                 thread_count, dma_chan_name(chan));
 851
 852         list_add_tail(&dtc->node, &info->channels);
 853         info->nr_channels++;
 854
 855         return 0;
 856 }
 857
 858 static bool filter(struct dma_chan *chan, void *param)
 859 {
 860         struct dmatest_params *params = param;
 861
 862         if (!dmatest_match_channel(params, chan) ||
 863             !dmatest_match_device(params, chan->device))
 864                 return false;
 865         else
 866                 return true;
 867 }
 868
 869 static int __run_threaded_test(struct dmatest_info *info)
 870 {
 871         dma_cap_mask_t mask;
 872         struct dma_chan *chan;
 873         struct dmatest_params *params = &info->params;
 874         int err = 0;
 875
 876         dma_cap_zero(mask);
 877         dma_cap_set(DMA_MEMCPY, mask);
 878         for (;;) {
 879                 chan = dma_request_channel(mask, filter, params);
 880                 if (chan) {
 881                         err = dmatest_add_channel(info, chan);
 882                         if (err) {
 883                                 dma_release_channel(chan);
 884                                 break; /* add_channel failed, punt */
 885                         }
 886                 } else
 887                         break; /* no more channels available */
 888                 if (params->max_channels &&
 889                     info->nr_channels >= params->max_channels)
 890                         break; /* we have all we need */
 891         }
 892         return err;
 893 }
 894
 895 #ifndef MODULE
 896 static int run_threaded_test(struct dmatest_info *info)
 897 {
 898         int ret;
 899
 900         mutex_lock(&info->lock);
 901         ret = __run_threaded_test(info);
 902         mutex_unlock(&info->lock);
 903         return ret;
 904 }
 905 #endif
 906
 907 static void __stop_threaded_test(struct dmatest_info *info)
 908 {
 909         struct dmatest_chan *dtc, *_dtc;
 910         struct dma_chan *chan;
 911
 912         list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
 913                 list_del(&dtc->node);
 914                 chan = dtc->chan;
 915                 dmatest_cleanup_channel(dtc);
 916                 pr_debug("dmatest: dropped channel %s\n", dma_chan_name(chan));
 917                 dma_release_channel(chan);
 918         }
 919
 920         info->nr_channels = 0;
 921 }
 922
 923 static void stop_threaded_test(struct dmatest_info *info)
 924 {
 925         mutex_lock(&info->lock);
 926         __stop_threaded_test(info);
 927         mutex_unlock(&info->lock);
 928 }
 929
 930 static int __restart_threaded_test(struct dmatest_info *info, bool run)
 931 {
 932         struct dmatest_params *params = &info->params;
 933
 934         /* Stop any running test first */
 935         __stop_threaded_test(info);
 936
 937         if (run == false)
 938                 return 0;
 939
 940         /* Clear results from previous run */
 941         result_free(info, NULL);
 942
 943         /* Copy test parameters */
 944         params->buf_size = test_buf_size;
 945         strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
 946         strlcpy(params->device, strim(test_device), sizeof(params->device));
 947         params->threads_per_chan = threads_per_chan;
 948         params->max_channels = max_channels;
 949         params->iterations = iterations;
 950         params->xor_sources = xor_sources;
 951         params->pq_sources = pq_sources;
 952         params->timeout = timeout;
 953
 954         /* Run test with new parameters */
 955         return __run_threaded_test(info);
 956 }
 957
 958 static bool __is_threaded_test_run(struct dmatest_info *info)
 959 {
 960         struct dmatest_chan *dtc;
 961
 962         list_for_each_entry(dtc, &info->channels, node) {
 963                 struct dmatest_thread *thread;
 964
 965                 list_for_each_entry(thread, &dtc->threads, node) {
 966                         if (!thread->done)
 967                                 return true;
 968                 }
 969         }
 970
 971         return false;
 972 }
 973
 974 static ssize_t dtf_read_run(struct file *file, char __user *user_buf,
 975                 size_t count, loff_t *ppos)
 976 {
 977         struct dmatest_info *info = file->private_data;
 978         char buf[3];
 979
 980         mutex_lock(&info->lock);
 981
 982         if (__is_threaded_test_run(info)) {
 983                 buf[0] = 'Y';
 984         } else {
 985                 __stop_threaded_test(info);
 986                 buf[0] = 'N';
 987         }
 988
 989         mutex_unlock(&info->lock);
 990         buf[1] = '\n';
 991         buf[2] = 0x00;
 992         return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
 993 }
 994
 995 static ssize_t dtf_write_run(struct file *file, const char __user *user_buf,
 996                 size_t count, loff_t *ppos)
 997 {
 998         struct dmatest_info *info = file->private_data;
 999         char buf[16];
1000         bool bv;
1001         int ret = 0;
1002
1003         if (copy_from_user(buf, user_buf, min(count, (sizeof(buf) - 1))))
1004                 return -EFAULT;
1005
1006         if (strtobool(buf, &bv) == 0) {
1007                 mutex_lock(&info->lock);
1008
1009                 if (__is_threaded_test_run(info))
1010                         ret = -EBUSY;
1011                 else
1012                         ret = __restart_threaded_test(info, bv);
1013
1014                 mutex_unlock(&info->lock);
1015         }
1016
1017         return ret ? ret : count;
1018 }
1019
1020 static const struct file_operations dtf_run_fops = {
1021         .read   = dtf_read_run,
1022         .write  = dtf_write_run,
1023         .open   = simple_open,
1024         .llseek = default_llseek,
1025 };
1026
1027 static int dtf_results_show(struct seq_file *sf, void *data)
1028 {
1029         struct dmatest_info *info = sf->private;
1030         struct dmatest_result *result;
1031         struct dmatest_thread_result *tr;
1032
1033         mutex_lock(&info->results_lock);
1034         list_for_each_entry(result, &info->results, node) {
1035                 list_for_each_entry(tr, &result->results, node)
1036                         seq_printf(sf, "%s\n",
1037                                 thread_result_get(result->name, tr));
1038         }
1039
1040         mutex_unlock(&info->results_lock);
1041         return 0;
1042 }
1043
1044 static int dtf_results_open(struct inode *inode, struct file *file)
1045 {
1046         return single_open(file, dtf_results_show, inode->i_private);
1047 }
1048
1049 static const struct file_operations dtf_results_fops = {
1050         .open           = dtf_results_open,
1051         .read           = seq_read,
1052         .llseek         = seq_lseek,
1053         .release        = single_release,
1054 };
1055
1056 static int dmatest_register_dbgfs(struct dmatest_info *info)
1057 {
1058         struct dentry *d;
1059
1060         d = debugfs_create_dir("dmatest", NULL);
1061         if (IS_ERR(d))
1062                 return PTR_ERR(d);
1063         if (!d)
1064                 goto err_root;
1065
1066         info->root = d;
1067
1068         /* Run or stop threaded test */
1069         debugfs_create_file("run", S_IWUSR | S_IRUGO, info->root, info,
1070                             &dtf_run_fops);
1071
1072         /* Results of test in progress */
1073         debugfs_create_file("results", S_IRUGO, info->root, info,
1074                             &dtf_results_fops);
1075
1076         return 0;
1077
1078 err_root:
1079         pr_err("dmatest: Failed to initialize debugfs\n");
1080         return -ENOMEM;
1081 }
1082
1083 static int __init dmatest_init(void)
1084 {
1085         struct dmatest_info *info = &test_info;
1086         int ret;
1087
1088         memset(info, 0, sizeof(*info));
1089
1090         mutex_init(&info->lock);
1091         INIT_LIST_HEAD(&info->channels);
1092
1093         mutex_init(&info->results_lock);
1094         INIT_LIST_HEAD(&info->results);
1095
1096         ret = dmatest_register_dbgfs(info);
1097         if (ret)
1098                 return ret;
1099
1100 #ifdef MODULE
1101         return 0;
1102 #else
1103         return run_threaded_test(info);
1104 #endif
1105 }
1106 /* when compiled-in wait for drivers to load first */
1107 late_initcall(dmatest_init);
1108
1109 static void __exit dmatest_exit(void)
1110 {
1111         struct dmatest_info *info = &test_info;
1112
1113         debugfs_remove_recursive(info->root);
1114         stop_threaded_test(info);
1115         result_free(info, NULL);
1116 }
1117 module_exit(dmatest_exit);
1118
1119 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1120 MODULE_LICENSE("GPL v2");