drivers/dma/dmatest.c

   1 /*
   2  * DMA Engine test module
   3  *
   4  * Copyright (C) 2007 Atmel Corporation
   5  * Copyright (C) 2013 Intel Corporation
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  12
  13 #include <linux/delay.h>
  14 #include <linux/dma-mapping.h>
  15 #include <linux/dmaengine.h>
  16 #include <linux/freezer.h>
  17 #include <linux/init.h>
  18 #include <linux/kthread.h>
  19 #include <linux/module.h>
  20 #include <linux/moduleparam.h>
  21 #include <linux/random.h>
  22 #include <linux/slab.h>
  23 #include <linux/wait.h>
  24
  25 static unsigned int test_buf_size = 16384;
  26 module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
  27 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
  28
  29 static char test_channel[20];
  30 module_param_string(channel, test_channel, sizeof(test_channel),
  31                 S_IRUGO | S_IWUSR);
  32 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
  33
  34 static char test_device[20];
  35 module_param_string(device, test_device, sizeof(test_device),
  36                 S_IRUGO | S_IWUSR);
  37 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
  38
  39 static unsigned int threads_per_chan = 1;
  40 module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
  41 MODULE_PARM_DESC(threads_per_chan,
  42                 "Number of threads to start per channel (default: 1)");
  43
  44 static unsigned int max_channels;
  45 module_param(max_channels, uint, S_IRUGO | S_IWUSR);
  46 MODULE_PARM_DESC(max_channels,
  47                 "Maximum number of channels to use (default: all)");
  48
  49 static unsigned int iterations;
  50 module_param(iterations, uint, S_IRUGO | S_IWUSR);
  51 MODULE_PARM_DESC(iterations,
  52                 "Iterations before stopping test (default: infinite)");
  53
  54 static unsigned int xor_sources = 3;
  55 module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
  56 MODULE_PARM_DESC(xor_sources,
  57                 "Number of xor source buffers (default: 3)");
  58
  59 static unsigned int pq_sources = 3;
  60 module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
  61 MODULE_PARM_DESC(pq_sources,
  62                 "Number of p+q source buffers (default: 3)");
  63
  64 static int timeout = 3000;
  65 module_param(timeout, uint, S_IRUGO | S_IWUSR);
  66 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
  67                  "Pass -1 for infinite timeout");
  68
  69 static bool noverify;
  70 module_param(noverify, bool, S_IRUGO | S_IWUSR);
  71 MODULE_PARM_DESC(noverify, "Disable random data setup and verification");
  72
  73 /**
  74  * struct dmatest_params - test parameters.
  75  * @buf_size:           size of the memcpy test buffer
  76  * @channel:            bus ID of the channel to test
  77  * @device:             bus ID of the DMA Engine to test
  78  * @threads_per_chan:   number of threads to start per channel
  79  * @max_channels:       maximum number of channels to use
  80  * @iterations:         iterations before stopping test
  81  * @xor_sources:        number of xor source buffers
  82  * @pq_sources:         number of p+q source buffers
  83  * @timeout:            transfer timeout in msec, -1 for infinite timeout
  84  */
  85 struct dmatest_params {
  86         unsigned int    buf_size;
  87         char            channel[20];
  88         char            device[20];
  89         unsigned int    threads_per_chan;
  90         unsigned int    max_channels;
  91         unsigned int    iterations;
  92         unsigned int    xor_sources;
  93         unsigned int    pq_sources;
  94         int             timeout;
  95         bool            noverify;
  96 };
  97
  98 /**
  99  * struct dmatest_info - test information.
 100  * @params:             test parameters
 101  * @lock:               access protection to the fields of this structure
 102  */
 103 static struct dmatest_info {
 104         /* Test parameters */
 105         struct dmatest_params   params;
 106
 107         /* Internal state */
 108         struct list_head        channels;
 109         unsigned int            nr_channels;
 110         struct mutex            lock;
 111         bool                    did_init;
 112 } test_info = {
 113         .channels = LIST_HEAD_INIT(test_info.channels),
 114         .lock = __MUTEX_INITIALIZER(test_info.lock),
 115 };
 116
 117 static int dmatest_run_set(const char *val, const struct kernel_param *kp);
 118 static int dmatest_run_get(char *val, const struct kernel_param *kp);
 119 static struct kernel_param_ops run_ops = {
 120         .set = dmatest_run_set,
 121         .get = dmatest_run_get,
 122 };
 123 static bool dmatest_run;
 124 module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
 125 MODULE_PARM_DESC(run, "Run the test (default: false)");
 126
 127 /* Maximum amount of mismatched bytes in buffer to print */
 128 #define MAX_ERROR_COUNT         32
 129
 130 /*
 131  * Initialization patterns. All bytes in the source buffer has bit 7
 132  * set, all bytes in the destination buffer has bit 7 cleared.
 133  *
 134  * Bit 6 is set for all bytes which are to be copied by the DMA
 135  * engine. Bit 5 is set for all bytes which are to be overwritten by
 136  * the DMA engine.
 137  *
 138  * The remaining bits are the inverse of a counter which increments by
 139  * one for each byte address.
 140  */
 141 #define PATTERN_SRC             0x80
 142 #define PATTERN_DST             0x00
 143 #define PATTERN_COPY            0x40
 144 #define PATTERN_OVERWRITE       0x20
 145 #define PATTERN_COUNT_MASK      0x1f
 146
 147 struct dmatest_thread {
 148         struct list_head        node;
 149         struct dmatest_info     *info;
 150         struct task_struct      *task;
 151         struct dma_chan         *chan;
 152         u8                      **srcs;
 153         u8                      **dsts;
 154         enum dma_transaction_type type;
 155         bool                    done;
 156 };
 157
 158 struct dmatest_chan {
 159         struct list_head        node;
 160         struct dma_chan         *chan;
 161         struct list_head        threads;
 162 };
 163
 164 static bool dmatest_match_channel(struct dmatest_params *params,
 165                 struct dma_chan *chan)
 166 {
 167         if (params->channel[0] == '\0')
 168                 return true;
 169         return strcmp(dma_chan_name(chan), params->channel) == 0;
 170 }
 171
 172 static bool dmatest_match_device(struct dmatest_params *params,
 173                 struct dma_device *device)
 174 {
 175         if (params->device[0] == '\0')
 176                 return true;
 177         return strcmp(dev_name(device->dev), params->device) == 0;
 178 }
 179
 180 static unsigned long dmatest_random(void)
 181 {
 182         unsigned long buf;
 183
 184         prandom_bytes(&buf, sizeof(buf));
 185         return buf;
 186 }
 187
 188 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
 189                 unsigned int buf_size)
 190 {
 191         unsigned int i;
 192         u8 *buf;
 193
 194         for (; (buf = *bufs); bufs++) {
 195                 for (i = 0; i < start; i++)
 196                         buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 197                 for ( ; i < start + len; i++)
 198                         buf[i] = PATTERN_SRC | PATTERN_COPY
 199                                 | (~i & PATTERN_COUNT_MASK);
 200                 for ( ; i < buf_size; i++)
 201                         buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 202                 buf++;
 203         }
 204 }
 205
 206 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
 207                 unsigned int buf_size)
 208 {
 209         unsigned int i;
 210         u8 *buf;
 211
 212         for (; (buf = *bufs); bufs++) {
 213                 for (i = 0; i < start; i++)
 214                         buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 215                 for ( ; i < start + len; i++)
 216                         buf[i] = PATTERN_DST | PATTERN_OVERWRITE
 217                                 | (~i & PATTERN_COUNT_MASK);
 218                 for ( ; i < buf_size; i++)
 219                         buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 220         }
 221 }
 222
 223 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
 224                 unsigned int counter, bool is_srcbuf)
 225 {
 226         u8              diff = actual ^ pattern;
 227         u8              expected = pattern | (~counter & PATTERN_COUNT_MASK);
 228         const char      *thread_name = current->comm;
 229
 230         if (is_srcbuf)
 231                 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
 232                         thread_name, index, expected, actual);
 233         else if ((pattern & PATTERN_COPY)
 234                         && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
 235                 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
 236                         thread_name, index, expected, actual);
 237         else if (diff & PATTERN_SRC)
 238                 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
 239                         thread_name, index, expected, actual);
 240         else
 241                 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
 242                         thread_name, index, expected, actual);
 243 }
 244
 245 static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
 246                 unsigned int end, unsigned int counter, u8 pattern,
 247                 bool is_srcbuf)
 248 {
 249         unsigned int i;
 250         unsigned int error_count = 0;
 251         u8 actual;
 252         u8 expected;
 253         u8 *buf;
 254         unsigned int counter_orig = counter;
 255
 256         for (; (buf = *bufs); bufs++) {
 257                 counter = counter_orig;
 258                 for (i = start; i < end; i++) {
 259                         actual = buf[i];
 260                         expected = pattern | (~counter & PATTERN_COUNT_MASK);
 261                         if (actual != expected) {
 262                                 if (error_count < MAX_ERROR_COUNT)
 263                                         dmatest_mismatch(actual, pattern, i,
 264                                                          counter, is_srcbuf);
 265                                 error_count++;
 266                         }
 267                         counter++;
 268                 }
 269         }
 270
 271         if (error_count > MAX_ERROR_COUNT)
 272                 pr_warn("%s: %u errors suppressed\n",
 273                         current->comm, error_count - MAX_ERROR_COUNT);
 274
 275         return error_count;
 276 }
 277
 278 /* poor man's completion - we want to use wait_event_freezable() on it */
 279 struct dmatest_done {
 280         bool                    done;
 281         wait_queue_head_t       *wait;
 282 };
 283
 284 static void dmatest_callback(void *arg)
 285 {
 286         struct dmatest_done *done = arg;
 287
 288         done->done = true;
 289         wake_up_all(done->wait);
 290 }
 291
 292 static inline void unmap_src(struct device *dev, dma_addr_t *addr, size_t len,
 293                              unsigned int count)
 294 {
 295         while (count--)
 296                 dma_unmap_single(dev, addr[count], len, DMA_TO_DEVICE);
 297 }
 298
 299 static inline void unmap_dst(struct device *dev, dma_addr_t *addr, size_t len,
 300                              unsigned int count)
 301 {
 302         while (count--)
 303                 dma_unmap_single(dev, addr[count], len, DMA_BIDIRECTIONAL);
 304 }
 305
 306 static unsigned int min_odd(unsigned int x, unsigned int y)
 307 {
 308         unsigned int val = min(x, y);
 309
 310         return val % 2 ? val : val - 1;
 311 }
 312
 313 static void result(const char *err, unsigned int n, unsigned int src_off,
 314                    unsigned int dst_off, unsigned int len, unsigned long data)
 315 {
 316         pr_info("%s: result #%u: '%s' with src_off=0x%x ""dst_off=0x%x len=0x%x (%lu)",
 317                 current->comm, n, err, src_off, dst_off, len, data);
 318 }
 319
 320 static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
 321                        unsigned int dst_off, unsigned int len,
 322                        unsigned long data)
 323 {
 324         pr_debug("%s: result #%u: '%s' with src_off=0x%x ""dst_off=0x%x len=0x%x (%lu)",
 325                  current->comm, n, err, src_off, dst_off, len, data);
 326 }
 327
 328 static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
 329 {
 330         unsigned long long per_sec = 1000000;
 331
 332         if (runtime <= 0)
 333                 return 0;
 334
 335         /* drop precision until runtime is 32-bits */
 336         while (runtime > UINT_MAX) {
 337                 runtime >>= 1;
 338                 per_sec <<= 1;
 339         }
 340
 341         per_sec *= val;
 342         do_div(per_sec, runtime);
 343         return per_sec;
 344 }
 345
 346 static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
 347 {
 348         return dmatest_persec(runtime, len >> 10);
 349 }
 350
 351 /*
 352  * This function repeatedly tests DMA transfers of various lengths and
 353  * offsets for a given operation type until it is told to exit by
 354  * kthread_stop(). There may be multiple threads running this function
 355  * in parallel for a single channel, and there may be multiple channels
 356  * being tested in parallel.
 357  *
 358  * Before each test, the source and destination buffer is initialized
 359  * with a known pattern. This pattern is different depending on
 360  * whether it's in an area which is supposed to be copied or
 361  * overwritten, and different in the source and destination buffers.
 362  * So if the DMA engine doesn't copy exactly what we tell it to copy,
 363  * we'll notice.
 364  */
 365 static int dmatest_func(void *data)
 366 {
 367         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait);
 368         struct dmatest_thread   *thread = data;
 369         struct dmatest_done     done = { .wait = &done_wait };
 370         struct dmatest_info     *info;
 371         struct dmatest_params   *params;
 372         struct dma_chan         *chan;
 373         struct dma_device       *dev;
 374         unsigned int            src_off, dst_off, len;
 375         unsigned int            error_count;
 376         unsigned int            failed_tests = 0;
 377         unsigned int            total_tests = 0;
 378         dma_cookie_t            cookie;
 379         enum dma_status         status;
 380         enum dma_ctrl_flags     flags;
 381         u8                      *pq_coefs = NULL;
 382         int                     ret;
 383         int                     src_cnt;
 384         int                     dst_cnt;
 385         int                     i;
 386         ktime_t                 ktime;
 387         s64                     runtime = 0;
 388         unsigned long long      total_len = 0;
 389
 390         set_freezable();
 391
 392         ret = -ENOMEM;
 393
 394         smp_rmb();
 395         info = thread->info;
 396         params = &info->params;
 397         chan = thread->chan;
 398         dev = chan->device;
 399         if (thread->type == DMA_MEMCPY)
 400                 src_cnt = dst_cnt = 1;
 401         else if (thread->type == DMA_XOR) {
 402                 /* force odd to ensure dst = src */
 403                 src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
 404                 dst_cnt = 1;
 405         } else if (thread->type == DMA_PQ) {
 406                 /* force odd to ensure dst = src */
 407                 src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
 408                 dst_cnt = 2;
 409
 410                 pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL);
 411                 if (!pq_coefs)
 412                         goto err_thread_type;
 413
 414                 for (i = 0; i < src_cnt; i++)
 415                         pq_coefs[i] = 1;
 416         } else
 417                 goto err_thread_type;
 418
 419         thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
 420         if (!thread->srcs)
 421                 goto err_srcs;
 422         for (i = 0; i < src_cnt; i++) {
 423                 thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL);
 424                 if (!thread->srcs[i])
 425                         goto err_srcbuf;
 426         }
 427         thread->srcs[i] = NULL;
 428
 429         thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
 430         if (!thread->dsts)
 431                 goto err_dsts;
 432         for (i = 0; i < dst_cnt; i++) {
 433                 thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL);
 434                 if (!thread->dsts[i])
 435                         goto err_dstbuf;
 436         }
 437         thread->dsts[i] = NULL;
 438
 439         set_user_nice(current, 10);
 440
 441         /*
 442          * src and dst buffers are freed by ourselves below
 443          */
 444         flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
 445
 446         ktime = ktime_get();
 447         while (!kthread_should_stop()
 448                && !(params->iterations && total_tests >= params->iterations)) {
 449                 struct dma_async_tx_descriptor *tx = NULL;
 450                 dma_addr_t dma_srcs[src_cnt];
 451                 dma_addr_t dma_dsts[dst_cnt];
 452                 u8 align = 0;
 453
 454                 total_tests++;
 455
 456                 /* honor alignment restrictions */
 457                 if (thread->type == DMA_MEMCPY)
 458                         align = dev->copy_align;
 459                 else if (thread->type == DMA_XOR)
 460                         align = dev->xor_align;
 461                 else if (thread->type == DMA_PQ)
 462                         align = dev->pq_align;
 463
 464                 if (1 << align > params->buf_size) {
 465                         pr_err("%u-byte buffer too small for %d-byte alignment\n",
 466                                params->buf_size, 1 << align);
 467                         break;
 468                 }
 469
 470                 if (params->noverify) {
 471                         len = params->buf_size;
 472                         src_off = 0;
 473                         dst_off = 0;
 474                 } else {
 475                         len = dmatest_random() % params->buf_size + 1;
 476                         len = (len >> align) << align;
 477                         if (!len)
 478                                 len = 1 << align;
 479                         src_off = dmatest_random() % (params->buf_size - len + 1);
 480                         dst_off = dmatest_random() % (params->buf_size - len + 1);
 481
 482                         src_off = (src_off >> align) << align;
 483                         dst_off = (dst_off >> align) << align;
 484
 485                         dmatest_init_srcs(thread->srcs, src_off, len,
 486                                           params->buf_size);
 487                         dmatest_init_dsts(thread->dsts, dst_off, len,
 488                                           params->buf_size);
 489                 }
 490
 491                 len = (len >> align) << align;
 492                 if (!len)
 493                         len = 1 << align;
 494                 total_len += len;
 495
 496                 for (i = 0; i < src_cnt; i++) {
 497                         u8 *buf = thread->srcs[i] + src_off;
 498
 499                         dma_srcs[i] = dma_map_single(dev->dev, buf, len,
 500                                                      DMA_TO_DEVICE);
 501                         ret = dma_mapping_error(dev->dev, dma_srcs[i]);
 502                         if (ret) {
 503                                 unmap_src(dev->dev, dma_srcs, len, i);
 504                                 result("src mapping error", total_tests,
 505                                        src_off, dst_off, len, ret);
 506                                 failed_tests++;
 507                                 continue;
 508                         }
 509                 }
 510                 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
 511                 for (i = 0; i < dst_cnt; i++) {
 512                         dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i],
 513                                                      params->buf_size,
 514                                                      DMA_BIDIRECTIONAL);
 515                         ret = dma_mapping_error(dev->dev, dma_dsts[i]);
 516                         if (ret) {
 517                                 unmap_src(dev->dev, dma_srcs, len, src_cnt);
 518                                 unmap_dst(dev->dev, dma_dsts, params->buf_size,
 519                                           i);
 520                                 result("dst mapping error", total_tests,
 521                                        src_off, dst_off, len, ret);
 522                                 failed_tests++;
 523                                 continue;
 524                         }
 525                 }
 526
 527                 if (thread->type == DMA_MEMCPY)
 528                         tx = dev->device_prep_dma_memcpy(chan,
 529                                                          dma_dsts[0] + dst_off,
 530                                                          dma_srcs[0], len,
 531                                                          flags);
 532                 else if (thread->type == DMA_XOR)
 533                         tx = dev->device_prep_dma_xor(chan,
 534                                                       dma_dsts[0] + dst_off,
 535                                                       dma_srcs, src_cnt,
 536                                                       len, flags);
 537                 else if (thread->type == DMA_PQ) {
 538                         dma_addr_t dma_pq[dst_cnt];
 539
 540                         for (i = 0; i < dst_cnt; i++)
 541                                 dma_pq[i] = dma_dsts[i] + dst_off;
 542                         tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs,
 543                                                      src_cnt, pq_coefs,
 544                                                      len, flags);
 545                 }
 546
 547                 if (!tx) {
 548                         unmap_src(dev->dev, dma_srcs, len, src_cnt);
 549                         unmap_dst(dev->dev, dma_dsts, params->buf_size,
 550                                   dst_cnt);
 551                         result("prep error", total_tests, src_off,
 552                                dst_off, len, ret);
 553                         msleep(100);
 554                         failed_tests++;
 555                         continue;
 556                 }
 557
 558                 done.done = false;
 559                 tx->callback = dmatest_callback;
 560                 tx->callback_param = &done;
 561                 cookie = tx->tx_submit(tx);
 562
 563                 if (dma_submit_error(cookie)) {
 564                         result("submit error", total_tests, src_off,
 565                                dst_off, len, ret);
 566                         msleep(100);
 567                         failed_tests++;
 568                         continue;
 569                 }
 570                 dma_async_issue_pending(chan);
 571
 572                 wait_event_freezable_timeout(done_wait, done.done,
 573                                              msecs_to_jiffies(params->timeout));
 574
 575                 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
 576
 577                 if (!done.done) {
 578                         /*
 579                          * We're leaving the timed out dma operation with
 580                          * dangling pointer to done_wait.  To make this
 581                          * correct, we'll need to allocate wait_done for
 582                          * each test iteration and perform "who's gonna
 583                          * free it this time?" dancing.  For now, just
 584                          * leave it dangling.
 585                          */
 586                         result("test timed out", total_tests, src_off, dst_off,
 587                                len, 0);
 588                         failed_tests++;
 589                         continue;
 590                 } else if (status != DMA_SUCCESS) {
 591                         result(status == DMA_ERROR ?
 592                                "completion error status" :
 593                                "completion busy status", total_tests, src_off,
 594                                dst_off, len, ret);
 595                         failed_tests++;
 596                         continue;
 597                 }
 598
 599                 /* Unmap by myself */
 600                 unmap_src(dev->dev, dma_srcs, len, src_cnt);
 601                 unmap_dst(dev->dev, dma_dsts, params->buf_size, dst_cnt);
 602
 603                 if (params->noverify) {
 604                         dbg_result("test passed", total_tests, src_off, dst_off,
 605                                    len, 0);
 606                         continue;
 607                 }
 608
 609                 pr_debug("%s: verifying source buffer...\n", current->comm);
 610                 error_count = dmatest_verify(thread->srcs, 0, src_off,
 611                                 0, PATTERN_SRC, true);
 612                 error_count += dmatest_verify(thread->srcs, src_off,
 613                                 src_off + len, src_off,
 614                                 PATTERN_SRC | PATTERN_COPY, true);
 615                 error_count += dmatest_verify(thread->srcs, src_off + len,
 616                                 params->buf_size, src_off + len,
 617                                 PATTERN_SRC, true);
 618
 619                 pr_debug("%s: verifying dest buffer...\n", current->comm);
 620                 error_count += dmatest_verify(thread->dsts, 0, dst_off,
 621                                 0, PATTERN_DST, false);
 622                 error_count += dmatest_verify(thread->dsts, dst_off,
 623                                 dst_off + len, src_off,
 624                                 PATTERN_SRC | PATTERN_COPY, false);
 625                 error_count += dmatest_verify(thread->dsts, dst_off + len,
 626                                 params->buf_size, dst_off + len,
 627                                 PATTERN_DST, false);
 628
 629                 if (error_count) {
 630                         result("data error", total_tests, src_off, dst_off,
 631                                len, error_count);
 632                         failed_tests++;
 633                 } else {
 634                         dbg_result("test passed", total_tests, src_off, dst_off,
 635                                    len, 0);
 636                 }
 637         }
 638         runtime = ktime_us_delta(ktime_get(), ktime);
 639
 640         ret = 0;
 641         for (i = 0; thread->dsts[i]; i++)
 642                 kfree(thread->dsts[i]);
 643 err_dstbuf:
 644         kfree(thread->dsts);
 645 err_dsts:
 646         for (i = 0; thread->srcs[i]; i++)
 647                 kfree(thread->srcs[i]);
 648 err_srcbuf:
 649         kfree(thread->srcs);
 650 err_srcs:
 651         kfree(pq_coefs);
 652 err_thread_type:
 653         pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n",
 654                 current->comm, total_tests, failed_tests,
 655                 dmatest_persec(runtime, total_tests),
 656                 dmatest_KBs(runtime, total_len), ret);
 657
 658         /* terminate all transfers on specified channels */
 659         if (ret)
 660                 dmaengine_terminate_all(chan);
 661
 662         thread->done = true;
 663
 664         if (params->iterations > 0)
 665                 while (!kthread_should_stop()) {
 666                         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit);
 667                         interruptible_sleep_on(&wait_dmatest_exit);
 668                 }
 669
 670         return ret;
 671 }
 672
 673 static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
 674 {
 675         struct dmatest_thread   *thread;
 676         struct dmatest_thread   *_thread;
 677         int                     ret;
 678
 679         list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
 680                 ret = kthread_stop(thread->task);
 681                 pr_debug("thread %s exited with status %d\n",
 682                          thread->task->comm, ret);
 683                 list_del(&thread->node);
 684                 kfree(thread);
 685         }
 686
 687         /* terminate all transfers on specified channels */
 688         dmaengine_terminate_all(dtc->chan);
 689
 690         kfree(dtc);
 691 }
 692
 693 static int dmatest_add_threads(struct dmatest_info *info,
 694                 struct dmatest_chan *dtc, enum dma_transaction_type type)
 695 {
 696         struct dmatest_params *params = &info->params;
 697         struct dmatest_thread *thread;
 698         struct dma_chan *chan = dtc->chan;
 699         char *op;
 700         unsigned int i;
 701
 702         if (type == DMA_MEMCPY)
 703                 op = "copy";
 704         else if (type == DMA_XOR)
 705                 op = "xor";
 706         else if (type == DMA_PQ)
 707                 op = "pq";
 708         else
 709                 return -EINVAL;
 710
 711         for (i = 0; i < params->threads_per_chan; i++) {
 712                 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
 713                 if (!thread) {
 714                         pr_warn("No memory for %s-%s%u\n",
 715                                 dma_chan_name(chan), op, i);
 716                         break;
 717                 }
 718                 thread->info = info;
 719                 thread->chan = dtc->chan;
 720                 thread->type = type;
 721                 smp_wmb();
 722                 thread->task = kthread_run(dmatest_func, thread, "%s-%s%u",
 723                                 dma_chan_name(chan), op, i);
 724                 if (IS_ERR(thread->task)) {
 725                         pr_warn("Failed to run thread %s-%s%u\n",
 726                                 dma_chan_name(chan), op, i);
 727                         kfree(thread);
 728                         break;
 729                 }
 730
 731                 /* srcbuf and dstbuf are allocated by the thread itself */
 732
 733                 list_add_tail(&thread->node, &dtc->threads);
 734         }
 735
 736         return i;
 737 }
 738
 739 static int dmatest_add_channel(struct dmatest_info *info,
 740                 struct dma_chan *chan)
 741 {
 742         struct dmatest_chan     *dtc;
 743         struct dma_device       *dma_dev = chan->device;
 744         unsigned int            thread_count = 0;
 745         int cnt;
 746
 747         dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
 748         if (!dtc) {
 749                 pr_warn("No memory for %s\n", dma_chan_name(chan));
 750                 return -ENOMEM;
 751         }
 752
 753         dtc->chan = chan;
 754         INIT_LIST_HEAD(&dtc->threads);
 755
 756         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
 757                 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
 758                 thread_count += cnt > 0 ? cnt : 0;
 759         }
 760         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
 761                 cnt = dmatest_add_threads(info, dtc, DMA_XOR);
 762                 thread_count += cnt > 0 ? cnt : 0;
 763         }
 764         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
 765                 cnt = dmatest_add_threads(info, dtc, DMA_PQ);
 766                 thread_count += cnt > 0 ? cnt : 0;
 767         }
 768
 769         pr_info("Started %u threads using %s\n",
 770                 thread_count, dma_chan_name(chan));
 771
 772         list_add_tail(&dtc->node, &info->channels);
 773         info->nr_channels++;
 774
 775         return 0;
 776 }
 777
 778 static bool filter(struct dma_chan *chan, void *param)
 779 {
 780         struct dmatest_params *params = param;
 781
 782         if (!dmatest_match_channel(params, chan) ||
 783             !dmatest_match_device(params, chan->device))
 784                 return false;
 785         else
 786                 return true;
 787 }
 788
 789 static void request_channels(struct dmatest_info *info,
 790                              enum dma_transaction_type type)
 791 {
 792         dma_cap_mask_t mask;
 793
 794         dma_cap_zero(mask);
 795         dma_cap_set(type, mask);
 796         for (;;) {
 797                 struct dmatest_params *params = &info->params;
 798                 struct dma_chan *chan;
 799
 800                 chan = dma_request_channel(mask, filter, params);
 801                 if (chan) {
 802                         if (dmatest_add_channel(info, chan)) {
 803                                 dma_release_channel(chan);
 804                                 break; /* add_channel failed, punt */
 805                         }
 806                 } else
 807                         break; /* no more channels available */
 808                 if (params->max_channels &&
 809                     info->nr_channels >= params->max_channels)
 810                         break; /* we have all we need */
 811         }
 812 }
 813
 814 static void run_threaded_test(struct dmatest_info *info)
 815 {
 816         struct dmatest_params *params = &info->params;
 817
 818         /* Copy test parameters */
 819         params->buf_size = test_buf_size;
 820         strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
 821         strlcpy(params->device, strim(test_device), sizeof(params->device));
 822         params->threads_per_chan = threads_per_chan;
 823         params->max_channels = max_channels;
 824         params->iterations = iterations;
 825         params->xor_sources = xor_sources;
 826         params->pq_sources = pq_sources;
 827         params->timeout = timeout;
 828         params->noverify = noverify;
 829
 830         request_channels(info, DMA_MEMCPY);
 831         request_channels(info, DMA_XOR);
 832         request_channels(info, DMA_PQ);
 833 }
 834
 835 static void stop_threaded_test(struct dmatest_info *info)
 836 {
 837         struct dmatest_chan *dtc, *_dtc;
 838         struct dma_chan *chan;
 839
 840         list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
 841                 list_del(&dtc->node);
 842                 chan = dtc->chan;
 843                 dmatest_cleanup_channel(dtc);
 844                 pr_debug("dropped channel %s\n", dma_chan_name(chan));
 845                 dma_release_channel(chan);
 846         }
 847
 848         info->nr_channels = 0;
 849 }
 850
 851 static void restart_threaded_test(struct dmatest_info *info, bool run)
 852 {
 853         /* we might be called early to set run=, defer running until all
 854          * parameters have been evaluated
 855          */
 856         if (!info->did_init)
 857                 return;
 858
 859         /* Stop any running test first */
 860         stop_threaded_test(info);
 861
 862         /* Run test with new parameters */
 863         run_threaded_test(info);
 864 }
 865
 866 static bool is_threaded_test_run(struct dmatest_info *info)
 867 {
 868         struct dmatest_chan *dtc;
 869
 870         list_for_each_entry(dtc, &info->channels, node) {
 871                 struct dmatest_thread *thread;
 872
 873                 list_for_each_entry(thread, &dtc->threads, node) {
 874                         if (!thread->done)
 875                                 return true;
 876                 }
 877         }
 878
 879         return false;
 880 }
 881
 882 static int dmatest_run_get(char *val, const struct kernel_param *kp)
 883 {
 884         struct dmatest_info *info = &test_info;
 885
 886         mutex_lock(&info->lock);
 887         if (is_threaded_test_run(info)) {
 888                 dmatest_run = true;
 889         } else {
 890                 stop_threaded_test(info);
 891                 dmatest_run = false;
 892         }
 893         mutex_unlock(&info->lock);
 894
 895         return param_get_bool(val, kp);
 896 }
 897
 898 static int dmatest_run_set(const char *val, const struct kernel_param *kp)
 899 {
 900         struct dmatest_info *info = &test_info;
 901         int ret;
 902
 903         mutex_lock(&info->lock);
 904         ret = param_set_bool(val, kp);
 905         if (ret) {
 906                 mutex_unlock(&info->lock);
 907                 return ret;
 908         }
 909
 910         if (is_threaded_test_run(info))
 911                 ret = -EBUSY;
 912         else if (dmatest_run)
 913                 restart_threaded_test(info, dmatest_run);
 914
 915         mutex_unlock(&info->lock);
 916
 917         return ret;
 918 }
 919
 920 static int __init dmatest_init(void)
 921 {
 922         struct dmatest_info *info = &test_info;
 923
 924         if (dmatest_run) {
 925                 mutex_lock(&info->lock);
 926                 run_threaded_test(info);
 927                 mutex_unlock(&info->lock);
 928         }
 929
 930         /* module parameters are stable, inittime tests are started,
 931          * let userspace take over 'run' control
 932          */
 933         info->did_init = true;
 934
 935         return 0;
 936 }
 937 /* when compiled-in wait for drivers to load first */
 938 late_initcall(dmatest_init);
 939
 940 static void __exit dmatest_exit(void)
 941 {
 942         struct dmatest_info *info = &test_info;
 943
 944         mutex_lock(&info->lock);
 945         stop_threaded_test(info);
 946         mutex_unlock(&info->lock);
 947 }
 948 module_exit(dmatest_exit);
 949
 950 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
 951 MODULE_LICENSE("GPL v2");