drivers/dma/iop-adma.c

   1 /*
   2  * offload engine driver for the Intel Xscale series of i/o processors
   3  * Copyright © 2006, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  */
  19
  20 /*
  21  * This driver supports the asynchrounous DMA copy and RAID engines available
  22  * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
  23  */
  24
  25 #include <linux/init.h>
  26 #include <linux/module.h>
  27 #include <linux/delay.h>
  28 #include <linux/dma-mapping.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/interrupt.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/memory.h>
  33 #include <linux/ioport.h>
  34 #include <linux/raid/pq.h>
  35 #include <linux/slab.h>
  36
  37 #include <mach/adma.h>
  38
  39 #include "dmaengine.h"
  40
  41 #define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common)
  42 #define to_iop_adma_device(dev) \
  43         container_of(dev, struct iop_adma_device, common)
  44 #define tx_to_iop_adma_slot(tx) \
  45         container_of(tx, struct iop_adma_desc_slot, async_tx)
  46
  47 /**
  48  * iop_adma_free_slots - flags descriptor slots for reuse
  49  * @slot: Slot to free
  50  * Caller must hold &iop_chan->lock while calling this function
  51  */
  52 static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
  53 {
  54         int stride = slot->slots_per_op;
  55
  56         while (stride--) {
  57                 slot->slots_per_op = 0;
  58                 slot = list_entry(slot->slot_node.next,
  59                                 struct iop_adma_desc_slot,
  60                                 slot_node);
  61         }
  62 }
  63
  64 static dma_cookie_t
  65 iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
  66         struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
  67 {
  68         struct dma_async_tx_descriptor *tx = &desc->async_tx;
  69
  70         BUG_ON(tx->cookie < 0);
  71         if (tx->cookie > 0) {
  72                 cookie = tx->cookie;
  73                 tx->cookie = 0;
  74
  75                 /* call the callback (must not sleep or submit new
  76                  * operations to this channel)
  77                  */
  78                 if (tx->callback)
  79                         tx->callback(tx->callback_param);
  80
  81                 dma_descriptor_unmap(tx);
  82                 if (desc->group_head)
  83                         desc->group_head = NULL;
  84         }
  85
  86         /* run dependent operations */
  87         dma_run_dependencies(tx);
  88
  89         return cookie;
  90 }
  91
  92 static int
  93 iop_adma_clean_slot(struct iop_adma_desc_slot *desc,
  94         struct iop_adma_chan *iop_chan)
  95 {
  96         /* the client is allowed to attach dependent operations
  97          * until 'ack' is set
  98          */
  99         if (!async_tx_test_ack(&desc->async_tx))
 100                 return 0;
 101
 102         /* leave the last descriptor in the chain
 103          * so we can append to it
 104          */
 105         if (desc->chain_node.next == &iop_chan->chain)
 106                 return 1;
 107
 108         dev_dbg(iop_chan->device->common.dev,
 109                 "\tfree slot: %d slots_per_op: %d\n",
 110                 desc->idx, desc->slots_per_op);
 111
 112         list_del(&desc->chain_node);
 113         iop_adma_free_slots(desc);
 114
 115         return 0;
 116 }
 117
 118 static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
 119 {
 120         struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL;
 121         dma_cookie_t cookie = 0;
 122         u32 current_desc = iop_chan_get_current_descriptor(iop_chan);
 123         int busy = iop_chan_is_busy(iop_chan);
 124         int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
 125
 126         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
 127         /* free completed slots from the chain starting with
 128          * the oldest descriptor
 129          */
 130         list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
 131                                         chain_node) {
 132                 pr_debug("\tcookie: %d slot: %d busy: %d "
 133                         "this_desc: %#x next_desc: %#x ack: %d\n",
 134                         iter->async_tx.cookie, iter->idx, busy,
 135                         iter->async_tx.phys, iop_desc_get_next_desc(iter),
 136                         async_tx_test_ack(&iter->async_tx));
 137                 prefetch(_iter);
 138                 prefetch(&_iter->async_tx);
 139
 140                 /* do not advance past the current descriptor loaded into the
 141                  * hardware channel, subsequent descriptors are either in
 142                  * process or have not been submitted
 143                  */
 144                 if (seen_current)
 145                         break;
 146
 147                 /* stop the search if we reach the current descriptor and the
 148                  * channel is busy, or if it appears that the current descriptor
 149                  * needs to be re-read (i.e. has been appended to)
 150                  */
 151                 if (iter->async_tx.phys == current_desc) {
 152                         BUG_ON(seen_current++);
 153                         if (busy || iop_desc_get_next_desc(iter))
 154                                 break;
 155                 }
 156
 157                 /* detect the start of a group transaction */
 158                 if (!slot_cnt && !slots_per_op) {
 159                         slot_cnt = iter->slot_cnt;
 160                         slots_per_op = iter->slots_per_op;
 161                         if (slot_cnt <= slots_per_op) {
 162                                 slot_cnt = 0;
 163                                 slots_per_op = 0;
 164                         }
 165                 }
 166
 167                 if (slot_cnt) {
 168                         pr_debug("\tgroup++\n");
 169                         if (!grp_start)
 170                                 grp_start = iter;
 171                         slot_cnt -= slots_per_op;
 172                 }
 173
 174                 /* all the members of a group are complete */
 175                 if (slots_per_op != 0 && slot_cnt == 0) {
 176                         struct iop_adma_desc_slot *grp_iter, *_grp_iter;
 177                         int end_of_chain = 0;
 178                         pr_debug("\tgroup end\n");
 179
 180                         /* collect the total results */
 181                         if (grp_start->xor_check_result) {
 182                                 u32 zero_sum_result = 0;
 183                                 slot_cnt = grp_start->slot_cnt;
 184                                 grp_iter = grp_start;
 185
 186                                 list_for_each_entry_from(grp_iter,
 187                                         &iop_chan->chain, chain_node) {
 188                                         zero_sum_result |=
 189                                             iop_desc_get_zero_result(grp_iter);
 190                                             pr_debug("\titer%d result: %d\n",
 191                                             grp_iter->idx, zero_sum_result);
 192                                         slot_cnt -= slots_per_op;
 193                                         if (slot_cnt == 0)
 194                                                 break;
 195                                 }
 196                                 pr_debug("\tgrp_start->xor_check_result: %p\n",
 197                                         grp_start->xor_check_result);
 198                                 *grp_start->xor_check_result = zero_sum_result;
 199                         }
 200
 201                         /* clean up the group */
 202                         slot_cnt = grp_start->slot_cnt;
 203                         grp_iter = grp_start;
 204                         list_for_each_entry_safe_from(grp_iter, _grp_iter,
 205                                 &iop_chan->chain, chain_node) {
 206                                 cookie = iop_adma_run_tx_complete_actions(
 207                                         grp_iter, iop_chan, cookie);
 208
 209                                 slot_cnt -= slots_per_op;
 210                                 end_of_chain = iop_adma_clean_slot(grp_iter,
 211                                         iop_chan);
 212
 213                                 if (slot_cnt == 0 || end_of_chain)
 214                                         break;
 215                         }
 216
 217                         /* the group should be complete at this point */
 218                         BUG_ON(slot_cnt);
 219
 220                         slots_per_op = 0;
 221                         grp_start = NULL;
 222                         if (end_of_chain)
 223                                 break;
 224                         else
 225                                 continue;
 226                 } else if (slots_per_op) /* wait for group completion */
 227                         continue;
 228
 229                 /* write back zero sum results (single descriptor case) */
 230                 if (iter->xor_check_result && iter->async_tx.cookie)
 231                         *iter->xor_check_result =
 232                                 iop_desc_get_zero_result(iter);
 233
 234                 cookie = iop_adma_run_tx_complete_actions(
 235                                         iter, iop_chan, cookie);
 236
 237                 if (iop_adma_clean_slot(iter, iop_chan))
 238                         break;
 239         }
 240
 241         if (cookie > 0) {
 242                 iop_chan->common.completed_cookie = cookie;
 243                 pr_debug("\tcompleted cookie %d\n", cookie);
 244         }
 245 }
 246
 247 static void
 248 iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
 249 {
 250         spin_lock_bh(&iop_chan->lock);
 251         __iop_adma_slot_cleanup(iop_chan);
 252         spin_unlock_bh(&iop_chan->lock);
 253 }
 254
 255 static void iop_adma_tasklet(unsigned long data)
 256 {
 257         struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data;
 258
 259         /* lockdep will flag depedency submissions as potentially
 260          * recursive locking, this is not the case as a dependency
 261          * submission will never recurse a channels submit routine.
 262          * There are checks in async_tx.c to prevent this.
 263          */
 264         spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING);
 265         __iop_adma_slot_cleanup(iop_chan);
 266         spin_unlock(&iop_chan->lock);
 267 }
 268
 269 static struct iop_adma_desc_slot *
 270 iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots,
 271                         int slots_per_op)
 272 {
 273         struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL;
 274         LIST_HEAD(chain);
 275         int slots_found, retry = 0;
 276
 277         /* start search from the last allocated descrtiptor
 278          * if a contiguous allocation can not be found start searching
 279          * from the beginning of the list
 280          */
 281 retry:
 282         slots_found = 0;
 283         if (retry == 0)
 284                 iter = iop_chan->last_used;
 285         else
 286                 iter = list_entry(&iop_chan->all_slots,
 287                         struct iop_adma_desc_slot,
 288                         slot_node);
 289
 290         list_for_each_entry_safe_continue(
 291                 iter, _iter, &iop_chan->all_slots, slot_node) {
 292                 prefetch(_iter);
 293                 prefetch(&_iter->async_tx);
 294                 if (iter->slots_per_op) {
 295                         /* give up after finding the first busy slot
 296                          * on the second pass through the list
 297                          */
 298                         if (retry)
 299                                 break;
 300
 301                         slots_found = 0;
 302                         continue;
 303                 }
 304
 305                 /* start the allocation if the slot is correctly aligned */
 306                 if (!slots_found++) {
 307                         if (iop_desc_is_aligned(iter, slots_per_op))
 308                                 alloc_start = iter;
 309                         else {
 310                                 slots_found = 0;
 311                                 continue;
 312                         }
 313                 }
 314
 315                 if (slots_found == num_slots) {
 316                         struct iop_adma_desc_slot *alloc_tail = NULL;
 317                         struct iop_adma_desc_slot *last_used = NULL;
 318                         iter = alloc_start;
 319                         while (num_slots) {
 320                                 int i;
 321                                 dev_dbg(iop_chan->device->common.dev,
 322                                         "allocated slot: %d "
 323                                         "(desc %p phys: %#x) slots_per_op %d\n",
 324                                         iter->idx, iter->hw_desc,
 325                                         iter->async_tx.phys, slots_per_op);
 326
 327                                 /* pre-ack all but the last descriptor */
 328                                 if (num_slots != slots_per_op)
 329                                         async_tx_ack(&iter->async_tx);
 330
 331                                 list_add_tail(&iter->chain_node, &chain);
 332                                 alloc_tail = iter;
 333                                 iter->async_tx.cookie = 0;
 334                                 iter->slot_cnt = num_slots;
 335                                 iter->xor_check_result = NULL;
 336                                 for (i = 0; i < slots_per_op; i++) {
 337                                         iter->slots_per_op = slots_per_op - i;
 338                                         last_used = iter;
 339                                         iter = list_entry(iter->slot_node.next,
 340                                                 struct iop_adma_desc_slot,
 341                                                 slot_node);
 342                                 }
 343                                 num_slots -= slots_per_op;
 344                         }
 345                         alloc_tail->group_head = alloc_start;
 346                         alloc_tail->async_tx.cookie = -EBUSY;
 347                         list_splice(&chain, &alloc_tail->tx_list);
 348                         iop_chan->last_used = last_used;
 349                         iop_desc_clear_next_desc(alloc_start);
 350                         iop_desc_clear_next_desc(alloc_tail);
 351                         return alloc_tail;
 352                 }
 353         }
 354         if (!retry++)
 355                 goto retry;
 356
 357         /* perform direct reclaim if the allocation fails */
 358         __iop_adma_slot_cleanup(iop_chan);
 359
 360         return NULL;
 361 }
 362
 363 static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan)
 364 {
 365         dev_dbg(iop_chan->device->common.dev, "pending: %d\n",
 366                 iop_chan->pending);
 367
 368         if (iop_chan->pending >= IOP_ADMA_THRESHOLD) {
 369                 iop_chan->pending = 0;
 370                 iop_chan_append(iop_chan);
 371         }
 372 }
 373
 374 static dma_cookie_t
 375 iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
 376 {
 377         struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
 378         struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan);
 379         struct iop_adma_desc_slot *grp_start, *old_chain_tail;
 380         int slot_cnt;
 381         int slots_per_op;
 382         dma_cookie_t cookie;
 383         dma_addr_t next_dma;
 384
 385         grp_start = sw_desc->group_head;
 386         slot_cnt = grp_start->slot_cnt;
 387         slots_per_op = grp_start->slots_per_op;
 388
 389         spin_lock_bh(&iop_chan->lock);
 390         cookie = dma_cookie_assign(tx);
 391
 392         old_chain_tail = list_entry(iop_chan->chain.prev,
 393                 struct iop_adma_desc_slot, chain_node);
 394         list_splice_init(&sw_desc->tx_list,
 395                          &old_chain_tail->chain_node);
 396
 397         /* fix up the hardware chain */
 398         next_dma = grp_start->async_tx.phys;
 399         iop_desc_set_next_desc(old_chain_tail, next_dma);
 400         BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */
 401
 402         /* check for pre-chained descriptors */
 403         iop_paranoia(iop_desc_get_next_desc(sw_desc));
 404
 405         /* increment the pending count by the number of slots
 406          * memcpy operations have a 1:1 (slot:operation) relation
 407          * other operations are heavier and will pop the threshold
 408          * more often.
 409          */
 410         iop_chan->pending += slot_cnt;
 411         iop_adma_check_threshold(iop_chan);
 412         spin_unlock_bh(&iop_chan->lock);
 413
 414         dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n",
 415                 __func__, sw_desc->async_tx.cookie, sw_desc->idx);
 416
 417         return cookie;
 418 }
 419
 420 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan);
 421 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan);
 422
 423 /**
 424  * iop_adma_alloc_chan_resources -  returns the number of allocated descriptors
 425  * @chan - allocate descriptor resources for this channel
 426  * @client - current client requesting the channel be ready for requests
 427  *
 428  * Note: We keep the slots for 1 operation on iop_chan->chain at all times.  To
 429  * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be
 430  * greater than 2x the number slots needed to satisfy a device->max_xor
 431  * request.
 432  * */
 433 static int iop_adma_alloc_chan_resources(struct dma_chan *chan)
 434 {
 435         char *hw_desc;
 436         int idx;
 437         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 438         struct iop_adma_desc_slot *slot = NULL;
 439         int init = iop_chan->slots_allocated ? 0 : 1;
 440         struct iop_adma_platform_data *plat_data =
 441                 dev_get_platdata(&iop_chan->device->pdev->dev);
 442         int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE;
 443
 444         /* Allocate descriptor slots */
 445         do {
 446                 idx = iop_chan->slots_allocated;
 447                 if (idx == num_descs_in_pool)
 448                         break;
 449
 450                 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
 451                 if (!slot) {
 452                         printk(KERN_INFO "IOP ADMA Channel only initialized"
 453                                 " %d descriptor slots", idx);
 454                         break;
 455                 }
 456                 hw_desc = (char *) iop_chan->device->dma_desc_pool_virt;
 457                 slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
 458
 459                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
 460                 slot->async_tx.tx_submit = iop_adma_tx_submit;
 461                 INIT_LIST_HEAD(&slot->tx_list);
 462                 INIT_LIST_HEAD(&slot->chain_node);
 463                 INIT_LIST_HEAD(&slot->slot_node);
 464                 hw_desc = (char *) iop_chan->device->dma_desc_pool;
 465                 slot->async_tx.phys =
 466                         (dma_addr_t) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
 467                 slot->idx = idx;
 468
 469                 spin_lock_bh(&iop_chan->lock);
 470                 iop_chan->slots_allocated++;
 471                 list_add_tail(&slot->slot_node, &iop_chan->all_slots);
 472                 spin_unlock_bh(&iop_chan->lock);
 473         } while (iop_chan->slots_allocated < num_descs_in_pool);
 474
 475         if (idx && !iop_chan->last_used)
 476                 iop_chan->last_used = list_entry(iop_chan->all_slots.next,
 477                                         struct iop_adma_desc_slot,
 478                                         slot_node);
 479
 480         dev_dbg(iop_chan->device->common.dev,
 481                 "allocated %d descriptor slots last_used: %p\n",
 482                 iop_chan->slots_allocated, iop_chan->last_used);
 483
 484         /* initialize the channel and the chain with a null operation */
 485         if (init) {
 486                 if (dma_has_cap(DMA_MEMCPY,
 487                         iop_chan->device->common.cap_mask))
 488                         iop_chan_start_null_memcpy(iop_chan);
 489                 else if (dma_has_cap(DMA_XOR,
 490                         iop_chan->device->common.cap_mask))
 491                         iop_chan_start_null_xor(iop_chan);
 492                 else
 493                         BUG();
 494         }
 495
 496         return (idx > 0) ? idx : -ENOMEM;
 497 }
 498
 499 static struct dma_async_tx_descriptor *
 500 iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
 501 {
 502         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 503         struct iop_adma_desc_slot *sw_desc, *grp_start;
 504         int slot_cnt, slots_per_op;
 505
 506         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
 507
 508         spin_lock_bh(&iop_chan->lock);
 509         slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan);
 510         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 511         if (sw_desc) {
 512                 grp_start = sw_desc->group_head;
 513                 iop_desc_init_interrupt(grp_start, iop_chan);
 514                 sw_desc->async_tx.flags = flags;
 515         }
 516         spin_unlock_bh(&iop_chan->lock);
 517
 518         return sw_desc ? &sw_desc->async_tx : NULL;
 519 }
 520
 521 static struct dma_async_tx_descriptor *
 522 iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
 523                          dma_addr_t dma_src, size_t len, unsigned long flags)
 524 {
 525         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 526         struct iop_adma_desc_slot *sw_desc, *grp_start;
 527         int slot_cnt, slots_per_op;
 528
 529         if (unlikely(!len))
 530                 return NULL;
 531         BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT);
 532
 533         dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
 534                 __func__, len);
 535
 536         spin_lock_bh(&iop_chan->lock);
 537         slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op);
 538         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 539         if (sw_desc) {
 540                 grp_start = sw_desc->group_head;
 541                 iop_desc_init_memcpy(grp_start, flags);
 542                 iop_desc_set_byte_count(grp_start, iop_chan, len);
 543                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 544                 iop_desc_set_memcpy_src_addr(grp_start, dma_src);
 545                 sw_desc->async_tx.flags = flags;
 546         }
 547         spin_unlock_bh(&iop_chan->lock);
 548
 549         return sw_desc ? &sw_desc->async_tx : NULL;
 550 }
 551
 552 static struct dma_async_tx_descriptor *
 553 iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
 554                       dma_addr_t *dma_src, unsigned int src_cnt, size_t len,
 555                       unsigned long flags)
 556 {
 557         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 558         struct iop_adma_desc_slot *sw_desc, *grp_start;
 559         int slot_cnt, slots_per_op;
 560
 561         if (unlikely(!len))
 562                 return NULL;
 563         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 564
 565         dev_dbg(iop_chan->device->common.dev,
 566                 "%s src_cnt: %d len: %u flags: %lx\n",
 567                 __func__, src_cnt, len, flags);
 568
 569         spin_lock_bh(&iop_chan->lock);
 570         slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op);
 571         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 572         if (sw_desc) {
 573                 grp_start = sw_desc->group_head;
 574                 iop_desc_init_xor(grp_start, src_cnt, flags);
 575                 iop_desc_set_byte_count(grp_start, iop_chan, len);
 576                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 577                 sw_desc->async_tx.flags = flags;
 578                 while (src_cnt--)
 579                         iop_desc_set_xor_src_addr(grp_start, src_cnt,
 580                                                   dma_src[src_cnt]);
 581         }
 582         spin_unlock_bh(&iop_chan->lock);
 583
 584         return sw_desc ? &sw_desc->async_tx : NULL;
 585 }
 586
 587 static struct dma_async_tx_descriptor *
 588 iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
 589                           unsigned int src_cnt, size_t len, u32 *result,
 590                           unsigned long flags)
 591 {
 592         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 593         struct iop_adma_desc_slot *sw_desc, *grp_start;
 594         int slot_cnt, slots_per_op;
 595
 596         if (unlikely(!len))
 597                 return NULL;
 598
 599         dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
 600                 __func__, src_cnt, len);
 601
 602         spin_lock_bh(&iop_chan->lock);
 603         slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op);
 604         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 605         if (sw_desc) {
 606                 grp_start = sw_desc->group_head;
 607                 iop_desc_init_zero_sum(grp_start, src_cnt, flags);
 608                 iop_desc_set_zero_sum_byte_count(grp_start, len);
 609                 grp_start->xor_check_result = result;
 610                 pr_debug("\t%s: grp_start->xor_check_result: %p\n",
 611                         __func__, grp_start->xor_check_result);
 612                 sw_desc->async_tx.flags = flags;
 613                 while (src_cnt--)
 614                         iop_desc_set_zero_sum_src_addr(grp_start, src_cnt,
 615                                                        dma_src[src_cnt]);
 616         }
 617         spin_unlock_bh(&iop_chan->lock);
 618
 619         return sw_desc ? &sw_desc->async_tx : NULL;
 620 }
 621
 622 static struct dma_async_tx_descriptor *
 623 iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
 624                      unsigned int src_cnt, const unsigned char *scf, size_t len,
 625                      unsigned long flags)
 626 {
 627         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 628         struct iop_adma_desc_slot *sw_desc, *g;
 629         int slot_cnt, slots_per_op;
 630         int continue_srcs;
 631
 632         if (unlikely(!len))
 633                 return NULL;
 634         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 635
 636         dev_dbg(iop_chan->device->common.dev,
 637                 "%s src_cnt: %d len: %u flags: %lx\n",
 638                 __func__, src_cnt, len, flags);
 639
 640         if (dmaf_p_disabled_continue(flags))
 641                 continue_srcs = 1+src_cnt;
 642         else if (dmaf_continue(flags))
 643                 continue_srcs = 3+src_cnt;
 644         else
 645                 continue_srcs = 0+src_cnt;
 646
 647         spin_lock_bh(&iop_chan->lock);
 648         slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op);
 649         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 650         if (sw_desc) {
 651                 int i;
 652
 653                 g = sw_desc->group_head;
 654                 iop_desc_set_byte_count(g, iop_chan, len);
 655
 656                 /* even if P is disabled its destination address (bits
 657                  * [3:0]) must match Q.  It is ok if P points to an
 658                  * invalid address, it won't be written.
 659                  */
 660                 if (flags & DMA_PREP_PQ_DISABLE_P)
 661                         dst[0] = dst[1] & 0x7;
 662
 663                 iop_desc_set_pq_addr(g, dst);
 664                 sw_desc->async_tx.flags = flags;
 665                 for (i = 0; i < src_cnt; i++)
 666                         iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);
 667
 668                 /* if we are continuing a previous operation factor in
 669                  * the old p and q values, see the comment for dma_maxpq
 670                  * in include/linux/dmaengine.h
 671                  */
 672                 if (dmaf_p_disabled_continue(flags))
 673                         iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
 674                 else if (dmaf_continue(flags)) {
 675                         iop_desc_set_pq_src_addr(g, i++, dst[0], 0);
 676                         iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
 677                         iop_desc_set_pq_src_addr(g, i++, dst[1], 0);
 678                 }
 679                 iop_desc_init_pq(g, i, flags);
 680         }
 681         spin_unlock_bh(&iop_chan->lock);
 682
 683         return sw_desc ? &sw_desc->async_tx : NULL;
 684 }
 685
 686 static struct dma_async_tx_descriptor *
 687 iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
 688                          unsigned int src_cnt, const unsigned char *scf,
 689                          size_t len, enum sum_check_flags *pqres,
 690                          unsigned long flags)
 691 {
 692         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 693         struct iop_adma_desc_slot *sw_desc, *g;
 694         int slot_cnt, slots_per_op;
 695
 696         if (unlikely(!len))
 697                 return NULL;
 698         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 699
 700         dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
 701                 __func__, src_cnt, len);
 702
 703         spin_lock_bh(&iop_chan->lock);
 704         slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op);
 705         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 706         if (sw_desc) {
 707                 /* for validate operations p and q are tagged onto the
 708                  * end of the source list
 709                  */
 710                 int pq_idx = src_cnt;
 711
 712                 g = sw_desc->group_head;
 713                 iop_desc_init_pq_zero_sum(g, src_cnt+2, flags);
 714                 iop_desc_set_pq_zero_sum_byte_count(g, len);
 715                 g->pq_check_result = pqres;
 716                 pr_debug("\t%s: g->pq_check_result: %p\n",
 717                         __func__, g->pq_check_result);
 718                 sw_desc->async_tx.flags = flags;
 719                 while (src_cnt--)
 720                         iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
 721                                                           src[src_cnt],
 722                                                           scf[src_cnt]);
 723                 iop_desc_set_pq_zero_sum_addr(g, pq_idx, src);
 724         }
 725         spin_unlock_bh(&iop_chan->lock);
 726
 727         return sw_desc ? &sw_desc->async_tx : NULL;
 728 }
 729
 730 static void iop_adma_free_chan_resources(struct dma_chan *chan)
 731 {
 732         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 733         struct iop_adma_desc_slot *iter, *_iter;
 734         int in_use_descs = 0;
 735
 736         iop_adma_slot_cleanup(iop_chan);
 737
 738         spin_lock_bh(&iop_chan->lock);
 739         list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
 740                                         chain_node) {
 741                 in_use_descs++;
 742                 list_del(&iter->chain_node);
 743         }
 744         list_for_each_entry_safe_reverse(
 745                 iter, _iter, &iop_chan->all_slots, slot_node) {
 746                 list_del(&iter->slot_node);
 747                 kfree(iter);
 748                 iop_chan->slots_allocated--;
 749         }
 750         iop_chan->last_used = NULL;
 751
 752         dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n",
 753                 __func__, iop_chan->slots_allocated);
 754         spin_unlock_bh(&iop_chan->lock);
 755
 756         /* one is ok since we left it on there on purpose */
 757         if (in_use_descs > 1)
 758                 printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n",
 759                         in_use_descs - 1);
 760 }
 761
 762 /**
 763  * iop_adma_status - poll the status of an ADMA transaction
 764  * @chan: ADMA channel handle
 765  * @cookie: ADMA transaction identifier
 766  * @txstate: a holder for the current state of the channel or NULL
 767  */
 768 static enum dma_status iop_adma_status(struct dma_chan *chan,
 769                                         dma_cookie_t cookie,
 770                                         struct dma_tx_state *txstate)
 771 {
 772         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 773         int ret;
 774
 775         ret = dma_cookie_status(chan, cookie, txstate);
 776         if (ret == DMA_COMPLETE)
 777                 return ret;
 778
 779         iop_adma_slot_cleanup(iop_chan);
 780
 781         return dma_cookie_status(chan, cookie, txstate);
 782 }
 783
 784 static irqreturn_t iop_adma_eot_handler(int irq, void *data)
 785 {
 786         struct iop_adma_chan *chan = data;
 787
 788         dev_dbg(chan->device->common.dev, "%s\n", __func__);
 789
 790         tasklet_schedule(&chan->irq_tasklet);
 791
 792         iop_adma_device_clear_eot_status(chan);
 793
 794         return IRQ_HANDLED;
 795 }
 796
 797 static irqreturn_t iop_adma_eoc_handler(int irq, void *data)
 798 {
 799         struct iop_adma_chan *chan = data;
 800
 801         dev_dbg(chan->device->common.dev, "%s\n", __func__);
 802
 803         tasklet_schedule(&chan->irq_tasklet);
 804
 805         iop_adma_device_clear_eoc_status(chan);
 806
 807         return IRQ_HANDLED;
 808 }
 809
 810 static irqreturn_t iop_adma_err_handler(int irq, void *data)
 811 {
 812         struct iop_adma_chan *chan = data;
 813         unsigned long status = iop_chan_get_status(chan);
 814
 815         dev_err(chan->device->common.dev,
 816                 "error ( %s%s%s%s%s%s%s)\n",
 817                 iop_is_err_int_parity(status, chan) ? "int_parity " : "",
 818                 iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "",
 819                 iop_is_err_int_tabort(status, chan) ? "int_tabort " : "",
 820                 iop_is_err_int_mabort(status, chan) ? "int_mabort " : "",
 821                 iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "",
 822                 iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "",
 823                 iop_is_err_split_tx(status, chan) ? "split_tx " : "");
 824
 825         iop_adma_device_clear_err_status(chan);
 826
 827         BUG();
 828
 829         return IRQ_HANDLED;
 830 }
 831
 832 static void iop_adma_issue_pending(struct dma_chan *chan)
 833 {
 834         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 835
 836         if (iop_chan->pending) {
 837                 iop_chan->pending = 0;
 838                 iop_chan_append(iop_chan);
 839         }
 840 }
 841
 842 /*
 843  * Perform a transaction to verify the HW works.
 844  */
 845 #define IOP_ADMA_TEST_SIZE 2000
 846
 847 static int iop_adma_memcpy_self_test(struct iop_adma_device *device)
 848 {
 849         int i;
 850         void *src, *dest;
 851         dma_addr_t src_dma, dest_dma;
 852         struct dma_chan *dma_chan;
 853         dma_cookie_t cookie;
 854         struct dma_async_tx_descriptor *tx;
 855         int err = 0;
 856         struct iop_adma_chan *iop_chan;
 857
 858         dev_dbg(device->common.dev, "%s\n", __func__);
 859
 860         src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
 861         if (!src)
 862                 return -ENOMEM;
 863         dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
 864         if (!dest) {
 865                 kfree(src);
 866                 return -ENOMEM;
 867         }
 868
 869         /* Fill in src buffer */
 870         for (i = 0; i < IOP_ADMA_TEST_SIZE; i++)
 871                 ((u8 *) src)[i] = (u8)i;
 872
 873         /* Start copy, using first DMA channel */
 874         dma_chan = container_of(device->common.channels.next,
 875                                 struct dma_chan,
 876                                 device_node);
 877         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
 878                 err = -ENODEV;
 879                 goto out;
 880         }
 881
 882         dest_dma = dma_map_single(dma_chan->device->dev, dest,
 883                                 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
 884         src_dma = dma_map_single(dma_chan->device->dev, src,
 885                                 IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE);
 886         tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
 887                                       IOP_ADMA_TEST_SIZE,
 888                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 889
 890         cookie = iop_adma_tx_submit(tx);
 891         iop_adma_issue_pending(dma_chan);
 892         msleep(1);
 893
 894         if (iop_adma_status(dma_chan, cookie, NULL) !=
 895                         DMA_COMPLETE) {
 896                 dev_err(dma_chan->device->dev,
 897                         "Self-test copy timed out, disabling\n");
 898                 err = -ENODEV;
 899                 goto free_resources;
 900         }
 901
 902         iop_chan = to_iop_adma_chan(dma_chan);
 903         dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
 904                 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
 905         if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) {
 906                 dev_err(dma_chan->device->dev,
 907                         "Self-test copy failed compare, disabling\n");
 908                 err = -ENODEV;
 909                 goto free_resources;
 910         }
 911
 912 free_resources:
 913         iop_adma_free_chan_resources(dma_chan);
 914 out:
 915         kfree(src);
 916         kfree(dest);
 917         return err;
 918 }
 919
 920 #define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */
 921 static int
 922 iop_adma_xor_val_self_test(struct iop_adma_device *device)
 923 {
 924         int i, src_idx;
 925         struct page *dest;
 926         struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST];
 927         struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
 928         dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
 929         dma_addr_t dest_dma;
 930         struct dma_async_tx_descriptor *tx;
 931         struct dma_chan *dma_chan;
 932         dma_cookie_t cookie;
 933         u8 cmp_byte = 0;
 934         u32 cmp_word;
 935         u32 zero_sum_result;
 936         int err = 0;
 937         struct iop_adma_chan *iop_chan;
 938
 939         dev_dbg(device->common.dev, "%s\n", __func__);
 940
 941         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
 942                 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
 943                 if (!xor_srcs[src_idx]) {
 944                         while (src_idx--)
 945                                 __free_page(xor_srcs[src_idx]);
 946                         return -ENOMEM;
 947                 }
 948         }
 949
 950         dest = alloc_page(GFP_KERNEL);
 951         if (!dest) {
 952                 while (src_idx--)
 953                         __free_page(xor_srcs[src_idx]);
 954                 return -ENOMEM;
 955         }
 956
 957         /* Fill in src buffers */
 958         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
 959                 u8 *ptr = page_address(xor_srcs[src_idx]);
 960                 for (i = 0; i < PAGE_SIZE; i++)
 961                         ptr[i] = (1 << src_idx);
 962         }
 963
 964         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++)
 965                 cmp_byte ^= (u8) (1 << src_idx);
 966
 967         cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
 968                         (cmp_byte << 8) | cmp_byte;
 969
 970         memset(page_address(dest), 0, PAGE_SIZE);
 971
 972         dma_chan = container_of(device->common.channels.next,
 973                                 struct dma_chan,
 974                                 device_node);
 975         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
 976                 err = -ENODEV;
 977                 goto out;
 978         }
 979
 980         /* test xor */
 981         dest_dma = dma_map_page(dma_chan->device->dev, dest, 0,
 982                                 PAGE_SIZE, DMA_FROM_DEVICE);
 983         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
 984                 dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
 985                                            0, PAGE_SIZE, DMA_TO_DEVICE);
 986         tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
 987                                    IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE,
 988                                    DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 989
 990         cookie = iop_adma_tx_submit(tx);
 991         iop_adma_issue_pending(dma_chan);
 992         msleep(8);
 993
 994         if (iop_adma_status(dma_chan, cookie, NULL) !=
 995                 DMA_COMPLETE) {
 996                 dev_err(dma_chan->device->dev,
 997                         "Self-test xor timed out, disabling\n");
 998                 err = -ENODEV;
 999                 goto free_resources;
1000         }
1001
1002         iop_chan = to_iop_adma_chan(dma_chan);
1003         dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
1004                 PAGE_SIZE, DMA_FROM_DEVICE);
1005         for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
1006                 u32 *ptr = page_address(dest);
1007                 if (ptr[i] != cmp_word) {
1008                         dev_err(dma_chan->device->dev,
1009                                 "Self-test xor failed compare, disabling\n");
1010                         err = -ENODEV;
1011                         goto free_resources;
1012                 }
1013         }
1014         dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma,
1015                 PAGE_SIZE, DMA_TO_DEVICE);
1016
1017         /* skip zero sum if the capability is not present */
1018         if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
1019                 goto free_resources;
1020
1021         /* zero sum the sources with the destintation page */
1022         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1023                 zero_sum_srcs[i] = xor_srcs[i];
1024         zero_sum_srcs[i] = dest;
1025
1026         zero_sum_result = 1;
1027
1028         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1029                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1030                                            zero_sum_srcs[i], 0, PAGE_SIZE,
1031                                            DMA_TO_DEVICE);
1032         tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1033                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1034                                        &zero_sum_result,
1035                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1036
1037         cookie = iop_adma_tx_submit(tx);
1038         iop_adma_issue_pending(dma_chan);
1039         msleep(8);
1040
1041         if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
1042                 dev_err(dma_chan->device->dev,
1043                         "Self-test zero sum timed out, disabling\n");
1044                 err = -ENODEV;
1045                 goto free_resources;
1046         }
1047
1048         if (zero_sum_result != 0) {
1049                 dev_err(dma_chan->device->dev,
1050                         "Self-test zero sum failed compare, disabling\n");
1051                 err = -ENODEV;
1052                 goto free_resources;
1053         }
1054
1055         /* test for non-zero parity sum */
1056         zero_sum_result = 0;
1057         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1058                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1059                                            zero_sum_srcs[i], 0, PAGE_SIZE,
1060                                            DMA_TO_DEVICE);
1061         tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1062                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1063                                        &zero_sum_result,
1064                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1065
1066         cookie = iop_adma_tx_submit(tx);
1067         iop_adma_issue_pending(dma_chan);
1068         msleep(8);
1069
1070         if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
1071                 dev_err(dma_chan->device->dev,
1072                         "Self-test non-zero sum timed out, disabling\n");
1073                 err = -ENODEV;
1074                 goto free_resources;
1075         }
1076
1077         if (zero_sum_result != 1) {
1078                 dev_err(dma_chan->device->dev,
1079                         "Self-test non-zero sum failed compare, disabling\n");
1080                 err = -ENODEV;
1081                 goto free_resources;
1082         }
1083
1084 free_resources:
1085         iop_adma_free_chan_resources(dma_chan);
1086 out:
1087         src_idx = IOP_ADMA_NUM_SRC_TEST;
1088         while (src_idx--)
1089                 __free_page(xor_srcs[src_idx]);
1090         __free_page(dest);
1091         return err;
1092 }
1093
1094 #ifdef CONFIG_RAID6_PQ
1095 static int
1096 iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device)
1097 {
1098         /* combined sources, software pq results, and extra hw pq results */
1099         struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2];
1100         /* ptr to the extra hw pq buffers defined above */
1101         struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2];
1102         /* address conversion buffers (dma_map / page_address) */
1103         void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2];
1104         dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST+2];
1105         dma_addr_t *pq_dest = &pq_src[IOP_ADMA_NUM_SRC_TEST];
1106
1107         int i;
1108         struct dma_async_tx_descriptor *tx;
1109         struct dma_chan *dma_chan;
1110         dma_cookie_t cookie;
1111         u32 zero_sum_result;
1112         int err = 0;
1113         struct device *dev;
1114
1115         dev_dbg(device->common.dev, "%s\n", __func__);
1116
1117         for (i = 0; i < ARRAY_SIZE(pq); i++) {
1118                 pq[i] = alloc_page(GFP_KERNEL);
1119                 if (!pq[i]) {
1120                         while (i--)
1121                                 __free_page(pq[i]);
1122                         return -ENOMEM;
1123                 }
1124         }
1125
1126         /* Fill in src buffers */
1127         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) {
1128                 pq_sw[i] = page_address(pq[i]);
1129                 memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE);
1130         }
1131         pq_sw[i] = page_address(pq[i]);
1132         pq_sw[i+1] = page_address(pq[i+1]);
1133
1134         dma_chan = container_of(device->common.channels.next,
1135                                 struct dma_chan,
1136                                 device_node);
1137         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1138                 err = -ENODEV;
1139                 goto out;
1140         }
1141
1142         dev = dma_chan->device->dev;
1143
1144         /* initialize the dests */
1145         memset(page_address(pq_hw[0]), 0 , PAGE_SIZE);
1146         memset(page_address(pq_hw[1]), 0 , PAGE_SIZE);
1147
1148         /* test pq */
1149         pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1150         pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1151         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1152                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1153                                          DMA_TO_DEVICE);
1154
1155         tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src,
1156                                   IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp,
1157                                   PAGE_SIZE,
1158                                   DMA_PREP_INTERRUPT |
1159                                   DMA_CTRL_ACK);
1160
1161         cookie = iop_adma_tx_submit(tx);
1162         iop_adma_issue_pending(dma_chan);
1163         msleep(8);
1164
1165         if (iop_adma_status(dma_chan, cookie, NULL) !=
1166                 DMA_COMPLETE) {
1167                 dev_err(dev, "Self-test pq timed out, disabling\n");
1168                 err = -ENODEV;
1169                 goto free_resources;
1170         }
1171
1172         raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw);
1173
1174         if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST],
1175                    page_address(pq_hw[0]), PAGE_SIZE) != 0) {
1176                 dev_err(dev, "Self-test p failed compare, disabling\n");
1177                 err = -ENODEV;
1178                 goto free_resources;
1179         }
1180         if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1],
1181                    page_address(pq_hw[1]), PAGE_SIZE) != 0) {
1182                 dev_err(dev, "Self-test q failed compare, disabling\n");
1183                 err = -ENODEV;
1184                 goto free_resources;
1185         }
1186
1187         /* test correct zero sum using the software generated pq values */
1188         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1189                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1190                                          DMA_TO_DEVICE);
1191
1192         zero_sum_result = ~0;
1193         tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1194                                       pq_src, IOP_ADMA_NUM_SRC_TEST,
1195                                       raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1196                                       DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1197
1198         cookie = iop_adma_tx_submit(tx);
1199         iop_adma_issue_pending(dma_chan);
1200         msleep(8);
1201
1202         if (iop_adma_status(dma_chan, cookie, NULL) !=
1203                 DMA_COMPLETE) {
1204                 dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n");
1205                 err = -ENODEV;
1206                 goto free_resources;
1207         }
1208
1209         if (zero_sum_result != 0) {
1210                 dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n",
1211                         zero_sum_result);
1212                 err = -ENODEV;
1213                 goto free_resources;
1214         }
1215
1216         /* test incorrect zero sum */
1217         i = IOP_ADMA_NUM_SRC_TEST;
1218         memset(pq_sw[i] + 100, 0, 100);
1219         memset(pq_sw[i+1] + 200, 0, 200);
1220         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1221                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1222                                          DMA_TO_DEVICE);
1223
1224         zero_sum_result = 0;
1225         tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1226                                       pq_src, IOP_ADMA_NUM_SRC_TEST,
1227                                       raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1228                                       DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1229
1230         cookie = iop_adma_tx_submit(tx);
1231         iop_adma_issue_pending(dma_chan);
1232         msleep(8);
1233
1234         if (iop_adma_status(dma_chan, cookie, NULL) !=
1235                 DMA_COMPLETE) {
1236                 dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n");
1237                 err = -ENODEV;
1238                 goto free_resources;
1239         }
1240
1241         if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) {
1242                 dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n",
1243                         zero_sum_result);
1244                 err = -ENODEV;
1245                 goto free_resources;
1246         }
1247
1248 free_resources:
1249         iop_adma_free_chan_resources(dma_chan);
1250 out:
1251         i = ARRAY_SIZE(pq);
1252         while (i--)
1253                 __free_page(pq[i]);
1254         return err;
1255 }
1256 #endif
1257
1258 static int iop_adma_remove(struct platform_device *dev)
1259 {
1260         struct iop_adma_device *device = platform_get_drvdata(dev);
1261         struct dma_chan *chan, *_chan;
1262         struct iop_adma_chan *iop_chan;
1263         struct iop_adma_platform_data *plat_data = dev_get_platdata(&dev->dev);
1264
1265         dma_async_device_unregister(&device->common);
1266
1267         dma_free_coherent(&dev->dev, plat_data->pool_size,
1268                         device->dma_desc_pool_virt, device->dma_desc_pool);
1269
1270         list_for_each_entry_safe(chan, _chan, &device->common.channels,
1271                                 device_node) {
1272                 iop_chan = to_iop_adma_chan(chan);
1273                 list_del(&chan->device_node);
1274                 kfree(iop_chan);
1275         }
1276         kfree(device);
1277
1278         return 0;
1279 }
1280
1281 static int iop_adma_probe(struct platform_device *pdev)
1282 {
1283         struct resource *res;
1284         int ret = 0, i;
1285         struct iop_adma_device *adev;
1286         struct iop_adma_chan *iop_chan;
1287         struct dma_device *dma_dev;
1288         struct iop_adma_platform_data *plat_data = dev_get_platdata(&pdev->dev);
1289
1290         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1291         if (!res)
1292                 return -ENODEV;
1293
1294         if (!devm_request_mem_region(&pdev->dev, res->start,
1295                                 resource_size(res), pdev->name))
1296                 return -EBUSY;
1297
1298         adev = kzalloc(sizeof(*adev), GFP_KERNEL);
1299         if (!adev)
1300                 return -ENOMEM;
1301         dma_dev = &adev->common;
1302
1303         /* allocate coherent memory for hardware descriptors
1304          * note: writecombine gives slightly better performance, but
1305          * requires that we explicitly flush the writes
1306          */
1307         if ((adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev,
1308                                         plat_data->pool_size,
1309                                         &adev->dma_desc_pool,
1310                                         GFP_KERNEL)) == NULL) {
1311                 ret = -ENOMEM;
1312                 goto err_free_adev;
1313         }
1314
1315         dev_dbg(&pdev->dev, "%s: allocated descriptor pool virt %p phys %p\n",
1316                 __func__, adev->dma_desc_pool_virt,
1317                 (void *) adev->dma_desc_pool);
1318
1319         adev->id = plat_data->hw_id;
1320
1321         /* discover transaction capabilites from the platform data */
1322         dma_dev->cap_mask = plat_data->cap_mask;
1323
1324         adev->pdev = pdev;
1325         platform_set_drvdata(pdev, adev);
1326
1327         INIT_LIST_HEAD(&dma_dev->channels);
1328
1329         /* set base routines */
1330         dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources;
1331         dma_dev->device_free_chan_resources = iop_adma_free_chan_resources;
1332         dma_dev->device_tx_status = iop_adma_status;
1333         dma_dev->device_issue_pending = iop_adma_issue_pending;
1334         dma_dev->dev = &pdev->dev;
1335
1336         /* set prep routines based on capability */
1337         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1338                 dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy;
1339         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1340                 dma_dev->max_xor = iop_adma_get_max_xor();
1341                 dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor;
1342         }
1343         if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask))
1344                 dma_dev->device_prep_dma_xor_val =
1345                         iop_adma_prep_dma_xor_val;
1346         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1347                 dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0);
1348                 dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq;
1349         }
1350         if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask))
1351                 dma_dev->device_prep_dma_pq_val =
1352                         iop_adma_prep_dma_pq_val;
1353         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1354                 dma_dev->device_prep_dma_interrupt =
1355                         iop_adma_prep_dma_interrupt;
1356
1357         iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL);
1358         if (!iop_chan) {
1359                 ret = -ENOMEM;
1360                 goto err_free_dma;
1361         }
1362         iop_chan->device = adev;
1363
1364         iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start,
1365                                         resource_size(res));
1366         if (!iop_chan->mmr_base) {
1367                 ret = -ENOMEM;
1368                 goto err_free_iop_chan;
1369         }
1370         tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long)
1371                 iop_chan);
1372
1373         /* clear errors before enabling interrupts */
1374         iop_adma_device_clear_err_status(iop_chan);
1375
1376         for (i = 0; i < 3; i++) {
1377                 irq_handler_t handler[] = { iop_adma_eot_handler,
1378                                         iop_adma_eoc_handler,
1379                                         iop_adma_err_handler };
1380                 int irq = platform_get_irq(pdev, i);
1381                 if (irq < 0) {
1382                         ret = -ENXIO;
1383                         goto err_free_iop_chan;
1384                 } else {
1385                         ret = devm_request_irq(&pdev->dev, irq,
1386                                         handler[i], 0, pdev->name, iop_chan);
1387                         if (ret)
1388                                 goto err_free_iop_chan;
1389                 }
1390         }
1391
1392         spin_lock_init(&iop_chan->lock);
1393         INIT_LIST_HEAD(&iop_chan->chain);
1394         INIT_LIST_HEAD(&iop_chan->all_slots);
1395         iop_chan->common.device = dma_dev;
1396         dma_cookie_init(&iop_chan->common);
1397         list_add_tail(&iop_chan->common.device_node, &dma_dev->channels);
1398
1399         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1400                 ret = iop_adma_memcpy_self_test(adev);
1401                 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1402                 if (ret)
1403                         goto err_free_iop_chan;
1404         }
1405
1406         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1407                 ret = iop_adma_xor_val_self_test(adev);
1408                 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1409                 if (ret)
1410                         goto err_free_iop_chan;
1411         }
1412
1413         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) &&
1414             dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) {
1415                 #ifdef CONFIG_RAID6_PQ
1416                 ret = iop_adma_pq_zero_sum_self_test(adev);
1417                 dev_dbg(&pdev->dev, "pq self test returned %d\n", ret);
1418                 #else
1419                 /* can not test raid6, so do not publish capability */
1420                 dma_cap_clear(DMA_PQ, dma_dev->cap_mask);
1421                 dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask);
1422                 ret = 0;
1423                 #endif
1424                 if (ret)
1425                         goto err_free_iop_chan;
1426         }
1427
1428         dev_info(&pdev->dev, "Intel(R) IOP: ( %s%s%s%s%s%s)\n",
1429                  dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "",
1430                  dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "",
1431                  dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1432                  dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "",
1433                  dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1434                  dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1435
1436         dma_async_device_register(dma_dev);
1437         goto out;
1438
1439  err_free_iop_chan:
1440         kfree(iop_chan);
1441  err_free_dma:
1442         dma_free_coherent(&adev->pdev->dev, plat_data->pool_size,
1443                         adev->dma_desc_pool_virt, adev->dma_desc_pool);
1444  err_free_adev:
1445         kfree(adev);
1446  out:
1447         return ret;
1448 }
1449
1450 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
1451 {
1452         struct iop_adma_desc_slot *sw_desc, *grp_start;
1453         dma_cookie_t cookie;
1454         int slot_cnt, slots_per_op;
1455
1456         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1457
1458         spin_lock_bh(&iop_chan->lock);
1459         slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op);
1460         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1461         if (sw_desc) {
1462                 grp_start = sw_desc->group_head;
1463
1464                 list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1465                 async_tx_ack(&sw_desc->async_tx);
1466                 iop_desc_init_memcpy(grp_start, 0);
1467                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1468                 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1469                 iop_desc_set_memcpy_src_addr(grp_start, 0);
1470
1471                 cookie = dma_cookie_assign(&sw_desc->async_tx);
1472
1473                 /* initialize the completed cookie to be less than
1474                  * the most recently used cookie
1475                  */
1476                 iop_chan->common.completed_cookie = cookie - 1;
1477
1478                 /* channel should not be busy */
1479                 BUG_ON(iop_chan_is_busy(iop_chan));
1480
1481                 /* clear any prior error-status bits */
1482                 iop_adma_device_clear_err_status(iop_chan);
1483
1484                 /* disable operation */
1485                 iop_chan_disable(iop_chan);
1486
1487                 /* set the descriptor address */
1488                 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1489
1490                 /* 1/ don't add pre-chained descriptors
1491                  * 2/ dummy read to flush next_desc write
1492                  */
1493                 BUG_ON(iop_desc_get_next_desc(sw_desc));
1494
1495                 /* run the descriptor */
1496                 iop_chan_enable(iop_chan);
1497         } else
1498                 dev_err(iop_chan->device->common.dev,
1499                         "failed to allocate null descriptor\n");
1500         spin_unlock_bh(&iop_chan->lock);
1501 }
1502
1503 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
1504 {
1505         struct iop_adma_desc_slot *sw_desc, *grp_start;
1506         dma_cookie_t cookie;
1507         int slot_cnt, slots_per_op;
1508
1509         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1510
1511         spin_lock_bh(&iop_chan->lock);
1512         slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op);
1513         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1514         if (sw_desc) {
1515                 grp_start = sw_desc->group_head;
1516                 list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1517                 async_tx_ack(&sw_desc->async_tx);
1518                 iop_desc_init_null_xor(grp_start, 2, 0);
1519                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1520                 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1521                 iop_desc_set_xor_src_addr(grp_start, 0, 0);
1522                 iop_desc_set_xor_src_addr(grp_start, 1, 0);
1523
1524                 cookie = dma_cookie_assign(&sw_desc->async_tx);
1525
1526                 /* initialize the completed cookie to be less than
1527                  * the most recently used cookie
1528                  */
1529                 iop_chan->common.completed_cookie = cookie - 1;
1530
1531                 /* channel should not be busy */
1532                 BUG_ON(iop_chan_is_busy(iop_chan));
1533
1534                 /* clear any prior error-status bits */
1535                 iop_adma_device_clear_err_status(iop_chan);
1536
1537                 /* disable operation */
1538                 iop_chan_disable(iop_chan);
1539
1540                 /* set the descriptor address */
1541                 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1542
1543                 /* 1/ don't add pre-chained descriptors
1544                  * 2/ dummy read to flush next_desc write
1545                  */
1546                 BUG_ON(iop_desc_get_next_desc(sw_desc));
1547
1548                 /* run the descriptor */
1549                 iop_chan_enable(iop_chan);
1550         } else
1551                 dev_err(iop_chan->device->common.dev,
1552                         "failed to allocate null descriptor\n");
1553         spin_unlock_bh(&iop_chan->lock);
1554 }
1555
1556 static struct platform_driver iop_adma_driver = {
1557         .probe          = iop_adma_probe,
1558         .remove         = iop_adma_remove,
1559         .driver         = {
1560                 .owner  = THIS_MODULE,
1561                 .name   = "iop-adma",
1562         },
1563 };
1564
1565 module_platform_driver(iop_adma_driver);
1566
1567 MODULE_AUTHOR("Intel Corporation");
1568 MODULE_DESCRIPTION("IOP ADMA Engine Driver");
1569 MODULE_LICENSE("GPL");
1570 MODULE_ALIAS("platform:iop-adma");