drivers/dma/ppc4xx/adma.c

   1 /*
   2  * Copyright (C) 2006-2009 DENX Software Engineering.
   3  *
   4  * Author: Yuri Tikhonov <yur@emcraft.com>
   5  *
   6  * Further porting to arch/powerpc by
   7  *      Anatolij Gustschin <agust@denx.de>
   8  *
   9  * This program is free software; you can redistribute it and/or modify it
  10  * under the terms of the GNU General Public License as published by the Free
  11  * Software Foundation; either version 2 of the License, or (at your option)
  12  * any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful, but WITHOUT
  15  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  16  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  17  * more details.
  18  *
  19  * You should have received a copy of the GNU General Public License along with
  20  * this program; if not, write to the Free Software Foundation, Inc., 59
  21  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  22  *
  23  * The full GNU General Public License is included in this distribution in the
  24  * file called COPYING.
  25  */
  26
  27 /*
  28  * This driver supports the asynchrounous DMA copy and RAID engines available
  29  * on the AMCC PPC440SPe Processors.
  30  * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
  31  * ADMA driver written by D.Williams.
  32  */
  33
  34 #include <linux/init.h>
  35 #include <linux/module.h>
  36 #include <linux/async_tx.h>
  37 #include <linux/delay.h>
  38 #include <linux/dma-mapping.h>
  39 #include <linux/spinlock.h>
  40 #include <linux/interrupt.h>
  41 #include <linux/slab.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/proc_fs.h>
  44 #include <linux/of.h>
  45 #include <linux/of_address.h>
  46 #include <linux/of_irq.h>
  47 #include <linux/of_platform.h>
  48 #include <asm/dcr.h>
  49 #include <asm/dcr-regs.h>
  50 #include "adma.h"
  51 #include "../dmaengine.h"
  52
  53 enum ppc_adma_init_code {
  54         PPC_ADMA_INIT_OK = 0,
  55         PPC_ADMA_INIT_MEMRES,
  56         PPC_ADMA_INIT_MEMREG,
  57         PPC_ADMA_INIT_ALLOC,
  58         PPC_ADMA_INIT_COHERENT,
  59         PPC_ADMA_INIT_CHANNEL,
  60         PPC_ADMA_INIT_IRQ1,
  61         PPC_ADMA_INIT_IRQ2,
  62         PPC_ADMA_INIT_REGISTER
  63 };
  64
  65 static char *ppc_adma_errors[] = {
  66         [PPC_ADMA_INIT_OK] = "ok",
  67         [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
  68         [PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
  69         [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
  70                                 "structure",
  71         [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
  72                                    "hardware descriptors",
  73         [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
  74         [PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
  75         [PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
  76         [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
  77 };
  78
  79 static enum ppc_adma_init_code
  80 ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
  81
  82 struct ppc_dma_chan_ref {
  83         struct dma_chan *chan;
  84         struct list_head node;
  85 };
  86
  87 /* The list of channels exported by ppc440spe ADMA */
  88 struct list_head
  89 ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
  90
  91 /* This flag is set when want to refetch the xor chain in the interrupt
  92  * handler
  93  */
  94 static u32 do_xor_refetch;
  95
  96 /* Pointer to DMA0, DMA1 CP/CS FIFO */
  97 static void *ppc440spe_dma_fifo_buf;
  98
  99 /* Pointers to last submitted to DMA0, DMA1 CDBs */
 100 static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
 101 static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
 102
 103 /* Pointer to last linked and submitted xor CB */
 104 static struct ppc440spe_adma_desc_slot *xor_last_linked;
 105 static struct ppc440spe_adma_desc_slot *xor_last_submit;
 106
 107 /* This array is used in data-check operations for storing a pattern */
 108 static char ppc440spe_qword[16];
 109
 110 static atomic_t ppc440spe_adma_err_irq_ref;
 111 static dcr_host_t ppc440spe_mq_dcr_host;
 112 static unsigned int ppc440spe_mq_dcr_len;
 113
 114 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
 115  * the block size in transactions, then we do not allow to activate more than
 116  * only one RXOR transactions simultaneously. So use this var to store
 117  * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
 118  * set) or not (PPC440SPE_RXOR_RUN is clear).
 119  */
 120 static unsigned long ppc440spe_rxor_state;
 121
 122 /* These are used in enable & check routines
 123  */
 124 static u32 ppc440spe_r6_enabled;
 125 static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
 126 static struct completion ppc440spe_r6_test_comp;
 127
 128 static int ppc440spe_adma_dma2rxor_prep_src(
 129                 struct ppc440spe_adma_desc_slot *desc,
 130                 struct ppc440spe_rxor *cursor, int index,
 131                 int src_cnt, u32 addr);
 132 static void ppc440spe_adma_dma2rxor_set_src(
 133                 struct ppc440spe_adma_desc_slot *desc,
 134                 int index, dma_addr_t addr);
 135 static void ppc440spe_adma_dma2rxor_set_mult(
 136                 struct ppc440spe_adma_desc_slot *desc,
 137                 int index, u8 mult);
 138
 139 #ifdef ADMA_LL_DEBUG
 140 #define ADMA_LL_DBG(x) ({ if (1) x; 0; })
 141 #else
 142 #define ADMA_LL_DBG(x) ({ if (0) x; 0; })
 143 #endif
 144
 145 static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
 146 {
 147         struct dma_cdb *cdb;
 148         struct xor_cb *cb;
 149         int i;
 150
 151         switch (chan->device->id) {
 152         case 0:
 153         case 1:
 154                 cdb = block;
 155
 156                 pr_debug("CDB at %p [%d]:\n"
 157                         "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
 158                         "\t sg1u 0x%08x sg1l 0x%08x\n"
 159                         "\t sg2u 0x%08x sg2l 0x%08x\n"
 160                         "\t sg3u 0x%08x sg3l 0x%08x\n",
 161                         cdb, chan->device->id,
 162                         cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
 163                         le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
 164                         le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
 165                         le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
 166                 );
 167                 break;
 168         case 2:
 169                 cb = block;
 170
 171                 pr_debug("CB at %p [%d]:\n"
 172                         "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
 173                         "\t cbtah 0x%08x cbtal 0x%08x\n"
 174                         "\t cblah 0x%08x cblal 0x%08x\n",
 175                         cb, chan->device->id,
 176                         cb->cbc, cb->cbbc, cb->cbs,
 177                         cb->cbtah, cb->cbtal,
 178                         cb->cblah, cb->cblal);
 179                 for (i = 0; i < 16; i++) {
 180                         if (i && !cb->ops[i].h && !cb->ops[i].l)
 181                                 continue;
 182                         pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
 183                                 i, cb->ops[i].h, cb->ops[i].l);
 184                 }
 185                 break;
 186         }
 187 }
 188
 189 static void print_cb_list(struct ppc440spe_adma_chan *chan,
 190                           struct ppc440spe_adma_desc_slot *iter)
 191 {
 192         for (; iter; iter = iter->hw_next)
 193                 print_cb(chan, iter->hw_desc);
 194 }
 195
 196 static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
 197                              unsigned int src_cnt)
 198 {
 199         int i;
 200
 201         pr_debug("\n%s(%d):\nsrc: ", __func__, id);
 202         for (i = 0; i < src_cnt; i++)
 203                 pr_debug("\t0x%016llx ", src[i]);
 204         pr_debug("dst:\n\t0x%016llx\n", dst);
 205 }
 206
 207 static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
 208                             unsigned int src_cnt)
 209 {
 210         int i;
 211
 212         pr_debug("\n%s(%d):\nsrc: ", __func__, id);
 213         for (i = 0; i < src_cnt; i++)
 214                 pr_debug("\t0x%016llx ", src[i]);
 215         pr_debug("dst: ");
 216         for (i = 0; i < 2; i++)
 217                 pr_debug("\t0x%016llx ", dst[i]);
 218 }
 219
 220 static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
 221                                     unsigned int src_cnt,
 222                                     const unsigned char *scf)
 223 {
 224         int i;
 225
 226         pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
 227         if (scf) {
 228                 for (i = 0; i < src_cnt; i++)
 229                         pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
 230         } else {
 231                 for (i = 0; i < src_cnt; i++)
 232                         pr_debug("\t0x%016llx(no) ", src[i]);
 233         }
 234
 235         pr_debug("dst: ");
 236         for (i = 0; i < 2; i++)
 237                 pr_debug("\t0x%016llx ", src[src_cnt + i]);
 238 }
 239
 240 /******************************************************************************
 241  * Command (Descriptor) Blocks low-level routines
 242  ******************************************************************************/
 243 /**
 244  * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
 245  * pseudo operation
 246  */
 247 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
 248                                           struct ppc440spe_adma_chan *chan)
 249 {
 250         struct xor_cb *p;
 251
 252         switch (chan->device->id) {
 253         case PPC440SPE_XOR_ID:
 254                 p = desc->hw_desc;
 255                 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 256                 /* NOP with Command Block Complete Enable */
 257                 p->cbc = XOR_CBCR_CBCE_BIT;
 258                 break;
 259         case PPC440SPE_DMA0_ID:
 260         case PPC440SPE_DMA1_ID:
 261                 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
 262                 /* NOP with interrupt */
 263                 set_bit(PPC440SPE_DESC_INT, &desc->flags);
 264                 break;
 265         default:
 266                 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
 267                                 __func__);
 268                 break;
 269         }
 270 }
 271
 272 /**
 273  * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
 274  * pseudo operation
 275  */
 276 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
 277 {
 278         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 279         desc->hw_next = NULL;
 280         desc->src_cnt = 0;
 281         desc->dst_cnt = 1;
 282 }
 283
 284 /**
 285  * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
 286  */
 287 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
 288                                          int src_cnt, unsigned long flags)
 289 {
 290         struct xor_cb *hw_desc = desc->hw_desc;
 291
 292         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 293         desc->hw_next = NULL;
 294         desc->src_cnt = src_cnt;
 295         desc->dst_cnt = 1;
 296
 297         hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
 298         if (flags & DMA_PREP_INTERRUPT)
 299                 /* Enable interrupt on completion */
 300                 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
 301 }
 302
 303 /**
 304  * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
 305  * operation in DMA2 controller
 306  */
 307 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
 308                 int dst_cnt, int src_cnt, unsigned long flags)
 309 {
 310         struct xor_cb *hw_desc = desc->hw_desc;
 311
 312         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 313         desc->hw_next = NULL;
 314         desc->src_cnt = src_cnt;
 315         desc->dst_cnt = dst_cnt;
 316         memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
 317         desc->descs_per_op = 0;
 318
 319         hw_desc->cbc = XOR_CBCR_TGT_BIT;
 320         if (flags & DMA_PREP_INTERRUPT)
 321                 /* Enable interrupt on completion */
 322                 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
 323 }
 324
 325 #define DMA_CTRL_FLAGS_LAST     DMA_PREP_FENCE
 326 #define DMA_PREP_ZERO_P         (DMA_CTRL_FLAGS_LAST << 1)
 327 #define DMA_PREP_ZERO_Q         (DMA_PREP_ZERO_P << 1)
 328
 329 /**
 330  * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
 331  * with DMA0/1
 332  */
 333 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
 334                                 int dst_cnt, int src_cnt, unsigned long flags,
 335                                 unsigned long op)
 336 {
 337         struct dma_cdb *hw_desc;
 338         struct ppc440spe_adma_desc_slot *iter;
 339         u8 dopc;
 340
 341         /* Common initialization of a PQ descriptors chain */
 342         set_bits(op, &desc->flags);
 343         desc->src_cnt = src_cnt;
 344         desc->dst_cnt = dst_cnt;
 345
 346         /* WXOR MULTICAST if both P and Q are being computed
 347          * MV_SG1_SG2 if Q only
 348          */
 349         dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
 350                 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
 351
 352         list_for_each_entry(iter, &desc->group_list, chain_node) {
 353                 hw_desc = iter->hw_desc;
 354                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
 355
 356                 if (likely(!list_is_last(&iter->chain_node,
 357                                 &desc->group_list))) {
 358                         /* set 'next' pointer */
 359                         iter->hw_next = list_entry(iter->chain_node.next,
 360                                 struct ppc440spe_adma_desc_slot, chain_node);
 361                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
 362                 } else {
 363                         /* this is the last descriptor.
 364                          * this slot will be pasted from ADMA level
 365                          * each time it wants to configure parameters
 366                          * of the transaction (src, dst, ...)
 367                          */
 368                         iter->hw_next = NULL;
 369                         if (flags & DMA_PREP_INTERRUPT)
 370                                 set_bit(PPC440SPE_DESC_INT, &iter->flags);
 371                         else
 372                                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
 373                 }
 374         }
 375
 376         /* Set OPS depending on WXOR/RXOR type of operation */
 377         if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
 378                 /* This is a WXOR only chain:
 379                  * - first descriptors are for zeroing destinations
 380                  *   if PPC440SPE_ZERO_P/Q set;
 381                  * - descriptors remained are for GF-XOR operations.
 382                  */
 383                 iter = list_first_entry(&desc->group_list,
 384                                         struct ppc440spe_adma_desc_slot,
 385                                         chain_node);
 386
 387                 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
 388                         hw_desc = iter->hw_desc;
 389                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 390                         iter = list_first_entry(&iter->chain_node,
 391                                         struct ppc440spe_adma_desc_slot,
 392                                         chain_node);
 393                 }
 394
 395                 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
 396                         hw_desc = iter->hw_desc;
 397                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 398                         iter = list_first_entry(&iter->chain_node,
 399                                         struct ppc440spe_adma_desc_slot,
 400                                         chain_node);
 401                 }
 402
 403                 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
 404                         hw_desc = iter->hw_desc;
 405                         hw_desc->opc = dopc;
 406                 }
 407         } else {
 408                 /* This is either RXOR-only or mixed RXOR/WXOR */
 409
 410                 /* The first 1 or 2 slots in chain are always RXOR,
 411                  * if need to calculate P & Q, then there are two
 412                  * RXOR slots; if only P or only Q, then there is one
 413                  */
 414                 iter = list_first_entry(&desc->group_list,
 415                                         struct ppc440spe_adma_desc_slot,
 416                                         chain_node);
 417                 hw_desc = iter->hw_desc;
 418                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 419
 420                 if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
 421                         iter = list_first_entry(&iter->chain_node,
 422                                                 struct ppc440spe_adma_desc_slot,
 423                                                 chain_node);
 424                         hw_desc = iter->hw_desc;
 425                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 426                 }
 427
 428                 /* The remaining descs (if any) are WXORs */
 429                 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
 430                         iter = list_first_entry(&iter->chain_node,
 431                                                 struct ppc440spe_adma_desc_slot,
 432                                                 chain_node);
 433                         list_for_each_entry_from(iter, &desc->group_list,
 434                                                 chain_node) {
 435                                 hw_desc = iter->hw_desc;
 436                                 hw_desc->opc = dopc;
 437                         }
 438                 }
 439         }
 440 }
 441
 442 /**
 443  * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
 444  * for PQ_ZERO_SUM operation
 445  */
 446 static void ppc440spe_desc_init_dma01pqzero_sum(
 447                                 struct ppc440spe_adma_desc_slot *desc,
 448                                 int dst_cnt, int src_cnt)
 449 {
 450         struct dma_cdb *hw_desc;
 451         struct ppc440spe_adma_desc_slot *iter;
 452         int i = 0;
 453         u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
 454                                    DMA_CDB_OPC_MV_SG1_SG2;
 455         /*
 456          * Initialize starting from 2nd or 3rd descriptor dependent
 457          * on dst_cnt. First one or two slots are for cloning P
 458          * and/or Q to chan->pdest and/or chan->qdest as we have
 459          * to preserve original P/Q.
 460          */
 461         iter = list_first_entry(&desc->group_list,
 462                                 struct ppc440spe_adma_desc_slot, chain_node);
 463         iter = list_entry(iter->chain_node.next,
 464                           struct ppc440spe_adma_desc_slot, chain_node);
 465
 466         if (dst_cnt > 1) {
 467                 iter = list_entry(iter->chain_node.next,
 468                                   struct ppc440spe_adma_desc_slot, chain_node);
 469         }
 470         /* initialize each source descriptor in chain */
 471         list_for_each_entry_from(iter, &desc->group_list, chain_node) {
 472                 hw_desc = iter->hw_desc;
 473                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
 474                 iter->src_cnt = 0;
 475                 iter->dst_cnt = 0;
 476
 477                 /* This is a ZERO_SUM operation:
 478                  * - <src_cnt> descriptors starting from 2nd or 3rd
 479                  *   descriptor are for GF-XOR operations;
 480                  * - remaining <dst_cnt> descriptors are for checking the result
 481                  */
 482                 if (i++ < src_cnt)
 483                         /* MV_SG1_SG2 if only Q is being verified
 484                          * MULTICAST if both P and Q are being verified
 485                          */
 486                         hw_desc->opc = dopc;
 487                 else
 488                         /* DMA_CDB_OPC_DCHECK128 operation */
 489                         hw_desc->opc = DMA_CDB_OPC_DCHECK128;
 490
 491                 if (likely(!list_is_last(&iter->chain_node,
 492                                          &desc->group_list))) {
 493                         /* set 'next' pointer */
 494                         iter->hw_next = list_entry(iter->chain_node.next,
 495                                                 struct ppc440spe_adma_desc_slot,
 496                                                 chain_node);
 497                 } else {
 498                         /* this is the last descriptor.
 499                          * this slot will be pasted from ADMA level
 500                          * each time it wants to configure parameters
 501                          * of the transaction (src, dst, ...)
 502                          */
 503                         iter->hw_next = NULL;
 504                         /* always enable interrupt generation since we get
 505                          * the status of pqzero from the handler
 506                          */
 507                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
 508                 }
 509         }
 510         desc->src_cnt = src_cnt;
 511         desc->dst_cnt = dst_cnt;
 512 }
 513
 514 /**
 515  * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
 516  */
 517 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
 518                                         unsigned long flags)
 519 {
 520         struct dma_cdb *hw_desc = desc->hw_desc;
 521
 522         memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
 523         desc->hw_next = NULL;
 524         desc->src_cnt = 1;
 525         desc->dst_cnt = 1;
 526
 527         if (flags & DMA_PREP_INTERRUPT)
 528                 set_bit(PPC440SPE_DESC_INT, &desc->flags);
 529         else
 530                 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
 531
 532         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 533 }
 534
 535 /**
 536  * ppc440spe_desc_set_src_addr - set source address into the descriptor
 537  */
 538 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
 539                                         struct ppc440spe_adma_chan *chan,
 540                                         int src_idx, dma_addr_t addrh,
 541                                         dma_addr_t addrl)
 542 {
 543         struct dma_cdb *dma_hw_desc;
 544         struct xor_cb *xor_hw_desc;
 545         phys_addr_t addr64, tmplow, tmphi;
 546
 547         switch (chan->device->id) {
 548         case PPC440SPE_DMA0_ID:
 549         case PPC440SPE_DMA1_ID:
 550                 if (!addrh) {
 551                         addr64 = addrl;
 552                         tmphi = (addr64 >> 32);
 553                         tmplow = (addr64 & 0xFFFFFFFF);
 554                 } else {
 555                         tmphi = addrh;
 556                         tmplow = addrl;
 557                 }
 558                 dma_hw_desc = desc->hw_desc;
 559                 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
 560                 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
 561                 break;
 562         case PPC440SPE_XOR_ID:
 563                 xor_hw_desc = desc->hw_desc;
 564                 xor_hw_desc->ops[src_idx].l = addrl;
 565                 xor_hw_desc->ops[src_idx].h |= addrh;
 566                 break;
 567         }
 568 }
 569
 570 /**
 571  * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
 572  */
 573 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
 574                         struct ppc440spe_adma_chan *chan, u32 mult_index,
 575                         int sg_index, unsigned char mult_value)
 576 {
 577         struct dma_cdb *dma_hw_desc;
 578         struct xor_cb *xor_hw_desc;
 579         u32 *psgu;
 580
 581         switch (chan->device->id) {
 582         case PPC440SPE_DMA0_ID:
 583         case PPC440SPE_DMA1_ID:
 584                 dma_hw_desc = desc->hw_desc;
 585
 586                 switch (sg_index) {
 587                 /* for RXOR operations set multiplier
 588                  * into source cued address
 589                  */
 590                 case DMA_CDB_SG_SRC:
 591                         psgu = &dma_hw_desc->sg1u;
 592                         break;
 593                 /* for WXOR operations set multiplier
 594                  * into destination cued address(es)
 595                  */
 596                 case DMA_CDB_SG_DST1:
 597                         psgu = &dma_hw_desc->sg2u;
 598                         break;
 599                 case DMA_CDB_SG_DST2:
 600                         psgu = &dma_hw_desc->sg3u;
 601                         break;
 602                 default:
 603                         BUG();
 604                 }
 605
 606                 *psgu |= cpu_to_le32(mult_value << mult_index);
 607                 break;
 608         case PPC440SPE_XOR_ID:
 609                 xor_hw_desc = desc->hw_desc;
 610                 break;
 611         default:
 612                 BUG();
 613         }
 614 }
 615
 616 /**
 617  * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
 618  */
 619 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
 620                                 struct ppc440spe_adma_chan *chan,
 621                                 dma_addr_t addrh, dma_addr_t addrl,
 622                                 u32 dst_idx)
 623 {
 624         struct dma_cdb *dma_hw_desc;
 625         struct xor_cb *xor_hw_desc;
 626         phys_addr_t addr64, tmphi, tmplow;
 627         u32 *psgu, *psgl;
 628
 629         switch (chan->device->id) {
 630         case PPC440SPE_DMA0_ID:
 631         case PPC440SPE_DMA1_ID:
 632                 if (!addrh) {
 633                         addr64 = addrl;
 634                         tmphi = (addr64 >> 32);
 635                         tmplow = (addr64 & 0xFFFFFFFF);
 636                 } else {
 637                         tmphi = addrh;
 638                         tmplow = addrl;
 639                 }
 640                 dma_hw_desc = desc->hw_desc;
 641
 642                 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
 643                 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
 644
 645                 *psgl = cpu_to_le32((u32)tmplow);
 646                 *psgu |= cpu_to_le32((u32)tmphi);
 647                 break;
 648         case PPC440SPE_XOR_ID:
 649                 xor_hw_desc = desc->hw_desc;
 650                 xor_hw_desc->cbtal = addrl;
 651                 xor_hw_desc->cbtah |= addrh;
 652                 break;
 653         }
 654 }
 655
 656 /**
 657  * ppc440spe_desc_set_byte_count - set number of data bytes involved
 658  * into the operation
 659  */
 660 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
 661                                 struct ppc440spe_adma_chan *chan,
 662                                 u32 byte_count)
 663 {
 664         struct dma_cdb *dma_hw_desc;
 665         struct xor_cb *xor_hw_desc;
 666
 667         switch (chan->device->id) {
 668         case PPC440SPE_DMA0_ID:
 669         case PPC440SPE_DMA1_ID:
 670                 dma_hw_desc = desc->hw_desc;
 671                 dma_hw_desc->cnt = cpu_to_le32(byte_count);
 672                 break;
 673         case PPC440SPE_XOR_ID:
 674                 xor_hw_desc = desc->hw_desc;
 675                 xor_hw_desc->cbbc = byte_count;
 676                 break;
 677         }
 678 }
 679
 680 /**
 681  * ppc440spe_desc_set_rxor_block_size - set RXOR block size
 682  */
 683 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
 684 {
 685         /* assume that byte_count is aligned on the 512-boundary;
 686          * thus write it directly to the register (bits 23:31 are
 687          * reserved there).
 688          */
 689         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
 690 }
 691
 692 /**
 693  * ppc440spe_desc_set_dcheck - set CHECK pattern
 694  */
 695 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
 696                                 struct ppc440spe_adma_chan *chan, u8 *qword)
 697 {
 698         struct dma_cdb *dma_hw_desc;
 699
 700         switch (chan->device->id) {
 701         case PPC440SPE_DMA0_ID:
 702         case PPC440SPE_DMA1_ID:
 703                 dma_hw_desc = desc->hw_desc;
 704                 iowrite32(qword[0], &dma_hw_desc->sg3l);
 705                 iowrite32(qword[4], &dma_hw_desc->sg3u);
 706                 iowrite32(qword[8], &dma_hw_desc->sg2l);
 707                 iowrite32(qword[12], &dma_hw_desc->sg2u);
 708                 break;
 709         default:
 710                 BUG();
 711         }
 712 }
 713
 714 /**
 715  * ppc440spe_xor_set_link - set link address in xor CB
 716  */
 717 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
 718                                 struct ppc440spe_adma_desc_slot *next_desc)
 719 {
 720         struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
 721
 722         if (unlikely(!next_desc || !(next_desc->phys))) {
 723                 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
 724                         __func__, next_desc,
 725                         next_desc ? next_desc->phys : 0);
 726                 BUG();
 727         }
 728
 729         xor_hw_desc->cbs = 0;
 730         xor_hw_desc->cblal = next_desc->phys;
 731         xor_hw_desc->cblah = 0;
 732         xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
 733 }
 734
 735 /**
 736  * ppc440spe_desc_set_link - set the address of descriptor following this
 737  * descriptor in chain
 738  */
 739 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
 740                                 struct ppc440spe_adma_desc_slot *prev_desc,
 741                                 struct ppc440spe_adma_desc_slot *next_desc)
 742 {
 743         unsigned long flags;
 744         struct ppc440spe_adma_desc_slot *tail = next_desc;
 745
 746         if (unlikely(!prev_desc || !next_desc ||
 747                 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
 748                 /* If previous next is overwritten something is wrong.
 749                  * though we may refetch from append to initiate list
 750                  * processing; in this case - it's ok.
 751                  */
 752                 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
 753                         "prev->hw_next=0x%p\n", __func__, prev_desc,
 754                         next_desc, prev_desc ? prev_desc->hw_next : 0);
 755                 BUG();
 756         }
 757
 758         local_irq_save(flags);
 759
 760         /* do s/w chaining both for DMA and XOR descriptors */
 761         prev_desc->hw_next = next_desc;
 762
 763         switch (chan->device->id) {
 764         case PPC440SPE_DMA0_ID:
 765         case PPC440SPE_DMA1_ID:
 766                 break;
 767         case PPC440SPE_XOR_ID:
 768                 /* bind descriptor to the chain */
 769                 while (tail->hw_next)
 770                         tail = tail->hw_next;
 771                 xor_last_linked = tail;
 772
 773                 if (prev_desc == xor_last_submit)
 774                         /* do not link to the last submitted CB */
 775                         break;
 776                 ppc440spe_xor_set_link(prev_desc, next_desc);
 777                 break;
 778         }
 779
 780         local_irq_restore(flags);
 781 }
 782
 783 /**
 784  * ppc440spe_desc_get_link - get the address of the descriptor that
 785  * follows this one
 786  */
 787 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
 788                                         struct ppc440spe_adma_chan *chan)
 789 {
 790         if (!desc->hw_next)
 791                 return 0;
 792
 793         return desc->hw_next->phys;
 794 }
 795
 796 /**
 797  * ppc440spe_desc_is_aligned - check alignment
 798  */
 799 static inline int ppc440spe_desc_is_aligned(
 800         struct ppc440spe_adma_desc_slot *desc, int num_slots)
 801 {
 802         return (desc->idx & (num_slots - 1)) ? 0 : 1;
 803 }
 804
 805 /**
 806  * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
 807  * XOR operation
 808  */
 809 static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
 810                         int *slots_per_op)
 811 {
 812         int slot_cnt;
 813
 814         /* each XOR descriptor provides up to 16 source operands */
 815         slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
 816
 817         if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
 818                 return slot_cnt;
 819
 820         printk(KERN_ERR "%s: len %d > max %d !!\n",
 821                 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
 822         BUG();
 823         return slot_cnt;
 824 }
 825
 826 /**
 827  * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
 828  * DMA2 PQ operation
 829  */
 830 static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
 831                 int src_cnt, size_t len)
 832 {
 833         signed long long order = 0;
 834         int state = 0;
 835         int addr_count = 0;
 836         int i;
 837         for (i = 1; i < src_cnt; i++) {
 838                 dma_addr_t cur_addr = srcs[i];
 839                 dma_addr_t old_addr = srcs[i-1];
 840                 switch (state) {
 841                 case 0:
 842                         if (cur_addr == old_addr + len) {
 843                                 /* direct RXOR */
 844                                 order = 1;
 845                                 state = 1;
 846                                 if (i == src_cnt-1)
 847                                         addr_count++;
 848                         } else if (old_addr == cur_addr + len) {
 849                                 /* reverse RXOR */
 850                                 order = -1;
 851                                 state = 1;
 852                                 if (i == src_cnt-1)
 853                                         addr_count++;
 854                         } else {
 855                                 state = 3;
 856                         }
 857                         break;
 858                 case 1:
 859                         if (i == src_cnt-2 || (order == -1
 860                                 && cur_addr != old_addr - len)) {
 861                                 order = 0;
 862                                 state = 0;
 863                                 addr_count++;
 864                         } else if (cur_addr == old_addr + len*order) {
 865                                 state = 2;
 866                                 if (i == src_cnt-1)
 867                                         addr_count++;
 868                         } else if (cur_addr == old_addr + 2*len) {
 869                                 state = 2;
 870                                 if (i == src_cnt-1)
 871                                         addr_count++;
 872                         } else if (cur_addr == old_addr + 3*len) {
 873                                 state = 2;
 874                                 if (i == src_cnt-1)
 875                                         addr_count++;
 876                         } else {
 877                                 order = 0;
 878                                 state = 0;
 879                                 addr_count++;
 880                         }
 881                         break;
 882                 case 2:
 883                         order = 0;
 884                         state = 0;
 885                         addr_count++;
 886                                 break;
 887                 }
 888                 if (state == 3)
 889                         break;
 890         }
 891         if (src_cnt <= 1 || (state != 1 && state != 2)) {
 892                 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
 893                         __func__, src_cnt, state, addr_count, order);
 894                 for (i = 0; i < src_cnt; i++)
 895                         pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
 896                 BUG();
 897         }
 898
 899         return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
 900 }
 901
 902
 903 /******************************************************************************
 904  * ADMA channel low-level routines
 905  ******************************************************************************/
 906
 907 static u32
 908 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
 909 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
 910
 911 /**
 912  * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
 913  */
 914 static void ppc440spe_adma_device_clear_eot_status(
 915                                         struct ppc440spe_adma_chan *chan)
 916 {
 917         struct dma_regs *dma_reg;
 918         struct xor_regs *xor_reg;
 919         u8 *p = chan->device->dma_desc_pool_virt;
 920         struct dma_cdb *cdb;
 921         u32 rv, i;
 922
 923         switch (chan->device->id) {
 924         case PPC440SPE_DMA0_ID:
 925         case PPC440SPE_DMA1_ID:
 926                 /* read FIFO to ack */
 927                 dma_reg = chan->device->dma_reg;
 928                 while ((rv = ioread32(&dma_reg->csfpl))) {
 929                         i = rv & DMA_CDB_ADDR_MSK;
 930                         cdb = (struct dma_cdb *)&p[i -
 931                             (u32)chan->device->dma_desc_pool];
 932
 933                         /* Clear opcode to ack. This is necessary for
 934                          * ZeroSum operations only
 935                          */
 936                         cdb->opc = 0;
 937
 938                         if (test_bit(PPC440SPE_RXOR_RUN,
 939                             &ppc440spe_rxor_state)) {
 940                                 /* probably this is a completed RXOR op,
 941                                  * get pointer to CDB using the fact that
 942                                  * physical and virtual addresses of CDB
 943                                  * in pools have the same offsets
 944                                  */
 945                                 if (le32_to_cpu(cdb->sg1u) &
 946                                     DMA_CUED_XOR_BASE) {
 947                                         /* this is a RXOR */
 948                                         clear_bit(PPC440SPE_RXOR_RUN,
 949                                                   &ppc440spe_rxor_state);
 950                                 }
 951                         }
 952
 953                         if (rv & DMA_CDB_STATUS_MSK) {
 954                                 /* ZeroSum check failed
 955                                  */
 956                                 struct ppc440spe_adma_desc_slot *iter;
 957                                 dma_addr_t phys = rv & ~DMA_CDB_MSK;
 958
 959                                 /*
 960                                  * Update the status of corresponding
 961                                  * descriptor.
 962                                  */
 963                                 list_for_each_entry(iter, &chan->chain,
 964                                     chain_node) {
 965                                         if (iter->phys == phys)
 966                                                 break;
 967                                 }
 968                                 /*
 969                                  * if cannot find the corresponding
 970                                  * slot it's a bug
 971                                  */
 972                                 BUG_ON(&iter->chain_node == &chan->chain);
 973
 974                                 if (iter->xor_check_result) {
 975                                         if (test_bit(PPC440SPE_DESC_PCHECK,
 976                                                      &iter->flags)) {
 977                                                 *iter->xor_check_result |=
 978                                                         SUM_CHECK_P_RESULT;
 979                                         } else
 980                                         if (test_bit(PPC440SPE_DESC_QCHECK,
 981                                                      &iter->flags)) {
 982                                                 *iter->xor_check_result |=
 983                                                         SUM_CHECK_Q_RESULT;
 984                                         } else
 985                                                 BUG();
 986                                 }
 987                         }
 988                 }
 989
 990                 rv = ioread32(&dma_reg->dsts);
 991                 if (rv) {
 992                         pr_err("DMA%d err status: 0x%x\n",
 993                                chan->device->id, rv);
 994                         /* write back to clear */
 995                         iowrite32(rv, &dma_reg->dsts);
 996                 }
 997                 break;
 998         case PPC440SPE_XOR_ID:
 999                 /* reset status bits to ack */
1000                 xor_reg = chan->device->xor_reg;
1001                 rv = ioread32be(&xor_reg->sr);
1002                 iowrite32be(rv, &xor_reg->sr);
1003
1004                 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
1005                         if (rv & XOR_IE_RPTIE_BIT) {
1006                                 /* Read PLB Timeout Error.
1007                                  * Try to resubmit the CB
1008                                  */
1009                                 u32 val = ioread32be(&xor_reg->ccbalr);
1010
1011                                 iowrite32be(val, &xor_reg->cblalr);
1012
1013                                 val = ioread32be(&xor_reg->crsr);
1014                                 iowrite32be(val | XOR_CRSR_XAE_BIT,
1015                                             &xor_reg->crsr);
1016                         } else
1017                                 pr_err("XOR ERR 0x%x status\n", rv);
1018                         break;
1019                 }
1020
1021                 /*  if the XORcore is idle, but there are unprocessed CBs
1022                  * then refetch the s/w chain here
1023                  */
1024                 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1025                     do_xor_refetch)
1026                         ppc440spe_chan_append(chan);
1027                 break;
1028         }
1029 }
1030
1031 /**
1032  * ppc440spe_chan_is_busy - get the channel status
1033  */
1034 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1035 {
1036         struct dma_regs *dma_reg;
1037         struct xor_regs *xor_reg;
1038         int busy = 0;
1039
1040         switch (chan->device->id) {
1041         case PPC440SPE_DMA0_ID:
1042         case PPC440SPE_DMA1_ID:
1043                 dma_reg = chan->device->dma_reg;
1044                 /*  if command FIFO's head and tail pointers are equal and
1045                  * status tail is the same as command, then channel is free
1046                  */
1047                 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1048                     ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1049                         busy = 1;
1050                 break;
1051         case PPC440SPE_XOR_ID:
1052                 /* use the special status bit for the XORcore
1053                  */
1054                 xor_reg = chan->device->xor_reg;
1055                 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1056                 break;
1057         }
1058
1059         return busy;
1060 }
1061
1062 /**
1063  * ppc440spe_chan_set_first_xor_descriptor -  init XORcore chain
1064  */
1065 static void ppc440spe_chan_set_first_xor_descriptor(
1066                                 struct ppc440spe_adma_chan *chan,
1067                                 struct ppc440spe_adma_desc_slot *next_desc)
1068 {
1069         struct xor_regs *xor_reg = chan->device->xor_reg;
1070
1071         if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1072                 printk(KERN_INFO "%s: Warn: XORcore is running "
1073                         "when try to set the first CDB!\n",
1074                         __func__);
1075
1076         xor_last_submit = xor_last_linked = next_desc;
1077
1078         iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1079
1080         iowrite32be(next_desc->phys, &xor_reg->cblalr);
1081         iowrite32be(0, &xor_reg->cblahr);
1082         iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1083                     &xor_reg->cbcr);
1084
1085         chan->hw_chain_inited = 1;
1086 }
1087
1088 /**
1089  * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1090  * called with irqs disabled
1091  */
1092 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1093                 struct ppc440spe_adma_desc_slot *desc)
1094 {
1095         u32 pcdb;
1096         struct dma_regs *dma_reg = chan->device->dma_reg;
1097
1098         pcdb = desc->phys;
1099         if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1100                 pcdb |= DMA_CDB_NO_INT;
1101
1102         chan_last_sub[chan->device->id] = desc;
1103
1104         ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1105
1106         iowrite32(pcdb, &dma_reg->cpfpl);
1107 }
1108
1109 /**
1110  * ppc440spe_chan_append - update the h/w chain in the channel
1111  */
1112 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1113 {
1114         struct xor_regs *xor_reg;
1115         struct ppc440spe_adma_desc_slot *iter;
1116         struct xor_cb *xcb;
1117         u32 cur_desc;
1118         unsigned long flags;
1119
1120         local_irq_save(flags);
1121
1122         switch (chan->device->id) {
1123         case PPC440SPE_DMA0_ID:
1124         case PPC440SPE_DMA1_ID:
1125                 cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1126
1127                 if (likely(cur_desc)) {
1128                         iter = chan_last_sub[chan->device->id];
1129                         BUG_ON(!iter);
1130                 } else {
1131                         /* first peer */
1132                         iter = chan_first_cdb[chan->device->id];
1133                         BUG_ON(!iter);
1134                         ppc440spe_dma_put_desc(chan, iter);
1135                         chan->hw_chain_inited = 1;
1136                 }
1137
1138                 /* is there something new to append */
1139                 if (!iter->hw_next)
1140                         break;
1141
1142                 /* flush descriptors from the s/w queue to fifo */
1143                 list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1144                         ppc440spe_dma_put_desc(chan, iter);
1145                         if (!iter->hw_next)
1146                                 break;
1147                 }
1148                 break;
1149         case PPC440SPE_XOR_ID:
1150                 /* update h/w links and refetch */
1151                 if (!xor_last_submit->hw_next)
1152                         break;
1153
1154                 xor_reg = chan->device->xor_reg;
1155                 /* the last linked CDB has to generate an interrupt
1156                  * that we'd be able to append the next lists to h/w
1157                  * regardless of the XOR engine state at the moment of
1158                  * appending of these next lists
1159                  */
1160                 xcb = xor_last_linked->hw_desc;
1161                 xcb->cbc |= XOR_CBCR_CBCE_BIT;
1162
1163                 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1164                         /* XORcore is idle. Refetch now */
1165                         do_xor_refetch = 0;
1166                         ppc440spe_xor_set_link(xor_last_submit,
1167                                 xor_last_submit->hw_next);
1168
1169                         ADMA_LL_DBG(print_cb_list(chan,
1170                                 xor_last_submit->hw_next));
1171
1172                         xor_last_submit = xor_last_linked;
1173                         iowrite32be(ioread32be(&xor_reg->crsr) |
1174                                     XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1175                                     &xor_reg->crsr);
1176                 } else {
1177                         /* XORcore is running. Refetch later in the handler */
1178                         do_xor_refetch = 1;
1179                 }
1180
1181                 break;
1182         }
1183
1184         local_irq_restore(flags);
1185 }
1186
1187 /**
1188  * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1189  */
1190 static u32
1191 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1192 {
1193         struct dma_regs *dma_reg;
1194         struct xor_regs *xor_reg;
1195
1196         if (unlikely(!chan->hw_chain_inited))
1197                 /* h/w descriptor chain is not initialized yet */
1198                 return 0;
1199
1200         switch (chan->device->id) {
1201         case PPC440SPE_DMA0_ID:
1202         case PPC440SPE_DMA1_ID:
1203                 dma_reg = chan->device->dma_reg;
1204                 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1205         case PPC440SPE_XOR_ID:
1206                 xor_reg = chan->device->xor_reg;
1207                 return ioread32be(&xor_reg->ccbalr);
1208         }
1209         return 0;
1210 }
1211
1212 /**
1213  * ppc440spe_chan_run - enable the channel
1214  */
1215 static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1216 {
1217         struct xor_regs *xor_reg;
1218
1219         switch (chan->device->id) {
1220         case PPC440SPE_DMA0_ID:
1221         case PPC440SPE_DMA1_ID:
1222                 /* DMAs are always enabled, do nothing */
1223                 break;
1224         case PPC440SPE_XOR_ID:
1225                 /* drain write buffer */
1226                 xor_reg = chan->device->xor_reg;
1227
1228                 /* fetch descriptor pointed to in <link> */
1229                 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1230                             &xor_reg->crsr);
1231                 break;
1232         }
1233 }
1234
1235 /******************************************************************************
1236  * ADMA device level
1237  ******************************************************************************/
1238
1239 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1240 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1241
1242 static dma_cookie_t
1243 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1244
1245 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1246                                     dma_addr_t addr, int index);
1247 static void
1248 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1249                                   dma_addr_t addr, int index);
1250
1251 static void
1252 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1253                            dma_addr_t *paddr, unsigned long flags);
1254 static void
1255 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1256                           dma_addr_t addr, int index);
1257 static void
1258 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1259                                unsigned char mult, int index, int dst_pos);
1260 static void
1261 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1262                                    dma_addr_t paddr, dma_addr_t qaddr);
1263
1264 static struct page *ppc440spe_rxor_srcs[32];
1265
1266 /**
1267  * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1268  */
1269 static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1270 {
1271         int i, order = 0, state = 0;
1272         int idx = 0;
1273
1274         if (unlikely(!(src_cnt > 1)))
1275                 return 0;
1276
1277         BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1278
1279         /* Skip holes in the source list before checking */
1280         for (i = 0; i < src_cnt; i++) {
1281                 if (!srcs[i])
1282                         continue;
1283                 ppc440spe_rxor_srcs[idx++] = srcs[i];
1284         }
1285         src_cnt = idx;
1286
1287         for (i = 1; i < src_cnt; i++) {
1288                 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1289                 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1290
1291                 switch (state) {
1292                 case 0:
1293                         if (cur_addr == old_addr + len) {
1294                                 /* direct RXOR */
1295                                 order = 1;
1296                                 state = 1;
1297                         } else if (old_addr == cur_addr + len) {
1298                                 /* reverse RXOR */
1299                                 order = -1;
1300                                 state = 1;
1301                         } else
1302                                 goto out;
1303                         break;
1304                 case 1:
1305                         if ((i == src_cnt - 2) ||
1306                             (order == -1 && cur_addr != old_addr - len)) {
1307                                 order = 0;
1308                                 state = 0;
1309                         } else if ((cur_addr == old_addr + len * order) ||
1310                                    (cur_addr == old_addr + 2 * len) ||
1311                                    (cur_addr == old_addr + 3 * len)) {
1312                                 state = 2;
1313                         } else {
1314                                 order = 0;
1315                                 state = 0;
1316                         }
1317                         break;
1318                 case 2:
1319                         order = 0;
1320                         state = 0;
1321                         break;
1322                 }
1323         }
1324
1325 out:
1326         if (state == 1 || state == 2)
1327                 return 1;
1328
1329         return 0;
1330 }
1331
1332 /**
1333  * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1334  *      the operation given on this channel. It's assumed that 'chan' is
1335  *      capable to process 'cap' type of operation.
1336  * @chan: channel to use
1337  * @cap: type of transaction
1338  * @dst_lst: array of destination pointers
1339  * @dst_cnt: number of destination operands
1340  * @src_lst: array of source pointers
1341  * @src_cnt: number of source operands
1342  * @src_sz: size of each source operand
1343  */
1344 static int ppc440spe_adma_estimate(struct dma_chan *chan,
1345         enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1346         struct page **src_lst, int src_cnt, size_t src_sz)
1347 {
1348         int ef = 1;
1349
1350         if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1351                 /* If RAID-6 capabilities were not activated don't try
1352                  * to use them
1353                  */
1354                 if (unlikely(!ppc440spe_r6_enabled))
1355                         return -1;
1356         }
1357         /*  In the current implementation of ppc440spe ADMA driver it
1358          * makes sense to pick out only pq case, because it may be
1359          * processed:
1360          * (1) either using Biskup method on DMA2;
1361          * (2) or on DMA0/1.
1362          *  Thus we give a favour to (1) if the sources are suitable;
1363          * else let it be processed on one of the DMA0/1 engines.
1364          *  In the sum_product case where destination is also the
1365          * source process it on DMA0/1 only.
1366          */
1367         if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1368
1369                 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1370                         ef = 0; /* sum_product case, process on DMA0/1 */
1371                 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
1372                         ef = 3; /* override (DMA0/1 + idle) */
1373                 else
1374                         ef = 0; /* can't process on DMA2 if !rxor */
1375         }
1376
1377         /* channel idleness increases the priority */
1378         if (likely(ef) &&
1379             !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1380                 ef++;
1381
1382         return ef;
1383 }
1384
1385 struct dma_chan *
1386 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1387         struct page **dst_lst, int dst_cnt, struct page **src_lst,
1388         int src_cnt, size_t src_sz)
1389 {
1390         struct dma_chan *best_chan = NULL;
1391         struct ppc_dma_chan_ref *ref;
1392         int best_rank = -1;
1393
1394         if (unlikely(!src_sz))
1395                 return NULL;
1396         if (src_sz > PAGE_SIZE) {
1397                 /*
1398                  * should a user of the api ever pass > PAGE_SIZE requests
1399                  * we sort out cases where temporary page-sized buffers
1400                  * are used.
1401                  */
1402                 switch (cap) {
1403                 case DMA_PQ:
1404                         if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1405                                 return NULL;
1406                         if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1407                                 return NULL;
1408                         break;
1409                 case DMA_PQ_VAL:
1410                 case DMA_XOR_VAL:
1411                         return NULL;
1412                 default:
1413                         break;
1414                 }
1415         }
1416
1417         list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1418                 if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1419                         int rank;
1420
1421                         rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
1422                                         dst_cnt, src_lst, src_cnt, src_sz);
1423                         if (rank > best_rank) {
1424                                 best_rank = rank;
1425                                 best_chan = ref->chan;
1426                         }
1427                 }
1428         }
1429
1430         return best_chan;
1431 }
1432 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1433
1434 /**
1435  * ppc440spe_get_group_entry - get group entry with index idx
1436  * @tdesc: is the last allocated slot in the group.
1437  */
1438 static struct ppc440spe_adma_desc_slot *
1439 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1440 {
1441         struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1442         int i = 0;
1443
1444         if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1445                 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1446                         __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1447                 BUG();
1448         }
1449
1450         list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1451                 if (i++ == entry_idx)
1452                         break;
1453         }
1454         return iter;
1455 }
1456
1457 /**
1458  * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1459  * @slot: Slot to free
1460  * Caller must hold &ppc440spe_chan->lock while calling this function
1461  */
1462 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1463                                       struct ppc440spe_adma_chan *chan)
1464 {
1465         int stride = slot->slots_per_op;
1466
1467         while (stride--) {
1468                 slot->slots_per_op = 0;
1469                 slot = list_entry(slot->slot_node.next,
1470                                 struct ppc440spe_adma_desc_slot,
1471                                 slot_node);
1472         }
1473 }
1474
1475 /**
1476  * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1477  * upon completion
1478  */
1479 static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1480                 struct ppc440spe_adma_desc_slot *desc,
1481                 struct ppc440spe_adma_chan *chan,
1482                 dma_cookie_t cookie)
1483 {
1484         BUG_ON(desc->async_tx.cookie < 0);
1485         if (desc->async_tx.cookie > 0) {
1486                 cookie = desc->async_tx.cookie;
1487                 desc->async_tx.cookie = 0;
1488
1489                 /* call the callback (must not sleep or submit new
1490                  * operations to this channel)
1491                  */
1492                 if (desc->async_tx.callback)
1493                         desc->async_tx.callback(
1494                                 desc->async_tx.callback_param);
1495
1496                 dma_descriptor_unmap(&desc->async_tx);
1497         }
1498
1499         /* run dependent operations */
1500         dma_run_dependencies(&desc->async_tx);
1501
1502         return cookie;
1503 }
1504
1505 /**
1506  * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1507  */
1508 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1509                 struct ppc440spe_adma_chan *chan)
1510 {
1511         /* the client is allowed to attach dependent operations
1512          * until 'ack' is set
1513          */
1514         if (!async_tx_test_ack(&desc->async_tx))
1515                 return 0;
1516
1517         /* leave the last descriptor in the chain
1518          * so we can append to it
1519          */
1520         if (list_is_last(&desc->chain_node, &chan->chain) ||
1521             desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1522                 return 1;
1523
1524         if (chan->device->id != PPC440SPE_XOR_ID) {
1525                 /* our DMA interrupt handler clears opc field of
1526                  * each processed descriptor. For all types of
1527                  * operations except for ZeroSum we do not actually
1528                  * need ack from the interrupt handler. ZeroSum is a
1529                  * special case since the result of this operation
1530                  * is available from the handler only, so if we see
1531                  * such type of descriptor (which is unprocessed yet)
1532                  * then leave it in chain.
1533                  */
1534                 struct dma_cdb *cdb = desc->hw_desc;
1535                 if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1536                         return 1;
1537         }
1538
1539         dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1540                 desc->phys, desc->idx, desc->slots_per_op);
1541
1542         list_del(&desc->chain_node);
1543         ppc440spe_adma_free_slots(desc, chan);
1544         return 0;
1545 }
1546
1547 /**
1548  * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1549  *      which runs through the channel CDBs list until reach the descriptor
1550  *      currently processed. When routine determines that all CDBs of group
1551  *      are completed then corresponding callbacks (if any) are called and slots
1552  *      are freed.
1553  */
1554 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1555 {
1556         struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1557         dma_cookie_t cookie = 0;
1558         u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1559         int busy = ppc440spe_chan_is_busy(chan);
1560         int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1561
1562         dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1563                 chan->device->id, __func__);
1564
1565         if (!current_desc) {
1566                 /*  There were no transactions yet, so
1567                  * nothing to clean
1568                  */
1569                 return;
1570         }
1571
1572         /* free completed slots from the chain starting with
1573          * the oldest descriptor
1574          */
1575         list_for_each_entry_safe(iter, _iter, &chan->chain,
1576                                         chain_node) {
1577                 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1578                     "busy: %d this_desc: %#llx next_desc: %#x "
1579                     "cur: %#x ack: %d\n",
1580                     iter->async_tx.cookie, iter->idx, busy, iter->phys,
1581                     ppc440spe_desc_get_link(iter, chan), current_desc,
1582                     async_tx_test_ack(&iter->async_tx));
1583                 prefetch(_iter);
1584                 prefetch(&_iter->async_tx);
1585
1586                 /* do not advance past the current descriptor loaded into the
1587                  * hardware channel,subsequent descriptors are either in process
1588                  * or have not been submitted
1589                  */
1590                 if (seen_current)
1591                         break;
1592
1593                 /* stop the search if we reach the current descriptor and the
1594                  * channel is busy, or if it appears that the current descriptor
1595                  * needs to be re-read (i.e. has been appended to)
1596                  */
1597                 if (iter->phys == current_desc) {
1598                         BUG_ON(seen_current++);
1599                         if (busy || ppc440spe_desc_get_link(iter, chan)) {
1600                                 /* not all descriptors of the group have
1601                                  * been completed; exit.
1602                                  */
1603                                 break;
1604                         }
1605                 }
1606
1607                 /* detect the start of a group transaction */
1608                 if (!slot_cnt && !slots_per_op) {
1609                         slot_cnt = iter->slot_cnt;
1610                         slots_per_op = iter->slots_per_op;
1611                         if (slot_cnt <= slots_per_op) {
1612                                 slot_cnt = 0;
1613                                 slots_per_op = 0;
1614                         }
1615                 }
1616
1617                 if (slot_cnt) {
1618                         if (!group_start)
1619                                 group_start = iter;
1620                         slot_cnt -= slots_per_op;
1621                 }
1622
1623                 /* all the members of a group are complete */
1624                 if (slots_per_op != 0 && slot_cnt == 0) {
1625                         struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1626                         int end_of_chain = 0;
1627
1628                         /* clean up the group */
1629                         slot_cnt = group_start->slot_cnt;
1630                         grp_iter = group_start;
1631                         list_for_each_entry_safe_from(grp_iter, _grp_iter,
1632                                 &chan->chain, chain_node) {
1633
1634                                 cookie = ppc440spe_adma_run_tx_complete_actions(
1635                                         grp_iter, chan, cookie);
1636
1637                                 slot_cnt -= slots_per_op;
1638                                 end_of_chain = ppc440spe_adma_clean_slot(
1639                                     grp_iter, chan);
1640                                 if (end_of_chain && slot_cnt) {
1641                                         /* Should wait for ZeroSum completion */
1642                                         if (cookie > 0)
1643                                                 chan->common.completed_cookie = cookie;
1644                                         return;
1645                                 }
1646
1647                                 if (slot_cnt == 0 || end_of_chain)
1648                                         break;
1649                         }
1650
1651                         /* the group should be complete at this point */
1652                         BUG_ON(slot_cnt);
1653
1654                         slots_per_op = 0;
1655                         group_start = NULL;
1656                         if (end_of_chain)
1657                                 break;
1658                         else
1659                                 continue;
1660                 } else if (slots_per_op) /* wait for group completion */
1661                         continue;
1662
1663                 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
1664                     cookie);
1665
1666                 if (ppc440spe_adma_clean_slot(iter, chan))
1667                         break;
1668         }
1669
1670         BUG_ON(!seen_current);
1671
1672         if (cookie > 0) {
1673                 chan->common.completed_cookie = cookie;
1674                 pr_debug("\tcompleted cookie %d\n", cookie);
1675         }
1676
1677 }
1678
1679 /**
1680  * ppc440spe_adma_tasklet - clean up watch-dog initiator
1681  */
1682 static void ppc440spe_adma_tasklet(unsigned long data)
1683 {
1684         struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data;
1685
1686         spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1687         __ppc440spe_adma_slot_cleanup(chan);
1688         spin_unlock(&chan->lock);
1689 }
1690
1691 /**
1692  * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1693  */
1694 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1695 {
1696         spin_lock_bh(&chan->lock);
1697         __ppc440spe_adma_slot_cleanup(chan);
1698         spin_unlock_bh(&chan->lock);
1699 }
1700
1701 /**
1702  * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1703  */
1704 static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1705                 struct ppc440spe_adma_chan *chan, int num_slots,
1706                 int slots_per_op)
1707 {
1708         struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
1709         struct ppc440spe_adma_desc_slot *alloc_start = NULL;
1710         struct list_head chain = LIST_HEAD_INIT(chain);
1711         int slots_found, retry = 0;
1712
1713
1714         BUG_ON(!num_slots || !slots_per_op);
1715         /* start search from the last allocated descrtiptor
1716          * if a contiguous allocation can not be found start searching
1717          * from the beginning of the list
1718          */
1719 retry:
1720         slots_found = 0;
1721         if (retry == 0)
1722                 iter = chan->last_used;
1723         else
1724                 iter = list_entry(&chan->all_slots,
1725                                   struct ppc440spe_adma_desc_slot,
1726                                   slot_node);
1727         list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
1728             slot_node) {
1729                 prefetch(_iter);
1730                 prefetch(&_iter->async_tx);
1731                 if (iter->slots_per_op) {
1732                         slots_found = 0;
1733                         continue;
1734                 }
1735
1736                 /* start the allocation if the slot is correctly aligned */
1737                 if (!slots_found++)
1738                         alloc_start = iter;
1739
1740                 if (slots_found == num_slots) {
1741                         struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
1742                         struct ppc440spe_adma_desc_slot *last_used = NULL;
1743
1744                         iter = alloc_start;
1745                         while (num_slots) {
1746                                 int i;
1747                                 /* pre-ack all but the last descriptor */
1748                                 if (num_slots != slots_per_op)
1749                                         async_tx_ack(&iter->async_tx);
1750
1751                                 list_add_tail(&iter->chain_node, &chain);
1752                                 alloc_tail = iter;
1753                                 iter->async_tx.cookie = 0;
1754                                 iter->hw_next = NULL;
1755                                 iter->flags = 0;
1756                                 iter->slot_cnt = num_slots;
1757                                 iter->xor_check_result = NULL;
1758                                 for (i = 0; i < slots_per_op; i++) {
1759                                         iter->slots_per_op = slots_per_op - i;
1760                                         last_used = iter;
1761                                         iter = list_entry(iter->slot_node.next,
1762                                                 struct ppc440spe_adma_desc_slot,
1763                                                 slot_node);
1764                                 }
1765                                 num_slots -= slots_per_op;
1766                         }
1767                         alloc_tail->group_head = alloc_start;
1768                         alloc_tail->async_tx.cookie = -EBUSY;
1769                         list_splice(&chain, &alloc_tail->group_list);
1770                         chan->last_used = last_used;
1771                         return alloc_tail;
1772                 }
1773         }
1774         if (!retry++)
1775                 goto retry;
1776
1777         /* try to free some slots if the allocation fails */
1778         tasklet_schedule(&chan->irq_tasklet);
1779         return NULL;
1780 }
1781
1782 /**
1783  * ppc440spe_adma_alloc_chan_resources -  allocate pools for CDB slots
1784  */
1785 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
1786 {
1787         struct ppc440spe_adma_chan *ppc440spe_chan;
1788         struct ppc440spe_adma_desc_slot *slot = NULL;
1789         char *hw_desc;
1790         int i, db_sz;
1791         int init;
1792
1793         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1794         init = ppc440spe_chan->slots_allocated ? 0 : 1;
1795         chan->chan_id = ppc440spe_chan->device->id;
1796
1797         /* Allocate descriptor slots */
1798         i = ppc440spe_chan->slots_allocated;
1799         if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
1800                 db_sz = sizeof(struct dma_cdb);
1801         else
1802                 db_sz = sizeof(struct xor_cb);
1803
1804         for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
1805                 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
1806                                GFP_KERNEL);
1807                 if (!slot) {
1808                         printk(KERN_INFO "SPE ADMA Channel only initialized"
1809                                 " %d descriptor slots", i--);
1810                         break;
1811                 }
1812
1813                 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
1814                 slot->hw_desc = (void *) &hw_desc[i * db_sz];
1815                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
1816                 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
1817                 INIT_LIST_HEAD(&slot->chain_node);
1818                 INIT_LIST_HEAD(&slot->slot_node);
1819                 INIT_LIST_HEAD(&slot->group_list);
1820                 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
1821                 slot->idx = i;
1822
1823                 spin_lock_bh(&ppc440spe_chan->lock);
1824                 ppc440spe_chan->slots_allocated++;
1825                 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
1826                 spin_unlock_bh(&ppc440spe_chan->lock);
1827         }
1828
1829         if (i && !ppc440spe_chan->last_used) {
1830                 ppc440spe_chan->last_used =
1831                         list_entry(ppc440spe_chan->all_slots.next,
1832                                 struct ppc440spe_adma_desc_slot,
1833                                 slot_node);
1834         }
1835
1836         dev_dbg(ppc440spe_chan->device->common.dev,
1837                 "ppc440spe adma%d: allocated %d descriptor slots\n",
1838                 ppc440spe_chan->device->id, i);
1839
1840         /* initialize the channel and the chain with a null operation */
1841         if (init) {
1842                 switch (ppc440spe_chan->device->id) {
1843                 case PPC440SPE_DMA0_ID:
1844                 case PPC440SPE_DMA1_ID:
1845                         ppc440spe_chan->hw_chain_inited = 0;
1846                         /* Use WXOR for self-testing */
1847                         if (!ppc440spe_r6_tchan)
1848                                 ppc440spe_r6_tchan = ppc440spe_chan;
1849                         break;
1850                 case PPC440SPE_XOR_ID:
1851                         ppc440spe_chan_start_null_xor(ppc440spe_chan);
1852                         break;
1853                 default:
1854                         BUG();
1855                 }
1856                 ppc440spe_chan->needs_unmap = 1;
1857         }
1858
1859         return (i > 0) ? i : -ENOMEM;
1860 }
1861
1862 /**
1863  * ppc440spe_rxor_set_region_data -
1864  */
1865 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
1866         u8 xor_arg_no, u32 mask)
1867 {
1868         struct xor_cb *xcb = desc->hw_desc;
1869
1870         xcb->ops[xor_arg_no].h |= mask;
1871 }
1872
1873 /**
1874  * ppc440spe_rxor_set_src -
1875  */
1876 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
1877         u8 xor_arg_no, dma_addr_t addr)
1878 {
1879         struct xor_cb *xcb = desc->hw_desc;
1880
1881         xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
1882         xcb->ops[xor_arg_no].l = addr;
1883 }
1884
1885 /**
1886  * ppc440spe_rxor_set_mult -
1887  */
1888 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
1889         u8 xor_arg_no, u8 idx, u8 mult)
1890 {
1891         struct xor_cb *xcb = desc->hw_desc;
1892
1893         xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
1894 }
1895
1896 /**
1897  * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
1898  *      has been achieved
1899  */
1900 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
1901 {
1902         dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
1903                 chan->device->id, chan->pending);
1904
1905         if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
1906                 chan->pending = 0;
1907                 ppc440spe_chan_append(chan);
1908         }
1909 }
1910
1911 /**
1912  * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
1913  *      (it's not necessary that descriptors will be submitted to the h/w
1914  *      chains too right now)
1915  */
1916 static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
1917 {
1918         struct ppc440spe_adma_desc_slot *sw_desc;
1919         struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
1920         struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
1921         int slot_cnt;
1922         int slots_per_op;
1923         dma_cookie_t cookie;
1924
1925         sw_desc = tx_to_ppc440spe_adma_slot(tx);
1926
1927         group_start = sw_desc->group_head;
1928         slot_cnt = group_start->slot_cnt;
1929         slots_per_op = group_start->slots_per_op;
1930
1931         spin_lock_bh(&chan->lock);
1932         cookie = dma_cookie_assign(tx);
1933
1934         if (unlikely(list_empty(&chan->chain))) {
1935                 /* first peer */
1936                 list_splice_init(&sw_desc->group_list, &chan->chain);
1937                 chan_first_cdb[chan->device->id] = group_start;
1938         } else {
1939                 /* isn't first peer, bind CDBs to chain */
1940                 old_chain_tail = list_entry(chan->chain.prev,
1941                                         struct ppc440spe_adma_desc_slot,
1942                                         chain_node);
1943                 list_splice_init(&sw_desc->group_list,
1944                     &old_chain_tail->chain_node);
1945                 /* fix up the hardware chain */
1946                 ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
1947         }
1948
1949         /* increment the pending count by the number of operations */
1950         chan->pending += slot_cnt / slots_per_op;
1951         ppc440spe_adma_check_threshold(chan);
1952         spin_unlock_bh(&chan->lock);
1953
1954         dev_dbg(chan->device->common.dev,
1955                 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
1956                 chan->device->id, __func__,
1957                 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
1958
1959         return cookie;
1960 }
1961
1962 /**
1963  * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
1964  */
1965 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
1966                 struct dma_chan *chan, unsigned long flags)
1967 {
1968         struct ppc440spe_adma_chan *ppc440spe_chan;
1969         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1970         int slot_cnt, slots_per_op;
1971
1972         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1973
1974         dev_dbg(ppc440spe_chan->device->common.dev,
1975                 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
1976                 __func__);
1977
1978         spin_lock_bh(&ppc440spe_chan->lock);
1979         slot_cnt = slots_per_op = 1;
1980         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
1981                         slots_per_op);
1982         if (sw_desc) {
1983                 group_start = sw_desc->group_head;
1984                 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
1985                 group_start->unmap_len = 0;
1986                 sw_desc->async_tx.flags = flags;
1987         }
1988         spin_unlock_bh(&ppc440spe_chan->lock);
1989
1990         return sw_desc ? &sw_desc->async_tx : NULL;
1991 }
1992
1993 /**
1994  * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
1995  */
1996 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
1997                 struct dma_chan *chan, dma_addr_t dma_dest,
1998                 dma_addr_t dma_src, size_t len, unsigned long flags)
1999 {
2000         struct ppc440spe_adma_chan *ppc440spe_chan;
2001         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2002         int slot_cnt, slots_per_op;
2003
2004         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2005
2006         if (unlikely(!len))
2007                 return NULL;
2008
2009         BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT);
2010
2011         spin_lock_bh(&ppc440spe_chan->lock);
2012
2013         dev_dbg(ppc440spe_chan->device->common.dev,
2014                 "ppc440spe adma%d: %s len: %u int_en %d\n",
2015                 ppc440spe_chan->device->id, __func__, len,
2016                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2017         slot_cnt = slots_per_op = 1;
2018         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2019                 slots_per_op);
2020         if (sw_desc) {
2021                 group_start = sw_desc->group_head;
2022                 ppc440spe_desc_init_memcpy(group_start, flags);
2023                 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2024                 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
2025                 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2026                 sw_desc->unmap_len = len;
2027                 sw_desc->async_tx.flags = flags;
2028         }
2029         spin_unlock_bh(&ppc440spe_chan->lock);
2030
2031         return sw_desc ? &sw_desc->async_tx : NULL;
2032 }
2033
2034 /**
2035  * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2036  */
2037 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2038                 struct dma_chan *chan, dma_addr_t dma_dest,
2039                 dma_addr_t *dma_src, u32 src_cnt, size_t len,
2040                 unsigned long flags)
2041 {
2042         struct ppc440spe_adma_chan *ppc440spe_chan;
2043         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2044         int slot_cnt, slots_per_op;
2045
2046         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2047
2048         ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2049                                      dma_dest, dma_src, src_cnt));
2050         if (unlikely(!len))
2051                 return NULL;
2052         BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2053
2054         dev_dbg(ppc440spe_chan->device->common.dev,
2055                 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2056                 ppc440spe_chan->device->id, __func__, src_cnt, len,
2057                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2058
2059         spin_lock_bh(&ppc440spe_chan->lock);
2060         slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
2061         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2062                         slots_per_op);
2063         if (sw_desc) {
2064                 group_start = sw_desc->group_head;
2065                 ppc440spe_desc_init_xor(group_start, src_cnt, flags);
2066                 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2067                 while (src_cnt--)
2068                         ppc440spe_adma_memcpy_xor_set_src(group_start,
2069                                 dma_src[src_cnt], src_cnt);
2070                 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2071                 sw_desc->unmap_len = len;
2072                 sw_desc->async_tx.flags = flags;
2073         }
2074         spin_unlock_bh(&ppc440spe_chan->lock);
2075
2076         return sw_desc ? &sw_desc->async_tx : NULL;
2077 }
2078
2079 static inline void
2080 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2081                                 int src_cnt);
2082 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2083
2084 /**
2085  * ppc440spe_adma_init_dma2rxor_slot -
2086  */
2087 static void ppc440spe_adma_init_dma2rxor_slot(
2088                 struct ppc440spe_adma_desc_slot *desc,
2089                 dma_addr_t *src, int src_cnt)
2090 {
2091         int i;
2092
2093         /* initialize CDB */
2094         for (i = 0; i < src_cnt; i++) {
2095                 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
2096                                                  desc->src_cnt, (u32)src[i]);
2097         }
2098 }
2099
2100 /**
2101  * ppc440spe_dma01_prep_mult -
2102  * for Q operation where destination is also the source
2103  */
2104 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2105                 struct ppc440spe_adma_chan *ppc440spe_chan,
2106                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2107                 const unsigned char *scf, size_t len, unsigned long flags)
2108 {
2109         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2110         unsigned long op = 0;
2111         int slot_cnt;
2112
2113         set_bit(PPC440SPE_DESC_WXOR, &op);
2114         slot_cnt = 2;
2115
2116         spin_lock_bh(&ppc440spe_chan->lock);
2117
2118         /* use WXOR, each descriptor occupies one slot */
2119         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2120         if (sw_desc) {
2121                 struct ppc440spe_adma_chan *chan;
2122                 struct ppc440spe_adma_desc_slot *iter;
2123                 struct dma_cdb *hw_desc;
2124
2125                 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2126                 set_bits(op, &sw_desc->flags);
2127                 sw_desc->src_cnt = src_cnt;
2128                 sw_desc->dst_cnt = dst_cnt;
2129                 /* First descriptor, zero data in the destination and copy it
2130                  * to q page using MULTICAST transfer.
2131                  */
2132                 iter = list_first_entry(&sw_desc->group_list,
2133                                         struct ppc440spe_adma_desc_slot,
2134                                         chain_node);
2135                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2136                 /* set 'next' pointer */
2137                 iter->hw_next = list_entry(iter->chain_node.next,
2138                                            struct ppc440spe_adma_desc_slot,
2139                                            chain_node);
2140                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2141                 hw_desc = iter->hw_desc;
2142                 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2143
2144                 ppc440spe_desc_set_dest_addr(iter, chan,
2145                                              DMA_CUED_XOR_BASE, dst[0], 0);
2146                 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
2147                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2148                                             src[0]);
2149                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2150                 iter->unmap_len = len;
2151
2152                 /*
2153                  * Second descriptor, multiply data from the q page
2154                  * and store the result in real destination.
2155                  */
2156                 iter = list_first_entry(&iter->chain_node,
2157                                         struct ppc440spe_adma_desc_slot,
2158                                         chain_node);
2159                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2160                 iter->hw_next = NULL;
2161                 if (flags & DMA_PREP_INTERRUPT)
2162                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2163                 else
2164                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2165
2166                 hw_desc = iter->hw_desc;
2167                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2168                 ppc440spe_desc_set_src_addr(iter, chan, 0,
2169                                             DMA_CUED_XOR_HB, dst[1]);
2170                 ppc440spe_desc_set_dest_addr(iter, chan,
2171                                              DMA_CUED_XOR_BASE, dst[0], 0);
2172
2173                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2174                                             DMA_CDB_SG_DST1, scf[0]);
2175                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2176                 iter->unmap_len = len;
2177                 sw_desc->async_tx.flags = flags;
2178         }
2179
2180         spin_unlock_bh(&ppc440spe_chan->lock);
2181
2182         return sw_desc;
2183 }
2184
2185 /**
2186  * ppc440spe_dma01_prep_sum_product -
2187  * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2188  * the source.
2189  */
2190 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2191                 struct ppc440spe_adma_chan *ppc440spe_chan,
2192                 dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2193                 const unsigned char *scf, size_t len, unsigned long flags)
2194 {
2195         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2196         unsigned long op = 0;
2197         int slot_cnt;
2198
2199         set_bit(PPC440SPE_DESC_WXOR, &op);
2200         slot_cnt = 3;
2201
2202         spin_lock_bh(&ppc440spe_chan->lock);
2203
2204         /* WXOR, each descriptor occupies one slot */
2205         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2206         if (sw_desc) {
2207                 struct ppc440spe_adma_chan *chan;
2208                 struct ppc440spe_adma_desc_slot *iter;
2209                 struct dma_cdb *hw_desc;
2210
2211                 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2212                 set_bits(op, &sw_desc->flags);
2213                 sw_desc->src_cnt = src_cnt;
2214                 sw_desc->dst_cnt = 1;
2215                 /* 1st descriptor, src[1] data to q page and zero destination */
2216                 iter = list_first_entry(&sw_desc->group_list,
2217                                         struct ppc440spe_adma_desc_slot,
2218                                         chain_node);
2219                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2220                 iter->hw_next = list_entry(iter->chain_node.next,
2221                                            struct ppc440spe_adma_desc_slot,
2222                                            chain_node);
2223                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2224                 hw_desc = iter->hw_desc;
2225                 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2226
2227                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2228                                              *dst, 0);
2229                 ppc440spe_desc_set_dest_addr(iter, chan, 0,
2230                                              ppc440spe_chan->qdest, 1);
2231                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2232                                             src[1]);
2233                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2234                 iter->unmap_len = len;
2235
2236                 /* 2nd descriptor, multiply src[1] data and store the
2237                  * result in destination */
2238                 iter = list_first_entry(&iter->chain_node,
2239                                         struct ppc440spe_adma_desc_slot,
2240                                         chain_node);
2241                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2242                 /* set 'next' pointer */
2243                 iter->hw_next = list_entry(iter->chain_node.next,
2244                                            struct ppc440spe_adma_desc_slot,
2245                                            chain_node);
2246                 if (flags & DMA_PREP_INTERRUPT)
2247                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2248                 else
2249                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2250
2251                 hw_desc = iter->hw_desc;
2252                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2253                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2254                                             ppc440spe_chan->qdest);
2255                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2256                                              *dst, 0);
2257                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2258                                             DMA_CDB_SG_DST1, scf[1]);
2259                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2260                 iter->unmap_len = len;
2261
2262                 /*
2263                  * 3rd descriptor, multiply src[0] data and xor it
2264                  * with destination
2265                  */
2266                 iter = list_first_entry(&iter->chain_node,
2267                                         struct ppc440spe_adma_desc_slot,
2268                                         chain_node);
2269                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2270                 iter->hw_next = NULL;
2271                 if (flags & DMA_PREP_INTERRUPT)
2272                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2273                 else
2274                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2275
2276                 hw_desc = iter->hw_desc;
2277                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2278                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2279                                             src[0]);
2280                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2281                                              *dst, 0);
2282                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2283                                             DMA_CDB_SG_DST1, scf[0]);
2284                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2285                 iter->unmap_len = len;
2286                 sw_desc->async_tx.flags = flags;
2287         }
2288
2289         spin_unlock_bh(&ppc440spe_chan->lock);
2290
2291         return sw_desc;
2292 }
2293
2294 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2295                 struct ppc440spe_adma_chan *ppc440spe_chan,
2296                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2297                 const unsigned char *scf, size_t len, unsigned long flags)
2298 {
2299         int slot_cnt;
2300         struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2301         unsigned long op = 0;
2302         unsigned char mult = 1;
2303
2304         pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2305                  __func__, dst_cnt, src_cnt, len);
2306         /*  select operations WXOR/RXOR depending on the
2307          * source addresses of operators and the number
2308          * of destinations (RXOR support only Q-parity calculations)
2309          */
2310         set_bit(PPC440SPE_DESC_WXOR, &op);
2311         if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
2312                 /* no active RXOR;
2313                  * do RXOR if:
2314                  * - there are more than 1 source,
2315                  * - len is aligned on 512-byte boundary,
2316                  * - source addresses fit to one of 4 possible regions.
2317                  */
2318                 if (src_cnt > 1 &&
2319                     !(len & MQ0_CF2H_RXOR_BS_MASK) &&
2320                     (src[0] + len) == src[1]) {
2321                         /* may do RXOR R1 R2 */
2322                         set_bit(PPC440SPE_DESC_RXOR, &op);
2323                         if (src_cnt != 2) {
2324                                 /* may try to enhance region of RXOR */
2325                                 if ((src[1] + len) == src[2]) {
2326                                         /* do RXOR R1 R2 R3 */
2327                                         set_bit(PPC440SPE_DESC_RXOR123,
2328                                                 &op);
2329                                 } else if ((src[1] + len * 2) == src[2]) {
2330                                         /* do RXOR R1 R2 R4 */
2331                                         set_bit(PPC440SPE_DESC_RXOR124, &op);
2332                                 } else if ((src[1] + len * 3) == src[2]) {
2333                                         /* do RXOR R1 R2 R5 */
2334                                         set_bit(PPC440SPE_DESC_RXOR125,
2335                                                 &op);
2336                                 } else {
2337                                         /* do RXOR R1 R2 */
2338                                         set_bit(PPC440SPE_DESC_RXOR12,
2339                                                 &op);
2340                                 }
2341                         } else {
2342                                 /* do RXOR R1 R2 */
2343                                 set_bit(PPC440SPE_DESC_RXOR12, &op);
2344                         }
2345                 }
2346
2347                 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2348                         /* can not do this operation with RXOR */
2349                         clear_bit(PPC440SPE_RXOR_RUN,
2350                                 &ppc440spe_rxor_state);
2351                 } else {
2352                         /* can do; set block size right now */
2353                         ppc440spe_desc_set_rxor_block_size(len);
2354                 }
2355         }
2356
2357         /* Number of necessary slots depends on operation type selected */
2358         if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2359                 /*  This is a WXOR only chain. Need descriptors for each
2360                  * source to GF-XOR them with WXOR, and need descriptors
2361                  * for each destination to zero them with WXOR
2362                  */
2363                 slot_cnt = src_cnt;
2364
2365                 if (flags & DMA_PREP_ZERO_P) {
2366                         slot_cnt++;
2367                         set_bit(PPC440SPE_ZERO_P, &op);
2368                 }
2369                 if (flags & DMA_PREP_ZERO_Q) {
2370                         slot_cnt++;
2371                         set_bit(PPC440SPE_ZERO_Q, &op);
2372                 }
2373         } else {
2374                 /*  Need 1/2 descriptor for RXOR operation, and
2375                  * need (src_cnt - (2 or 3)) for WXOR of sources
2376                  * remained (if any)
2377                  */
2378                 slot_cnt = dst_cnt;
2379
2380                 if (flags & DMA_PREP_ZERO_P)
2381                         set_bit(PPC440SPE_ZERO_P, &op);
2382                 if (flags & DMA_PREP_ZERO_Q)
2383                         set_bit(PPC440SPE_ZERO_Q, &op);
2384
2385                 if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2386                         slot_cnt += src_cnt - 2;
2387                 else
2388                         slot_cnt += src_cnt - 3;
2389
2390                 /*  Thus we have either RXOR only chain or
2391                  * mixed RXOR/WXOR
2392                  */
2393                 if (slot_cnt == dst_cnt)
2394                         /* RXOR only chain */
2395                         clear_bit(PPC440SPE_DESC_WXOR, &op);
2396         }
2397
2398         spin_lock_bh(&ppc440spe_chan->lock);
2399         /* for both RXOR/WXOR each descriptor occupies one slot */
2400         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2401         if (sw_desc) {
2402                 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2403                                 flags, op);
2404
2405                 /* setup dst/src/mult */
2406                 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2407                          __func__, dst[0], dst[1]);
2408                 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2409                 while (src_cnt--) {
2410                         ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2411                                                   src_cnt);
2412
2413                         /* NOTE: "Multi = 0 is equivalent to = 1" as it
2414                          * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2415                          * doesn't work for RXOR with DMA0/1! Instead, multi=0
2416                          * leads to zeroing source data after RXOR.
2417                          * So, for P case set-up mult=1 explicitly.
2418                          */
2419                         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2420                                 mult = scf[src_cnt];
2421                         ppc440spe_adma_pq_set_src_mult(sw_desc,
2422                                 mult, src_cnt,  dst_cnt - 1);
2423                 }
2424
2425                 /* Setup byte count foreach slot just allocated */
2426                 sw_desc->async_tx.flags = flags;
2427                 list_for_each_entry(iter, &sw_desc->group_list,
2428                                 chain_node) {
2429                         ppc440spe_desc_set_byte_count(iter,
2430                                 ppc440spe_chan, len);
2431                         iter->unmap_len = len;
2432                 }
2433         }
2434         spin_unlock_bh(&ppc440spe_chan->lock);
2435
2436         return sw_desc;
2437 }
2438
2439 static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2440                 struct ppc440spe_adma_chan *ppc440spe_chan,
2441                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2442                 const unsigned char *scf, size_t len, unsigned long flags)
2443 {
2444         int slot_cnt, descs_per_op;
2445         struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2446         unsigned long op = 0;
2447         unsigned char mult = 1;
2448
2449         BUG_ON(!dst_cnt);
2450         /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2451                  __func__, dst_cnt, src_cnt, len);*/
2452
2453         spin_lock_bh(&ppc440spe_chan->lock);
2454         descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
2455         if (descs_per_op < 0) {
2456                 spin_unlock_bh(&ppc440spe_chan->lock);
2457                 return NULL;
2458         }
2459
2460         /* depending on number of sources we have 1 or 2 RXOR chains */
2461         slot_cnt = descs_per_op * dst_cnt;
2462
2463         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2464         if (sw_desc) {
2465                 op = slot_cnt;
2466                 sw_desc->async_tx.flags = flags;
2467                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2468                         ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
2469                                 --op ? 0 : flags);
2470                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2471                                 len);
2472                         iter->unmap_len = len;
2473
2474                         ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
2475                         iter->rxor_cursor.len = len;
2476                         iter->descs_per_op = descs_per_op;
2477                 }
2478                 op = 0;
2479                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2480                         op++;
2481                         if (op % descs_per_op == 0)
2482                                 ppc440spe_adma_init_dma2rxor_slot(iter, src,
2483                                                                   src_cnt);
2484                         if (likely(!list_is_last(&iter->chain_node,
2485                                                  &sw_desc->group_list))) {
2486                                 /* set 'next' pointer */
2487                                 iter->hw_next =
2488                                         list_entry(iter->chain_node.next,
2489                                                 struct ppc440spe_adma_desc_slot,
2490                                                 chain_node);
2491                                 ppc440spe_xor_set_link(iter, iter->hw_next);
2492                         } else {
2493                                 /* this is the last descriptor. */
2494                                 iter->hw_next = NULL;
2495                         }
2496                 }
2497
2498                 /* fixup head descriptor */
2499                 sw_desc->dst_cnt = dst_cnt;
2500                 if (flags & DMA_PREP_ZERO_P)
2501                         set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2502                 if (flags & DMA_PREP_ZERO_Q)
2503                         set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2504
2505                 /* setup dst/src/mult */
2506                 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2507
2508                 while (src_cnt--) {
2509                         /* handle descriptors (if dst_cnt == 2) inside
2510                          * the ppc440spe_adma_pq_set_srcxxx() functions
2511                          */
2512                         ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2513                                                   src_cnt);
2514                         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2515                                 mult = scf[src_cnt];
2516                         ppc440spe_adma_pq_set_src_mult(sw_desc,
2517                                         mult, src_cnt, dst_cnt - 1);
2518                 }
2519         }
2520         spin_unlock_bh(&ppc440spe_chan->lock);
2521         ppc440spe_desc_set_rxor_block_size(len);
2522         return sw_desc;
2523 }
2524
2525 /**
2526  * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2527  */
2528 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2529                 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2530                 unsigned int src_cnt, const unsigned char *scf,
2531                 size_t len, unsigned long flags)
2532 {
2533         struct ppc440spe_adma_chan *ppc440spe_chan;
2534         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2535         int dst_cnt = 0;
2536
2537         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2538
2539         ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2540                                     dst, src, src_cnt));
2541         BUG_ON(!len);
2542         BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2543         BUG_ON(!src_cnt);
2544
2545         if (src_cnt == 1 && dst[1] == src[0]) {
2546                 dma_addr_t dest[2];
2547
2548                 /* dst[1] is real destination (Q) */
2549                 dest[0] = dst[1];
2550                 /* this is the page to multicast source data to */
2551                 dest[1] = ppc440spe_chan->qdest;
2552                 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2553                                 dest, 2, src, src_cnt, scf, len, flags);
2554                 return sw_desc ? &sw_desc->async_tx : NULL;
2555         }
2556
2557         if (src_cnt == 2 && dst[1] == src[1]) {
2558                 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2559                                         &dst[1], src, 2, scf, len, flags);
2560                 return sw_desc ? &sw_desc->async_tx : NULL;
2561         }
2562
2563         if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2564                 BUG_ON(!dst[0]);
2565                 dst_cnt++;
2566                 flags |= DMA_PREP_ZERO_P;
2567         }
2568
2569         if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2570                 BUG_ON(!dst[1]);
2571                 dst_cnt++;
2572                 flags |= DMA_PREP_ZERO_Q;
2573         }
2574
2575         BUG_ON(!dst_cnt);
2576
2577         dev_dbg(ppc440spe_chan->device->common.dev,
2578                 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2579                 ppc440spe_chan->device->id, __func__, src_cnt, len,
2580                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2581
2582         switch (ppc440spe_chan->device->id) {
2583         case PPC440SPE_DMA0_ID:
2584         case PPC440SPE_DMA1_ID:
2585                 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2586                                 dst, dst_cnt, src, src_cnt, scf,
2587                                 len, flags);
2588                 break;
2589
2590         case PPC440SPE_XOR_ID:
2591                 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2592                                 dst, dst_cnt, src, src_cnt, scf,
2593                                 len, flags);
2594                 break;
2595         }
2596
2597         return sw_desc ? &sw_desc->async_tx : NULL;
2598 }
2599
2600 /**
2601  * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2602  * a PQ_ZERO_SUM operation
2603  */
2604 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2605                 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2606                 unsigned int src_cnt, const unsigned char *scf, size_t len,
2607                 enum sum_check_flags *pqres, unsigned long flags)
2608 {
2609         struct ppc440spe_adma_chan *ppc440spe_chan;
2610         struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2611         dma_addr_t pdest, qdest;
2612         int slot_cnt, slots_per_op, idst, dst_cnt;
2613
2614         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2615
2616         if (flags & DMA_PREP_PQ_DISABLE_P)
2617                 pdest = 0;
2618         else
2619                 pdest = pq[0];
2620
2621         if (flags & DMA_PREP_PQ_DISABLE_Q)
2622                 qdest = 0;
2623         else
2624                 qdest = pq[1];
2625
2626         ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2627                                             src, src_cnt, scf));
2628
2629         /* Always use WXOR for P/Q calculations (two destinations).
2630          * Need 1 or 2 extra slots to verify results are zero.
2631          */
2632         idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2633
2634         /* One additional slot per destination to clone P/Q
2635          * before calculation (we have to preserve destinations).
2636          */
2637         slot_cnt = src_cnt + dst_cnt * 2;
2638         slots_per_op = 1;
2639
2640         spin_lock_bh(&ppc440spe_chan->lock);
2641         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2642                                              slots_per_op);
2643         if (sw_desc) {
2644                 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2645
2646                 /* Setup byte count for each slot just allocated */
2647                 sw_desc->async_tx.flags = flags;
2648                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2649                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2650                                                       len);
2651                         iter->unmap_len = len;
2652                 }
2653
2654                 if (pdest) {
2655                         struct dma_cdb *hw_desc;
2656                         struct ppc440spe_adma_chan *chan;
2657
2658                         iter = sw_desc->group_head;
2659                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2660                         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2661                         iter->hw_next = list_entry(iter->chain_node.next,
2662                                                 struct ppc440spe_adma_desc_slot,
2663                                                 chain_node);
2664                         hw_desc = iter->hw_desc;
2665                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2666                         iter->src_cnt = 0;
2667                         iter->dst_cnt = 0;
2668                         ppc440spe_desc_set_dest_addr(iter, chan, 0,
2669                                                      ppc440spe_chan->pdest, 0);
2670                         ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
2671                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2672                                                       len);
2673                         iter->unmap_len = 0;
2674                         /* override pdest to preserve original P */
2675                         pdest = ppc440spe_chan->pdest;
2676                 }
2677                 if (qdest) {
2678                         struct dma_cdb *hw_desc;
2679                         struct ppc440spe_adma_chan *chan;
2680
2681                         iter = list_first_entry(&sw_desc->group_list,
2682                                                 struct ppc440spe_adma_desc_slot,
2683                                                 chain_node);
2684                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2685
2686                         if (pdest) {
2687                                 iter = list_entry(iter->chain_node.next,
2688                                                 struct ppc440spe_adma_desc_slot,
2689                                                 chain_node);
2690                         }
2691
2692                         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2693                         iter->hw_next = list_entry(iter->chain_node.next,
2694                                                 struct ppc440spe_adma_desc_slot,
2695                                                 chain_node);
2696                         hw_desc = iter->hw_desc;
2697                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2698                         iter->src_cnt = 0;
2699                         iter->dst_cnt = 0;
2700                         ppc440spe_desc_set_dest_addr(iter, chan, 0,
2701                                                      ppc440spe_chan->qdest, 0);
2702                         ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
2703                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2704                                                       len);
2705                         iter->unmap_len = 0;
2706                         /* override qdest to preserve original Q */
2707                         qdest = ppc440spe_chan->qdest;
2708                 }
2709
2710                 /* Setup destinations for P/Q ops */
2711                 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
2712
2713                 /* Setup zero QWORDs into DCHECK CDBs */
2714                 idst = dst_cnt;
2715                 list_for_each_entry_reverse(iter, &sw_desc->group_list,
2716                                             chain_node) {
2717                         /*
2718                          * The last CDB corresponds to Q-parity check,
2719                          * the one before last CDB corresponds
2720                          * P-parity check
2721                          */
2722                         if (idst == DMA_DEST_MAX_NUM) {
2723                                 if (idst == dst_cnt) {
2724                                         set_bit(PPC440SPE_DESC_QCHECK,
2725                                                 &iter->flags);
2726                                 } else {
2727                                         set_bit(PPC440SPE_DESC_PCHECK,
2728                                                 &iter->flags);
2729                                 }
2730                         } else {
2731                                 if (qdest) {
2732                                         set_bit(PPC440SPE_DESC_QCHECK,
2733                                                 &iter->flags);
2734                                 } else {
2735                                         set_bit(PPC440SPE_DESC_PCHECK,
2736                                                 &iter->flags);
2737                                 }
2738                         }
2739                         iter->xor_check_result = pqres;
2740
2741                         /*
2742                          * set it to zero, if check fail then result will
2743                          * be updated
2744                          */
2745                         *iter->xor_check_result = 0;
2746                         ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
2747                                 ppc440spe_qword);
2748
2749                         if (!(--dst_cnt))
2750                                 break;
2751                 }
2752
2753                 /* Setup sources and mults for P/Q ops */
2754                 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
2755                                                      chain_node) {
2756                         struct ppc440spe_adma_chan *chan;
2757                         u32 mult_dst;
2758
2759                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2760                         ppc440spe_desc_set_src_addr(iter, chan, 0,
2761                                                     DMA_CUED_XOR_HB,
2762                                                     src[src_cnt - 1]);
2763                         if (qdest) {
2764                                 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
2765                                                            DMA_CDB_SG_DST1;
2766                                 ppc440spe_desc_set_src_mult(iter, chan,
2767                                                             DMA_CUED_MULT1_OFF,
2768                                                             mult_dst,
2769                                                             scf[src_cnt - 1]);
2770                         }
2771                         if (!(--src_cnt))
2772                                 break;
2773                 }
2774         }
2775         spin_unlock_bh(&ppc440spe_chan->lock);
2776         return sw_desc ? &sw_desc->async_tx : NULL;
2777 }
2778
2779 /**
2780  * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
2781  * XOR ZERO_SUM operation
2782  */
2783 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
2784                 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
2785                 size_t len, enum sum_check_flags *result, unsigned long flags)
2786 {
2787         struct dma_async_tx_descriptor *tx;
2788         dma_addr_t pq[2];
2789
2790         /* validate P, disable Q */
2791         pq[0] = src[0];
2792         pq[1] = 0;
2793         flags |= DMA_PREP_PQ_DISABLE_Q;
2794
2795         tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
2796                                                 src_cnt - 1, 0, len,
2797                                                 result, flags);
2798         return tx;
2799 }
2800
2801 /**
2802  * ppc440spe_adma_set_dest - set destination address into descriptor
2803  */
2804 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2805                 dma_addr_t addr, int index)
2806 {
2807         struct ppc440spe_adma_chan *chan;
2808
2809         BUG_ON(index >= sw_desc->dst_cnt);
2810
2811         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2812
2813         switch (chan->device->id) {
2814         case PPC440SPE_DMA0_ID:
2815         case PPC440SPE_DMA1_ID:
2816                 /* to do: support transfers lengths >
2817                  * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
2818                  */
2819                 ppc440spe_desc_set_dest_addr(sw_desc->group_head,
2820                         chan, 0, addr, index);
2821                 break;
2822         case PPC440SPE_XOR_ID:
2823                 sw_desc = ppc440spe_get_group_entry(sw_desc, index);
2824                 ppc440spe_desc_set_dest_addr(sw_desc,
2825                         chan, 0, addr, index);
2826                 break;
2827         }
2828 }
2829
2830 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
2831                 struct ppc440spe_adma_chan *chan, dma_addr_t addr)
2832 {
2833         /*  To clear destinations update the descriptor
2834          * (P or Q depending on index) as follows:
2835          * addr is destination (0 corresponds to SG2):
2836          */
2837         ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
2838
2839         /* ... and the addr is source: */
2840         ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
2841
2842         /* addr is always SG2 then the mult is always DST1 */
2843         ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2844                                     DMA_CDB_SG_DST1, 1);
2845 }
2846
2847 /**
2848  * ppc440spe_adma_pq_set_dest - set destination address into descriptor
2849  * for the PQXOR operation
2850  */
2851 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2852                 dma_addr_t *addrs, unsigned long flags)
2853 {
2854         struct ppc440spe_adma_desc_slot *iter;
2855         struct ppc440spe_adma_chan *chan;
2856         dma_addr_t paddr, qaddr;
2857         dma_addr_t addr = 0, ppath, qpath;
2858         int index = 0, i;
2859
2860         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2861
2862         if (flags & DMA_PREP_PQ_DISABLE_P)
2863                 paddr = 0;
2864         else
2865                 paddr = addrs[0];
2866
2867         if (flags & DMA_PREP_PQ_DISABLE_Q)
2868                 qaddr = 0;
2869         else
2870                 qaddr = addrs[1];
2871
2872         if (!paddr || !qaddr)
2873                 addr = paddr ? paddr : qaddr;
2874
2875         switch (chan->device->id) {
2876         case PPC440SPE_DMA0_ID:
2877         case PPC440SPE_DMA1_ID:
2878                 /* walk through the WXOR source list and set P/Q-destinations
2879                  * for each slot:
2880                  */
2881                 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
2882                         /* This is WXOR-only chain; may have 1/2 zero descs */
2883                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
2884                                 index++;
2885                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
2886                                 index++;
2887
2888                         iter = ppc440spe_get_group_entry(sw_desc, index);
2889                         if (addr) {
2890                                 /* one destination */
2891                                 list_for_each_entry_from(iter,
2892                                         &sw_desc->group_list, chain_node)
2893                                         ppc440spe_desc_set_dest_addr(iter, chan,
2894                                                 DMA_CUED_XOR_BASE, addr, 0);
2895                         } else {
2896                                 /* two destinations */
2897                                 list_for_each_entry_from(iter,
2898                                         &sw_desc->group_list, chain_node) {
2899                                         ppc440spe_desc_set_dest_addr(iter, chan,
2900                                                 DMA_CUED_XOR_BASE, paddr, 0);
2901                                         ppc440spe_desc_set_dest_addr(iter, chan,
2902                                                 DMA_CUED_XOR_BASE, qaddr, 1);
2903                                 }
2904                         }
2905
2906                         if (index) {
2907                                 /*  To clear destinations update the descriptor
2908                                  * (1st,2nd, or both depending on flags)
2909                                  */
2910                                 index = 0;
2911                                 if (test_bit(PPC440SPE_ZERO_P,
2912                                                 &sw_desc->flags)) {
2913                                         iter = ppc440spe_get_group_entry(
2914                                                         sw_desc, index++);
2915                                         ppc440spe_adma_pq_zero_op(iter, chan,
2916                                                         paddr);
2917                                 }
2918
2919                                 if (test_bit(PPC440SPE_ZERO_Q,
2920                                                 &sw_desc->flags)) {
2921                                         iter = ppc440spe_get_group_entry(
2922                                                         sw_desc, index++);
2923                                         ppc440spe_adma_pq_zero_op(iter, chan,
2924                                                         qaddr);
2925                                 }
2926
2927                                 return;
2928                         }
2929                 } else {
2930                         /* This is RXOR-only or RXOR/WXOR mixed chain */
2931
2932                         /* If we want to include destination into calculations,
2933                          * then make dest addresses cued with mult=1 (XOR).
2934                          */
2935                         ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2936                                         DMA_CUED_XOR_HB :
2937                                         DMA_CUED_XOR_BASE |
2938                                                 (1 << DMA_CUED_MULT1_OFF);
2939                         qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2940                                         DMA_CUED_XOR_HB :
2941                                         DMA_CUED_XOR_BASE |
2942                                                 (1 << DMA_CUED_MULT1_OFF);
2943
2944                         /* Setup destination(s) in RXOR slot(s) */
2945                         iter = ppc440spe_get_group_entry(sw_desc, index++);
2946                         ppc440spe_desc_set_dest_addr(iter, chan,
2947                                                 paddr ? ppath : qpath,
2948                                                 paddr ? paddr : qaddr, 0);
2949                         if (!addr) {
2950                                 /* two destinations */
2951                                 iter = ppc440spe_get_group_entry(sw_desc,
2952                                                                  index++);
2953                                 ppc440spe_desc_set_dest_addr(iter, chan,
2954                                                 qpath, qaddr, 0);
2955                         }
2956
2957                         if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
2958                                 /* Setup destination(s) in remaining WXOR
2959                                  * slots
2960                                  */
2961                                 iter = ppc440spe_get_group_entry(sw_desc,
2962                                                                  index);
2963                                 if (addr) {
2964                                         /* one destination */
2965                                         list_for_each_entry_from(iter,
2966                                             &sw_desc->group_list,
2967                                             chain_node)
2968                                                 ppc440spe_desc_set_dest_addr(
2969                                                         iter, chan,
2970                                                         DMA_CUED_XOR_BASE,
2971                                                         addr, 0);
2972
2973                                 } else {
2974                                         /* two destinations */
2975                                         list_for_each_entry_from(iter,
2976                                             &sw_desc->group_list,
2977                                             chain_node) {
2978                                                 ppc440spe_desc_set_dest_addr(
2979                                                         iter, chan,
2980                                                         DMA_CUED_XOR_BASE,
2981                                                         paddr, 0);
2982                                                 ppc440spe_desc_set_dest_addr(
2983                                                         iter, chan,
2984                                                         DMA_CUED_XOR_BASE,
2985                                                         qaddr, 1);
2986                                         }
2987                                 }
2988                         }
2989
2990                 }
2991                 break;
2992
2993         case PPC440SPE_XOR_ID:
2994                 /* DMA2 descriptors have only 1 destination, so there are
2995                  * two chains - one for each dest.
2996                  * If we want to include destination into calculations,
2997                  * then make dest addresses cued with mult=1 (XOR).
2998                  */
2999                 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
3000                                 DMA_CUED_XOR_HB :
3001                                 DMA_CUED_XOR_BASE |
3002                                         (1 << DMA_CUED_MULT1_OFF);
3003
3004                 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
3005                                 DMA_CUED_XOR_HB :
3006                                 DMA_CUED_XOR_BASE |
3007                                         (1 << DMA_CUED_MULT1_OFF);
3008
3009                 iter = ppc440spe_get_group_entry(sw_desc, 0);
3010                 for (i = 0; i < sw_desc->descs_per_op; i++) {
3011                         ppc440spe_desc_set_dest_addr(iter, chan,
3012                                 paddr ? ppath : qpath,
3013                                 paddr ? paddr : qaddr, 0);
3014                         iter = list_entry(iter->chain_node.next,
3015                                           struct ppc440spe_adma_desc_slot,
3016                                           chain_node);
3017                 }
3018
3019                 if (!addr) {
3020                         /* Two destinations; setup Q here */
3021                         iter = ppc440spe_get_group_entry(sw_desc,
3022                                 sw_desc->descs_per_op);
3023                         for (i = 0; i < sw_desc->descs_per_op; i++) {
3024                                 ppc440spe_desc_set_dest_addr(iter,
3025                                         chan, qpath, qaddr, 0);
3026                                 iter = list_entry(iter->chain_node.next,
3027                                                 struct ppc440spe_adma_desc_slot,
3028                                                 chain_node);
3029                         }
3030                 }
3031
3032                 break;
3033         }
3034 }
3035
3036 /**
3037  * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3038  * for the PQ_ZERO_SUM operation
3039  */
3040 static void ppc440spe_adma_pqzero_sum_set_dest(
3041                 struct ppc440spe_adma_desc_slot *sw_desc,
3042                 dma_addr_t paddr, dma_addr_t qaddr)
3043 {
3044         struct ppc440spe_adma_desc_slot *iter, *end;
3045         struct ppc440spe_adma_chan *chan;
3046         dma_addr_t addr = 0;
3047         int idx;
3048
3049         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3050
3051         /* walk through the WXOR source list and set P/Q-destinations
3052          * for each slot
3053          */
3054         idx = (paddr && qaddr) ? 2 : 1;
3055         /* set end */
3056         list_for_each_entry_reverse(end, &sw_desc->group_list,
3057                                     chain_node) {
3058                 if (!(--idx))
3059                         break;
3060         }
3061         /* set start */
3062         idx = (paddr && qaddr) ? 2 : 1;
3063         iter = ppc440spe_get_group_entry(sw_desc, idx);
3064
3065         if (paddr && qaddr) {
3066                 /* two destinations */
3067                 list_for_each_entry_from(iter, &sw_desc->group_list,
3068                                          chain_node) {
3069                         if (unlikely(iter == end))
3070                                 break;
3071                         ppc440spe_desc_set_dest_addr(iter, chan,
3072                                                 DMA_CUED_XOR_BASE, paddr, 0);
3073                         ppc440spe_desc_set_dest_addr(iter, chan,
3074                                                 DMA_CUED_XOR_BASE, qaddr, 1);
3075                 }
3076         } else {
3077                 /* one destination */
3078                 addr = paddr ? paddr : qaddr;
3079                 list_for_each_entry_from(iter, &sw_desc->group_list,
3080                                          chain_node) {
3081                         if (unlikely(iter == end))
3082                                 break;
3083                         ppc440spe_desc_set_dest_addr(iter, chan,
3084                                                 DMA_CUED_XOR_BASE, addr, 0);
3085                 }
3086         }
3087
3088         /*  The remaining descriptors are DATACHECK. These have no need in
3089          * destination. Actually, these destinations are used there
3090          * as sources for check operation. So, set addr as source.
3091          */
3092         ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
3093
3094         if (!addr) {
3095                 end = list_entry(end->chain_node.next,
3096                                  struct ppc440spe_adma_desc_slot, chain_node);
3097                 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
3098         }
3099 }
3100
3101 /**
3102  * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3103  */
3104 static inline void ppc440spe_desc_set_xor_src_cnt(
3105                         struct ppc440spe_adma_desc_slot *desc,
3106                         int src_cnt)
3107 {
3108         struct xor_cb *hw_desc = desc->hw_desc;
3109
3110         hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3111         hw_desc->cbc |= src_cnt;
3112 }
3113
3114 /**
3115  * ppc440spe_adma_pq_set_src - set source address into descriptor
3116  */
3117 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3118                 dma_addr_t addr, int index)
3119 {
3120         struct ppc440spe_adma_chan *chan;
3121         dma_addr_t haddr = 0;
3122         struct ppc440spe_adma_desc_slot *iter = NULL;
3123
3124         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3125
3126         switch (chan->device->id) {
3127         case PPC440SPE_DMA0_ID:
3128         case PPC440SPE_DMA1_ID:
3129                 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3130                  */
3131                 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3132                         /* RXOR-only or RXOR/WXOR operation */
3133                         int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3134                                 &sw_desc->flags) ?  2 : 3;
3135
3136                         if (index == 0) {
3137                                 /* 1st slot (RXOR) */
3138                                 /* setup sources region (R1-2-3, R1-2-4,
3139                                  * or R1-2-5)
3140                                  */
3141                                 if (test_bit(PPC440SPE_DESC_RXOR12,
3142                                                 &sw_desc->flags))
3143                                         haddr = DMA_RXOR12 <<
3144                                                 DMA_CUED_REGION_OFF;
3145                                 else if (test_bit(PPC440SPE_DESC_RXOR123,
3146                                     &sw_desc->flags))
3147                                         haddr = DMA_RXOR123 <<
3148                                                 DMA_CUED_REGION_OFF;
3149                                 else if (test_bit(PPC440SPE_DESC_RXOR124,
3150                                     &sw_desc->flags))
3151                                         haddr = DMA_RXOR124 <<
3152                                                 DMA_CUED_REGION_OFF;
3153                                 else if (test_bit(PPC440SPE_DESC_RXOR125,
3154                                     &sw_desc->flags))
3155                                         haddr = DMA_RXOR125 <<
3156                                                 DMA_CUED_REGION_OFF;
3157                                 else
3158                                         BUG();
3159                                 haddr |= DMA_CUED_XOR_BASE;
3160                                 iter = ppc440spe_get_group_entry(sw_desc, 0);
3161                         } else if (index < iskip) {
3162                                 /* 1st slot (RXOR)
3163                                  * shall actually set source address only once
3164                                  * instead of first <iskip>
3165                                  */
3166                                 iter = NULL;
3167                         } else {
3168                                 /* 2nd/3d and next slots (WXOR);
3169                                  * skip first slot with RXOR
3170                                  */
3171                                 haddr = DMA_CUED_XOR_HB;
3172                                 iter = ppc440spe_get_group_entry(sw_desc,
3173                                     index - iskip + sw_desc->dst_cnt);
3174                         }
3175                 } else {
3176                         int znum = 0;
3177
3178                         /* WXOR-only operation; skip first slots with
3179                          * zeroing destinations
3180                          */
3181                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3182                                 znum++;
3183                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3184                                 znum++;
3185
3186                         haddr = DMA_CUED_XOR_HB;
3187                         iter = ppc440spe_get_group_entry(sw_desc,
3188                                         index + znum);
3189                 }
3190
3191                 if (likely(iter)) {
3192                         ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
3193
3194                         if (!index &&
3195                             test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3196                             sw_desc->dst_cnt == 2) {
3197                                 /* if we have two destinations for RXOR, then
3198                                  * setup source in the second descr too
3199                                  */
3200                                 iter = ppc440spe_get_group_entry(sw_desc, 1);
3201                                 ppc440spe_desc_set_src_addr(iter, chan, 0,
3202                                         haddr, addr);
3203                         }
3204                 }
3205                 break;
3206
3207         case PPC440SPE_XOR_ID:
3208                 /* DMA2 may do Biskup */
3209                 iter = sw_desc->group_head;
3210                 if (iter->dst_cnt == 2) {
3211                         /* both P & Q calculations required; set P src here */
3212                         ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3213
3214                         /* this is for Q */
3215                         iter = ppc440spe_get_group_entry(sw_desc,
3216                                 sw_desc->descs_per_op);
3217                 }
3218                 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3219                 break;
3220         }
3221 }
3222
3223 /**
3224  * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3225  */
3226 static void ppc440spe_adma_memcpy_xor_set_src(
3227                 struct ppc440spe_adma_desc_slot *sw_desc,
3228                 dma_addr_t addr, int index)
3229 {
3230         struct ppc440spe_adma_chan *chan;
3231
3232         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3233         sw_desc = sw_desc->group_head;
3234
3235         if (likely(sw_desc))
3236                 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3237 }
3238
3239 /**
3240  * ppc440spe_adma_dma2rxor_inc_addr  -
3241  */
3242 static void ppc440spe_adma_dma2rxor_inc_addr(
3243                 struct ppc440spe_adma_desc_slot *desc,
3244                 struct ppc440spe_rxor *cursor, int index, int src_cnt)
3245 {
3246         cursor->addr_count++;
3247         if (index == src_cnt - 1) {
3248                 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3249         } else if (cursor->addr_count == XOR_MAX_OPS) {
3250                 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3251                 cursor->addr_count = 0;
3252                 cursor->desc_count++;
3253         }
3254 }
3255
3256 /**
3257  * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3258  */
3259 static int ppc440spe_adma_dma2rxor_prep_src(
3260                 struct ppc440spe_adma_desc_slot *hdesc,
3261                 struct ppc440spe_rxor *cursor, int index,
3262                 int src_cnt, u32 addr)
3263 {
3264         int rval = 0;
3265         u32 sign;
3266         struct ppc440spe_adma_desc_slot *desc = hdesc;
3267         int i;
3268
3269         for (i = 0; i < cursor->desc_count; i++) {
3270                 desc = list_entry(hdesc->chain_node.next,
3271                                   struct ppc440spe_adma_desc_slot,
3272                                   chain_node);
3273         }
3274
3275         switch (cursor->state) {
3276         case 0:
3277                 if (addr == cursor->addrl + cursor->len) {
3278                         /* direct RXOR */
3279                         cursor->state = 1;
3280                         cursor->xor_count++;
3281                         if (index == src_cnt-1) {
3282                                 ppc440spe_rxor_set_region(desc,
3283                                         cursor->addr_count,
3284                                         DMA_RXOR12 << DMA_CUED_REGION_OFF);
3285                                 ppc440spe_adma_dma2rxor_inc_addr(
3286                                         desc, cursor, index, src_cnt);
3287                         }
3288                 } else if (cursor->addrl == addr + cursor->len) {
3289                         /* reverse RXOR */
3290                         cursor->state = 1;
3291                         cursor->xor_count++;
3292                         set_bit(cursor->addr_count, &desc->reverse_flags[0]);
3293                         if (index == src_cnt-1) {
3294                                 ppc440spe_rxor_set_region(desc,
3295                                         cursor->addr_count,
3296                                         DMA_RXOR12 << DMA_CUED_REGION_OFF);
3297                                 ppc440spe_adma_dma2rxor_inc_addr(
3298                                         desc, cursor, index, src_cnt);
3299                         }
3300                 } else {
3301                         printk(KERN_ERR "Cannot build "
3302                                 "DMA2 RXOR command block.\n");
3303                         BUG();
3304                 }
3305                 break;
3306         case 1:
3307                 sign = test_bit(cursor->addr_count,
3308                                 desc->reverse_flags)
3309                         ? -1 : 1;
3310                 if (index == src_cnt-2 || (sign == -1
3311                         && addr != cursor->addrl - 2*cursor->len)) {
3312                         cursor->state = 0;
3313                         cursor->xor_count = 1;
3314                         cursor->addrl = addr;
3315                         ppc440spe_rxor_set_region(desc,
3316                                 cursor->addr_count,
3317                                 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3318                         ppc440spe_adma_dma2rxor_inc_addr(
3319                                 desc, cursor, index, src_cnt);
3320                 } else if (addr == cursor->addrl + 2*sign*cursor->len) {
3321                         cursor->state = 2;
3322                         cursor->xor_count = 0;
3323                         ppc440spe_rxor_set_region(desc,
3324                                 cursor->addr_count,
3325                                 DMA_RXOR123 << DMA_CUED_REGION_OFF);
3326                         if (index == src_cnt-1) {
3327                                 ppc440spe_adma_dma2rxor_inc_addr(
3328                                         desc, cursor, index, src_cnt);
3329                         }
3330                 } else if (addr == cursor->addrl + 3*cursor->len) {
3331                         cursor->state = 2;
3332                         cursor->xor_count = 0;
3333                         ppc440spe_rxor_set_region(desc,
3334                                 cursor->addr_count,
3335                                 DMA_RXOR124 << DMA_CUED_REGION_OFF);
3336                         if (index == src_cnt-1) {
3337                                 ppc440spe_adma_dma2rxor_inc_addr(
3338                                         desc, cursor, index, src_cnt);
3339                         }
3340                 } else if (addr == cursor->addrl + 4*cursor->len) {
3341                         cursor->state = 2;
3342                         cursor->xor_count = 0;
3343                         ppc440spe_rxor_set_region(desc,
3344                                 cursor->addr_count,
3345                                 DMA_RXOR125 << DMA_CUED_REGION_OFF);
3346                         if (index == src_cnt-1) {
3347                                 ppc440spe_adma_dma2rxor_inc_addr(
3348                                         desc, cursor, index, src_cnt);
3349                         }
3350                 } else {
3351                         cursor->state = 0;
3352                         cursor->xor_count = 1;
3353                         cursor->addrl = addr;
3354                         ppc440spe_rxor_set_region(desc,
3355                                 cursor->addr_count,
3356                                 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3357                         ppc440spe_adma_dma2rxor_inc_addr(
3358                                 desc, cursor, index, src_cnt);
3359                 }
3360                 break;
3361         case 2:
3362                 cursor->state = 0;
3363                 cursor->addrl = addr;
3364                 cursor->xor_count++;
3365                 if (index) {
3366                         ppc440spe_adma_dma2rxor_inc_addr(
3367                                 desc, cursor, index, src_cnt);
3368                 }
3369                 break;
3370         }
3371
3372         return rval;
3373 }
3374
3375 /**
3376  * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3377  *      ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3378  */
3379 static void ppc440spe_adma_dma2rxor_set_src(
3380                 struct ppc440spe_adma_desc_slot *desc,
3381                 int index, dma_addr_t addr)
3382 {
3383         struct xor_cb *xcb = desc->hw_desc;
3384         int k = 0, op = 0, lop = 0;
3385
3386         /* get the RXOR operand which corresponds to index addr */
3387         while (op <= index) {
3388                 lop = op;
3389                 if (k == XOR_MAX_OPS) {
3390                         k = 0;
3391                         desc = list_entry(desc->chain_node.next,
3392                                 struct ppc440spe_adma_desc_slot, chain_node);
3393                         xcb = desc->hw_desc;
3394
3395                 }
3396                 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3397                     (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3398                         op += 2;
3399                 else
3400                         op += 3;
3401         }
3402
3403         BUG_ON(k < 1);
3404
3405         if (test_bit(k-1, desc->reverse_flags)) {
3406                 /* reverse operand order; put last op in RXOR group */
3407                 if (index == op - 1)
3408                         ppc440spe_rxor_set_src(desc, k - 1, addr);
3409         } else {
3410                 /* direct operand order; put first op in RXOR group */
3411                 if (index == lop)
3412                         ppc440spe_rxor_set_src(desc, k - 1, addr);
3413         }
3414 }
3415
3416 /**
3417  * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3418  *      ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3419  */
3420 static void ppc440spe_adma_dma2rxor_set_mult(
3421                 struct ppc440spe_adma_desc_slot *desc,
3422                 int index, u8 mult)
3423 {
3424         struct xor_cb *xcb = desc->hw_desc;
3425         int k = 0, op = 0, lop = 0;
3426
3427         /* get the RXOR operand which corresponds to index mult */
3428         while (op <= index) {
3429                 lop = op;
3430                 if (k == XOR_MAX_OPS) {
3431                         k = 0;
3432                         desc = list_entry(desc->chain_node.next,
3433                                           struct ppc440spe_adma_desc_slot,
3434                                           chain_node);
3435                         xcb = desc->hw_desc;
3436
3437                 }
3438                 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3439                     (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3440                         op += 2;
3441                 else
3442                         op += 3;
3443         }
3444
3445         BUG_ON(k < 1);
3446         if (test_bit(k-1, desc->reverse_flags)) {
3447                 /* reverse order */
3448                 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
3449         } else {
3450                 /* direct order */
3451                 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
3452         }
3453 }
3454
3455 /**
3456  * ppc440spe_init_rxor_cursor -
3457  */
3458 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3459 {
3460         memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3461         cursor->state = 2;
3462 }
3463
3464 /**
3465  * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3466  * descriptor for the PQXOR operation
3467  */
3468 static void ppc440spe_adma_pq_set_src_mult(
3469                 struct ppc440spe_adma_desc_slot *sw_desc,
3470                 unsigned char mult, int index, int dst_pos)
3471 {
3472         struct ppc440spe_adma_chan *chan;
3473         u32 mult_idx, mult_dst;
3474         struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3475
3476         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3477
3478         switch (chan->device->id) {
3479         case PPC440SPE_DMA0_ID:
3480         case PPC440SPE_DMA1_ID:
3481                 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3482                         int region = test_bit(PPC440SPE_DESC_RXOR12,
3483                                         &sw_desc->flags) ? 2 : 3;
3484
3485                         if (index < region) {
3486                                 /* RXOR multipliers */
3487                                 iter = ppc440spe_get_group_entry(sw_desc,
3488                                         sw_desc->dst_cnt - 1);
3489                                 if (sw_desc->dst_cnt == 2)
3490                                         iter1 = ppc440spe_get_group_entry(
3491                                                         sw_desc, 0);
3492
3493                                 mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3494                                 mult_dst = DMA_CDB_SG_SRC;
3495                         } else {
3496                                 /* WXOR multiplier */
3497                                 iter = ppc440spe_get_group_entry(sw_desc,
3498                                                         index - region +
3499                                                         sw_desc->dst_cnt);
3500                                 mult_idx = DMA_CUED_MULT1_OFF;
3501                                 mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3502                                                      DMA_CDB_SG_DST1;
3503                         }
3504                 } else {
3505                         int znum = 0;
3506
3507                         /* WXOR-only;
3508                          * skip first slots with destinations (if ZERO_DST has
3509                          * place)
3510                          */
3511                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3512                                 znum++;
3513                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3514                                 znum++;
3515
3516                         iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3517                         mult_idx = DMA_CUED_MULT1_OFF;
3518                         mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3519                 }
3520
3521                 if (likely(iter)) {
3522                         ppc440spe_desc_set_src_mult(iter, chan,
3523                                 mult_idx, mult_dst, mult);
3524
3525                         if (unlikely(iter1)) {
3526                                 /* if we have two destinations for RXOR, then
3527                                  * we've just set Q mult. Set-up P now.
3528                                  */
3529                                 ppc440spe_desc_set_src_mult(iter1, chan,
3530                                         mult_idx, mult_dst, 1);
3531                         }
3532
3533                 }
3534                 break;
3535
3536         case PPC440SPE_XOR_ID:
3537                 iter = sw_desc->group_head;
3538                 if (sw_desc->dst_cnt == 2) {
3539                         /* both P & Q calculations required; set P mult here */
3540                         ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
3541
3542                         /* and then set Q mult */
3543                         iter = ppc440spe_get_group_entry(sw_desc,
3544                                sw_desc->descs_per_op);
3545                 }
3546                 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
3547                 break;
3548         }
3549 }
3550
3551 /**
3552  * ppc440spe_adma_free_chan_resources - free the resources allocated
3553  */
3554 static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3555 {
3556         struct ppc440spe_adma_chan *ppc440spe_chan;
3557         struct ppc440spe_adma_desc_slot *iter, *_iter;
3558         int in_use_descs = 0;
3559
3560         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3561         ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3562
3563         spin_lock_bh(&ppc440spe_chan->lock);
3564         list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3565                                         chain_node) {
3566                 in_use_descs++;
3567                 list_del(&iter->chain_node);
3568         }
3569         list_for_each_entry_safe_reverse(iter, _iter,
3570                         &ppc440spe_chan->all_slots, slot_node) {
3571                 list_del(&iter->slot_node);
3572                 kfree(iter);
3573                 ppc440spe_chan->slots_allocated--;
3574         }
3575         ppc440spe_chan->last_used = NULL;
3576
3577         dev_dbg(ppc440spe_chan->device->common.dev,
3578                 "ppc440spe adma%d %s slots_allocated %d\n",
3579                 ppc440spe_chan->device->id,
3580                 __func__, ppc440spe_chan->slots_allocated);
3581         spin_unlock_bh(&ppc440spe_chan->lock);
3582
3583         /* one is ok since we left it on there on purpose */
3584         if (in_use_descs > 1)
3585                 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3586                         in_use_descs - 1);
3587 }
3588
3589 /**
3590  * ppc440spe_adma_tx_status - poll the status of an ADMA transaction
3591  * @chan: ADMA channel handle
3592  * @cookie: ADMA transaction identifier
3593  * @txstate: a holder for the current state of the channel
3594  */
3595 static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan,
3596                         dma_cookie_t cookie, struct dma_tx_state *txstate)
3597 {
3598         struct ppc440spe_adma_chan *ppc440spe_chan;
3599         enum dma_status ret;
3600
3601         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3602         ret = dma_cookie_status(chan, cookie, txstate);
3603         if (ret == DMA_COMPLETE)
3604                 return ret;
3605
3606         ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3607
3608         return dma_cookie_status(chan, cookie, txstate);
3609 }
3610
3611 /**
3612  * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3613  */
3614 static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3615 {
3616         struct ppc440spe_adma_chan *chan = data;
3617
3618         dev_dbg(chan->device->common.dev,
3619                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3620
3621         tasklet_schedule(&chan->irq_tasklet);
3622         ppc440spe_adma_device_clear_eot_status(chan);
3623
3624         return IRQ_HANDLED;
3625 }
3626
3627 /**
3628  * ppc440spe_adma_err_handler - DMA error interrupt handler;
3629  *      do the same things as a eot handler
3630  */
3631 static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3632 {
3633         struct ppc440spe_adma_chan *chan = data;
3634
3635         dev_dbg(chan->device->common.dev,
3636                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3637
3638         tasklet_schedule(&chan->irq_tasklet);
3639         ppc440spe_adma_device_clear_eot_status(chan);
3640
3641         return IRQ_HANDLED;
3642 }
3643
3644 /**
3645  * ppc440spe_test_callback - called when test operation has been done
3646  */
3647 static void ppc440spe_test_callback(void *unused)
3648 {
3649         complete(&ppc440spe_r6_test_comp);
3650 }
3651
3652 /**
3653  * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
3654  */
3655 static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
3656 {
3657         struct ppc440spe_adma_chan *ppc440spe_chan;
3658
3659         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3660         dev_dbg(ppc440spe_chan->device->common.dev,
3661                 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
3662                 __func__, ppc440spe_chan->pending);
3663
3664         if (ppc440spe_chan->pending) {
3665                 ppc440spe_chan->pending = 0;
3666                 ppc440spe_chan_append(ppc440spe_chan);
3667         }
3668 }
3669
3670 /**
3671  * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
3672  *      use FIFOs (as opposite to chains used in XOR) so this is a XOR
3673  *      specific operation)
3674  */
3675 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
3676 {
3677         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
3678         dma_cookie_t cookie;
3679         int slot_cnt, slots_per_op;
3680
3681         dev_dbg(chan->device->common.dev,
3682                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3683
3684         spin_lock_bh(&chan->lock);
3685         slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
3686         sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
3687         if (sw_desc) {
3688                 group_start = sw_desc->group_head;
3689                 list_splice_init(&sw_desc->group_list, &chan->chain);
3690                 async_tx_ack(&sw_desc->async_tx);
3691                 ppc440spe_desc_init_null_xor(group_start);
3692
3693                 cookie = dma_cookie_assign(&sw_desc->async_tx);
3694
3695                 /* initialize the completed cookie to be less than
3696                  * the most recently used cookie
3697                  */
3698                 chan->common.completed_cookie = cookie - 1;
3699
3700                 /* channel should not be busy */
3701                 BUG_ON(ppc440spe_chan_is_busy(chan));
3702
3703                 /* set the descriptor address */
3704                 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
3705
3706                 /* run the descriptor */
3707                 ppc440spe_chan_run(chan);
3708         } else
3709                 printk(KERN_ERR "ppc440spe adma%d"
3710                         " failed to allocate null descriptor\n",
3711                         chan->device->id);
3712         spin_unlock_bh(&chan->lock);
3713 }
3714
3715 /**
3716  * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
3717  *      For this we just perform one WXOR operation with the same source
3718  *      and destination addresses, the GF-multiplier is 1; so if RAID-6
3719  *      capabilities are enabled then we'll get src/dst filled with zero.
3720  */
3721 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
3722 {
3723         struct ppc440spe_adma_desc_slot *sw_desc, *iter;
3724         struct page *pg;
3725         char *a;
3726         dma_addr_t dma_addr, addrs[2];
3727         unsigned long op = 0;
3728         int rval = 0;
3729
3730         set_bit(PPC440SPE_DESC_WXOR, &op);
3731
3732         pg = alloc_page(GFP_KERNEL);
3733         if (!pg)
3734                 return -ENOMEM;
3735
3736         spin_lock_bh(&chan->lock);
3737         sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
3738         if (sw_desc) {
3739                 /* 1 src, 1 dsr, int_ena, WXOR */
3740                 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
3741                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
3742                         ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
3743                         iter->unmap_len = PAGE_SIZE;
3744                 }
3745         } else {
3746                 rval = -EFAULT;
3747                 spin_unlock_bh(&chan->lock);
3748                 goto exit;
3749         }
3750         spin_unlock_bh(&chan->lock);
3751
3752         /* Fill the test page with ones */
3753         memset(page_address(pg), 0xFF, PAGE_SIZE);
3754         dma_addr = dma_map_page(chan->device->dev, pg, 0,
3755                                 PAGE_SIZE, DMA_BIDIRECTIONAL);
3756
3757         /* Setup addresses */
3758         ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
3759         ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
3760         addrs[0] = dma_addr;
3761         addrs[1] = 0;
3762         ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
3763
3764         async_tx_ack(&sw_desc->async_tx);
3765         sw_desc->async_tx.callback = ppc440spe_test_callback;
3766         sw_desc->async_tx.callback_param = NULL;
3767
3768         init_completion(&ppc440spe_r6_test_comp);
3769
3770         ppc440spe_adma_tx_submit(&sw_desc->async_tx);
3771         ppc440spe_adma_issue_pending(&chan->common);
3772
3773         wait_for_completion(&ppc440spe_r6_test_comp);
3774
3775         /* Now check if the test page is zeroed */
3776         a = page_address(pg);
3777         if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
3778                 /* page is zero - RAID-6 enabled */
3779                 rval = 0;
3780         } else {
3781                 /* RAID-6 was not enabled */
3782                 rval = -EINVAL;
3783         }
3784 exit:
3785         __free_page(pg);
3786         return rval;
3787 }
3788
3789 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
3790 {
3791         switch (adev->id) {
3792         case PPC440SPE_DMA0_ID:
3793         case PPC440SPE_DMA1_ID:
3794                 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
3795                 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3796                 dma_cap_set(DMA_PQ, adev->common.cap_mask);
3797                 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
3798                 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
3799                 break;
3800         case PPC440SPE_XOR_ID:
3801                 dma_cap_set(DMA_XOR, adev->common.cap_mask);
3802                 dma_cap_set(DMA_PQ, adev->common.cap_mask);
3803                 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3804                 adev->common.cap_mask = adev->common.cap_mask;
3805                 break;
3806         }
3807
3808         /* Set base routines */
3809         adev->common.device_alloc_chan_resources =
3810                                 ppc440spe_adma_alloc_chan_resources;
3811         adev->common.device_free_chan_resources =
3812                                 ppc440spe_adma_free_chan_resources;
3813         adev->common.device_tx_status = ppc440spe_adma_tx_status;
3814         adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
3815
3816         /* Set prep routines based on capability */
3817         if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
3818                 adev->common.device_prep_dma_memcpy =
3819                         ppc440spe_adma_prep_dma_memcpy;
3820         }
3821         if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
3822                 adev->common.max_xor = XOR_MAX_OPS;
3823                 adev->common.device_prep_dma_xor =
3824                         ppc440spe_adma_prep_dma_xor;
3825         }
3826         if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
3827                 switch (adev->id) {
3828                 case PPC440SPE_DMA0_ID:
3829                         dma_set_maxpq(&adev->common,
3830                                 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3831                         break;
3832                 case PPC440SPE_DMA1_ID:
3833                         dma_set_maxpq(&adev->common,
3834                                 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3835                         break;
3836                 case PPC440SPE_XOR_ID:
3837                         adev->common.max_pq = XOR_MAX_OPS * 3;
3838                         break;
3839                 }
3840                 adev->common.device_prep_dma_pq =
3841                         ppc440spe_adma_prep_dma_pq;
3842         }
3843         if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
3844                 switch (adev->id) {
3845                 case PPC440SPE_DMA0_ID:
3846                         adev->common.max_pq = DMA0_FIFO_SIZE /
3847                                                 sizeof(struct dma_cdb);
3848                         break;
3849                 case PPC440SPE_DMA1_ID:
3850                         adev->common.max_pq = DMA1_FIFO_SIZE /
3851                                                 sizeof(struct dma_cdb);
3852                         break;
3853                 }
3854                 adev->common.device_prep_dma_pq_val =
3855                         ppc440spe_adma_prep_dma_pqzero_sum;
3856         }
3857         if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
3858                 switch (adev->id) {
3859                 case PPC440SPE_DMA0_ID:
3860                         adev->common.max_xor = DMA0_FIFO_SIZE /
3861                                                 sizeof(struct dma_cdb);
3862                         break;
3863                 case PPC440SPE_DMA1_ID:
3864                         adev->common.max_xor = DMA1_FIFO_SIZE /
3865                                                 sizeof(struct dma_cdb);
3866                         break;
3867                 }
3868                 adev->common.device_prep_dma_xor_val =
3869                         ppc440spe_adma_prep_dma_xor_zero_sum;
3870         }
3871         if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
3872                 adev->common.device_prep_dma_interrupt =
3873                         ppc440spe_adma_prep_dma_interrupt;
3874         }
3875         pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
3876           "( %s%s%s%s%s%s)\n",
3877           dev_name(adev->dev),
3878           dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
3879           dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
3880           dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
3881           dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
3882           dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
3883           dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
3884 }
3885
3886 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
3887                                      struct ppc440spe_adma_chan *chan,
3888                                      int *initcode)
3889 {
3890         struct platform_device *ofdev;
3891         struct device_node *np;
3892         int ret;
3893
3894         ofdev = container_of(adev->dev, struct platform_device, dev);
3895         np = ofdev->dev.of_node;
3896         if (adev->id != PPC440SPE_XOR_ID) {
3897                 adev->err_irq = irq_of_parse_and_map(np, 1);
3898                 if (adev->err_irq == NO_IRQ) {
3899                         dev_warn(adev->dev, "no err irq resource?\n");
3900                         *initcode = PPC_ADMA_INIT_IRQ2;
3901                         adev->err_irq = -ENXIO;
3902                 } else
3903                         atomic_inc(&ppc440spe_adma_err_irq_ref);
3904         } else {
3905                 adev->err_irq = -ENXIO;
3906         }
3907
3908         adev->irq = irq_of_parse_and_map(np, 0);
3909         if (adev->irq == NO_IRQ) {
3910                 dev_err(adev->dev, "no irq resource\n");
3911                 *initcode = PPC_ADMA_INIT_IRQ1;
3912                 ret = -ENXIO;
3913                 goto err_irq_map;
3914         }
3915         dev_dbg(adev->dev, "irq %d, err irq %d\n",
3916                 adev->irq, adev->err_irq);
3917
3918         ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
3919                           0, dev_driver_string(adev->dev), chan);
3920         if (ret) {
3921                 dev_err(adev->dev, "can't request irq %d\n",
3922                         adev->irq);
3923                 *initcode = PPC_ADMA_INIT_IRQ1;
3924                 ret = -EIO;
3925                 goto err_req1;
3926         }
3927
3928         /* only DMA engines have a separate error IRQ
3929          * so it's Ok if err_irq < 0 in XOR engine case.
3930          */
3931         if (adev->err_irq > 0) {
3932                 /* both DMA engines share common error IRQ */
3933                 ret = request_irq(adev->err_irq,
3934                                   ppc440spe_adma_err_handler,
3935                                   IRQF_SHARED,
3936                                   dev_driver_string(adev->dev),
3937                                   chan);
3938                 if (ret) {
3939                         dev_err(adev->dev, "can't request irq %d\n",
3940                                 adev->err_irq);
3941                         *initcode = PPC_ADMA_INIT_IRQ2;
3942                         ret = -EIO;
3943                         goto err_req2;
3944                 }
3945         }
3946
3947         if (adev->id == PPC440SPE_XOR_ID) {
3948                 /* enable XOR engine interrupts */
3949                 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3950                             XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
3951                             &adev->xor_reg->ier);
3952         } else {
3953                 u32 mask, enable;
3954
3955                 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
3956                 if (!np) {
3957                         pr_err("%s: can't find I2O device tree node\n",
3958                                 __func__);
3959                         ret = -ENODEV;
3960                         goto err_req2;
3961                 }
3962                 adev->i2o_reg = of_iomap(np, 0);
3963                 if (!adev->i2o_reg) {
3964                         pr_err("%s: failed to map I2O registers\n", __func__);
3965                         of_node_put(np);
3966                         ret = -EINVAL;
3967                         goto err_req2;
3968                 }
3969                 of_node_put(np);
3970                 /* Unmask 'CS FIFO Attention' interrupts and
3971                  * enable generating interrupts on errors
3972                  */
3973                 enable = (adev->id == PPC440SPE_DMA0_ID) ?
3974                          ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3975                          ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3976                 mask = ioread32(&adev->i2o_reg->iopim) & enable;
3977                 iowrite32(mask, &adev->i2o_reg->iopim);
3978         }
3979         return 0;
3980
3981 err_req2:
3982         free_irq(adev->irq, chan);
3983 err_req1:
3984         irq_dispose_mapping(adev->irq);
3985 err_irq_map:
3986         if (adev->err_irq > 0) {
3987                 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
3988                         irq_dispose_mapping(adev->err_irq);
3989         }
3990         return ret;
3991 }
3992
3993 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
3994                                         struct ppc440spe_adma_chan *chan)
3995 {
3996         u32 mask, disable;
3997
3998         if (adev->id == PPC440SPE_XOR_ID) {
3999                 /* disable XOR engine interrupts */
4000                 mask = ioread32be(&adev->xor_reg->ier);
4001                 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
4002                           XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
4003                 iowrite32be(mask, &adev->xor_reg->ier);
4004         } else {
4005                 /* disable DMAx engine interrupts */
4006                 disable = (adev->id == PPC440SPE_DMA0_ID) ?
4007                           (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
4008                           (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
4009                 mask = ioread32(&adev->i2o_reg->iopim) | disable;
4010                 iowrite32(mask, &adev->i2o_reg->iopim);
4011         }
4012         free_irq(adev->irq, chan);
4013         irq_dispose_mapping(adev->irq);
4014         if (adev->err_irq > 0) {
4015                 free_irq(adev->err_irq, chan);
4016                 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
4017                         irq_dispose_mapping(adev->err_irq);
4018                         iounmap(adev->i2o_reg);
4019                 }
4020         }
4021 }
4022
4023 /**
4024  * ppc440spe_adma_probe - probe the asynch device
4025  */
4026 static int ppc440spe_adma_probe(struct platform_device *ofdev)
4027 {
4028         struct device_node *np = ofdev->dev.of_node;
4029         struct resource res;
4030         struct ppc440spe_adma_device *adev;
4031         struct ppc440spe_adma_chan *chan;
4032         struct ppc_dma_chan_ref *ref, *_ref;
4033         int ret = 0, initcode = PPC_ADMA_INIT_OK;
4034         const u32 *idx;
4035         int len;
4036         void *regs;
4037         u32 id, pool_size;
4038
4039         if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
4040                 id = PPC440SPE_XOR_ID;
4041                 /* As far as the XOR engine is concerned, it does not
4042                  * use FIFOs but uses linked list. So there is no dependency
4043                  * between pool size to allocate and the engine configuration.
4044                  */
4045                 pool_size = PAGE_SIZE << 1;
4046         } else {
4047                 /* it is DMA0 or DMA1 */
4048                 idx = of_get_property(np, "cell-index", &len);
4049                 if (!idx || (len != sizeof(u32))) {
4050                         dev_err(&ofdev->dev, "Device node %s has missing "
4051                                 "or invalid cell-index property\n",
4052                                 np->full_name);
4053                         return -EINVAL;
4054                 }
4055                 id = *idx;
4056                 /* DMA0,1 engines use FIFO to maintain CDBs, so we
4057                  * should allocate the pool accordingly to size of this
4058                  * FIFO. Thus, the pool size depends on the FIFO depth:
4059                  * how much CDBs pointers the FIFO may contain then so
4060                  * much CDBs we should provide in the pool.
4061                  * That is
4062                  *   CDB size = 32B;
4063                  *   CDBs number = (DMA0_FIFO_SIZE >> 3);
4064                  *   Pool size = CDBs number * CDB size =
4065                  *      = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4066                  */
4067                 pool_size = (id == PPC440SPE_DMA0_ID) ?
4068                             DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4069                 pool_size <<= 2;
4070         }
4071
4072         if (of_address_to_resource(np, 0, &res)) {
4073                 dev_err(&ofdev->dev, "failed to get memory resource\n");
4074                 initcode = PPC_ADMA_INIT_MEMRES;
4075                 ret = -ENODEV;
4076                 goto out;
4077         }
4078
4079         if (!request_mem_region(res.start, resource_size(&res),
4080                                 dev_driver_string(&ofdev->dev))) {
4081                 dev_err(&ofdev->dev, "failed to request memory region %pR\n",
4082                         &res);
4083                 initcode = PPC_ADMA_INIT_MEMREG;
4084                 ret = -EBUSY;
4085                 goto out;
4086         }
4087
4088         /* create a device */
4089         adev = kzalloc(sizeof(*adev), GFP_KERNEL);
4090         if (!adev) {
4091                 dev_err(&ofdev->dev, "failed to allocate device\n");
4092                 initcode = PPC_ADMA_INIT_ALLOC;
4093                 ret = -ENOMEM;
4094                 goto err_adev_alloc;
4095         }
4096
4097         adev->id = id;
4098         adev->pool_size = pool_size;
4099         /* allocate coherent memory for hardware descriptors */
4100         adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
4101                                         adev->pool_size, &adev->dma_desc_pool,
4102                                         GFP_KERNEL);
4103         if (adev->dma_desc_pool_virt == NULL) {
4104                 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4105                         "memory for hardware descriptors\n",
4106                         adev->pool_size);
4107                 initcode = PPC_ADMA_INIT_COHERENT;
4108                 ret = -ENOMEM;
4109                 goto err_dma_alloc;
4110         }
4111         dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n",
4112                 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4113
4114         regs = ioremap(res.start, resource_size(&res));
4115         if (!regs) {
4116                 dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4117                 goto err_regs_alloc;
4118         }
4119
4120         if (adev->id == PPC440SPE_XOR_ID) {
4121                 adev->xor_reg = regs;
4122                 /* Reset XOR */
4123                 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4124                 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4125         } else {
4126                 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4127                                    DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4128                 adev->dma_reg = regs;
4129                 /* DMAx_FIFO_SIZE is defined in bytes,
4130                  * <fsiz> - is defined in number of CDB pointers (8byte).
4131                  * DMA FIFO Length = CSlength + CPlength, where
4132                  * CSlength = CPlength = (fsiz + 1) * 8.
4133                  */
4134                 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4135                           &adev->dma_reg->fsiz);
4136                 /* Configure DMA engine */
4137                 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4138                           &adev->dma_reg->cfg);
4139                 /* Clear Status */
4140                 iowrite32(~0, &adev->dma_reg->dsts);
4141         }
4142
4143         adev->dev = &ofdev->dev;
4144         adev->common.dev = &ofdev->dev;
4145         INIT_LIST_HEAD(&adev->common.channels);
4146         platform_set_drvdata(ofdev, adev);
4147
4148         /* create a channel */
4149         chan = kzalloc(sizeof(*chan), GFP_KERNEL);
4150         if (!chan) {
4151                 dev_err(&ofdev->dev, "can't allocate channel structure\n");
4152                 initcode = PPC_ADMA_INIT_CHANNEL;
4153                 ret = -ENOMEM;
4154                 goto err_chan_alloc;
4155         }
4156
4157         spin_lock_init(&chan->lock);
4158         INIT_LIST_HEAD(&chan->chain);
4159         INIT_LIST_HEAD(&chan->all_slots);
4160         chan->device = adev;
4161         chan->common.device = &adev->common;
4162         dma_cookie_init(&chan->common);
4163         list_add_tail(&chan->common.device_node, &adev->common.channels);
4164         tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet,
4165                      (unsigned long)chan);
4166
4167         /* allocate and map helper pages for async validation or
4168          * async_mult/async_sum_product operations on DMA0/1.
4169          */
4170         if (adev->id != PPC440SPE_XOR_ID) {
4171                 chan->pdest_page = alloc_page(GFP_KERNEL);
4172                 chan->qdest_page = alloc_page(GFP_KERNEL);
4173                 if (!chan->pdest_page ||
4174                     !chan->qdest_page) {
4175                         if (chan->pdest_page)
4176                                 __free_page(chan->pdest_page);
4177                         if (chan->qdest_page)
4178                                 __free_page(chan->qdest_page);
4179                         ret = -ENOMEM;
4180                         goto err_page_alloc;
4181                 }
4182                 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4183                                            PAGE_SIZE, DMA_BIDIRECTIONAL);
4184                 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4185                                            PAGE_SIZE, DMA_BIDIRECTIONAL);
4186         }
4187
4188         ref = kmalloc(sizeof(*ref), GFP_KERNEL);
4189         if (ref) {
4190                 ref->chan = &chan->common;
4191                 INIT_LIST_HEAD(&ref->node);
4192                 list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
4193         } else {
4194                 dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4195                 ret = -ENOMEM;
4196                 goto err_ref_alloc;
4197         }
4198
4199         ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
4200         if (ret)
4201                 goto err_irq;
4202
4203         ppc440spe_adma_init_capabilities(adev);
4204
4205         ret = dma_async_device_register(&adev->common);
4206         if (ret) {
4207                 initcode = PPC_ADMA_INIT_REGISTER;
4208                 dev_err(&ofdev->dev, "failed to register dma device\n");
4209                 goto err_dev_reg;
4210         }
4211
4212         goto out;
4213
4214 err_dev_reg:
4215         ppc440spe_adma_release_irqs(adev, chan);
4216 err_irq:
4217         list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4218                 if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4219                         list_del(&ref->node);
4220                         kfree(ref);
4221                 }
4222         }
4223 err_ref_alloc:
4224         if (adev->id != PPC440SPE_XOR_ID) {
4225                 dma_unmap_page(&ofdev->dev, chan->pdest,
4226                                PAGE_SIZE, DMA_BIDIRECTIONAL);
4227                 dma_unmap_page(&ofdev->dev, chan->qdest,
4228                                PAGE_SIZE, DMA_BIDIRECTIONAL);
4229                 __free_page(chan->pdest_page);
4230                 __free_page(chan->qdest_page);
4231         }
4232 err_page_alloc:
4233         kfree(chan);
4234 err_chan_alloc:
4235         if (adev->id == PPC440SPE_XOR_ID)
4236                 iounmap(adev->xor_reg);
4237         else
4238                 iounmap(adev->dma_reg);
4239 err_regs_alloc:
4240         dma_free_coherent(adev->dev, adev->pool_size,
4241                           adev->dma_desc_pool_virt,
4242                           adev->dma_desc_pool);
4243 err_dma_alloc:
4244         kfree(adev);
4245 err_adev_alloc:
4246         release_mem_region(res.start, resource_size(&res));
4247 out:
4248         if (id < PPC440SPE_ADMA_ENGINES_NUM)
4249                 ppc440spe_adma_devices[id] = initcode;
4250
4251         return ret;
4252 }
4253
4254 /**
4255  * ppc440spe_adma_remove - remove the asynch device
4256  */
4257 static int ppc440spe_adma_remove(struct platform_device *ofdev)
4258 {
4259         struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev);
4260         struct device_node *np = ofdev->dev.of_node;
4261         struct resource res;
4262         struct dma_chan *chan, *_chan;
4263         struct ppc_dma_chan_ref *ref, *_ref;
4264         struct ppc440spe_adma_chan *ppc440spe_chan;
4265
4266         if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4267                 ppc440spe_adma_devices[adev->id] = -1;
4268
4269         dma_async_device_unregister(&adev->common);
4270
4271         list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4272                                  device_node) {
4273                 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4274                 ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
4275                 tasklet_kill(&ppc440spe_chan->irq_tasklet);
4276                 if (adev->id != PPC440SPE_XOR_ID) {
4277                         dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4278                                         PAGE_SIZE, DMA_BIDIRECTIONAL);
4279                         dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4280                                         PAGE_SIZE, DMA_BIDIRECTIONAL);
4281                         __free_page(ppc440spe_chan->pdest_page);
4282                         __free_page(ppc440spe_chan->qdest_page);
4283                 }
4284                 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4285                                          node) {
4286                         if (ppc440spe_chan ==
4287                             to_ppc440spe_adma_chan(ref->chan)) {
4288                                 list_del(&ref->node);
4289                                 kfree(ref);
4290                         }
4291                 }
4292                 list_del(&chan->device_node);
4293                 kfree(ppc440spe_chan);
4294         }
4295
4296         dma_free_coherent(adev->dev, adev->pool_size,
4297                           adev->dma_desc_pool_virt, adev->dma_desc_pool);
4298         if (adev->id == PPC440SPE_XOR_ID)
4299                 iounmap(adev->xor_reg);
4300         else
4301                 iounmap(adev->dma_reg);
4302         of_address_to_resource(np, 0, &res);
4303         release_mem_region(res.start, resource_size(&res));
4304         kfree(adev);
4305         return 0;
4306 }
4307
4308 /*
4309  * /sys driver interface to enable h/w RAID-6 capabilities
4310  * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4311  * directory are "devices", "enable" and "poly".
4312  * "devices" shows available engines.
4313  * "enable" is used to enable RAID-6 capabilities or to check
4314  * whether these has been activated.
4315  * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4316  */
4317
4318 static ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf)
4319 {
4320         ssize_t size = 0;
4321         int i;
4322
4323         for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4324                 if (ppc440spe_adma_devices[i] == -1)
4325                         continue;
4326                 size += snprintf(buf + size, PAGE_SIZE - size,
4327                                  "PPC440SP(E)-ADMA.%d: %s\n", i,
4328                                  ppc_adma_errors[ppc440spe_adma_devices[i]]);
4329         }
4330         return size;
4331 }
4332
4333 static ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf)
4334 {
4335         return snprintf(buf, PAGE_SIZE,
4336                         "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4337                         ppc440spe_r6_enabled ? "EN" : "DIS");
4338 }
4339
4340 static ssize_t store_ppc440spe_r6enable(struct device_driver *dev,
4341                                         const char *buf, size_t count)
4342 {
4343         unsigned long val;
4344
4345         if (!count || count > 11)
4346                 return -EINVAL;
4347
4348         if (!ppc440spe_r6_tchan)
4349                 return -EFAULT;
4350
4351         /* Write a key */
4352         sscanf(buf, "%lx", &val);
4353         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4354         isync();
4355
4356         /* Verify whether it really works now */
4357         if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
4358                 pr_info("PPC440SP(e) RAID-6 has been activated "
4359                         "successfully\n");
4360                 ppc440spe_r6_enabled = 1;
4361         } else {
4362                 pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4363                         " Error key ?\n");
4364                 ppc440spe_r6_enabled = 0;
4365         }
4366         return count;
4367 }
4368
4369 static ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf)
4370 {
4371         ssize_t size = 0;
4372         u32 reg;
4373
4374 #ifdef CONFIG_440SP
4375         /* 440SP has fixed polynomial */
4376         reg = 0x4d;
4377 #else
4378         reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4379         reg >>= MQ0_CFBHL_POLY;
4380         reg &= 0xFF;
4381 #endif
4382
4383         size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver "
4384                         "uses 0x1%02x polynomial.\n", reg);
4385         return size;
4386 }
4387
4388 static ssize_t store_ppc440spe_r6poly(struct device_driver *dev,
4389                                       const char *buf, size_t count)
4390 {
4391         unsigned long reg, val;
4392
4393 #ifdef CONFIG_440SP
4394         /* 440SP uses default 0x14D polynomial only */
4395         return -EINVAL;
4396 #endif
4397
4398         if (!count || count > 6)
4399                 return -EINVAL;
4400
4401         /* e.g., 0x14D or 0x11D */
4402         sscanf(buf, "%lx", &val);
4403
4404         if (val & ~0x1FF)
4405                 return -EINVAL;
4406
4407         val &= 0xFF;
4408         reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4409         reg &= ~(0xFF << MQ0_CFBHL_POLY);
4410         reg |= val << MQ0_CFBHL_POLY;
4411         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4412
4413         return count;
4414 }
4415
4416 static DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL);
4417 static DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable,
4418                    store_ppc440spe_r6enable);
4419 static DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly,
4420                    store_ppc440spe_r6poly);
4421
4422 /*
4423  * Common initialisation for RAID engines; allocate memory for
4424  * DMAx FIFOs, perform configuration common for all DMA engines.
4425  * Further DMA engine specific configuration is done at probe time.
4426  */
4427 static int ppc440spe_configure_raid_devices(void)
4428 {
4429         struct device_node *np;
4430         struct resource i2o_res;
4431         struct i2o_regs __iomem *i2o_reg;
4432         dcr_host_t i2o_dcr_host;
4433         unsigned int dcr_base, dcr_len;
4434         int i, ret;
4435
4436         np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4437         if (!np) {
4438                 pr_err("%s: can't find I2O device tree node\n",
4439                         __func__);
4440                 return -ENODEV;
4441         }
4442
4443         if (of_address_to_resource(np, 0, &i2o_res)) {
4444                 of_node_put(np);
4445                 return -EINVAL;
4446         }
4447
4448         i2o_reg = of_iomap(np, 0);
4449         if (!i2o_reg) {
4450                 pr_err("%s: failed to map I2O registers\n", __func__);
4451                 of_node_put(np);
4452                 return -EINVAL;
4453         }
4454
4455         /* Get I2O DCRs base */
4456         dcr_base = dcr_resource_start(np, 0);
4457         dcr_len = dcr_resource_len(np, 0);
4458         if (!dcr_base && !dcr_len) {
4459                 pr_err("%s: can't get DCR registers base/len!\n",
4460                         np->full_name);
4461                 of_node_put(np);
4462                 iounmap(i2o_reg);
4463                 return -ENODEV;
4464         }
4465
4466         i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4467         if (!DCR_MAP_OK(i2o_dcr_host)) {
4468                 pr_err("%s: failed to map DCRs!\n", np->full_name);
4469                 of_node_put(np);
4470                 iounmap(i2o_reg);
4471                 return -ENODEV;
4472         }
4473         of_node_put(np);
4474
4475         /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4476          * the base address of FIFO memory space.
4477          * Actually we need twice more physical memory than programmed in the
4478          * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4479          */
4480         ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4481                                          GFP_KERNEL);
4482         if (!ppc440spe_dma_fifo_buf) {
4483                 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4484                 iounmap(i2o_reg);
4485                 dcr_unmap(i2o_dcr_host, dcr_len);
4486                 return -ENOMEM;
4487         }
4488
4489         /*
4490          * Configure h/w
4491          */
4492         /* Reset I2O/DMA */
4493         mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4494         mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4495
4496         /* Setup the base address of mmaped registers */
4497         dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4498         dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4499                                                 I2O_REG_ENABLE);
4500         dcr_unmap(i2o_dcr_host, dcr_len);
4501
4502         /* Setup FIFO memory space base address */
4503         iowrite32(0, &i2o_reg->ifbah);
4504         iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4505
4506         /* set zero FIFO size for I2O, so the whole
4507          * ppc440spe_dma_fifo_buf is used by DMAs.
4508          * DMAx_FIFOs will be configured while probe.
4509          */
4510         iowrite32(0, &i2o_reg->ifsiz);
4511         iounmap(i2o_reg);
4512
4513         /* To prepare WXOR/RXOR functionality we need access to
4514          * Memory Queue Module DCRs (finally it will be enabled
4515          * via /sys interface of the ppc440spe ADMA driver).
4516          */
4517         np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
4518         if (!np) {
4519                 pr_err("%s: can't find MQ device tree node\n",
4520                         __func__);
4521                 ret = -ENODEV;
4522                 goto out_free;
4523         }
4524
4525         /* Get MQ DCRs base */
4526         dcr_base = dcr_resource_start(np, 0);
4527         dcr_len = dcr_resource_len(np, 0);
4528         if (!dcr_base && !dcr_len) {
4529                 pr_err("%s: can't get DCR registers base/len!\n",
4530                         np->full_name);
4531                 ret = -ENODEV;
4532                 goto out_mq;
4533         }
4534
4535         ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4536         if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4537                 pr_err("%s: failed to map DCRs!\n", np->full_name);
4538                 ret = -ENODEV;
4539                 goto out_mq;
4540         }
4541         of_node_put(np);
4542         ppc440spe_mq_dcr_len = dcr_len;
4543
4544         /* Set HB alias */
4545         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4546
4547         /* Set:
4548          * - LL transaction passing limit to 1;
4549          * - Memory controller cycle limit to 1;
4550          * - Galois Polynomial to 0x14d (default)
4551          */
4552         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4553                   (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4554                   (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4555
4556         atomic_set(&ppc440spe_adma_err_irq_ref, 0);
4557         for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4558                 ppc440spe_adma_devices[i] = -1;
4559
4560         return 0;
4561
4562 out_mq:
4563         of_node_put(np);
4564 out_free:
4565         kfree(ppc440spe_dma_fifo_buf);
4566         return ret;
4567 }
4568
4569 static const struct of_device_id ppc440spe_adma_of_match[] = {
4570         { .compatible   = "ibm,dma-440spe", },
4571         { .compatible   = "amcc,xor-accelerator", },
4572         {},
4573 };
4574 MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4575
4576 static struct platform_driver ppc440spe_adma_driver = {
4577         .probe = ppc440spe_adma_probe,
4578         .remove = ppc440spe_adma_remove,
4579         .driver = {
4580                 .name = "PPC440SP(E)-ADMA",
4581                 .owner = THIS_MODULE,
4582                 .of_match_table = ppc440spe_adma_of_match,
4583         },
4584 };
4585
4586 static __init int ppc440spe_adma_init(void)
4587 {
4588         int ret;
4589
4590         ret = ppc440spe_configure_raid_devices();
4591         if (ret)
4592                 return ret;
4593
4594         ret = platform_driver_register(&ppc440spe_adma_driver);
4595         if (ret) {
4596                 pr_err("%s: failed to register platform driver\n",
4597                         __func__);
4598                 goto out_reg;
4599         }
4600
4601         /* Initialization status */
4602         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4603                                  &driver_attr_devices);
4604         if (ret)
4605                 goto out_dev;
4606
4607         /* RAID-6 h/w enable entry */
4608         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4609                                  &driver_attr_enable);
4610         if (ret)
4611                 goto out_en;
4612
4613         /* GF polynomial to use */
4614         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4615                                  &driver_attr_poly);
4616         if (!ret)
4617                 return ret;
4618
4619         driver_remove_file(&ppc440spe_adma_driver.driver,
4620                            &driver_attr_enable);
4621 out_en:
4622         driver_remove_file(&ppc440spe_adma_driver.driver,
4623                            &driver_attr_devices);
4624 out_dev:
4625         /* User will not be able to enable h/w RAID-6 */
4626         pr_err("%s: failed to create RAID-6 driver interface\n",
4627                 __func__);
4628         platform_driver_unregister(&ppc440spe_adma_driver);
4629 out_reg:
4630         dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4631         kfree(ppc440spe_dma_fifo_buf);
4632         return ret;
4633 }
4634
4635 static void __exit ppc440spe_adma_exit(void)
4636 {
4637         driver_remove_file(&ppc440spe_adma_driver.driver,
4638                            &driver_attr_poly);
4639         driver_remove_file(&ppc440spe_adma_driver.driver,
4640                            &driver_attr_enable);
4641         driver_remove_file(&ppc440spe_adma_driver.driver,
4642                            &driver_attr_devices);
4643         platform_driver_unregister(&ppc440spe_adma_driver);
4644         dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4645         kfree(ppc440spe_dma_fifo_buf);
4646 }
4647
4648 arch_initcall(ppc440spe_adma_init);
4649 module_exit(ppc440spe_adma_exit);
4650
4651 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
4652 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
4653 MODULE_LICENSE("GPL");