drivers/dma/sa11x0-dma.c

   1 /*
   2  * SA11x0 DMAengine support
   3  *
   4  * Copyright (C) 2012 Russell King
   5  *   Derived in part from arch/arm/mach-sa1100/dma.c,
   6  *   Copyright (C) 2000, 2001 by Nicolas Pitre
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  */
  12 #include <linux/sched.h>
  13 #include <linux/device.h>
  14 #include <linux/dmaengine.h>
  15 #include <linux/init.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/kernel.h>
  18 #include <linux/module.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/sa11x0-dma.h>
  21 #include <linux/slab.h>
  22 #include <linux/spinlock.h>
  23
  24 #include "virt-dma.h"
  25
  26 #define NR_PHY_CHAN     6
  27 #define DMA_ALIGN       3
  28 #define DMA_MAX_SIZE    0x1fff
  29 #define DMA_CHUNK_SIZE  0x1000
  30
  31 #define DMA_DDAR        0x00
  32 #define DMA_DCSR_S      0x04
  33 #define DMA_DCSR_C      0x08
  34 #define DMA_DCSR_R      0x0c
  35 #define DMA_DBSA        0x10
  36 #define DMA_DBTA        0x14
  37 #define DMA_DBSB        0x18
  38 #define DMA_DBTB        0x1c
  39 #define DMA_SIZE        0x20
  40
  41 #define DCSR_RUN        (1 << 0)
  42 #define DCSR_IE         (1 << 1)
  43 #define DCSR_ERROR      (1 << 2)
  44 #define DCSR_DONEA      (1 << 3)
  45 #define DCSR_STRTA      (1 << 4)
  46 #define DCSR_DONEB      (1 << 5)
  47 #define DCSR_STRTB      (1 << 6)
  48 #define DCSR_BIU        (1 << 7)
  49
  50 #define DDAR_RW         (1 << 0)        /* 0 = W, 1 = R */
  51 #define DDAR_E          (1 << 1)        /* 0 = LE, 1 = BE */
  52 #define DDAR_BS         (1 << 2)        /* 0 = BS4, 1 = BS8 */
  53 #define DDAR_DW         (1 << 3)        /* 0 = 8b, 1 = 16b */
  54 #define DDAR_Ser0UDCTr  (0x0 << 4)
  55 #define DDAR_Ser0UDCRc  (0x1 << 4)
  56 #define DDAR_Ser1SDLCTr (0x2 << 4)
  57 #define DDAR_Ser1SDLCRc (0x3 << 4)
  58 #define DDAR_Ser1UARTTr (0x4 << 4)
  59 #define DDAR_Ser1UARTRc (0x5 << 4)
  60 #define DDAR_Ser2ICPTr  (0x6 << 4)
  61 #define DDAR_Ser2ICPRc  (0x7 << 4)
  62 #define DDAR_Ser3UARTTr (0x8 << 4)
  63 #define DDAR_Ser3UARTRc (0x9 << 4)
  64 #define DDAR_Ser4MCP0Tr (0xa << 4)
  65 #define DDAR_Ser4MCP0Rc (0xb << 4)
  66 #define DDAR_Ser4MCP1Tr (0xc << 4)
  67 #define DDAR_Ser4MCP1Rc (0xd << 4)
  68 #define DDAR_Ser4SSPTr  (0xe << 4)
  69 #define DDAR_Ser4SSPRc  (0xf << 4)
  70
  71 struct sa11x0_dma_sg {
  72         u32                     addr;
  73         u32                     len;
  74 };
  75
  76 struct sa11x0_dma_desc {
  77         struct virt_dma_desc    vd;
  78
  79         u32                     ddar;
  80         size_t                  size;
  81         unsigned                period;
  82         bool                    cyclic;
  83
  84         unsigned                sglen;
  85         struct sa11x0_dma_sg    sg[0];
  86 };
  87
  88 struct sa11x0_dma_phy;
  89
  90 struct sa11x0_dma_chan {
  91         struct virt_dma_chan    vc;
  92
  93         /* protected by c->vc.lock */
  94         struct sa11x0_dma_phy   *phy;
  95         enum dma_status         status;
  96
  97         /* protected by d->lock */
  98         struct list_head        node;
  99
 100         u32                     ddar;
 101         const char              *name;
 102 };
 103
 104 struct sa11x0_dma_phy {
 105         void __iomem            *base;
 106         struct sa11x0_dma_dev   *dev;
 107         unsigned                num;
 108
 109         struct sa11x0_dma_chan  *vchan;
 110
 111         /* Protected by c->vc.lock */
 112         unsigned                sg_load;
 113         struct sa11x0_dma_desc  *txd_load;
 114         unsigned                sg_done;
 115         struct sa11x0_dma_desc  *txd_done;
 116         u32                     dbs[2];
 117         u32                     dbt[2];
 118         u32                     dcsr;
 119 };
 120
 121 struct sa11x0_dma_dev {
 122         struct dma_device       slave;
 123         void __iomem            *base;
 124         spinlock_t              lock;
 125         struct tasklet_struct   task;
 126         struct list_head        chan_pending;
 127         struct sa11x0_dma_phy   phy[NR_PHY_CHAN];
 128 };
 129
 130 static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan)
 131 {
 132         return container_of(chan, struct sa11x0_dma_chan, vc.chan);
 133 }
 134
 135 static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev)
 136 {
 137         return container_of(dmadev, struct sa11x0_dma_dev, slave);
 138 }
 139
 140 static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c)
 141 {
 142         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 143
 144         return vd ? container_of(vd, struct sa11x0_dma_desc, vd) : NULL;
 145 }
 146
 147 static void sa11x0_dma_free_desc(struct virt_dma_desc *vd)
 148 {
 149         kfree(container_of(vd, struct sa11x0_dma_desc, vd));
 150 }
 151
 152 static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd)
 153 {
 154         list_del(&txd->vd.node);
 155         p->txd_load = txd;
 156         p->sg_load = 0;
 157
 158         dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n",
 159                 p->num, &txd->vd, txd->vd.tx.cookie, txd->ddar);
 160 }
 161
 162 static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p,
 163         struct sa11x0_dma_chan *c)
 164 {
 165         struct sa11x0_dma_desc *txd = p->txd_load;
 166         struct sa11x0_dma_sg *sg;
 167         void __iomem *base = p->base;
 168         unsigned dbsx, dbtx;
 169         u32 dcsr;
 170
 171         if (!txd)
 172                 return;
 173
 174         dcsr = readl_relaxed(base + DMA_DCSR_R);
 175
 176         /* Don't try to load the next transfer if both buffers are started */
 177         if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB))
 178                 return;
 179
 180         if (p->sg_load == txd->sglen) {
 181                 if (!txd->cyclic) {
 182                         struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c);
 183
 184                         /*
 185                          * We have reached the end of the current descriptor.
 186                          * Peek at the next descriptor, and if compatible with
 187                          * the current, start processing it.
 188                          */
 189                         if (txn && txn->ddar == txd->ddar) {
 190                                 txd = txn;
 191                                 sa11x0_dma_start_desc(p, txn);
 192                         } else {
 193                                 p->txd_load = NULL;
 194                                 return;
 195                         }
 196                 } else {
 197                         /* Cyclic: reset back to beginning */
 198                         p->sg_load = 0;
 199                 }
 200         }
 201
 202         sg = &txd->sg[p->sg_load++];
 203
 204         /* Select buffer to load according to channel status */
 205         if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) ||
 206             ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) {
 207                 dbsx = DMA_DBSA;
 208                 dbtx = DMA_DBTA;
 209                 dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN;
 210         } else {
 211                 dbsx = DMA_DBSB;
 212                 dbtx = DMA_DBTB;
 213                 dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN;
 214         }
 215
 216         writel_relaxed(sg->addr, base + dbsx);
 217         writel_relaxed(sg->len, base + dbtx);
 218         writel(dcsr, base + DMA_DCSR_S);
 219
 220         dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n",
 221                 p->num, dcsr,
 222                 'A' + (dbsx == DMA_DBSB), sg->addr,
 223                 'A' + (dbtx == DMA_DBTB), sg->len);
 224 }
 225
 226 static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p,
 227         struct sa11x0_dma_chan *c)
 228 {
 229         struct sa11x0_dma_desc *txd = p->txd_done;
 230
 231         if (++p->sg_done == txd->sglen) {
 232                 if (!txd->cyclic) {
 233                         vchan_cookie_complete(&txd->vd);
 234
 235                         p->sg_done = 0;
 236                         p->txd_done = p->txd_load;
 237
 238                         if (!p->txd_done)
 239                                 tasklet_schedule(&p->dev->task);
 240                 } else {
 241                         if ((p->sg_done % txd->period) == 0)
 242                                 vchan_cyclic_callback(&txd->vd);
 243
 244                         /* Cyclic: reset back to beginning */
 245                         p->sg_done = 0;
 246                 }
 247         }
 248
 249         sa11x0_dma_start_sg(p, c);
 250 }
 251
 252 static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id)
 253 {
 254         struct sa11x0_dma_phy *p = dev_id;
 255         struct sa11x0_dma_dev *d = p->dev;
 256         struct sa11x0_dma_chan *c;
 257         u32 dcsr;
 258
 259         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 260         if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB)))
 261                 return IRQ_NONE;
 262
 263         /* Clear reported status bits */
 264         writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB),
 265                 p->base + DMA_DCSR_C);
 266
 267         dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr);
 268
 269         if (dcsr & DCSR_ERROR) {
 270                 dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n",
 271                         p->num, dcsr,
 272                         readl_relaxed(p->base + DMA_DDAR),
 273                         readl_relaxed(p->base + DMA_DBSA),
 274                         readl_relaxed(p->base + DMA_DBTA),
 275                         readl_relaxed(p->base + DMA_DBSB),
 276                         readl_relaxed(p->base + DMA_DBTB));
 277         }
 278
 279         c = p->vchan;
 280         if (c) {
 281                 unsigned long flags;
 282
 283                 spin_lock_irqsave(&c->vc.lock, flags);
 284                 /*
 285                  * Now that we're holding the lock, check that the vchan
 286                  * really is associated with this pchan before touching the
 287                  * hardware.  This should always succeed, because we won't
 288                  * change p->vchan or c->phy while the channel is actively
 289                  * transferring.
 290                  */
 291                 if (c->phy == p) {
 292                         if (dcsr & DCSR_DONEA)
 293                                 sa11x0_dma_complete(p, c);
 294                         if (dcsr & DCSR_DONEB)
 295                                 sa11x0_dma_complete(p, c);
 296                 }
 297                 spin_unlock_irqrestore(&c->vc.lock, flags);
 298         }
 299
 300         return IRQ_HANDLED;
 301 }
 302
 303 static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c)
 304 {
 305         struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c);
 306
 307         /* If the issued list is empty, we have no further txds to process */
 308         if (txd) {
 309                 struct sa11x0_dma_phy *p = c->phy;
 310
 311                 sa11x0_dma_start_desc(p, txd);
 312                 p->txd_done = txd;
 313                 p->sg_done = 0;
 314
 315                 /* The channel should not have any transfers started */
 316                 WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) &
 317                                       (DCSR_STRTA | DCSR_STRTB));
 318
 319                 /* Clear the run and start bits before changing DDAR */
 320                 writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB,
 321                                p->base + DMA_DCSR_C);
 322                 writel_relaxed(txd->ddar, p->base + DMA_DDAR);
 323
 324                 /* Try to start both buffers */
 325                 sa11x0_dma_start_sg(p, c);
 326                 sa11x0_dma_start_sg(p, c);
 327         }
 328 }
 329
 330 static void sa11x0_dma_tasklet(unsigned long arg)
 331 {
 332         struct sa11x0_dma_dev *d = (struct sa11x0_dma_dev *)arg;
 333         struct sa11x0_dma_phy *p;
 334         struct sa11x0_dma_chan *c;
 335         unsigned pch, pch_alloc = 0;
 336
 337         dev_dbg(d->slave.dev, "tasklet enter\n");
 338
 339         list_for_each_entry(c, &d->slave.channels, vc.chan.device_node) {
 340                 spin_lock_irq(&c->vc.lock);
 341                 p = c->phy;
 342                 if (p && !p->txd_done) {
 343                         sa11x0_dma_start_txd(c);
 344                         if (!p->txd_done) {
 345                                 /* No current txd associated with this channel */
 346                                 dev_dbg(d->slave.dev, "pchan %u: free\n", p->num);
 347
 348                                 /* Mark this channel free */
 349                                 c->phy = NULL;
 350                                 p->vchan = NULL;
 351                         }
 352                 }
 353                 spin_unlock_irq(&c->vc.lock);
 354         }
 355
 356         spin_lock_irq(&d->lock);
 357         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 358                 p = &d->phy[pch];
 359
 360                 if (p->vchan == NULL && !list_empty(&d->chan_pending)) {
 361                         c = list_first_entry(&d->chan_pending,
 362                                 struct sa11x0_dma_chan, node);
 363                         list_del_init(&c->node);
 364
 365                         pch_alloc |= 1 << pch;
 366
 367                         /* Mark this channel allocated */
 368                         p->vchan = c;
 369
 370                         dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc);
 371                 }
 372         }
 373         spin_unlock_irq(&d->lock);
 374
 375         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 376                 if (pch_alloc & (1 << pch)) {
 377                         p = &d->phy[pch];
 378                         c = p->vchan;
 379
 380                         spin_lock_irq(&c->vc.lock);
 381                         c->phy = p;
 382
 383                         sa11x0_dma_start_txd(c);
 384                         spin_unlock_irq(&c->vc.lock);
 385                 }
 386         }
 387
 388         dev_dbg(d->slave.dev, "tasklet exit\n");
 389 }
 390
 391
 392 static int sa11x0_dma_alloc_chan_resources(struct dma_chan *chan)
 393 {
 394         return 0;
 395 }
 396
 397 static void sa11x0_dma_free_chan_resources(struct dma_chan *chan)
 398 {
 399         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 400         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 401         unsigned long flags;
 402
 403         spin_lock_irqsave(&d->lock, flags);
 404         list_del_init(&c->node);
 405         spin_unlock_irqrestore(&d->lock, flags);
 406
 407         vchan_free_chan_resources(&c->vc);
 408 }
 409
 410 static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p)
 411 {
 412         unsigned reg;
 413         u32 dcsr;
 414
 415         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 416
 417         if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA ||
 418             (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU)
 419                 reg = DMA_DBSA;
 420         else
 421                 reg = DMA_DBSB;
 422
 423         return readl_relaxed(p->base + reg);
 424 }
 425
 426 static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
 427         dma_cookie_t cookie, struct dma_tx_state *state)
 428 {
 429         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 430         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 431         struct sa11x0_dma_phy *p;
 432         struct virt_dma_desc *vd;
 433         unsigned long flags;
 434         enum dma_status ret;
 435
 436         ret = dma_cookie_status(&c->vc.chan, cookie, state);
 437         if (ret == DMA_COMPLETE)
 438                 return ret;
 439
 440         if (!state)
 441                 return c->status;
 442
 443         spin_lock_irqsave(&c->vc.lock, flags);
 444         p = c->phy;
 445
 446         /*
 447          * If the cookie is on our issue queue, then the residue is
 448          * its total size.
 449          */
 450         vd = vchan_find_desc(&c->vc, cookie);
 451         if (vd) {
 452                 state->residue = container_of(vd, struct sa11x0_dma_desc, vd)->size;
 453         } else if (!p) {
 454                 state->residue = 0;
 455         } else {
 456                 struct sa11x0_dma_desc *txd;
 457                 size_t bytes = 0;
 458
 459                 if (p->txd_done && p->txd_done->vd.tx.cookie == cookie)
 460                         txd = p->txd_done;
 461                 else if (p->txd_load && p->txd_load->vd.tx.cookie == cookie)
 462                         txd = p->txd_load;
 463                 else
 464                         txd = NULL;
 465
 466                 ret = c->status;
 467                 if (txd) {
 468                         dma_addr_t addr = sa11x0_dma_pos(p);
 469                         unsigned i;
 470
 471                         dev_vdbg(d->slave.dev, "tx_status: addr:%x\n", addr);
 472
 473                         for (i = 0; i < txd->sglen; i++) {
 474                                 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",
 475                                         i, txd->sg[i].addr, txd->sg[i].len);
 476                                 if (addr >= txd->sg[i].addr &&
 477                                     addr < txd->sg[i].addr + txd->sg[i].len) {
 478                                         unsigned len;
 479
 480                                         len = txd->sg[i].len -
 481                                                 (addr - txd->sg[i].addr);
 482                                         dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n",
 483                                                 i, len);
 484                                         bytes += len;
 485                                         i++;
 486                                         break;
 487                                 }
 488                         }
 489                         for (; i < txd->sglen; i++) {
 490                                 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n",
 491                                         i, txd->sg[i].addr, txd->sg[i].len);
 492                                 bytes += txd->sg[i].len;
 493                         }
 494                 }
 495                 state->residue = bytes;
 496         }
 497         spin_unlock_irqrestore(&c->vc.lock, flags);
 498
 499         dev_vdbg(d->slave.dev, "tx_status: bytes 0x%zx\n", state->residue);
 500
 501         return ret;
 502 }
 503
 504 /*
 505  * Move pending txds to the issued list, and re-init pending list.
 506  * If not already pending, add this channel to the list of pending
 507  * channels and trigger the tasklet to run.
 508  */
 509 static void sa11x0_dma_issue_pending(struct dma_chan *chan)
 510 {
 511         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 512         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 513         unsigned long flags;
 514
 515         spin_lock_irqsave(&c->vc.lock, flags);
 516         if (vchan_issue_pending(&c->vc)) {
 517                 if (!c->phy) {
 518                         spin_lock(&d->lock);
 519                         if (list_empty(&c->node)) {
 520                                 list_add_tail(&c->node, &d->chan_pending);
 521                                 tasklet_schedule(&d->task);
 522                                 dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
 523                         }
 524                         spin_unlock(&d->lock);
 525                 }
 526         } else
 527                 dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
 528         spin_unlock_irqrestore(&c->vc.lock, flags);
 529 }
 530
 531 static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
 532         struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen,
 533         enum dma_transfer_direction dir, unsigned long flags, void *context)
 534 {
 535         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 536         struct sa11x0_dma_desc *txd;
 537         struct scatterlist *sgent;
 538         unsigned i, j = sglen;
 539         size_t size = 0;
 540
 541         /* SA11x0 channels can only operate in their native direction */
 542         if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 543                 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 544                         &c->vc, c->ddar, dir);
 545                 return NULL;
 546         }
 547
 548         /* Do not allow zero-sized txds */
 549         if (sglen == 0)
 550                 return NULL;
 551
 552         for_each_sg(sg, sgent, sglen, i) {
 553                 dma_addr_t addr = sg_dma_address(sgent);
 554                 unsigned int len = sg_dma_len(sgent);
 555
 556                 if (len > DMA_MAX_SIZE)
 557                         j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;
 558                 if (addr & DMA_ALIGN) {
 559                         dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %08x\n",
 560                                 &c->vc, addr);
 561                         return NULL;
 562                 }
 563         }
 564
 565         txd = kzalloc(sizeof(*txd) + j * sizeof(txd->sg[0]), GFP_ATOMIC);
 566         if (!txd) {
 567                 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
 568                 return NULL;
 569         }
 570
 571         j = 0;
 572         for_each_sg(sg, sgent, sglen, i) {
 573                 dma_addr_t addr = sg_dma_address(sgent);
 574                 unsigned len = sg_dma_len(sgent);
 575
 576                 size += len;
 577
 578                 do {
 579                         unsigned tlen = len;
 580
 581                         /*
 582                          * Check whether the transfer will fit.  If not, try
 583                          * to split the transfer up such that we end up with
 584                          * equal chunks - but make sure that we preserve the
 585                          * alignment.  This avoids small segments.
 586                          */
 587                         if (tlen > DMA_MAX_SIZE) {
 588                                 unsigned mult = DIV_ROUND_UP(tlen,
 589                                         DMA_MAX_SIZE & ~DMA_ALIGN);
 590
 591                                 tlen = (tlen / mult) & ~DMA_ALIGN;
 592                         }
 593
 594                         txd->sg[j].addr = addr;
 595                         txd->sg[j].len = tlen;
 596
 597                         addr += tlen;
 598                         len -= tlen;
 599                         j++;
 600                 } while (len);
 601         }
 602
 603         txd->ddar = c->ddar;
 604         txd->size = size;
 605         txd->sglen = j;
 606
 607         dev_dbg(chan->device->dev, "vchan %p: txd %p: size %u nr %u\n",
 608                 &c->vc, &txd->vd, txd->size, txd->sglen);
 609
 610         return vchan_tx_prep(&c->vc, &txd->vd, flags);
 611 }
 612
 613 static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic(
 614         struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
 615         enum dma_transfer_direction dir, unsigned long flags)
 616 {
 617         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 618         struct sa11x0_dma_desc *txd;
 619         unsigned i, j, k, sglen, sgperiod;
 620
 621         /* SA11x0 channels can only operate in their native direction */
 622         if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 623                 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 624                         &c->vc, c->ddar, dir);
 625                 return NULL;
 626         }
 627
 628         sgperiod = DIV_ROUND_UP(period, DMA_MAX_SIZE & ~DMA_ALIGN);
 629         sglen = size * sgperiod / period;
 630
 631         /* Do not allow zero-sized txds */
 632         if (sglen == 0)
 633                 return NULL;
 634
 635         txd = kzalloc(sizeof(*txd) + sglen * sizeof(txd->sg[0]), GFP_ATOMIC);
 636         if (!txd) {
 637                 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
 638                 return NULL;
 639         }
 640
 641         for (i = k = 0; i < size / period; i++) {
 642                 size_t tlen, len = period;
 643
 644                 for (j = 0; j < sgperiod; j++, k++) {
 645                         tlen = len;
 646
 647                         if (tlen > DMA_MAX_SIZE) {
 648                                 unsigned mult = DIV_ROUND_UP(tlen, DMA_MAX_SIZE & ~DMA_ALIGN);
 649                                 tlen = (tlen / mult) & ~DMA_ALIGN;
 650                         }
 651
 652                         txd->sg[k].addr = addr;
 653                         txd->sg[k].len = tlen;
 654                         addr += tlen;
 655                         len -= tlen;
 656                 }
 657
 658                 WARN_ON(len != 0);
 659         }
 660
 661         WARN_ON(k != sglen);
 662
 663         txd->ddar = c->ddar;
 664         txd->size = size;
 665         txd->sglen = sglen;
 666         txd->cyclic = 1;
 667         txd->period = sgperiod;
 668
 669         return vchan_tx_prep(&c->vc, &txd->vd, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 670 }
 671
 672 static int sa11x0_dma_device_config(struct dma_chan *chan,
 673                                     struct dma_slave_config *cfg)
 674 {
 675         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 676         u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW);
 677         dma_addr_t addr;
 678         enum dma_slave_buswidth width;
 679         u32 maxburst;
 680
 681         if (ddar & DDAR_RW) {
 682                 addr = cfg->src_addr;
 683                 width = cfg->src_addr_width;
 684                 maxburst = cfg->src_maxburst;
 685         } else {
 686                 addr = cfg->dst_addr;
 687                 width = cfg->dst_addr_width;
 688                 maxburst = cfg->dst_maxburst;
 689         }
 690
 691         if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE &&
 692              width != DMA_SLAVE_BUSWIDTH_2_BYTES) ||
 693             (maxburst != 4 && maxburst != 8))
 694                 return -EINVAL;
 695
 696         if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
 697                 ddar |= DDAR_DW;
 698         if (maxburst == 8)
 699                 ddar |= DDAR_BS;
 700
 701         dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %x width %u burst %u\n",
 702                 &c->vc, addr, width, maxburst);
 703
 704         c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;
 705
 706         return 0;
 707 }
 708
 709 static int sa11x0_dma_device_pause(struct dma_chan *chan)
 710 {
 711         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 712         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 713         struct sa11x0_dma_phy *p;
 714         LIST_HEAD(head);
 715         unsigned long flags;
 716
 717         dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc);
 718         spin_lock_irqsave(&c->vc.lock, flags);
 719         if (c->status == DMA_IN_PROGRESS) {
 720                 c->status = DMA_PAUSED;
 721
 722                 p = c->phy;
 723                 if (p) {
 724                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
 725                 } else {
 726                         spin_lock(&d->lock);
 727                         list_del_init(&c->node);
 728                         spin_unlock(&d->lock);
 729                 }
 730         }
 731         spin_unlock_irqrestore(&c->vc.lock, flags);
 732
 733         return 0;
 734 }
 735
 736 static int sa11x0_dma_device_resume(struct dma_chan *chan)
 737 {
 738         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 739         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 740         struct sa11x0_dma_phy *p;
 741         LIST_HEAD(head);
 742         unsigned long flags;
 743
 744         dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc);
 745         spin_lock_irqsave(&c->vc.lock, flags);
 746         if (c->status == DMA_PAUSED) {
 747                 c->status = DMA_IN_PROGRESS;
 748
 749                 p = c->phy;
 750                 if (p) {
 751                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S);
 752                 } else if (!list_empty(&c->vc.desc_issued)) {
 753                         spin_lock(&d->lock);
 754                         list_add_tail(&c->node, &d->chan_pending);
 755                         spin_unlock(&d->lock);
 756                 }
 757         }
 758         spin_unlock_irqrestore(&c->vc.lock, flags);
 759
 760         return 0;
 761 }
 762
 763 static int sa11x0_dma_device_terminate_all(struct dma_chan *chan)
 764 {
 765         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 766         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 767         struct sa11x0_dma_phy *p;
 768         LIST_HEAD(head);
 769         unsigned long flags;
 770
 771         dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
 772         /* Clear the tx descriptor lists */
 773         spin_lock_irqsave(&c->vc.lock, flags);
 774         vchan_get_all_descriptors(&c->vc, &head);
 775
 776         p = c->phy;
 777         if (p) {
 778                 dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num);
 779                 /* vchan is assigned to a pchan - stop the channel */
 780                 writel(DCSR_RUN | DCSR_IE |
 781                        DCSR_STRTA | DCSR_DONEA |
 782                        DCSR_STRTB | DCSR_DONEB,
 783                        p->base + DMA_DCSR_C);
 784
 785                 if (p->txd_load) {
 786                         if (p->txd_load != p->txd_done)
 787                                 list_add_tail(&p->txd_load->vd.node, &head);
 788                         p->txd_load = NULL;
 789                 }
 790                 if (p->txd_done) {
 791                         list_add_tail(&p->txd_done->vd.node, &head);
 792                         p->txd_done = NULL;
 793                 }
 794                 c->phy = NULL;
 795                 spin_lock(&d->lock);
 796                 p->vchan = NULL;
 797                 spin_unlock(&d->lock);
 798                 tasklet_schedule(&d->task);
 799         }
 800         spin_unlock_irqrestore(&c->vc.lock, flags);
 801         vchan_dma_desc_free_list(&c->vc, &head);
 802
 803         return 0;
 804 }
 805
 806 struct sa11x0_dma_channel_desc {
 807         u32 ddar;
 808         const char *name;
 809 };
 810
 811 #define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 }
 812 static const struct sa11x0_dma_channel_desc chan_desc[] = {
 813         CD(Ser0UDCTr, 0),
 814         CD(Ser0UDCRc, DDAR_RW),
 815         CD(Ser1SDLCTr, 0),
 816         CD(Ser1SDLCRc, DDAR_RW),
 817         CD(Ser1UARTTr, 0),
 818         CD(Ser1UARTRc, DDAR_RW),
 819         CD(Ser2ICPTr, 0),
 820         CD(Ser2ICPRc, DDAR_RW),
 821         CD(Ser3UARTTr, 0),
 822         CD(Ser3UARTRc, DDAR_RW),
 823         CD(Ser4MCP0Tr, 0),
 824         CD(Ser4MCP0Rc, DDAR_RW),
 825         CD(Ser4MCP1Tr, 0),
 826         CD(Ser4MCP1Rc, DDAR_RW),
 827         CD(Ser4SSPTr, 0),
 828         CD(Ser4SSPRc, DDAR_RW),
 829 };
 830
 831 static int sa11x0_dma_init_dmadev(struct dma_device *dmadev,
 832         struct device *dev)
 833 {
 834         unsigned i;
 835
 836         INIT_LIST_HEAD(&dmadev->channels);
 837         dmadev->dev = dev;
 838         dmadev->device_alloc_chan_resources = sa11x0_dma_alloc_chan_resources;
 839         dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources;
 840         dmadev->device_config = sa11x0_dma_device_config;
 841         dmadev->device_pause = sa11x0_dma_device_pause;
 842         dmadev->device_resume = sa11x0_dma_device_resume;
 843         dmadev->device_terminate_all = sa11x0_dma_device_terminate_all;
 844         dmadev->device_tx_status = sa11x0_dma_tx_status;
 845         dmadev->device_issue_pending = sa11x0_dma_issue_pending;
 846
 847         for (i = 0; i < ARRAY_SIZE(chan_desc); i++) {
 848                 struct sa11x0_dma_chan *c;
 849
 850                 c = kzalloc(sizeof(*c), GFP_KERNEL);
 851                 if (!c) {
 852                         dev_err(dev, "no memory for channel %u\n", i);
 853                         return -ENOMEM;
 854                 }
 855
 856                 c->status = DMA_IN_PROGRESS;
 857                 c->ddar = chan_desc[i].ddar;
 858                 c->name = chan_desc[i].name;
 859                 INIT_LIST_HEAD(&c->node);
 860
 861                 c->vc.desc_free = sa11x0_dma_free_desc;
 862                 vchan_init(&c->vc, dmadev);
 863         }
 864
 865         return dma_async_device_register(dmadev);
 866 }
 867
 868 static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr,
 869         void *data)
 870 {
 871         int irq = platform_get_irq(pdev, nr);
 872
 873         if (irq <= 0)
 874                 return -ENXIO;
 875
 876         return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data);
 877 }
 878
 879 static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr,
 880         void *data)
 881 {
 882         int irq = platform_get_irq(pdev, nr);
 883         if (irq > 0)
 884                 free_irq(irq, data);
 885 }
 886
 887 static void sa11x0_dma_free_channels(struct dma_device *dmadev)
 888 {
 889         struct sa11x0_dma_chan *c, *cn;
 890
 891         list_for_each_entry_safe(c, cn, &dmadev->channels, vc.chan.device_node) {
 892                 list_del(&c->vc.chan.device_node);
 893                 tasklet_kill(&c->vc.task);
 894                 kfree(c);
 895         }
 896 }
 897
 898 static int sa11x0_dma_probe(struct platform_device *pdev)
 899 {
 900         struct sa11x0_dma_dev *d;
 901         struct resource *res;
 902         unsigned i;
 903         int ret;
 904
 905         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 906         if (!res)
 907                 return -ENXIO;
 908
 909         d = kzalloc(sizeof(*d), GFP_KERNEL);
 910         if (!d) {
 911                 ret = -ENOMEM;
 912                 goto err_alloc;
 913         }
 914
 915         spin_lock_init(&d->lock);
 916         INIT_LIST_HEAD(&d->chan_pending);
 917
 918         d->base = ioremap(res->start, resource_size(res));
 919         if (!d->base) {
 920                 ret = -ENOMEM;
 921                 goto err_ioremap;
 922         }
 923
 924         tasklet_init(&d->task, sa11x0_dma_tasklet, (unsigned long)d);
 925
 926         for (i = 0; i < NR_PHY_CHAN; i++) {
 927                 struct sa11x0_dma_phy *p = &d->phy[i];
 928
 929                 p->dev = d;
 930                 p->num = i;
 931                 p->base = d->base + i * DMA_SIZE;
 932                 writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR |
 933                         DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB,
 934                         p->base + DMA_DCSR_C);
 935                 writel_relaxed(0, p->base + DMA_DDAR);
 936
 937                 ret = sa11x0_dma_request_irq(pdev, i, p);
 938                 if (ret) {
 939                         while (i) {
 940                                 i--;
 941                                 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 942                         }
 943                         goto err_irq;
 944                 }
 945         }
 946
 947         dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
 948         dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
 949         d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg;
 950         d->slave.device_prep_dma_cyclic = sa11x0_dma_prep_dma_cyclic;
 951         ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev);
 952         if (ret) {
 953                 dev_warn(d->slave.dev, "failed to register slave async device: %d\n",
 954                         ret);
 955                 goto err_slave_reg;
 956         }
 957
 958         platform_set_drvdata(pdev, d);
 959         return 0;
 960
 961  err_slave_reg:
 962         sa11x0_dma_free_channels(&d->slave);
 963         for (i = 0; i < NR_PHY_CHAN; i++)
 964                 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 965  err_irq:
 966         tasklet_kill(&d->task);
 967         iounmap(d->base);
 968  err_ioremap:
 969         kfree(d);
 970  err_alloc:
 971         return ret;
 972 }
 973
 974 static int sa11x0_dma_remove(struct platform_device *pdev)
 975 {
 976         struct sa11x0_dma_dev *d = platform_get_drvdata(pdev);
 977         unsigned pch;
 978
 979         dma_async_device_unregister(&d->slave);
 980
 981         sa11x0_dma_free_channels(&d->slave);
 982         for (pch = 0; pch < NR_PHY_CHAN; pch++)
 983                 sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]);
 984         tasklet_kill(&d->task);
 985         iounmap(d->base);
 986         kfree(d);
 987
 988         return 0;
 989 }
 990
 991 static int sa11x0_dma_suspend(struct device *dev)
 992 {
 993         struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
 994         unsigned pch;
 995
 996         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 997                 struct sa11x0_dma_phy *p = &d->phy[pch];
 998                 u32 dcsr, saved_dcsr;
 999
1000                 dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1001                 if (dcsr & DCSR_RUN) {
1002                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
1003                         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1004                 }
1005
1006                 saved_dcsr &= DCSR_RUN | DCSR_IE;
1007                 if (dcsr & DCSR_BIU) {
1008                         p->dbs[0] = readl_relaxed(p->base + DMA_DBSB);
1009                         p->dbt[0] = readl_relaxed(p->base + DMA_DBTB);
1010                         p->dbs[1] = readl_relaxed(p->base + DMA_DBSA);
1011                         p->dbt[1] = readl_relaxed(p->base + DMA_DBTA);
1012                         saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) |
1013                                       (dcsr & DCSR_STRTB ? DCSR_STRTA : 0);
1014                 } else {
1015                         p->dbs[0] = readl_relaxed(p->base + DMA_DBSA);
1016                         p->dbt[0] = readl_relaxed(p->base + DMA_DBTA);
1017                         p->dbs[1] = readl_relaxed(p->base + DMA_DBSB);
1018                         p->dbt[1] = readl_relaxed(p->base + DMA_DBTB);
1019                         saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB);
1020                 }
1021                 p->dcsr = saved_dcsr;
1022
1023                 writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C);
1024         }
1025
1026         return 0;
1027 }
1028
1029 static int sa11x0_dma_resume(struct device *dev)
1030 {
1031         struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1032         unsigned pch;
1033
1034         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1035                 struct sa11x0_dma_phy *p = &d->phy[pch];
1036                 struct sa11x0_dma_desc *txd = NULL;
1037                 u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1038
1039                 WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN));
1040
1041                 if (p->txd_done)
1042                         txd = p->txd_done;
1043                 else if (p->txd_load)
1044                         txd = p->txd_load;
1045
1046                 if (!txd)
1047                         continue;
1048
1049                 writel_relaxed(txd->ddar, p->base + DMA_DDAR);
1050
1051                 writel_relaxed(p->dbs[0], p->base + DMA_DBSA);
1052                 writel_relaxed(p->dbt[0], p->base + DMA_DBTA);
1053                 writel_relaxed(p->dbs[1], p->base + DMA_DBSB);
1054                 writel_relaxed(p->dbt[1], p->base + DMA_DBTB);
1055                 writel_relaxed(p->dcsr, p->base + DMA_DCSR_S);
1056         }
1057
1058         return 0;
1059 }
1060
1061 static const struct dev_pm_ops sa11x0_dma_pm_ops = {
1062         .suspend_noirq = sa11x0_dma_suspend,
1063         .resume_noirq = sa11x0_dma_resume,
1064         .freeze_noirq = sa11x0_dma_suspend,
1065         .thaw_noirq = sa11x0_dma_resume,
1066         .poweroff_noirq = sa11x0_dma_suspend,
1067         .restore_noirq = sa11x0_dma_resume,
1068 };
1069
1070 static struct platform_driver sa11x0_dma_driver = {
1071         .driver = {
1072                 .name   = "sa11x0-dma",
1073                 .pm     = &sa11x0_dma_pm_ops,
1074         },
1075         .probe          = sa11x0_dma_probe,
1076         .remove         = sa11x0_dma_remove,
1077 };
1078
1079 bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param)
1080 {
1081         if (chan->device->dev->driver == &sa11x0_dma_driver.driver) {
1082                 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
1083                 const char *p = param;
1084
1085                 return !strcmp(c->name, p);
1086         }
1087         return false;
1088 }
1089 EXPORT_SYMBOL(sa11x0_dma_filter_fn);
1090
1091 static int __init sa11x0_dma_init(void)
1092 {
1093         return platform_driver_register(&sa11x0_dma_driver);
1094 }
1095 subsys_initcall(sa11x0_dma_init);
1096
1097 static void __exit sa11x0_dma_exit(void)
1098 {
1099         platform_driver_unregister(&sa11x0_dma_driver);
1100 }
1101 module_exit(sa11x0_dma_exit);
1102
1103 MODULE_AUTHOR("Russell King");
1104 MODULE_DESCRIPTION("SA-11x0 DMA driver");
1105 MODULE_LICENSE("GPL v2");
1106 MODULE_ALIAS("platform:sa11x0-dma");