drivers/usb/musb/musb_cppi41.c

   1 #include <linux/device.h>
   2 #include <linux/dma-mapping.h>
   3 #include <linux/dmaengine.h>
   4 #include <linux/sizes.h>
   5 #include <linux/platform_device.h>
   6 #include <linux/of.h>
   7
   8 #include "musb_core.h"
   9
  10 #define RNDIS_REG(x) (0x80 + ((x - 1) * 4))
  11
  12 #define EP_MODE_AUTOREG_NONE            0
  13 #define EP_MODE_AUTOREG_ALL_NEOP        1
  14 #define EP_MODE_AUTOREG_ALWAYS          3
  15
  16 #define EP_MODE_DMA_TRANSPARENT         0
  17 #define EP_MODE_DMA_RNDIS               1
  18 #define EP_MODE_DMA_GEN_RNDIS           3
  19
  20 #define USB_CTRL_TX_MODE        0x70
  21 #define USB_CTRL_RX_MODE        0x74
  22 #define USB_CTRL_AUTOREQ        0xd0
  23 #define USB_TDOWN               0xd8
  24
  25 struct cppi41_dma_channel {
  26         struct dma_channel channel;
  27         struct cppi41_dma_controller *controller;
  28         struct musb_hw_ep *hw_ep;
  29         struct dma_chan *dc;
  30         dma_cookie_t cookie;
  31         u8 port_num;
  32         u8 is_tx;
  33         u8 is_allocated;
  34         u8 usb_toggle;
  35
  36         dma_addr_t buf_addr;
  37         u32 total_len;
  38         u32 prog_len;
  39         u32 transferred;
  40         u32 packet_sz;
  41 };
  42
  43 #define MUSB_DMA_NUM_CHANNELS 15
  44
  45 struct cppi41_dma_controller {
  46         struct dma_controller controller;
  47         struct cppi41_dma_channel rx_channel[MUSB_DMA_NUM_CHANNELS];
  48         struct cppi41_dma_channel tx_channel[MUSB_DMA_NUM_CHANNELS];
  49         struct musb *musb;
  50         u32 rx_mode;
  51         u32 tx_mode;
  52         u32 auto_req;
  53 };
  54
  55 static void save_rx_toggle(struct cppi41_dma_channel *cppi41_channel)
  56 {
  57         u16 csr;
  58         u8 toggle;
  59
  60         if (cppi41_channel->is_tx)
  61                 return;
  62         if (!is_host_active(cppi41_channel->controller->musb))
  63                 return;
  64
  65         csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR);
  66         toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0;
  67
  68         cppi41_channel->usb_toggle = toggle;
  69 }
  70
  71 static void update_rx_toggle(struct cppi41_dma_channel *cppi41_channel)
  72 {
  73         u16 csr;
  74         u8 toggle;
  75
  76         if (cppi41_channel->is_tx)
  77                 return;
  78         if (!is_host_active(cppi41_channel->controller->musb))
  79                 return;
  80
  81         csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR);
  82         toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0;
  83
  84         /*
  85          * AM335x Advisory 1.0.13: Due to internal synchronisation error the
  86          * data toggle may reset from DATA1 to DATA0 during receiving data from
  87          * more than one endpoint.
  88          */
  89         if (!toggle && toggle == cppi41_channel->usb_toggle) {
  90                 csr |= MUSB_RXCSR_H_DATATOGGLE | MUSB_RXCSR_H_WR_DATATOGGLE;
  91                 musb_writew(cppi41_channel->hw_ep->regs, MUSB_RXCSR, csr);
  92                 dev_dbg(cppi41_channel->controller->musb->controller,
  93                                 "Restoring DATA1 toggle.\n");
  94         }
  95
  96         cppi41_channel->usb_toggle = toggle;
  97 }
  98
  99 static void cppi41_dma_callback(void *private_data)
 100 {
 101         struct dma_channel *channel = private_data;
 102         struct cppi41_dma_channel *cppi41_channel = channel->private_data;
 103         struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
 104         struct musb *musb = hw_ep->musb;
 105         unsigned long flags;
 106         struct dma_tx_state txstate;
 107         u32 transferred;
 108
 109         spin_lock_irqsave(&musb->lock, flags);
 110
 111         dmaengine_tx_status(cppi41_channel->dc, cppi41_channel->cookie,
 112                         &txstate);
 113         transferred = cppi41_channel->prog_len - txstate.residue;
 114         cppi41_channel->transferred += transferred;
 115
 116         dev_dbg(musb->controller, "DMA transfer done on hw_ep=%d bytes=%d/%d\n",
 117                 hw_ep->epnum, cppi41_channel->transferred,
 118                 cppi41_channel->total_len);
 119
 120         update_rx_toggle(cppi41_channel);
 121
 122         if (cppi41_channel->transferred == cppi41_channel->total_len ||
 123                         transferred < cppi41_channel->packet_sz) {
 124
 125                 /* done, complete */
 126                 cppi41_channel->channel.actual_len =
 127                         cppi41_channel->transferred;
 128                 cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;
 129                 musb_dma_completion(musb, hw_ep->epnum, cppi41_channel->is_tx);
 130         } else {
 131                 /* next iteration, reload */
 132                 struct dma_chan *dc = cppi41_channel->dc;
 133                 struct dma_async_tx_descriptor *dma_desc;
 134                 enum dma_transfer_direction direction;
 135                 u16 csr;
 136                 u32 remain_bytes;
 137                 void __iomem *epio = cppi41_channel->hw_ep->regs;
 138
 139                 cppi41_channel->buf_addr += cppi41_channel->packet_sz;
 140
 141                 remain_bytes = cppi41_channel->total_len;
 142                 remain_bytes -= cppi41_channel->transferred;
 143                 remain_bytes = min(remain_bytes, cppi41_channel->packet_sz);
 144                 cppi41_channel->prog_len = remain_bytes;
 145
 146                 direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV
 147                         : DMA_DEV_TO_MEM;
 148                 dma_desc = dmaengine_prep_slave_single(dc,
 149                                 cppi41_channel->buf_addr,
 150                                 remain_bytes,
 151                                 direction,
 152                                 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 153                 if (WARN_ON(!dma_desc)) {
 154                         spin_unlock_irqrestore(&musb->lock, flags);
 155                         return;
 156                 }
 157
 158                 dma_desc->callback = cppi41_dma_callback;
 159                 dma_desc->callback_param = channel;
 160                 cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);
 161                 dma_async_issue_pending(dc);
 162
 163                 if (!cppi41_channel->is_tx) {
 164                         csr = musb_readw(epio, MUSB_RXCSR);
 165                         csr |= MUSB_RXCSR_H_REQPKT;
 166                         musb_writew(epio, MUSB_RXCSR, csr);
 167                 }
 168         }
 169         spin_unlock_irqrestore(&musb->lock, flags);
 170 }
 171
 172 static u32 update_ep_mode(unsigned ep, unsigned mode, u32 old)
 173 {
 174         unsigned shift;
 175
 176         shift = (ep - 1) * 2;
 177         old &= ~(3 << shift);
 178         old |= mode << shift;
 179         return old;
 180 }
 181
 182 static void cppi41_set_dma_mode(struct cppi41_dma_channel *cppi41_channel,
 183                 unsigned mode)
 184 {
 185         struct cppi41_dma_controller *controller = cppi41_channel->controller;
 186         u32 port;
 187         u32 new_mode;
 188         u32 old_mode;
 189
 190         if (cppi41_channel->is_tx)
 191                 old_mode = controller->tx_mode;
 192         else
 193                 old_mode = controller->rx_mode;
 194         port = cppi41_channel->port_num;
 195         new_mode = update_ep_mode(port, mode, old_mode);
 196
 197         if (new_mode == old_mode)
 198                 return;
 199         if (cppi41_channel->is_tx) {
 200                 controller->tx_mode = new_mode;
 201                 musb_writel(controller->musb->ctrl_base, USB_CTRL_TX_MODE,
 202                                 new_mode);
 203         } else {
 204                 controller->rx_mode = new_mode;
 205                 musb_writel(controller->musb->ctrl_base, USB_CTRL_RX_MODE,
 206                                 new_mode);
 207         }
 208 }
 209
 210 static void cppi41_set_autoreq_mode(struct cppi41_dma_channel *cppi41_channel,
 211                 unsigned mode)
 212 {
 213         struct cppi41_dma_controller *controller = cppi41_channel->controller;
 214         u32 port;
 215         u32 new_mode;
 216         u32 old_mode;
 217
 218         old_mode = controller->auto_req;
 219         port = cppi41_channel->port_num;
 220         new_mode = update_ep_mode(port, mode, old_mode);
 221
 222         if (new_mode == old_mode)
 223                 return;
 224         controller->auto_req = new_mode;
 225         musb_writel(controller->musb->ctrl_base, USB_CTRL_AUTOREQ, new_mode);
 226 }
 227
 228 static bool cppi41_configure_channel(struct dma_channel *channel,
 229                                 u16 packet_sz, u8 mode,
 230                                 dma_addr_t dma_addr, u32 len)
 231 {
 232         struct cppi41_dma_channel *cppi41_channel = channel->private_data;
 233         struct dma_chan *dc = cppi41_channel->dc;
 234         struct dma_async_tx_descriptor *dma_desc;
 235         enum dma_transfer_direction direction;
 236         struct musb *musb = cppi41_channel->controller->musb;
 237         unsigned use_gen_rndis = 0;
 238
 239         dev_dbg(musb->controller,
 240                 "configure ep%d/%x packet_sz=%d, mode=%d, dma_addr=0x%llx, len=%d is_tx=%d\n",
 241                 cppi41_channel->port_num, RNDIS_REG(cppi41_channel->port_num),
 242                 packet_sz, mode, (unsigned long long) dma_addr,
 243                 len, cppi41_channel->is_tx);
 244
 245         cppi41_channel->buf_addr = dma_addr;
 246         cppi41_channel->total_len = len;
 247         cppi41_channel->transferred = 0;
 248         cppi41_channel->packet_sz = packet_sz;
 249
 250         /*
 251          * Due to AM335x' Advisory 1.0.13 we are not allowed to transfer more
 252          * than max packet size at a time.
 253          */
 254         if (cppi41_channel->is_tx)
 255                 use_gen_rndis = 1;
 256
 257         if (use_gen_rndis) {
 258                 /* RNDIS mode */
 259                 if (len > packet_sz) {
 260                         musb_writel(musb->ctrl_base,
 261                                 RNDIS_REG(cppi41_channel->port_num), len);
 262                         /* gen rndis */
 263                         cppi41_set_dma_mode(cppi41_channel,
 264                                         EP_MODE_DMA_GEN_RNDIS);
 265
 266                         /* auto req */
 267                         cppi41_set_autoreq_mode(cppi41_channel,
 268                                         EP_MODE_AUTOREG_ALL_NEOP);
 269                 } else {
 270                         musb_writel(musb->ctrl_base,
 271                                         RNDIS_REG(cppi41_channel->port_num), 0);
 272                         cppi41_set_dma_mode(cppi41_channel,
 273                                         EP_MODE_DMA_TRANSPARENT);
 274                         cppi41_set_autoreq_mode(cppi41_channel,
 275                                         EP_MODE_AUTOREG_NONE);
 276                 }
 277         } else {
 278                 /* fallback mode */
 279                 cppi41_set_dma_mode(cppi41_channel, EP_MODE_DMA_TRANSPARENT);
 280                 cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREG_NONE);
 281                 len = min_t(u32, packet_sz, len);
 282         }
 283         cppi41_channel->prog_len = len;
 284         direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
 285         dma_desc = dmaengine_prep_slave_single(dc, dma_addr, len, direction,
 286                         DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 287         if (!dma_desc)
 288                 return false;
 289
 290         dma_desc->callback = cppi41_dma_callback;
 291         dma_desc->callback_param = channel;
 292         cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);
 293
 294         save_rx_toggle(cppi41_channel);
 295         dma_async_issue_pending(dc);
 296         return true;
 297 }
 298
 299 static struct dma_channel *cppi41_dma_channel_allocate(struct dma_controller *c,
 300                                 struct musb_hw_ep *hw_ep, u8 is_tx)
 301 {
 302         struct cppi41_dma_controller *controller = container_of(c,
 303                         struct cppi41_dma_controller, controller);
 304         struct cppi41_dma_channel *cppi41_channel = NULL;
 305         u8 ch_num = hw_ep->epnum - 1;
 306
 307         if (ch_num >= MUSB_DMA_NUM_CHANNELS)
 308                 return NULL;
 309
 310         if (is_tx)
 311                 cppi41_channel = &controller->tx_channel[ch_num];
 312         else
 313                 cppi41_channel = &controller->rx_channel[ch_num];
 314
 315         if (!cppi41_channel->dc)
 316                 return NULL;
 317
 318         if (cppi41_channel->is_allocated)
 319                 return NULL;
 320
 321         cppi41_channel->hw_ep = hw_ep;
 322         cppi41_channel->is_allocated = 1;
 323
 324         return &cppi41_channel->channel;
 325 }
 326
 327 static void cppi41_dma_channel_release(struct dma_channel *channel)
 328 {
 329         struct cppi41_dma_channel *cppi41_channel = channel->private_data;
 330
 331         if (cppi41_channel->is_allocated) {
 332                 cppi41_channel->is_allocated = 0;
 333                 channel->status = MUSB_DMA_STATUS_FREE;
 334                 channel->actual_len = 0;
 335         }
 336 }
 337
 338 static int cppi41_dma_channel_program(struct dma_channel *channel,
 339                                 u16 packet_sz, u8 mode,
 340                                 dma_addr_t dma_addr, u32 len)
 341 {
 342         int ret;
 343
 344         BUG_ON(channel->status == MUSB_DMA_STATUS_UNKNOWN ||
 345                 channel->status == MUSB_DMA_STATUS_BUSY);
 346
 347         channel->status = MUSB_DMA_STATUS_BUSY;
 348         channel->actual_len = 0;
 349         ret = cppi41_configure_channel(channel, packet_sz, mode, dma_addr, len);
 350         if (!ret)
 351                 channel->status = MUSB_DMA_STATUS_FREE;
 352
 353         return ret;
 354 }
 355
 356 static int cppi41_is_compatible(struct dma_channel *channel, u16 maxpacket,
 357                 void *buf, u32 length)
 358 {
 359         struct cppi41_dma_channel *cppi41_channel = channel->private_data;
 360         struct cppi41_dma_controller *controller = cppi41_channel->controller;
 361         struct musb *musb = controller->musb;
 362
 363         if (is_host_active(musb)) {
 364                 WARN_ON(1);
 365                 return 1;
 366         }
 367         if (cppi41_channel->is_tx)
 368                 return 1;
 369         /* AM335x Advisory 1.0.13. No workaround for device RX mode */
 370         return 0;
 371 }
 372
 373 static int cppi41_dma_channel_abort(struct dma_channel *channel)
 374 {
 375         struct cppi41_dma_channel *cppi41_channel = channel->private_data;
 376         struct cppi41_dma_controller *controller = cppi41_channel->controller;
 377         struct musb *musb = controller->musb;
 378         void __iomem *epio = cppi41_channel->hw_ep->regs;
 379         int tdbit;
 380         int ret;
 381         unsigned is_tx;
 382         u16 csr;
 383
 384         is_tx = cppi41_channel->is_tx;
 385         dev_dbg(musb->controller, "abort channel=%d, is_tx=%d\n",
 386                         cppi41_channel->port_num, is_tx);
 387
 388         if (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)
 389                 return 0;
 390
 391         if (is_tx) {
 392                 csr = musb_readw(epio, MUSB_TXCSR);
 393                 csr &= ~MUSB_TXCSR_DMAENAB;
 394                 musb_writew(epio, MUSB_TXCSR, csr);
 395         } else {
 396                 csr = musb_readw(epio, MUSB_RXCSR);
 397                 csr &= ~(MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_DMAENAB);
 398                 musb_writew(epio, MUSB_RXCSR, csr);
 399
 400                 csr = musb_readw(epio, MUSB_RXCSR);
 401                 if (csr & MUSB_RXCSR_RXPKTRDY) {
 402                         csr |= MUSB_RXCSR_FLUSHFIFO;
 403                         musb_writew(epio, MUSB_RXCSR, csr);
 404                         musb_writew(epio, MUSB_RXCSR, csr);
 405                 }
 406         }
 407
 408         tdbit = 1 << cppi41_channel->port_num;
 409         if (is_tx)
 410                 tdbit <<= 16;
 411
 412         do {
 413                 musb_writel(musb->ctrl_base, USB_TDOWN, tdbit);
 414                 ret = dmaengine_terminate_all(cppi41_channel->dc);
 415         } while (ret == -EAGAIN);
 416
 417         musb_writel(musb->ctrl_base, USB_TDOWN, tdbit);
 418
 419         if (is_tx) {
 420                 csr = musb_readw(epio, MUSB_TXCSR);
 421                 if (csr & MUSB_TXCSR_TXPKTRDY) {
 422                         csr |= MUSB_TXCSR_FLUSHFIFO;
 423                         musb_writew(epio, MUSB_TXCSR, csr);
 424                 }
 425         }
 426
 427         cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;
 428         return 0;
 429 }
 430
 431 static void cppi41_release_all_dma_chans(struct cppi41_dma_controller *ctrl)
 432 {
 433         struct dma_chan *dc;
 434         int i;
 435
 436         for (i = 0; i < MUSB_DMA_NUM_CHANNELS; i++) {
 437                 dc = ctrl->tx_channel[i].dc;
 438                 if (dc)
 439                         dma_release_channel(dc);
 440                 dc = ctrl->rx_channel[i].dc;
 441                 if (dc)
 442                         dma_release_channel(dc);
 443         }
 444 }
 445
 446 static void cppi41_dma_controller_stop(struct cppi41_dma_controller *controller)
 447 {
 448         cppi41_release_all_dma_chans(controller);
 449 }
 450
 451 static int cppi41_dma_controller_start(struct cppi41_dma_controller *controller)
 452 {
 453         struct musb *musb = controller->musb;
 454         struct device *dev = musb->controller;
 455         struct device_node *np = dev->of_node;
 456         struct cppi41_dma_channel *cppi41_channel;
 457         int count;
 458         int i;
 459         int ret;
 460
 461         count = of_property_count_strings(np, "dma-names");
 462         if (count < 0)
 463                 return count;
 464
 465         for (i = 0; i < count; i++) {
 466                 struct dma_chan *dc;
 467                 struct dma_channel *musb_dma;
 468                 const char *str;
 469                 unsigned is_tx;
 470                 unsigned int port;
 471
 472                 ret = of_property_read_string_index(np, "dma-names", i, &str);
 473                 if (ret)
 474                         goto err;
 475                 if (!strncmp(str, "tx", 2))
 476                         is_tx = 1;
 477                 else if (!strncmp(str, "rx", 2))
 478                         is_tx = 0;
 479                 else {
 480                         dev_err(dev, "Wrong dmatype %s\n", str);
 481                         goto err;
 482                 }
 483                 ret = kstrtouint(str + 2, 0, &port);
 484                 if (ret)
 485                         goto err;
 486
 487                 ret = -EINVAL;
 488                 if (port > MUSB_DMA_NUM_CHANNELS || !port)
 489                         goto err;
 490                 if (is_tx)
 491                         cppi41_channel = &controller->tx_channel[port - 1];
 492                 else
 493                         cppi41_channel = &controller->rx_channel[port - 1];
 494
 495                 cppi41_channel->controller = controller;
 496                 cppi41_channel->port_num = port;
 497                 cppi41_channel->is_tx = is_tx;
 498
 499                 musb_dma = &cppi41_channel->channel;
 500                 musb_dma->private_data = cppi41_channel;
 501                 musb_dma->status = MUSB_DMA_STATUS_FREE;
 502                 musb_dma->max_len = SZ_4M;
 503
 504                 dc = dma_request_slave_channel(dev, str);
 505                 if (!dc) {
 506                         dev_err(dev, "Falied to request %s.\n", str);
 507                         ret = -EPROBE_DEFER;
 508                         goto err;
 509                 }
 510                 cppi41_channel->dc = dc;
 511         }
 512         return 0;
 513 err:
 514         cppi41_release_all_dma_chans(controller);
 515         return ret;
 516 }
 517
 518 void dma_controller_destroy(struct dma_controller *c)
 519 {
 520         struct cppi41_dma_controller *controller = container_of(c,
 521                         struct cppi41_dma_controller, controller);
 522
 523         cppi41_dma_controller_stop(controller);
 524         kfree(controller);
 525 }
 526
 527 struct dma_controller *dma_controller_create(struct musb *musb,
 528                                         void __iomem *base)
 529 {
 530         struct cppi41_dma_controller *controller;
 531         int ret = 0;
 532
 533         if (!musb->controller->of_node) {
 534                 dev_err(musb->controller, "Need DT for the DMA engine.\n");
 535                 return NULL;
 536         }
 537
 538         controller = kzalloc(sizeof(*controller), GFP_KERNEL);
 539         if (!controller)
 540                 goto kzalloc_fail;
 541
 542         controller->musb = musb;
 543
 544         controller->controller.channel_alloc = cppi41_dma_channel_allocate;
 545         controller->controller.channel_release = cppi41_dma_channel_release;
 546         controller->controller.channel_program = cppi41_dma_channel_program;
 547         controller->controller.channel_abort = cppi41_dma_channel_abort;
 548         controller->controller.is_compatible = cppi41_is_compatible;
 549
 550         ret = cppi41_dma_controller_start(controller);
 551         if (ret)
 552                 goto plat_get_fail;
 553         return &controller->controller;
 554
 555 plat_get_fail:
 556         kfree(controller);
 557 kzalloc_fail:
 558         if (ret == -EPROBE_DEFER)
 559                 return ERR_PTR(ret);
 560         return NULL;
 561 }