drivers/staging/octeon-usb/octeon-hcd.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2008 Cavium Networks
   7  *
   8  * Some parts of the code were originally released under BSD license:
   9  *
  10  * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
  11  * reserved.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions are
  15  * met:
  16  *
  17  *   * Redistributions of source code must retain the above copyright
  18  *     notice, this list of conditions and the following disclaimer.
  19  *
  20  *   * Redistributions in binary form must reproduce the above
  21  *     copyright notice, this list of conditions and the following
  22  *     disclaimer in the documentation and/or other materials provided
  23  *     with the distribution.
  24  *
  25  *   * Neither the name of Cavium Networks nor the names of
  26  *     its contributors may be used to endorse or promote products
  27  *     derived from this software without specific prior written
  28  *     permission.
  29  *
  30  * This Software, including technical data, may be subject to U.S. export
  31  * control laws, including the U.S. Export Administration Act and its associated
  32  * regulations, and may be subject to export or import regulations in other
  33  * countries.
  34  *
  35  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  36  * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  37  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  38  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
  39  * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  40  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  41  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  42  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  43  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
  44  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  45  */
  46 #include <linux/kernel.h>
  47 #include <linux/module.h>
  48 #include <linux/init.h>
  49 #include <linux/pci.h>
  50 #include <linux/prefetch.h>
  51 #include <linux/interrupt.h>
  52 #include <linux/platform_device.h>
  53 #include <linux/usb.h>
  54
  55 #include <linux/time.h>
  56 #include <linux/delay.h>
  57
  58 #include <asm/octeon/cvmx.h>
  59 #include <asm/octeon/cvmx-iob-defs.h>
  60
  61 #include <linux/usb/hcd.h>
  62
  63 #include <linux/err.h>
  64
  65 #include <asm/octeon/octeon.h>
  66 #include <asm/octeon/cvmx-helper.h>
  67 #include <asm/octeon/cvmx-sysinfo.h>
  68 #include <asm/octeon/cvmx-helper-board.h>
  69
  70 #include "octeon-hcd.h"
  71
  72 /**
  73  * enum cvmx_usb_speed - the possible USB device speeds
  74  *
  75  * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
  76  * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
  77  * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
  78  */
  79 enum cvmx_usb_speed {
  80         CVMX_USB_SPEED_HIGH = 0,
  81         CVMX_USB_SPEED_FULL = 1,
  82         CVMX_USB_SPEED_LOW = 2,
  83 };
  84
  85 /**
  86  * enum cvmx_usb_transfer - the possible USB transfer types
  87  *
  88  * @CVMX_USB_TRANSFER_CONTROL:     USB transfer type control for hub and status
  89  *                                 transfers
  90  * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
  91  *                                 priority periodic transfers
  92  * @CVMX_USB_TRANSFER_BULK:        USB transfer type bulk for large low priority
  93  *                                 transfers
  94  * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
  95  *                                 periodic transfers
  96  */
  97 enum cvmx_usb_transfer {
  98         CVMX_USB_TRANSFER_CONTROL = 0,
  99         CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
 100         CVMX_USB_TRANSFER_BULK = 2,
 101         CVMX_USB_TRANSFER_INTERRUPT = 3,
 102 };
 103
 104 /**
 105  * enum cvmx_usb_direction - the transfer directions
 106  *
 107  * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
 108  * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
 109  */
 110 enum cvmx_usb_direction {
 111         CVMX_USB_DIRECTION_OUT,
 112         CVMX_USB_DIRECTION_IN,
 113 };
 114
 115 /**
 116  * enum cvmx_usb_complete - possible callback function status codes
 117  *
 118  * @CVMX_USB_COMPLETE_SUCCESS:    The transaction / operation finished without
 119  *                                any errors
 120  * @CVMX_USB_COMPLETE_SHORT:      FIXME: This is currently not implemented
 121  * @CVMX_USB_COMPLETE_CANCEL:     The transaction was canceled while in flight
 122  *                                by a user call to cvmx_usb_cancel
 123  * @CVMX_USB_COMPLETE_ERROR:      The transaction aborted with an unexpected
 124  *                                error status
 125  * @CVMX_USB_COMPLETE_STALL:      The transaction received a USB STALL response
 126  *                                from the device
 127  * @CVMX_USB_COMPLETE_XACTERR:    The transaction failed with an error from the
 128  *                                device even after a number of retries
 129  * @CVMX_USB_COMPLETE_DATATGLERR: The transaction failed with a data toggle
 130  *                                error even after a number of retries
 131  * @CVMX_USB_COMPLETE_BABBLEERR:  The transaction failed with a babble error
 132  * @CVMX_USB_COMPLETE_FRAMEERR:   The transaction failed with a frame error
 133  *                                even after a number of retries
 134  */
 135 enum cvmx_usb_complete {
 136         CVMX_USB_COMPLETE_SUCCESS,
 137         CVMX_USB_COMPLETE_SHORT,
 138         CVMX_USB_COMPLETE_CANCEL,
 139         CVMX_USB_COMPLETE_ERROR,
 140         CVMX_USB_COMPLETE_STALL,
 141         CVMX_USB_COMPLETE_XACTERR,
 142         CVMX_USB_COMPLETE_DATATGLERR,
 143         CVMX_USB_COMPLETE_BABBLEERR,
 144         CVMX_USB_COMPLETE_FRAMEERR,
 145 };
 146
 147 /**
 148  * struct cvmx_usb_port_status - the USB port status information
 149  *
 150  * @port_enabled:       1 = Usb port is enabled, 0 = disabled
 151  * @port_over_current:  1 = Over current detected, 0 = Over current not
 152  *                      detected. Octeon doesn't support over current detection.
 153  * @port_powered:       1 = Port power is being supplied to the device, 0 =
 154  *                      power is off. Octeon doesn't support turning port power
 155  *                      off.
 156  * @port_speed:         Current port speed.
 157  * @connected:          1 = A device is connected to the port, 0 = No device is
 158  *                      connected.
 159  * @connect_change:     1 = Device connected state changed since the last set
 160  *                      status call.
 161  */
 162 struct cvmx_usb_port_status {
 163         uint32_t reserved               : 25;
 164         uint32_t port_enabled           : 1;
 165         uint32_t port_over_current      : 1;
 166         uint32_t port_powered           : 1;
 167         enum cvmx_usb_speed port_speed  : 2;
 168         uint32_t connected              : 1;
 169         uint32_t connect_change         : 1;
 170 };
 171
 172 /**
 173  * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
 174  *
 175  * @offset:     This is the offset in bytes into the main buffer where this data
 176  *              is stored.
 177  * @length:     This is the length in bytes of the data.
 178  * @status:     This is the status of this individual packet transfer.
 179  */
 180 struct cvmx_usb_iso_packet {
 181         int offset;
 182         int length;
 183         enum cvmx_usb_complete status;
 184 };
 185
 186 /**
 187  * enum cvmx_usb_initialize_flags - flags used by the initialization function
 188  *
 189  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
 190  *                                            as clock source at USB_XO and
 191  *                                            USB_XI.
 192  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
 193  *                                            board clock source at USB_XO.
 194  *                                            USB_XI should be tied to GND.
 195  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
 196  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
 197  *                                            crystal
 198  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
 199  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
 200  * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:         Disable DMA and used polled IO for
 201  *                                            data transfer use for the USB
 202  */
 203 enum cvmx_usb_initialize_flags {
 204         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI           = 1 << 0,
 205         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND          = 1 << 1,
 206         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK        = 3 << 3,
 207         CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ           = 1 << 3,
 208         CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ           = 2 << 3,
 209         CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ           = 3 << 3,
 210         /* Bits 3-4 used to encode the clock frequency */
 211         CVMX_USB_INITIALIZE_FLAGS_NO_DMA                = 1 << 5,
 212 };
 213
 214 /**
 215  * enum cvmx_usb_pipe_flags - internal flags for a pipe.
 216  *
 217  * @__CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
 218  *                                   actively using hardware. Do not use.
 219  * @__CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high
 220  *                                   speed pipe is in the ping state. Do not
 221  *                                   use.
 222  */
 223 enum cvmx_usb_pipe_flags {
 224         __CVMX_USB_PIPE_FLAGS_SCHEDULED = 1 << 17,
 225         __CVMX_USB_PIPE_FLAGS_NEED_PING = 1 << 18,
 226 };
 227
 228 /* Maximum number of times to retry failed transactions */
 229 #define MAX_RETRIES             3
 230
 231 /* Maximum number of hardware channels supported by the USB block */
 232 #define MAX_CHANNELS            8
 233
 234 /*
 235  * The low level hardware can transfer a maximum of this number of bytes in each
 236  * transfer. The field is 19 bits wide
 237  */
 238 #define MAX_TRANSFER_BYTES      ((1<<19)-1)
 239
 240 /*
 241  * The low level hardware can transfer a maximum of this number of packets in
 242  * each transfer. The field is 10 bits wide
 243  */
 244 #define MAX_TRANSFER_PACKETS    ((1<<10)-1)
 245
 246 /**
 247  * Logical transactions may take numerous low level
 248  * transactions, especially when splits are concerned. This
 249  * enum represents all of the possible stages a transaction can
 250  * be in. Note that split completes are always even. This is so
 251  * the NAK handler can backup to the previous low level
 252  * transaction with a simple clearing of bit 0.
 253  */
 254 enum cvmx_usb_stage {
 255         CVMX_USB_STAGE_NON_CONTROL,
 256         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
 257         CVMX_USB_STAGE_SETUP,
 258         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
 259         CVMX_USB_STAGE_DATA,
 260         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
 261         CVMX_USB_STAGE_STATUS,
 262         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 263 };
 264
 265 /**
 266  * struct cvmx_usb_transaction - describes each pending USB transaction
 267  *                               regardless of type. These are linked together
 268  *                               to form a list of pending requests for a pipe.
 269  *
 270  * @node:               List node for transactions in the pipe.
 271  * @type:               Type of transaction, duplicated of the pipe.
 272  * @flags:              State flags for this transaction.
 273  * @buffer:             User's physical buffer address to read/write.
 274  * @buffer_length:      Size of the user's buffer in bytes.
 275  * @control_header:     For control transactions, physical address of the 8
 276  *                      byte standard header.
 277  * @iso_start_frame:    For ISO transactions, the starting frame number.
 278  * @iso_number_packets: For ISO transactions, the number of packets in the
 279  *                      request.
 280  * @iso_packets:        For ISO transactions, the sub packets in the request.
 281  * @actual_bytes:       Actual bytes transfer for this transaction.
 282  * @stage:              For control transactions, the current stage.
 283  * @urb:                URB.
 284  */
 285 struct cvmx_usb_transaction {
 286         struct list_head node;
 287         enum cvmx_usb_transfer type;
 288         uint64_t buffer;
 289         int buffer_length;
 290         uint64_t control_header;
 291         int iso_start_frame;
 292         int iso_number_packets;
 293         struct cvmx_usb_iso_packet *iso_packets;
 294         int xfersize;
 295         int pktcnt;
 296         int retries;
 297         int actual_bytes;
 298         enum cvmx_usb_stage stage;
 299         struct urb *urb;
 300 };
 301
 302 /**
 303  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
 304  *                        and some USB device. It contains a list of pending
 305  *                        request to the device.
 306  *
 307  * @node:               List node for pipe list
 308  * @next:               Pipe after this one in the list
 309  * @transactions:       List of pending transactions
 310  * @interval:           For periodic pipes, the interval between packets in
 311  *                      frames
 312  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
 313  * @flags:              State flags for this pipe
 314  * @device_speed:       Speed of device connected to this pipe
 315  * @transfer_type:      Type of transaction supported by this pipe
 316  * @transfer_dir:       IN or OUT. Ignored for Control
 317  * @multi_count:        Max packet in a row for the device
 318  * @max_packet:         The device's maximum packet size in bytes
 319  * @device_addr:        USB device address at other end of pipe
 320  * @endpoint_num:       USB endpoint number at other end of pipe
 321  * @hub_device_addr:    Hub address this device is connected to
 322  * @hub_port:           Hub port this device is connected to
 323  * @pid_toggle:         This toggles between 0/1 on every packet send to track
 324  *                      the data pid needed
 325  * @channel:            Hardware DMA channel for this pipe
 326  * @split_sc_frame:     The low order bits of the frame number the split
 327  *                      complete should be sent on
 328  */
 329 struct cvmx_usb_pipe {
 330         struct list_head node;
 331         struct list_head transactions;
 332         uint64_t interval;
 333         uint64_t next_tx_frame;
 334         enum cvmx_usb_pipe_flags flags;
 335         enum cvmx_usb_speed device_speed;
 336         enum cvmx_usb_transfer transfer_type;
 337         enum cvmx_usb_direction transfer_dir;
 338         int multi_count;
 339         uint16_t max_packet;
 340         uint8_t device_addr;
 341         uint8_t endpoint_num;
 342         uint8_t hub_device_addr;
 343         uint8_t hub_port;
 344         uint8_t pid_toggle;
 345         uint8_t channel;
 346         int8_t split_sc_frame;
 347 };
 348
 349 struct cvmx_usb_tx_fifo {
 350         struct {
 351                 int channel;
 352                 int size;
 353                 uint64_t address;
 354         } entry[MAX_CHANNELS+1];
 355         int head;
 356         int tail;
 357 };
 358
 359 /**
 360  * struct cvmx_usb_state - the state of the USB block
 361  *
 362  * init_flags:             Flags passed to initialize.
 363  * index:                  Which USB block this is for.
 364  * idle_hardware_channels: Bit set for every idle hardware channel.
 365  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
 366  *                         determine splits.
 367  * pipe_for_channel:       Map channels to pipes.
 368  * pipe:                   Storage for pipes.
 369  * indent:                 Used by debug output to indent functions.
 370  * port_status:            Last port status used for change notification.
 371  * idle_pipes:             List of open pipes that have no transactions.
 372  * active_pipes:           Active pipes indexed by transfer type.
 373  * frame_number:           Increments every SOF interrupt for time keeping.
 374  * active_split:           Points to the current active split, or NULL.
 375  */
 376 struct cvmx_usb_state {
 377         int init_flags;
 378         int index;
 379         int idle_hardware_channels;
 380         union cvmx_usbcx_hprt usbcx_hprt;
 381         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
 382         int indent;
 383         struct cvmx_usb_port_status port_status;
 384         struct list_head idle_pipes;
 385         struct list_head active_pipes[4];
 386         uint64_t frame_number;
 387         struct cvmx_usb_transaction *active_split;
 388         struct cvmx_usb_tx_fifo periodic;
 389         struct cvmx_usb_tx_fifo nonperiodic;
 390 };
 391
 392 struct octeon_hcd {
 393         spinlock_t lock;
 394         struct cvmx_usb_state usb;
 395 };
 396
 397 /* This macro spins on a field waiting for it to reach a value */
 398 #define CVMX_WAIT_FOR_FIELD32(address, type, field, op, value, timeout_usec)\
 399         ({int result;                                                       \
 400         do {                                                                \
 401                 uint64_t done = cvmx_get_cycle() + (uint64_t)timeout_usec * \
 402                         octeon_get_clock_rate() / 1000000;                  \
 403                 type c;                                                     \
 404                 while (1) {                                                 \
 405                         c.u32 = __cvmx_usb_read_csr32(usb, address);        \
 406                         if (c.s.field op (value)) {                         \
 407                                 result = 0;                                 \
 408                                 break;                                      \
 409                         } else if (cvmx_get_cycle() > done) {               \
 410                                 result = -1;                                \
 411                                 break;                                      \
 412                         } else                                              \
 413                                 cvmx_wait(100);                             \
 414                 }                                                           \
 415         } while (0);                                                        \
 416         result; })
 417
 418 /*
 419  * This macro logically sets a single field in a CSR. It does the sequence
 420  * read, modify, and write
 421  */
 422 #define USB_SET_FIELD32(address, type, field, value)            \
 423         do {                                                    \
 424                 type c;                                         \
 425                 c.u32 = __cvmx_usb_read_csr32(usb, address);    \
 426                 c.s.field = value;                              \
 427                 __cvmx_usb_write_csr32(usb, address, c.u32);    \
 428         } while (0)
 429
 430 /* Returns the IO address to push/pop stuff data from the FIFOs */
 431 #define USB_FIFO_ADDRESS(channel, usb_index) \
 432         (CVMX_USBCX_GOTGCTL(usb_index) + ((channel)+1)*0x1000)
 433
 434 /**
 435  * struct octeon_temp_buffer - a bounce buffer for USB transfers
 436  * @temp_buffer: the newly allocated temporary buffer (including meta-data)
 437  * @orig_buffer: the original buffer passed by the USB stack
 438  * @data:        the newly allocated temporary buffer (excluding meta-data)
 439  *
 440  * Both the DMA engine and FIFO mode will always transfer full 32-bit words. If
 441  * the buffer is too short, we need to allocate a temporary one, and this struct
 442  * represents it.
 443  */
 444 struct octeon_temp_buffer {
 445         void *temp_buffer;
 446         void *orig_buffer;
 447         u8 data[0];
 448 };
 449
 450 static inline struct octeon_hcd *cvmx_usb_to_octeon(struct cvmx_usb_state *p)
 451 {
 452         return container_of(p, struct octeon_hcd, usb);
 453 }
 454
 455 static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
 456 {
 457         return container_of((void *)p, struct usb_hcd, hcd_priv);
 458 }
 459
 460 /**
 461  * octeon_alloc_temp_buffer - allocate a temporary buffer for USB transfer
 462  *                            (if needed)
 463  * @urb:        URB.
 464  * @mem_flags:  Memory allocation flags.
 465  *
 466  * This function allocates a temporary bounce buffer whenever it's needed
 467  * due to HW limitations.
 468  */
 469 static int octeon_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)
 470 {
 471         struct octeon_temp_buffer *temp;
 472
 473         if (urb->num_sgs || urb->sg ||
 474             (urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) ||
 475             !(urb->transfer_buffer_length % sizeof(u32)))
 476                 return 0;
 477
 478         temp = kmalloc(ALIGN(urb->transfer_buffer_length, sizeof(u32)) +
 479                        sizeof(*temp), mem_flags);
 480         if (!temp)
 481                 return -ENOMEM;
 482
 483         temp->temp_buffer = temp;
 484         temp->orig_buffer = urb->transfer_buffer;
 485         if (usb_urb_dir_out(urb))
 486                 memcpy(temp->data, urb->transfer_buffer,
 487                        urb->transfer_buffer_length);
 488         urb->transfer_buffer = temp->data;
 489         urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;
 490
 491         return 0;
 492 }
 493
 494 /**
 495  * octeon_free_temp_buffer - free a temporary buffer used by USB transfers.
 496  * @urb: URB.
 497  *
 498  * Frees a buffer allocated by octeon_alloc_temp_buffer().
 499  */
 500 static void octeon_free_temp_buffer(struct urb *urb)
 501 {
 502         struct octeon_temp_buffer *temp;
 503
 504         if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))
 505                 return;
 506
 507         temp = container_of(urb->transfer_buffer, struct octeon_temp_buffer,
 508                             data);
 509         if (usb_urb_dir_in(urb))
 510                 memcpy(temp->orig_buffer, urb->transfer_buffer,
 511                        urb->actual_length);
 512         urb->transfer_buffer = temp->orig_buffer;
 513         urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;
 514         kfree(temp->temp_buffer);
 515 }
 516
 517 /**
 518  * octeon_map_urb_for_dma - Octeon-specific map_urb_for_dma().
 519  * @hcd:        USB HCD structure.
 520  * @urb:        URB.
 521  * @mem_flags:  Memory allocation flags.
 522  */
 523 static int octeon_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
 524                                   gfp_t mem_flags)
 525 {
 526         int ret;
 527
 528         ret = octeon_alloc_temp_buffer(urb, mem_flags);
 529         if (ret)
 530                 return ret;
 531
 532         ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
 533         if (ret)
 534                 octeon_free_temp_buffer(urb);
 535
 536         return ret;
 537 }
 538
 539 /**
 540  * octeon_unmap_urb_for_dma - Octeon-specific unmap_urb_for_dma()
 541  * @hcd:        USB HCD structure.
 542  * @urb:        URB.
 543  */
 544 static void octeon_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
 545 {
 546         usb_hcd_unmap_urb_for_dma(hcd, urb);
 547         octeon_free_temp_buffer(urb);
 548 }
 549
 550 /**
 551  * Read a USB 32bit CSR. It performs the necessary address swizzle
 552  * for 32bit CSRs and logs the value in a readable format if
 553  * debugging is on.
 554  *
 555  * @usb:     USB block this access is for
 556  * @address: 64bit address to read
 557  *
 558  * Returns: Result of the read
 559  */
 560 static inline uint32_t __cvmx_usb_read_csr32(struct cvmx_usb_state *usb,
 561                                              uint64_t address)
 562 {
 563         uint32_t result = cvmx_read64_uint32(address ^ 4);
 564         return result;
 565 }
 566
 567
 568 /**
 569  * Write a USB 32bit CSR. It performs the necessary address
 570  * swizzle for 32bit CSRs and logs the value in a readable format
 571  * if debugging is on.
 572  *
 573  * @usb:     USB block this access is for
 574  * @address: 64bit address to write
 575  * @value:   Value to write
 576  */
 577 static inline void __cvmx_usb_write_csr32(struct cvmx_usb_state *usb,
 578                                           uint64_t address, uint32_t value)
 579 {
 580         cvmx_write64_uint32(address ^ 4, value);
 581         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 582 }
 583
 584 /**
 585  * Return non zero if this pipe connects to a non HIGH speed
 586  * device through a high speed hub.
 587  *
 588  * @usb:    USB block this access is for
 589  * @pipe:   Pipe to check
 590  *
 591  * Returns: Non zero if we need to do split transactions
 592  */
 593 static inline int __cvmx_usb_pipe_needs_split(struct cvmx_usb_state *usb,
 594                                               struct cvmx_usb_pipe *pipe)
 595 {
 596         return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
 597                usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
 598 }
 599
 600
 601 /**
 602  * Trivial utility function to return the correct PID for a pipe
 603  *
 604  * @pipe:   pipe to check
 605  *
 606  * Returns: PID for pipe
 607  */
 608 static inline int __cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
 609 {
 610         if (pipe->pid_toggle)
 611                 return 2; /* Data1 */
 612         return 0; /* Data0 */
 613 }
 614
 615 /**
 616  * Initialize a USB port for use. This must be called before any
 617  * other access to the Octeon USB port is made. The port starts
 618  * off in the disabled state.
 619  *
 620  * @usb:         Pointer to an empty struct cvmx_usb_state
 621  *               that will be populated by the initialize call.
 622  *               This structure is then passed to all other USB
 623  *               functions.
 624  * @usb_port_number:
 625  *               Which Octeon USB port to initialize.
 626  *
 627  * Returns: 0 or a negative error code.
 628  */
 629 static int cvmx_usb_initialize(struct cvmx_usb_state *usb,
 630                                int usb_port_number,
 631                                enum cvmx_usb_initialize_flags flags)
 632 {
 633         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 634         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
 635         int i;
 636
 637         memset(usb, 0, sizeof(*usb));
 638         usb->init_flags = flags;
 639
 640         /* Initialize the USB state structure */
 641         usb->index = usb_port_number;
 642         INIT_LIST_HEAD(&usb->idle_pipes);
 643         for (i = 0; i < ARRAY_SIZE(usb->active_pipes); i++)
 644                 INIT_LIST_HEAD(&usb->active_pipes[i]);
 645
 646         /*
 647          * Power On Reset and PHY Initialization
 648          *
 649          * 1. Wait for DCOK to assert (nothing to do)
 650          *
 651          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
 652          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
 653          */
 654         usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
 655         usbn_clk_ctl.s.por = 1;
 656         usbn_clk_ctl.s.hrst = 0;
 657         usbn_clk_ctl.s.prst = 0;
 658         usbn_clk_ctl.s.hclk_rst = 0;
 659         usbn_clk_ctl.s.enable = 0;
 660         /*
 661          * 2b. Select the USB reference clock/crystal parameters by writing
 662          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
 663          */
 664         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 665                 /*
 666                  * The USB port uses 12/24/48MHz 2.5V board clock
 667                  * source at USB_XO. USB_XI should be tied to GND.
 668                  * Most Octeon evaluation boards require this setting
 669                  */
 670                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
 671                     OCTEON_IS_MODEL(OCTEON_CN56XX) ||
 672                     OCTEON_IS_MODEL(OCTEON_CN50XX))
 673                         /* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
 674                         usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
 675                 else
 676                         /* From CN52XX manual */
 677                         usbn_clk_ctl.s.p_rtype = 1;
 678
 679                 switch (flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
 680                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
 681                         usbn_clk_ctl.s.p_c_sel = 0;
 682                         break;
 683                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
 684                         usbn_clk_ctl.s.p_c_sel = 1;
 685                         break;
 686                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
 687                         usbn_clk_ctl.s.p_c_sel = 2;
 688                         break;
 689                 }
 690         } else {
 691                 /*
 692                  * The USB port uses a 12MHz crystal as clock source
 693                  * at USB_XO and USB_XI
 694                  */
 695                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
 696                         /* From CN31XX,CN30XX manual */
 697                         usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
 698                 else
 699                         /* From CN56XX,CN52XX,CN50XX manuals. */
 700                         usbn_clk_ctl.s.p_rtype = 0;
 701
 702                 usbn_clk_ctl.s.p_c_sel = 0;
 703         }
 704         /*
 705          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
 706          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
 707          *     such that USB is as close as possible to 125Mhz
 708          */
 709         {
 710                 int divisor = DIV_ROUND_UP(octeon_get_clock_rate(), 125000000);
 711                 /* Lower than 4 doesn't seem to work properly */
 712                 if (divisor < 4)
 713                         divisor = 4;
 714                 usbn_clk_ctl.s.divide = divisor;
 715                 usbn_clk_ctl.s.divide2 = 0;
 716         }
 717         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 718         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
 719         usbn_clk_ctl.s.hclk_rst = 1;
 720         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 721         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
 722         cvmx_wait(64);
 723         /*
 724          * 3. Program the power-on reset field in the USBN clock-control
 725          *    register:
 726          *    USBN_CLK_CTL[POR] = 0
 727          */
 728         usbn_clk_ctl.s.por = 0;
 729         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 730         /* 4. Wait 1 ms for PHY clock to start */
 731         mdelay(1);
 732         /*
 733          * 5. Program the Reset input from automatic test equipment field in the
 734          *    USBP control and status register:
 735          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
 736          */
 737         usbn_usbp_ctl_status.u64 =
 738                 cvmx_read64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index));
 739         usbn_usbp_ctl_status.s.ate_reset = 1;
 740         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 741                             usbn_usbp_ctl_status.u64);
 742         /* 6. Wait 10 cycles */
 743         cvmx_wait(10);
 744         /*
 745          * 7. Clear ATE_RESET field in the USBN clock-control register:
 746          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
 747          */
 748         usbn_usbp_ctl_status.s.ate_reset = 0;
 749         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 750                             usbn_usbp_ctl_status.u64);
 751         /*
 752          * 8. Program the PHY reset field in the USBN clock-control register:
 753          *    USBN_CLK_CTL[PRST] = 1
 754          */
 755         usbn_clk_ctl.s.prst = 1;
 756         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 757         /*
 758          * 9. Program the USBP control and status register to select host or
 759          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
 760          *    device
 761          */
 762         usbn_usbp_ctl_status.s.hst_mode = 0;
 763         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 764                             usbn_usbp_ctl_status.u64);
 765         /* 10. Wait 1 us */
 766         udelay(1);
 767         /*
 768          * 11. Program the hreset_n field in the USBN clock-control register:
 769          *     USBN_CLK_CTL[HRST] = 1
 770          */
 771         usbn_clk_ctl.s.hrst = 1;
 772         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 773         /* 12. Proceed to USB core initialization */
 774         usbn_clk_ctl.s.enable = 1;
 775         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 776         udelay(1);
 777
 778         /*
 779          * USB Core Initialization
 780          *
 781          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
 782          *    determine USB core configuration parameters.
 783          *
 784          *    Nothing needed
 785          *
 786          * 2. Program the following fields in the global AHB configuration
 787          *    register (USBC_GAHBCFG)
 788          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
 789          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
 790          *    Nonperiodic TxFIFO empty level (slave mode only),
 791          *    USBC_GAHBCFG[NPTXFEMPLVL]
 792          *    Periodic TxFIFO empty level (slave mode only),
 793          *    USBC_GAHBCFG[PTXFEMPLVL]
 794          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 795          */
 796         {
 797                 union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
 798                 /* Due to an errata, CN31XX doesn't support DMA */
 799                 if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 800                         usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
 801                 usbcx_gahbcfg.u32 = 0;
 802                 usbcx_gahbcfg.s.dmaen = !(usb->init_flags &
 803                                           CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
 804                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 805                         /* Only use one channel with non DMA */
 806                         usb->idle_hardware_channels = 0x1;
 807                 else if (OCTEON_IS_MODEL(OCTEON_CN5XXX))
 808                         /* CN5XXX have an errata with channel 3 */
 809                         usb->idle_hardware_channels = 0xf7;
 810                 else
 811                         usb->idle_hardware_channels = 0xff;
 812                 usbcx_gahbcfg.s.hbstlen = 0;
 813                 usbcx_gahbcfg.s.nptxfemplvl = 1;
 814                 usbcx_gahbcfg.s.ptxfemplvl = 1;
 815                 usbcx_gahbcfg.s.glblintrmsk = 1;
 816                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
 817                                        usbcx_gahbcfg.u32);
 818         }
 819         /*
 820          * 3. Program the following fields in USBC_GUSBCFG register.
 821          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
 822          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
 823          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
 824          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
 825          */
 826         {
 827                 union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
 828
 829                 usbcx_gusbcfg.u32 = __cvmx_usb_read_csr32(usb,
 830                                 CVMX_USBCX_GUSBCFG(usb->index));
 831                 usbcx_gusbcfg.s.toutcal = 0;
 832                 usbcx_gusbcfg.s.ddrsel = 0;
 833                 usbcx_gusbcfg.s.usbtrdtim = 0x5;
 834                 usbcx_gusbcfg.s.phylpwrclksel = 0;
 835                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
 836                                        usbcx_gusbcfg.u32);
 837         }
 838         /*
 839          * 4. The software must unmask the following bits in the USBC_GINTMSK
 840          *    register.
 841          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
 842          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
 843          */
 844         {
 845                 union cvmx_usbcx_gintmsk usbcx_gintmsk;
 846                 int channel;
 847
 848                 usbcx_gintmsk.u32 = __cvmx_usb_read_csr32(usb,
 849                                 CVMX_USBCX_GINTMSK(usb->index));
 850                 usbcx_gintmsk.s.otgintmsk = 1;
 851                 usbcx_gintmsk.s.modemismsk = 1;
 852                 usbcx_gintmsk.s.hchintmsk = 1;
 853                 usbcx_gintmsk.s.sofmsk = 0;
 854                 /* We need RX FIFO interrupts if we don't have DMA */
 855                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 856                         usbcx_gintmsk.s.rxflvlmsk = 1;
 857                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
 858                                        usbcx_gintmsk.u32);
 859
 860                 /*
 861                  * Disable all channel interrupts. We'll enable them per channel
 862                  * later.
 863                  */
 864                 for (channel = 0; channel < 8; channel++)
 865                         __cvmx_usb_write_csr32(usb,
 866                                 CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
 867         }
 868
 869         {
 870                 /*
 871                  * Host Port Initialization
 872                  *
 873                  * 1. Program the host-port interrupt-mask field to unmask,
 874                  *    USBC_GINTMSK[PRTINT] = 1
 875                  */
 876                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 877                                 union cvmx_usbcx_gintmsk, prtintmsk, 1);
 878                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 879                                 union cvmx_usbcx_gintmsk, disconnintmsk, 1);
 880                 /*
 881                  * 2. Program the USBC_HCFG register to select full-speed host
 882                  *    or high-speed host.
 883                  */
 884                 {
 885                         union cvmx_usbcx_hcfg usbcx_hcfg;
 886
 887                         usbcx_hcfg.u32 = __cvmx_usb_read_csr32(usb,
 888                                         CVMX_USBCX_HCFG(usb->index));
 889                         usbcx_hcfg.s.fslssupp = 0;
 890                         usbcx_hcfg.s.fslspclksel = 0;
 891                         __cvmx_usb_write_csr32(usb,
 892                                         CVMX_USBCX_HCFG(usb->index),
 893                                         usbcx_hcfg.u32);
 894                 }
 895                 /*
 896                  * 3. Program the port power bit to drive VBUS on the USB,
 897                  *    USBC_HPRT[PRTPWR] = 1
 898                  */
 899                 USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index),
 900                                 union cvmx_usbcx_hprt, prtpwr, 1);
 901
 902                 /*
 903                  * Steps 4-15 from the manual are done later in the port enable
 904                  */
 905         }
 906
 907         return 0;
 908 }
 909
 910
 911 /**
 912  * Shutdown a USB port after a call to cvmx_usb_initialize().
 913  * The port should be disabled with all pipes closed when this
 914  * function is called.
 915  *
 916  * @usb: USB device state populated by cvmx_usb_initialize().
 917  *
 918  * Returns: 0 or a negative error code.
 919  */
 920 static int cvmx_usb_shutdown(struct cvmx_usb_state *usb)
 921 {
 922         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 923
 924         /* Make sure all pipes are closed */
 925         if (!list_empty(&usb->idle_pipes) ||
 926             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
 927             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
 928             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
 929             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
 930                 return -EBUSY;
 931
 932         /* Disable the clocks and put them in power on reset */
 933         usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
 934         usbn_clk_ctl.s.enable = 1;
 935         usbn_clk_ctl.s.por = 1;
 936         usbn_clk_ctl.s.hclk_rst = 1;
 937         usbn_clk_ctl.s.prst = 0;
 938         usbn_clk_ctl.s.hrst = 0;
 939         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 940         return 0;
 941 }
 942
 943
 944 /**
 945  * Enable a USB port. After this call succeeds, the USB port is
 946  * online and servicing requests.
 947  *
 948  * @usb: USB device state populated by cvmx_usb_initialize().
 949  *
 950  * Returns: 0 or a negative error code.
 951  */
 952 static int cvmx_usb_enable(struct cvmx_usb_state *usb)
 953 {
 954         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
 955
 956         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb,
 957                         CVMX_USBCX_HPRT(usb->index));
 958
 959         /*
 960          * If the port is already enabled the just return. We don't need to do
 961          * anything
 962          */
 963         if (usb->usbcx_hprt.s.prtena)
 964                 return 0;
 965
 966         /* If there is nothing plugged into the port then fail immediately */
 967         if (!usb->usbcx_hprt.s.prtconnsts)
 968                 return -ETIMEDOUT;
 969
 970         /* Program the port reset bit to start the reset process */
 971         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
 972                         prtrst, 1);
 973
 974         /*
 975          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
 976          * process to complete.
 977          */
 978         mdelay(50);
 979
 980         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
 981         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
 982                         prtrst, 0);
 983
 984         /* Wait for the USBC_HPRT[PRTENA]. */
 985         if (CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_HPRT(usb->index),
 986                                 union cvmx_usbcx_hprt, prtena, ==, 1, 100000))
 987                 return -ETIMEDOUT;
 988
 989         /*
 990          * Read the port speed field to get the enumerated speed,
 991          * USBC_HPRT[PRTSPD].
 992          */
 993         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb,
 994                         CVMX_USBCX_HPRT(usb->index));
 995         usbcx_ghwcfg3.u32 = __cvmx_usb_read_csr32(usb,
 996                         CVMX_USBCX_GHWCFG3(usb->index));
 997
 998         /*
 999          * 13. Program the USBC_GRXFSIZ register to select the size of the
1000          *     receive FIFO (25%).
1001          */
1002         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index),
1003                         union cvmx_usbcx_grxfsiz, rxfdep,
1004                         usbcx_ghwcfg3.s.dfifodepth / 4);
1005         /*
1006          * 14. Program the USBC_GNPTXFSIZ register to select the size and the
1007          *     start address of the non- periodic transmit FIFO for nonperiodic
1008          *     transactions (50%).
1009          */
1010         {
1011                 union cvmx_usbcx_gnptxfsiz siz;
1012
1013                 siz.u32 = __cvmx_usb_read_csr32(usb,
1014                                 CVMX_USBCX_GNPTXFSIZ(usb->index));
1015                 siz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
1016                 siz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
1017                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index),
1018                                        siz.u32);
1019         }
1020         /*
1021          * 15. Program the USBC_HPTXFSIZ register to select the size and start
1022          *     address of the periodic transmit FIFO for periodic transactions
1023          *     (25%).
1024          */
1025         {
1026                 union cvmx_usbcx_hptxfsiz siz;
1027
1028                 siz.u32 = __cvmx_usb_read_csr32(usb,
1029                                 CVMX_USBCX_HPTXFSIZ(usb->index));
1030                 siz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
1031                 siz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
1032                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index),
1033                                        siz.u32);
1034         }
1035         /* Flush all FIFOs */
1036         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1037                         union cvmx_usbcx_grstctl, txfnum, 0x10);
1038         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1039                         union cvmx_usbcx_grstctl, txfflsh, 1);
1040         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1041                               union cvmx_usbcx_grstctl,
1042                               txfflsh, ==, 0, 100);
1043         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1044                         union cvmx_usbcx_grstctl, rxfflsh, 1);
1045         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1046                               union cvmx_usbcx_grstctl,
1047                               rxfflsh, ==, 0, 100);
1048
1049         return 0;
1050 }
1051
1052
1053 /**
1054  * Disable a USB port. After this call the USB port will not
1055  * generate data transfers and will not generate events.
1056  * Transactions in process will fail and call their
1057  * associated callbacks.
1058  *
1059  * @usb: USB device state populated by cvmx_usb_initialize().
1060  *
1061  * Returns: 0 or a negative error code.
1062  */
1063 static int cvmx_usb_disable(struct cvmx_usb_state *usb)
1064 {
1065         /* Disable the port */
1066         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1067                         prtena, 1);
1068         return 0;
1069 }
1070
1071
1072 /**
1073  * Get the current state of the USB port. Use this call to
1074  * determine if the usb port has anything connected, is enabled,
1075  * or has some sort of error condition. The return value of this
1076  * call has "changed" bits to signal of the value of some fields
1077  * have changed between calls.
1078  *
1079  * @usb: USB device state populated by cvmx_usb_initialize().
1080  *
1081  * Returns: Port status information
1082  */
1083 static struct cvmx_usb_port_status cvmx_usb_get_status(
1084                 struct cvmx_usb_state *usb)
1085 {
1086         union cvmx_usbcx_hprt usbc_hprt;
1087         struct cvmx_usb_port_status result;
1088
1089         memset(&result, 0, sizeof(result));
1090
1091         usbc_hprt.u32 = __cvmx_usb_read_csr32(usb,
1092                         CVMX_USBCX_HPRT(usb->index));
1093         result.port_enabled = usbc_hprt.s.prtena;
1094         result.port_over_current = usbc_hprt.s.prtovrcurract;
1095         result.port_powered = usbc_hprt.s.prtpwr;
1096         result.port_speed = usbc_hprt.s.prtspd;
1097         result.connected = usbc_hprt.s.prtconnsts;
1098         result.connect_change =
1099                 (result.connected != usb->port_status.connected);
1100
1101         return result;
1102 }
1103
1104 /**
1105  * Open a virtual pipe between the host and a USB device. A pipe
1106  * must be opened before data can be transferred between a device
1107  * and Octeon.
1108  *
1109  * @usb:             USB device state populated by cvmx_usb_initialize().
1110  * @device_addr:
1111  *                   USB device address to open the pipe to
1112  *                   (0-127).
1113  * @endpoint_num:
1114  *                   USB endpoint number to open the pipe to
1115  *                   (0-15).
1116  * @device_speed:
1117  *                   The speed of the device the pipe is going
1118  *                   to. This must match the device's speed,
1119  *                   which may be different than the port speed.
1120  * @max_packet:      The maximum packet length the device can
1121  *                   transmit/receive (low speed=0-8, full
1122  *                   speed=0-1023, high speed=0-1024). This value
1123  *                   comes from the standard endpoint descriptor
1124  *                   field wMaxPacketSize bits <10:0>.
1125  * @transfer_type:
1126  *                   The type of transfer this pipe is for.
1127  * @transfer_dir:
1128  *                   The direction the pipe is in. This is not
1129  *                   used for control pipes.
1130  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1131  *                   this is how often the transfer is scheduled
1132  *                   for. All other transfers should specify
1133  *                   zero. The units are in frames (8000/sec at
1134  *                   high speed, 1000/sec for full speed).
1135  * @multi_count:
1136  *                   For high speed devices, this is the maximum
1137  *                   allowed number of packet per microframe.
1138  *                   Specify zero for non high speed devices. This
1139  *                   value comes from the standard endpoint descriptor
1140  *                   field wMaxPacketSize bits <12:11>.
1141  * @hub_device_addr:
1142  *                   Hub device address this device is connected
1143  *                   to. Devices connected directly to Octeon
1144  *                   use zero. This is only used when the device
1145  *                   is full/low speed behind a high speed hub.
1146  *                   The address will be of the high speed hub,
1147  *                   not and full speed hubs after it.
1148  * @hub_port:        Which port on the hub the device is
1149  *                   connected. Use zero for devices connected
1150  *                   directly to Octeon. Like hub_device_addr,
1151  *                   this is only used for full/low speed
1152  *                   devices behind a high speed hub.
1153  *
1154  * Returns: A non-NULL value is a pipe. NULL means an error.
1155  */
1156 static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct cvmx_usb_state *usb,
1157                                                 int device_addr,
1158                                                 int endpoint_num,
1159                                                 enum cvmx_usb_speed
1160                                                         device_speed,
1161                                                 int max_packet,
1162                                                 enum cvmx_usb_transfer
1163                                                         transfer_type,
1164                                                 enum cvmx_usb_direction
1165                                                         transfer_dir,
1166                                                 int interval, int multi_count,
1167                                                 int hub_device_addr,
1168                                                 int hub_port)
1169 {
1170         struct cvmx_usb_pipe *pipe;
1171
1172         pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
1173         if (!pipe)
1174                 return NULL;
1175         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1176                 (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1177                 (transfer_type == CVMX_USB_TRANSFER_BULK))
1178                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
1179         pipe->device_addr = device_addr;
1180         pipe->endpoint_num = endpoint_num;
1181         pipe->device_speed = device_speed;
1182         pipe->max_packet = max_packet;
1183         pipe->transfer_type = transfer_type;
1184         pipe->transfer_dir = transfer_dir;
1185         INIT_LIST_HEAD(&pipe->transactions);
1186
1187         /*
1188          * All pipes use interval to rate limit NAK processing. Force an
1189          * interval if one wasn't supplied
1190          */
1191         if (!interval)
1192                 interval = 1;
1193         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1194                 pipe->interval = interval*8;
1195                 /* Force start splits to be schedule on uFrame 0 */
1196                 pipe->next_tx_frame = ((usb->frame_number+7)&~7) +
1197                                         pipe->interval;
1198         } else {
1199                 pipe->interval = interval;
1200                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1201         }
1202         pipe->multi_count = multi_count;
1203         pipe->hub_device_addr = hub_device_addr;
1204         pipe->hub_port = hub_port;
1205         pipe->pid_toggle = 0;
1206         pipe->split_sc_frame = -1;
1207         list_add_tail(&pipe->node, &usb->idle_pipes);
1208
1209         /*
1210          * We don't need to tell the hardware about this pipe yet since
1211          * it doesn't have any submitted requests
1212          */
1213
1214         return pipe;
1215 }
1216
1217
1218 /**
1219  * Poll the RX FIFOs and remove data as needed. This function is only used
1220  * in non DMA mode. It is very important that this function be called quickly
1221  * enough to prevent FIFO overflow.
1222  *
1223  * @usb:        USB device state populated by cvmx_usb_initialize().
1224  */
1225 static void __cvmx_usb_poll_rx_fifo(struct cvmx_usb_state *usb)
1226 {
1227         union cvmx_usbcx_grxstsph rx_status;
1228         int channel;
1229         int bytes;
1230         uint64_t address;
1231         uint32_t *ptr;
1232
1233         rx_status.u32 = __cvmx_usb_read_csr32(usb,
1234                         CVMX_USBCX_GRXSTSPH(usb->index));
1235         /* Only read data if IN data is there */
1236         if (rx_status.s.pktsts != 2)
1237                 return;
1238         /* Check if no data is available */
1239         if (!rx_status.s.bcnt)
1240                 return;
1241
1242         channel = rx_status.s.chnum;
1243         bytes = rx_status.s.bcnt;
1244         if (!bytes)
1245                 return;
1246
1247         /* Get where the DMA engine would have written this data */
1248         address = cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) +
1249                                      channel * 8);
1250
1251         ptr = cvmx_phys_to_ptr(address);
1252         cvmx_write64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel * 8,
1253                             address + bytes);
1254
1255         /* Loop writing the FIFO data for this packet into memory */
1256         while (bytes > 0) {
1257                 *ptr++ = __cvmx_usb_read_csr32(usb,
1258                                 USB_FIFO_ADDRESS(channel, usb->index));
1259                 bytes -= 4;
1260         }
1261         CVMX_SYNCW;
1262 }
1263
1264
1265 /**
1266  * Fill the TX hardware fifo with data out of the software
1267  * fifos
1268  *
1269  * @usb:            USB device state populated by cvmx_usb_initialize().
1270  * @fifo:           Software fifo to use
1271  * @available:      Amount of space in the hardware fifo
1272  *
1273  * Returns: Non zero if the hardware fifo was too small and needs
1274  *          to be serviced again.
1275  */
1276 static int __cvmx_usb_fill_tx_hw(struct cvmx_usb_state *usb,
1277                                  struct cvmx_usb_tx_fifo *fifo, int available)
1278 {
1279         /*
1280          * We're done either when there isn't anymore space or the software FIFO
1281          * is empty
1282          */
1283         while (available && (fifo->head != fifo->tail)) {
1284                 int i = fifo->tail;
1285                 const uint32_t *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1286                 uint64_t csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel,
1287                                                         usb->index) ^ 4;
1288                 int words = available;
1289
1290                 /* Limit the amount of data to waht the SW fifo has */
1291                 if (fifo->entry[i].size <= available) {
1292                         words = fifo->entry[i].size;
1293                         fifo->tail++;
1294                         if (fifo->tail > MAX_CHANNELS)
1295                                 fifo->tail = 0;
1296                 }
1297
1298                 /* Update the next locations and counts */
1299                 available -= words;
1300                 fifo->entry[i].address += words * 4;
1301                 fifo->entry[i].size -= words;
1302
1303                 /*
1304                  * Write the HW fifo data. The read every three writes is due
1305                  * to an errata on CN3XXX chips
1306                  */
1307                 while (words > 3) {
1308                         cvmx_write64_uint32(csr_address, *ptr++);
1309                         cvmx_write64_uint32(csr_address, *ptr++);
1310                         cvmx_write64_uint32(csr_address, *ptr++);
1311                         cvmx_read64_uint64(
1312                                         CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1313                         words -= 3;
1314                 }
1315                 cvmx_write64_uint32(csr_address, *ptr++);
1316                 if (--words) {
1317                         cvmx_write64_uint32(csr_address, *ptr++);
1318                         if (--words)
1319                                 cvmx_write64_uint32(csr_address, *ptr++);
1320                 }
1321                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1322         }
1323         return fifo->head != fifo->tail;
1324 }
1325
1326
1327 /**
1328  * Check the hardware FIFOs and fill them as needed
1329  *
1330  * @usb:        USB device state populated by cvmx_usb_initialize().
1331  */
1332 static void __cvmx_usb_poll_tx_fifo(struct cvmx_usb_state *usb)
1333 {
1334         if (usb->periodic.head != usb->periodic.tail) {
1335                 union cvmx_usbcx_hptxsts tx_status;
1336
1337                 tx_status.u32 = __cvmx_usb_read_csr32(usb,
1338                                 CVMX_USBCX_HPTXSTS(usb->index));
1339                 if (__cvmx_usb_fill_tx_hw(usb, &usb->periodic,
1340                                           tx_status.s.ptxfspcavail))
1341                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1342                                         union cvmx_usbcx_gintmsk,
1343                                         ptxfempmsk, 1);
1344                 else
1345                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1346                                         union cvmx_usbcx_gintmsk,
1347                                         ptxfempmsk, 0);
1348         }
1349
1350         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1351                 union cvmx_usbcx_gnptxsts tx_status;
1352
1353                 tx_status.u32 = __cvmx_usb_read_csr32(usb,
1354                                 CVMX_USBCX_GNPTXSTS(usb->index));
1355                 if (__cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic,
1356                                           tx_status.s.nptxfspcavail))
1357                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1358                                         union cvmx_usbcx_gintmsk,
1359                                         nptxfempmsk, 1);
1360                 else
1361                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1362                                         union cvmx_usbcx_gintmsk,
1363                                         nptxfempmsk, 0);
1364         }
1365 }
1366
1367
1368 /**
1369  * Fill the TX FIFO with an outgoing packet
1370  *
1371  * @usb:          USB device state populated by cvmx_usb_initialize().
1372  * @channel:      Channel number to get packet from
1373  */
1374 static void __cvmx_usb_fill_tx_fifo(struct cvmx_usb_state *usb, int channel)
1375 {
1376         union cvmx_usbcx_hccharx hcchar;
1377         union cvmx_usbcx_hcspltx usbc_hcsplt;
1378         union cvmx_usbcx_hctsizx usbc_hctsiz;
1379         struct cvmx_usb_tx_fifo *fifo;
1380
1381         /* We only need to fill data on outbound channels */
1382         hcchar.u32 = __cvmx_usb_read_csr32(usb,
1383                         CVMX_USBCX_HCCHARX(channel, usb->index));
1384         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1385                 return;
1386
1387         /* OUT Splits only have data on the start and not the complete */
1388         usbc_hcsplt.u32 = __cvmx_usb_read_csr32(usb,
1389                         CVMX_USBCX_HCSPLTX(channel, usb->index));
1390         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1391                 return;
1392
1393         /*
1394          * Find out how many bytes we need to fill and convert it into 32bit
1395          * words.
1396          */
1397         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
1398                         CVMX_USBCX_HCTSIZX(channel, usb->index));
1399         if (!usbc_hctsiz.s.xfersize)
1400                 return;
1401
1402         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1403                 (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1404                 fifo = &usb->periodic;
1405         else
1406                 fifo = &usb->nonperiodic;
1407
1408         fifo->entry[fifo->head].channel = channel;
1409         fifo->entry[fifo->head].address =
1410                 cvmx_read64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
1411                                    channel * 8);
1412         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize+3)>>2;
1413         fifo->head++;
1414         if (fifo->head > MAX_CHANNELS)
1415                 fifo->head = 0;
1416
1417         __cvmx_usb_poll_tx_fifo(usb);
1418 }
1419
1420 /**
1421  * Perform channel specific setup for Control transactions. All
1422  * the generic stuff will already have been done in
1423  * __cvmx_usb_start_channel()
1424  *
1425  * @usb:          USB device state populated by cvmx_usb_initialize().
1426  * @channel:      Channel to setup
1427  * @pipe:         Pipe for control transaction
1428  */
1429 static void __cvmx_usb_start_channel_control(struct cvmx_usb_state *usb,
1430                                              int channel,
1431                                              struct cvmx_usb_pipe *pipe)
1432 {
1433         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
1434         struct usb_hcd *hcd = octeon_to_hcd(priv);
1435         struct device *dev = hcd->self.controller;
1436         struct cvmx_usb_transaction *transaction =
1437                 list_first_entry(&pipe->transactions, typeof(*transaction),
1438                                  node);
1439         struct usb_ctrlrequest *header =
1440                 cvmx_phys_to_ptr(transaction->control_header);
1441         int bytes_to_transfer = transaction->buffer_length -
1442                 transaction->actual_bytes;
1443         int packets_to_transfer;
1444         union cvmx_usbcx_hctsizx usbc_hctsiz;
1445
1446         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
1447                         CVMX_USBCX_HCTSIZX(channel, usb->index));
1448
1449         switch (transaction->stage) {
1450         case CVMX_USB_STAGE_NON_CONTROL:
1451         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1452                 dev_err(dev, "%s: ERROR - Non control stage\n", __func__);
1453                 break;
1454         case CVMX_USB_STAGE_SETUP:
1455                 usbc_hctsiz.s.pid = 3; /* Setup */
1456                 bytes_to_transfer = sizeof(*header);
1457                 /* All Control operations start with a setup going OUT */
1458                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1459                                 union cvmx_usbcx_hccharx, epdir,
1460                                 CVMX_USB_DIRECTION_OUT);
1461                 /*
1462                  * Setup send the control header instead of the buffer data. The
1463                  * buffer data will be used in the next stage
1464                  */
1465                 cvmx_write64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
1466                                         channel * 8,
1467                                     transaction->control_header);
1468                 break;
1469         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1470                 usbc_hctsiz.s.pid = 3; /* Setup */
1471                 bytes_to_transfer = 0;
1472                 /* All Control operations start with a setup going OUT */
1473                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1474                                 union cvmx_usbcx_hccharx, epdir,
1475                                 CVMX_USB_DIRECTION_OUT);
1476
1477                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1478                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1479                 break;
1480         case CVMX_USB_STAGE_DATA:
1481                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1482                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1483                         if (header->bRequestType & USB_DIR_IN)
1484                                 bytes_to_transfer = 0;
1485                         else if (bytes_to_transfer > pipe->max_packet)
1486                                 bytes_to_transfer = pipe->max_packet;
1487                 }
1488                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1489                                 union cvmx_usbcx_hccharx, epdir,
1490                                 ((header->bRequestType & USB_DIR_IN) ?
1491                                         CVMX_USB_DIRECTION_IN :
1492                                         CVMX_USB_DIRECTION_OUT));
1493                 break;
1494         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1495                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1496                 if (!(header->bRequestType & USB_DIR_IN))
1497                         bytes_to_transfer = 0;
1498                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1499                                 union cvmx_usbcx_hccharx, epdir,
1500                                 ((header->bRequestType & USB_DIR_IN) ?
1501                                         CVMX_USB_DIRECTION_IN :
1502                                         CVMX_USB_DIRECTION_OUT));
1503                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1504                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1505                 break;
1506         case CVMX_USB_STAGE_STATUS:
1507                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1508                 bytes_to_transfer = 0;
1509                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1510                                 union cvmx_usbcx_hccharx, epdir,
1511                                 ((header->bRequestType & USB_DIR_IN) ?
1512                                         CVMX_USB_DIRECTION_OUT :
1513                                         CVMX_USB_DIRECTION_IN));
1514                 break;
1515         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1516                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1517                 bytes_to_transfer = 0;
1518                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1519                                 union cvmx_usbcx_hccharx, epdir,
1520                                 ((header->bRequestType & USB_DIR_IN) ?
1521                                         CVMX_USB_DIRECTION_OUT :
1522                                         CVMX_USB_DIRECTION_IN));
1523                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1524                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1525                 break;
1526         }
1527
1528         /*
1529          * Make sure the transfer never exceeds the byte limit of the hardware.
1530          * Further bytes will be sent as continued transactions
1531          */
1532         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1533                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1534                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1535                 bytes_to_transfer *= pipe->max_packet;
1536         }
1537
1538         /*
1539          * Calculate the number of packets to transfer. If the length is zero
1540          * we still need to transfer one packet
1541          */
1542         packets_to_transfer = DIV_ROUND_UP(bytes_to_transfer,
1543                                            pipe->max_packet);
1544         if (packets_to_transfer == 0)
1545                 packets_to_transfer = 1;
1546         else if ((packets_to_transfer > 1) &&
1547                         (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1548                 /*
1549                  * Limit to one packet when not using DMA. Channels must be
1550                  * restarted between every packet for IN transactions, so there
1551                  * is no reason to do multiple packets in a row
1552                  */
1553                 packets_to_transfer = 1;
1554                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1555         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1556                 /*
1557                  * Limit the number of packet and data transferred to what the
1558                  * hardware can handle
1559                  */
1560                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1561                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1562         }
1563
1564         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1565         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1566
1567         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index),
1568                                usbc_hctsiz.u32);
1569 }
1570
1571
1572 /**
1573  * Start a channel to perform the pipe's head transaction
1574  *
1575  * @usb:          USB device state populated by cvmx_usb_initialize().
1576  * @channel:      Channel to setup
1577  * @pipe:         Pipe to start
1578  */
1579 static void __cvmx_usb_start_channel(struct cvmx_usb_state *usb,
1580                                      int channel,
1581                                      struct cvmx_usb_pipe *pipe)
1582 {
1583         struct cvmx_usb_transaction *transaction =
1584                 list_first_entry(&pipe->transactions, typeof(*transaction),
1585                                  node);
1586
1587         /* Make sure all writes to the DMA region get flushed */
1588         CVMX_SYNCW;
1589
1590         /* Attach the channel to the pipe */
1591         usb->pipe_for_channel[channel] = pipe;
1592         pipe->channel = channel;
1593         pipe->flags |= __CVMX_USB_PIPE_FLAGS_SCHEDULED;
1594
1595         /* Mark this channel as in use */
1596         usb->idle_hardware_channels &= ~(1<<channel);
1597
1598         /* Enable the channel interrupt bits */
1599         {
1600                 union cvmx_usbcx_hcintx usbc_hcint;
1601                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1602                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1603
1604                 /* Clear all channel status bits */
1605                 usbc_hcint.u32 = __cvmx_usb_read_csr32(usb,
1606                                 CVMX_USBCX_HCINTX(channel, usb->index));
1607
1608                 __cvmx_usb_write_csr32(usb,
1609                                        CVMX_USBCX_HCINTX(channel, usb->index),
1610                                        usbc_hcint.u32);
1611
1612                 usbc_hcintmsk.u32 = 0;
1613                 usbc_hcintmsk.s.chhltdmsk = 1;
1614                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1615                         /*
1616                          * Channels need these extra interrupts when we aren't
1617                          * in DMA mode.
1618                          */
1619                         usbc_hcintmsk.s.datatglerrmsk = 1;
1620                         usbc_hcintmsk.s.frmovrunmsk = 1;
1621                         usbc_hcintmsk.s.bblerrmsk = 1;
1622                         usbc_hcintmsk.s.xacterrmsk = 1;
1623                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1624                                 /*
1625                                  * Splits don't generate xfercompl, so we need
1626                                  * ACK and NYET.
1627                                  */
1628                                 usbc_hcintmsk.s.nyetmsk = 1;
1629                                 usbc_hcintmsk.s.ackmsk = 1;
1630                         }
1631                         usbc_hcintmsk.s.nakmsk = 1;
1632                         usbc_hcintmsk.s.stallmsk = 1;
1633                         usbc_hcintmsk.s.xfercomplmsk = 1;
1634                 }
1635                 __cvmx_usb_write_csr32(usb,
1636                                 CVMX_USBCX_HCINTMSKX(channel, usb->index),
1637                                 usbc_hcintmsk.u32);
1638
1639                 /* Enable the channel interrupt to propagate */
1640                 usbc_haintmsk.u32 = __cvmx_usb_read_csr32(usb,
1641                                         CVMX_USBCX_HAINTMSK(usb->index));
1642                 usbc_haintmsk.s.haintmsk |= 1<<channel;
1643                 __cvmx_usb_write_csr32(usb,
1644                                         CVMX_USBCX_HAINTMSK(usb->index),
1645                                         usbc_haintmsk.u32);
1646         }
1647
1648         /* Setup the locations the DMA engines use  */
1649         {
1650                 uint64_t dma_address = transaction->buffer +
1651                                         transaction->actual_bytes;
1652
1653                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1654                         dma_address = transaction->buffer +
1655                                         transaction->iso_packets[0].offset +
1656                                         transaction->actual_bytes;
1657
1658                 cvmx_write64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
1659                                         channel * 8,
1660                                     dma_address);
1661                 cvmx_write64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) +
1662                                         channel * 8,
1663                                     dma_address);
1664         }
1665
1666         /* Setup both the size of the transfer and the SPLIT characteristics */
1667         {
1668                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1669                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1670                 int packets_to_transfer;
1671                 int bytes_to_transfer = transaction->buffer_length -
1672                         transaction->actual_bytes;
1673
1674                 /*
1675                  * ISOCHRONOUS transactions store each individual transfer size
1676                  * in the packet structure, not the global buffer_length
1677                  */
1678                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1679                         bytes_to_transfer =
1680                                 transaction->iso_packets[0].length -
1681                                 transaction->actual_bytes;
1682
1683                 /*
1684                  * We need to do split transactions when we are talking to non
1685                  * high speed devices that are behind a high speed hub
1686                  */
1687                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1688                         /*
1689                          * On the start split phase (stage is even) record the
1690                          * frame number we will need to send the split complete.
1691                          * We only store the lower two bits since the time ahead
1692                          * can only be two frames
1693                          */
1694                         if ((transaction->stage&1) == 0) {
1695                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1696                                         pipe->split_sc_frame =
1697                                                 (usb->frame_number + 1) & 0x7f;
1698                                 else
1699                                         pipe->split_sc_frame =
1700                                                 (usb->frame_number + 2) & 0x7f;
1701                         } else
1702                                 pipe->split_sc_frame = -1;
1703
1704                         usbc_hcsplt.s.spltena = 1;
1705                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1706                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1707                         usbc_hcsplt.s.compsplt = (transaction->stage ==
1708                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1709
1710                         /*
1711                          * SPLIT transactions can only ever transmit one data
1712                          * packet so limit the transfer size to the max packet
1713                          * size
1714                          */
1715                         if (bytes_to_transfer > pipe->max_packet)
1716                                 bytes_to_transfer = pipe->max_packet;
1717
1718                         /*
1719                          * ISOCHRONOUS OUT splits are unique in that they limit
1720                          * data transfers to 188 byte chunks representing the
1721                          * begin/middle/end of the data or all
1722                          */
1723                         if (!usbc_hcsplt.s.compsplt &&
1724                                 (pipe->transfer_dir ==
1725                                  CVMX_USB_DIRECTION_OUT) &&
1726                                 (pipe->transfer_type ==
1727                                  CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1728                                 /*
1729                                  * Clear the split complete frame number as
1730                                  * there isn't going to be a split complete
1731                                  */
1732                                 pipe->split_sc_frame = -1;
1733                                 /*
1734                                  * See if we've started this transfer and sent
1735                                  * data
1736                                  */
1737                                 if (transaction->actual_bytes == 0) {
1738                                         /*
1739                                          * Nothing sent yet, this is either a
1740                                          * begin or the entire payload
1741                                          */
1742                                         if (bytes_to_transfer <= 188)
1743                                                 /* Entire payload in one go */
1744                                                 usbc_hcsplt.s.xactpos = 3;
1745                                         else
1746                                                 /* First part of payload */
1747                                                 usbc_hcsplt.s.xactpos = 2;
1748                                 } else {
1749                                         /*
1750                                          * Continuing the previous data, we must
1751                                          * either be in the middle or at the end
1752                                          */
1753                                         if (bytes_to_transfer <= 188)
1754                                                 /* End of payload */
1755                                                 usbc_hcsplt.s.xactpos = 1;
1756                                         else
1757                                                 /* Middle of payload */
1758                                                 usbc_hcsplt.s.xactpos = 0;
1759                                 }
1760                                 /*
1761                                  * Again, the transfer size is limited to 188
1762                                  * bytes
1763                                  */
1764                                 if (bytes_to_transfer > 188)
1765                                         bytes_to_transfer = 188;
1766                         }
1767                 }
1768
1769                 /*
1770                  * Make sure the transfer never exceeds the byte limit of the
1771                  * hardware. Further bytes will be sent as continued
1772                  * transactions
1773                  */
1774                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1775                         /*
1776                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1777                          * size
1778                          */
1779                         bytes_to_transfer = MAX_TRANSFER_BYTES /
1780                                 pipe->max_packet;
1781                         bytes_to_transfer *= pipe->max_packet;
1782                 }
1783
1784                 /*
1785                  * Calculate the number of packets to transfer. If the length is
1786                  * zero we still need to transfer one packet
1787                  */
1788                 packets_to_transfer =
1789                         DIV_ROUND_UP(bytes_to_transfer, pipe->max_packet);
1790                 if (packets_to_transfer == 0)
1791                         packets_to_transfer = 1;
1792                 else if ((packets_to_transfer > 1) &&
1793                                 (usb->init_flags &
1794                                  CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1795                         /*
1796                          * Limit to one packet when not using DMA. Channels must
1797                          * be restarted between every packet for IN
1798                          * transactions, so there is no reason to do multiple
1799                          * packets in a row
1800                          */
1801                         packets_to_transfer = 1;
1802                         bytes_to_transfer = packets_to_transfer *
1803                                 pipe->max_packet;
1804                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1805                         /*
1806                          * Limit the number of packet and data transferred to
1807                          * what the hardware can handle
1808                          */
1809                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1810                         bytes_to_transfer = packets_to_transfer *
1811                                 pipe->max_packet;
1812                 }
1813
1814                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1815                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1816
1817                 /* Update the DATA0/DATA1 toggle */
1818                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1819                 /*
1820                  * High speed pipes may need a hardware ping before they start
1821                  */
1822                 if (pipe->flags & __CVMX_USB_PIPE_FLAGS_NEED_PING)
1823                         usbc_hctsiz.s.dopng = 1;
1824
1825                 __cvmx_usb_write_csr32(usb,
1826                                        CVMX_USBCX_HCSPLTX(channel, usb->index),
1827                                        usbc_hcsplt.u32);
1828                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel,
1829                                         usb->index), usbc_hctsiz.u32);
1830         }
1831
1832         /* Setup the Host Channel Characteristics Register */
1833         {
1834                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1835
1836                 /*
1837                  * Set the startframe odd/even properly. This is only used for
1838                  * periodic
1839                  */
1840                 usbc_hcchar.s.oddfrm = usb->frame_number&1;
1841
1842                 /*
1843                  * Set the number of back to back packets allowed by this
1844                  * endpoint. Split transactions interpret "ec" as the number of
1845                  * immediate retries of failure. These retries happen too
1846                  * quickly, so we disable these entirely for splits
1847                  */
1848                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
1849                         usbc_hcchar.s.ec = 1;
1850                 else if (pipe->multi_count < 1)
1851                         usbc_hcchar.s.ec = 1;
1852                 else if (pipe->multi_count > 3)
1853                         usbc_hcchar.s.ec = 3;
1854                 else
1855                         usbc_hcchar.s.ec = pipe->multi_count;
1856
1857                 /* Set the rest of the endpoint specific settings */
1858                 usbc_hcchar.s.devaddr = pipe->device_addr;
1859                 usbc_hcchar.s.eptype = transaction->type;
1860                 usbc_hcchar.s.lspddev =
1861                         (pipe->device_speed == CVMX_USB_SPEED_LOW);
1862                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1863                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1864                 usbc_hcchar.s.mps = pipe->max_packet;
1865                 __cvmx_usb_write_csr32(usb,
1866                                        CVMX_USBCX_HCCHARX(channel, usb->index),
1867                                        usbc_hcchar.u32);
1868         }
1869
1870         /* Do transaction type specific fixups as needed */
1871         switch (transaction->type) {
1872         case CVMX_USB_TRANSFER_CONTROL:
1873                 __cvmx_usb_start_channel_control(usb, channel, pipe);
1874                 break;
1875         case CVMX_USB_TRANSFER_BULK:
1876         case CVMX_USB_TRANSFER_INTERRUPT:
1877                 break;
1878         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1879                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
1880                         /*
1881                          * ISO transactions require different PIDs depending on
1882                          * direction and how many packets are needed
1883                          */
1884                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1885                                 if (pipe->multi_count < 2) /* Need DATA0 */
1886                                         USB_SET_FIELD32(
1887                                                 CVMX_USBCX_HCTSIZX(channel,
1888                                                                    usb->index),
1889                                                 union cvmx_usbcx_hctsizx,
1890                                                 pid, 0);
1891                                 else /* Need MDATA */
1892                                         USB_SET_FIELD32(
1893                                                 CVMX_USBCX_HCTSIZX(channel,
1894                                                                    usb->index),
1895                                                 union cvmx_usbcx_hctsizx,
1896                                                 pid, 3);
1897                         }
1898                 }
1899                 break;
1900         }
1901         {
1902                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 =
1903                         __cvmx_usb_read_csr32(usb,
1904                                 CVMX_USBCX_HCTSIZX(channel, usb->index))};
1905                 transaction->xfersize = usbc_hctsiz.s.xfersize;
1906                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
1907         }
1908         /* Remeber when we start a split transaction */
1909         if (__cvmx_usb_pipe_needs_split(usb, pipe))
1910                 usb->active_split = transaction;
1911         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1912                         union cvmx_usbcx_hccharx, chena, 1);
1913         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
1914                 __cvmx_usb_fill_tx_fifo(usb, channel);
1915 }
1916
1917
1918 /**
1919  * Find a pipe that is ready to be scheduled to hardware.
1920  * @usb:         USB device state populated by cvmx_usb_initialize().
1921  * @list:        Pipe list to search
1922  * @current_frame:
1923  *               Frame counter to use as a time reference.
1924  *
1925  * Returns: Pipe or NULL if none are ready
1926  */
1927 static struct cvmx_usb_pipe *__cvmx_usb_find_ready_pipe(
1928                 struct cvmx_usb_state *usb,
1929                 struct list_head *list,
1930                 uint64_t current_frame)
1931 {
1932         struct cvmx_usb_pipe *pipe;
1933
1934         list_for_each_entry(pipe, list, node) {
1935                 struct cvmx_usb_transaction *t =
1936                         list_first_entry(&pipe->transactions, typeof(*t),
1937                                          node);
1938                 if (!(pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED) && t &&
1939                         (pipe->next_tx_frame <= current_frame) &&
1940                         ((pipe->split_sc_frame == -1) ||
1941                          ((((int)current_frame - (int)pipe->split_sc_frame)
1942                            & 0x7f) < 0x40)) &&
1943                         (!usb->active_split || (usb->active_split == t))) {
1944                         prefetch(t);
1945                         return pipe;
1946                 }
1947         }
1948         return NULL;
1949 }
1950
1951
1952 /**
1953  * Called whenever a pipe might need to be scheduled to the
1954  * hardware.
1955  *
1956  * @usb:         USB device state populated by cvmx_usb_initialize().
1957  * @is_sof:      True if this schedule was called on a SOF interrupt.
1958  */
1959 static void __cvmx_usb_schedule(struct cvmx_usb_state *usb, int is_sof)
1960 {
1961         int channel;
1962         struct cvmx_usb_pipe *pipe;
1963         int need_sof;
1964         enum cvmx_usb_transfer ttype;
1965
1966         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1967                 /*
1968                  * Without DMA we need to be careful to not schedule something
1969                  * at the end of a frame and cause an overrun.
1970                  */
1971                 union cvmx_usbcx_hfnum hfnum = {
1972                         .u32 = __cvmx_usb_read_csr32(usb,
1973                                                 CVMX_USBCX_HFNUM(usb->index))
1974                 };
1975
1976                 union cvmx_usbcx_hfir hfir = {
1977                         .u32 = __cvmx_usb_read_csr32(usb,
1978                                                 CVMX_USBCX_HFIR(usb->index))
1979                 };
1980
1981                 if (hfnum.s.frrem < hfir.s.frint/4)
1982                         goto done;
1983         }
1984
1985         while (usb->idle_hardware_channels) {
1986                 /* Find an idle channel */
1987                 channel = __fls(usb->idle_hardware_channels);
1988                 if (unlikely(channel > 7))
1989                         break;
1990
1991                 /* Find a pipe needing service */
1992                 pipe = NULL;
1993                 if (is_sof) {
1994                         /*
1995                          * Only process periodic pipes on SOF interrupts. This
1996                          * way we are sure that the periodic data is sent in the
1997                          * beginning of the frame
1998                          */
1999                         pipe = __cvmx_usb_find_ready_pipe(usb,
2000                                         usb->active_pipes +
2001                                         CVMX_USB_TRANSFER_ISOCHRONOUS,
2002                                         usb->frame_number);
2003                         if (likely(!pipe))
2004                                 pipe = __cvmx_usb_find_ready_pipe(usb,
2005                                                 usb->active_pipes +
2006                                                 CVMX_USB_TRANSFER_INTERRUPT,
2007                                                 usb->frame_number);
2008                 }
2009                 if (likely(!pipe)) {
2010                         pipe = __cvmx_usb_find_ready_pipe(usb,
2011                                         usb->active_pipes +
2012                                         CVMX_USB_TRANSFER_CONTROL,
2013                                         usb->frame_number);
2014                         if (likely(!pipe))
2015                                 pipe = __cvmx_usb_find_ready_pipe(usb,
2016                                                 usb->active_pipes +
2017                                                 CVMX_USB_TRANSFER_BULK,
2018                                                 usb->frame_number);
2019                 }
2020                 if (!pipe)
2021                         break;
2022
2023                 __cvmx_usb_start_channel(usb, channel, pipe);
2024         }
2025
2026 done:
2027         /*
2028          * Only enable SOF interrupts when we have transactions pending in the
2029          * future that might need to be scheduled
2030          */
2031         need_sof = 0;
2032         for (ttype = CVMX_USB_TRANSFER_CONTROL;
2033                         ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
2034                 list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
2035                         if (pipe->next_tx_frame > usb->frame_number) {
2036                                 need_sof = 1;
2037                                 break;
2038                         }
2039                 }
2040         }
2041         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
2042                         union cvmx_usbcx_gintmsk, sofmsk, need_sof);
2043 }
2044
2045 static void octeon_usb_urb_complete_callback(struct cvmx_usb_state *usb,
2046                                              enum cvmx_usb_complete status,
2047                                              struct cvmx_usb_pipe *pipe,
2048                                              struct cvmx_usb_transaction
2049                                                 *transaction,
2050                                              int bytes_transferred,
2051                                              struct urb *urb)
2052 {
2053         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
2054         struct usb_hcd *hcd = octeon_to_hcd(priv);
2055         struct device *dev = hcd->self.controller;
2056
2057         if (likely(status == CVMX_USB_COMPLETE_SUCCESS))
2058                 urb->actual_length = bytes_transferred;
2059         else
2060                 urb->actual_length = 0;
2061
2062         urb->hcpriv = NULL;
2063
2064         /* For Isochronous transactions we need to update the URB packet status
2065            list from data in our private copy */
2066         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
2067                 int i;
2068                 /*
2069                  * The pointer to the private list is stored in the setup_packet
2070                  * field.
2071                  */
2072                 struct cvmx_usb_iso_packet *iso_packet =
2073                         (struct cvmx_usb_iso_packet *) urb->setup_packet;
2074                 /* Recalculate the transfer size by adding up each packet */
2075                 urb->actual_length = 0;
2076                 for (i = 0; i < urb->number_of_packets; i++) {
2077                         if (iso_packet[i].status ==
2078                                         CVMX_USB_COMPLETE_SUCCESS) {
2079                                 urb->iso_frame_desc[i].status = 0;
2080                                 urb->iso_frame_desc[i].actual_length =
2081                                         iso_packet[i].length;
2082                                 urb->actual_length +=
2083                                         urb->iso_frame_desc[i].actual_length;
2084                         } else {
2085                                 dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
2086                                         i, urb->number_of_packets,
2087                                         iso_packet[i].status, pipe,
2088                                         transaction, iso_packet[i].length);
2089                                 urb->iso_frame_desc[i].status = -EREMOTEIO;
2090                         }
2091                 }
2092                 /* Free the private list now that we don't need it anymore */
2093                 kfree(iso_packet);
2094                 urb->setup_packet = NULL;
2095         }
2096
2097         switch (status) {
2098         case CVMX_USB_COMPLETE_SUCCESS:
2099                 urb->status = 0;
2100                 break;
2101         case CVMX_USB_COMPLETE_CANCEL:
2102                 if (urb->status == 0)
2103                         urb->status = -ENOENT;
2104                 break;
2105         case CVMX_USB_COMPLETE_STALL:
2106                 dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
2107                         pipe, transaction, bytes_transferred);
2108                 urb->status = -EPIPE;
2109                 break;
2110         case CVMX_USB_COMPLETE_BABBLEERR:
2111                 dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
2112                         pipe, transaction, bytes_transferred);
2113                 urb->status = -EPIPE;
2114                 break;
2115         case CVMX_USB_COMPLETE_SHORT:
2116                 dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
2117                         pipe, transaction, bytes_transferred);
2118                 urb->status = -EREMOTEIO;
2119                 break;
2120         case CVMX_USB_COMPLETE_ERROR:
2121         case CVMX_USB_COMPLETE_XACTERR:
2122         case CVMX_USB_COMPLETE_DATATGLERR:
2123         case CVMX_USB_COMPLETE_FRAMEERR:
2124                 dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
2125                         status, pipe, transaction, bytes_transferred);
2126                 urb->status = -EPROTO;
2127                 break;
2128         }
2129         usb_hcd_unlink_urb_from_ep(octeon_to_hcd(priv), urb);
2130         spin_unlock(&priv->lock);
2131         usb_hcd_giveback_urb(octeon_to_hcd(priv), urb, urb->status);
2132         spin_lock(&priv->lock);
2133 }
2134
2135 /**
2136  * Signal the completion of a transaction and free it. The
2137  * transaction will be removed from the pipe transaction list.
2138  *
2139  * @usb:         USB device state populated by cvmx_usb_initialize().
2140  * @pipe:        Pipe the transaction is on
2141  * @transaction:
2142  *               Transaction that completed
2143  * @complete_code:
2144  *               Completion code
2145  */
2146 static void __cvmx_usb_perform_complete(
2147                                 struct cvmx_usb_state *usb,
2148                                 struct cvmx_usb_pipe *pipe,
2149                                 struct cvmx_usb_transaction *transaction,
2150                                 enum cvmx_usb_complete complete_code)
2151 {
2152         /* If this was a split then clear our split in progress marker */
2153         if (usb->active_split == transaction)
2154                 usb->active_split = NULL;
2155
2156         /*
2157          * Isochronous transactions need extra processing as they might not be
2158          * done after a single data transfer
2159          */
2160         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2161                 /* Update the number of bytes transferred in this ISO packet */
2162                 transaction->iso_packets[0].length = transaction->actual_bytes;
2163                 transaction->iso_packets[0].status = complete_code;
2164
2165                 /*
2166                  * If there are more ISOs pending and we succeeded, schedule the
2167                  * next one
2168                  */
2169                 if ((transaction->iso_number_packets > 1) &&
2170                         (complete_code == CVMX_USB_COMPLETE_SUCCESS)) {
2171                         /* No bytes transferred for this packet as of yet */
2172                         transaction->actual_bytes = 0;
2173                         /* One less ISO waiting to transfer */
2174                         transaction->iso_number_packets--;
2175                         /* Increment to the next location in our packet array */
2176                         transaction->iso_packets++;
2177                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2178                         return;
2179                 }
2180         }
2181
2182         /* Remove the transaction from the pipe list */
2183         list_del(&transaction->node);
2184         if (list_empty(&pipe->transactions))
2185                 list_move_tail(&pipe->node, &usb->idle_pipes);
2186         octeon_usb_urb_complete_callback(usb, complete_code, pipe,
2187                                          transaction,
2188                                          transaction->actual_bytes,
2189                                          transaction->urb);
2190         kfree(transaction);
2191 }
2192
2193
2194 /**
2195  * Submit a usb transaction to a pipe. Called for all types
2196  * of transactions.
2197  *
2198  * @usb:
2199  * @pipe:           Which pipe to submit to.
2200  * @type:           Transaction type
2201  * @buffer:         User buffer for the transaction
2202  * @buffer_length:
2203  *                  User buffer's length in bytes
2204  * @control_header:
2205  *                  For control transactions, the 8 byte standard header
2206  * @iso_start_frame:
2207  *                  For ISO transactions, the start frame
2208  * @iso_number_packets:
2209  *                  For ISO, the number of packet in the transaction.
2210  * @iso_packets:
2211  *                  A description of each ISO packet
2212  * @urb:            URB for the callback
2213  *
2214  * Returns: Transaction or NULL on failure.
2215  */
2216 static struct cvmx_usb_transaction *__cvmx_usb_submit_transaction(
2217                                 struct cvmx_usb_state *usb,
2218                                 struct cvmx_usb_pipe *pipe,
2219                                 enum cvmx_usb_transfer type,
2220                                 uint64_t buffer,
2221                                 int buffer_length,
2222                                 uint64_t control_header,
2223                                 int iso_start_frame,
2224                                 int iso_number_packets,
2225                                 struct cvmx_usb_iso_packet *iso_packets,
2226                                 struct urb *urb)
2227 {
2228         struct cvmx_usb_transaction *transaction;
2229
2230         if (unlikely(pipe->transfer_type != type))
2231                 return NULL;
2232
2233         transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
2234         if (unlikely(!transaction))
2235                 return NULL;
2236
2237         transaction->type = type;
2238         transaction->buffer = buffer;
2239         transaction->buffer_length = buffer_length;
2240         transaction->control_header = control_header;
2241         /* FIXME: This is not used, implement it. */
2242         transaction->iso_start_frame = iso_start_frame;
2243         transaction->iso_number_packets = iso_number_packets;
2244         transaction->iso_packets = iso_packets;
2245         transaction->urb = urb;
2246         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2247                 transaction->stage = CVMX_USB_STAGE_SETUP;
2248         else
2249                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2250
2251         if (!list_empty(&pipe->transactions)) {
2252                 list_add_tail(&transaction->node, &pipe->transactions);
2253         } else {
2254                 list_add_tail(&transaction->node, &pipe->transactions);
2255                 list_move_tail(&pipe->node,
2256                                &usb->active_pipes[pipe->transfer_type]);
2257
2258                 /*
2259                  * We may need to schedule the pipe if this was the head of the
2260                  * pipe.
2261                  */
2262                 __cvmx_usb_schedule(usb, 0);
2263         }
2264
2265         return transaction;
2266 }
2267
2268
2269 /**
2270  * Call to submit a USB Bulk transfer to a pipe.
2271  *
2272  * @usb:            USB device state populated by cvmx_usb_initialize().
2273  * @pipe:           Handle to the pipe for the transfer.
2274  * @urb:            URB.
2275  *
2276  * Returns: A submitted transaction or NULL on failure.
2277  */
2278 static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(
2279                                                 struct cvmx_usb_state *usb,
2280                                                 struct cvmx_usb_pipe *pipe,
2281                                                 struct urb *urb)
2282 {
2283         return __cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
2284                                              urb->transfer_dma,
2285                                              urb->transfer_buffer_length,
2286                                              0, /* control_header */
2287                                              0, /* iso_start_frame */
2288                                              0, /* iso_number_packets */
2289                                              NULL, /* iso_packets */
2290                                              urb);
2291 }
2292
2293
2294 /**
2295  * Call to submit a USB Interrupt transfer to a pipe.
2296  *
2297  * @usb:            USB device state populated by cvmx_usb_initialize().
2298  * @pipe:           Handle to the pipe for the transfer.
2299  * @urb:            URB returned when the callback is called.
2300  *
2301  * Returns: A submitted transaction or NULL on failure.
2302  */
2303 static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(
2304                                                 struct cvmx_usb_state *usb,
2305                                                 struct cvmx_usb_pipe *pipe,
2306                                                 struct urb *urb)
2307 {
2308         return __cvmx_usb_submit_transaction(usb, pipe,
2309                                              CVMX_USB_TRANSFER_INTERRUPT,
2310                                              urb->transfer_dma,
2311                                              urb->transfer_buffer_length,
2312                                              0, /* control_header */
2313                                              0, /* iso_start_frame */
2314                                              0, /* iso_number_packets */
2315                                              NULL, /* iso_packets */
2316                                              urb);
2317 }
2318
2319
2320 /**
2321  * Call to submit a USB Control transfer to a pipe.
2322  *
2323  * @usb:            USB device state populated by cvmx_usb_initialize().
2324  * @pipe:           Handle to the pipe for the transfer.
2325  * @urb:            URB.
2326  *
2327  * Returns: A submitted transaction or NULL on failure.
2328  */
2329 static struct cvmx_usb_transaction *cvmx_usb_submit_control(
2330                                                 struct cvmx_usb_state *usb,
2331                                                 struct cvmx_usb_pipe *pipe,
2332                                                 struct urb *urb)
2333 {
2334         int buffer_length = urb->transfer_buffer_length;
2335         uint64_t control_header = urb->setup_dma;
2336         struct usb_ctrlrequest *header = cvmx_phys_to_ptr(control_header);
2337
2338         if ((header->bRequestType & USB_DIR_IN) == 0)
2339                 buffer_length = le16_to_cpu(header->wLength);
2340
2341         return __cvmx_usb_submit_transaction(usb, pipe,
2342                                              CVMX_USB_TRANSFER_CONTROL,
2343                                              urb->transfer_dma, buffer_length,
2344                                              control_header,
2345                                              0, /* iso_start_frame */
2346                                              0, /* iso_number_packets */
2347                                              NULL, /* iso_packets */
2348                                              urb);
2349 }
2350
2351
2352 /**
2353  * Call to submit a USB Isochronous transfer to a pipe.
2354  *
2355  * @usb:            USB device state populated by cvmx_usb_initialize().
2356  * @pipe:           Handle to the pipe for the transfer.
2357  * @urb:            URB returned when the callback is called.
2358  *
2359  * Returns: A submitted transaction or NULL on failure.
2360  */
2361 static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(
2362                                                 struct cvmx_usb_state *usb,
2363                                                 struct cvmx_usb_pipe *pipe,
2364                                                 struct urb *urb)
2365 {
2366         struct cvmx_usb_iso_packet *packets;
2367
2368         packets = (struct cvmx_usb_iso_packet *) urb->setup_packet;
2369         return __cvmx_usb_submit_transaction(usb, pipe,
2370                                              CVMX_USB_TRANSFER_ISOCHRONOUS,
2371                                              urb->transfer_dma,
2372                                              urb->transfer_buffer_length,
2373                                              0, /* control_header */
2374                                              urb->start_frame,
2375                                              urb->number_of_packets,
2376                                              packets, urb);
2377 }
2378
2379
2380 /**
2381  * Cancel one outstanding request in a pipe. Canceling a request
2382  * can fail if the transaction has already completed before cancel
2383  * is called. Even after a successful cancel call, it may take
2384  * a frame or two for the cvmx_usb_poll() function to call the
2385  * associated callback.
2386  *
2387  * @usb:         USB device state populated by cvmx_usb_initialize().
2388  * @pipe:        Pipe to cancel requests in.
2389  * @transaction: Transaction to cancel, returned by the submit function.
2390  *
2391  * Returns: 0 or a negative error code.
2392  */
2393 static int cvmx_usb_cancel(struct cvmx_usb_state *usb,
2394                            struct cvmx_usb_pipe *pipe,
2395                            struct cvmx_usb_transaction *transaction)
2396 {
2397         /*
2398          * If the transaction is the HEAD of the queue and scheduled. We need to
2399          * treat it special
2400          */
2401         if (list_first_entry(&pipe->transactions, typeof(*transaction), node) ==
2402             transaction && (pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2403                 union cvmx_usbcx_hccharx usbc_hcchar;
2404
2405                 usb->pipe_for_channel[pipe->channel] = NULL;
2406                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2407
2408                 CVMX_SYNCW;
2409
2410                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2411                                 CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2412                 /*
2413                  * If the channel isn't enabled then the transaction already
2414                  * completed.
2415                  */
2416                 if (usbc_hcchar.s.chena) {
2417                         usbc_hcchar.s.chdis = 1;
2418                         __cvmx_usb_write_csr32(usb,
2419                                         CVMX_USBCX_HCCHARX(pipe->channel,
2420                                                 usb->index),
2421                                         usbc_hcchar.u32);
2422                 }
2423         }
2424         __cvmx_usb_perform_complete(usb, pipe, transaction,
2425                                     CVMX_USB_COMPLETE_CANCEL);
2426         return 0;
2427 }
2428
2429
2430 /**
2431  * Cancel all outstanding requests in a pipe. Logically all this
2432  * does is call cvmx_usb_cancel() in a loop.
2433  *
2434  * @usb:         USB device state populated by cvmx_usb_initialize().
2435  * @pipe:        Pipe to cancel requests in.
2436  *
2437  * Returns: 0 or a negative error code.
2438  */
2439 static int cvmx_usb_cancel_all(struct cvmx_usb_state *usb,
2440                                struct cvmx_usb_pipe *pipe)
2441 {
2442         struct cvmx_usb_transaction *transaction, *next;
2443
2444         /* Simply loop through and attempt to cancel each transaction */
2445         list_for_each_entry_safe(transaction, next, &pipe->transactions, node) {
2446                 int result = cvmx_usb_cancel(usb, pipe, transaction);
2447
2448                 if (unlikely(result != 0))
2449                         return result;
2450         }
2451         return 0;
2452 }
2453
2454
2455 /**
2456  * Close a pipe created with cvmx_usb_open_pipe().
2457  *
2458  * @usb:         USB device state populated by cvmx_usb_initialize().
2459  * @pipe:        Pipe to close.
2460  *
2461  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2462  *          outstanding transfers.
2463  */
2464 static int cvmx_usb_close_pipe(struct cvmx_usb_state *usb,
2465                                struct cvmx_usb_pipe *pipe)
2466 {
2467         /* Fail if the pipe has pending transactions */
2468         if (!list_empty(&pipe->transactions))
2469                 return -EBUSY;
2470
2471         list_del(&pipe->node);
2472         kfree(pipe);
2473
2474         return 0;
2475 }
2476
2477 /**
2478  * Get the current USB protocol level frame number. The frame
2479  * number is always in the range of 0-0x7ff.
2480  *
2481  * @usb: USB device state populated by cvmx_usb_initialize().
2482  *
2483  * Returns: USB frame number
2484  */
2485 static int cvmx_usb_get_frame_number(struct cvmx_usb_state *usb)
2486 {
2487         int frame_number;
2488         union cvmx_usbcx_hfnum usbc_hfnum;
2489
2490         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb,
2491                         CVMX_USBCX_HFNUM(usb->index));
2492         frame_number = usbc_hfnum.s.frnum;
2493
2494         return frame_number;
2495 }
2496
2497
2498 /**
2499  * Poll a channel for status
2500  *
2501  * @usb:     USB device
2502  * @channel: Channel to poll
2503  *
2504  * Returns: Zero on success
2505  */
2506 static int __cvmx_usb_poll_channel(struct cvmx_usb_state *usb, int channel)
2507 {
2508         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
2509         struct usb_hcd *hcd = octeon_to_hcd(priv);
2510         struct device *dev = hcd->self.controller;
2511         union cvmx_usbcx_hcintx usbc_hcint;
2512         union cvmx_usbcx_hctsizx usbc_hctsiz;
2513         union cvmx_usbcx_hccharx usbc_hcchar;
2514         struct cvmx_usb_pipe *pipe;
2515         struct cvmx_usb_transaction *transaction;
2516         int bytes_this_transfer;
2517         int bytes_in_last_packet;
2518         int packets_processed;
2519         int buffer_space_left;
2520
2521         /* Read the interrupt status bits for the channel */
2522         usbc_hcint.u32 = __cvmx_usb_read_csr32(usb,
2523                         CVMX_USBCX_HCINTX(channel, usb->index));
2524
2525         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2526                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2527                                 CVMX_USBCX_HCCHARX(channel, usb->index));
2528
2529                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2530                         /*
2531                          * There seems to be a bug in CN31XX which can cause
2532                          * interrupt IN transfers to get stuck until we do a
2533                          * write of HCCHARX without changing things
2534                          */
2535                         __cvmx_usb_write_csr32(usb,
2536                                         CVMX_USBCX_HCCHARX(channel,
2537                                                            usb->index),
2538                                         usbc_hcchar.u32);
2539                         return 0;
2540                 }
2541
2542                 /*
2543                  * In non DMA mode the channels don't halt themselves. We need
2544                  * to manually disable channels that are left running
2545                  */
2546                 if (!usbc_hcint.s.chhltd) {
2547                         if (usbc_hcchar.s.chena) {
2548                                 union cvmx_usbcx_hcintmskx hcintmsk;
2549                                 /* Disable all interrupts except CHHLTD */
2550                                 hcintmsk.u32 = 0;
2551                                 hcintmsk.s.chhltdmsk = 1;
2552                                 __cvmx_usb_write_csr32(usb,
2553                                                 CVMX_USBCX_HCINTMSKX(channel,
2554                                                         usb->index),
2555                                                 hcintmsk.u32);
2556                                 usbc_hcchar.s.chdis = 1;
2557                                 __cvmx_usb_write_csr32(usb,
2558                                                 CVMX_USBCX_HCCHARX(channel,
2559                                                         usb->index),
2560                                                 usbc_hcchar.u32);
2561                                 return 0;
2562                         } else if (usbc_hcint.s.xfercompl) {
2563                                 /*
2564                                  * Successful IN/OUT with transfer complete.
2565                                  * Channel halt isn't needed.
2566                                  */
2567                         } else {
2568                                 dev_err(dev, "USB%d: Channel %d interrupt without halt\n",
2569                                         usb->index, channel);
2570                                 return 0;
2571                         }
2572                 }
2573         } else {
2574                 /*
2575                  * There is are no interrupts that we need to process when the
2576                  * channel is still running
2577                  */
2578                 if (!usbc_hcint.s.chhltd)
2579                         return 0;
2580         }
2581
2582         /* Disable the channel interrupts now that it is done */
2583         __cvmx_usb_write_csr32(usb,
2584                                 CVMX_USBCX_HCINTMSKX(channel, usb->index),
2585                                 0);
2586         usb->idle_hardware_channels |= (1<<channel);
2587
2588         /* Make sure this channel is tied to a valid pipe */
2589         pipe = usb->pipe_for_channel[channel];
2590         prefetch(pipe);
2591         if (!pipe)
2592                 return 0;
2593         transaction = list_first_entry(&pipe->transactions,
2594                                        typeof(*transaction),
2595                                        node);
2596         prefetch(transaction);
2597
2598         /*
2599          * Disconnect this pipe from the HW channel. Later the schedule
2600          * function will figure out which pipe needs to go
2601          */
2602         usb->pipe_for_channel[channel] = NULL;
2603         pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2604
2605         /*
2606          * Read the channel config info so we can figure out how much data
2607          * transferred
2608          */
2609         usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2610                         CVMX_USBCX_HCCHARX(channel, usb->index));
2611         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
2612                         CVMX_USBCX_HCTSIZX(channel, usb->index));
2613
2614         /*
2615          * Calculating the number of bytes successfully transferred is dependent
2616          * on the transfer direction
2617          */
2618         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2619         if (usbc_hcchar.s.epdir) {
2620                 /*
2621                  * IN transactions are easy. For every byte received the
2622                  * hardware decrements xfersize. All we need to do is subtract
2623                  * the current value of xfersize from its starting value and we
2624                  * know how many bytes were written to the buffer
2625                  */
2626                 bytes_this_transfer = transaction->xfersize -
2627                         usbc_hctsiz.s.xfersize;
2628         } else {
2629                 /*
2630                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2631                  * decremented on every successful packet send. The hardware
2632                  * does this when it receives an ACK, or NYET. If it doesn't
2633                  * receive one of these responses pktcnt doesn't change
2634                  */
2635                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2636                 /*
2637                  * The last packet may not be a full transfer if we didn't have
2638                  * enough data
2639                  */
2640                 if (bytes_this_transfer > transaction->xfersize)
2641                         bytes_this_transfer = transaction->xfersize;
2642         }
2643         /* Figure out how many bytes were in the last packet of the transfer */
2644         if (packets_processed)
2645                 bytes_in_last_packet = bytes_this_transfer -
2646                         (packets_processed - 1) * usbc_hcchar.s.mps;
2647         else
2648                 bytes_in_last_packet = bytes_this_transfer;
2649
2650         /*
2651          * As a special case, setup transactions output the setup header, not
2652          * the user's data. For this reason we don't count setup data as bytes
2653          * transferred
2654          */
2655         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2656                 (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2657                 bytes_this_transfer = 0;
2658
2659         /*
2660          * Add the bytes transferred to the running total. It is important that
2661          * bytes_this_transfer doesn't count any data that needs to be
2662          * retransmitted
2663          */
2664         transaction->actual_bytes += bytes_this_transfer;
2665         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2666                 buffer_space_left = transaction->iso_packets[0].length -
2667                         transaction->actual_bytes;
2668         else
2669                 buffer_space_left = transaction->buffer_length -
2670                         transaction->actual_bytes;
2671
2672         /*
2673          * We need to remember the PID toggle state for the next transaction.
2674          * The hardware already updated it for the next transaction
2675          */
2676         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2677
2678         /*
2679          * For high speed bulk out, assume the next transaction will need to do
2680          * a ping before proceeding. If this isn't true the ACK processing below
2681          * will clear this flag
2682          */
2683         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2684                 (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2685                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2686                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2687
2688         if (usbc_hcint.s.stall) {
2689                 /*
2690                  * STALL as a response means this transaction cannot be
2691                  * completed because the device can't process transactions. Tell
2692                  * the user. Any data that was transferred will be counted on
2693                  * the actual bytes transferred
2694                  */
2695                 pipe->pid_toggle = 0;
2696                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2697                                             CVMX_USB_COMPLETE_STALL);
2698         } else if (usbc_hcint.s.xacterr) {
2699                 /*
2700                  * We know at least one packet worked if we get a ACK or NAK.
2701                  * Reset the retry counter
2702                  */
2703                 if (usbc_hcint.s.nak || usbc_hcint.s.ack)
2704                         transaction->retries = 0;
2705                 transaction->retries++;
2706                 if (transaction->retries > MAX_RETRIES) {
2707                         /*
2708                          * XactErr as a response means the device signaled
2709                          * something wrong with the transfer. For example, PID
2710                          * toggle errors cause these
2711                          */
2712                         __cvmx_usb_perform_complete(usb, pipe, transaction,
2713                                                     CVMX_USB_COMPLETE_XACTERR);
2714                 } else {
2715                         /*
2716                          * If this was a split then clear our split in progress
2717                          * marker
2718                          */
2719                         if (usb->active_split == transaction)
2720                                 usb->active_split = NULL;
2721                         /*
2722                          * Rewind to the beginning of the transaction by anding
2723                          * off the split complete bit
2724                          */
2725                         transaction->stage &= ~1;
2726                         pipe->split_sc_frame = -1;
2727                         pipe->next_tx_frame += pipe->interval;
2728                         if (pipe->next_tx_frame < usb->frame_number)
2729                                 pipe->next_tx_frame =
2730                                         usb->frame_number + pipe->interval -
2731                                         (usb->frame_number -
2732                                          pipe->next_tx_frame) % pipe->interval;
2733                 }
2734         } else if (usbc_hcint.s.bblerr) {
2735                 /* Babble Error (BblErr) */
2736                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2737                                             CVMX_USB_COMPLETE_BABBLEERR);
2738         } else if (usbc_hcint.s.datatglerr) {
2739                 /* Data toggle error */
2740                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2741                                             CVMX_USB_COMPLETE_DATATGLERR);
2742         } else if (usbc_hcint.s.nyet) {
2743                 /*
2744                  * NYET as a response is only allowed in three cases: as a
2745                  * response to a ping, as a response to a split transaction, and
2746                  * as a response to a bulk out. The ping case is handled by
2747                  * hardware, so we only have splits and bulk out
2748                  */
2749                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
2750                         transaction->retries = 0;
2751                         /*
2752                          * If there is more data to go then we need to try
2753                          * again. Otherwise this transaction is complete
2754                          */
2755                         if ((buffer_space_left == 0) ||
2756                                 (bytes_in_last_packet < pipe->max_packet))
2757                                 __cvmx_usb_perform_complete(usb, pipe,
2758                                                 transaction,
2759                                                 CVMX_USB_COMPLETE_SUCCESS);
2760                 } else {
2761                         /*
2762                          * Split transactions retry the split complete 4 times
2763                          * then rewind to the start split and do the entire
2764                          * transactions again
2765                          */
2766                         transaction->retries++;
2767                         if ((transaction->retries & 0x3) == 0) {
2768                                 /*
2769                                  * Rewind to the beginning of the transaction by
2770                                  * anding off the split complete bit
2771                                  */
2772                                 transaction->stage &= ~1;
2773                                 pipe->split_sc_frame = -1;
2774                         }
2775                 }
2776         } else if (usbc_hcint.s.ack) {
2777                 transaction->retries = 0;
2778                 /*
2779                  * The ACK bit can only be checked after the other error bits.
2780                  * This is because a multi packet transfer may succeed in a
2781                  * number of packets and then get a different response on the
2782                  * last packet. In this case both ACK and the last response bit
2783                  * will be set. If none of the other response bits is set, then
2784                  * the last packet must have been an ACK
2785                  *
2786                  * Since we got an ACK, we know we don't need to do a ping on
2787                  * this pipe
2788                  */
2789                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_NEED_PING;
2790
2791                 switch (transaction->type) {
2792                 case CVMX_USB_TRANSFER_CONTROL:
2793                         switch (transaction->stage) {
2794                         case CVMX_USB_STAGE_NON_CONTROL:
2795                         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2796                                 /* This should be impossible */
2797                                 __cvmx_usb_perform_complete(usb, pipe,
2798                                         transaction, CVMX_USB_COMPLETE_ERROR);
2799                                 break;
2800                         case CVMX_USB_STAGE_SETUP:
2801                                 pipe->pid_toggle = 1;
2802                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2803                                         transaction->stage =
2804                                                 CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2805                                 else {
2806                                         struct usb_ctrlrequest *header =
2807                                                 cvmx_phys_to_ptr(transaction->control_header);
2808                                         if (header->wLength)
2809                                                 transaction->stage =
2810                                                         CVMX_USB_STAGE_DATA;
2811                                         else
2812                                                 transaction->stage =
2813                                                         CVMX_USB_STAGE_STATUS;
2814                                 }
2815                                 break;
2816                         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2817                                 {
2818                                         struct usb_ctrlrequest *header =
2819                                                 cvmx_phys_to_ptr(transaction->control_header);
2820                                         if (header->wLength)
2821                                                 transaction->stage =
2822                                                         CVMX_USB_STAGE_DATA;
2823                                         else
2824                                                 transaction->stage =
2825                                                         CVMX_USB_STAGE_STATUS;
2826                                 }
2827                                 break;
2828                         case CVMX_USB_STAGE_DATA:
2829                                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2830                                         transaction->stage =
2831                                                 CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2832                                         /*
2833                                          * For setup OUT data that are splits,
2834                                          * the hardware doesn't appear to count
2835                                          * transferred data. Here we manually
2836                                          * update the data transferred
2837                                          */
2838                                         if (!usbc_hcchar.s.epdir) {
2839                                                 if (buffer_space_left < pipe->max_packet)
2840                                                         transaction->actual_bytes +=
2841                                                                 buffer_space_left;
2842                                                 else
2843                                                         transaction->actual_bytes +=
2844                                                                 pipe->max_packet;
2845                                         }
2846                                 } else if ((buffer_space_left == 0) ||
2847                                                 (bytes_in_last_packet <
2848                                                  pipe->max_packet)) {
2849                                         pipe->pid_toggle = 1;
2850                                         transaction->stage =
2851                                                 CVMX_USB_STAGE_STATUS;
2852                                 }
2853                                 break;
2854                         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2855                                 if ((buffer_space_left == 0) ||
2856                                                 (bytes_in_last_packet <
2857                                                  pipe->max_packet)) {
2858                                         pipe->pid_toggle = 1;
2859                                         transaction->stage =
2860                                                 CVMX_USB_STAGE_STATUS;
2861                                 } else {
2862                                         transaction->stage =
2863                                                 CVMX_USB_STAGE_DATA;
2864                                 }
2865                                 break;
2866                         case CVMX_USB_STAGE_STATUS:
2867                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2868                                         transaction->stage =
2869                                                 CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2870                                 else
2871                                         __cvmx_usb_perform_complete(usb, pipe,
2872                                                 transaction,
2873                                                 CVMX_USB_COMPLETE_SUCCESS);
2874                                 break;
2875                         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2876                                 __cvmx_usb_perform_complete(usb, pipe,
2877                                                 transaction,
2878                                                 CVMX_USB_COMPLETE_SUCCESS);
2879                                 break;
2880                         }
2881                         break;
2882                 case CVMX_USB_TRANSFER_BULK:
2883                 case CVMX_USB_TRANSFER_INTERRUPT:
2884                         /*
2885                          * The only time a bulk transfer isn't complete when it
2886                          * finishes with an ACK is during a split transaction.
2887                          * For splits we need to continue the transfer if more
2888                          * data is needed
2889                          */
2890                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2891                                 if (transaction->stage ==
2892                                                 CVMX_USB_STAGE_NON_CONTROL)
2893                                         transaction->stage =
2894                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2895                                 else {
2896                                         if (buffer_space_left &&
2897                                                 (bytes_in_last_packet ==
2898                                                  pipe->max_packet))
2899                                                 transaction->stage =
2900                                                         CVMX_USB_STAGE_NON_CONTROL;
2901                                         else {
2902                                                 if (transaction->type ==
2903                                                         CVMX_USB_TRANSFER_INTERRUPT)
2904                                                         pipe->next_tx_frame +=
2905                                                                 pipe->interval;
2906                                                         __cvmx_usb_perform_complete(
2907                                                                 usb,
2908                                                                 pipe,
2909                                                                 transaction,
2910                                                                 CVMX_USB_COMPLETE_SUCCESS);
2911                                         }
2912                                 }
2913                         } else {
2914                                 if ((pipe->device_speed ==
2915                                         CVMX_USB_SPEED_HIGH) &&
2916                                     (pipe->transfer_type ==
2917                                      CVMX_USB_TRANSFER_BULK) &&
2918                                     (pipe->transfer_dir ==
2919                                      CVMX_USB_DIRECTION_OUT) &&
2920                                     (usbc_hcint.s.nak))
2921                                         pipe->flags |=
2922                                                 __CVMX_USB_PIPE_FLAGS_NEED_PING;
2923                                 if (!buffer_space_left ||
2924                                         (bytes_in_last_packet <
2925                                          pipe->max_packet)) {
2926                                         if (transaction->type ==
2927                                                 CVMX_USB_TRANSFER_INTERRUPT)
2928                                                 pipe->next_tx_frame +=
2929                                                         pipe->interval;
2930                                         __cvmx_usb_perform_complete(usb,
2931                                                 pipe,
2932                                                 transaction,
2933                                                 CVMX_USB_COMPLETE_SUCCESS);
2934                                 }
2935                         }
2936                         break;
2937                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
2938                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2939                                 /*
2940                                  * ISOCHRONOUS OUT splits don't require a
2941                                  * complete split stage. Instead they use a
2942                                  * sequence of begin OUT splits to transfer the
2943                                  * data 188 bytes at a time. Once the transfer
2944                                  * is complete, the pipe sleeps until the next
2945                                  * schedule interval
2946                                  */
2947                                 if (pipe->transfer_dir ==
2948                                         CVMX_USB_DIRECTION_OUT) {
2949                                         /*
2950                                          * If no space left or this wasn't a max
2951                                          * size packet then this transfer is
2952                                          * complete. Otherwise start it again to
2953                                          * send the next 188 bytes
2954                                          */
2955                                         if (!buffer_space_left ||
2956                                                 (bytes_this_transfer < 188)) {
2957                                                 pipe->next_tx_frame +=
2958                                                         pipe->interval;
2959                                                 __cvmx_usb_perform_complete(
2960                                                         usb,
2961                                                         pipe,
2962                                                         transaction,
2963                                                         CVMX_USB_COMPLETE_SUCCESS);
2964                                         }
2965                                 } else {
2966                                         if (transaction->stage ==
2967                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
2968                                                 /*
2969                                                  * We are in the incoming data
2970                                                  * phase. Keep getting data
2971                                                  * until we run out of space or
2972                                                  * get a small packet
2973                                                  */
2974                                                 if ((buffer_space_left == 0) ||
2975                                                         (bytes_in_last_packet <
2976                                                          pipe->max_packet)) {
2977                                                         pipe->next_tx_frame +=
2978                                                                 pipe->interval;
2979                                                         __cvmx_usb_perform_complete(
2980                                                                 usb,
2981                                                                 pipe,
2982                                                                 transaction,
2983                                                                 CVMX_USB_COMPLETE_SUCCESS);
2984                                                 }
2985                                         } else
2986                                                 transaction->stage =
2987                                                         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2988                                 }
2989                         } else {
2990                                 pipe->next_tx_frame += pipe->interval;
2991                                 __cvmx_usb_perform_complete(usb,
2992                                                 pipe,
2993                                                 transaction,
2994                                                 CVMX_USB_COMPLETE_SUCCESS);
2995                         }
2996                         break;
2997                 }
2998         } else if (usbc_hcint.s.nak) {
2999                 /*
3000                  * If this was a split then clear our split in progress marker.
3001                  */
3002                 if (usb->active_split == transaction)
3003                         usb->active_split = NULL;
3004                 /*
3005                  * NAK as a response means the device couldn't accept the
3006                  * transaction, but it should be retried in the future. Rewind
3007                  * to the beginning of the transaction by anding off the split
3008                  * complete bit. Retry in the next interval
3009                  */
3010                 transaction->retries = 0;
3011                 transaction->stage &= ~1;
3012                 pipe->next_tx_frame += pipe->interval;
3013                 if (pipe->next_tx_frame < usb->frame_number)
3014                         pipe->next_tx_frame = usb->frame_number +
3015                                 pipe->interval -
3016                                 (usb->frame_number - pipe->next_tx_frame) %
3017                                 pipe->interval;
3018         } else {
3019                 struct cvmx_usb_port_status port;
3020
3021                 port = cvmx_usb_get_status(usb);
3022                 if (port.port_enabled) {
3023                         /* We'll retry the exact same transaction again */
3024                         transaction->retries++;
3025                 } else {
3026                         /*
3027                          * We get channel halted interrupts with no result bits
3028                          * sets when the cable is unplugged
3029                          */
3030                         __cvmx_usb_perform_complete(usb, pipe, transaction,
3031                                         CVMX_USB_COMPLETE_ERROR);
3032                 }
3033         }
3034         return 0;
3035 }
3036
3037 static void octeon_usb_port_callback(struct cvmx_usb_state *usb)
3038 {
3039         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
3040
3041         spin_unlock(&priv->lock);
3042         usb_hcd_poll_rh_status(octeon_to_hcd(priv));
3043         spin_lock(&priv->lock);
3044 }
3045
3046 /**
3047  * Poll the USB block for status and call all needed callback
3048  * handlers. This function is meant to be called in the interrupt
3049  * handler for the USB controller. It can also be called
3050  * periodically in a loop for non-interrupt based operation.
3051  *
3052  * @usb: USB device state populated by cvmx_usb_initialize().
3053  *
3054  * Returns: 0 or a negative error code.
3055  */
3056 static int cvmx_usb_poll(struct cvmx_usb_state *usb)
3057 {
3058         union cvmx_usbcx_hfnum usbc_hfnum;
3059         union cvmx_usbcx_gintsts usbc_gintsts;
3060
3061         prefetch_range(usb, sizeof(*usb));
3062
3063         /* Update the frame counter */
3064         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb,
3065                                                 CVMX_USBCX_HFNUM(usb->index));
3066         if ((usb->frame_number&0x3fff) > usbc_hfnum.s.frnum)
3067                 usb->frame_number += 0x4000;
3068         usb->frame_number &= ~0x3fffull;
3069         usb->frame_number |= usbc_hfnum.s.frnum;
3070
3071         /* Read the pending interrupts */
3072         usbc_gintsts.u32 = __cvmx_usb_read_csr32(usb,
3073                                                 CVMX_USBCX_GINTSTS(usb->index));
3074
3075         /* Clear the interrupts now that we know about them */
3076         __cvmx_usb_write_csr32(usb,
3077                                 CVMX_USBCX_GINTSTS(usb->index),
3078                                 usbc_gintsts.u32);
3079
3080         if (usbc_gintsts.s.rxflvl) {
3081                 /*
3082                  * RxFIFO Non-Empty (RxFLvl)
3083                  * Indicates that there is at least one packet pending to be
3084                  * read from the RxFIFO.
3085                  *
3086                  * In DMA mode this is handled by hardware
3087                  */
3088                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3089                         __cvmx_usb_poll_rx_fifo(usb);
3090         }
3091         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
3092                 /* Fill the Tx FIFOs when not in DMA mode */
3093                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3094                         __cvmx_usb_poll_tx_fifo(usb);
3095         }
3096         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
3097                 union cvmx_usbcx_hprt usbc_hprt;
3098                 /*
3099                  * Disconnect Detected Interrupt (DisconnInt)
3100                  * Asserted when a device disconnect is detected.
3101                  *
3102                  * Host Port Interrupt (PrtInt)
3103                  * The core sets this bit to indicate a change in port status of
3104                  * one of the O2P USB core ports in Host mode. The application
3105                  * must read the Host Port Control and Status (HPRT) register to
3106                  * determine the exact event that caused this interrupt. The
3107                  * application must clear the appropriate status bit in the Host
3108                  * Port Control and Status register to clear this bit.
3109                  *
3110                  * Call the user's port callback
3111                  */
3112                 octeon_usb_port_callback(usb);
3113                 /* Clear the port change bits */
3114                 usbc_hprt.u32 = __cvmx_usb_read_csr32(usb,
3115                                 CVMX_USBCX_HPRT(usb->index));
3116                 usbc_hprt.s.prtena = 0;
3117                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index),
3118                                        usbc_hprt.u32);
3119         }
3120         if (usbc_gintsts.s.hchint) {
3121                 /*
3122                  * Host Channels Interrupt (HChInt)
3123                  * The core sets this bit to indicate that an interrupt is
3124                  * pending on one of the channels of the core (in Host mode).
3125                  * The application must read the Host All Channels Interrupt
3126                  * (HAINT) register to determine the exact number of the channel
3127                  * on which the interrupt occurred, and then read the
3128                  * corresponding Host Channel-n Interrupt (HCINTn) register to
3129                  * determine the exact cause of the interrupt. The application
3130                  * must clear the appropriate status bit in the HCINTn register
3131                  * to clear this bit.
3132                  */
3133                 union cvmx_usbcx_haint usbc_haint;
3134
3135                 usbc_haint.u32 = __cvmx_usb_read_csr32(usb,
3136                                         CVMX_USBCX_HAINT(usb->index));
3137                 while (usbc_haint.u32) {
3138                         int channel;
3139
3140                         channel = __fls(usbc_haint.u32);
3141                         __cvmx_usb_poll_channel(usb, channel);
3142                         usbc_haint.u32 ^= 1<<channel;
3143                 }
3144         }
3145
3146         __cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
3147
3148         return 0;
3149 }
3150
3151 /* convert between an HCD pointer and the corresponding struct octeon_hcd */
3152 static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
3153 {
3154         return (struct octeon_hcd *)(hcd->hcd_priv);
3155 }
3156
3157 static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
3158 {
3159         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3160         unsigned long flags;
3161
3162         spin_lock_irqsave(&priv->lock, flags);
3163         cvmx_usb_poll(&priv->usb);
3164         spin_unlock_irqrestore(&priv->lock, flags);
3165         return IRQ_HANDLED;
3166 }
3167
3168 static int octeon_usb_start(struct usb_hcd *hcd)
3169 {
3170         hcd->state = HC_STATE_RUNNING;
3171         return 0;
3172 }
3173
3174 static void octeon_usb_stop(struct usb_hcd *hcd)
3175 {
3176         hcd->state = HC_STATE_HALT;
3177 }
3178
3179 static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
3180 {
3181         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3182
3183         return cvmx_usb_get_frame_number(&priv->usb);
3184 }
3185
3186 static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
3187                                   struct urb *urb,
3188                                   gfp_t mem_flags)
3189 {
3190         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3191         struct device *dev = hcd->self.controller;
3192         struct cvmx_usb_transaction *transaction = NULL;
3193         struct cvmx_usb_pipe *pipe;
3194         unsigned long flags;
3195         struct cvmx_usb_iso_packet *iso_packet;
3196         struct usb_host_endpoint *ep = urb->ep;
3197         int rc;
3198
3199         urb->status = 0;
3200         spin_lock_irqsave(&priv->lock, flags);
3201
3202         rc = usb_hcd_link_urb_to_ep(hcd, urb);
3203         if (rc) {
3204                 spin_unlock_irqrestore(&priv->lock, flags);
3205                 return rc;
3206         }
3207
3208         if (!ep->hcpriv) {
3209                 enum cvmx_usb_transfer transfer_type;
3210                 enum cvmx_usb_speed speed;
3211                 int split_device = 0;
3212                 int split_port = 0;
3213
3214                 switch (usb_pipetype(urb->pipe)) {
3215                 case PIPE_ISOCHRONOUS:
3216                         transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
3217                         break;
3218                 case PIPE_INTERRUPT:
3219                         transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
3220                         break;
3221                 case PIPE_CONTROL:
3222                         transfer_type = CVMX_USB_TRANSFER_CONTROL;
3223                         break;
3224                 default:
3225                         transfer_type = CVMX_USB_TRANSFER_BULK;
3226                         break;
3227                 }
3228                 switch (urb->dev->speed) {
3229                 case USB_SPEED_LOW:
3230                         speed = CVMX_USB_SPEED_LOW;
3231                         break;
3232                 case USB_SPEED_FULL:
3233                         speed = CVMX_USB_SPEED_FULL;
3234                         break;
3235                 default:
3236                         speed = CVMX_USB_SPEED_HIGH;
3237                         break;
3238                 }
3239                 /*
3240                  * For slow devices on high speed ports we need to find the hub
3241                  * that does the speed translation so we know where to send the
3242                  * split transactions.
3243                  */
3244                 if (speed != CVMX_USB_SPEED_HIGH) {
3245                         /*
3246                          * Start at this device and work our way up the usb
3247                          * tree.
3248                          */
3249                         struct usb_device *dev = urb->dev;
3250
3251                         while (dev->parent) {
3252                                 /*
3253                                  * If our parent is high speed then he'll
3254                                  * receive the splits.
3255                                  */
3256                                 if (dev->parent->speed == USB_SPEED_HIGH) {
3257                                         split_device = dev->parent->devnum;
3258                                         split_port = dev->portnum;
3259                                         break;
3260                                 }
3261                                 /*
3262                                  * Move up the tree one level. If we make it all
3263                                  * the way up the tree, then the port must not
3264                                  * be in high speed mode and we don't need a
3265                                  * split.
3266                                  */
3267                                 dev = dev->parent;
3268                         }
3269                 }
3270                 pipe = cvmx_usb_open_pipe(&priv->usb, usb_pipedevice(urb->pipe),
3271                                           usb_pipeendpoint(urb->pipe), speed,
3272                                           le16_to_cpu(ep->desc.wMaxPacketSize)
3273                                           & 0x7ff,
3274                                           transfer_type,
3275                                           usb_pipein(urb->pipe) ?
3276                                                 CVMX_USB_DIRECTION_IN :
3277                                                 CVMX_USB_DIRECTION_OUT,
3278                                           urb->interval,
3279                                           (le16_to_cpu(ep->desc.wMaxPacketSize)
3280                                            >> 11) & 0x3,
3281                                           split_device, split_port);
3282                 if (!pipe) {
3283                         usb_hcd_unlink_urb_from_ep(hcd, urb);
3284                         spin_unlock_irqrestore(&priv->lock, flags);
3285                         dev_dbg(dev, "Failed to create pipe\n");
3286                         return -ENOMEM;
3287                 }
3288                 ep->hcpriv = pipe;
3289         } else {
3290                 pipe = ep->hcpriv;
3291         }
3292
3293         switch (usb_pipetype(urb->pipe)) {
3294         case PIPE_ISOCHRONOUS:
3295                 dev_dbg(dev, "Submit isochronous to %d.%d\n",
3296                         usb_pipedevice(urb->pipe),
3297                         usb_pipeendpoint(urb->pipe));
3298                 /*
3299                  * Allocate a structure to use for our private list of
3300                  * isochronous packets.
3301                  */
3302                 iso_packet = kmalloc(urb->number_of_packets *
3303                                      sizeof(struct cvmx_usb_iso_packet),
3304                                      GFP_ATOMIC);
3305                 if (iso_packet) {
3306                         int i;
3307                         /* Fill the list with the data from the URB */
3308                         for (i = 0; i < urb->number_of_packets; i++) {
3309                                 iso_packet[i].offset =
3310                                         urb->iso_frame_desc[i].offset;
3311                                 iso_packet[i].length =
3312                                         urb->iso_frame_desc[i].length;
3313                                 iso_packet[i].status =
3314                                         CVMX_USB_COMPLETE_ERROR;
3315                         }
3316                         /*
3317                          * Store a pointer to the list in the URB setup_packet
3318                          * field. We know this currently isn't being used and
3319                          * this saves us a bunch of logic.
3320                          */
3321                         urb->setup_packet = (char *)iso_packet;
3322                         transaction = cvmx_usb_submit_isochronous(&priv->usb,
3323                                                                   pipe, urb);
3324                         /*
3325                          * If submit failed we need to free our private packet
3326                          * list.
3327                          */
3328                         if (!transaction) {
3329                                 urb->setup_packet = NULL;
3330                                 kfree(iso_packet);
3331                         }
3332                 }
3333                 break;
3334         case PIPE_INTERRUPT:
3335                 dev_dbg(dev, "Submit interrupt to %d.%d\n",
3336                         usb_pipedevice(urb->pipe),
3337                         usb_pipeendpoint(urb->pipe));
3338                 transaction = cvmx_usb_submit_interrupt(&priv->usb, pipe, urb);
3339                 break;
3340         case PIPE_CONTROL:
3341                 dev_dbg(dev, "Submit control to %d.%d\n",
3342                         usb_pipedevice(urb->pipe),
3343                         usb_pipeendpoint(urb->pipe));
3344                 transaction = cvmx_usb_submit_control(&priv->usb, pipe, urb);
3345                 break;
3346         case PIPE_BULK:
3347                 dev_dbg(dev, "Submit bulk to %d.%d\n",
3348                         usb_pipedevice(urb->pipe),
3349                         usb_pipeendpoint(urb->pipe));
3350                 transaction = cvmx_usb_submit_bulk(&priv->usb, pipe, urb);
3351                 break;
3352         }
3353         if (!transaction) {
3354                 usb_hcd_unlink_urb_from_ep(hcd, urb);
3355                 spin_unlock_irqrestore(&priv->lock, flags);
3356                 dev_dbg(dev, "Failed to submit\n");
3357                 return -ENOMEM;
3358         }
3359         urb->hcpriv = transaction;
3360         spin_unlock_irqrestore(&priv->lock, flags);
3361         return 0;
3362 }
3363
3364 static int octeon_usb_urb_dequeue(struct usb_hcd *hcd,
3365                                   struct urb *urb,
3366                                   int status)
3367 {
3368         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3369         unsigned long flags;
3370         int rc;
3371
3372         if (!urb->dev)
3373                 return -EINVAL;
3374
3375         spin_lock_irqsave(&priv->lock, flags);
3376
3377         rc = usb_hcd_check_unlink_urb(hcd, urb, status);
3378         if (rc)
3379                 goto out;
3380
3381         urb->status = status;
3382         cvmx_usb_cancel(&priv->usb, urb->ep->hcpriv, urb->hcpriv);
3383
3384 out:
3385         spin_unlock_irqrestore(&priv->lock, flags);
3386
3387         return rc;
3388 }
3389
3390 static void octeon_usb_endpoint_disable(struct usb_hcd *hcd,
3391                                         struct usb_host_endpoint *ep)
3392 {
3393         struct device *dev = hcd->self.controller;
3394
3395         if (ep->hcpriv) {
3396                 struct octeon_hcd *priv = hcd_to_octeon(hcd);
3397                 struct cvmx_usb_pipe *pipe = ep->hcpriv;
3398                 unsigned long flags;
3399
3400                 spin_lock_irqsave(&priv->lock, flags);
3401                 cvmx_usb_cancel_all(&priv->usb, pipe);
3402                 if (cvmx_usb_close_pipe(&priv->usb, pipe))
3403                         dev_dbg(dev, "Closing pipe %p failed\n", pipe);
3404                 spin_unlock_irqrestore(&priv->lock, flags);
3405                 ep->hcpriv = NULL;
3406         }
3407 }
3408
3409 static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
3410 {
3411         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3412         struct cvmx_usb_port_status port_status;
3413         unsigned long flags;
3414
3415         spin_lock_irqsave(&priv->lock, flags);
3416         port_status = cvmx_usb_get_status(&priv->usb);
3417         spin_unlock_irqrestore(&priv->lock, flags);
3418         buf[0] = 0;
3419         buf[0] = port_status.connect_change << 1;
3420
3421         return buf[0] != 0;
3422 }
3423
3424 static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
3425                                 u16 wIndex, char *buf, u16 wLength)
3426 {
3427         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3428         struct device *dev = hcd->self.controller;
3429         struct cvmx_usb_port_status usb_port_status;
3430         int port_status;
3431         struct usb_hub_descriptor *desc;
3432         unsigned long flags;
3433
3434         switch (typeReq) {
3435         case ClearHubFeature:
3436                 dev_dbg(dev, "ClearHubFeature\n");
3437                 switch (wValue) {
3438                 case C_HUB_LOCAL_POWER:
3439                 case C_HUB_OVER_CURRENT:
3440                         /* Nothing required here */
3441                         break;
3442                 default:
3443                         return -EINVAL;
3444                 }
3445                 break;
3446         case ClearPortFeature:
3447                 dev_dbg(dev, "ClearPortFeature\n");
3448                 if (wIndex != 1) {
3449                         dev_dbg(dev, " INVALID\n");
3450                         return -EINVAL;
3451                 }
3452
3453                 switch (wValue) {
3454                 case USB_PORT_FEAT_ENABLE:
3455                         dev_dbg(dev, " ENABLE\n");
3456                         spin_lock_irqsave(&priv->lock, flags);
3457                         cvmx_usb_disable(&priv->usb);
3458                         spin_unlock_irqrestore(&priv->lock, flags);
3459                         break;
3460                 case USB_PORT_FEAT_SUSPEND:
3461                         dev_dbg(dev, " SUSPEND\n");
3462                         /* Not supported on Octeon */
3463                         break;
3464                 case USB_PORT_FEAT_POWER:
3465                         dev_dbg(dev, " POWER\n");
3466                         /* Not supported on Octeon */
3467                         break;
3468                 case USB_PORT_FEAT_INDICATOR:
3469                         dev_dbg(dev, " INDICATOR\n");
3470                         /* Port inidicator not supported */
3471                         break;
3472                 case USB_PORT_FEAT_C_CONNECTION:
3473                         dev_dbg(dev, " C_CONNECTION\n");
3474                         /* Clears drivers internal connect status change flag */
3475                         spin_lock_irqsave(&priv->lock, flags);
3476                         priv->usb.port_status =
3477                                 cvmx_usb_get_status(&priv->usb);
3478                         spin_unlock_irqrestore(&priv->lock, flags);
3479                         break;
3480                 case USB_PORT_FEAT_C_RESET:
3481                         dev_dbg(dev, " C_RESET\n");
3482                         /*
3483                          * Clears the driver's internal Port Reset Change flag.
3484                          */
3485                         spin_lock_irqsave(&priv->lock, flags);
3486                         priv->usb.port_status =
3487                                 cvmx_usb_get_status(&priv->usb);
3488                         spin_unlock_irqrestore(&priv->lock, flags);
3489                         break;
3490                 case USB_PORT_FEAT_C_ENABLE:
3491                         dev_dbg(dev, " C_ENABLE\n");
3492                         /*
3493                          * Clears the driver's internal Port Enable/Disable
3494                          * Change flag.
3495                          */
3496                         spin_lock_irqsave(&priv->lock, flags);
3497                         priv->usb.port_status =
3498                                 cvmx_usb_get_status(&priv->usb);
3499                         spin_unlock_irqrestore(&priv->lock, flags);
3500                         break;
3501                 case USB_PORT_FEAT_C_SUSPEND:
3502                         dev_dbg(dev, " C_SUSPEND\n");
3503                         /*
3504                          * Clears the driver's internal Port Suspend Change
3505                          * flag, which is set when resume signaling on the host
3506                          * port is complete.
3507                          */
3508                         break;
3509                 case USB_PORT_FEAT_C_OVER_CURRENT:
3510                         dev_dbg(dev, " C_OVER_CURRENT\n");
3511                         /* Clears the driver's overcurrent Change flag */
3512                         spin_lock_irqsave(&priv->lock, flags);
3513                         priv->usb.port_status =
3514                                 cvmx_usb_get_status(&priv->usb);
3515                         spin_unlock_irqrestore(&priv->lock, flags);
3516                         break;
3517                 default:
3518                         dev_dbg(dev, " UNKNOWN\n");
3519                         return -EINVAL;
3520                 }
3521                 break;
3522         case GetHubDescriptor:
3523                 dev_dbg(dev, "GetHubDescriptor\n");
3524                 desc = (struct usb_hub_descriptor *)buf;
3525                 desc->bDescLength = 9;
3526                 desc->bDescriptorType = 0x29;
3527                 desc->bNbrPorts = 1;
3528                 desc->wHubCharacteristics = cpu_to_le16(0x08);
3529                 desc->bPwrOn2PwrGood = 1;
3530                 desc->bHubContrCurrent = 0;
3531                 desc->u.hs.DeviceRemovable[0] = 0;
3532                 desc->u.hs.DeviceRemovable[1] = 0xff;
3533                 break;
3534         case GetHubStatus:
3535                 dev_dbg(dev, "GetHubStatus\n");
3536                 *(__le32 *) buf = 0;
3537                 break;
3538         case GetPortStatus:
3539                 dev_dbg(dev, "GetPortStatus\n");
3540                 if (wIndex != 1) {
3541                         dev_dbg(dev, " INVALID\n");
3542                         return -EINVAL;
3543                 }
3544
3545                 spin_lock_irqsave(&priv->lock, flags);
3546                 usb_port_status = cvmx_usb_get_status(&priv->usb);
3547                 spin_unlock_irqrestore(&priv->lock, flags);
3548                 port_status = 0;
3549
3550                 if (usb_port_status.connect_change) {
3551                         port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
3552                         dev_dbg(dev, " C_CONNECTION\n");
3553                 }
3554
3555                 if (usb_port_status.port_enabled) {
3556                         port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
3557                         dev_dbg(dev, " C_ENABLE\n");
3558                 }
3559
3560                 if (usb_port_status.connected) {
3561                         port_status |= (1 << USB_PORT_FEAT_CONNECTION);
3562                         dev_dbg(dev, " CONNECTION\n");
3563                 }
3564
3565                 if (usb_port_status.port_enabled) {
3566                         port_status |= (1 << USB_PORT_FEAT_ENABLE);
3567                         dev_dbg(dev, " ENABLE\n");
3568                 }
3569
3570                 if (usb_port_status.port_over_current) {
3571                         port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
3572                         dev_dbg(dev, " OVER_CURRENT\n");
3573                 }
3574
3575                 if (usb_port_status.port_powered) {
3576                         port_status |= (1 << USB_PORT_FEAT_POWER);
3577                         dev_dbg(dev, " POWER\n");
3578                 }
3579
3580                 if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
3581                         port_status |= USB_PORT_STAT_HIGH_SPEED;
3582                         dev_dbg(dev, " HIGHSPEED\n");
3583                 } else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
3584                         port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
3585                         dev_dbg(dev, " LOWSPEED\n");
3586                 }
3587
3588                 *((__le32 *) buf) = cpu_to_le32(port_status);
3589                 break;
3590         case SetHubFeature:
3591                 dev_dbg(dev, "SetHubFeature\n");
3592                 /* No HUB features supported */
3593                 break;
3594         case SetPortFeature:
3595                 dev_dbg(dev, "SetPortFeature\n");
3596                 if (wIndex != 1) {
3597                         dev_dbg(dev, " INVALID\n");
3598                         return -EINVAL;
3599                 }
3600
3601                 switch (wValue) {
3602                 case USB_PORT_FEAT_SUSPEND:
3603                         dev_dbg(dev, " SUSPEND\n");
3604                         return -EINVAL;
3605                 case USB_PORT_FEAT_POWER:
3606                         dev_dbg(dev, " POWER\n");
3607                         return -EINVAL;
3608                 case USB_PORT_FEAT_RESET:
3609                         dev_dbg(dev, " RESET\n");
3610                         spin_lock_irqsave(&priv->lock, flags);
3611                         cvmx_usb_disable(&priv->usb);
3612                         if (cvmx_usb_enable(&priv->usb))
3613                                 dev_dbg(dev, "Failed to enable the port\n");
3614                         spin_unlock_irqrestore(&priv->lock, flags);
3615                         return 0;
3616                 case USB_PORT_FEAT_INDICATOR:
3617                         dev_dbg(dev, " INDICATOR\n");
3618                         /* Not supported */
3619                         break;
3620                 default:
3621                         dev_dbg(dev, " UNKNOWN\n");
3622                         return -EINVAL;
3623                 }
3624                 break;
3625         default:
3626                 dev_dbg(dev, "Unknown root hub request\n");
3627                 return -EINVAL;
3628         }
3629         return 0;
3630 }
3631
3632 static const struct hc_driver octeon_hc_driver = {
3633         .description            = "Octeon USB",
3634         .product_desc           = "Octeon Host Controller",
3635         .hcd_priv_size          = sizeof(struct octeon_hcd),
3636         .irq                    = octeon_usb_irq,
3637         .flags                  = HCD_MEMORY | HCD_USB2,
3638         .start                  = octeon_usb_start,
3639         .stop                   = octeon_usb_stop,
3640         .urb_enqueue            = octeon_usb_urb_enqueue,
3641         .urb_dequeue            = octeon_usb_urb_dequeue,
3642         .endpoint_disable       = octeon_usb_endpoint_disable,
3643         .get_frame_number       = octeon_usb_get_frame_number,
3644         .hub_status_data        = octeon_usb_hub_status_data,
3645         .hub_control            = octeon_usb_hub_control,
3646         .map_urb_for_dma        = octeon_map_urb_for_dma,
3647         .unmap_urb_for_dma      = octeon_unmap_urb_for_dma,
3648 };
3649
3650 static int octeon_usb_probe(struct platform_device *pdev)
3651 {
3652         int status;
3653         int initialize_flags;
3654         int usb_num;
3655         struct resource *res_mem;
3656         struct device_node *usbn_node;
3657         int irq = platform_get_irq(pdev, 0);
3658         struct device *dev = &pdev->dev;
3659         struct octeon_hcd *priv;
3660         struct usb_hcd *hcd;
3661         unsigned long flags;
3662         u32 clock_rate = 48000000;
3663         bool is_crystal_clock = false;
3664         const char *clock_type;
3665         int i;
3666
3667         if (dev->of_node == NULL) {
3668                 dev_err(dev, "Error: empty of_node\n");
3669                 return -ENXIO;
3670         }
3671         usbn_node = dev->of_node->parent;
3672
3673         i = of_property_read_u32(usbn_node,
3674                                  "refclk-frequency", &clock_rate);
3675         if (i) {
3676                 dev_err(dev, "No USBN \"refclk-frequency\"\n");
3677                 return -ENXIO;
3678         }
3679         switch (clock_rate) {
3680         case 12000000:
3681                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
3682                 break;
3683         case 24000000:
3684                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
3685                 break;
3686         case 48000000:
3687                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
3688                 break;
3689         default:
3690                 dev_err(dev, "Illebal USBN \"refclk-frequency\" %u\n",
3691                                 clock_rate);
3692                 return -ENXIO;
3693
3694         }
3695
3696         i = of_property_read_string(usbn_node,
3697                                     "refclk-type", &clock_type);
3698
3699         if (!i && strcmp("crystal", clock_type) == 0)
3700                 is_crystal_clock = true;
3701
3702         if (is_crystal_clock)
3703                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
3704         else
3705                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
3706
3707         res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3708         if (res_mem == NULL) {
3709                 dev_err(dev, "found no memory resource\n");
3710                 return -ENXIO;
3711         }
3712         usb_num = (res_mem->start >> 44) & 1;
3713
3714         if (irq < 0) {
3715                 /* Defective device tree, but we know how to fix it. */
3716                 irq_hw_number_t hwirq = usb_num ? (1 << 6) + 17 : 56;
3717
3718                 irq = irq_create_mapping(NULL, hwirq);
3719         }
3720
3721         /*
3722          * Set the DMA mask to 64bits so we get buffers already translated for
3723          * DMA.
3724          */
3725         dev->coherent_dma_mask = ~0;
3726         dev->dma_mask = &dev->coherent_dma_mask;
3727
3728         /*
3729          * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
3730          * IOB priority registers.  Under heavy network load USB
3731          * hardware can be starved by the IOB causing a crash.  Give
3732          * it a priority boost if it has been waiting more than 400
3733          * cycles to avoid this situation.
3734          *
3735          * Testing indicates that a cnt_val of 8192 is not sufficient,
3736          * but no failures are seen with 4096.  We choose a value of
3737          * 400 to give a safety factor of 10.
3738          */
3739         if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
3740                 union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;
3741
3742                 pri_cnt.u64 = 0;
3743                 pri_cnt.s.cnt_enb = 1;
3744                 pri_cnt.s.cnt_val = 400;
3745                 cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
3746         }
3747
3748         hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
3749         if (!hcd) {
3750                 dev_dbg(dev, "Failed to allocate memory for HCD\n");
3751                 return -1;
3752         }
3753         hcd->uses_new_polling = 1;
3754         priv = (struct octeon_hcd *)hcd->hcd_priv;
3755
3756         spin_lock_init(&priv->lock);
3757
3758         status = cvmx_usb_initialize(&priv->usb, usb_num, initialize_flags);
3759         if (status) {
3760                 dev_dbg(dev, "USB initialization failed with %d\n", status);
3761                 kfree(hcd);
3762                 return -1;
3763         }
3764
3765         /* This delay is needed for CN3010, but I don't know why... */
3766         mdelay(10);
3767
3768         spin_lock_irqsave(&priv->lock, flags);
3769         cvmx_usb_poll(&priv->usb);
3770         spin_unlock_irqrestore(&priv->lock, flags);
3771
3772         status = usb_add_hcd(hcd, irq, 0);
3773         if (status) {
3774                 dev_dbg(dev, "USB add HCD failed with %d\n", status);
3775                 kfree(hcd);
3776                 return -1;
3777         }
3778         device_wakeup_enable(hcd->self.controller);
3779
3780         dev_info(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);
3781
3782         return 0;
3783 }
3784
3785 static int octeon_usb_remove(struct platform_device *pdev)
3786 {
3787         int status;
3788         struct device *dev = &pdev->dev;
3789         struct usb_hcd *hcd = dev_get_drvdata(dev);
3790         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3791         unsigned long flags;
3792
3793         usb_remove_hcd(hcd);
3794         spin_lock_irqsave(&priv->lock, flags);
3795         status = cvmx_usb_shutdown(&priv->usb);
3796         spin_unlock_irqrestore(&priv->lock, flags);
3797         if (status)
3798                 dev_dbg(dev, "USB shutdown failed with %d\n", status);
3799
3800         kfree(hcd);
3801
3802         return 0;
3803 }
3804
3805 static const struct of_device_id octeon_usb_match[] = {
3806         {
3807                 .compatible = "cavium,octeon-5750-usbc",
3808         },
3809         {},
3810 };
3811
3812 static struct platform_driver octeon_usb_driver = {
3813         .driver = {
3814                 .name       = "OcteonUSB",
3815                 .of_match_table = octeon_usb_match,
3816         },
3817         .probe      = octeon_usb_probe,
3818         .remove     = octeon_usb_remove,
3819 };
3820
3821 static int __init octeon_usb_driver_init(void)
3822 {
3823         if (usb_disabled())
3824                 return 0;
3825
3826         return platform_driver_register(&octeon_usb_driver);
3827 }
3828 module_init(octeon_usb_driver_init);
3829
3830 static void __exit octeon_usb_driver_exit(void)
3831 {
3832         if (usb_disabled())
3833                 return;
3834
3835         platform_driver_unregister(&octeon_usb_driver);
3836 }
3837 module_exit(octeon_usb_driver_exit);
3838
3839 MODULE_LICENSE("GPL");
3840 MODULE_AUTHOR("Cavium, Inc. <support@cavium.com>");
3841 MODULE_DESCRIPTION("Cavium Inc. OCTEON USB Host driver.");